ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'dev' into d_prefetch_refactor

Browse Source
This commit is contained in:
Yann Collet
2021-05-05 09:13:38 -07:00
parent f5434663ea 455fd1a067
commit 8cde167a27
226 changed files with 4424 additions and 2472 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/generic-dev.yml
+10 -2
View File
@@ -55,11 +55,9 @@ jobs:
CC=gcc-7 CFLAGS=-Werror make -j all
make clean
LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt
make -C tests zbufftest-dll
# candidate test (to check) : underlink test
# LDFLAGS=-Wl,--no-undefined : will make the linker fail if dll is underlinked
# zbufftest-dll : test that a user program can link to multi-threaded libzstd without specifying -pthread
gcc-8-asan-ubsan-testzstd:
runs-on: ubuntu-latest
@@ -126,6 +124,16 @@ jobs:
make libc6install
CFLAGS="-O2 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
clang-msan-fuzz:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: clang + MSan + Fuzz Test
run: |
sudo apt-get update
sudo apt-get install clang
CC=clang FUZZER_FLAGS="--long-tests" make clean msan-fuzztest
asan-ubsan-msan-regression:
runs-on: ubuntu-latest
steps:
.github/workflows/generic-release.yml
-8
View File
@@ -25,14 +25,6 @@ jobs:
make test
# make -c lib all (need to fix. not working right now)
zbuff:
runs-on: ubuntu-16.04
steps:
- uses: actions/checkout@v2
- name: zbuff test
run: |
make -C tests test-zbuff
tsan:
runs-on: ubuntu-latest
steps:
.gitignore
+2
View File
@@ -50,3 +50,5 @@ googletest/
*.code-workspace
compile_commands.json
.clangd
perf.data
perf.data.old
build/VS2008/fullbench/fullbench.vcproj
+1 -1
View File
@@ -463,7 +463,7 @@
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_errors.h"
RelativePath="..\..\..\lib\zstd_errors.h"
>
</File>
<File
build/VS2008/fuzzer/fuzzer.vcproj
+2 -2
View File
@@ -483,7 +483,7 @@
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_errors.h"
RelativePath="..\..\..\lib\zstd_errors.h"
>
</File>
<File
@@ -511,7 +511,7 @@
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
RelativePath="..\..\..\lib\zdict.h"
>
</File>
<File
build/VS2008/zstd/zstd.vcproj
+2 -2
View File
@@ -559,7 +559,7 @@
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
RelativePath="..\..\..\lib\zdict.h"
>
</File>
<File
@@ -575,7 +575,7 @@
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_errors.h"
RelativePath="..\..\..\lib\zstd_errors.h"
>
</File>
<File
build/VS2008/zstdlib/zstdlib.vcproj
+2 -2
View File
@@ -495,7 +495,7 @@
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_errors.h"
RelativePath="..\..\..\lib\zstd_errors.h"
>
</File>
<File
@@ -523,7 +523,7 @@
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
RelativePath="..\..\..\lib\zdict.h"
>
</File>
<File
build/VS2010/fullbench/fullbench.vcxproj
+1 -1
View File
@@ -190,7 +190,7 @@
<ClInclude Include="..\..\..\lib\zstd.h" />
<ClInclude Include="..\..\..\lib\common\fse.h" />
<ClInclude Include="..\..\..\lib\common\huf.h" />
<ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
<ClInclude Include="..\..\..\lib\zstd_errors.h" />
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
<ClInclude Include="..\..\..\lib\common\pool.h" />
<ClInclude Include="..\..\..\lib\common\threading.h" />
build/VS2010/fuzzer/fuzzer.vcxproj
+2 -2
View File
@@ -196,7 +196,7 @@
<ClInclude Include="..\..\..\lib\common\huf.h" />
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
<ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
<ClInclude Include="..\..\..\lib\zstd_errors.h" />
<ClInclude Include="..\..\..\lib\zstd.h" />
<ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
<ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
@@ -211,7 +211,7 @@
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
<ClInclude Include="..\..\..\lib\decompress\zstd_ddict.h" />
<ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
<ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
<ClInclude Include="..\..\..\lib\zdict.h" />
<ClInclude Include="..\..\..\lib\dictBuilder\cover.h" />
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
<ClInclude Include="..\..\..\programs\datagen.h" />
build/VS2010/libzstd-dll/libzstd-dll.vcxproj
+1 -5
View File
@@ -44,9 +44,6 @@
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
<ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
<ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
<ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
<ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
<ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
@@ -64,12 +61,11 @@
<ClInclude Include="..\..\..\lib\common\threading.h" />
<ClInclude Include="..\..\..\lib\common\bitstream.h" />
<ClInclude Include="..\..\..\lib\common\error_private.h" />
<ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
<ClInclude Include="..\..\..\lib\zstd_errors.h" />
<ClInclude Include="..\..\..\lib\common\mem.h" />
<ClInclude Include="..\..\..\lib\common\fse.h" />
<ClInclude Include="..\..\..\lib\common\huf.h" />
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
<ClInclude Include="..\..\..\lib\deprecated\zbuff.h" />
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
<ClInclude Include="..\..\..\lib\legacy\zstd_v01.h" />
<ClInclude Include="..\..\..\lib\legacy\zstd_v02.h" />
build/VS2010/libzstd/libzstd.vcxproj
+1 -5
View File
@@ -44,9 +44,6 @@
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress.c" />
<ClCompile Include="..\..\..\lib\decompress\zstd_decompress_block.c" />
<ClCompile Include="..\..\..\lib\decompress\zstd_ddict.c" />
<ClCompile Include="..\..\..\lib\deprecated\zbuff_common.c" />
<ClCompile Include="..\..\..\lib\deprecated\zbuff_compress.c" />
<ClCompile Include="..\..\..\lib\deprecated\zbuff_decompress.c" />
<ClCompile Include="..\..\..\lib\dictBuilder\cover.c" />
<ClCompile Include="..\..\..\lib\dictBuilder\fastcover.c" />
<ClCompile Include="..\..\..\lib\dictBuilder\divsufsort.c" />
@@ -64,12 +61,11 @@
<ClInclude Include="..\..\..\lib\common\threading.h" />
<ClInclude Include="..\..\..\lib\common\bitstream.h" />
<ClInclude Include="..\..\..\lib\common\error_private.h" />
<ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
<ClInclude Include="..\..\..\lib\zstd_errors.h" />
<ClInclude Include="..\..\..\lib\common\mem.h" />
<ClInclude Include="..\..\..\lib\common\fse.h" />
<ClInclude Include="..\..\..\lib\common\huf.h" />
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
<ClInclude Include="..\..\..\lib\deprecated\zbuff.h" />
<ClInclude Include="..\..\..\lib\legacy\zstd_legacy.h" />
<ClInclude Include="..\..\..\lib\legacy\zstd_v01.h" />
<ClInclude Include="..\..\..\lib\legacy\zstd_v02.h" />
build/VS2010/zstd/zstd.vcxproj
+2 -2
View File
@@ -70,14 +70,14 @@
<ClInclude Include="..\..\..\lib\common\threading.h" />
<ClInclude Include="..\..\..\lib\common\xxhash.h" />
<ClInclude Include="..\..\..\lib\compress\zstdmt_compress.h" />
<ClInclude Include="..\..\..\lib\dictBuilder\zdict.h" />
<ClInclude Include="..\..\..\lib\zdict.h" />
<ClInclude Include="..\..\..\lib\dictBuilder\cover.h" />
<ClInclude Include="..\..\..\lib\dictBuilder\divsufsort.h" />
<ClInclude Include="..\..\..\lib\common\fse.h" />
<ClInclude Include="..\..\..\lib\common\huf.h" />
<ClInclude Include="..\..\..\lib\zstd.h" />
<ClInclude Include="..\..\..\lib\common\zstd_internal.h" />
<ClInclude Include="..\..\..\lib\common\zstd_errors.h" />
<ClInclude Include="..\..\..\lib\zstd_errors.h" />
<ClInclude Include="..\..\..\lib\compress\zstd_compress.h" />
<ClInclude Include="..\..\..\lib\compress\zstd_compress_literals.h" />
<ClInclude Include="..\..\..\lib\compress\zstd_compress_sequences.h" />
build/cmake/lib/CMakeLists.txt
+2 -2
View File
@@ -162,8 +162,8 @@ endif ()
# install target
install(FILES
"${LIBRARY_DIR}/zstd.h"
"${LIBRARY_DIR}/dictBuilder/zdict.h"
"${LIBRARY_DIR}/common/zstd_errors.h"
"${LIBRARY_DIR}/zdict.h"
"${LIBRARY_DIR}/zstd_errors.h"
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
install(TARGETS ${library_targets}
build/cmake/tests/.gitignore
-1
View File
@@ -3,5 +3,4 @@ datagen
fullbench
fuzzer
paramgrill
zbufftest
build/meson/lib/meson.build
+3 -6
View File
@@ -44,10 +44,7 @@ libzstd_sources = [join_paths(zstd_rootdir, 'lib/common/entropy_common.c'),
join_paths(zstd_rootdir, 'lib/dictBuilder/cover.c'),
join_paths(zstd_rootdir, 'lib/dictBuilder/fastcover.c'),
join_paths(zstd_rootdir, 'lib/dictBuilder/divsufsort.c'),
join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.c'),
join_paths(zstd_rootdir, 'lib/deprecated/zbuff_common.c'),
join_paths(zstd_rootdir, 'lib/deprecated/zbuff_compress.c'),
join_paths(zstd_rootdir, 'lib/deprecated/zbuff_decompress.c')]
join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.c')]
# Explicit define legacy support
add_project_arguments('-DZSTD_LEGACY_SUPPORT=@0@'.format(legacy_level),
@@ -127,5 +124,5 @@ pkgconfig.generate(libzstd,
url: 'http://www.zstd.net/')
install_headers(join_paths(zstd_rootdir, 'lib/zstd.h'),
join_paths(zstd_rootdir, 'lib/dictBuilder/zdict.h'),
join_paths(zstd_rootdir, 'lib/common/zstd_errors.h'))
join_paths(zstd_rootdir, 'lib/zdict.h'),
join_paths(zstd_rootdir, 'lib/zstd_errors.h'))
contrib/VS2005/fullbench/fullbench.vcproj
+1 -1
View File
@@ -390,7 +390,7 @@
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_errors.h"
RelativePath="..\..\..\lib\zstd_errors.h"
>
</File>
<File
contrib/VS2005/fuzzer/fuzzer.vcproj
+2 -2
View File
@@ -426,7 +426,7 @@
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_errors.h"
RelativePath="..\..\..\lib\zstd_errors.h"
>
</File>
<File
@@ -454,7 +454,7 @@
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
RelativePath="..\..\..\lib\zdict.h"
>
</File>
<File
contrib/VS2005/zstd/zstd.vcproj
+2 -2
View File
@@ -454,7 +454,7 @@
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_errors.h"
RelativePath="..\..\..\lib\zstd_errors.h"
>
</File>
<File
@@ -482,7 +482,7 @@
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
RelativePath="..\..\..\lib\zdict.h"
>
</File>
<File
contrib/VS2005/zstdlib/zstdlib.vcproj
+2 -14
View File
@@ -371,18 +371,6 @@
RelativePath="..\..\..\lib\common\xxhash.c"
>
</File>
<File
RelativePath="..\..\..\lib\deprecated\zbuff_common.c"
>
</File>
<File
RelativePath="..\..\..\lib\deprecated\zbuff_compress.c"
>
</File>
<File
RelativePath="..\..\..\lib\deprecated\zbuff_decompress.c"
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\zdict.c"
>
@@ -458,7 +446,7 @@
>
</File>
<File
RelativePath="..\..\..\lib\common\zstd_errors.h"
RelativePath="..\..\..\lib\zstd_errors.h"
>
</File>
<File
@@ -490,7 +478,7 @@
>
</File>
<File
RelativePath="..\..\..\lib\dictBuilder\zdict.h"
RelativePath="..\..\..\lib\zdict.h"
>
</File>
<File
contrib/freestanding_lib/freestanding.py
+63 -9
View File
@@ -27,6 +27,8 @@ SKIPPED_FILES = [
"common/pool.h",
"common/threading.c",
"common/threading.h",
"common/zstd_trace.c",
"common/zstd_trace.h",
"compress/zstdmt_compress.h",
"compress/zstdmt_compress.c",
]
@@ -430,7 +432,7 @@ class Freestanding(object):
external_xxhash: bool, xxh64_state: Optional[str],
xxh64_prefix: Optional[str], rewritten_includes: [(str, str)],
defs: [(str, Optional[str])], replaces: [(str, str)],
undefs: [str], excludes: [str]
undefs: [str], excludes: [str], seds: [str],
):
self._zstd_deps = zstd_deps
self._mem = mem
@@ -444,6 +446,7 @@ class Freestanding(object):
self._replaces = replaces
self._undefs = undefs
self._excludes = excludes
self._seds = seds
def _dst_lib_file_paths(self):
"""
@@ -471,24 +474,25 @@ class Freestanding(object):
dst_path = os.path.join(self._dst_lib, lib_path)
self._log(f"\tCopying: {src_path} -> {dst_path}")
shutil.copyfile(src_path, dst_path)
def _copy_source_lib(self):
self._log("Copying source library into output library")
assert os.path.exists(self._src_lib)
os.makedirs(self._dst_lib, exist_ok=True)
self._copy_file("zstd.h")
self._copy_file("zstd_errors.h")
for subdir in INCLUDED_SUBDIRS:
src_dir = os.path.join(self._src_lib, subdir)
dst_dir = os.path.join(self._dst_lib, subdir)
assert os.path.exists(src_dir)
os.makedirs(dst_dir, exist_ok=True)
for filename in os.listdir(src_dir):
lib_path = os.path.join(subdir, filename)
self._copy_file(lib_path)
def _copy_zstd_deps(self):
dst_zstd_deps = os.path.join(self._dst_lib, "common", "zstd_deps.h")
self._log(f"Copying zstd_deps: {self._zstd_deps} -> {dst_zstd_deps}")
@@ -508,7 +512,7 @@ class Freestanding(object):
assert not (undef and value is not None)
for filepath in self._dst_lib_file_paths():
file = FileLines(filepath)
def _hardwire_defines(self):
self._log("Hardwiring macros")
partial_preprocessor = PartialPreprocessor(self._defs, self._replaces, self._undefs)
@@ -536,7 +540,7 @@ class Freestanding(object):
skipped.append(line)
if end_re.search(line) is not None:
assert begin_re.search(line) is None
self._log(f"\t\tRemoving excluded section: {exclude}")
self._log(f"\t\tRemoving excluded section: {exclude}")
for s in skipped:
self._log(f"\t\t\t- {s}")
emit = True
@@ -559,12 +563,12 @@ class Freestanding(object):
e = match.end('include')
file.lines[i] = line[:s] + rewritten + line[e:]
file.write()
def _rewrite_includes(self):
self._log("Rewriting includes")
for original, rewritten in self._rewritten_includes:
self._rewrite_include(original, rewritten)
def _replace_xxh64_prefix(self):
if self._xxh64_prefix is None:
return
@@ -596,6 +600,48 @@ class Freestanding(object):
file.lines[i] = line
file.write()
def _parse_sed(self, sed):
assert sed[0] == 's'
delim = sed[1]
match = re.fullmatch(f's{delim}(.+){delim}(.*){delim}(.*)', sed)
assert match is not None
regex = re.compile(match.group(1))
format_str = match.group(2)
is_global = match.group(3) == 'g'
return regex, format_str, is_global
def _process_sed(self, sed):
self._log(f"Processing sed: {sed}")
regex, format_str, is_global = self._parse_sed(sed)
for filepath in self._dst_lib_file_paths():
file = FileLines(filepath)
for i, line in enumerate(file.lines):
modified = False
while True:
match = regex.search(line)
if match is None:
break
replacement = format_str.format(match.groups(''), match.groupdict(''))
b = match.start()
e = match.end()
line = line[:b] + replacement + line[e:]
modified = True
if not is_global:
break
if modified:
self._log(f"\t- {file.lines[i][:-1]}")
self._log(f"\t+ {line[:-1]}")
file.lines[i] = line
file.write()
def _process_seds(self):
self._log("Processing seds")
for sed in self._seds:
self._process_sed(sed)
def go(self):
self._copy_source_lib()
self._copy_zstd_deps()
@@ -604,6 +650,7 @@ class Freestanding(object):
self._remove_excludes()
self._rewrite_includes()
self._replace_xxh64_prefix()
self._process_seds()
def parse_optional_pair(defines: [str]) -> [(str, Optional[str])]:
@@ -641,6 +688,7 @@ def main(name, args):
parser.add_argument("--xxh64-state", default=None, help="Alternate XXH64 state type (excluding _) e.g. --xxh64-state='struct xxh64_state'")
parser.add_argument("--xxh64-prefix", default=None, help="Alternate XXH64 function prefix (excluding _) e.g. --xxh64-prefix=xxh64")
parser.add_argument("--rewrite-include", default=[], dest="rewritten_includes", action="append", help="Rewrite an include REGEX=NEW (e.g. '<stddef\\.h>=<linux/types.h>')")
parser.add_argument("--sed", default=[], dest="seds", action="append", help="Apply a sed replacement. Format: `s/REGEX/FORMAT/[g]`. REGEX is a Python regex. FORMAT is a Python format string formatted by the regex dict.")
parser.add_argument("-D", "--define", default=[], dest="defs", action="append", help="Pre-define this macro (can be passed multiple times)")
parser.add_argument("-U", "--undefine", default=[], dest="undefs", action="append", help="Pre-undefine this macro (can be passed mutliple times)")
parser.add_argument("-R", "--replace", default=[], dest="replaces", action="append", help="Pre-define this macro and replace the first ifndef block with its definition")
@@ -656,6 +704,11 @@ def main(name, args):
if name in args.undefs:
raise RuntimeError(f"{name} is both defined and undefined!")
# Always set tracing to 0
if "ZSTD_NO_TRACE" not in (arg[0] for arg in args.defs):
args.defs.append(("ZSTD_NO_TRACE", None))
args.defs.append(("ZSTD_TRACE", "0"))
args.replaces = parse_pair(args.replaces)
for name, _ in args.replaces:
if name in args.undefs or name in args.defs:
@@ -688,7 +741,8 @@ def main(name, args):
args.defs,
args.replaces,
args.undefs,
args.excludes
args.excludes,
args.seds,
).go()
if __name__ == "__main__":
contrib/linux-kernel/Makefile
+13 -3
View File
@@ -1,5 +1,5 @@
# ################################################################
# Copyright (c) 2015-2021, Facebook, Inc.
# Copyright (c) Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
@@ -22,6 +22,10 @@ libzstd:
--xxh64-prefix 'xxh64' \
--rewrite-include '<limits\.h>=<linux/limits.h>' \
--rewrite-include '<stddef\.h>=<linux/types.h>' \
--rewrite-include '"\.\./zstd.h"=<linux/zstd.h>' \
--rewrite-include '"(\.\./)?zstd_errors.h"=<linux/zstd_errors.h>' \
--sed 's,/\*\*\*,/* *,g' \
--sed 's,/\*\*,/*,g' \
-DZSTD_NO_INTRINSICS \
-DZSTD_NO_UNUSED_FUNCTIONS \
-DZSTD_LEGACY_SUPPORT=0 \
@@ -47,7 +51,10 @@ libzstd:
-RZSTDLIB_VISIBILITY= \
-RZSTDERRORLIB_VISIBILITY= \
-DZSTD_HAVE_WEAK_SYMBOLS=0 \
-DZSTD_TRACE=0
-DZSTD_TRACE=0 \
-DZSTD_NO_TRACE
mv linux/lib/zstd/zstd.h linux/include/linux/zstd_lib.h
mv linux/lib/zstd/zstd_errors.h linux/include/linux/
cp linux_zstd.h linux/include/linux/zstd.h
cp zstd_compress_module.c linux/lib/zstd
cp zstd_decompress_module.c linux/lib/zstd
@@ -62,15 +69,18 @@ import: libzstd
rm -f $(LINUX)/include/linux/zstd_errors.h
rm -rf $(LINUX)/lib/zstd
cp linux/include/linux/zstd.h $(LINUX)/include/linux
cp linux/include/linux/zstd_lib.h $(LINUX)/include/linux
cp linux/include/linux/zstd_errors.h $(LINUX)/include/linux
cp -r linux/lib/zstd $(LINUX)/lib
import-upstream:
rm -rf $(LINUX)/lib/zstd
mkdir $(LINUX)/lib/zstd
cp ../../lib/zstd.h $(LINUX)/lib/zstd
cp ../../lib/zstd.h $(LINUX)/include/linux/zstd_lib.h
cp -r ../../lib/common $(LINUX)/lib/zstd
cp -r ../../lib/compress $(LINUX)/lib/zstd
cp -r ../../lib/decompress $(LINUX)/lib/zstd
mv $(LINUX)/lib/zstd/zstd_errors.h $(LINUX)/include/linux
rm $(LINUX)/lib/zstd/common/threading.*
rm $(LINUX)/lib/zstd/common/pool.*
rm $(LINUX)/lib/zstd/common/xxhash.*
contrib/linux-kernel/decompress_sources.h
+9
View File
@@ -1,4 +1,13 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/*
* This file includes every .c file needed for decompression.
contrib/linux-kernel/linux.mk
+9
View File
@@ -1,4 +1,13 @@
# SPDX-License-Identifier: GPL-2.0-only
# ################################################################
# Copyright (c) Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# You may select, at your option, one of the above-listed licenses.
# ################################################################
obj-$(CONFIG_ZSTD_COMPRESS) += zstd_compress.o
obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd_decompress.o
contrib/linux-kernel/linux_zstd.h
+90 -102
View File
@@ -1,18 +1,13 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of https://github.com/facebook/zstd.
* An additional grant of patent rights can be found in the PATENTS file in the
* same directory.
*
* This program is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License version 2 as published by the
* Free Software Foundation. This program is dual-licensed; you may select
* either version 2 of the GNU General Public License ("GPL") or BSD license
* ("BSD").
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of https://github.com/facebook/zstd) and
* the GPLv2 (found in the COPYING file in the root directory of
* https://github.com/facebook/zstd). You may select, at your option, one of the
* above-listed licenses.
*/
#ifndef LINUX_ZSTD_H
@@ -27,6 +22,8 @@
/* ====== Dependency ====== */
#include <linux/types.h>
#include <linux/zstd_errors.h>
#include <linux/zstd_lib.h>
/* ====== Helper Functions ====== */
/**
@@ -45,13 +42,18 @@ size_t zstd_compress_bound(size_t src_size);
*/
unsigned int zstd_is_error(size_t code);
/**
* enum zstd_error_code - zstd error codes
*/
typedef ZSTD_ErrorCode zstd_error_code;
/**
* zstd_get_error_code() - translates an error function result to an error code
* @code: The function result for which zstd_is_error(code) is true.
*
* Return: A unique error code for this error.
*/
int zstd_get_error_code(size_t code);
zstd_error_code zstd_get_error_code(size_t code);
/**
* zstd_get_error_name() - translates an error function result to a string
@@ -61,76 +63,67 @@ int zstd_get_error_code(size_t code);
*/
const char *zstd_get_error_name(size_t code);
/**
* zstd_min_clevel() - minimum allowed compression level
*
* Return: The minimum allowed compression level.
*/
int zstd_min_clevel(void);
/**
* zstd_max_clevel() - maximum allowed compression level
*
* Return: The maximum allowed compression level.
*/
int zstd_max_clevel(void);
/* ====== Parameter Selection ====== */
/**
* enum zstd_strategy - zstd compression search strategy
*
* From faster to stronger.
* From faster to stronger. See zstd_lib.h.
*/
enum zstd_strategy {
zstd_fast = 1,
zstd_dfast = 2,
zstd_greedy = 3,
zstd_lazy = 4,
zstd_lazy2 = 5,
zstd_btlazy2 = 6,
zstd_btopt = 7,
zstd_btultra = 8,
zstd_btultra2 = 9
};
typedef ZSTD_strategy zstd_strategy;
/**
* struct zstd_compression_parameters - zstd compression parameters
* @window_log: Log of the largest match distance. Larger means more
* compression, and more memory needed during decompression.
* @chain_log: Fully searched segment. Larger means more compression,
* slower, and more memory (useless for fast).
* @hash_log: Dispatch table. Larger means more compression,
* slower, and more memory.
* @search_log: Number of searches. Larger means more compression and slower.
* @search_length: Match length searched. Larger means faster decompression,
* sometimes less compression.
* @target_length: Acceptable match size for optimal parser (only). Larger means
* more compression, and slower.
* @strategy: The zstd compression strategy.
* @windowLog: Log of the largest match distance. Larger means more
* compression, and more memory needed during decompression.
* @chainLog: Fully searched segment. Larger means more compression,
* slower, and more memory (useless for fast).
* @hashLog: Dispatch table. Larger means more compression,
* slower, and more memory.
* @searchLog: Number of searches. Larger means more compression and slower.
* @searchLength: Match length searched. Larger means faster decompression,
* sometimes less compression.
* @targetLength: Acceptable match size for optimal parser (only). Larger means
* more compression, and slower.
* @strategy: The zstd compression strategy.
*
* See zstd_lib.h.
*/
struct zstd_compression_parameters {
unsigned int window_log;
unsigned int chain_log;
unsigned int hash_log;
unsigned int search_log;
unsigned int search_length;
unsigned int target_length;
enum zstd_strategy strategy;
};
typedef ZSTD_compressionParameters zstd_compression_parameters;
/**
* struct zstd_frame_parameters - zstd frame parameters
* @content_size_flag: Controls whether content size will be present in the
* frame header (when known).
* @checksum_flag: Controls whether a 32-bit checksum is generated at the
* end of the frame for error detection.
* @no_dict_id_flag: Controls whether dictID will be saved into the frame
* header when using dictionary compression.
* @contentSizeFlag: Controls whether content size will be present in the
* frame header (when known).
* @checksumFlag: Controls whether a 32-bit checksum is generated at the
* end of the frame for error detection.
* @noDictIDFlag: Controls whether dictID will be saved into the frame
* header when using dictionary compression.
*
* The default value is all fields set to 0.
* The default value is all fields set to 0. See zstd_lib.h.
*/
struct zstd_frame_parameters {
unsigned int content_size_flag;
unsigned int checksum_flag;
unsigned int no_dict_id_flag;
};
typedef ZSTD_frameParameters zstd_frame_parameters;
/**
* struct zstd_parameters - zstd parameters
* @cparams: The compression parameters.
* @fparams: The frame parameters.
* @cParams: The compression parameters.
* @fParams: The frame parameters.
*/
struct zstd_parameters {
struct zstd_compression_parameters cparams;
struct zstd_frame_parameters fparams;
};
typedef ZSTD_parameters zstd_parameters;
/**
* zstd_get_params() - returns zstd_parameters for selected level
@@ -140,12 +133,12 @@ struct zstd_parameters {
*
* Return: The selected zstd_parameters.
*/
struct zstd_parameters zstd_get_params(int level,
zstd_parameters zstd_get_params(int level,
unsigned long long estimated_src_size);
/* ====== Single-pass Compression ====== */
typedef struct ZSTD_CCtx_s zstd_cctx;
typedef ZSTD_CCtx zstd_cctx;
/**
* zstd_cctx_workspace_bound() - max memory needed to initialize a zstd_cctx
@@ -158,8 +151,7 @@ typedef struct ZSTD_CCtx_s zstd_cctx;
* Return: A lower bound on the size of the workspace that is passed to
* zstd_init_cctx().
*/
size_t zstd_cctx_workspace_bound(
const struct zstd_compression_parameters *parameters);
size_t zstd_cctx_workspace_bound(const zstd_compression_parameters *parameters);
/**
* zstd_init_cctx() - initialize a zstd compression context
@@ -186,11 +178,11 @@ zstd_cctx *zstd_init_cctx(void *workspace, size_t workspace_size);
* zstd_is_error().
*/
size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity,
const void *src, size_t src_size, const struct zstd_parameters *parameters);
const void *src, size_t src_size, const zstd_parameters *parameters);
/* ====== Single-pass Decompression ====== */
typedef struct ZSTD_DCtx_s zstd_dctx;
typedef ZSTD_DCtx zstd_dctx;
/**
* zstd_dctx_workspace_bound() - max memory needed to initialize a zstd_dctx
@@ -236,12 +228,10 @@ size_t zstd_decompress_dctx(zstd_dctx *dctx, void *dst, size_t dst_capacity,
* @size: Size of the input buffer.
* @pos: Position where reading stopped. Will be updated.
* Necessarily 0 <= pos <= size.
*
* See zstd_lib.h.
*/
struct zstd_in_buffer {
const void *src;
size_t size;
size_t pos;
};
typedef ZSTD_inBuffer zstd_in_buffer;
/**
* struct zstd_out_buffer - output buffer for streaming
@@ -249,16 +239,14 @@ struct zstd_in_buffer {
* @size: Size of the output buffer.
* @pos: Position where writing stopped. Will be updated.
* Necessarily 0 <= pos <= size.
*
* See zstd_lib.h.
*/
struct zstd_out_buffer {
void *dst;
size_t size;
size_t pos;
};
typedef ZSTD_outBuffer zstd_out_buffer;
/* ====== Streaming Compression ====== */
typedef struct ZSTD_CCtx_s zstd_cstream;
typedef ZSTD_CStream zstd_cstream;
/**
* zstd_cstream_workspace_bound() - memory needed to initialize a zstd_cstream
@@ -267,8 +255,7 @@ typedef struct ZSTD_CCtx_s zstd_cstream;
* Return: A lower bound on the size of the workspace that is passed to
* zstd_init_cstream().
*/
size_t zstd_cstream_workspace_bound(
const struct zstd_compression_parameters *cparams);
size_t zstd_cstream_workspace_bound(const zstd_compression_parameters *cparams);
/**
* zstd_init_cstream() - initialize a zstd streaming compression context
@@ -285,7 +272,7 @@ size_t zstd_cstream_workspace_bound(
*
* Return: The zstd streaming compression context or NULL on error.
*/
zstd_cstream *zstd_init_cstream(const struct zstd_parameters *parameters,
zstd_cstream *zstd_init_cstream(const zstd_parameters *parameters,
unsigned long long pledged_src_size, void *workspace, size_t workspace_size);
/**
@@ -320,8 +307,8 @@ size_t zstd_reset_cstream(zstd_cstream *cstream,
* function call or an error, which can be checked using
* zstd_is_error().
*/
size_t zstd_compress_stream(zstd_cstream *cstream,
struct zstd_out_buffer *output, struct zstd_in_buffer *input);
size_t zstd_compress_stream(zstd_cstream *cstream, zstd_out_buffer *output,
zstd_in_buffer *input);
/**
* zstd_flush_stream() - flush internal buffers into output
@@ -336,7 +323,7 @@ size_t zstd_compress_stream(zstd_cstream *cstream,
* Return: The number of bytes still present within internal buffers or an
* error, which can be checked using zstd_is_error().
*/
size_t zstd_flush_stream(zstd_cstream *cstream, struct zstd_out_buffer *output);
size_t zstd_flush_stream(zstd_cstream *cstream, zstd_out_buffer *output);
/**
* zstd_end_stream() - flush internal buffers into output and end the frame
@@ -350,11 +337,11 @@ size_t zstd_flush_stream(zstd_cstream *cstream, struct zstd_out_buffer *output);
* Return: The number of bytes still present within internal buffers or an
* error, which can be checked using zstd_is_error().
*/
size_t zstd_end_stream(zstd_cstream *cstream, struct zstd_out_buffer *output);
size_t zstd_end_stream(zstd_cstream *cstream, zstd_out_buffer *output);
/* ====== Streaming Decompression ====== */
typedef struct ZSTD_DCtx_s zstd_dstream;
typedef ZSTD_DStream zstd_dstream;
/**
* zstd_dstream_workspace_bound() - memory needed to initialize a zstd_dstream
@@ -411,8 +398,8 @@ size_t zstd_reset_dstream(zstd_dstream *dstream);
* using zstd_is_error(). The size hint will never load more than the
* frame.
*/
size_t zstd_decompress_stream(zstd_dstream *dstream,
struct zstd_out_buffer *output, struct zstd_in_buffer *input);
size_t zstd_decompress_stream(zstd_dstream *dstream, zstd_out_buffer *output,
zstd_in_buffer *input);
/* ====== Frame Inspection Functions ====== */
@@ -431,20 +418,21 @@ size_t zstd_find_frame_compressed_size(const void *src, size_t src_size);
/**
* struct zstd_frame_params - zstd frame parameters stored in the frame header
* @frame_content_size: The frame content size, or 0 if not present.
* @window_size: The window size, or 0 if the frame is a skippable frame.
* @dict_id: The dictionary id, or 0 if not present.
* @checksum_flag: Whether a checksum was used.
* @frameContentSize: The frame content size, or ZSTD_CONTENTSIZE_UNKNOWN if not
* present.
* @windowSize: The window size, or 0 if the frame is a skippable frame.
* @blockSizeMax: The maximum block size.
* @frameType: The frame type (zstd or skippable)
* @headerSize: The size of the frame header.
* @dictID: The dictionary id, or 0 if not present.
* @checksumFlag: Whether a checksum was used.
*
* See zstd_lib.h.
*/
struct zstd_frame_params {
unsigned long long frame_content_size;
unsigned int window_size;
unsigned int dict_id;
unsigned int checksum_flag;
};
typedef ZSTD_frameHeader zstd_frame_header;
/**
* zstd_get_frame_params() - extracts parameters from a zstd or skippable frame
* zstd_get_frame_header() - extracts parameters from a zstd or skippable frame
* @params: On success the frame parameters are written here.
* @src: The source buffer. It must point to a zstd or skippable frame.
* @src_size: The size of the source buffer.
@@ -453,7 +441,7 @@ struct zstd_frame_params {
* must be provided to make forward progress. Otherwise it returns
* an error, which can be checked using zstd_is_error().
*/
size_t zstd_get_frame_params(struct zstd_frame_params *params, const void *src,
size_t zstd_get_frame_header(zstd_frame_header *params, const void *src,
size_t src_size);
#endif /* LINUX_ZSTD_H */
contrib/linux-kernel/mem.h
+2 -1
View File
@@ -1,5 +1,6 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
contrib/linux-kernel/test/Makefile
+13 -3
View File
@@ -1,3 +1,12 @@
# ################################################################
# Copyright (c) Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# You may select, at your option, one of the above-listed licenses.
# ################################################################
LINUX := ../linux
LINUX_ZSTDLIB := $(LINUX)/lib/zstd
@@ -7,9 +16,9 @@ CPPFLAGS += -I$(LINUX)/include -I$(LINUX_ZSTDLIB) -Iinclude -DNDEBUG
CPPFLAGS += -DZSTD_ASAN_DONT_POISON_WORKSPACE
LINUX_ZSTD_MODULE := $(wildcard $(LINUX_ZSTDLIB)/*.c)
LINUX_ZSTD_COMMON := $(wildcard $(LINUX_ZSTDLIB)/common/*.c)
LINUX_ZSTD_COMPRESS := $(wildcard $(LINUX_ZSTDLIB)/compress/*.c)
LINUX_ZSTD_DECOMPRESS := $(wildcard $(LINUX_ZSTDLIB)/decompress/*.c)
LINUX_ZSTD_COMMON := $(wildcard $(LINUX_ZSTDLIB)/common/*.c)
LINUX_ZSTD_COMPRESS := $(wildcard $(LINUX_ZSTDLIB)/compress/*.c)
LINUX_ZSTD_DECOMPRESS := $(wildcard $(LINUX_ZSTDLIB)/decompress/*.c)
LINUX_ZSTD_FILES := $(LINUX_ZSTD_MODULE) $(LINUX_ZSTD_COMMON) $(LINUX_ZSTD_COMPRESS) $(LINUX_ZSTD_DECOMPRESS)
LINUX_ZSTD_OBJECTS := $(LINUX_ZSTD_FILES:.c=.o)
@@ -29,6 +38,7 @@ run-test: test static_test
.PHONY:
clean:
$(RM) -f $(LINUX_ZSTDLIB)/*.o
$(RM) -f $(LINUX_ZSTDLIB)/**/*.o
$(RM) -f *.o *.a
$(RM) -f test
contrib/linux-kernel/test/include/asm/unaligned.h
+14 -4
View File
@@ -4,13 +4,23 @@
#include <assert.h>
#include <linux/types.h>
#define _LITTLE_ENDIAN 1
#ifndef __LITTLE_ENDIAN
# if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) || defined(__LITTLE_ENDIAN__)
# define __LITTLE_ENDIAN 1
# endif
#endif
#ifdef __LITTLE_ENDIAN
# define _IS_LITTLE_ENDIAN 1
#else
# define _IS_LITTLE_ENDIAN 0
#endif
static unsigned _isLittleEndian(void)
{
const union { uint32_t u; uint8_t c[4]; } one = { 1 };
assert(_LITTLE_ENDIAN == one.c[0]);
return _LITTLE_ENDIAN;
assert(_IS_LITTLE_ENDIAN == one.c[0]);
return _IS_LITTLE_ENDIAN;
}
static uint16_t _swap16(uint16_t in)
@@ -165,7 +175,7 @@ extern void __bad_unaligned_access_size(void);
(void)0; \
})
#if _LITTLE_ENDIAN
#if _IS_LITTLE_ENDIAN
# define get_unaligned __get_unaligned_le
# define put_unaligned __put_unaligned_le
#else
contrib/linux-kernel/test/include/linux/compiler.h
+4
View File
@@ -14,4 +14,8 @@
#define inline __inline __attribute__((unused))
#endif
#ifndef noinline
#define noinline __attribute__((noinline))
#endif
#endif
contrib/linux-kernel/test/include/linux/kernel.h
+4
View File
@@ -12,4 +12,8 @@
#define WARN_ON(x)
#define PTR_ALIGN(p, a) (typeof(p))ALIGN((unsigned long long)(p), (a))
#define ALIGN(x, a) ALIGN_MASK((x), (a) - 1)
#define ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask))
#endif
contrib/linux-kernel/test/static_test.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
contrib/linux-kernel/test/test.c
+20 -11
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 7-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -57,10 +57,10 @@ static void test_btrfs(test_data_t const *data) {
fprintf(stderr, "testing btrfs use cases... ");
size_t const size = MIN(data->dataSize, 128 * 1024);
for (int level = -1; level < 16; ++level) {
struct zstd_parameters params = zstd_get_params(level, size);
CONTROL(params.cparams.window_log <= 17);
zstd_parameters params = zstd_get_params(level, size);
CONTROL(params.cParams.windowLog <= 17);
size_t const workspaceSize =
MAX(zstd_cstream_workspace_bound(&params.cparams),
MAX(zstd_cstream_workspace_bound(&params.cParams),
zstd_dstream_workspace_bound(size));
void *workspace = malloc(workspaceSize);
CONTROL(workspace != NULL);
@@ -72,8 +72,8 @@ static void test_btrfs(test_data_t const *data) {
{
zstd_cstream *cctx = zstd_init_cstream(&params, size, workspace, workspaceSize);
CONTROL(cctx != NULL);
struct zstd_out_buffer out = {NULL, 0, 0};
struct zstd_in_buffer in = {NULL, 0, 0};
zstd_out_buffer out = {NULL, 0, 0};
zstd_in_buffer in = {NULL, 0, 0};
for (;;) {
if (in.pos == in.size) {
in.src = ip;
@@ -107,10 +107,10 @@ static void test_btrfs(test_data_t const *data) {
op = data->data2;
oend = op + size;
{
zstd_dstream *dctx = zstd_init_dstream(1ULL << params.cparams.window_log, workspace, workspaceSize);
zstd_dstream *dctx = zstd_init_dstream(1ULL << params.cParams.windowLog, workspace, workspaceSize);
CONTROL(dctx != NULL);
struct zstd_out_buffer out = {NULL, 0, 0};
struct zstd_in_buffer in = {NULL, 0, 0};
zstd_out_buffer out = {NULL, 0, 0};
zstd_in_buffer in = {NULL, 0, 0};
for (;;) {
if (in.pos == in.size) {
in.src = ip;
@@ -144,8 +144,8 @@ static void test_decompress_unzstd(test_data_t const *data) {
fprintf(stderr, "Testing decompress unzstd... ");
size_t cSize;
{
struct zstd_parameters params = zstd_get_params(19, 0);
size_t const wkspSize = zstd_cctx_workspace_bound(&params.cparams);
zstd_parameters params = zstd_get_params(19, 0);
size_t const wkspSize = zstd_cctx_workspace_bound(&params.cParams);
void* wksp = malloc(wkspSize);
CONTROL(wksp != NULL);
zstd_cctx* cctx = zstd_init_cctx(wksp, wkspSize);
@@ -169,6 +169,13 @@ static void test_decompress_unzstd(test_data_t const *data) {
fprintf(stderr, "Ok\n");
}
static void test_f2fs() {
fprintf(stderr, "testing f2fs uses... ");
CONTROL(zstd_min_clevel() < 0);
CONTROL(zstd_max_clevel() == 22);
fprintf(stderr, "Ok\n");
}
static char *g_stack = NULL;
static void __attribute__((noinline)) use(void *x) {
@@ -195,6 +202,7 @@ static void __attribute__((noinline)) check_stack() {
static void test_stack_usage(test_data_t const *data) {
set_stack();
test_f2fs();
test_btrfs(data);
test_decompress_unzstd(data);
check_stack();
@@ -202,6 +210,7 @@ static void test_stack_usage(test_data_t const *data) {
int main(void) {
test_data_t data = create_test_data();
test_f2fs();
test_btrfs(&data);
test_decompress_unzstd(&data);
test_stack_usage(&data);
contrib/linux-kernel/zstd_compress_module.c
+37 -119
View File
@@ -1,75 +1,33 @@
// SPDX-License-Identifier: GPL-2.0
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/string.h>
#include <linux/zstd.h>
#include "zstd.h"
#include "common/zstd_deps.h"
#include "common/zstd_internal.h"
static void zstd_check_structs(void) {
/* Check that the structs have the same size. */
ZSTD_STATIC_ASSERT(sizeof(ZSTD_parameters) ==
sizeof(struct zstd_parameters));
ZSTD_STATIC_ASSERT(sizeof(ZSTD_compressionParameters) ==
sizeof(struct zstd_compression_parameters));
ZSTD_STATIC_ASSERT(sizeof(ZSTD_frameParameters) ==
sizeof(struct zstd_frame_parameters));
/* Zstd guarantees that the layout of the structs never change. Verify it. */
ZSTD_STATIC_ASSERT(offsetof(ZSTD_parameters, cParams) ==
offsetof(struct zstd_parameters, cparams));
ZSTD_STATIC_ASSERT(offsetof(ZSTD_parameters, fParams) ==
offsetof(struct zstd_parameters, fparams));
ZSTD_STATIC_ASSERT(offsetof(ZSTD_compressionParameters, windowLog) ==
offsetof(struct zstd_compression_parameters, window_log));
ZSTD_STATIC_ASSERT(offsetof(ZSTD_compressionParameters, chainLog) ==
offsetof(struct zstd_compression_parameters, chain_log));
ZSTD_STATIC_ASSERT(offsetof(ZSTD_compressionParameters, hashLog) ==
offsetof(struct zstd_compression_parameters, hash_log));
ZSTD_STATIC_ASSERT(offsetof(ZSTD_compressionParameters, searchLog) ==
offsetof(struct zstd_compression_parameters, search_log));
ZSTD_STATIC_ASSERT(offsetof(ZSTD_compressionParameters, minMatch) ==
offsetof(struct zstd_compression_parameters, search_length));
ZSTD_STATIC_ASSERT(offsetof(ZSTD_compressionParameters, targetLength) ==
offsetof(struct zstd_compression_parameters, target_length));
ZSTD_STATIC_ASSERT(offsetof(ZSTD_compressionParameters, strategy) ==
offsetof(struct zstd_compression_parameters, strategy));
ZSTD_STATIC_ASSERT(offsetof(ZSTD_frameParameters, contentSizeFlag) ==
offsetof(struct zstd_frame_parameters, content_size_flag));
ZSTD_STATIC_ASSERT(offsetof(ZSTD_frameParameters, checksumFlag) ==
offsetof(struct zstd_frame_parameters, checksum_flag));
ZSTD_STATIC_ASSERT(offsetof(ZSTD_frameParameters, noDictIDFlag) ==
offsetof(struct zstd_frame_parameters, no_dict_id_flag));
/* Check that the strategies are the same. This can change. */
ZSTD_STATIC_ASSERT((int)ZSTD_fast == (int)zstd_fast);
ZSTD_STATIC_ASSERT((int)ZSTD_dfast == (int)zstd_dfast);
ZSTD_STATIC_ASSERT((int)ZSTD_greedy == (int)zstd_greedy);
ZSTD_STATIC_ASSERT((int)ZSTD_lazy == (int)zstd_lazy);
ZSTD_STATIC_ASSERT((int)ZSTD_lazy2 == (int)zstd_lazy2);
ZSTD_STATIC_ASSERT((int)ZSTD_btlazy2 == (int)zstd_btlazy2);
ZSTD_STATIC_ASSERT((int)ZSTD_btopt == (int)zstd_btopt);
ZSTD_STATIC_ASSERT((int)ZSTD_btultra == (int)zstd_btultra);
ZSTD_STATIC_ASSERT((int)ZSTD_btultra2 == (int)zstd_btultra2);
/* Check input buffer */
ZSTD_STATIC_ASSERT(sizeof(ZSTD_inBuffer) == sizeof(struct zstd_in_buffer));
ZSTD_STATIC_ASSERT(offsetof(ZSTD_inBuffer, src) ==
offsetof(struct zstd_in_buffer, src));
ZSTD_STATIC_ASSERT(offsetof(ZSTD_inBuffer, size) ==
offsetof(struct zstd_in_buffer, size));
ZSTD_STATIC_ASSERT(offsetof(ZSTD_inBuffer, pos) ==
offsetof(struct zstd_in_buffer, pos));
/* Check output buffer */
ZSTD_STATIC_ASSERT(sizeof(ZSTD_outBuffer) ==
sizeof(struct zstd_out_buffer));
ZSTD_STATIC_ASSERT(offsetof(ZSTD_outBuffer, dst) ==
offsetof(struct zstd_out_buffer, dst));
ZSTD_STATIC_ASSERT(offsetof(ZSTD_outBuffer, size) ==
offsetof(struct zstd_out_buffer, size));
ZSTD_STATIC_ASSERT(offsetof(ZSTD_outBuffer, pos) ==
offsetof(struct zstd_out_buffer, pos));
int zstd_min_clevel(void)
{
return ZSTD_minCLevel();
}
EXPORT_SYMBOL(zstd_min_clevel);
int zstd_max_clevel(void)
{
return ZSTD_maxCLevel();
}
EXPORT_SYMBOL(zstd_max_clevel);
size_t zstd_compress_bound(size_t src_size)
{
@@ -77,26 +35,16 @@ size_t zstd_compress_bound(size_t src_size)
}
EXPORT_SYMBOL(zstd_compress_bound);
struct zstd_parameters zstd_get_params(int level,
zstd_parameters zstd_get_params(int level,
unsigned long long estimated_src_size)
{
const ZSTD_parameters params = ZSTD_getParams(level, estimated_src_size, 0);
struct zstd_parameters out;
/* no-op */
zstd_check_structs();
ZSTD_memcpy(&out, &params, sizeof(out));
return out;
return ZSTD_getParams(level, estimated_src_size, 0);
}
EXPORT_SYMBOL(zstd_get_params);
size_t zstd_cctx_workspace_bound(
const struct zstd_compression_parameters *cparams)
size_t zstd_cctx_workspace_bound(const zstd_compression_parameters *cparams)
{
ZSTD_compressionParameters p;
ZSTD_memcpy(&p, cparams, sizeof(p));
return ZSTD_estimateCCtxSize_usingCParams(p);
return ZSTD_estimateCCtxSize_usingCParams(*cparams);
}
EXPORT_SYMBOL(zstd_cctx_workspace_bound);
@@ -109,29 +57,21 @@ zstd_cctx *zstd_init_cctx(void *workspace, size_t workspace_size)
EXPORT_SYMBOL(zstd_init_cctx);
size_t zstd_compress_cctx(zstd_cctx *cctx, void *dst, size_t dst_capacity,
const void *src, size_t src_size, const struct zstd_parameters *parameters)
const void *src, size_t src_size, const zstd_parameters *parameters)
{
ZSTD_parameters p;
ZSTD_memcpy(&p, parameters, sizeof(p));
return ZSTD_compress_advanced(cctx, dst, dst_capacity, src, src_size, NULL, 0, p);
return ZSTD_compress_advanced(cctx, dst, dst_capacity, src, src_size, NULL, 0, *parameters);
}
EXPORT_SYMBOL(zstd_compress_cctx);
size_t zstd_cstream_workspace_bound(
const struct zstd_compression_parameters *cparams)
size_t zstd_cstream_workspace_bound(const zstd_compression_parameters *cparams)
{
ZSTD_compressionParameters p;
ZSTD_memcpy(&p, cparams, sizeof(p));
return ZSTD_estimateCStreamSize_usingCParams(p);
return ZSTD_estimateCStreamSize_usingCParams(*cparams);
}
EXPORT_SYMBOL(zstd_cstream_workspace_bound);
zstd_cstream *zstd_init_cstream(const struct zstd_parameters *parameters,
zstd_cstream *zstd_init_cstream(const zstd_parameters *parameters,
unsigned long long pledged_src_size, void *workspace, size_t workspace_size)
{
ZSTD_parameters p;
zstd_cstream *cstream;
size_t ret;
@@ -146,8 +86,7 @@ zstd_cstream *zstd_init_cstream(const struct zstd_parameters *parameters,
if (pledged_src_size == 0)
pledged_src_size = ZSTD_CONTENTSIZE_UNKNOWN;
ZSTD_memcpy(&p, parameters, sizeof(p));
ret = ZSTD_initCStream_advanced(cstream, NULL, 0, p, pledged_src_size);
ret = ZSTD_initCStream_advanced(cstream, NULL, 0, *parameters, pledged_src_size);
if (ZSTD_isError(ret))
return NULL;
@@ -162,43 +101,22 @@ size_t zstd_reset_cstream(zstd_cstream *cstream,
}
EXPORT_SYMBOL(zstd_reset_cstream);
size_t zstd_compress_stream(zstd_cstream *cstream,
struct zstd_out_buffer *output, struct zstd_in_buffer *input)
size_t zstd_compress_stream(zstd_cstream *cstream, zstd_out_buffer *output,
zstd_in_buffer *input)
{
ZSTD_outBuffer o;
ZSTD_inBuffer i;
size_t ret;
ZSTD_memcpy(&o, output, sizeof(o));
ZSTD_memcpy(&i, input, sizeof(i));
ret = ZSTD_compressStream(cstream, &o, &i);
ZSTD_memcpy(output, &o, sizeof(o));
ZSTD_memcpy(input, &i, sizeof(i));
return ret;
return ZSTD_compressStream(cstream, output, input);
}
EXPORT_SYMBOL(zstd_compress_stream);
size_t zstd_flush_stream(zstd_cstream *cstream, struct zstd_out_buffer *output)
size_t zstd_flush_stream(zstd_cstream *cstream, zstd_out_buffer *output)
{
ZSTD_outBuffer o;
size_t ret;
ZSTD_memcpy(&o, output, sizeof(o));
ret = ZSTD_flushStream(cstream, &o);
ZSTD_memcpy(output, &o, sizeof(o));
return ret;
return ZSTD_flushStream(cstream, output);
}
EXPORT_SYMBOL(zstd_flush_stream);
size_t zstd_end_stream(zstd_cstream *cstream, struct zstd_out_buffer *output)
size_t zstd_end_stream(zstd_cstream *cstream, zstd_out_buffer *output)
{
ZSTD_outBuffer o;
size_t ret;
ZSTD_memcpy(&o, output, sizeof(o));
ret = ZSTD_endStream(cstream, &o);
ZSTD_memcpy(output, &o, sizeof(o));
return ret;
return ZSTD_endStream(cstream, output);
}
EXPORT_SYMBOL(zstd_end_stream);
contrib/linux-kernel/zstd_decompress_module.c
+17 -34
View File
@@ -1,13 +1,20 @@
// SPDX-License-Identifier: GPL-2.0
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/string.h>
#include <linux/zstd.h>
#include "zstd.h"
#include "common/zstd_deps.h"
#include "common/zstd_errors.h"
/* Common symbols. zstd_compress must depend on zstd_decompress. */
@@ -17,7 +24,7 @@ unsigned int zstd_is_error(size_t code)
}
EXPORT_SYMBOL(zstd_is_error);
int zstd_get_error_code(size_t code)
zstd_error_code zstd_get_error_code(size_t code)
{
return ZSTD_getErrorCode(code);
}
@@ -74,19 +81,10 @@ size_t zstd_reset_dstream(zstd_dstream *dstream)
}
EXPORT_SYMBOL(zstd_reset_dstream);
size_t zstd_decompress_stream(zstd_dstream *dstream,
struct zstd_out_buffer *output, struct zstd_in_buffer *input)
size_t zstd_decompress_stream(zstd_dstream *dstream, zstd_out_buffer *output,
zstd_in_buffer *input)
{
ZSTD_outBuffer o;
ZSTD_inBuffer i;
size_t ret;
ZSTD_memcpy(&o, output, sizeof(o));
ZSTD_memcpy(&i, input, sizeof(i));
ret = ZSTD_decompressStream(dstream, &o, &i);
ZSTD_memcpy(output, &o, sizeof(o));
ZSTD_memcpy(input, &i, sizeof(i));
return ret;
return ZSTD_decompressStream(dstream, output, input);
}
EXPORT_SYMBOL(zstd_decompress_stream);
@@ -96,27 +94,12 @@ size_t zstd_find_frame_compressed_size(const void *src, size_t src_size)
}
EXPORT_SYMBOL(zstd_find_frame_compressed_size);
size_t zstd_get_frame_params(struct zstd_frame_params *params, const void *src,
size_t zstd_get_frame_header(zstd_frame_header *header, const void *src,
size_t src_size)
{
ZSTD_frameHeader h;
const size_t ret = ZSTD_getFrameHeader(&h, src, src_size);
if (ret != 0)
return ret;
if (h.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN)
params->frame_content_size = h.frameContentSize;
else
params->frame_content_size = 0;
params->window_size = h.windowSize;
params->dict_id = h.dictID;
params->checksum_flag = h.checksumFlag;
return ret;
return ZSTD_getFrameHeader(header, src, src_size);
}
EXPORT_SYMBOL(zstd_get_frame_params);
EXPORT_SYMBOL(zstd_get_frame_header);
MODULE_LICENSE("Dual BSD/GPL");
MODULE_DESCRIPTION("Zstd Decompressor");
contrib/linux-kernel/zstd_deps.h
+5 -4
View File
@@ -1,5 +1,6 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -71,7 +72,7 @@ static uint64_t ZSTD_div64(uint64_t dividend, uint32_t divisor) {
#endif /* ZSTD_DEPS_MATH64 */
#endif /* ZSTD_DEPS_NEED_MATH64 */
/*
/*
* This is only requested when DEBUGLEVEL >= 1, meaning
* it is disabled in production.
* Need:
@@ -88,7 +89,7 @@ static uint64_t ZSTD_div64(uint64_t dividend, uint32_t divisor) {
#endif /* ZSTD_DEPS_ASSERT */
#endif /* ZSTD_DEPS_NEED_ASSERT */
/*
/*
* This is only requested when DEBUGLEVEL >= 2, meaning
* it is disabled in production.
* Need:
@@ -105,7 +106,7 @@ static uint64_t ZSTD_div64(uint64_t dividend, uint32_t divisor) {
#endif /* ZSTD_DEPS_IO */
#endif /* ZSTD_DEPS_NEED_IO */
/*
/*
* Only requested when MSAN is enabled.
* Need:
* intptr_t
contrib/seekable_format/examples/seekable_decompression.c
+3
View File
@@ -99,6 +99,9 @@ static void decompressFile_orDie(const char* fname, off_t startOffset, off_t end
while (startOffset < endOffset) {
size_t const result = ZSTD_seekable_decompress(seekable, buffOut, MIN(endOffset - startOffset, buffOutSize), startOffset);
if (!result) {
break;
}
if (ZSTD_isError(result)) {
fprintf(stderr, "ZSTD_seekable_decompress() error : %s \n",
contrib/seekable_format/examples/seekable_decompression_mem.c
+3
View File
@@ -104,6 +104,9 @@ static void decompressFile_orDie(const char* fname, off_t startOffset, off_t end
while (startOffset < endOffset) {
size_t const result = ZSTD_seekable_decompress(seekable, buffOut, MIN(endOffset - startOffset, buffOutSize), startOffset);
if (!result) {
break;
}
if (ZSTD_isError(result)) {
fprintf(stderr, "ZSTD_seekable_decompress() error : %s \n",
contrib/seekable_format/zstdseek_decompress.c
+8 -1
View File
@@ -433,6 +433,11 @@ size_t ZSTD_seekable_initAdvanced(ZSTD_seekable* zs, ZSTD_seekable_customFile sr
size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsigned long long offset)
{
unsigned long long const eos = zs->seekTable.entries[zs->seekTable.tableLen].dOffset;
if (offset + len > eos) {
len = eos - offset;
}
U32 targetFrame = ZSTD_seekable_offsetToFrameIndex(zs, offset);
U32 noOutputProgressCount = 0;
size_t srcBytesRead = 0;
@@ -449,7 +454,7 @@ size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsign
zs->in = (ZSTD_inBuffer){zs->inBuff, 0, 0};
XXH64_reset(&zs->xxhState, 0);
ZSTD_DCtx_reset(zs->dstream, ZSTD_reset_session_only);
if (srcBytesRead > zs->buffWrapper.size) {
if (zs->buffWrapper.size && srcBytesRead > zs->buffWrapper.size) {
return ERROR(seekableIO);
}
}
@@ -502,6 +507,8 @@ size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t len, unsign
if (zs->decompressedOffset < offset + len) {
/* go back to the start and force a reset of the stream */
targetFrame = ZSTD_seekable_offsetToFrameIndex(zs, zs->decompressedOffset);
/* in this case it will fail later with corruption_detected, since last block does not have checksum */
assert(targetFrame != zs->seekTable.tableLen);
}
break;
}
doc/educational_decoder/Makefile
+1 -1
View File
@@ -1,5 +1,5 @@
# ################################################################
# Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
# Copyright (c) Yann Collet, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
doc/educational_decoder/harness.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2017-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
doc/educational_decoder/zstd_decompress.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2017-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
doc/educational_decoder/zstd_decompress.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
doc/zstd_manual.html
+11 -8
View File
@@ -143,6 +143,7 @@ unsigned ZSTD_isError(size_t code); </b>/*!< tells if a `size_t` fun
const char* ZSTD_getErrorName(size_t code); </b>/*!< provides readable string from an error code */<b>
int ZSTD_minCLevel(void); </b>/*!< minimum negative compression level allowed */<b>
int ZSTD_maxCLevel(void); </b>/*!< maximum compression level available */<b>
int ZSTD_defaultCLevel(void); </b>/*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT */<b>
</pre></b><BR>
<a name="Chapter4"></a><h2>Explicit context</h2><pre></pre>
@@ -270,6 +271,10 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
* The higher the value of selected strategy, the more complex it is,
* resulting in stronger and slower compression.
* Special: value 0 means "use default strategy". */
ZSTD_c_literalCompressionMode=108, </b>/* Controls how the literals are compressed (default is auto).<b>
* The value must be of type ZSTD_literalCompressionMode_e.
* See ZSTD_literalCompressionMode_t enum definition for details.
*/
</b>/* LDM mode parameters */<b>
ZSTD_c_enableLongDistanceMatching=160, </b>/* Enable long distance matching.<b>
@@ -349,7 +354,6 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
* ZSTD_c_format
* ZSTD_c_forceMaxWindow
* ZSTD_c_forceAttachDict
* ZSTD_c_literalCompressionMode
* ZSTD_c_targetCBlockSize
* ZSTD_c_srcSizeHint
* ZSTD_c_enableDedicatedDictSearch
@@ -365,7 +369,6 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
ZSTD_c_experimentalParam2=10,
ZSTD_c_experimentalParam3=1000,
ZSTD_c_experimentalParam4=1001,
ZSTD_c_experimentalParam5=1002,
ZSTD_c_experimentalParam6=1003,
ZSTD_c_experimentalParam7=1004,
ZSTD_c_experimentalParam8=1005,
@@ -770,6 +773,12 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds);
It can still be loaded, but as a content-only dictionary.
</p></pre><BR>
<pre><b>unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict);
</b><p> Provides the dictID of the dictionary loaded into `cdict`.
If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
Non-conformant dictionaries can still be loaded, but as content-only dictionaries.
</p></pre><BR>
<pre><b>unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
</b><p> Provides the dictID of the dictionary loaded into `ddict`.
If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
@@ -1294,12 +1303,6 @@ ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; </b>/**< this con
note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef
</p></pre><BR>
<pre><b>unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict);
</b><p> Provides the dictID of the dictionary loaded into `cdict`.
If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
Non-conformant dictionaries can still be loaded, but as content-only dictionaries.
</p></pre><BR>
<pre><b>ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize);
</b><p> @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
`estimatedSrcSize` value is optional, select 0 if not known
examples/Makefile
+1 -1
View File
@@ -1,5 +1,5 @@
# ################################################################
# Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
# Copyright (c) Yann Collet, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
examples/common.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
examples/dictionary_compression.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021 Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
examples/dictionary_decompression.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
examples/multiple_simple_compression.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
examples/multiple_streaming_compression.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
examples/simple_compression.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
examples/simple_decompression.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
examples/streaming_compression.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
examples/streaming_compression_thread_pool.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2021, Martin Liska, SUSE, Facebook, Inc.
* Copyright (c) Martin Liska, SUSE, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
examples/streaming_decompression.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
examples/streaming_memory_usage.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2017-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/BUCK
+5 -7
View File
@@ -65,9 +65,7 @@ cxx_library(
name='zdict',
header_namespace='',
visibility=['PUBLIC'],
exported_headers=subdir_glob([
('dictBuilder', 'zdict.h'),
]),
exported_headers=['zdict.h'],
headers=subdir_glob([
('dictBuilder', 'divsufsort.h'),
('dictBuilder', 'cover.h'),
@@ -131,10 +129,10 @@ cxx_library(
name='errors',
header_namespace='',
visibility=['PUBLIC'],
exported_headers=subdir_glob([
('common', 'error_private.h'),
('common', 'zstd_errors.h'),
]),
exported_headers=[
'zstd_errors.h',
'common/error_private.h',
]
srcs=['common/error_private.c'],
)
lib/Makefile
+7 -5
View File
@@ -1,5 +1,5 @@
# ################################################################
# Copyright (c) 2015-2021, Yann Collet, Facebook, Inc.
# Copyright (c) Yann Collet, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
@@ -91,7 +91,7 @@ endif
ZSTD_LIB_COMPRESSION ?= 1
ZSTD_LIB_DECOMPRESSION ?= 1
ZSTD_LIB_DICTBUILDER ?= 1
ZSTD_LIB_DEPRECATED ?= 1
ZSTD_LIB_DEPRECATED ?= 0
# Legacy support
ZSTD_LEGACY_SUPPORT ?= 5
@@ -176,7 +176,9 @@ UNAME := $(shell uname)
ifndef BUILD_DIR
ifeq ($(UNAME), Darwin)
HASH ?= md5
ifeq ($(shell md5 < /dev/null > /dev/null; echo $$?), 0)
HASH ?= md5
endif
else ifeq ($(UNAME), FreeBSD)
HASH ?= gmd5sum
else ifeq ($(UNAME), NetBSD)
@@ -435,8 +437,8 @@ install-includes:
[ -e $(DESTDIR)$(INCLUDEDIR) ] || $(INSTALL) -d -m 755 $(DESTDIR)$(INCLUDEDIR)/
@echo Installing includes
$(INSTALL_DATA) zstd.h $(DESTDIR)$(INCLUDEDIR)
$(INSTALL_DATA) common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)
$(INSTALL_DATA) dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)
$(INSTALL_DATA) zstd_errors.h $(DESTDIR)$(INCLUDEDIR)
$(INSTALL_DATA) zdict.h $(DESTDIR)$(INCLUDEDIR)
.PHONY: uninstall
uninstall:
lib/README.md
+2 -2
View File
@@ -42,8 +42,8 @@ Zstandard's stable API is exposed within [lib/zstd.h](zstd.h).
Optional advanced features are exposed via :
- `lib/common/zstd_errors.h` : translates `size_t` function results
into a `ZSTD_ErrorCode`, for accurate error handling.
- `lib/zstd_errors.h` : translates `size_t` function results
into a `ZSTD_ErrorCode`, for accurate error handling.
- `ZSTD_STATIC_LINKING_ONLY` : if this macro is defined _before_ including `zstd.h`,
it unlocks access to the experimental API,
lib/common/bitstream.h
+1 -1
View File
@@ -1,7 +1,7 @@
/* ******************************************************************
* bitstream
* Part of FSE library
* Copyright (c) 2013-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
*
* You can contact the author at :
* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
lib/common/compiler.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/common/cpu.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2018-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/common/debug.c
+1 -1
View File
@@ -1,7 +1,7 @@
/* ******************************************************************
* debug
* Part of FSE library
* Copyright (c) 2013-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
*
* You can contact the author at :
* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
lib/common/debug.h
+1 -1
View File
@@ -1,7 +1,7 @@
/* ******************************************************************
* debug
* Part of FSE library
* Copyright (c) 2013-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
*
* You can contact the author at :
* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
lib/common/entropy_common.c
+1 -1
View File
@@ -1,6 +1,6 @@
/* ******************************************************************
* Common functions of New Generation Entropy library
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
*
* You can contact the author at :
* - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
lib/common/error_private.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/common/error_private.h
+3 -3
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -21,8 +21,8 @@ extern "C" {
/* ****************************************
* Dependencies
******************************************/
#include "zstd_deps.h" /* size_t */
#include "zstd_errors.h" /* enum list */
#include "../zstd_errors.h" /* enum list */
#include "zstd_deps.h" /* size_t */
/* ****************************************
lib/common/fse.h
+2 -2
View File
@@ -1,7 +1,7 @@
/* ******************************************************************
* FSE : Finite State Entropy codec
* Public Prototypes declaration
* Copyright (c) 2013-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
*
* You can contact the author at :
* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -352,7 +352,7 @@ size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits);
size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
/**< build a fake FSE_DTable, designed to always generate the same symbolValue */
#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue))
#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1)
#define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned))
size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize);
/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)` */
lib/common/fse_decompress.c
+25 -15
View File
@@ -1,6 +1,6 @@
/* ******************************************************************
* FSE : Finite State Entropy decoder
* Copyright (c) 2013-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
*
* You can contact the author at :
* - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -310,6 +310,12 @@ size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size
return FSE_decompress_wksp_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, /* bmi2 */ 0);
}
typedef struct {
short ncount[FSE_MAX_SYMBOL_VALUE + 1];
FSE_DTable dtable[1]; /* Dynamically sized */
} FSE_DecompressWksp;
FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body(
void* dst, size_t dstCapacity,
const void* cSrc, size_t cSrcSize,
@@ -318,33 +324,37 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body(
{
const BYTE* const istart = (const BYTE*)cSrc;
const BYTE* ip = istart;
short counting[FSE_MAX_SYMBOL_VALUE+1];
unsigned tableLog;
unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
FSE_DTable* const dtable = (FSE_DTable*)workSpace;
FSE_DecompressWksp* const wksp = (FSE_DecompressWksp*)workSpace;
DEBUG_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0);
if (wkspSize < sizeof(*wksp)) return ERROR(GENERIC);
/* normal FSE decoding mode */
size_t const NCountLength = FSE_readNCount_bmi2(counting, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2);
if (FSE_isError(NCountLength)) return NCountLength;
if (tableLog > maxLog) return ERROR(tableLog_tooLarge);
assert(NCountLength <= cSrcSize);
ip += NCountLength;
cSrcSize -= NCountLength;
{
size_t const NCountLength = FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2);
if (FSE_isError(NCountLength)) return NCountLength;
if (tableLog > maxLog) return ERROR(tableLog_tooLarge);
assert(NCountLength <= cSrcSize);
ip += NCountLength;
cSrcSize -= NCountLength;
}
if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge);
workSpace = dtable + FSE_DTABLE_SIZE_U32(tableLog);
wkspSize -= FSE_DTABLE_SIZE(tableLog);
workSpace = wksp->dtable + FSE_DTABLE_SIZE_U32(tableLog);
wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog);
CHECK_F( FSE_buildDTable_internal(dtable, counting, maxSymbolValue, tableLog, workSpace, wkspSize) );
CHECK_F( FSE_buildDTable_internal(wksp->dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) );
{
const void* ptr = dtable;
const void* ptr = wksp->dtable;
const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
const U32 fastMode = DTableH->fastMode;
/* select fast mode (static) */
if (fastMode) return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, dtable, 1);
return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, dtable, 0);
if (fastMode) return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 1);
return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 0);
}
}
lib/common/huf.h
+2 -2
View File
@@ -1,7 +1,7 @@
/* ******************************************************************
* huff0 huffman codec,
* part of Finite State Entropy library
* Copyright (c) 2013-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
*
* You can contact the author at :
* - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -279,7 +279,7 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
* a required workspace size greater than that specified in the following
* macro.
*/
#define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10)
#define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9))
#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
#ifndef HUF_FORCE_DECOMPRESS_X2
lib/common/mem.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/common/pool.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/common/pool.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/common/xxhash.c
+1 -1
View File
@@ -1,6 +1,6 @@
/*
* xxHash - Fast Hash algorithm
* Copyright (c) 2012-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
*
* You can contact the author at :
* - xxHash homepage: http://www.xxhash.com
lib/common/xxhash.h
+1 -1
View File
@@ -1,7 +1,7 @@
/*
* xxHash - Extremely Fast Hash algorithm
* Header File
* Copyright (c) 2012-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
*
* You can contact the author at :
* - xxHash source repository : https://github.com/Cyan4973/xxHash
lib/common/zstd_common.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/common/zstd_deps.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/common/zstd_internal.h
+21 -9
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -36,6 +36,11 @@
# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
#endif
#include "xxhash.h" /* XXH_reset, update, digest */
#ifndef ZSTD_NO_TRACE
# include "zstd_trace.h"
#else
# define ZSTD_TRACE 0
#endif
#if defined (__cplusplus)
extern "C" {
@@ -347,11 +352,18 @@ typedef enum {
* Private declarations
*********************************************/
typedef struct seqDef_s {
U32 offset; /* Offset code of the sequence */
U32 offset; /* offset == rawOffset + ZSTD_REP_NUM, or equivalently, offCode + 1 */
U16 litLength;
U16 matchLength;
} seqDef;
/* Controls whether seqStore has a single "long" litLength or matchLength. See seqStore_t. */
typedef enum {
ZSTD_llt_none = 0, /* no longLengthType */
ZSTD_llt_literalLength = 1, /* represents a long literal */
ZSTD_llt_matchLength = 2 /* represents a long match */
} ZSTD_longLengthType_e;
typedef struct {
seqDef* sequencesStart;
seqDef* sequences; /* ptr to end of sequences */
@@ -363,12 +375,12 @@ typedef struct {
size_t maxNbSeq;
size_t maxNbLit;
/* longLengthPos and longLengthID to allow us to represent either a single litLength or matchLength
/* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength
* in the seqStore that has a value larger than U16 (if it exists). To do so, we increment
* the existing value of the litLength or matchLength by 0x10000.
* the existing value of the litLength or matchLength by 0x10000.
*/
U32 longLengthID; /* 0 == no longLength; 1 == Represent the long literal; 2 == Represent the long match; */
U32 longLengthPos; /* Index of the sequence to apply long length modification to */
ZSTD_longLengthType_e longLengthType;
U32 longLengthPos; /* Index of the sequence to apply long length modification to */
} seqStore_t;
typedef struct {
@@ -378,7 +390,7 @@ typedef struct {
/**
* Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences
* indicated by longLengthPos and longLengthID, and adds MINMATCH back to matchLength.
* indicated by longLengthPos and longLengthType, and adds MINMATCH back to matchLength.
*/
MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq)
{
@@ -386,10 +398,10 @@ MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore
seqLen.litLength = seq->litLength;
seqLen.matchLength = seq->matchLength + MINMATCH;
if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) {
if (seqStore->longLengthID == 1) {
if (seqStore->longLengthType == ZSTD_llt_literalLength) {
seqLen.litLength += 0xFFFF;
}
if (seqStore->longLengthID == 2) {
if (seqStore->longLengthType == ZSTD_llt_matchLength) {
seqLen.matchLength += 0xFFFF;
}
}
lib/common/zstd_trace.c
-42
View File
@@ -1,42 +0,0 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#include "zstd_trace.h"
#include "../zstd.h"
#include "compiler.h"
#if ZSTD_TRACE && ZSTD_HAVE_WEAK_SYMBOLS
ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_compress_begin(ZSTD_CCtx const* cctx)
{
(void)cctx;
return 0;
}
ZSTD_WEAK_ATTR void ZSTD_trace_compress_end(ZSTD_TraceCtx ctx, ZSTD_Trace const* trace)
{
(void)ctx;
(void)trace;
}
ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_decompress_begin(ZSTD_DCtx const* dctx)
{
(void)dctx;
return 0;
}
ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end(ZSTD_TraceCtx ctx, ZSTD_Trace const* trace)
{
(void)ctx;
(void)trace;
}
#endif
lib/common/zstd_trace.h
+7 -5
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -114,14 +114,15 @@ typedef unsigned long long ZSTD_TraceCtx;
* @returns Non-zero if tracing is enabled. The return value is
* passed to ZSTD_trace_compress_end().
*/
ZSTD_TraceCtx ZSTD_trace_compress_begin(struct ZSTD_CCtx_s const* cctx);
ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_compress_begin(
struct ZSTD_CCtx_s const* cctx);
/**
* Trace the end of a compression call.
* @param ctx The return value of ZSTD_trace_compress_begin().
* @param trace The zstd tracing info.
*/
void ZSTD_trace_compress_end(
ZSTD_WEAK_ATTR void ZSTD_trace_compress_end(
ZSTD_TraceCtx ctx,
ZSTD_Trace const* trace);
@@ -132,14 +133,15 @@ void ZSTD_trace_compress_end(
* @returns Non-zero if tracing is enabled. The return value is
* passed to ZSTD_trace_compress_end().
*/
ZSTD_TraceCtx ZSTD_trace_decompress_begin(struct ZSTD_DCtx_s const* dctx);
ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_decompress_begin(
struct ZSTD_DCtx_s const* dctx);
/**
* Trace the end of a decompression call.
* @param ctx The return value of ZSTD_trace_decompress_begin().
* @param trace The zstd tracing info.
*/
void ZSTD_trace_decompress_end(
ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end(
ZSTD_TraceCtx ctx,
ZSTD_Trace const* trace);
lib/compress/fse_compress.c
+1 -1
View File
@@ -1,6 +1,6 @@
/* ******************************************************************
* FSE : Finite State Entropy encoder
* Copyright (c) 2013-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
*
* You can contact the author at :
* - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
lib/compress/hist.c
+1 -1
View File
@@ -1,7 +1,7 @@
/* ******************************************************************
* hist : Histogram functions
* part of Finite State Entropy project
* Copyright (c) 2013-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
*
* You can contact the author at :
* - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
lib/compress/hist.h
+1 -1
View File
@@ -1,7 +1,7 @@
/* ******************************************************************
* hist : Histogram functions
* part of Finite State Entropy project
* Copyright (c) 2013-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
*
* You can contact the author at :
* - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
lib/compress/huf_compress.c
+1 -1
View File
@@ -1,6 +1,6 @@
/* ******************************************************************
* Huffman encoder, part of New Generation Entropy library
* Copyright (c) 2013-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
*
* You can contact the author at :
* - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
lib/compress/zstd_compress.c
+1267 -234
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -14,7 +14,6 @@
#include "../common/zstd_deps.h" /* INT_MAX, ZSTD_memset, ZSTD_memcpy */
#include "../common/cpu.h"
#include "../common/mem.h"
#include "../common/zstd_trace.h"
#include "hist.h" /* HIST_countFast_wksp */
#define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */
#include "../common/fse.h"
@@ -73,6 +72,10 @@ struct ZSTD_CDict_s {
ZSTD_customMem customMem;
U32 dictID;
int compressionLevel; /* 0 indicates that advanced API was used to select CDict params */
ZSTD_useRowMatchFinderMode_e useRowMatchFinder; /* Indicates whether the CDict was created with params that would use
* row-based matchfinder. Unless the cdict is reloaded, we will use
* the same greedy/lazy matchfinder at compression time.
*/
}; /* typedef'd to ZSTD_CDict within "zstd.h" */
ZSTD_CCtx* ZSTD_createCCtx(void)
@@ -203,6 +206,49 @@ size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
/* private API call, for dictBuilder only */
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); }
/* Returns true if the strategy supports using a row based matchfinder */
static int ZSTD_rowMatchFinderSupported(const ZSTD_strategy strategy) {
return (strategy >= ZSTD_greedy && strategy <= ZSTD_lazy2);
}
/* Returns true if the strategy and useRowMatchFinder mode indicate that we will use the row based matchfinder
* for this compression.
*/
static int ZSTD_rowMatchFinderUsed(const ZSTD_strategy strategy, const ZSTD_useRowMatchFinderMode_e mode) {
assert(mode != ZSTD_urm_auto);
return ZSTD_rowMatchFinderSupported(strategy) && (mode == ZSTD_urm_enableRowMatchFinder);
}
/* Returns row matchfinder usage enum given an initial mode and cParams */
static ZSTD_useRowMatchFinderMode_e ZSTD_resolveRowMatchFinderMode(ZSTD_useRowMatchFinderMode_e mode,
const ZSTD_compressionParameters* const cParams) {
#if !defined(ZSTD_NO_INTRINSICS) && (defined(__SSE2__) || defined(__ARM_NEON))
int const kHasSIMD128 = 1;
#else
int const kHasSIMD128 = 0;
#endif
if (mode != ZSTD_urm_auto) return mode; /* if requested enabled, but no SIMD, we still will use row matchfinder */
mode = ZSTD_urm_disableRowMatchFinder;
if (!ZSTD_rowMatchFinderSupported(cParams->strategy)) return mode;
if (kHasSIMD128) {
if (cParams->windowLog > 14) mode = ZSTD_urm_enableRowMatchFinder;
} else {
if (cParams->windowLog > 17) mode = ZSTD_urm_enableRowMatchFinder;
}
return mode;
}
/* Returns 1 if the arguments indicate that we should allocate a chainTable, 0 otherwise */
static int ZSTD_allocateChainTable(const ZSTD_strategy strategy,
const ZSTD_useRowMatchFinderMode_e useRowMatchFinder,
const U32 forDDSDict) {
assert(useRowMatchFinder != ZSTD_urm_auto);
/* We always should allocate a chaintable if we are allocating a matchstate for a DDS dictionary matchstate.
* We do not allocate a chaintable if we are using ZSTD_fast, or are using the row-based matchfinder.
*/
return forDDSDict || ((strategy != ZSTD_fast) && !ZSTD_rowMatchFinderUsed(strategy, useRowMatchFinder));
}
/* Returns 1 if compression parameters are such that we should
* enable long distance matching (wlog >= 27, strategy >= btopt).
* Returns 0 otherwise.
@@ -211,6 +257,14 @@ static U32 ZSTD_CParams_shouldEnableLdm(const ZSTD_compressionParameters* const
return cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27;
}
/* Returns 1 if compression parameters are such that we should
* enable blockSplitter (wlog >= 17, strategy >= btopt).
* Returns 0 otherwise.
*/
static U32 ZSTD_CParams_useBlockSplitter(const ZSTD_compressionParameters* const cParams) {
return cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 17;
}
static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
ZSTD_compressionParameters cParams)
{
@@ -219,6 +273,7 @@ static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
ZSTD_CCtxParams_init(&cctxParams, ZSTD_CLEVEL_DEFAULT);
cctxParams.cParams = cParams;
/* Adjust advanced params according to cParams */
if (ZSTD_CParams_shouldEnableLdm(&cParams)) {
DEBUGLOG(4, "ZSTD_makeCCtxParamsFromCParams(): Including LDM into cctx params");
cctxParams.ldmParams.enableLdm = 1;
@@ -228,6 +283,12 @@ static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
assert(cctxParams.ldmParams.hashRateLog < 32);
}
if (ZSTD_CParams_useBlockSplitter(&cParams)) {
DEBUGLOG(4, "ZSTD_makeCCtxParamsFromCParams(): Including block splitting into cctx params");
cctxParams.splitBlocks = 1;
}
cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams);
assert(!ZSTD_checkCParams(cParams));
return cctxParams;
}
@@ -286,6 +347,8 @@ static void ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, ZSTD_par
* But, set it for tracing anyway.
*/
cctxParams->compressionLevel = compressionLevel;
cctxParams->useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams->useRowMatchFinder, &params->cParams);
DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d", cctxParams->useRowMatchFinder);
}
size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params)
@@ -486,6 +549,16 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param)
bounds.upperBound = 1;
return bounds;
case ZSTD_c_splitBlocks:
bounds.lowerBound = 0;
bounds.upperBound = 1;
return bounds;
case ZSTD_c_useRowMatchFinder:
bounds.lowerBound = (int)ZSTD_urm_auto;
bounds.upperBound = (int)ZSTD_urm_enableRowMatchFinder;
return bounds;
default:
bounds.error = ERROR(parameter_unsupported);
return bounds;
@@ -547,6 +620,8 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param)
case ZSTD_c_stableOutBuffer:
case ZSTD_c_blockDelimiters:
case ZSTD_c_validateSequences:
case ZSTD_c_splitBlocks:
case ZSTD_c_useRowMatchFinder:
default:
return 0;
}
@@ -599,6 +674,8 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value)
case ZSTD_c_stableOutBuffer:
case ZSTD_c_blockDelimiters:
case ZSTD_c_validateSequences:
case ZSTD_c_splitBlocks:
case ZSTD_c_useRowMatchFinder:
break;
default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
@@ -810,6 +887,16 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams,
CCtxParams->validateSequences = value;
return CCtxParams->validateSequences;
case ZSTD_c_splitBlocks:
BOUNDCHECK(ZSTD_c_splitBlocks, value);
CCtxParams->splitBlocks = value;
return CCtxParams->splitBlocks;
case ZSTD_c_useRowMatchFinder:
BOUNDCHECK(ZSTD_c_useRowMatchFinder, value);
CCtxParams->useRowMatchFinder = (ZSTD_useRowMatchFinderMode_e)value;
return CCtxParams->useRowMatchFinder;
default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
}
}
@@ -933,6 +1020,12 @@ size_t ZSTD_CCtxParams_getParameter(
case ZSTD_c_validateSequences :
*value = (int)CCtxParams->validateSequences;
break;
case ZSTD_c_splitBlocks :
*value = (int)CCtxParams->splitBlocks;
break;
case ZSTD_c_useRowMatchFinder :
*value = (int)CCtxParams->useRowMatchFinder;
break;
default: RETURN_ERROR(parameter_unsupported, "unknown parameter");
}
return 0;
@@ -1299,9 +1392,14 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
static size_t
ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
const ZSTD_useRowMatchFinderMode_e useRowMatchFinder,
const U32 enableDedicatedDictSearch,
const U32 forCCtx)
{
size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog);
/* chain table size should be 0 for fast or row-hash strategies */
size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder, enableDedicatedDictSearch && !forCCtx)
? ((size_t)1 << cParams->chainLog)
: 0;
size_t const hSize = ((size_t)1) << cParams->hashLog;
U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0;
size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0;
@@ -1311,24 +1409,34 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
+ hSize * sizeof(U32)
+ h3Size * sizeof(U32);
size_t const optPotentialSpace =
ZSTD_cwksp_alloc_size((MaxML+1) * sizeof(U32))
+ ZSTD_cwksp_alloc_size((MaxLL+1) * sizeof(U32))
+ ZSTD_cwksp_alloc_size((MaxOff+1) * sizeof(U32))
+ ZSTD_cwksp_alloc_size((1<<Litbits) * sizeof(U32))
+ ZSTD_cwksp_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t))
+ ZSTD_cwksp_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t));
ZSTD_cwksp_aligned_alloc_size((MaxML+1) * sizeof(U32))
+ ZSTD_cwksp_aligned_alloc_size((MaxLL+1) * sizeof(U32))
+ ZSTD_cwksp_aligned_alloc_size((MaxOff+1) * sizeof(U32))
+ ZSTD_cwksp_aligned_alloc_size((1<<Litbits) * sizeof(U32))
+ ZSTD_cwksp_aligned_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t))
+ ZSTD_cwksp_aligned_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t));
size_t const lazyAdditionalSpace = ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)
? ZSTD_cwksp_aligned_alloc_size(hSize*sizeof(U16))
: 0;
size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt))
? optPotentialSpace
: 0;
size_t const slackSpace = ZSTD_cwksp_slack_space_required();
/* tables are guaranteed to be sized in multiples of 64 bytes (or 16 uint32_t) */
ZSTD_STATIC_ASSERT(ZSTD_HASHLOG_MIN >= 4 && ZSTD_WINDOWLOG_MIN >= 4 && ZSTD_CHAINLOG_MIN >= 4);
assert(useRowMatchFinder != ZSTD_urm_auto);
DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u",
(U32)chainSize, (U32)hSize, (U32)h3Size);
return tableSpace + optSpace;
return tableSpace + optSpace + slackSpace + lazyAdditionalSpace;
}
static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
const ZSTD_compressionParameters* cParams,
const ldmParams_t* ldmParams,
const int isStatic,
const ZSTD_useRowMatchFinderMode_e useRowMatchFinder,
const size_t buffInSize,
const size_t buffOutSize,
const U64 pledgedSrcSize)
@@ -1338,16 +1446,16 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal(
U32 const divider = (cParams->minMatch==3) ? 3 : 4;
size_t const maxNbSeq = blockSize / divider;
size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize)
+ ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(seqDef))
+ ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef))
+ 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE));
size_t const entropySpace = ZSTD_cwksp_alloc_size(ENTROPY_WORKSPACE_SIZE);
size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t));
size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, /* forCCtx */ 1);
size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 0, /* forCCtx */ 1);
size_t const ldmSpace = ZSTD_ldm_getTableSize(*ldmParams);
size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(*ldmParams, blockSize);
size_t const ldmSeqSpace = ldmParams->enableLdm ?
ZSTD_cwksp_alloc_size(maxNbLdmSeq * sizeof(rawSeq)) : 0;
ZSTD_cwksp_aligned_alloc_size(maxNbLdmSeq * sizeof(rawSeq)) : 0;
size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize)
@@ -1373,25 +1481,45 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params)
{
ZSTD_compressionParameters const cParams =
ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
ZSTD_useRowMatchFinderMode_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder,
&cParams);
RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only.");
/* estimateCCtxSize is for one-shot compression. So no buffers should
* be needed. However, we still allocate two 0-sized buffers, which can
* take space under ASAN. */
return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
&cParams, &params->ldmParams, 1, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN);
&cParams, &params->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN);
}
size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams)
{
ZSTD_CCtx_params const params = ZSTD_makeCCtxParamsFromCParams(cParams);
return ZSTD_estimateCCtxSize_usingCCtxParams(&params);
ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams);
if (ZSTD_rowMatchFinderSupported(cParams.strategy)) {
/* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */
size_t noRowCCtxSize;
size_t rowCCtxSize;
initialParams.useRowMatchFinder = ZSTD_urm_disableRowMatchFinder;
noRowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams);
initialParams.useRowMatchFinder = ZSTD_urm_enableRowMatchFinder;
rowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams);
return MAX(noRowCCtxSize, rowCCtxSize);
} else {
return ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams);
}
}
static size_t ZSTD_estimateCCtxSize_internal(int compressionLevel)
{
ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict);
return ZSTD_estimateCCtxSize_usingCParams(cParams);
int tier = 0;
size_t largestSize = 0;
static const unsigned long long srcSizeTiers[4] = {16 KB, 128 KB, 256 KB, ZSTD_CONTENTSIZE_UNKNOWN};
for (; tier < 4; ++tier) {
/* Choose the set of cParams for a given level across all srcSizes that give the largest cctxSize */
ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeTiers[tier], 0, ZSTD_cpm_noAttachDict);
largestSize = MAX(ZSTD_estimateCCtxSize_usingCParams(cParams), largestSize);
}
return largestSize;
}
size_t ZSTD_estimateCCtxSize(int compressionLevel)
@@ -1399,6 +1527,7 @@ size_t ZSTD_estimateCCtxSize(int compressionLevel)
int level;
size_t memBudget = 0;
for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) {
/* Ensure monotonically increasing memory usage as compression level increases */
size_t const newMB = ZSTD_estimateCCtxSize_internal(level);
if (newMB > memBudget) memBudget = newMB;
}
@@ -1417,17 +1546,29 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params)
size_t const outBuffSize = (params->outBufferMode == ZSTD_bm_buffered)
? ZSTD_compressBound(blockSize) + 1
: 0;
ZSTD_useRowMatchFinderMode_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, &params->cParams);
return ZSTD_estimateCCtxSize_usingCCtxParams_internal(
&cParams, &params->ldmParams, 1, inBuffSize, outBuffSize,
&cParams, &params->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize,
ZSTD_CONTENTSIZE_UNKNOWN);
}
}
size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams)
{
ZSTD_CCtx_params const params = ZSTD_makeCCtxParamsFromCParams(cParams);
return ZSTD_estimateCStreamSize_usingCCtxParams(&params);
ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams);
if (ZSTD_rowMatchFinderSupported(cParams.strategy)) {
/* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */
size_t noRowCCtxSize;
size_t rowCCtxSize;
initialParams.useRowMatchFinder = ZSTD_urm_disableRowMatchFinder;
noRowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams);
initialParams.useRowMatchFinder = ZSTD_urm_enableRowMatchFinder;
rowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams);
return MAX(noRowCCtxSize, rowCCtxSize);
} else {
return ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams);
}
}
static size_t ZSTD_estimateCStreamSize_internal(int compressionLevel)
@@ -1552,20 +1693,27 @@ typedef enum {
ZSTD_resetTarget_CCtx
} ZSTD_resetTarget_e;
static size_t
ZSTD_reset_matchState(ZSTD_matchState_t* ms,
ZSTD_cwksp* ws,
const ZSTD_compressionParameters* cParams,
const ZSTD_useRowMatchFinderMode_e useRowMatchFinder,
const ZSTD_compResetPolicy_e crp,
const ZSTD_indexResetPolicy_e forceResetIndex,
const ZSTD_resetTarget_e forWho)
{
size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog);
/* disable chain table allocation for fast or row-based strategies */
size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder,
ms->dedicatedDictSearch && (forWho == ZSTD_resetTarget_CDict))
? ((size_t)1 << cParams->chainLog)
: 0;
size_t const hSize = ((size_t)1) << cParams->hashLog;
U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0;
size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0;
DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset);
assert(useRowMatchFinder != ZSTD_urm_auto);
if (forceResetIndex == ZSTDirp_reset) {
ZSTD_window_init(&ms->window);
ZSTD_cwksp_mark_tables_dirty(ws);
@@ -1604,11 +1752,23 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms,
ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t));
}
if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) {
{ /* Row match finder needs an additional table of hashes ("tags") */
size_t const tagTableSize = hSize*sizeof(U16);
ms->tagTable = (U16*)ZSTD_cwksp_reserve_aligned(ws, tagTableSize);
if (ms->tagTable) ZSTD_memset(ms->tagTable, 0, tagTableSize);
}
{ /* Switch to 32-entry rows if searchLog is 5 (or more) */
U32 const rowLog = cParams->searchLog < 5 ? 4 : 5;
assert(cParams->hashLog > rowLog);
ms->rowHashLog = cParams->hashLog - rowLog;
}
}
ms->cParams = *cParams;
RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation,
"failed a workspace allocation in ZSTD_reset_matchState");
return 0;
}
@@ -1625,21 +1785,37 @@ static int ZSTD_indexTooCloseToMax(ZSTD_window_t w)
return (size_t)(w.nextSrc - w.base) > (ZSTD_CURRENT_MAX - ZSTD_INDEXOVERFLOW_MARGIN);
}
/** ZSTD_dictTooBig():
* When dictionaries are larger than ZSTD_CHUNKSIZE_MAX they can't be loaded in
* one go generically. So we ensure that in that case we reset the tables to zero,
* so that we can load as much of the dictionary as possible.
*/
static int ZSTD_dictTooBig(size_t const loadedDictSize)
{
return loadedDictSize > ZSTD_CHUNKSIZE_MAX;
}
/*! ZSTD_resetCCtx_internal() :
note : `params` are assumed fully validated at this stage */
* @param loadedDictSize The size of the dictionary to be loaded
* into the context, if any. If no dictionary is used, or the
* dictionary is being attached / copied, then pass 0.
* note : `params` are assumed fully validated at this stage.
*/
static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
ZSTD_CCtx_params params,
U64 const pledgedSrcSize,
size_t const loadedDictSize,
ZSTD_compResetPolicy_e const crp,
ZSTD_buffered_policy_e const zbuff)
{
ZSTD_cwksp* const ws = &zc->workspace;
DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u",
(U32)pledgedSrcSize, params.cParams.windowLog);
DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u, useRowMatchFinder=%d",
(U32)pledgedSrcSize, params.cParams.windowLog, (int)params.useRowMatchFinder);
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
zc->isFirstBlock = 1;
assert(params.useRowMatchFinder != ZSTD_urm_auto);
if (params.ldmParams.enableLdm) {
/* Adjust long distance matching parameters */
ZSTD_ldm_adjustParameters(&params.ldmParams, &params.cParams);
@@ -1660,26 +1836,28 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize);
int const indexTooClose = ZSTD_indexTooCloseToMax(zc->blockState.matchState.window);
int const dictTooBig = ZSTD_dictTooBig(loadedDictSize);
ZSTD_indexResetPolicy_e needsIndexReset =
(!indexTooClose && zc->initialized) ? ZSTDirp_continue : ZSTDirp_reset;
(indexTooClose || dictTooBig || !zc->initialized) ? ZSTDirp_reset : ZSTDirp_continue;
size_t const neededSpace =
ZSTD_estimateCCtxSize_usingCCtxParams_internal(
&params.cParams, &params.ldmParams, zc->staticSize != 0,
&params.cParams, &params.ldmParams, zc->staticSize != 0, params.useRowMatchFinder,
buffInSize, buffOutSize, pledgedSrcSize);
int resizeWorkspace;
FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!");
if (!zc->staticSize) ZSTD_cwksp_bump_oversized_duration(ws, 0);
/* Check if workspace is large enough, alloc a new one if needed */
{
{ /* Check if workspace is large enough, alloc a new one if needed */
int const workspaceTooSmall = ZSTD_cwksp_sizeof(ws) < neededSpace;
int const workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace);
resizeWorkspace = workspaceTooSmall || workspaceWasteful;
DEBUGLOG(4, "Need %zu B workspace", neededSpace);
DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize);
if (workspaceTooSmall || workspaceWasteful) {
if (resizeWorkspace) {
DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB",
ZSTD_cwksp_sizeof(ws) >> 10,
neededSpace >> 10);
@@ -1760,6 +1938,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
&zc->blockState.matchState,
ws,
&params.cParams,
params.useRowMatchFinder,
crp,
needsIndexReset,
ZSTD_resetTarget_CCtx), "");
@@ -1774,17 +1953,12 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
zc->maxNbLdmSequences = maxNbLdmSeq;
ZSTD_window_init(&zc->ldmState.window);
ZSTD_window_clear(&zc->ldmState.window);
zc->ldmState.loadedDictEnd = 0;
}
/* Due to alignment, when reusing a workspace, we can actually consume
* up to 3 extra bytes for alignment. See the comments in zstd_cwksp.h
*/
assert(ZSTD_cwksp_used(ws) >= neededSpace &&
ZSTD_cwksp_used(ws) <= neededSpace + 3);
assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace, resizeWorkspace));
DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws));
zc->initialized = 1;
return 0;
@@ -1840,6 +2014,7 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,
U64 pledgedSrcSize,
ZSTD_buffered_policy_e zbuff)
{
DEBUGLOG(4, "ZSTD_resetCCtx_byAttachingCDict() pledgedSrcSize=%zu", pledgedSrcSize);
{
ZSTD_compressionParameters adjusted_cdict_cParams = cdict->matchState.cParams;
unsigned const windowLog = params.cParams.windowLog;
@@ -1855,7 +2030,9 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx,
params.cParams = ZSTD_adjustCParams_internal(adjusted_cdict_cParams, pledgedSrcSize,
cdict->dictContentSize, ZSTD_cpm_attachDict);
params.cParams.windowLog = windowLog;
params.useRowMatchFinder = cdict->useRowMatchFinder; /* cdict overrides */
FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
/* loadedDictSize */ 0,
ZSTDcrp_makeClean, zbuff), "");
assert(cctx->appliedParams.cParams.strategy == adjusted_cdict_cParams.strategy);
}
@@ -1899,15 +2076,16 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx,
const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams;
assert(!cdict->matchState.dedicatedDictSearch);
DEBUGLOG(4, "copying dictionary into context");
DEBUGLOG(4, "ZSTD_resetCCtx_byCopyingCDict() pledgedSrcSize=%zu", pledgedSrcSize);
{ unsigned const windowLog = params.cParams.windowLog;
assert(windowLog != 0);
/* Copy only compression parameters related to tables. */
params.cParams = *cdict_cParams;
params.cParams.windowLog = windowLog;
params.useRowMatchFinder = cdict->useRowMatchFinder;
FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize,
/* loadedDictSize */ 0,
ZSTDcrp_leaveDirty, zbuff), "");
assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy);
assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog);
@@ -1915,17 +2093,30 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx,
}
ZSTD_cwksp_mark_tables_dirty(&cctx->workspace);
assert(params.useRowMatchFinder != ZSTD_urm_auto);
/* copy tables */
{ size_t const chainSize = (cdict_cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cdict_cParams->chainLog);
{ size_t const chainSize = ZSTD_allocateChainTable(cdict_cParams->strategy, cdict->useRowMatchFinder, 0 /* DDS guaranteed disabled */)
? ((size_t)1 << cdict_cParams->chainLog)
: 0;
size_t const hSize = (size_t)1 << cdict_cParams->hashLog;
ZSTD_memcpy(cctx->blockState.matchState.hashTable,
cdict->matchState.hashTable,
hSize * sizeof(U32));
ZSTD_memcpy(cctx->blockState.matchState.chainTable,
/* Do not copy cdict's chainTable if cctx has parameters such that it would not use chainTable */
if (ZSTD_allocateChainTable(cctx->appliedParams.cParams.strategy, cctx->appliedParams.useRowMatchFinder, 0 /* forDDSDict */)) {
ZSTD_memcpy(cctx->blockState.matchState.chainTable,
cdict->matchState.chainTable,
chainSize * sizeof(U32));
}
/* copy tag table */
if (ZSTD_rowMatchFinderUsed(cdict_cParams->strategy, cdict->useRowMatchFinder)) {
size_t const tagTableSize = hSize*sizeof(U16);
ZSTD_memcpy(cctx->blockState.matchState.tagTable,
cdict->matchState.tagTable,
tagTableSize);
}
}
/* Zero the hashTable3, since the cdict never fills it */
@@ -1989,16 +2180,18 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
U64 pledgedSrcSize,
ZSTD_buffered_policy_e zbuff)
{
DEBUGLOG(5, "ZSTD_copyCCtx_internal");
RETURN_ERROR_IF(srcCCtx->stage!=ZSTDcs_init, stage_wrong,
"Can't copy a ctx that's not in init stage.");
DEBUGLOG(5, "ZSTD_copyCCtx_internal");
ZSTD_memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
{ ZSTD_CCtx_params params = dstCCtx->requestedParams;
/* Copy only compression parameters related to tables. */
params.cParams = srcCCtx->appliedParams.cParams;
assert(srcCCtx->appliedParams.useRowMatchFinder != ZSTD_urm_auto);
params.useRowMatchFinder = srcCCtx->appliedParams.useRowMatchFinder;
params.fParams = fParams;
ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize,
/* loadedDictSize */ 0,
ZSTDcrp_leaveDirty, zbuff);
assert(dstCCtx->appliedParams.cParams.windowLog == srcCCtx->appliedParams.cParams.windowLog);
assert(dstCCtx->appliedParams.cParams.strategy == srcCCtx->appliedParams.cParams.strategy);
@@ -2010,7 +2203,11 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx,
ZSTD_cwksp_mark_tables_dirty(&dstCCtx->workspace);
/* copy tables */
{ size_t const chainSize = (srcCCtx->appliedParams.cParams.strategy == ZSTD_fast) ? 0 : ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog);
{ size_t const chainSize = ZSTD_allocateChainTable(srcCCtx->appliedParams.cParams.strategy,
srcCCtx->appliedParams.useRowMatchFinder,
0 /* forDDSDict */)
? ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog)
: 0;
size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog;
int const h3log = srcCCtx->blockState.matchState.hashLog3;
size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0;
@@ -2124,7 +2321,7 @@ static void ZSTD_reduceIndex (ZSTD_matchState_t* ms, ZSTD_CCtx_params const* par
ZSTD_reduceTable(ms->hashTable, hSize, reducerValue);
}
if (params->cParams.strategy != ZSTD_fast) {
if (ZSTD_allocateChainTable(params->cParams.strategy, params->useRowMatchFinder, (U32)ms->dedicatedDictSearch)) {
U32 const chainSize = (U32)1 << params->cParams.chainLog;
if (params->cParams.strategy == ZSTD_btlazy2)
ZSTD_reduceTable_btlazy2(ms->chainTable, chainSize, reducerValue);
@@ -2161,9 +2358,9 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
ofCodeTable[u] = (BYTE)ZSTD_highbit32(sequences[u].offset);
mlCodeTable[u] = (BYTE)ZSTD_MLcode(mlv);
}
if (seqStorePtr->longLengthID==1)
if (seqStorePtr->longLengthType==ZSTD_llt_literalLength)
llCodeTable[seqStorePtr->longLengthPos] = MaxLL;
if (seqStorePtr->longLengthID==2)
if (seqStorePtr->longLengthType==ZSTD_llt_matchLength)
mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
}
@@ -2177,10 +2374,156 @@ static int ZSTD_useTargetCBlockSize(const ZSTD_CCtx_params* cctxParams)
return (cctxParams->targetCBlockSize != 0);
}
/* ZSTD_entropyCompressSequences_internal():
* actually compresses both literals and sequences */
/* ZSTD_blockSplitterEnabled():
* Returns if block splitting param is being used
* If used, compression will do best effort to split a block in order to improve compression ratio.
* Returns 1 if true, 0 otherwise. */
static int ZSTD_blockSplitterEnabled(ZSTD_CCtx_params* cctxParams)
{
DEBUGLOG(5, "ZSTD_blockSplitterEnabled(splitBlocks=%d)", cctxParams->splitBlocks);
return (cctxParams->splitBlocks != 0);
}
/* Type returned by ZSTD_buildSequencesStatistics containing finalized symbol encoding types
* and size of the sequences statistics
*/
typedef struct {
U32 LLtype;
U32 Offtype;
U32 MLtype;
size_t size;
size_t lastCountSize; /* Accounts for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */
} ZSTD_symbolEncodingTypeStats_t;
/* ZSTD_buildSequencesStatistics():
* Returns the size of the statistics for a given set of sequences, or a ZSTD error code,
* Also modifies LLtype, Offtype, MLtype, and lastNCount to the appropriate values.
*
* entropyWkspSize must be of size at least ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1)*sizeof(U32)
*/
static ZSTD_symbolEncodingTypeStats_t
ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq,
const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy,
BYTE* dst, const BYTE* const dstEnd,
ZSTD_strategy strategy, unsigned* countWorkspace,
void* entropyWorkspace, size_t entropyWkspSize) {
BYTE* const ostart = dst;
const BYTE* const oend = dstEnd;
BYTE* op = ostart;
FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable;
FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable;
FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable;
const BYTE* const ofCodeTable = seqStorePtr->ofCode;
const BYTE* const llCodeTable = seqStorePtr->llCode;
const BYTE* const mlCodeTable = seqStorePtr->mlCode;
ZSTD_symbolEncodingTypeStats_t stats;
stats.lastCountSize = 0;
/* convert length/distances into codes */
ZSTD_seqToCodes(seqStorePtr);
assert(op <= oend);
/* build CTable for Literal Lengths */
{ unsigned max = MaxLL;
size_t const mostFrequent = HIST_countFast_wksp(countWorkspace, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */
DEBUGLOG(5, "Building LL table");
nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode;
stats.LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode,
countWorkspace, max, mostFrequent, nbSeq,
LLFSELog, prevEntropy->litlengthCTable,
LL_defaultNorm, LL_defaultNormLog,
ZSTD_defaultAllowed, strategy);
assert(set_basic < set_compressed && set_rle < set_compressed);
assert(!(stats.LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(
op, (size_t)(oend - op),
CTable_LitLength, LLFSELog, (symbolEncodingType_e)stats.LLtype,
countWorkspace, max, llCodeTable, nbSeq,
LL_defaultNorm, LL_defaultNormLog, MaxLL,
prevEntropy->litlengthCTable,
sizeof(prevEntropy->litlengthCTable),
entropyWorkspace, entropyWkspSize);
if (ZSTD_isError(countSize)) {
DEBUGLOG(3, "ZSTD_buildCTable for LitLens failed");
stats.size = countSize;
return stats;
}
if (stats.LLtype == set_compressed)
stats.lastCountSize = countSize;
op += countSize;
assert(op <= oend);
} }
/* build CTable for Offsets */
{ unsigned max = MaxOff;
size_t const mostFrequent = HIST_countFast_wksp(
countWorkspace, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */
/* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */
ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed;
DEBUGLOG(5, "Building OF table");
nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode;
stats.Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode,
countWorkspace, max, mostFrequent, nbSeq,
OffFSELog, prevEntropy->offcodeCTable,
OF_defaultNorm, OF_defaultNormLog,
defaultPolicy, strategy);
assert(!(stats.Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(
op, (size_t)(oend - op),
CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)stats.Offtype,
countWorkspace, max, ofCodeTable, nbSeq,
OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
prevEntropy->offcodeCTable,
sizeof(prevEntropy->offcodeCTable),
entropyWorkspace, entropyWkspSize);
if (ZSTD_isError(countSize)) {
DEBUGLOG(3, "ZSTD_buildCTable for Offsets failed");
stats.size = countSize;
return stats;
}
if (stats.Offtype == set_compressed)
stats.lastCountSize = countSize;
op += countSize;
assert(op <= oend);
} }
/* build CTable for MatchLengths */
{ unsigned max = MaxML;
size_t const mostFrequent = HIST_countFast_wksp(
countWorkspace, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */
DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op));
nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode;
stats.MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode,
countWorkspace, max, mostFrequent, nbSeq,
MLFSELog, prevEntropy->matchlengthCTable,
ML_defaultNorm, ML_defaultNormLog,
ZSTD_defaultAllowed, strategy);
assert(!(stats.MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(
op, (size_t)(oend - op),
CTable_MatchLength, MLFSELog, (symbolEncodingType_e)stats.MLtype,
countWorkspace, max, mlCodeTable, nbSeq,
ML_defaultNorm, ML_defaultNormLog, MaxML,
prevEntropy->matchlengthCTable,
sizeof(prevEntropy->matchlengthCTable),
entropyWorkspace, entropyWkspSize);
if (ZSTD_isError(countSize)) {
DEBUGLOG(3, "ZSTD_buildCTable for MatchLengths failed");
stats.size = countSize;
return stats;
}
if (stats.MLtype == set_compressed)
stats.lastCountSize = countSize;
op += countSize;
assert(op <= oend);
} }
stats.size = (size_t)(op-ostart);
return stats;
}
/* ZSTD_entropyCompressSeqStore_internal():
* compresses both literals and sequences
* Returns compressed size of block, or a zstd error.
*/
MEM_STATIC size_t
ZSTD_entropyCompressSequences_internal(seqStore_t* seqStorePtr,
ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr,
const ZSTD_entropyCTables_t* prevEntropy,
ZSTD_entropyCTables_t* nextEntropy,
const ZSTD_CCtx_params* cctxParams,
@@ -2194,22 +2537,20 @@ ZSTD_entropyCompressSequences_internal(seqStore_t* seqStorePtr,
FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable;
FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable;
FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable;
U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */
const seqDef* const sequences = seqStorePtr->sequencesStart;
const size_t nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
const BYTE* const ofCodeTable = seqStorePtr->ofCode;
const BYTE* const llCodeTable = seqStorePtr->llCode;
const BYTE* const mlCodeTable = seqStorePtr->mlCode;
BYTE* const ostart = (BYTE*)dst;
BYTE* const oend = ostart + dstCapacity;
BYTE* op = ostart;
size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
BYTE* seqHead;
BYTE* lastNCount = NULL;
size_t lastCountSize;
entropyWorkspace = count + (MaxSeq + 1);
entropyWkspSize -= (MaxSeq + 1) * sizeof(*count);
DEBUGLOG(4, "ZSTD_entropyCompressSequences_internal (nbSeq=%zu)", nbSeq);
DEBUGLOG(4, "ZSTD_entropyCompressSeqStore_internal (nbSeq=%zu)", nbSeq);
ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog)));
assert(entropyWkspSize >= HUF_WORKSPACE_SIZE);
@@ -2249,95 +2590,20 @@ ZSTD_entropyCompressSequences_internal(seqStore_t* seqStorePtr,
ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse));
return (size_t)(op - ostart);
}
/* seqHead : flags for FSE encoding type */
seqHead = op++;
assert(op <= oend);
/* convert length/distances into codes */
ZSTD_seqToCodes(seqStorePtr);
/* build CTable for Literal Lengths */
{ unsigned max = MaxLL;
size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */
DEBUGLOG(5, "Building LL table");
nextEntropy->fse.litlength_repeatMode = prevEntropy->fse.litlength_repeatMode;
LLtype = ZSTD_selectEncodingType(&nextEntropy->fse.litlength_repeatMode,
count, max, mostFrequent, nbSeq,
LLFSELog, prevEntropy->fse.litlengthCTable,
LL_defaultNorm, LL_defaultNormLog,
ZSTD_defaultAllowed, strategy);
assert(set_basic < set_compressed && set_rle < set_compressed);
assert(!(LLtype < set_compressed && nextEntropy->fse.litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(
op, (size_t)(oend - op),
CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype,
count, max, llCodeTable, nbSeq,
LL_defaultNorm, LL_defaultNormLog, MaxLL,
prevEntropy->fse.litlengthCTable,
sizeof(prevEntropy->fse.litlengthCTable),
entropyWorkspace, entropyWkspSize);
FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for LitLens failed");
if (LLtype == set_compressed)
lastNCount = op;
op += countSize;
assert(op <= oend);
} }
/* build CTable for Offsets */
{ unsigned max = MaxOff;
size_t const mostFrequent = HIST_countFast_wksp(
count, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */
/* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */
ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed;
DEBUGLOG(5, "Building OF table");
nextEntropy->fse.offcode_repeatMode = prevEntropy->fse.offcode_repeatMode;
Offtype = ZSTD_selectEncodingType(&nextEntropy->fse.offcode_repeatMode,
count, max, mostFrequent, nbSeq,
OffFSELog, prevEntropy->fse.offcodeCTable,
OF_defaultNorm, OF_defaultNormLog,
defaultPolicy, strategy);
assert(!(Offtype < set_compressed && nextEntropy->fse.offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(
op, (size_t)(oend - op),
CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype,
count, max, ofCodeTable, nbSeq,
OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
prevEntropy->fse.offcodeCTable,
sizeof(prevEntropy->fse.offcodeCTable),
entropyWorkspace, entropyWkspSize);
FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for Offsets failed");
if (Offtype == set_compressed)
lastNCount = op;
op += countSize;
assert(op <= oend);
} }
/* build CTable for MatchLengths */
{ unsigned max = MaxML;
size_t const mostFrequent = HIST_countFast_wksp(
count, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */
DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op));
nextEntropy->fse.matchlength_repeatMode = prevEntropy->fse.matchlength_repeatMode;
MLtype = ZSTD_selectEncodingType(&nextEntropy->fse.matchlength_repeatMode,
count, max, mostFrequent, nbSeq,
MLFSELog, prevEntropy->fse.matchlengthCTable,
ML_defaultNorm, ML_defaultNormLog,
ZSTD_defaultAllowed, strategy);
assert(!(MLtype < set_compressed && nextEntropy->fse.matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(
op, (size_t)(oend - op),
CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype,
count, max, mlCodeTable, nbSeq,
ML_defaultNorm, ML_defaultNormLog, MaxML,
prevEntropy->fse.matchlengthCTable,
sizeof(prevEntropy->fse.matchlengthCTable),
entropyWorkspace, entropyWkspSize);
FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for MatchLengths failed");
if (MLtype == set_compressed)
lastNCount = op;
op += countSize;
assert(op <= oend);
} }
*seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
{
ZSTD_symbolEncodingTypeStats_t stats;
BYTE* seqHead = op++;
/* build stats for sequences */
stats = ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq,
&prevEntropy->fse, &nextEntropy->fse,
op, oend,
strategy, count,
entropyWorkspace, entropyWkspSize);
FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!");
*seqHead = (BYTE)((stats.LLtype<<6) + (stats.Offtype<<4) + (stats.MLtype<<2));
lastCountSize = stats.lastCountSize;
op += stats.size;
}
{ size_t const bitstreamSize = ZSTD_encodeSequences(
op, (size_t)(oend - op),
@@ -2357,9 +2623,9 @@ ZSTD_entropyCompressSequences_internal(seqStore_t* seqStorePtr,
* In this exceedingly rare case, we will simply emit an uncompressed
* block, since it isn't worth optimizing.
*/
if (lastNCount && (op - lastNCount) < 4) {
/* NCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */
assert(op - lastNCount == 3);
if (lastCountSize && (lastCountSize + bitstreamSize) < 4) {
/* lastCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */
assert(lastCountSize + bitstreamSize == 3);
DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by "
"emitting an uncompressed block.");
return 0;
@@ -2371,7 +2637,7 @@ ZSTD_entropyCompressSequences_internal(seqStore_t* seqStorePtr,
}
MEM_STATIC size_t
ZSTD_entropyCompressSequences(seqStore_t* seqStorePtr,
ZSTD_entropyCompressSeqStore(seqStore_t* seqStorePtr,
const ZSTD_entropyCTables_t* prevEntropy,
ZSTD_entropyCTables_t* nextEntropy,
const ZSTD_CCtx_params* cctxParams,
@@ -2380,7 +2646,7 @@ ZSTD_entropyCompressSequences(seqStore_t* seqStorePtr,
void* entropyWorkspace, size_t entropyWkspSize,
int bmi2)
{
size_t const cSize = ZSTD_entropyCompressSequences_internal(
size_t const cSize = ZSTD_entropyCompressSeqStore_internal(
seqStorePtr, prevEntropy, nextEntropy, cctxParams,
dst, dstCapacity,
entropyWorkspace, entropyWkspSize, bmi2);
@@ -2390,20 +2656,20 @@ ZSTD_entropyCompressSequences(seqStore_t* seqStorePtr,
*/
if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity))
return 0; /* block not compressed */
FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSequences_internal failed");
FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSeqStore_internal failed");
/* Check compressibility */
{ size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy);
if (cSize >= maxCSize) return 0; /* block not compressed */
}
DEBUGLOG(4, "ZSTD_entropyCompressSequences() cSize: %zu\n", cSize);
DEBUGLOG(4, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize);
return cSize;
}
/* ZSTD_selectBlockCompressor() :
* Not static, but internal use only (used by long distance matcher)
* assumption : strat is a valid strategy */
ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMode_e dictMode)
ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_useRowMatchFinderMode_e useRowMatchFinder, ZSTD_dictMode_e dictMode)
{
static const ZSTD_blockCompressor blockCompressor[4][ZSTD_STRATEGY_MAX+1] = {
{ ZSTD_compressBlock_fast /* default for 0 */,
@@ -2451,7 +2717,28 @@ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMo
ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1);
assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat));
selectedCompressor = blockCompressor[(int)dictMode][(int)strat];
DEBUGLOG(4, "Selected block compressor: dictMode=%d strat=%d rowMatchfinder=%d", (int)dictMode, (int)strat, (int)useRowMatchFinder);
if (ZSTD_rowMatchFinderUsed(strat, useRowMatchFinder)) {
static const ZSTD_blockCompressor rowBasedBlockCompressors[4][3] = {
{ ZSTD_compressBlock_greedy_row,
ZSTD_compressBlock_lazy_row,
ZSTD_compressBlock_lazy2_row },
{ ZSTD_compressBlock_greedy_extDict_row,
ZSTD_compressBlock_lazy_extDict_row,
ZSTD_compressBlock_lazy2_extDict_row },
{ ZSTD_compressBlock_greedy_dictMatchState_row,
ZSTD_compressBlock_lazy_dictMatchState_row,
ZSTD_compressBlock_lazy2_dictMatchState_row },
{ ZSTD_compressBlock_greedy_dedicatedDictSearch_row,
ZSTD_compressBlock_lazy_dedicatedDictSearch_row,
ZSTD_compressBlock_lazy2_dedicatedDictSearch_row }
};
DEBUGLOG(4, "Selecting a row-based matchfinder");
assert(useRowMatchFinder != ZSTD_urm_auto);
selectedCompressor = rowBasedBlockCompressors[(int)dictMode][(int)strat - (int)ZSTD_greedy];
} else {
selectedCompressor = blockCompressor[(int)dictMode][(int)strat];
}
assert(selectedCompressor != NULL);
return selectedCompressor;
}
@@ -2467,7 +2754,7 @@ void ZSTD_resetSeqStore(seqStore_t* ssPtr)
{
ssPtr->lit = ssPtr->litStart;
ssPtr->sequences = ssPtr->sequencesStart;
ssPtr->longLengthID = 0;
ssPtr->longLengthType = ZSTD_llt_none;
}
typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e;
@@ -2520,6 +2807,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
ZSTD_ldm_blockCompress(&zc->externSeqStore,
ms, &zc->seqStore,
zc->blockState.nextCBlock->rep,
zc->appliedParams.useRowMatchFinder,
src, srcSize);
assert(zc->externSeqStore.pos <= zc->externSeqStore.size);
} else if (zc->appliedParams.ldmParams.enableLdm) {
@@ -2536,10 +2824,13 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
ZSTD_ldm_blockCompress(&ldmSeqStore,
ms, &zc->seqStore,
zc->blockState.nextCBlock->rep,
zc->appliedParams.useRowMatchFinder,
src, srcSize);
assert(ldmSeqStore.pos == ldmSeqStore.size);
} else { /* not long range mode */
ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, dictMode);
ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy,
zc->appliedParams.useRowMatchFinder,
dictMode);
ms->ldmSeqStore = NULL;
lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize);
}
@@ -2573,9 +2864,9 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc)
outSeqs[i].rep = 0;
if (i == seqStore->longLengthPos) {
if (seqStore->longLengthID == 1) {
if (seqStore->longLengthType == ZSTD_llt_literalLength) {
outSeqs[i].litLength += 0x10000;
} else if (seqStore->longLengthID == 2) {
} else if (seqStore->longLengthType == ZSTD_llt_matchLength) {
outSeqs[i].matchLength += 0x10000;
}
}
@@ -2686,11 +2977,698 @@ static int ZSTD_maybeRLE(seqStore_t const* seqStore)
return nbSeqs < 4 && nbLits < 10;
}
static void ZSTD_confirmRepcodesAndEntropyTables(ZSTD_CCtx* zc)
static void ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs)
{
ZSTD_compressedBlockState_t* const tmp = zc->blockState.prevCBlock;
zc->blockState.prevCBlock = zc->blockState.nextCBlock;
zc->blockState.nextCBlock = tmp;
ZSTD_compressedBlockState_t* const tmp = bs->prevCBlock;
bs->prevCBlock = bs->nextCBlock;
bs->nextCBlock = tmp;
}
/* Writes the block header */
static void writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock) {
U32 const cBlockHeader = cSize == 1 ?
lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) :
lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
MEM_writeLE24(op, cBlockHeader);
DEBUGLOG(3, "writeBlockHeader: cSize: %zu blockSize: %zu lastBlock: %u", cSize, blockSize, lastBlock);
}
/** ZSTD_buildBlockEntropyStats_literals() :
* Builds entropy for the literals.
* Stores literals block type (raw, rle, compressed, repeat) and
* huffman description table to hufMetadata.
* Requires ENTROPY_WORKSPACE_SIZE workspace
* @return : size of huffman description table or error code */
static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize,
const ZSTD_hufCTables_t* prevHuf,
ZSTD_hufCTables_t* nextHuf,
ZSTD_hufCTablesMetadata_t* hufMetadata,
const int disableLiteralsCompression,
void* workspace, size_t wkspSize)
{
BYTE* const wkspStart = (BYTE*)workspace;
BYTE* const wkspEnd = wkspStart + wkspSize;
BYTE* const countWkspStart = wkspStart;
unsigned* const countWksp = (unsigned*)workspace;
const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned);
BYTE* const nodeWksp = countWkspStart + countWkspSize;
const size_t nodeWkspSize = wkspEnd-nodeWksp;
unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX;
unsigned huffLog = HUF_TABLELOG_DEFAULT;
HUF_repeat repeat = prevHuf->repeatMode;
DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_literals (srcSize=%zu)", srcSize);
/* Prepare nextEntropy assuming reusing the existing table */
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
if (disableLiteralsCompression) {
DEBUGLOG(5, "set_basic - disabled");
hufMetadata->hType = set_basic;
return 0;
}
/* small ? don't even attempt compression (speed opt) */
#ifndef COMPRESS_LITERALS_SIZE_MIN
#define COMPRESS_LITERALS_SIZE_MIN 63
#endif
{ size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN;
if (srcSize <= minLitSize) {
DEBUGLOG(5, "set_basic - too small");
hufMetadata->hType = set_basic;
return 0;
}
}
/* Scan input and build symbol stats */
{ size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)src, srcSize, workspace, wkspSize);
FORWARD_IF_ERROR(largest, "HIST_count_wksp failed");
if (largest == srcSize) {
DEBUGLOG(5, "set_rle");
hufMetadata->hType = set_rle;
return 0;
}
if (largest <= (srcSize >> 7)+4) {
DEBUGLOG(5, "set_basic - no gain");
hufMetadata->hType = set_basic;
return 0;
}
}
/* Validate the previous Huffman table */
if (repeat == HUF_repeat_check && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) {
repeat = HUF_repeat_none;
}
/* Build Huffman Tree */
ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable));
huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
{ size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp,
maxSymbolValue, huffLog,
nodeWksp, nodeWkspSize);
FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp");
huffLog = (U32)maxBits;
{ /* Build and write the CTable */
size_t const newCSize = HUF_estimateCompressedSize(
(HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue);
size_t const hSize = HUF_writeCTable_wksp(
hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer),
(HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog,
nodeWksp, nodeWkspSize);
/* Check against repeating the previous CTable */
if (repeat != HUF_repeat_none) {
size_t const oldCSize = HUF_estimateCompressedSize(
(HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue);
if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) {
DEBUGLOG(5, "set_repeat - smaller");
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
hufMetadata->hType = set_repeat;
return 0;
}
}
if (newCSize + hSize >= srcSize) {
DEBUGLOG(5, "set_basic - no gains");
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
hufMetadata->hType = set_basic;
return 0;
}
DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize);
hufMetadata->hType = set_compressed;
nextHuf->repeatMode = HUF_repeat_check;
return hSize;
}
}
}
/** ZSTD_buildBlockEntropyStats_sequences() :
* Builds entropy for the sequences.
* Stores symbol compression modes and fse table to fseMetadata.
* Requires ENTROPY_WORKSPACE_SIZE wksp.
* @return : size of fse tables or error code */
static size_t ZSTD_buildBlockEntropyStats_sequences(seqStore_t* seqStorePtr,
const ZSTD_fseCTables_t* prevEntropy,
ZSTD_fseCTables_t* nextEntropy,
const ZSTD_CCtx_params* cctxParams,
ZSTD_fseCTablesMetadata_t* fseMetadata,
void* workspace, size_t wkspSize)
{
ZSTD_strategy const strategy = cctxParams->cParams.strategy;
size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
BYTE* const ostart = fseMetadata->fseTablesBuffer;
BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer);
BYTE* op = ostart;
unsigned* countWorkspace = (unsigned*)workspace;
unsigned* entropyWorkspace = countWorkspace + (MaxSeq + 1);
size_t entropyWorkspaceSize = wkspSize - (MaxSeq + 1) * sizeof(*countWorkspace);
ZSTD_symbolEncodingTypeStats_t stats;
DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_sequences (nbSeq=%zu)", nbSeq);
stats = ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq,
prevEntropy, nextEntropy, op, oend,
strategy, countWorkspace,
entropyWorkspace, entropyWorkspaceSize);
FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!");
fseMetadata->llType = (symbolEncodingType_e) stats.LLtype;
fseMetadata->ofType = (symbolEncodingType_e) stats.Offtype;
fseMetadata->mlType = (symbolEncodingType_e) stats.MLtype;
fseMetadata->lastCountSize = stats.lastCountSize;
return stats.size;
}
/** ZSTD_buildBlockEntropyStats() :
* Builds entropy for the block.
* Requires workspace size ENTROPY_WORKSPACE_SIZE
*
* @return : 0 on success or error code
*/
size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr,
const ZSTD_entropyCTables_t* prevEntropy,
ZSTD_entropyCTables_t* nextEntropy,
const ZSTD_CCtx_params* cctxParams,
ZSTD_entropyCTablesMetadata_t* entropyMetadata,
void* workspace, size_t wkspSize)
{
size_t const litSize = seqStorePtr->lit - seqStorePtr->litStart;
entropyMetadata->hufMetadata.hufDesSize =
ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize,
&prevEntropy->huf, &nextEntropy->huf,
&entropyMetadata->hufMetadata,
ZSTD_disableLiteralsCompression(cctxParams),
workspace, wkspSize);
FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed");
entropyMetadata->fseMetadata.fseTablesSize =
ZSTD_buildBlockEntropyStats_sequences(seqStorePtr,
&prevEntropy->fse, &nextEntropy->fse,
cctxParams,
&entropyMetadata->fseMetadata,
workspace, wkspSize);
FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildBlockEntropyStats_sequences failed");
return 0;
}
/* Returns the size estimate for the literals section (header + content) of a block */
static size_t ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSize,
const ZSTD_hufCTables_t* huf,
const ZSTD_hufCTablesMetadata_t* hufMetadata,
void* workspace, size_t wkspSize,
int writeEntropy)
{
unsigned* const countWksp = (unsigned*)workspace;
unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX;
size_t literalSectionHeaderSize = 3 + (litSize >= 1 KB) + (litSize >= 16 KB);
U32 singleStream = litSize < 256;
if (hufMetadata->hType == set_basic) return litSize;
else if (hufMetadata->hType == set_rle) return 1;
else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) {
size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize, workspace, wkspSize);
if (ZSTD_isError(largest)) return litSize;
{ size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, maxSymbolValue);
if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize;
if (!singleStream) cLitSizeEstimate += 6; /* multi-stream huffman uses 6-byte jump table */
return cLitSizeEstimate + literalSectionHeaderSize;
} }
assert(0); /* impossible */
return 0;
}
/* Returns the size estimate for the FSE-compressed symbols (of, ml, ll) of a block */
static size_t ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type,
const BYTE* codeTable, size_t nbSeq, unsigned maxCode,
const FSE_CTable* fseCTable,
const U32* additionalBits,
short const* defaultNorm, U32 defaultNormLog, U32 defaultMax,
void* workspace, size_t wkspSize)
{
unsigned* const countWksp = (unsigned*)workspace;
const BYTE* ctp = codeTable;
const BYTE* const ctStart = ctp;
const BYTE* const ctEnd = ctStart + nbSeq;
size_t cSymbolTypeSizeEstimateInBits = 0;
unsigned max = maxCode;
HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */
if (type == set_basic) {
/* We selected this encoding type, so it must be valid. */
assert(max <= defaultMax);
(void)defaultMax;
cSymbolTypeSizeEstimateInBits = ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max);
} else if (type == set_rle) {
cSymbolTypeSizeEstimateInBits = 0;
} else if (type == set_compressed || type == set_repeat) {
cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max);
}
if (ZSTD_isError(cSymbolTypeSizeEstimateInBits)) {
return nbSeq * 10;
}
while (ctp < ctEnd) {
if (additionalBits) cSymbolTypeSizeEstimateInBits += additionalBits[*ctp];
else cSymbolTypeSizeEstimateInBits += *ctp; /* for offset, offset code is also the number of additional bits */
ctp++;
}
return cSymbolTypeSizeEstimateInBits >> 3;
}
/* Returns the size estimate for the sequences section (header + content) of a block */
static size_t ZSTD_estimateBlockSize_sequences(const BYTE* ofCodeTable,
const BYTE* llCodeTable,
const BYTE* mlCodeTable,
size_t nbSeq,
const ZSTD_fseCTables_t* fseTables,
const ZSTD_fseCTablesMetadata_t* fseMetadata,
void* workspace, size_t wkspSize,
int writeEntropy)
{
size_t sequencesSectionHeaderSize = 1 /* seqHead */ + 1 /* min seqSize size */ + (nbSeq >= 128) + (nbSeq >= LONGNBSEQ);
size_t cSeqSizeEstimate = 0;
cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, nbSeq, MaxOff,
fseTables->offcodeCTable, NULL,
OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
workspace, wkspSize);
cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->llType, llCodeTable, nbSeq, MaxLL,
fseTables->litlengthCTable, LL_bits,
LL_defaultNorm, LL_defaultNormLog, MaxLL,
workspace, wkspSize);
cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, nbSeq, MaxML,
fseTables->matchlengthCTable, ML_bits,
ML_defaultNorm, ML_defaultNormLog, MaxML,
workspace, wkspSize);
if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize;
return cSeqSizeEstimate + sequencesSectionHeaderSize;
}
/* Returns the size estimate for a given stream of literals, of, ll, ml */
static size_t ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize,
const BYTE* ofCodeTable,
const BYTE* llCodeTable,
const BYTE* mlCodeTable,
size_t nbSeq,
const ZSTD_entropyCTables_t* entropy,
const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
void* workspace, size_t wkspSize,
int writeLitEntropy, int writeSeqEntropy) {
size_t const literalsSize = ZSTD_estimateBlockSize_literal(literals, litSize,
&entropy->huf, &entropyMetadata->hufMetadata,
workspace, wkspSize, writeLitEntropy);
size_t const seqSize = ZSTD_estimateBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable,
nbSeq, &entropy->fse, &entropyMetadata->fseMetadata,
workspace, wkspSize, writeSeqEntropy);
return seqSize + literalsSize + ZSTD_blockHeaderSize;
}
/* Builds entropy statistics and uses them for blocksize estimation.
*
* Returns the estimated compressed size of the seqStore, or a zstd error.
*/
static size_t ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(seqStore_t* seqStore, const ZSTD_CCtx* zc) {
ZSTD_entropyCTablesMetadata_t entropyMetadata;
FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(seqStore,
&zc->blockState.prevCBlock->entropy,
&zc->blockState.nextCBlock->entropy,
&zc->appliedParams,
&entropyMetadata,
zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), "");
return ZSTD_estimateBlockSize(seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart),
seqStore->ofCode, seqStore->llCode, seqStore->mlCode,
(size_t)(seqStore->sequences - seqStore->sequencesStart),
&zc->blockState.nextCBlock->entropy, &entropyMetadata, zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE,
(int)(entropyMetadata.hufMetadata.hType == set_compressed), 1);
}
/* Returns literals bytes represented in a seqStore */
static size_t ZSTD_countSeqStoreLiteralsBytes(const seqStore_t* const seqStore) {
size_t literalsBytes = 0;
size_t const nbSeqs = seqStore->sequences - seqStore->sequencesStart;
size_t i;
for (i = 0; i < nbSeqs; ++i) {
seqDef seq = seqStore->sequencesStart[i];
literalsBytes += seq.litLength;
if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_literalLength) {
literalsBytes += 0x10000;
}
}
return literalsBytes;
}
/* Returns match bytes represented in a seqStore */
static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* const seqStore) {
size_t matchBytes = 0;
size_t const nbSeqs = seqStore->sequences - seqStore->sequencesStart;
size_t i;
for (i = 0; i < nbSeqs; ++i) {
seqDef seq = seqStore->sequencesStart[i];
matchBytes += seq.matchLength + MINMATCH;
if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_matchLength) {
matchBytes += 0x10000;
}
}
return matchBytes;
}
/* Derives the seqStore that is a chunk of the originalSeqStore from [startIdx, endIdx).
* Stores the result in resultSeqStore.
*/
static void ZSTD_deriveSeqStoreChunk(seqStore_t* resultSeqStore,
const seqStore_t* originalSeqStore,
size_t startIdx, size_t endIdx) {
BYTE* const litEnd = originalSeqStore->lit;
size_t literalsBytes;
size_t literalsBytesPreceding = 0;
*resultSeqStore = *originalSeqStore;
if (startIdx > 0) {
resultSeqStore->sequences = originalSeqStore->sequencesStart + startIdx;
literalsBytesPreceding = ZSTD_countSeqStoreLiteralsBytes(resultSeqStore);
}
/* Move longLengthPos into the correct position if necessary */
if (originalSeqStore->longLengthType != ZSTD_llt_none) {
if (originalSeqStore->longLengthPos < startIdx || originalSeqStore->longLengthPos > endIdx) {
resultSeqStore->longLengthType = ZSTD_llt_none;
} else {
resultSeqStore->longLengthPos -= (U32)startIdx;
}
}
resultSeqStore->sequencesStart = originalSeqStore->sequencesStart + startIdx;
resultSeqStore->sequences = originalSeqStore->sequencesStart + endIdx;
literalsBytes = ZSTD_countSeqStoreLiteralsBytes(resultSeqStore);
resultSeqStore->litStart += literalsBytesPreceding;
if (endIdx == (size_t)(originalSeqStore->sequences - originalSeqStore->sequencesStart)) {
/* This accounts for possible last literals if the derived chunk reaches the end of the block */
resultSeqStore->lit = litEnd;
} else {
resultSeqStore->lit = resultSeqStore->litStart+literalsBytes;
}
resultSeqStore->llCode += startIdx;
resultSeqStore->mlCode += startIdx;
resultSeqStore->ofCode += startIdx;
}
/**
* Returns the raw offset represented by the combination of offCode, ll0, and repcode history.
* offCode must be an offCode representing a repcode, therefore in the range of [0, 2].
*/
static U32 ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offCode, const U32 ll0) {
U32 const adjustedOffCode = offCode + ll0;
assert(offCode < ZSTD_REP_NUM);
if (adjustedOffCode == ZSTD_REP_NUM) {
/* litlength == 0 and offCode == 2 implies selection of first repcode - 1 */
assert(rep[0] > 0);
return rep[0] - 1;
}
return rep[adjustedOffCode];
}
/**
* ZSTD_seqStore_resolveOffCodes() reconciles any possible divergences in offset history that may arise
* due to emission of RLE/raw blocks that disturb the offset history, and replaces any repcodes within
* the seqStore that may be invalid.
*
* dRepcodes are updated as would be on the decompression side. cRepcodes are updated exactly in
* accordance with the seqStore.
*/
static void ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRepcodes,
seqStore_t* const seqStore, U32 const nbSeq) {
U32 idx = 0;
for (; idx < nbSeq; ++idx) {
seqDef* const seq = seqStore->sequencesStart + idx;
U32 const ll0 = (seq->litLength == 0);
U32 offCode = seq->offset - 1;
assert(seq->offset > 0);
if (offCode <= ZSTD_REP_MOVE) {
U32 const dRawOffset = ZSTD_resolveRepcodeToRawOffset(dRepcodes->rep, offCode, ll0);
U32 const cRawOffset = ZSTD_resolveRepcodeToRawOffset(cRepcodes->rep, offCode, ll0);
/* Adjust simulated decompression repcode history if we come across a mismatch. Replace
* the repcode with the offset it actually references, determined by the compression
* repcode history.
*/
if (dRawOffset != cRawOffset) {
seq->offset = cRawOffset + ZSTD_REP_NUM;
}
}
/* Compression repcode history is always updated with values directly from the unmodified seqStore.
* Decompression repcode history may use modified seq->offset value taken from compression repcode history.
*/
*dRepcodes = ZSTD_updateRep(dRepcodes->rep, seq->offset - 1, ll0);
*cRepcodes = ZSTD_updateRep(cRepcodes->rep, offCode, ll0);
}
}
/* ZSTD_compressSeqStore_singleBlock():
* Compresses a seqStore into a block with a block header, into the buffer dst.
*
* Returns the total size of that block (including header) or a ZSTD error code.
*/
static size_t ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc, seqStore_t* const seqStore,
repcodes_t* const dRep, repcodes_t* const cRep,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
U32 lastBlock, U32 isPartition) {
const U32 rleMaxLength = 25;
BYTE* op = (BYTE*)dst;
const BYTE* ip = (const BYTE*)src;
size_t cSize;
size_t cSeqsSize;
/* In case of an RLE or raw block, the simulated decompression repcode history must be reset */
repcodes_t const dRepOriginal = *dRep;
if (isPartition)
ZSTD_seqStore_resolveOffCodes(dRep, cRep, seqStore, (U32)(seqStore->sequences - seqStore->sequencesStart));
cSeqsSize = ZSTD_entropyCompressSeqStore(seqStore,
&zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy,
&zc->appliedParams,
op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize,
srcSize,
zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */,
zc->bmi2);
FORWARD_IF_ERROR(cSeqsSize, "ZSTD_entropyCompressSeqStore failed!");
if (!zc->isFirstBlock &&
cSeqsSize < rleMaxLength &&
ZSTD_isRLE((BYTE const*)src, srcSize)) {
/* We don't want to emit our first block as a RLE even if it qualifies because
* doing so will cause the decoder (cli only) to throw a "should consume all input error."
* This is only an issue for zstd <= v1.4.3
*/
cSeqsSize = 1;
}
if (zc->seqCollector.collectSequences) {
ZSTD_copyBlockSequences(zc);
ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState);
return 0;
}
if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
if (cSeqsSize == 0) {
cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, srcSize, lastBlock);
FORWARD_IF_ERROR(cSize, "Nocompress block failed");
DEBUGLOG(4, "Writing out nocompress block, size: %zu", cSize);
*dRep = dRepOriginal; /* reset simulated decompression repcode history */
} else if (cSeqsSize == 1) {
cSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, srcSize, lastBlock);
FORWARD_IF_ERROR(cSize, "RLE compress block failed");
DEBUGLOG(4, "Writing out RLE block, size: %zu", cSize);
*dRep = dRepOriginal; /* reset simulated decompression repcode history */
} else {
ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState);
writeBlockHeader(op, cSeqsSize, srcSize, lastBlock);
cSize = ZSTD_blockHeaderSize + cSeqsSize;
DEBUGLOG(4, "Writing out compressed block, size: %zu", cSize);
}
return cSize;
}
/* Struct to keep track of where we are in our recursive calls. */
typedef struct {
U32* splitLocations; /* Array of split indices */
size_t idx; /* The current index within splitLocations being worked on */
} seqStoreSplits;
#define MIN_SEQUENCES_BLOCK_SPLITTING 300
#define MAX_NB_SPLITS 196
/* Helper function to perform the recursive search for block splits.
* Estimates the cost of seqStore prior to split, and estimates the cost of splitting the sequences in half.
* If advantageous to split, then we recurse down the two sub-blocks. If not, or if an error occurred in estimation, then
* we do not recurse.
*
* Note: The recursion depth is capped by a heuristic minimum number of sequences, defined by MIN_SEQUENCES_BLOCK_SPLITTING.
* In theory, this means the absolute largest recursion depth is 10 == log2(maxNbSeqInBlock/MIN_SEQUENCES_BLOCK_SPLITTING).
* In practice, recursion depth usually doesn't go beyond 4.
*
* Furthermore, the number of splits is capped by MAX_NB_SPLITS. At MAX_NB_SPLITS == 196 with the current existing blockSize
* maximum of 128 KB, this value is actually impossible to reach.
*/
static void ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t endIdx,
const ZSTD_CCtx* zc, const seqStore_t* origSeqStore) {
seqStore_t fullSeqStoreChunk;
seqStore_t firstHalfSeqStore;
seqStore_t secondHalfSeqStore;
size_t estimatedOriginalSize;
size_t estimatedFirstHalfSize;
size_t estimatedSecondHalfSize;
size_t midIdx = (startIdx + endIdx)/2;
if (endIdx - startIdx < MIN_SEQUENCES_BLOCK_SPLITTING || splits->idx >= MAX_NB_SPLITS) {
return;
}
ZSTD_deriveSeqStoreChunk(&fullSeqStoreChunk, origSeqStore, startIdx, endIdx);
ZSTD_deriveSeqStoreChunk(&firstHalfSeqStore, origSeqStore, startIdx, midIdx);
ZSTD_deriveSeqStoreChunk(&secondHalfSeqStore, origSeqStore, midIdx, endIdx);
estimatedOriginalSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(&fullSeqStoreChunk, zc);
estimatedFirstHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(&firstHalfSeqStore, zc);
estimatedSecondHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(&secondHalfSeqStore, zc);
DEBUGLOG(5, "Estimated original block size: %zu -- First half split: %zu -- Second half split: %zu",
estimatedOriginalSize, estimatedFirstHalfSize, estimatedSecondHalfSize);
if (ZSTD_isError(estimatedOriginalSize) || ZSTD_isError(estimatedFirstHalfSize) || ZSTD_isError(estimatedSecondHalfSize)) {
return;
}
if (estimatedFirstHalfSize + estimatedSecondHalfSize < estimatedOriginalSize) {
ZSTD_deriveBlockSplitsHelper(splits, startIdx, midIdx, zc, origSeqStore);
splits->splitLocations[splits->idx] = (U32)midIdx;
splits->idx++;
ZSTD_deriveBlockSplitsHelper(splits, midIdx, endIdx, zc, origSeqStore);
}
}
/* Base recursive function. Populates a table with intra-block partition indices that can improve compression ratio.
*
* Returns the number of splits made (which equals the size of the partition table - 1).
*/
static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq) {
seqStoreSplits splits = {partitions, 0};
if (nbSeq <= 4) {
DEBUGLOG(4, "ZSTD_deriveBlockSplits: Too few sequences to split");
/* Refuse to try and split anything with less than 4 sequences */
return 0;
}
ZSTD_deriveBlockSplitsHelper(&splits, 0, nbSeq, zc, &zc->seqStore);
splits.splitLocations[splits.idx] = nbSeq;
DEBUGLOG(5, "ZSTD_deriveBlockSplits: final nb partitions: %zu", splits.idx+1);
return splits.idx;
}
/* ZSTD_compressBlock_splitBlock():
* Attempts to split a given block into multiple blocks to improve compression ratio.
*
* Returns combined size of all blocks (which includes headers), or a ZSTD error code.
*/
static size_t ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity,
const void* src, size_t blockSize, U32 lastBlock, U32 nbSeq) {
size_t cSize = 0;
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
U32 partitions[MAX_NB_SPLITS];
size_t i = 0;
size_t srcBytesTotal = 0;
size_t numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq);
seqStore_t nextSeqStore;
seqStore_t currSeqStore;
/* If a block is split and some partitions are emitted as RLE/uncompressed, then repcode history
* may become invalid. In order to reconcile potentially invalid repcodes, we keep track of two
* separate repcode histories that simulate repcode history on compression and decompression side,
* and use the histories to determine whether we must replace a particular repcode with its raw offset.
*
* 1) cRep gets updated for each partition, regardless of whether the block was emitted as uncompressed
* or RLE. This allows us to retrieve the offset value that an invalid repcode references within
* a nocompress/RLE block.
* 2) dRep gets updated only for compressed partitions, and when a repcode gets replaced, will use
* the replacement offset value rather than the original repcode to update the repcode history.
* dRep also will be the final repcode history sent to the next block.
*
* See ZSTD_seqStore_resolveOffCodes() for more details.
*/
repcodes_t dRep;
repcodes_t cRep;
ZSTD_memcpy(dRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t));
ZSTD_memcpy(cRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t));
DEBUGLOG(4, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)",
(unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit,
(unsigned)zc->blockState.matchState.nextToUpdate);
if (numSplits == 0) {
size_t cSizeSingleBlock = ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore,
&dRep, &cRep,
op, dstCapacity,
ip, blockSize,
lastBlock, 0 /* isPartition */);
FORWARD_IF_ERROR(cSizeSingleBlock, "Compressing single block from splitBlock_internal() failed!");
DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal: No splits");
assert(cSizeSingleBlock <= ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize);
return cSizeSingleBlock;
}
ZSTD_deriveSeqStoreChunk(&currSeqStore, &zc->seqStore, 0, partitions[0]);
for (i = 0; i <= numSplits; ++i) {
size_t srcBytes;
size_t cSizeChunk;
U32 const lastPartition = (i == numSplits);
U32 lastBlockEntireSrc = 0;
srcBytes = ZSTD_countSeqStoreLiteralsBytes(&currSeqStore) + ZSTD_countSeqStoreMatchBytes(&currSeqStore);
srcBytesTotal += srcBytes;
if (lastPartition) {
/* This is the final partition, need to account for possible last literals */
srcBytes += blockSize - srcBytesTotal;
lastBlockEntireSrc = lastBlock;
} else {
ZSTD_deriveSeqStoreChunk(&nextSeqStore, &zc->seqStore, partitions[i], partitions[i+1]);
}
cSizeChunk = ZSTD_compressSeqStore_singleBlock(zc, &currSeqStore,
&dRep, &cRep,
op, dstCapacity,
ip, srcBytes,
lastBlockEntireSrc, 1 /* isPartition */);
DEBUGLOG(5, "Estimated size: %zu actual size: %zu", ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(&currSeqStore, zc), cSizeChunk);
FORWARD_IF_ERROR(cSizeChunk, "Compressing chunk failed!");
ip += srcBytes;
op += cSizeChunk;
dstCapacity -= cSizeChunk;
cSize += cSizeChunk;
currSeqStore = nextSeqStore;
assert(cSizeChunk <= ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize);
}
/* cRep and dRep may have diverged during the compression. If so, we use the dRep repcodes
* for the next block.
*/
ZSTD_memcpy(zc->blockState.prevCBlock->rep, dRep.rep, sizeof(repcodes_t));
return cSize;
}
static size_t ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize, U32 lastBlock) {
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
U32 nbSeq;
size_t cSize;
DEBUGLOG(4, "ZSTD_compressBlock_splitBlock");
{ const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize);
FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed");
if (bss == ZSTDbss_noCompress) {
if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, srcSize, lastBlock);
FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed");
DEBUGLOG(4, "ZSTD_compressBlock_splitBlock: Nocompress block");
return cSize;
}
nbSeq = (U32)(zc->seqStore.sequences - zc->seqStore.sequencesStart);
}
assert(zc->appliedParams.splitBlocks == 1);
cSize = ZSTD_compressBlock_splitBlock_internal(zc, dst, dstCapacity, src, srcSize, lastBlock, nbSeq);
FORWARD_IF_ERROR(cSize, "Splitting blocks failed!");
return cSize;
}
static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
@@ -2716,12 +3694,12 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
if (zc->seqCollector.collectSequences) {
ZSTD_copyBlockSequences(zc);
ZSTD_confirmRepcodesAndEntropyTables(zc);
ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState);
return 0;
}
/* encode sequences and literals */
cSize = ZSTD_entropyCompressSequences(&zc->seqStore,
cSize = ZSTD_entropyCompressSeqStore(&zc->seqStore,
&zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy,
&zc->appliedParams,
dst, dstCapacity,
@@ -2750,7 +3728,7 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc,
out:
if (!ZSTD_isError(cSize) && cSize > 1) {
ZSTD_confirmRepcodesAndEntropyTables(zc);
ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState);
}
/* We check that dictionaries have offset codes available for the first
* block. After the first block, the offcode table might not have large
@@ -2803,7 +3781,7 @@ static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc,
size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy);
FORWARD_IF_ERROR(cSize, "ZSTD_compressSuperBlock failed");
if (cSize != 0 && cSize < maxCSize + ZSTD_blockHeaderSize) {
ZSTD_confirmRepcodesAndEntropyTables(zc);
ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState);
return cSize;
}
}
@@ -2843,9 +3821,9 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms,
void const* ip,
void const* iend)
{
if (ZSTD_window_needOverflowCorrection(ms->window, iend)) {
U32 const maxDist = (U32)1 << params->cParams.windowLog;
U32 const cycleLog = ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy);
U32 const cycleLog = ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy);
U32 const maxDist = (U32)1 << params->cParams.windowLog;
if (ZSTD_window_needOverflowCorrection(ms->window, cycleLog, maxDist, ms->loadedDictEnd, ip, iend)) {
U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip);
ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30);
ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30);
@@ -2868,7 +3846,7 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms,
* Frame is supposed already started (header already produced)
* @return : compressed size, or an error code
*/
static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
U32 lastFrameChunk)
@@ -2908,6 +3886,10 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx,
FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize failed");
assert(cSize > 0);
assert(cSize <= blockSize + ZSTD_blockHeaderSize);
} else if (ZSTD_blockSplitterEnabled(&cctx->appliedParams)) {
cSize = ZSTD_compressBlock_splitBlock(cctx, op, dstCapacity, ip, blockSize, lastBlock);
FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_splitBlock failed");
assert(cSize > 0 || cctx->seqCollector.collectSequences == 1);
} else {
cSize = ZSTD_compressBlock_internal(cctx,
op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize,
@@ -3135,63 +4117,85 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms,
{
const BYTE* ip = (const BYTE*) src;
const BYTE* const iend = ip + srcSize;
ZSTD_window_update(&ms->window, src, srcSize);
ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base);
if (params->ldmParams.enableLdm && ls != NULL) {
ZSTD_window_update(&ls->window, src, srcSize);
ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base);
}
int const loadLdmDict = params->ldmParams.enableLdm && ls != NULL;
/* Assert that we the ms params match the params we're being given */
ZSTD_assertEqualCParams(params->cParams, ms->cParams);
if (srcSize > ZSTD_CHUNKSIZE_MAX) {
/* Allow the dictionary to set indices up to exactly ZSTD_CURRENT_MAX.
* Dictionaries right at the edge will immediately trigger overflow
* correction, but I don't want to insert extra constraints here.
*/
U32 const maxDictSize = ZSTD_CURRENT_MAX - 1;
/* We must have cleared our windows when our source is this large. */
assert(ZSTD_window_isEmpty(ms->window));
if (loadLdmDict)
assert(ZSTD_window_isEmpty(ls->window));
/* If the dictionary is too large, only load the suffix of the dictionary. */
if (srcSize > maxDictSize) {
ip = iend - maxDictSize;
src = ip;
srcSize = maxDictSize;
}
}
DEBUGLOG(4, "ZSTD_loadDictionaryContent(): useRowMatchFinder=%d", (int)params->useRowMatchFinder);
ZSTD_window_update(&ms->window, src, srcSize);
ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base);
if (loadLdmDict) {
ZSTD_window_update(&ls->window, src, srcSize);
ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base);
}
if (srcSize <= HASH_READ_SIZE) return 0;
while (iend - ip > HASH_READ_SIZE) {
size_t const remaining = (size_t)(iend - ip);
size_t const chunk = MIN(remaining, ZSTD_CHUNKSIZE_MAX);
const BYTE* const ichunk = ip + chunk;
ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, iend);
ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, ichunk);
if (loadLdmDict)
ZSTD_ldm_fillHashTable(ls, ip, iend, &params->ldmParams);
if (params->ldmParams.enableLdm && ls != NULL)
ZSTD_ldm_fillHashTable(ls, (const BYTE*)src, (const BYTE*)src + srcSize, &params->ldmParams);
switch(params->cParams.strategy)
{
case ZSTD_fast:
ZSTD_fillHashTable(ms, iend, dtlm);
break;
case ZSTD_dfast:
ZSTD_fillDoubleHashTable(ms, iend, dtlm);
break;
switch(params->cParams.strategy)
{
case ZSTD_fast:
ZSTD_fillHashTable(ms, ichunk, dtlm);
break;
case ZSTD_dfast:
ZSTD_fillDoubleHashTable(ms, ichunk, dtlm);
break;
case ZSTD_greedy:
case ZSTD_lazy:
case ZSTD_lazy2:
if (chunk >= HASH_READ_SIZE && ms->dedicatedDictSearch) {
assert(chunk == remaining); /* must load everything in one go */
ZSTD_dedicatedDictSearch_lazy_loadDictionary(ms, ichunk-HASH_READ_SIZE);
} else if (chunk >= HASH_READ_SIZE) {
ZSTD_insertAndFindFirstIndex(ms, ichunk-HASH_READ_SIZE);
case ZSTD_greedy:
case ZSTD_lazy:
case ZSTD_lazy2:
assert(srcSize >= HASH_READ_SIZE);
if (ms->dedicatedDictSearch) {
assert(ms->chainTable != NULL);
ZSTD_dedicatedDictSearch_lazy_loadDictionary(ms, iend-HASH_READ_SIZE);
} else {
assert(params->useRowMatchFinder != ZSTD_urm_auto);
if (params->useRowMatchFinder == ZSTD_urm_enableRowMatchFinder) {
size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog) * sizeof(U16);
ZSTD_memset(ms->tagTable, 0, tagTableSize);
ZSTD_row_update(ms, iend-HASH_READ_SIZE);
DEBUGLOG(4, "Using row-based hash table for lazy dict");
} else {
ZSTD_insertAndFindFirstIndex(ms, iend-HASH_READ_SIZE);
DEBUGLOG(4, "Using chain-based hash table for lazy dict");
}
break;
case ZSTD_btlazy2: /* we want the dictionary table fully sorted */
case ZSTD_btopt:
case ZSTD_btultra:
case ZSTD_btultra2:
if (chunk >= HASH_READ_SIZE)
ZSTD_updateTree(ms, ichunk-HASH_READ_SIZE, ichunk);
break;
default:
assert(0); /* not possible : not a valid strategy id */
}
break;
ip = ichunk;
case ZSTD_btlazy2: /* we want the dictionary table fully sorted */
case ZSTD_btopt:
case ZSTD_btultra:
case ZSTD_btultra2:
assert(srcSize >= HASH_READ_SIZE);
ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend);
break;
default:
assert(0); /* not possible : not a valid strategy id */
}
ms->nextToUpdate = (U32)(iend - ms->window.base);
@@ -3330,7 +4334,6 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs,
const BYTE* const dictEnd = dictPtr + dictSize;
size_t dictID;
size_t eSize;
ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog)));
assert(dictSize >= 8);
assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY);
@@ -3401,8 +4404,9 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
const ZSTD_CCtx_params* params, U64 pledgedSrcSize,
ZSTD_buffered_policy_e zbuff)
{
size_t const dictContentSize = cdict ? cdict->dictContentSize : dictSize;
#if ZSTD_TRACE
cctx->traceCtx = ZSTD_trace_compress_begin(cctx);
cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0;
#endif
DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params->cParams.windowLog);
/* params are supposed to be fully validated at this point */
@@ -3419,6 +4423,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
}
FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, *params, pledgedSrcSize,
dictContentSize,
ZSTDcrp_makeClean, zbuff) , "");
{ size_t const dictID = cdict ?
ZSTD_compress_insertDictionary(
@@ -3433,7 +4438,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed");
assert(dictID <= UINT_MAX);
cctx->dictID = (U32)dictID;
cctx->dictContentSize = cdict ? cdict->dictContentSize : dictSize;
cctx->dictContentSize = dictContentSize;
}
return 0;
}
@@ -3533,7 +4538,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity)
void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize)
{
#if ZSTD_TRACE
if (cctx->traceCtx) {
if (cctx->traceCtx && ZSTD_trace_compress_end != NULL) {
int const streaming = cctx->inBuffSize > 0 || cctx->outBuffSize > 0 || cctx->appliedParams.nbWorkers > 0;
ZSTD_Trace trace;
ZSTD_memset(&trace, 0, sizeof(trace));
@@ -3669,7 +4674,10 @@ size_t ZSTD_estimateCDictSize_advanced(
DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict));
return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict))
+ ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE)
+ ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0)
/* enableDedicatedDictSearch == 1 ensures that CDict estimation will not be too small
* in case we are using DDS with row-hash. */
+ ZSTD_sizeof_matchState(&cParams, ZSTD_resolveRowMatchFinderMode(ZSTD_urm_auto, &cParams),
/* enableDedicatedDictSearch */ 1, /* forCCtx */ 0)
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0
: ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void *))));
}
@@ -3700,9 +4708,6 @@ static size_t ZSTD_initCDict_internal(
assert(!ZSTD_checkCParams(params.cParams));
cdict->matchState.cParams = params.cParams;
cdict->matchState.dedicatedDictSearch = params.enableDedicatedDictSearch;
if (cdict->matchState.dedicatedDictSearch && dictSize > ZSTD_CHUNKSIZE_MAX) {
cdict->matchState.dedicatedDictSearch = 0;
}
if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) {
cdict->dictContent = dictBuffer;
} else {
@@ -3723,6 +4728,7 @@ static size_t ZSTD_initCDict_internal(
&cdict->matchState,
&cdict->workspace,
&params.cParams,
params.useRowMatchFinder,
ZSTDcrp_makeClean,
ZSTDirp_reset,
ZSTD_resetTarget_CDict), "");
@@ -3746,14 +4752,17 @@ static size_t ZSTD_initCDict_internal(
static ZSTD_CDict* ZSTD_createCDict_advanced_internal(size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_compressionParameters cParams, ZSTD_customMem customMem)
ZSTD_compressionParameters cParams,
ZSTD_useRowMatchFinderMode_e useRowMatchFinder,
U32 enableDedicatedDictSearch,
ZSTD_customMem customMem)
{
if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL;
{ size_t const workspaceSize =
ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) +
ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) +
ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) +
ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, enableDedicatedDictSearch, /* forCCtx */ 0) +
(dictLoadMethod == ZSTD_dlm_byRef ? 0
: ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*))));
void* const workspace = ZSTD_customMalloc(workspaceSize, customMem);
@@ -3772,7 +4781,7 @@ static ZSTD_CDict* ZSTD_createCDict_advanced_internal(size_t dictSize,
ZSTD_cwksp_move(&cdict->workspace, &ws);
cdict->customMem = customMem;
cdict->compressionLevel = ZSTD_NO_CLEVEL; /* signals advanced API usage */
cdict->useRowMatchFinder = useRowMatchFinder;
return cdict;
}
}
@@ -3824,10 +4833,13 @@ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced2(
&cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict);
}
DEBUGLOG(3, "ZSTD_createCDict_advanced2: DDS: %u", cctxParams.enableDedicatedDictSearch);
cctxParams.cParams = cParams;
cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams);
cdict = ZSTD_createCDict_advanced_internal(dictSize,
dictLoadMethod, cctxParams.cParams,
cctxParams.useRowMatchFinder, cctxParams.enableDedicatedDictSearch,
customMem);
if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
@@ -3896,7 +4908,9 @@ const ZSTD_CDict* ZSTD_initStaticCDict(
ZSTD_dictContentType_e dictContentType,
ZSTD_compressionParameters cParams)
{
size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0);
ZSTD_useRowMatchFinderMode_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(ZSTD_urm_auto, &cParams);
/* enableDedicatedDictSearch == 1 ensures matchstate is not too small in case this CDict will be used for DDS + row hash */
size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0);
size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict))
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0
: ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*))))
@@ -3921,6 +4935,8 @@ const ZSTD_CDict* ZSTD_initStaticCDict(
ZSTD_CCtxParams_init(&params, 0);
params.cParams = cParams;
params.useRowMatchFinder = useRowMatchFinder;
cdict->useRowMatchFinder = useRowMatchFinder;
if (ZSTD_isError( ZSTD_initCDict_internal(cdict,
dict, dictSize,
@@ -4427,8 +5443,13 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */
ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */
assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */
if (cctx->cdict)
params.compressionLevel = cctx->cdict->compressionLevel; /* let cdict take priority in terms of compression level */
if (cctx->cdict && !cctx->localDict.cdict) {
/* Let the cdict's compression level take priority over the requested params.
* But do not take the cdict's compression level if the "cdict" is actually a localDict
* generated from ZSTD_initLocalDict().
*/
params.compressionLevel = cctx->cdict->compressionLevel;
}
DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage");
if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = inSize + 1; /* auto-fix pledgedSrcSize */
{
@@ -4447,13 +5468,20 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx,
params.ldmParams.enableLdm = 1;
}
if (ZSTD_CParams_useBlockSplitter(&params.cParams)) {
DEBUGLOG(4, "Block splitter enabled by default (window size >= 128K, strategy >= btopt)");
params.splitBlocks = 1;
}
params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, &params.cParams);
#ifdef ZSTD_MULTITHREAD
if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) {
params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */
}
if (params.nbWorkers > 0) {
#if ZSTD_TRACE
cctx->traceCtx = ZSTD_trace_compress_begin(cctx);
cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0;
#endif
/* mt context creation */
if (cctx->mtctx == NULL) {
@@ -4921,7 +5949,7 @@ static size_t ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
continue;
}
compressedSeqsSize = ZSTD_entropyCompressSequences(&cctx->seqStore,
compressedSeqsSize = ZSTD_entropyCompressSeqStore(&cctx->seqStore,
&cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy,
&cctx->appliedParams,
op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize,
@@ -4953,7 +5981,7 @@ static size_t ZSTD_compressSequences_internal(ZSTD_CCtx* cctx,
} else {
U32 cBlockHeader;
/* Error checking and repcodes update */
ZSTD_confirmRepcodesAndEntropyTables(cctx);
ZSTD_blockState_confirmRepcodesAndEntropyTables(&cctx->blockState);
if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid)
cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check;
@@ -5054,6 +6082,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
#define ZSTD_MAX_CLEVEL 22
int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; }
int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; }
int ZSTD_defaultCLevel(void) { return ZSTD_CLEVEL_DEFAULT; }
static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = {
{ /* "default" - for any srcSize > 256 KB */
@@ -5188,7 +6217,7 @@ static int ZSTD_dedicatedDictSearch_isSupported(
{
return (cParams->strategy >= ZSTD_greedy)
&& (cParams->strategy <= ZSTD_lazy2)
&& (cParams->hashLog >= cParams->chainLog)
&& (cParams->hashLog > cParams->chainLog)
&& (cParams->chainLog <= 24);
}
@@ -5207,6 +6236,9 @@ static void ZSTD_dedicatedDictSearch_revertCParams(
case ZSTD_lazy:
case ZSTD_lazy2:
cParams->hashLog -= ZSTD_LAZY_DDSS_BUCKET_LOG;
if (cParams->hashLog < ZSTD_HASHLOG_MIN) {
cParams->hashLog = ZSTD_HASHLOG_MIN;
}
break;
case ZSTD_btlazy2:
case ZSTD_btopt:
@@ -5255,6 +6287,7 @@ static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel,
else row = compressionLevel;
{ ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row];
DEBUGLOG(5, "ZSTD_getCParams_internal selected tableID: %u row: %u strat: %u", tableID, row, (U32)cp.strategy);
/* acceleration factor */
if (compressionLevel < 0) {
int const clampedCompressionLevel = MAX(ZSTD_minCLevel(), compressionLevel);
lib/compress/zstd_compress_internal.h
+129 -14
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -19,7 +19,6 @@
* Dependencies
***************************************/
#include "../common/zstd_internal.h"
#include "../common/zstd_trace.h" /* ZSTD_TraceCtx */
#include "zstd_cwksp.h"
#ifdef ZSTD_MULTITHREAD
# include "zstdmt_compress.h"
@@ -82,6 +81,53 @@ typedef struct {
ZSTD_fseCTables_t fse;
} ZSTD_entropyCTables_t;
/***********************************************
* Entropy buffer statistics structs and funcs *
***********************************************/
/** ZSTD_hufCTablesMetadata_t :
* Stores Literals Block Type for a super-block in hType, and
* huffman tree description in hufDesBuffer.
* hufDesSize refers to the size of huffman tree description in bytes.
* This metadata is populated in ZSTD_buildBlockEntropyStats_literals() */
typedef struct {
symbolEncodingType_e hType;
BYTE hufDesBuffer[ZSTD_MAX_HUF_HEADER_SIZE];
size_t hufDesSize;
} ZSTD_hufCTablesMetadata_t;
/** ZSTD_fseCTablesMetadata_t :
* Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and
* fse tables in fseTablesBuffer.
* fseTablesSize refers to the size of fse tables in bytes.
* This metadata is populated in ZSTD_buildBlockEntropyStats_sequences() */
typedef struct {
symbolEncodingType_e llType;
symbolEncodingType_e ofType;
symbolEncodingType_e mlType;
BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE];
size_t fseTablesSize;
size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */
} ZSTD_fseCTablesMetadata_t;
typedef struct {
ZSTD_hufCTablesMetadata_t hufMetadata;
ZSTD_fseCTablesMetadata_t fseMetadata;
} ZSTD_entropyCTablesMetadata_t;
/** ZSTD_buildBlockEntropyStats() :
* Builds entropy for the block.
* @return : 0 on success or error code */
size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr,
const ZSTD_entropyCTables_t* prevEntropy,
ZSTD_entropyCTables_t* nextEntropy,
const ZSTD_CCtx_params* cctxParams,
ZSTD_entropyCTablesMetadata_t* entropyMetadata,
void* workspace, size_t wkspSize);
/*********************************
* Compression internals structs *
*********************************/
typedef struct {
U32 off; /* Offset code (offset + ZSTD_REP_MOVE) for the match */
U32 len; /* Raw length of match */
@@ -150,6 +196,9 @@ typedef struct {
} ZSTD_window_t;
typedef struct ZSTD_matchState_t ZSTD_matchState_t;
#define ZSTD_ROW_HASH_CACHE_SIZE 8 /* Size of prefetching hash cache for row-based matchfinder */
struct ZSTD_matchState_t {
ZSTD_window_t window; /* State for window round buffer management */
U32 loadedDictEnd; /* index of end of dictionary, within context's referential.
@@ -161,9 +210,15 @@ struct ZSTD_matchState_t {
*/
U32 nextToUpdate; /* index from which to continue table update */
U32 hashLog3; /* dispatch table for matches of len==3 : larger == faster, more memory */
U32 rowHashLog; /* For row-based matchfinder: Hashlog based on nb of rows in the hashTable.*/
U16* tagTable; /* For row-based matchFinder: A row-based table containing the hashes and head index. */
U32 hashCache[ZSTD_ROW_HASH_CACHE_SIZE]; /* For row-based matchFinder: a cache of hashes to improve speed */
U32* hashTable;
U32* hashTable3;
U32* chainTable;
int dedicatedDictSearch; /* Indicates whether this matchState is using the
* dedicated dictionary search structure.
*/
@@ -256,6 +311,12 @@ struct ZSTD_CCtx_params_s {
ZSTD_sequenceFormat_e blockDelimiters;
int validateSequences;
/* Block splitting */
int splitBlocks;
/* Param for deciding whether to use row-based matchfinder */
ZSTD_useRowMatchFinderMode_e useRowMatchFinder;
/* Internal use, for createCCtxParams() and freeCCtxParams() only */
ZSTD_customMem customMem;
}; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */
@@ -371,7 +432,7 @@ typedef enum {
typedef size_t (*ZSTD_blockCompressor) (
ZSTD_matchState_t* bs, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMode_e dictMode);
ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_useRowMatchFinderMode_e rowMatchfinderMode, ZSTD_dictMode_e dictMode);
MEM_STATIC U32 ZSTD_LLcode(U32 litLength)
@@ -548,8 +609,8 @@ void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* litera
/* literal Length */
if (litLength>0xFFFF) {
assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */
seqStorePtr->longLengthID = 1;
assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
seqStorePtr->longLengthType = ZSTD_llt_literalLength;
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
}
seqStorePtr->sequences[0].litLength = (U16)litLength;
@@ -559,8 +620,8 @@ void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* litera
/* match Length */
if (mlBase>0xFFFF) {
assert(seqStorePtr->longLengthID == 0); /* there can only be a single long length */
seqStorePtr->longLengthID = 2;
assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */
seqStorePtr->longLengthType = ZSTD_llt_matchLength;
seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart);
}
seqStorePtr->sequences[0].matchLength = (U16)mlBase;
@@ -811,6 +872,13 @@ MEM_STATIC void ZSTD_window_clear(ZSTD_window_t* window)
window->dictLimit = end;
}
MEM_STATIC U32 ZSTD_window_isEmpty(ZSTD_window_t const window)
{
return window.dictLimit == 1 &&
window.lowLimit == 1 &&
(window.nextSrc - window.base) == 1;
}
/**
* ZSTD_window_hasExtDict():
* Returns non-zero if the window has a non-empty extDict.
@@ -834,15 +902,55 @@ MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_matchState_t *ms)
ZSTD_noDict;
}
/* Defining this macro to non-zero tells zstd to run the overflow correction
* code much more frequently. This is very inefficient, and should only be
* used for tests and fuzzers.
*/
#ifndef ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY
# ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 1
# else
# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 0
# endif
#endif
/**
* ZSTD_window_canOverflowCorrect():
* Returns non-zero if the indices are large enough for overflow correction
* to work correctly without impacting compression ratio.
*/
MEM_STATIC U32 ZSTD_window_canOverflowCorrect(ZSTD_window_t const window,
U32 cycleLog,
U32 maxDist,
U32 loadedDictEnd,
void const* src)
{
U32 const cycleSize = 1u << cycleLog;
U32 const curr = (U32)((BYTE const*)src - window.base);
U32 const minIndexToOverflowCorrect = cycleSize + MAX(maxDist, cycleSize);
U32 const indexLargeEnough = curr > minIndexToOverflowCorrect;
U32 const dictionaryInvalidated = curr > maxDist + loadedDictEnd;
return indexLargeEnough && dictionaryInvalidated;
}
/**
* ZSTD_window_needOverflowCorrection():
* Returns non-zero if the indices are getting too large and need overflow
* protection.
*/
MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window,
U32 cycleLog,
U32 maxDist,
U32 loadedDictEnd,
void const* src,
void const* srcEnd)
{
U32 const curr = (U32)((BYTE const*)srcEnd - window.base);
if (ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) {
if (ZSTD_window_canOverflowCorrect(window, cycleLog, maxDist, loadedDictEnd, src)) {
return 1;
}
}
return curr > ZSTD_CURRENT_MAX;
}
@@ -854,7 +962,6 @@ MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window,
*
* The least significant cycleLog bits of the indices must remain the same,
* which may be 0. Every index up to maxDist in the past must be valid.
* NOTE: (maxDist & cycleMask) must be zero.
*/
MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog,
U32 maxDist, void const* src)
@@ -878,17 +985,25 @@ MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog,
* 3. (cctx->lowLimit + 1<<windowLog) < 1<<32:
* windowLog <= 31 ==> 3<<29 + 1<<windowLog < 7<<29 < 1<<32.
*/
U32 const cycleMask = (1U << cycleLog) - 1;
U32 const cycleSize = 1u << cycleLog;
U32 const cycleMask = cycleSize - 1;
U32 const curr = (U32)((BYTE const*)src - window->base);
U32 const currentCycle0 = curr & cycleMask;
/* Exclude zero so that newCurrent - maxDist >= 1. */
U32 const currentCycle1 = currentCycle0 == 0 ? (1U << cycleLog) : currentCycle0;
U32 const newCurrent = currentCycle1 + maxDist;
U32 const currentCycle1 = currentCycle0 == 0 ? cycleSize : currentCycle0;
U32 const newCurrent = currentCycle1 + MAX(maxDist, cycleSize);
U32 const correction = curr - newCurrent;
assert((maxDist & cycleMask) == 0);
/* maxDist must be a power of two so that:
* (newCurrent & cycleMask) == (curr & cycleMask)
* This is required to not corrupt the chains / binary tree.
*/
assert((maxDist & (maxDist - 1)) == 0);
assert((curr & cycleMask) == (newCurrent & cycleMask));
assert(curr > newCurrent);
/* Loose bound, should be around 1<<29 (see above) */
assert(correction > 1<<28);
if (!ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) {
/* Loose bound, should be around 1<<29 (see above) */
assert(correction > 1<<28);
}
window->base += correction;
window->dictBase += correction;
lib/compress/zstd_compress_literals.c
+2 -2
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -117,7 +117,7 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf,
}
}
if ((cLitSize==0) | (cLitSize >= srcSize - minGain) | ERR_isError(cLitSize)) {
if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) {
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
}
lib/compress/zstd_compress_literals.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/compress/zstd_compress_sequences.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/compress/zstd_compress_sequences.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/compress/zstd_compress_superblock.c
+3 -282
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -15,289 +15,10 @@
#include "../common/zstd_internal.h" /* ZSTD_getSequenceLength */
#include "hist.h" /* HIST_countFast_wksp */
#include "zstd_compress_internal.h"
#include "zstd_compress_internal.h" /* ZSTD_[huf|fse|entropy]CTablesMetadata_t */
#include "zstd_compress_sequences.h"
#include "zstd_compress_literals.h"
/*-*************************************
* Superblock entropy buffer structs
***************************************/
/** ZSTD_hufCTablesMetadata_t :
* Stores Literals Block Type for a super-block in hType, and
* huffman tree description in hufDesBuffer.
* hufDesSize refers to the size of huffman tree description in bytes.
* This metadata is populated in ZSTD_buildSuperBlockEntropy_literal() */
typedef struct {
symbolEncodingType_e hType;
BYTE hufDesBuffer[ZSTD_MAX_HUF_HEADER_SIZE];
size_t hufDesSize;
} ZSTD_hufCTablesMetadata_t;
/** ZSTD_fseCTablesMetadata_t :
* Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and
* fse tables in fseTablesBuffer.
* fseTablesSize refers to the size of fse tables in bytes.
* This metadata is populated in ZSTD_buildSuperBlockEntropy_sequences() */
typedef struct {
symbolEncodingType_e llType;
symbolEncodingType_e ofType;
symbolEncodingType_e mlType;
BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE];
size_t fseTablesSize;
size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_compressSubBlock_sequences() */
} ZSTD_fseCTablesMetadata_t;
typedef struct {
ZSTD_hufCTablesMetadata_t hufMetadata;
ZSTD_fseCTablesMetadata_t fseMetadata;
} ZSTD_entropyCTablesMetadata_t;
/** ZSTD_buildSuperBlockEntropy_literal() :
* Builds entropy for the super-block literals.
* Stores literals block type (raw, rle, compressed, repeat) and
* huffman description table to hufMetadata.
* @return : size of huffman description table or error code */
static size_t ZSTD_buildSuperBlockEntropy_literal(void* const src, size_t srcSize,
const ZSTD_hufCTables_t* prevHuf,
ZSTD_hufCTables_t* nextHuf,
ZSTD_hufCTablesMetadata_t* hufMetadata,
const int disableLiteralsCompression,
void* workspace, size_t wkspSize)
{
BYTE* const wkspStart = (BYTE*)workspace;
BYTE* const wkspEnd = wkspStart + wkspSize;
BYTE* const countWkspStart = wkspStart;
unsigned* const countWksp = (unsigned*)workspace;
const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned);
BYTE* const nodeWksp = countWkspStart + countWkspSize;
const size_t nodeWkspSize = wkspEnd-nodeWksp;
unsigned maxSymbolValue = 255;
unsigned huffLog = HUF_TABLELOG_DEFAULT;
HUF_repeat repeat = prevHuf->repeatMode;
DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_literal (srcSize=%zu)", srcSize);
/* Prepare nextEntropy assuming reusing the existing table */
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
if (disableLiteralsCompression) {
DEBUGLOG(5, "set_basic - disabled");
hufMetadata->hType = set_basic;
return 0;
}
/* small ? don't even attempt compression (speed opt) */
# define COMPRESS_LITERALS_SIZE_MIN 63
{ size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN;
if (srcSize <= minLitSize) {
DEBUGLOG(5, "set_basic - too small");
hufMetadata->hType = set_basic;
return 0;
}
}
/* Scan input and build symbol stats */
{ size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)src, srcSize, workspace, wkspSize);
FORWARD_IF_ERROR(largest, "HIST_count_wksp failed");
if (largest == srcSize) {
DEBUGLOG(5, "set_rle");
hufMetadata->hType = set_rle;
return 0;
}
if (largest <= (srcSize >> 7)+4) {
DEBUGLOG(5, "set_basic - no gain");
hufMetadata->hType = set_basic;
return 0;
}
}
/* Validate the previous Huffman table */
if (repeat == HUF_repeat_check && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) {
repeat = HUF_repeat_none;
}
/* Build Huffman Tree */
ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable));
huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
{ size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp,
maxSymbolValue, huffLog,
nodeWksp, nodeWkspSize);
FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp");
huffLog = (U32)maxBits;
{ /* Build and write the CTable */
size_t const newCSize = HUF_estimateCompressedSize(
(HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue);
size_t const hSize = HUF_writeCTable_wksp(
hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer),
(HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog,
nodeWksp, nodeWkspSize);
/* Check against repeating the previous CTable */
if (repeat != HUF_repeat_none) {
size_t const oldCSize = HUF_estimateCompressedSize(
(HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue);
if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) {
DEBUGLOG(5, "set_repeat - smaller");
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
hufMetadata->hType = set_repeat;
return 0;
}
}
if (newCSize + hSize >= srcSize) {
DEBUGLOG(5, "set_basic - no gains");
ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
hufMetadata->hType = set_basic;
return 0;
}
DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize);
hufMetadata->hType = set_compressed;
nextHuf->repeatMode = HUF_repeat_check;
return hSize;
}
}
}
/** ZSTD_buildSuperBlockEntropy_sequences() :
* Builds entropy for the super-block sequences.
* Stores symbol compression modes and fse table to fseMetadata.
* @return : size of fse tables or error code */
static size_t ZSTD_buildSuperBlockEntropy_sequences(seqStore_t* seqStorePtr,
const ZSTD_fseCTables_t* prevEntropy,
ZSTD_fseCTables_t* nextEntropy,
const ZSTD_CCtx_params* cctxParams,
ZSTD_fseCTablesMetadata_t* fseMetadata,
void* workspace, size_t wkspSize)
{
BYTE* const wkspStart = (BYTE*)workspace;
BYTE* const wkspEnd = wkspStart + wkspSize;
BYTE* const countWkspStart = wkspStart;
unsigned* const countWksp = (unsigned*)workspace;
const size_t countWkspSize = (MaxSeq + 1) * sizeof(unsigned);
BYTE* const cTableWksp = countWkspStart + countWkspSize;
const size_t cTableWkspSize = wkspEnd-cTableWksp;
ZSTD_strategy const strategy = cctxParams->cParams.strategy;
FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable;
FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable;
FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable;
const BYTE* const ofCodeTable = seqStorePtr->ofCode;
const BYTE* const llCodeTable = seqStorePtr->llCode;
const BYTE* const mlCodeTable = seqStorePtr->mlCode;
size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
BYTE* const ostart = fseMetadata->fseTablesBuffer;
BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer);
BYTE* op = ostart;
assert(cTableWkspSize >= (1 << MaxFSELog) * sizeof(FSE_FUNCTION_TYPE));
DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_sequences (nbSeq=%zu)", nbSeq);
ZSTD_memset(workspace, 0, wkspSize);
fseMetadata->lastCountSize = 0;
/* convert length/distances into codes */
ZSTD_seqToCodes(seqStorePtr);
/* build CTable for Literal Lengths */
{ U32 LLtype;
unsigned max = MaxLL;
size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, llCodeTable, nbSeq, workspace, wkspSize); /* can't fail */
DEBUGLOG(5, "Building LL table");
nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode;
LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode,
countWksp, max, mostFrequent, nbSeq,
LLFSELog, prevEntropy->litlengthCTable,
LL_defaultNorm, LL_defaultNormLog,
ZSTD_defaultAllowed, strategy);
assert(set_basic < set_compressed && set_rle < set_compressed);
assert(!(LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype,
countWksp, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL,
prevEntropy->litlengthCTable, sizeof(prevEntropy->litlengthCTable),
cTableWksp, cTableWkspSize);
FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for LitLens failed");
if (LLtype == set_compressed)
fseMetadata->lastCountSize = countSize;
op += countSize;
fseMetadata->llType = (symbolEncodingType_e) LLtype;
} }
/* build CTable for Offsets */
{ U32 Offtype;
unsigned max = MaxOff;
size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, ofCodeTable, nbSeq, workspace, wkspSize); /* can't fail */
/* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */
ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed;
DEBUGLOG(5, "Building OF table");
nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode;
Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode,
countWksp, max, mostFrequent, nbSeq,
OffFSELog, prevEntropy->offcodeCTable,
OF_defaultNorm, OF_defaultNormLog,
defaultPolicy, strategy);
assert(!(Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype,
countWksp, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
prevEntropy->offcodeCTable, sizeof(prevEntropy->offcodeCTable),
cTableWksp, cTableWkspSize);
FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for Offsets failed");
if (Offtype == set_compressed)
fseMetadata->lastCountSize = countSize;
op += countSize;
fseMetadata->ofType = (symbolEncodingType_e) Offtype;
} }
/* build CTable for MatchLengths */
{ U32 MLtype;
unsigned max = MaxML;
size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, mlCodeTable, nbSeq, workspace, wkspSize); /* can't fail */
DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op));
nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode;
MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode,
countWksp, max, mostFrequent, nbSeq,
MLFSELog, prevEntropy->matchlengthCTable,
ML_defaultNorm, ML_defaultNormLog,
ZSTD_defaultAllowed, strategy);
assert(!(MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
{ size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype,
countWksp, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML,
prevEntropy->matchlengthCTable, sizeof(prevEntropy->matchlengthCTable),
cTableWksp, cTableWkspSize);
FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for MatchLengths failed");
if (MLtype == set_compressed)
fseMetadata->lastCountSize = countSize;
op += countSize;
fseMetadata->mlType = (symbolEncodingType_e) MLtype;
} }
assert((size_t) (op-ostart) <= sizeof(fseMetadata->fseTablesBuffer));
return op-ostart;
}
/** ZSTD_buildSuperBlockEntropy() :
* Builds entropy for the super-block.
* @return : 0 on success or error code */
static size_t
ZSTD_buildSuperBlockEntropy(seqStore_t* seqStorePtr,
const ZSTD_entropyCTables_t* prevEntropy,
ZSTD_entropyCTables_t* nextEntropy,
const ZSTD_CCtx_params* cctxParams,
ZSTD_entropyCTablesMetadata_t* entropyMetadata,
void* workspace, size_t wkspSize)
{
size_t const litSize = seqStorePtr->lit - seqStorePtr->litStart;
DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy");
entropyMetadata->hufMetadata.hufDesSize =
ZSTD_buildSuperBlockEntropy_literal(seqStorePtr->litStart, litSize,
&prevEntropy->huf, &nextEntropy->huf,
&entropyMetadata->hufMetadata,
ZSTD_disableLiteralsCompression(cctxParams),
workspace, wkspSize);
FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildSuperBlockEntropy_literal failed");
entropyMetadata->fseMetadata.fseTablesSize =
ZSTD_buildSuperBlockEntropy_sequences(seqStorePtr,
&prevEntropy->fse, &nextEntropy->fse,
cctxParams,
&entropyMetadata->fseMetadata,
workspace, wkspSize);
FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildSuperBlockEntropy_sequences failed");
return 0;
}
/** ZSTD_compressSubBlock_literal() :
* Compresses literals section for a sub-block.
* When we have to write the Huffman table we will sometimes choose a header
@@ -831,7 +552,7 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
unsigned lastBlock) {
ZSTD_entropyCTablesMetadata_t entropyMetadata;
FORWARD_IF_ERROR(ZSTD_buildSuperBlockEntropy(&zc->seqStore,
FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(&zc->seqStore,
&zc->blockState.prevCBlock->entropy,
&zc->blockState.nextCBlock->entropy,
&zc->appliedParams,
lib/compress/zstd_compress_superblock.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/compress/zstd_cwksp.h
+147 -46
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -35,6 +35,10 @@ extern "C" {
#define ZSTD_CWKSP_ASAN_REDZONE_SIZE 128
#endif
/* Set our tables and aligneds to align by 64 bytes */
#define ZSTD_CWKSP_ALIGNMENT_BYTES 64
/*-*************************************
* Structures
***************************************/
@@ -117,10 +121,11 @@ typedef enum {
* - Tables: these are any of several different datastructures (hash tables,
* chain tables, binary trees) that all respect a common format: they are
* uint32_t arrays, all of whose values are between 0 and (nextSrc - base).
* Their sizes depend on the cparams.
* Their sizes depend on the cparams. These tables are 64-byte aligned.
*
* - Aligned: these buffers are used for various purposes that require 4 byte
* alignment, but don't require any initialization before they're used.
* alignment, but don't require any initialization before they're used. These
* buffers are each aligned to 64 bytes.
*
* - Buffers: these buffers are used for various purposes that don't require
* any alignment or initialization before they're used. This means they can
@@ -133,8 +138,7 @@ typedef enum {
*
* 1. Objects
* 2. Buffers
* 3. Aligned
* 4. Tables
* 3. Aligned/Tables
*
* Attempts to reserve objects of different types out of order will fail.
*/
@@ -187,6 +191,8 @@ MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) {
* Since tables aren't currently redzoned, you don't need to call through this
* to figure out how much space you need for the matchState tables. Everything
* else is though.
*
* Do not use for sizing aligned buffers. Instead, use ZSTD_cwksp_aligned_alloc_size().
*/
MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) {
if (size == 0)
@@ -198,30 +204,110 @@ MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) {
#endif
}
MEM_STATIC void ZSTD_cwksp_internal_advance_phase(
/**
* Returns an adjusted alloc size that is the nearest larger multiple of 64 bytes.
* Used to determine the number of bytes required for a given "aligned".
*/
MEM_STATIC size_t ZSTD_cwksp_aligned_alloc_size(size_t size) {
return ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(size, ZSTD_CWKSP_ALIGNMENT_BYTES));
}
/**
* Returns the amount of additional space the cwksp must allocate
* for internal purposes (currently only alignment).
*/
MEM_STATIC size_t ZSTD_cwksp_slack_space_required(void) {
/* For alignment, the wksp will always allocate an additional n_1=[1, 64] bytes
* to align the beginning of tables section, as well as another n_2=[0, 63] bytes
* to align the beginning of the aligned secion.
*
* n_1 + n_2 == 64 bytes if the cwksp is freshly allocated, due to tables and
* aligneds being sized in multiples of 64 bytes.
*/
size_t const slackSpace = ZSTD_CWKSP_ALIGNMENT_BYTES;
return slackSpace;
}
/**
* Return the number of additional bytes required to align a pointer to the given number of bytes.
* alignBytes must be a power of two.
*/
MEM_STATIC size_t ZSTD_cwksp_bytes_to_align_ptr(void* ptr, const size_t alignBytes) {
size_t const alignBytesMask = alignBytes - 1;
size_t const bytes = (alignBytes - ((size_t)ptr & (alignBytesMask))) & alignBytesMask;
assert((alignBytes & alignBytesMask) == 0);
assert(bytes != ZSTD_CWKSP_ALIGNMENT_BYTES);
return bytes;
}
/**
* Internal function. Do not use directly.
* Reserves the given number of bytes within the aligned/buffer segment of the wksp, which
* counts from the end of the wksp. (as opposed to the object/table segment)
*
* Returns a pointer to the beginning of that space.
*/
MEM_STATIC void* ZSTD_cwksp_reserve_internal_buffer_space(ZSTD_cwksp* ws, size_t const bytes) {
void* const alloc = (BYTE*)ws->allocStart - bytes;
void* const bottom = ws->tableEnd;
DEBUGLOG(5, "cwksp: reserving %p %zd bytes, %zd bytes remaining",
alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);
ZSTD_cwksp_assert_internal_consistency(ws);
assert(alloc >= bottom);
if (alloc < bottom) {
DEBUGLOG(4, "cwksp: alloc failed!");
ws->allocFailed = 1;
return NULL;
}
if (alloc < ws->tableValidEnd) {
ws->tableValidEnd = alloc;
}
ws->allocStart = alloc;
return alloc;
}
/**
* Moves the cwksp to the next phase, and does any necessary allocations.
* Returns a 0 on success, or zstd error
*/
MEM_STATIC size_t ZSTD_cwksp_internal_advance_phase(
ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) {
assert(phase >= ws->phase);
if (phase > ws->phase) {
/* Going from allocating objects to allocating buffers */
if (ws->phase < ZSTD_cwksp_alloc_buffers &&
phase >= ZSTD_cwksp_alloc_buffers) {
ws->tableValidEnd = ws->objectEnd;
}
/* Going from allocating buffers to allocating aligneds/tables */
if (ws->phase < ZSTD_cwksp_alloc_aligned &&
phase >= ZSTD_cwksp_alloc_aligned) {
/* If unaligned allocations down from a too-large top have left us
* unaligned, we need to realign our alloc ptr. Technically, this
* can consume space that is unaccounted for in the neededSpace
* calculation. However, I believe this can only happen when the
* workspace is too large, and specifically when it is too large
* by a larger margin than the space that will be consumed. */
/* TODO: cleaner, compiler warning friendly way to do this??? */
ws->allocStart = (BYTE*)ws->allocStart - ((size_t)ws->allocStart & (sizeof(U32)-1));
if (ws->allocStart < ws->tableValidEnd) {
ws->tableValidEnd = ws->allocStart;
{ /* Align the start of the "aligned" to 64 bytes. Use [1, 64] bytes. */
size_t const bytesToAlign =
ZSTD_CWKSP_ALIGNMENT_BYTES - ZSTD_cwksp_bytes_to_align_ptr(ws->allocStart, ZSTD_CWKSP_ALIGNMENT_BYTES);
DEBUGLOG(5, "reserving aligned alignment addtl space: %zu", bytesToAlign);
ZSTD_STATIC_ASSERT((ZSTD_CWKSP_ALIGNMENT_BYTES & (ZSTD_CWKSP_ALIGNMENT_BYTES - 1)) == 0); /* power of 2 */
RETURN_ERROR_IF(!ZSTD_cwksp_reserve_internal_buffer_space(ws, bytesToAlign),
memory_allocation, "aligned phase - alignment initial allocation failed!");
}
{ /* Align the start of the tables to 64 bytes. Use [0, 63] bytes */
void* const alloc = ws->objectEnd;
size_t const bytesToAlign = ZSTD_cwksp_bytes_to_align_ptr(alloc, ZSTD_CWKSP_ALIGNMENT_BYTES);
void* const end = (BYTE*)alloc + bytesToAlign;
DEBUGLOG(5, "reserving table alignment addtl space: %zu", bytesToAlign);
RETURN_ERROR_IF(end > ws->workspaceEnd, memory_allocation,
"table phase - alignment initial allocation failed!");
ws->objectEnd = end;
ws->tableEnd = end;
ws->tableValidEnd = end;
}
}
ws->phase = phase;
ZSTD_cwksp_assert_internal_consistency(ws);
}
return 0;
}
/**
@@ -237,38 +323,25 @@ MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr) {
MEM_STATIC void* ZSTD_cwksp_reserve_internal(
ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) {
void* alloc;
void* bottom = ws->tableEnd;
ZSTD_cwksp_internal_advance_phase(ws, phase);
alloc = (BYTE *)ws->allocStart - bytes;
if (bytes == 0)
if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase)) || bytes == 0) {
return NULL;
}
#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
/* over-reserve space */
alloc = (BYTE *)alloc - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
bytes += 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE;
#endif
DEBUGLOG(5, "cwksp: reserving %p %zd bytes, %zd bytes remaining",
alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);
ZSTD_cwksp_assert_internal_consistency(ws);
assert(alloc >= bottom);
if (alloc < bottom) {
DEBUGLOG(4, "cwksp: alloc failed!");
ws->allocFailed = 1;
return NULL;
}
if (alloc < ws->tableValidEnd) {
ws->tableValidEnd = alloc;
}
ws->allocStart = alloc;
alloc = ZSTD_cwksp_reserve_internal_buffer_space(ws, bytes);
#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE)
/* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on
* either size. */
alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE;
if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) {
__asan_unpoison_memory_region(alloc, bytes);
if (alloc) {
alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE;
if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) {
__asan_unpoison_memory_region(alloc, bytes);
}
}
#endif
@@ -283,28 +356,36 @@ MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) {
}
/**
* Reserves and returns memory sized on and aligned on sizeof(unsigned).
* Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes).
*/
MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) {
assert((bytes & (sizeof(U32)-1)) == 0);
return ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, sizeof(U32)), ZSTD_cwksp_alloc_aligned);
void* ptr = ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES),
ZSTD_cwksp_alloc_aligned);
assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0);
return ptr;
}
/**
* Aligned on sizeof(unsigned). These buffers have the special property that
* Aligned on 64 bytes. These buffers have the special property that
* their values remain constrained, allowing us to re-use them without
* memset()-ing them.
*/
MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) {
const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned;
void* alloc = ws->tableEnd;
void* end = (BYTE *)alloc + bytes;
void* top = ws->allocStart;
void* alloc;
void* end;
void* top;
if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase))) {
return NULL;
}
alloc = ws->tableEnd;
end = (BYTE *)alloc + bytes;
top = ws->allocStart;
DEBUGLOG(5, "cwksp: reserving %p table %zd bytes, %zd bytes remaining",
alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);
assert((bytes & (sizeof(U32)-1)) == 0);
ZSTD_cwksp_internal_advance_phase(ws, phase);
ZSTD_cwksp_assert_internal_consistency(ws);
assert(end <= top);
if (end > top) {
@@ -320,6 +401,8 @@ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) {
}
#endif
assert((bytes & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0);
assert(((size_t)alloc & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0);
return alloc;
}
@@ -527,6 +610,24 @@ MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) {
* Functions Checking Free Space
***************************************/
/* ZSTD_alignmentSpaceWithinBounds() :
* Returns if the estimated space needed for a wksp is within an acceptable limit of the
* actual amount of space used.
*/
MEM_STATIC int ZSTD_cwksp_estimated_space_within_bounds(const ZSTD_cwksp* const ws,
size_t const estimatedSpace, int resizedWorkspace) {
if (resizedWorkspace) {
/* Resized/newly allocated wksp should have exact bounds */
return ZSTD_cwksp_used(ws) == estimatedSpace;
} else {
/* Due to alignment, when reusing a workspace, we can actually consume 63 fewer or more bytes
* than estimatedSpace. See the comments in zstd_cwksp.h for details.
*/
return (ZSTD_cwksp_used(ws) >= estimatedSpace - 63) && (ZSTD_cwksp_used(ws) <= estimatedSpace + 63);
}
}
MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) {
return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd);
}
lib/compress/zstd_double_fast.c
+3 -3
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -409,7 +409,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */
if ((((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex doesn't overlap dict + prefix */
& (repIndex > dictStartIndex))
& (offset_1 < curr+1 - dictStartIndex)) /* note: we are searching at curr+1 */
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4;
@@ -477,7 +477,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
U32 const repIndex2 = current2 - offset_2;
const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2;
if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) /* intentional overflow : ensure repIndex2 doesn't overlap dict + prefix */
& (repIndex2 > dictStartIndex))
& (offset_2 < current2 - dictStartIndex))
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
lib/compress/zstd_double_fast.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/compress/zstd_fast.c
+4 -4
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -416,9 +416,9 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
const BYTE* const repMatch = repBase + repIndex;
hashTable[h] = curr; /* update hash table */
DEBUGLOG(7, "offset_1 = %u , curr = %u", offset_1, curr);
assert(offset_1 <= curr +1); /* check repIndex */
if ( (((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > dictStartIndex))
if ( ( ((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow */
& (offset_1 < curr+1 - dictStartIndex) ) /* note: we are searching at curr+1 */
&& (MEM_read32(repMatch) == MEM_read32(ip+1)) ) {
const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend;
size_t const rLength = ZSTD_count_2segments(ip+1 +4, repMatch +4, iend, repMatchEnd, prefixStart) + 4;
@@ -453,7 +453,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
U32 const current2 = (U32)(ip-base);
U32 const repIndex2 = current2 - offset_2;
const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2;
if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (repIndex2 > dictStartIndex)) /* intentional overflow */
if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (offset_2 < curr - dictStartIndex)) /* intentional overflow */
&& (MEM_read32(repMatch2) == MEM_read32(ip)) ) {
const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend;
size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4;
lib/compress/zstd_fast.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/compress/zstd_lazy.c
+914 -142
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -438,43 +438,9 @@ static size_t ZSTD_BtFindBestMatch_extDict_selectMLS (
}
}
/* *********************************
* Hash Chain
/***********************************
* Dedicated dict search
***********************************/
#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & (mask)]
/* Update chains up to ip (excluded)
Assumption : always within prefix (i.e. not within extDict) */
FORCE_INLINE_TEMPLATE U32 ZSTD_insertAndFindFirstIndex_internal(
ZSTD_matchState_t* ms,
const ZSTD_compressionParameters* const cParams,
const BYTE* ip, U32 const mls)
{
U32* const hashTable = ms->hashTable;
const U32 hashLog = cParams->hashLog;
U32* const chainTable = ms->chainTable;
const U32 chainMask = (1 << cParams->chainLog) - 1;
const BYTE* const base = ms->window.base;
const U32 target = (U32)(ip - base);
U32 idx = ms->nextToUpdate;
while(idx < target) { /* catch up */
size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls);
NEXT_IN_CHAIN(idx, chainMask) = hashTable[h];
hashTable[h] = idx;
idx++;
}
ms->nextToUpdate = target;
return hashTable[ZSTD_hashPtr(ip, hashLog, mls)];
}
U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) {
const ZSTD_compressionParameters* const cParams = &ms->cParams;
return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch);
}
void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip)
{
@@ -500,11 +466,10 @@ void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const B
U32* const tmpChainTable = hashTable + ((size_t)1 << hashLog);
U32 const tmpChainSize = ((1 << ZSTD_LAZY_DDSS_BUCKET_LOG) - 1) << hashLog;
U32 const tmpMinChain = tmpChainSize < target ? target - tmpChainSize : idx;
U32 hashIdx;
assert(ms->cParams.chainLog <= 24);
assert(ms->cParams.hashLog >= ms->cParams.chainLog);
assert(ms->cParams.hashLog > ms->cParams.chainLog);
assert(idx != 0);
assert(tmpMinChain <= minChain);
@@ -535,7 +500,7 @@ void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const B
if (count == cacheSize) {
for (count = 0; count < chainLimit;) {
if (i < minChain) {
if (!i || countBeyondMinChain++ > cacheSize) {
if (!i || ++countBeyondMinChain > cacheSize) {
/* only allow pulling `cacheSize` number of entries
* into the cache or chainTable beyond `minChain`,
* to replace the entries pulled out of the
@@ -591,6 +556,139 @@ void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const B
ms->nextToUpdate = target;
}
/* Returns the longest match length found in the dedicated dict search structure.
* If none are longer than the argument ml, then ml will be returned.
*/
FORCE_INLINE_TEMPLATE
size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nbAttempts,
const ZSTD_matchState_t* const dms,
const BYTE* const ip, const BYTE* const iLimit,
const BYTE* const prefixStart, const U32 curr,
const U32 dictLimit, const size_t ddsIdx) {
const U32 ddsLowestIndex = dms->window.dictLimit;
const BYTE* const ddsBase = dms->window.base;
const BYTE* const ddsEnd = dms->window.nextSrc;
const U32 ddsSize = (U32)(ddsEnd - ddsBase);
const U32 ddsIndexDelta = dictLimit - ddsSize;
const U32 bucketSize = (1 << ZSTD_LAZY_DDSS_BUCKET_LOG);
const U32 bucketLimit = nbAttempts < bucketSize - 1 ? nbAttempts : bucketSize - 1;
U32 ddsAttempt;
U32 matchIndex;
for (ddsAttempt = 0; ddsAttempt < bucketSize - 1; ddsAttempt++) {
PREFETCH_L1(ddsBase + dms->hashTable[ddsIdx + ddsAttempt]);
}
{
U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1];
U32 const chainIndex = chainPackedPointer >> 8;
PREFETCH_L1(&dms->chainTable[chainIndex]);
}
for (ddsAttempt = 0; ddsAttempt < bucketLimit; ddsAttempt++) {
size_t currentMl=0;
const BYTE* match;
matchIndex = dms->hashTable[ddsIdx + ddsAttempt];
match = ddsBase + matchIndex;
if (!matchIndex) {
return ml;
}
/* guaranteed by table construction */
(void)ddsLowestIndex;
assert(matchIndex >= ddsLowestIndex);
assert(match+4 <= ddsEnd);
if (MEM_read32(match) == MEM_read32(ip)) {
/* assumption : matchIndex <= dictLimit-4 (by table construction) */
currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4;
}
/* save best solution */
if (currentMl > ml) {
ml = currentMl;
*offsetPtr = curr - (matchIndex + ddsIndexDelta) + ZSTD_REP_MOVE;
if (ip+currentMl == iLimit) {
/* best possible, avoids read overflow on next attempt */
return ml;
}
}
}
{
U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1];
U32 chainIndex = chainPackedPointer >> 8;
U32 const chainLength = chainPackedPointer & 0xFF;
U32 const chainAttempts = nbAttempts - ddsAttempt;
U32 const chainLimit = chainAttempts > chainLength ? chainLength : chainAttempts;
U32 chainAttempt;
for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++) {
PREFETCH_L1(ddsBase + dms->chainTable[chainIndex + chainAttempt]);
}
for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++, chainIndex++) {
size_t currentMl=0;
const BYTE* match;
matchIndex = dms->chainTable[chainIndex];
match = ddsBase + matchIndex;
/* guaranteed by table construction */
assert(matchIndex >= ddsLowestIndex);
assert(match+4 <= ddsEnd);
if (MEM_read32(match) == MEM_read32(ip)) {
/* assumption : matchIndex <= dictLimit-4 (by table construction) */
currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4;
}
/* save best solution */
if (currentMl > ml) {
ml = currentMl;
*offsetPtr = curr - (matchIndex + ddsIndexDelta) + ZSTD_REP_MOVE;
if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
}
}
}
return ml;
}
/* *********************************
* Hash Chain
***********************************/
#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & (mask)]
/* Update chains up to ip (excluded)
Assumption : always within prefix (i.e. not within extDict) */
FORCE_INLINE_TEMPLATE U32 ZSTD_insertAndFindFirstIndex_internal(
ZSTD_matchState_t* ms,
const ZSTD_compressionParameters* const cParams,
const BYTE* ip, U32 const mls)
{
U32* const hashTable = ms->hashTable;
const U32 hashLog = cParams->hashLog;
U32* const chainTable = ms->chainTable;
const U32 chainMask = (1 << cParams->chainLog) - 1;
const BYTE* const base = ms->window.base;
const U32 target = (U32)(ip - base);
U32 idx = ms->nextToUpdate;
while(idx < target) { /* catch up */
size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls);
NEXT_IN_CHAIN(idx, chainMask) = hashTable[h];
hashTable[h] = idx;
idx++;
}
ms->nextToUpdate = target;
return hashTable[ZSTD_hashPtr(ip, hashLog, mls)];
}
U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) {
const ZSTD_compressionParameters* const cParams = &ms->cParams;
return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch);
}
/* inlining is important to hardwire a hot branch (template emulation) */
FORCE_INLINE_TEMPLATE
@@ -661,90 +759,8 @@ size_t ZSTD_HcFindBestMatch_generic (
}
if (dictMode == ZSTD_dedicatedDictSearch) {
const U32 ddsLowestIndex = dms->window.dictLimit;
const BYTE* const ddsBase = dms->window.base;
const BYTE* const ddsEnd = dms->window.nextSrc;
const U32 ddsSize = (U32)(ddsEnd - ddsBase);
const U32 ddsIndexDelta = dictLimit - ddsSize;
const U32 bucketSize = (1 << ZSTD_LAZY_DDSS_BUCKET_LOG);
const U32 bucketLimit = nbAttempts < bucketSize - 1 ? nbAttempts : bucketSize - 1;
U32 ddsAttempt;
for (ddsAttempt = 0; ddsAttempt < bucketSize - 1; ddsAttempt++) {
PREFETCH_L1(ddsBase + dms->hashTable[ddsIdx + ddsAttempt]);
}
{
U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1];
U32 const chainIndex = chainPackedPointer >> 8;
PREFETCH_L1(&dms->chainTable[chainIndex]);
}
for (ddsAttempt = 0; ddsAttempt < bucketLimit; ddsAttempt++) {
size_t currentMl=0;
const BYTE* match;
matchIndex = dms->hashTable[ddsIdx + ddsAttempt];
match = ddsBase + matchIndex;
if (!matchIndex) {
return ml;
}
/* guaranteed by table construction */
(void)ddsLowestIndex;
assert(matchIndex >= ddsLowestIndex);
assert(match+4 <= ddsEnd);
if (MEM_read32(match) == MEM_read32(ip)) {
/* assumption : matchIndex <= dictLimit-4 (by table construction) */
currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4;
}
/* save best solution */
if (currentMl > ml) {
ml = currentMl;
*offsetPtr = curr - (matchIndex + ddsIndexDelta) + ZSTD_REP_MOVE;
if (ip+currentMl == iLimit) {
/* best possible, avoids read overflow on next attempt */
return ml;
}
}
}
{
U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1];
U32 chainIndex = chainPackedPointer >> 8;
U32 const chainLength = chainPackedPointer & 0xFF;
U32 const chainAttempts = nbAttempts - ddsAttempt;
U32 const chainLimit = chainAttempts > chainLength ? chainLength : chainAttempts;
U32 chainAttempt;
for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++) {
PREFETCH_L1(ddsBase + dms->chainTable[chainIndex + chainAttempt]);
}
for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++, chainIndex++) {
size_t currentMl=0;
const BYTE* match;
matchIndex = dms->chainTable[chainIndex];
match = ddsBase + matchIndex;
/* guaranteed by table construction */
assert(matchIndex >= ddsLowestIndex);
assert(match+4 <= ddsEnd);
if (MEM_read32(match) == MEM_read32(ip)) {
/* assumption : matchIndex <= dictLimit-4 (by table construction) */
currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4;
}
/* save best solution */
if (currentMl > ml) {
ml = currentMl;
*offsetPtr = curr - (matchIndex + ddsIndexDelta) + ZSTD_REP_MOVE;
if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
}
}
}
ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts, dms,
ip, iLimit, prefixStart, curr, dictLimit, ddsIdx);
} else if (dictMode == ZSTD_dictMatchState) {
const U32* const dmsChainTable = dms->chainTable;
const U32 dmsChainSize = (1 << dms->cParams.chainLog);
@@ -845,11 +861,657 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS (
}
}
/* *********************************
* (SIMD) Row-based matchfinder
***********************************/
/* Constants for row-based hash */
#define ZSTD_ROW_HASH_TAG_OFFSET 1 /* byte offset of hashes in the match state's tagTable from the beginning of a row */
#define ZSTD_ROW_HASH_TAG_BITS 8 /* nb bits to use for the tag */
#define ZSTD_ROW_HASH_TAG_MASK ((1u << ZSTD_ROW_HASH_TAG_BITS) - 1)
#define ZSTD_ROW_HASH_CACHE_MASK (ZSTD_ROW_HASH_CACHE_SIZE - 1)
typedef U32 ZSTD_VecMask; /* Clarifies when we are interacting with a U32 representing a mask of matches */
#if !defined(ZSTD_NO_INTRINSICS) && defined(__SSE2__) /* SIMD SSE version */
#include <emmintrin.h>
typedef __m128i ZSTD_Vec128;
/* Returns a 128-bit container with 128-bits from src */
static ZSTD_Vec128 ZSTD_Vec128_read(const void* const src) {
return _mm_loadu_si128((ZSTD_Vec128 const*)src);
}
/* Returns a ZSTD_Vec128 with the byte "val" packed 16 times */
static ZSTD_Vec128 ZSTD_Vec128_set8(BYTE val) {
return _mm_set1_epi8((char)val);
}
/* Do byte-by-byte comparison result of x and y. Then collapse 128-bit resultant mask
* into a 32-bit mask that is the MSB of each byte.
* */
static ZSTD_VecMask ZSTD_Vec128_cmpMask8(ZSTD_Vec128 x, ZSTD_Vec128 y) {
return (ZSTD_VecMask)_mm_movemask_epi8(_mm_cmpeq_epi8(x, y));
}
typedef struct {
__m128i fst;
__m128i snd;
} ZSTD_Vec256;
static ZSTD_Vec256 ZSTD_Vec256_read(const void* const ptr) {
ZSTD_Vec256 v;
v.fst = ZSTD_Vec128_read(ptr);
v.snd = ZSTD_Vec128_read((ZSTD_Vec128 const*)ptr + 1);
return v;
}
static ZSTD_Vec256 ZSTD_Vec256_set8(BYTE val) {
ZSTD_Vec256 v;
v.fst = ZSTD_Vec128_set8(val);
v.snd = ZSTD_Vec128_set8(val);
return v;
}
static ZSTD_VecMask ZSTD_Vec256_cmpMask8(ZSTD_Vec256 x, ZSTD_Vec256 y) {
ZSTD_VecMask fstMask;
ZSTD_VecMask sndMask;
fstMask = ZSTD_Vec128_cmpMask8(x.fst, y.fst);
sndMask = ZSTD_Vec128_cmpMask8(x.snd, y.snd);
return fstMask | (sndMask << 16);
}
#elif !defined(ZSTD_NO_INTRINSICS) && defined(__ARM_NEON) /* SIMD ARM NEON Version */
#include <arm_neon.h>
typedef uint8x16_t ZSTD_Vec128;
static ZSTD_Vec128 ZSTD_Vec128_read(const void* const src) {
return vld1q_u8((const BYTE* const)src);
}
static ZSTD_Vec128 ZSTD_Vec128_set8(BYTE val) {
return vdupq_n_u8(val);
}
/* Mimics '_mm_movemask_epi8()' from SSE */
static U32 ZSTD_vmovmaskq_u8(ZSTD_Vec128 val) {
/* Shift out everything but the MSB bits in each byte */
uint16x8_t highBits = vreinterpretq_u16_u8(vshrq_n_u8(val, 7));
/* Merge the even lanes together with vsra (right shift and add) */
uint32x4_t paired16 = vreinterpretq_u32_u16(vsraq_n_u16(highBits, highBits, 7));
uint64x2_t paired32 = vreinterpretq_u64_u32(vsraq_n_u32(paired16, paired16, 14));
uint8x16_t paired64 = vreinterpretq_u8_u64(vsraq_n_u64(paired32, paired32, 28));
/* Extract the low 8 bits from each lane, merge */
return vgetq_lane_u8(paired64, 0) | ((U32)vgetq_lane_u8(paired64, 8) << 8);
}
static ZSTD_VecMask ZSTD_Vec128_cmpMask8(ZSTD_Vec128 x, ZSTD_Vec128 y) {
return (ZSTD_VecMask)ZSTD_vmovmaskq_u8(vceqq_u8(x, y));
}
typedef struct {
uint8x16_t fst;
uint8x16_t snd;
} ZSTD_Vec256;
static ZSTD_Vec256 ZSTD_Vec256_read(const void* const ptr) {
ZSTD_Vec256 v;
v.fst = ZSTD_Vec128_read(ptr);
v.snd = ZSTD_Vec128_read((ZSTD_Vec128 const*)ptr + 1);
return v;
}
static ZSTD_Vec256 ZSTD_Vec256_set8(BYTE val) {
ZSTD_Vec256 v;
v.fst = ZSTD_Vec128_set8(val);
v.snd = ZSTD_Vec128_set8(val);
return v;
}
static ZSTD_VecMask ZSTD_Vec256_cmpMask8(ZSTD_Vec256 x, ZSTD_Vec256 y) {
ZSTD_VecMask fstMask;
ZSTD_VecMask sndMask;
fstMask = ZSTD_Vec128_cmpMask8(x.fst, y.fst);
sndMask = ZSTD_Vec128_cmpMask8(x.snd, y.snd);
return fstMask | (sndMask << 16);
}
#else /* Scalar fallback version */
#define VEC128_NB_SIZE_T (16 / sizeof(size_t))
typedef struct {
size_t vec[VEC128_NB_SIZE_T];
} ZSTD_Vec128;
static ZSTD_Vec128 ZSTD_Vec128_read(const void* const src) {
ZSTD_Vec128 ret;
ZSTD_memcpy(ret.vec, src, VEC128_NB_SIZE_T*sizeof(size_t));
return ret;
}
static ZSTD_Vec128 ZSTD_Vec128_set8(BYTE val) {
ZSTD_Vec128 ret = { {0} };
int startBit = sizeof(size_t) * 8 - 8;
for (;startBit >= 0; startBit -= 8) {
unsigned j = 0;
for (;j < VEC128_NB_SIZE_T; ++j) {
ret.vec[j] |= ((size_t)val << startBit);
}
}
return ret;
}
/* Compare x to y, byte by byte, generating a "matches" bitfield */
static ZSTD_VecMask ZSTD_Vec128_cmpMask8(ZSTD_Vec128 x, ZSTD_Vec128 y) {
ZSTD_VecMask res = 0;
unsigned i = 0;
unsigned l = 0;
for (; i < VEC128_NB_SIZE_T; ++i) {
const size_t cmp1 = x.vec[i];
const size_t cmp2 = y.vec[i];
unsigned j = 0;
for (; j < sizeof(size_t); ++j, ++l) {
if (((cmp1 >> j*8) & 0xFF) == ((cmp2 >> j*8) & 0xFF)) {
res |= ((U32)1 << (j+i*sizeof(size_t)));
}
}
}
return res;
}
#define VEC256_NB_SIZE_T 2*VEC128_NB_SIZE_T
typedef struct {
size_t vec[VEC256_NB_SIZE_T];
} ZSTD_Vec256;
static ZSTD_Vec256 ZSTD_Vec256_read(const void* const src) {
ZSTD_Vec256 ret;
ZSTD_memcpy(ret.vec, src, VEC256_NB_SIZE_T*sizeof(size_t));
return ret;
}
static ZSTD_Vec256 ZSTD_Vec256_set8(BYTE val) {
ZSTD_Vec256 ret = { {0} };
int startBit = sizeof(size_t) * 8 - 8;
for (;startBit >= 0; startBit -= 8) {
unsigned j = 0;
for (;j < VEC256_NB_SIZE_T; ++j) {
ret.vec[j] |= ((size_t)val << startBit);
}
}
return ret;
}
/* Compare x to y, byte by byte, generating a "matches" bitfield */
static ZSTD_VecMask ZSTD_Vec256_cmpMask8(ZSTD_Vec256 x, ZSTD_Vec256 y) {
ZSTD_VecMask res = 0;
unsigned i = 0;
unsigned l = 0;
for (; i < VEC256_NB_SIZE_T; ++i) {
const size_t cmp1 = x.vec[i];
const size_t cmp2 = y.vec[i];
unsigned j = 0;
for (; j < sizeof(size_t); ++j, ++l) {
if (((cmp1 >> j*8) & 0xFF) == ((cmp2 >> j*8) & 0xFF)) {
res |= ((U32)1 << (j+i*sizeof(size_t)));
}
}
}
return res;
}
#endif /* !defined(ZSTD_NO_INTRINSICS) && defined(__SSE2__) */
/* ZSTD_VecMask_next():
* Starting from the LSB, returns the idx of the next non-zero bit.
* Basically counting the nb of trailing zeroes.
*/
static U32 ZSTD_VecMask_next(ZSTD_VecMask val) {
# if defined(_MSC_VER) /* Visual */
unsigned long r=0;
return _BitScanForward(&r, val) ? (U32)r : 0;
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (U32)__builtin_ctz(val);
# else
/* Software ctz version: http://graphics.stanford.edu/~seander/bithacks.html#ZerosOnRightMultLookup */
static const U32 multiplyDeBruijnBitPosition[32] =
{
0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8,
31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9
};
return multiplyDeBruijnBitPosition[((U32)((v & -(int)v) * 0x077CB531U)) >> 27];
# endif
}
/* ZSTD_VecMask_rotateRight():
* Rotates a bitfield to the right by "rotation" bits.
* If the rotation is greater than totalBits, the returned mask is 0.
*/
FORCE_INLINE_TEMPLATE ZSTD_VecMask
ZSTD_VecMask_rotateRight(ZSTD_VecMask mask, U32 const rotation, U32 const totalBits) {
if (rotation == 0)
return mask;
switch (totalBits) {
default:
assert(0);
case 16:
return (mask >> rotation) | (U16)(mask << (16 - rotation));
case 32:
return (mask >> rotation) | (U32)(mask << (32 - rotation));
}
}
/* ZSTD_row_nextIndex():
* Returns the next index to insert at within a tagTable row, and updates the "head"
* value to reflect the update. Essentially cycles backwards from [0, {entries per row})
*/
FORCE_INLINE_TEMPLATE U32 ZSTD_row_nextIndex(BYTE* const tagRow, U32 const rowMask) {
U32 const next = (*tagRow - 1) & rowMask;
*tagRow = (BYTE)next;
return next;
}
/* ZSTD_isAligned():
* Checks that a pointer is aligned to "align" bytes which must be a power of 2.
*/
MEM_STATIC int ZSTD_isAligned(void const* ptr, size_t align) {
assert((align & (align - 1)) == 0);
return (((size_t)ptr) & (align - 1)) == 0;
}
/* ZSTD_row_prefetch():
* Performs prefetching for the hashTable and tagTable at a given row.
*/
FORCE_INLINE_TEMPLATE void ZSTD_row_prefetch(U32 const* hashTable, U16 const* tagTable, U32 const relRow, U32 const rowLog) {
PREFETCH_L1(hashTable + relRow);
if (rowLog == 5) {
PREFETCH_L1(hashTable + relRow + 16);
}
PREFETCH_L1(tagTable + relRow);
assert(rowLog == 4 || rowLog == 5);
assert(ZSTD_isAligned(hashTable + relRow, 64)); /* prefetched hash row always 64-byte aligned */
assert(ZSTD_isAligned(tagTable + relRow, (size_t)1 << rowLog)); /* prefetched tagRow sits on a multiple of 32 or 64 bytes */
}
/* ZSTD_row_fillHashCache():
* Fill up the hash cache starting at idx, prefetching up to ZSTD_ROW_HASH_CACHE_SIZE entries,
* but not beyond iLimit.
*/
static void ZSTD_row_fillHashCache(ZSTD_matchState_t* ms, const BYTE* base,
U32 const rowLog, U32 const mls,
U32 idx, const BYTE* const iLimit)
{
U32 const* const hashTable = ms->hashTable;
U16 const* const tagTable = ms->tagTable;
U32 const hashLog = ms->rowHashLog;
U32 const maxElemsToPrefetch = (base + idx) > iLimit ? 0 : (U32)(iLimit - (base + idx) + 1);
U32 const lim = idx + MIN(ZSTD_ROW_HASH_CACHE_SIZE, maxElemsToPrefetch);
for (; idx < lim; ++idx) {
U32 const hash = (U32)ZSTD_hashPtr(base + idx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls);
U32 const row = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
ZSTD_row_prefetch(hashTable, tagTable, row, rowLog);
ms->hashCache[idx & ZSTD_ROW_HASH_CACHE_MASK] = hash;
}
DEBUGLOG(6, "ZSTD_row_fillHashCache(): [%u %u %u %u %u %u %u %u]", ms->hashCache[0], ms->hashCache[1],
ms->hashCache[2], ms->hashCache[3], ms->hashCache[4],
ms->hashCache[5], ms->hashCache[6], ms->hashCache[7]);
}
/* ZSTD_row_nextCachedHash():
* Returns the hash of base + idx, and replaces the hash in the hash cache with the byte at
* base + idx + ZSTD_ROW_HASH_CACHE_SIZE. Also prefetches the appropriate rows from hashTable and tagTable.
*/
FORCE_INLINE_TEMPLATE U32 ZSTD_row_nextCachedHash(U32* cache, U32 const* hashTable,
U16 const* tagTable, BYTE const* base,
U32 idx, U32 const hashLog,
U32 const rowLog, U32 const mls)
{
U32 const newHash = (U32)ZSTD_hashPtr(base+idx+ZSTD_ROW_HASH_CACHE_SIZE, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls);
U32 const row = (newHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
ZSTD_row_prefetch(hashTable, tagTable, row, rowLog);
{ U32 const hash = cache[idx & ZSTD_ROW_HASH_CACHE_MASK];
cache[idx & ZSTD_ROW_HASH_CACHE_MASK] = newHash;
return hash;
}
}
/* ZSTD_row_update_internal():
* Inserts the byte at ip into the appropriate position in the hash table.
* Determines the relative row, and the position within the {16, 32} entry row to insert at.
*/
FORCE_INLINE_TEMPLATE void ZSTD_row_update_internal(ZSTD_matchState_t* ms, const BYTE* ip,
U32 const mls, U32 const rowLog,
U32 const rowMask, U32 const useCache)
{
U32* const hashTable = ms->hashTable;
U16* const tagTable = ms->tagTable;
U32 const hashLog = ms->rowHashLog;
const BYTE* const base = ms->window.base;
const U32 target = (U32)(ip - base);
U32 idx = ms->nextToUpdate;
DEBUGLOG(6, "ZSTD_row_update_internal(): nextToUpdate=%u, current=%u", idx, target);
for (; idx < target; ++idx) {
U32 const hash = useCache ? ZSTD_row_nextCachedHash(ms->hashCache, hashTable, tagTable, base, idx, hashLog, rowLog, mls)
: (U32)ZSTD_hashPtr(base + idx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls);
U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
U32* const row = hashTable + relRow;
BYTE* tagRow = (BYTE*)(tagTable + relRow); /* Though tagTable is laid out as a table of U16, each tag is only 1 byte.
Explicit cast allows us to get exact desired position within each row */
U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask);
assert(hash == ZSTD_hashPtr(base + idx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls));
((BYTE*)tagRow)[pos + ZSTD_ROW_HASH_TAG_OFFSET] = hash & ZSTD_ROW_HASH_TAG_MASK;
row[pos] = idx;
}
ms->nextToUpdate = target;
}
/* ZSTD_row_update():
* External wrapper for ZSTD_row_update_internal(). Used for filling the hashtable during dictionary
* processing.
*/
void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip) {
const U32 rowLog = ms->cParams.searchLog < 5 ? 4 : 5;
const U32 rowMask = (1u << rowLog) - 1;
const U32 mls = MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */);
DEBUGLOG(5, "ZSTD_row_update(), rowLog=%u", rowLog);
ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 0 /* dont use cache */);
}
/* Returns a ZSTD_VecMask (U32) that has the nth bit set to 1 if the newly-computed "tag" matches
* the hash at the nth position in a row of the tagTable.
*/
FORCE_INLINE_TEMPLATE
ZSTD_VecMask ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 head, const U32 rowEntries) {
ZSTD_VecMask matches = 0;
if (rowEntries == 16) {
ZSTD_Vec128 hashes = ZSTD_Vec128_read(tagRow + ZSTD_ROW_HASH_TAG_OFFSET);
ZSTD_Vec128 expandedTags = ZSTD_Vec128_set8(tag);
matches = ZSTD_Vec128_cmpMask8(hashes, expandedTags);
} else if (rowEntries == 32) {
ZSTD_Vec256 hashes = ZSTD_Vec256_read(tagRow + ZSTD_ROW_HASH_TAG_OFFSET);
ZSTD_Vec256 expandedTags = ZSTD_Vec256_set8(tag);
matches = ZSTD_Vec256_cmpMask8(hashes, expandedTags);
} else {
assert(0);
}
/* Each row is a circular buffer beginning at the value of "head". So we must rotate the "matches" bitfield
to match up with the actual layout of the entries within the hashTable */
return ZSTD_VecMask_rotateRight(matches, head, rowEntries);
}
/* The high-level approach of the SIMD row based match finder is as follows:
* - Figure out where to insert the new entry:
* - Generate a hash from a byte along with an additional 1-byte "short hash". The additional byte is our "tag"
* - The hashTable is effectively split into groups or "rows" of 16 or 32 entries of U32, and the hash determines
* which row to insert into.
* - Determine the correct position within the row to insert the entry into. Each row of 16 or 32 can
* be considered as a circular buffer with a "head" index that resides in the tagTable.
* - Also insert the "tag" into the equivalent row and position in the tagTable.
* - Note: The tagTable has 17 or 33 1-byte entries per row, due to 16 or 32 tags, and 1 "head" entry.
* The 17 or 33 entry rows are spaced out to occur every 32 or 64 bytes, respectively,
* for alignment/performance reasons, leaving some bytes unused.
* - Use SIMD to efficiently compare the tags in the tagTable to the 1-byte "short hash" and
* generate a bitfield that we can cycle through to check the collisions in the hash table.
* - Pick the longest match.
*/
FORCE_INLINE_TEMPLATE
size_t ZSTD_RowFindBestMatch_generic (
ZSTD_matchState_t* ms,
const BYTE* const ip, const BYTE* const iLimit,
size_t* offsetPtr,
const U32 mls, const ZSTD_dictMode_e dictMode,
const U32 rowLog)
{
U32* const hashTable = ms->hashTable;
U16* const tagTable = ms->tagTable;
U32* const hashCache = ms->hashCache;
const U32 hashLog = ms->rowHashLog;
const ZSTD_compressionParameters* const cParams = &ms->cParams;
const BYTE* const base = ms->window.base;
const BYTE* const dictBase = ms->window.dictBase;
const U32 dictLimit = ms->window.dictLimit;
const BYTE* const prefixStart = base + dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const U32 curr = (U32)(ip-base);
const U32 maxDistance = 1U << cParams->windowLog;
const U32 lowestValid = ms->window.lowLimit;
const U32 withinMaxDistance = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid;
const U32 isDictionary = (ms->loadedDictEnd != 0);
const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance;
const U32 rowEntries = (1U << rowLog);
const U32 rowMask = rowEntries - 1;
const U32 cappedSearchLog = MIN(cParams->searchLog, rowLog); /* nb of searches is capped at nb entries per row */
U32 nbAttempts = 1U << cappedSearchLog;
size_t ml=4-1;
/* DMS/DDS variables that may be referenced laster */
const ZSTD_matchState_t* const dms = ms->dictMatchState;
size_t ddsIdx;
U32 ddsExtraAttempts; /* cctx hash tables are limited in searches, but allow extra searches into DDS */
U32 dmsTag;
U32* dmsRow;
BYTE* dmsTagRow;
if (dictMode == ZSTD_dedicatedDictSearch) {
const U32 ddsHashLog = dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG;
{ /* Prefetch DDS hashtable entry */
ddsIdx = ZSTD_hashPtr(ip, ddsHashLog, mls) << ZSTD_LAZY_DDSS_BUCKET_LOG;
PREFETCH_L1(&dms->hashTable[ddsIdx]);
}
ddsExtraAttempts = cParams->searchLog > rowLog ? 1U << (cParams->searchLog - rowLog) : 0;
}
if (dictMode == ZSTD_dictMatchState) {
/* Prefetch DMS rows */
U32* const dmsHashTable = dms->hashTable;
U16* const dmsTagTable = dms->tagTable;
U32 const dmsHash = (U32)ZSTD_hashPtr(ip, dms->rowHashLog + ZSTD_ROW_HASH_TAG_BITS, mls);
U32 const dmsRelRow = (dmsHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
dmsTag = dmsHash & ZSTD_ROW_HASH_TAG_MASK;
dmsTagRow = (BYTE*)(dmsTagTable + dmsRelRow);
dmsRow = dmsHashTable + dmsRelRow;
ZSTD_row_prefetch(dmsHashTable, dmsTagTable, dmsRelRow, rowLog);
}
/* Update the hashTable and tagTable up to (but not including) ip */
ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 1 /* useCache */);
{ /* Get the hash for ip, compute the appropriate row */
U32 const hash = ZSTD_row_nextCachedHash(hashCache, hashTable, tagTable, base, curr, hashLog, rowLog, mls);
U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog;
U32 const tag = hash & ZSTD_ROW_HASH_TAG_MASK;
U32* const row = hashTable + relRow;
BYTE* tagRow = (BYTE*)(tagTable + relRow);
U32 const head = *tagRow & rowMask;
U32 matchBuffer[32 /* maximum nb entries per row */];
size_t numMatches = 0;
size_t currMatch = 0;
ZSTD_VecMask matches = ZSTD_row_getMatchMask(tagRow, (BYTE)tag, head, rowEntries);
/* Cycle through the matches and prefetch */
for (; (matches > 0) && (nbAttempts > 0); --nbAttempts, matches &= (matches - 1)) {
U32 const matchPos = (head + ZSTD_VecMask_next(matches)) & rowMask;
U32 const matchIndex = row[matchPos];
assert(numMatches < rowEntries);
if (matchIndex < lowLimit)
break;
if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) {
PREFETCH_L1(base + matchIndex);
} else {
PREFETCH_L1(dictBase + matchIndex);
}
matchBuffer[numMatches++] = matchIndex;
}
/* Speed opt: insert current byte into hashtable too. This allows us to avoid one iteration of the loop
in ZSTD_row_update_internal() at the next search. */
{
U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask);
tagRow[pos + ZSTD_ROW_HASH_TAG_OFFSET] = (BYTE)tag;
row[pos] = ms->nextToUpdate++;
}
/* Return the longest match */
for (; currMatch < numMatches; ++currMatch) {
U32 const matchIndex = matchBuffer[currMatch];
size_t currentMl=0;
assert(matchIndex < curr);
assert(matchIndex >= lowLimit);
if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) {
const BYTE* const match = base + matchIndex;
assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */
if (match[ml] == ip[ml]) /* potentially better */
currentMl = ZSTD_count(ip, match, iLimit);
} else {
const BYTE* const match = dictBase + matchIndex;
assert(match+4 <= dictEnd);
if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */
currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4;
}
/* Save best solution */
if (currentMl > ml) {
ml = currentMl;
*offsetPtr = curr - matchIndex + ZSTD_REP_MOVE;
if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
}
}
}
if (dictMode == ZSTD_dedicatedDictSearch) {
ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts + ddsExtraAttempts, dms,
ip, iLimit, prefixStart, curr, dictLimit, ddsIdx);
} else if (dictMode == ZSTD_dictMatchState) {
/* TODO: Measure and potentially add prefetching to DMS */
const U32 dmsLowestIndex = dms->window.dictLimit;
const BYTE* const dmsBase = dms->window.base;
const BYTE* const dmsEnd = dms->window.nextSrc;
const U32 dmsSize = (U32)(dmsEnd - dmsBase);
const U32 dmsIndexDelta = dictLimit - dmsSize;
{ U32 const head = *dmsTagRow & rowMask;
U32 matchBuffer[32 /* maximum nb row entries */];
size_t numMatches = 0;
size_t currMatch = 0;
ZSTD_VecMask matches = ZSTD_row_getMatchMask(dmsTagRow, (BYTE)dmsTag, head, rowEntries);
for (; (matches > 0) && (nbAttempts > 0); --nbAttempts, matches &= (matches - 1)) {
U32 const matchPos = (head + ZSTD_VecMask_next(matches)) & rowMask;
U32 const matchIndex = dmsRow[matchPos];
if (matchIndex < dmsLowestIndex)
break;
PREFETCH_L1(dmsBase + matchIndex);
matchBuffer[numMatches++] = matchIndex;
}
/* Return the longest match */
for (; currMatch < numMatches; ++currMatch) {
U32 const matchIndex = matchBuffer[currMatch];
size_t currentMl=0;
assert(matchIndex >= dmsLowestIndex);
assert(matchIndex < curr);
{ const BYTE* const match = dmsBase + matchIndex;
assert(match+4 <= dmsEnd);
if (MEM_read32(match) == MEM_read32(ip))
currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dmsEnd, prefixStart) + 4;
}
if (currentMl > ml) {
ml = currentMl;
*offsetPtr = curr - (matchIndex + dmsIndexDelta) + ZSTD_REP_MOVE;
if (ip+currentMl == iLimit) break;
}
}
}
}
return ml;
}
/* Inlining is important to hardwire a hot branch (template emulation) */
FORCE_INLINE_TEMPLATE size_t ZSTD_RowFindBestMatch_selectMLS (
ZSTD_matchState_t* ms,
const BYTE* ip, const BYTE* const iLimit,
const ZSTD_dictMode_e dictMode, size_t* offsetPtr, const U32 rowLog)
{
switch(ms->cParams.minMatch)
{
default : /* includes case 3 */
case 4 : return ZSTD_RowFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, dictMode, rowLog);
case 5 : return ZSTD_RowFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, dictMode, rowLog);
case 7 :
case 6 : return ZSTD_RowFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, dictMode, rowLog);
}
}
FORCE_INLINE_TEMPLATE size_t ZSTD_RowFindBestMatch_selectRowLog (
ZSTD_matchState_t* ms,
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
const U32 cappedSearchLog = MIN(ms->cParams.searchLog, 5);
switch(cappedSearchLog)
{
default :
case 4 : return ZSTD_RowFindBestMatch_selectMLS(ms, ip, iLimit, ZSTD_noDict, offsetPtr, 4);
case 5 : return ZSTD_RowFindBestMatch_selectMLS(ms, ip, iLimit, ZSTD_noDict, offsetPtr, 5);
}
}
FORCE_INLINE_TEMPLATE size_t ZSTD_RowFindBestMatch_dictMatchState_selectRowLog(
ZSTD_matchState_t* ms,
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
const U32 cappedSearchLog = MIN(ms->cParams.searchLog, 5);
switch(cappedSearchLog)
{
default :
case 4 : return ZSTD_RowFindBestMatch_selectMLS(ms, ip, iLimit, ZSTD_dictMatchState, offsetPtr, 4);
case 5 : return ZSTD_RowFindBestMatch_selectMLS(ms, ip, iLimit, ZSTD_dictMatchState, offsetPtr, 5);
}
}
FORCE_INLINE_TEMPLATE size_t ZSTD_RowFindBestMatch_dedicatedDictSearch_selectRowLog(
ZSTD_matchState_t* ms,
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
const U32 cappedSearchLog = MIN(ms->cParams.searchLog, 5);
switch(cappedSearchLog)
{
default :
case 4 : return ZSTD_RowFindBestMatch_selectMLS(ms, ip, iLimit, ZSTD_dedicatedDictSearch, offsetPtr, 4);
case 5 : return ZSTD_RowFindBestMatch_selectMLS(ms, ip, iLimit, ZSTD_dedicatedDictSearch, offsetPtr, 5);
}
}
FORCE_INLINE_TEMPLATE size_t ZSTD_RowFindBestMatch_extDict_selectRowLog (
ZSTD_matchState_t* ms,
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
const U32 cappedSearchLog = MIN(ms->cParams.searchLog, 5);
switch(cappedSearchLog)
{
default :
case 4 : return ZSTD_RowFindBestMatch_selectMLS(ms, ip, iLimit, ZSTD_extDict, offsetPtr, 4);
case 5 : return ZSTD_RowFindBestMatch_selectMLS(ms, ip, iLimit, ZSTD_extDict, offsetPtr, 5);
}
}
/* *******************************
* Common parser - lazy strategy
*********************************/
typedef enum { search_hashChain, search_binaryTree } searchMethod_e;
typedef enum { search_hashChain=0, search_binaryTree=1, search_rowHash=2 } searchMethod_e;
FORCE_INLINE_TEMPLATE size_t
ZSTD_compressBlock_lazy_generic(
@@ -863,10 +1525,11 @@ ZSTD_compressBlock_lazy_generic(
const BYTE* ip = istart;
const BYTE* anchor = istart;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8;
const BYTE* const ilimit = searchMethod == search_rowHash ? iend - 8 - ZSTD_ROW_HASH_CACHE_SIZE : iend - 8;
const BYTE* const base = ms->window.base;
const U32 prefixLowestIndex = ms->window.dictLimit;
const BYTE* const prefixLowest = base + prefixLowestIndex;
const U32 rowLog = ms->cParams.searchLog < 5 ? 4 : 5;
typedef size_t (*searchMax_f)(
ZSTD_matchState_t* ms,
@@ -878,26 +1541,30 @@ ZSTD_compressBlock_lazy_generic(
* that should never occur (extDict modes go to the other implementation
* below and there is no DDSS for binary tree search yet).
*/
const searchMax_f searchFuncs[4][2] = {
const searchMax_f searchFuncs[4][3] = {
{
ZSTD_HcFindBestMatch_selectMLS,
ZSTD_BtFindBestMatch_selectMLS
ZSTD_BtFindBestMatch_selectMLS,
ZSTD_RowFindBestMatch_selectRowLog
},
{
NULL,
NULL,
NULL
},
{
ZSTD_HcFindBestMatch_dictMatchState_selectMLS,
ZSTD_BtFindBestMatch_dictMatchState_selectMLS
ZSTD_BtFindBestMatch_dictMatchState_selectMLS,
ZSTD_RowFindBestMatch_dictMatchState_selectRowLog
},
{
ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS,
NULL
NULL,
ZSTD_RowFindBestMatch_dedicatedDictSearch_selectRowLog
}
};
searchMax_f const searchMax = searchFuncs[dictMode][searchMethod == search_binaryTree];
searchMax_f const searchMax = searchFuncs[dictMode][(int)searchMethod];
U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0;
const int isDMS = dictMode == ZSTD_dictMatchState;
@@ -915,9 +1582,7 @@ ZSTD_compressBlock_lazy_generic(
assert(searchMax != NULL);
DEBUGLOG(5, "ZSTD_compressBlock_lazy_generic (dictMode=%u)", (U32)dictMode);
/* init */
DEBUGLOG(5, "ZSTD_compressBlock_lazy_generic (dictMode=%u) (searchFunc=%u)", (U32)dictMode, (U32)searchMethod);
ip += (dictAndPrefixLength == 0);
if (dictMode == ZSTD_noDict) {
U32 const curr = (U32)(ip - base);
@@ -933,6 +1598,12 @@ ZSTD_compressBlock_lazy_generic(
assert(offset_2 <= dictAndPrefixLength);
}
if (searchMethod == search_rowHash) {
ZSTD_row_fillHashCache(ms, base, rowLog,
MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */),
ms->nextToUpdate, ilimit);
}
/* Match Loop */
#if defined(__GNUC__) && defined(__x86_64__)
/* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the
@@ -1198,6 +1869,70 @@ size_t ZSTD_compressBlock_greedy_dedicatedDictSearch(
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dedicatedDictSearch);
}
/* Row-based matchfinder */
size_t ZSTD_compressBlock_lazy2_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_noDict);
}
size_t ZSTD_compressBlock_lazy_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_noDict);
}
size_t ZSTD_compressBlock_greedy_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_noDict);
}
size_t ZSTD_compressBlock_lazy2_dictMatchState_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dictMatchState);
}
size_t ZSTD_compressBlock_lazy_dictMatchState_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dictMatchState);
}
size_t ZSTD_compressBlock_greedy_dictMatchState_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dictMatchState);
}
size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dedicatedDictSearch);
}
size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dedicatedDictSearch);
}
size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dedicatedDictSearch);
}
FORCE_INLINE_TEMPLATE
size_t ZSTD_compressBlock_lazy_extDict_generic(
@@ -1210,7 +1945,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
const BYTE* ip = istart;
const BYTE* anchor = istart;
const BYTE* const iend = istart + srcSize;
const BYTE* const ilimit = iend - 8;
const BYTE* const ilimit = searchMethod == search_rowHash ? iend - 8 - ZSTD_ROW_HASH_CACHE_SIZE : iend - 8;
const BYTE* const base = ms->window.base;
const U32 dictLimit = ms->window.dictLimit;
const BYTE* const prefixStart = base + dictLimit;
@@ -1218,18 +1953,28 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const dictStart = dictBase + ms->window.lowLimit;
const U32 windowLog = ms->cParams.windowLog;
const U32 rowLog = ms->cParams.searchLog < 5 ? 4 : 5;
typedef size_t (*searchMax_f)(
ZSTD_matchState_t* ms,
const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr);
searchMax_f searchMax = searchMethod==search_binaryTree ? ZSTD_BtFindBestMatch_extDict_selectMLS : ZSTD_HcFindBestMatch_extDict_selectMLS;
const searchMax_f searchFuncs[3] = {
ZSTD_HcFindBestMatch_extDict_selectMLS,
ZSTD_BtFindBestMatch_extDict_selectMLS,
ZSTD_RowFindBestMatch_extDict_selectRowLog
};
searchMax_f searchMax = searchFuncs[(int)searchMethod];
U32 offset_1 = rep[0], offset_2 = rep[1];
DEBUGLOG(5, "ZSTD_compressBlock_lazy_extDict_generic");
DEBUGLOG(5, "ZSTD_compressBlock_lazy_extDict_generic (searchFunc=%u)", (U32)searchMethod);
/* init */
ip += (ip == prefixStart);
if (searchMethod == search_rowHash) {
ZSTD_row_fillHashCache(ms, base, rowLog,
MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */),
ms->nextToUpdate, ilimit);
}
/* Match Loop */
#if defined(__GNUC__) && defined(__x86_64__)
@@ -1249,7 +1994,8 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
const U32 repIndex = (U32)(curr+1 - offset_1);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > windowLow)) /* intentional overflow */
if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */
& (offset_1 < curr+1 - windowLow) ) /* note: we are searching at curr+1 */
if (MEM_read32(ip+1) == MEM_read32(repMatch)) {
/* repcode detected we should take it */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
@@ -1280,7 +2026,8 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
const U32 repIndex = (U32)(curr - offset_1);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > windowLow)) /* intentional overflow */
if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */
& (offset_1 < curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */
if (MEM_read32(ip) == MEM_read32(repMatch)) {
/* repcode detected */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
@@ -1311,7 +2058,8 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
const U32 repIndex = (U32)(curr - offset_1);
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > windowLow)) /* intentional overflow */
if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */
& (offset_1 < curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */
if (MEM_read32(ip) == MEM_read32(repMatch)) {
/* repcode detected */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
@@ -1357,7 +2105,8 @@ _storeSequence:
const U32 repIndex = repCurrent - offset_2;
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
const BYTE* const repMatch = repBase + repIndex;
if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > windowLow)) /* intentional overflow */
if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */
& (offset_2 < repCurrent - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */
if (MEM_read32(ip) == MEM_read32(repMatch)) {
/* repcode detected we should take it */
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
@@ -1410,3 +2159,26 @@ size_t ZSTD_compressBlock_btlazy2_extDict(
{
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2);
}
size_t ZSTD_compressBlock_greedy_extDict_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0);
}
size_t ZSTD_compressBlock_lazy_extDict_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1);
}
size_t ZSTD_compressBlock_lazy2_extDict_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2);
}
lib/compress/zstd_lazy.h
+39 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -26,6 +26,7 @@ extern "C" {
#define ZSTD_LAZY_DDSS_BUCKET_LOG 2
U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip);
void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip);
void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip);
@@ -43,6 +44,15 @@ size_t ZSTD_compressBlock_lazy(
size_t ZSTD_compressBlock_greedy(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy2_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_greedy_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btlazy2_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
@@ -56,6 +66,15 @@ size_t ZSTD_compressBlock_lazy_dictMatchState(
size_t ZSTD_compressBlock_greedy_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy2_dictMatchState_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy_dictMatchState_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_greedy_dictMatchState_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
@@ -66,6 +85,15 @@ size_t ZSTD_compressBlock_lazy_dedicatedDictSearch(
size_t ZSTD_compressBlock_greedy_dedicatedDictSearch(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_greedy_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
@@ -76,9 +104,19 @@ size_t ZSTD_compressBlock_lazy_extDict(
size_t ZSTD_compressBlock_lazy2_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_greedy_extDict_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy_extDict_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_lazy2_extDict_row(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btlazy2_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
#if defined (__cplusplus)
}
lib/compress/zstd_ldm.c
+51 -15
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -57,6 +57,33 @@ static void ZSTD_ldm_gear_init(ldmRollingHashState_t* state, ldmParams_t const*
}
}
/** ZSTD_ldm_gear_reset()
* Feeds [data, data + minMatchLength) into the hash without registering any
* splits. This effectively resets the hash state. This is used when skipping
* over data, either at the beginning of a block, or skipping sections.
*/
static void ZSTD_ldm_gear_reset(ldmRollingHashState_t* state,
BYTE const* data, size_t minMatchLength)
{
U64 hash = state->rolling;
size_t n = 0;
#define GEAR_ITER_ONCE() do { \
hash = (hash << 1) + ZSTD_ldm_gearTab[data[n] & 0xff]; \
n += 1; \
} while (0)
while (n + 3 < minMatchLength) {
GEAR_ITER_ONCE();
GEAR_ITER_ONCE();
GEAR_ITER_ONCE();
GEAR_ITER_ONCE();
}
while (n < minMatchLength) {
GEAR_ITER_ONCE();
}
#undef GEAR_ITER_ONCE
}
/** ZSTD_ldm_gear_feed():
*
* Registers in the splits array all the split points found in the first
@@ -255,7 +282,7 @@ void ZSTD_ldm_fillHashTable(
while (ip < iend) {
size_t hashed;
unsigned n;
numSplits = 0;
hashed = ZSTD_ldm_gear_feed(&hashState, ip, iend - ip, splits, &numSplits);
@@ -327,16 +354,8 @@ static size_t ZSTD_ldm_generateSequences_internal(
/* Initialize the rolling hash state with the first minMatchLength bytes */
ZSTD_ldm_gear_init(&hashState, params);
{
size_t n = 0;
while (n < minMatchLength) {
numSplits = 0;
n += ZSTD_ldm_gear_feed(&hashState, ip + n, minMatchLength - n,
splits, &numSplits);
}
ip += minMatchLength;
}
ZSTD_ldm_gear_reset(&hashState, ip, minMatchLength);
ip += minMatchLength;
while (ip < ilimit) {
size_t hashed;
@@ -361,6 +380,7 @@ static size_t ZSTD_ldm_generateSequences_internal(
for (n = 0; n < numSplits; n++) {
size_t forwardMatchLength = 0, backwardMatchLength = 0,
bestMatchLength = 0, mLength;
U32 offset;
BYTE const* const split = candidates[n].split;
U32 const checksum = candidates[n].checksum;
U32 const hash = candidates[n].hash;
@@ -428,9 +448,9 @@ static size_t ZSTD_ldm_generateSequences_internal(
}
/* Match found */
offset = (U32)(split - base) - bestEntry->offset;
mLength = forwardMatchLength + backwardMatchLength;
{
U32 const offset = (U32)(split - base) - bestEntry->offset;
rawSeq* const seq = rawSeqStore->seq + rawSeqStore->size;
/* Out of sequence storage */
@@ -447,6 +467,21 @@ static size_t ZSTD_ldm_generateSequences_internal(
ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params);
anchor = split + forwardMatchLength;
/* If we find a match that ends after the data that we've hashed
* then we have a repeating, overlapping, pattern. E.g. all zeros.
* If one repetition of the pattern matches our `stopMask` then all
* repetitions will. We don't need to insert them all into out table,
* only the first one. So skip over overlapping matches.
* This is a major speed boost (20x) for compressing a single byte
* repeated, when that byte ends up in the table.
*/
if (anchor > ip + hashed) {
ZSTD_ldm_gear_reset(&hashState, anchor - minMatchLength, minMatchLength);
/* Continue the outter loop at anchor (ip + hashed == anchor). */
ip = anchor - hashed;
break;
}
}
ip += hashed;
@@ -500,7 +535,7 @@ size_t ZSTD_ldm_generateSequences(
assert(chunkStart < iend);
/* 1. Perform overflow correction if necessary. */
if (ZSTD_window_needOverflowCorrection(ldmState->window, chunkEnd)) {
if (ZSTD_window_needOverflowCorrection(ldmState->window, 0, maxDist, ldmState->loadedDictEnd, chunkStart, chunkEnd)) {
U32 const ldmHSize = 1U << params->hashLog;
U32 const correction = ZSTD_window_correctOverflow(
&ldmState->window, /* cycleLog */ 0, maxDist, chunkStart);
@@ -622,12 +657,13 @@ void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) {
size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_useRowMatchFinderMode_e useRowMatchFinder,
void const* src, size_t srcSize)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
unsigned const minMatch = cParams->minMatch;
ZSTD_blockCompressor const blockCompressor =
ZSTD_selectBlockCompressor(cParams->strategy, ZSTD_matchState_dictMode(ms));
ZSTD_selectBlockCompressor(cParams->strategy, useRowMatchFinder, ZSTD_matchState_dictMode(ms));
/* Input bounds */
BYTE const* const istart = (BYTE const*)src;
BYTE const* const iend = istart + srcSize;
lib/compress/zstd_ldm.h
+2 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -66,6 +66,7 @@ size_t ZSTD_ldm_generateSequences(
*/
size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
ZSTD_useRowMatchFinderMode_e useRowMatchFinder,
void const* src, size_t srcSize);
/**
lib/compress/zstd_ldm_geartab.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/compress/zstd_opt.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/compress/zstd_opt.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/compress/zstdmt_compress.c
+3 -4
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -1240,9 +1240,8 @@ size_t ZSTDMT_initCStream_internal(
if (params.rsyncable) {
/* Aim for the targetsectionSize as the average job size. */
U32 const jobSizeMB = (U32)(mtctx->targetSectionSize >> 20);
U32 const rsyncBits = ZSTD_highbit32(jobSizeMB) + 20;
assert(jobSizeMB >= 1);
U32 const jobSizeKB = (U32)(mtctx->targetSectionSize >> 10);
U32 const rsyncBits = (assert(jobSizeKB >= 1), ZSTD_highbit32(jobSizeKB) + 10);
DEBUGLOG(4, "rsyncLog = %u", rsyncBits);
mtctx->rsync.hash = 0;
mtctx->rsync.hitMask = (1ULL << rsyncBits) - 1;
lib/compress/zstdmt_compress.h
+4 -4
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -32,11 +32,11 @@
/* === Constants === */
#ifndef ZSTDMT_NBWORKERS_MAX
#ifndef ZSTDMT_NBWORKERS_MAX /* can be modified at compile time */
# define ZSTDMT_NBWORKERS_MAX 200
#endif
#ifndef ZSTDMT_JOBSIZE_MIN
# define ZSTDMT_JOBSIZE_MIN (1 MB)
#ifndef ZSTDMT_JOBSIZE_MIN /* can be modified at compile time */
# define ZSTDMT_JOBSIZE_MIN (512 KB)
#endif
#define ZSTDMT_JOBLOG_MAX (MEM_32bits() ? 29 : 30)
#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1024 MB))
lib/decompress/huf_decompress.c
+42 -41
View File
@@ -1,7 +1,7 @@
/* ******************************************************************
* huff0 huffman decoder,
* part of Finite State Entropy library
* Copyright (c) 2013-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
*
* You can contact the author at :
* - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
@@ -528,13 +528,15 @@ typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX];
static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 sizeLog, const U32 consumed,
const U32* rankValOrigin, const int minWeight,
const sortedSymbol_t* sortedSymbols, const U32 sortedListSize,
U32 nbBitsBaseline, U16 baseSeq)
U32 nbBitsBaseline, U16 baseSeq, U32* wksp, size_t wkspSize)
{
HUF_DEltX2 DElt;
U32 rankVal[HUF_TABLELOG_MAX + 1];
U32* rankVal = wksp;
assert(wkspSize >= HUF_TABLELOG_MAX + 1);
(void)wkspSize;
/* get pre-calculated rankVal */
ZSTD_memcpy(rankVal, rankValOrigin, sizeof(rankVal));
ZSTD_memcpy(rankVal, rankValOrigin, sizeof(U32) * (HUF_TABLELOG_MAX + 1));
/* fill skipped values */
if (minWeight>1) {
@@ -569,14 +571,18 @@ static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 sizeLog, const U32 co
static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog,
const sortedSymbol_t* sortedList, const U32 sortedListSize,
const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight,
const U32 nbBitsBaseline)
const U32 nbBitsBaseline, U32* wksp, size_t wkspSize)
{
U32 rankVal[HUF_TABLELOG_MAX + 1];
U32* rankVal = wksp;
const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */
const U32 minBits = nbBitsBaseline - maxWeight;
U32 s;
ZSTD_memcpy(rankVal, rankValOrigin, sizeof(rankVal));
assert(wkspSize >= HUF_TABLELOG_MAX + 1);
wksp += HUF_TABLELOG_MAX + 1;
wkspSize -= HUF_TABLELOG_MAX + 1;
ZSTD_memcpy(rankVal, rankValOrigin, sizeof(U32) * (HUF_TABLELOG_MAX + 1));
/* fill DTable */
for (s=0; s<sortedListSize; s++) {
@@ -594,7 +600,7 @@ static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog,
HUF_fillDTableX2Level2(DTable+start, targetLog-nbBits, nbBits,
rankValOrigin[nbBits], minWeight,
sortedList+sortedRank, sortedListSize-sortedRank,
nbBitsBaseline, symbol);
nbBitsBaseline, symbol, wksp, wkspSize);
} else {
HUF_DEltX2 DElt;
MEM_writeLE16(&(DElt.sequence), symbol);
@@ -608,6 +614,15 @@ static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog,
}
}
typedef struct {
rankValCol_t rankVal[HUF_TABLELOG_MAX];
U32 rankStats[HUF_TABLELOG_MAX + 1];
U32 rankStart0[HUF_TABLELOG_MAX + 2];
sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1];
BYTE weightList[HUF_SYMBOLVALUE_MAX + 1];
U32 calleeWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
} HUF_ReadDTableX2_Workspace;
size_t HUF_readDTableX2_wksp(HUF_DTable* DTable,
const void* src, size_t srcSize,
void* workSpace, size_t wkspSize)
@@ -620,47 +635,32 @@ size_t HUF_readDTableX2_wksp(HUF_DTable* DTable,
HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr;
U32 *rankStart;
rankValCol_t* rankVal;
U32* rankStats;
U32* rankStart0;
sortedSymbol_t* sortedSymbol;
BYTE* weightList;
size_t spaceUsed32 = 0;
HUF_ReadDTableX2_Workspace* const wksp = (HUF_ReadDTableX2_Workspace*)workSpace;
rankVal = (rankValCol_t *)((U32 *)workSpace + spaceUsed32);
spaceUsed32 += (sizeof(rankValCol_t) * HUF_TABLELOG_MAX) >> 2;
rankStats = (U32 *)workSpace + spaceUsed32;
spaceUsed32 += HUF_TABLELOG_MAX + 1;
rankStart0 = (U32 *)workSpace + spaceUsed32;
spaceUsed32 += HUF_TABLELOG_MAX + 2;
sortedSymbol = (sortedSymbol_t *)workSpace + (spaceUsed32 * sizeof(U32)) / sizeof(sortedSymbol_t);
spaceUsed32 += HUF_ALIGN(sizeof(sortedSymbol_t) * (HUF_SYMBOLVALUE_MAX + 1), sizeof(U32)) >> 2;
weightList = (BYTE *)((U32 *)workSpace + spaceUsed32);
spaceUsed32 += HUF_ALIGN(HUF_SYMBOLVALUE_MAX + 1, sizeof(U32)) >> 2;
if (sizeof(*wksp) > wkspSize) return ERROR(GENERIC);
if ((spaceUsed32 << 2) > wkspSize) return ERROR(tableLog_tooLarge);
rankStart = rankStart0 + 1;
ZSTD_memset(rankStats, 0, sizeof(U32) * (2 * HUF_TABLELOG_MAX + 2 + 1));
rankStart = wksp->rankStart0 + 1;
ZSTD_memset(wksp->rankStats, 0, sizeof(wksp->rankStats));
ZSTD_memset(wksp->rankStart0, 0, sizeof(wksp->rankStart0));
DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */
if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
/* ZSTD_memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), /* bmi2 */ 0);
if (HUF_isError(iSize)) return iSize;
/* check result */
if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */
/* find maxWeight */
for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */
for (maxW = tableLog; wksp->rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */
/* Get start index of each weight */
{ U32 w, nextRankStart = 0;
for (w=1; w<maxW+1; w++) {
U32 curr = nextRankStart;
nextRankStart += rankStats[w];
nextRankStart += wksp->rankStats[w];
rankStart[w] = curr;
}
rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/
@@ -670,37 +670,38 @@ size_t HUF_readDTableX2_wksp(HUF_DTable* DTable,
/* sort symbols by weight */
{ U32 s;
for (s=0; s<nbSymbols; s++) {
U32 const w = weightList[s];
U32 const w = wksp->weightList[s];
U32 const r = rankStart[w]++;
sortedSymbol[r].symbol = (BYTE)s;
sortedSymbol[r].weight = (BYTE)w;
wksp->sortedSymbol[r].symbol = (BYTE)s;
wksp->sortedSymbol[r].weight = (BYTE)w;
}
rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
}
/* Build rankVal */
{ U32* const rankVal0 = rankVal[0];
{ U32* const rankVal0 = wksp->rankVal[0];
{ int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */
U32 nextRankVal = 0;
U32 w;
for (w=1; w<maxW+1; w++) {
U32 curr = nextRankVal;
nextRankVal += rankStats[w] << (w+rescale);
nextRankVal += wksp->rankStats[w] << (w+rescale);
rankVal0[w] = curr;
} }
{ U32 const minBits = tableLog+1 - maxW;
U32 consumed;
for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) {
U32* const rankValPtr = rankVal[consumed];
U32* const rankValPtr = wksp->rankVal[consumed];
U32 w;
for (w = 1; w < maxW+1; w++) {
rankValPtr[w] = rankVal0[w] >> consumed;
} } } }
HUF_fillDTableX2(dt, maxTableLog,
sortedSymbol, sizeOfSort,
rankStart0, rankVal, maxW,
tableLog+1);
wksp->sortedSymbol, sizeOfSort,
wksp->rankStart0, wksp->rankVal, maxW,
tableLog+1,
wksp->calleeWksp, sizeof(wksp->calleeWksp) / sizeof(U32));
dtd.tableLog = (BYTE)maxTableLog;
dtd.tableType = 1;
@@ -1225,7 +1226,7 @@ size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cS
HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX);
return HUF_decompress1X1_DCtx (DTable, dst, dstSize, cSrc, cSrcSize);
}
#endif
#endif
#ifndef HUF_FORCE_DECOMPRESS_X1
size_t HUF_readDTableX2(HUF_DTable* DTable, const void* src, size_t srcSize)
lib/decompress/zstd_ddict.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/decompress/zstd_ddict.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/decompress/zstd_decompress.c
+3 -4
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -58,7 +58,6 @@
#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */
#include "../common/cpu.h" /* bmi2 */
#include "../common/mem.h" /* low level memory routines */
#include "../common/zstd_trace.h"
#define FSE_STATIC_LINKING_ONLY
#include "../common/fse.h"
#define HUF_STATIC_LINKING_ONLY
@@ -789,7 +788,7 @@ static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity,
static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming)
{
#if ZSTD_TRACE
if (dctx->traceCtx) {
if (dctx->traceCtx && ZSTD_trace_decompress_end != NULL) {
ZSTD_Trace trace;
ZSTD_memset(&trace, 0, sizeof(trace));
trace.version = ZSTD_VERSION_NUMBER;
@@ -1384,7 +1383,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
{
assert(dctx != NULL);
#if ZSTD_TRACE
dctx->traceCtx = ZSTD_trace_decompress_begin(dctx);
dctx->traceCtx = (ZSTD_trace_decompress_begin != NULL) ? ZSTD_trace_decompress_begin(dctx) : 0;
#endif
dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */
dctx->stage = ZSTDds_getFrameHeaderSize;
lib/decompress/zstd_decompress_block.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/decompress/zstd_decompress_block.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/decompress/zstd_decompress_internal.h
+1 -2
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -21,7 +21,6 @@
*********************************************************/
#include "../common/mem.h" /* BYTE, U16, U32 */
#include "../common/zstd_internal.h" /* ZSTD_seqSymbol */
#include "../common/zstd_trace.h" /* ZSTD_TraceCtx */
lib/deprecated/zbuff.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/deprecated/zbuff_common.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/deprecated/zbuff_compress.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/deprecated/zbuff_decompress.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/dictBuilder/cover.c
+7 -6
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -26,15 +26,16 @@
#include <string.h> /* memset */
#include <time.h> /* clock */
#ifndef ZDICT_STATIC_LINKING_ONLY
# define ZDICT_STATIC_LINKING_ONLY
#endif
#include "../common/mem.h" /* read */
#include "../common/pool.h"
#include "../common/threading.h"
#include "cover.h"
#include "../common/zstd_internal.h" /* includes zstd.h */
#ifndef ZDICT_STATIC_LINKING_ONLY
#define ZDICT_STATIC_LINKING_ONLY
#endif
#include "zdict.h"
#include "../zdict.h"
#include "cover.h"
/*-*************************************
* Constants
lib/dictBuilder/cover.h
+6 -5
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2017-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -8,6 +8,10 @@
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef ZDICT_STATIC_LINKING_ONLY
# define ZDICT_STATIC_LINKING_ONLY
#endif
#include <stdio.h> /* fprintf */
#include <stdlib.h> /* malloc, free, qsort */
#include <string.h> /* memset */
@@ -16,10 +20,7 @@
#include "../common/pool.h"
#include "../common/threading.h"
#include "../common/zstd_internal.h" /* includes zstd.h */
#ifndef ZDICT_STATIC_LINKING_ONLY
#define ZDICT_STATIC_LINKING_ONLY
#endif
#include "zdict.h"
#include "../zdict.h"
/**
* COVER_best_t is used for two purposes:
lib/dictBuilder/fastcover.c
+7 -6
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2018-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -16,16 +16,17 @@
#include <string.h> /* memset */
#include <time.h> /* clock */
#ifndef ZDICT_STATIC_LINKING_ONLY
# define ZDICT_STATIC_LINKING_ONLY
#endif
#include "../common/mem.h" /* read */
#include "../common/pool.h"
#include "../common/threading.h"
#include "cover.h"
#include "../common/zstd_internal.h" /* includes zstd.h */
#include "../compress/zstd_compress_internal.h" /* ZSTD_hash*() */
#ifndef ZDICT_STATIC_LINKING_ONLY
#define ZDICT_STATIC_LINKING_ONLY
#endif
#include "zdict.h"
#include "../zdict.h"
#include "cover.h"
/*-*************************************
lib/dictBuilder/zdict.c
+10 -9
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -41,18 +41,19 @@
#include <stdio.h> /* fprintf, fopen, ftello64 */
#include <time.h> /* clock */
#include "../common/mem.h" /* read */
#include "../common/fse.h" /* FSE_normalizeCount, FSE_writeNCount */
#define HUF_STATIC_LINKING_ONLY
#include "../common/huf.h" /* HUF_buildCTable, HUF_writeCTable */
#include "../common/zstd_internal.h" /* includes zstd.h */
#include "../common/xxhash.h" /* XXH64 */
#include "divsufsort.h"
#ifndef ZDICT_STATIC_LINKING_ONLY
# define ZDICT_STATIC_LINKING_ONLY
#endif
#include "zdict.h"
#define HUF_STATIC_LINKING_ONLY
#include "../common/mem.h" /* read */
#include "../common/fse.h" /* FSE_normalizeCount, FSE_writeNCount */
#include "../common/huf.h" /* HUF_buildCTable, HUF_writeCTable */
#include "../common/zstd_internal.h" /* includes zstd.h */
#include "../common/xxhash.h" /* XXH64 */
#include "../compress/zstd_compress_internal.h" /* ZSTD_loadCEntropy() */
#include "../zdict.h"
#include "divsufsort.h"
/*-*************************************
lib/dll/example/Makefile
+1 -1
View File
@@ -1,5 +1,5 @@
# ################################################################
# Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
# Copyright (c) Yann Collet, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
lib/legacy/zstd_legacy.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/legacy/zstd_v01.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/legacy/zstd_v01.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/legacy/zstd_v02.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/legacy/zstd_v02.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/legacy/zstd_v03.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/legacy/zstd_v03.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/legacy/zstd_v04.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/legacy/zstd_v04.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/legacy/zstd_v05.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/legacy/zstd_v05.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/legacy/zstd_v06.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/legacy/zstd_v06.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/legacy/zstd_v07.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/legacy/zstd_v07.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/dictBuilder/zdict.h → lib/zdict.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
lib/zstd.h
+77 -52
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -72,7 +72,7 @@ extern "C" {
/*------ Version ------*/
#define ZSTD_VERSION_MAJOR 1
#define ZSTD_VERSION_MINOR 4
#define ZSTD_VERSION_RELEASE 9
#define ZSTD_VERSION_RELEASE 10
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
/*! ZSTD_versionNumber() :
@@ -109,7 +109,6 @@ ZSTDLIB_API const char* ZSTD_versionString(void);
#define ZSTD_BLOCKSIZE_MAX (1<<ZSTD_BLOCKSIZELOG_MAX)
/***************************************
* Simple API
***************************************/
@@ -166,7 +165,7 @@ ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t
* @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */
ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
/*! ZSTD_findFrameCompressedSize() :
/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+
* `src` should point to the start of a ZSTD frame or skippable frame.
* `srcSize` must be >= first frame size
* @return : the compressed size of the first frame starting at `src`,
@@ -180,8 +179,9 @@ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */
ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */
ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed */
ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */
ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */
ZSTDLIB_API int ZSTD_defaultCLevel(void); /*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */
/***************************************
@@ -234,9 +234,9 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx,
const void* src, size_t srcSize);
/***************************************
* Advanced compression API
***************************************/
/*********************************************
* Advanced compression API (Requires v1.4.0+)
**********************************************/
/* API design :
* Parameters are pushed one by one into an existing context,
@@ -266,6 +266,14 @@ typedef enum { ZSTD_fast=1,
Only the order (from fast to strong) is guaranteed */
} ZSTD_strategy;
typedef enum {
ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level.
* Negative compression levels will be uncompressed, and positive compression
* levels will be compressed. */
ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be
* emitted if Huffman compression is not profitable. */
ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */
} ZSTD_literalCompressionMode_e; /* Requires v1.5.0+ */
typedef enum {
@@ -332,6 +340,11 @@ typedef enum {
* The higher the value of selected strategy, the more complex it is,
* resulting in stronger and slower compression.
* Special: value 0 means "use default strategy". */
ZSTD_c_literalCompressionMode=108, /* Note : requires v1.5.0+
* Controls how the literals are compressed (default is auto).
* The value must be of type ZSTD_literalCompressionMode_e.
* See ZSTD_literalCompressionMode_e enum definition for details.
*/
/* LDM mode parameters */
ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching.
@@ -389,7 +402,7 @@ typedef enum {
ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1.
* Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads.
* 0 means default, which is dynamically determined based on compression parameters.
* Job size must be a minimum of overlap size, or 1 MB, whichever is largest.
* Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest.
* The minimum size is automatically and transparently enforced. */
ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size.
* The overlap size is an amount of data reloaded from previous job at the beginning of a new job.
@@ -411,7 +424,6 @@ typedef enum {
* ZSTD_c_format
* ZSTD_c_forceMaxWindow
* ZSTD_c_forceAttachDict
* ZSTD_c_literalCompressionMode
* ZSTD_c_targetCBlockSize
* ZSTD_c_srcSizeHint
* ZSTD_c_enableDedicatedDictSearch
@@ -419,6 +431,8 @@ typedef enum {
* ZSTD_c_stableOutBuffer
* ZSTD_c_blockDelimiters
* ZSTD_c_validateSequences
* ZSTD_c_splitBlocks
* ZSTD_c_useRowMatchFinder
* Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them.
* note : never ever use experimentalParam? names directly;
* also, the enums values themselves are unstable and can still change.
@@ -427,14 +441,15 @@ typedef enum {
ZSTD_c_experimentalParam2=10,
ZSTD_c_experimentalParam3=1000,
ZSTD_c_experimentalParam4=1001,
ZSTD_c_experimentalParam5=1002,
ZSTD_c_experimentalParam6=1003,
ZSTD_c_experimentalParam7=1004,
ZSTD_c_experimentalParam8=1005,
ZSTD_c_experimentalParam9=1006,
ZSTD_c_experimentalParam10=1007,
ZSTD_c_experimentalParam11=1008,
ZSTD_c_experimentalParam12=1009
ZSTD_c_experimentalParam12=1009,
ZSTD_c_experimentalParam13=1010,
ZSTD_c_experimentalParam14=1011
} ZSTD_cParameter;
typedef struct {
@@ -519,9 +534,9 @@ ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx,
const void* src, size_t srcSize);
/***************************************
* Advanced decompression API
***************************************/
/***********************************************
* Advanced decompression API (Requires v1.4.0+)
************************************************/
/* The advanced API pushes parameters one by one into an existing DCtx context.
* Parameters are sticky, and remain valid for all following frames
@@ -683,7 +698,7 @@ typedef enum {
: note : multithreaded compression will block to flush as much output as possible. */
} ZSTD_EndDirective;
/*! ZSTD_compressStream2() :
/*! ZSTD_compressStream2() : Requires v1.4.0+
* Behaves about the same as ZSTD_compressStream, with additional control on end directive.
* - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*()
* - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode)
@@ -729,11 +744,11 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output
/* *****************************************************************************
* This following is a legacy streaming API.
* This following is a legacy streaming API, available since v1.0+ .
* It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2().
* It is redundant, but remains fully supported.
* Advanced parameters and dictionary compression can only be used through the
* new API.
* Streaming in combination with advanced parameters and dictionary compression
* can only be used through the new API.
******************************************************************************/
/*!
@@ -811,7 +826,7 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
/*! ZSTD_compress_usingDict() :
* Compression at an explicit compression level using a Dictionary.
* A dictionary can be any arbitrary data segment (also called a prefix),
* or a buffer with specified information (see dictBuilder/zdict.h).
* or a buffer with specified information (see zdict.h).
* Note : This function loads the dictionary, resulting in significant startup delay.
* It's intended for a dictionary used only once.
* Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */
@@ -894,19 +909,25 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
* Dictionary helper functions
*******************************/
/*! ZSTD_getDictID_fromDict() :
/*! ZSTD_getDictID_fromDict() : Requires v1.4.0+
* Provides the dictID stored within dictionary.
* if @return == 0, the dictionary is not conformant with Zstandard specification.
* It can still be loaded, but as a content-only dictionary. */
ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
/*! ZSTD_getDictID_fromDDict() :
/*! ZSTD_getDictID_fromCDict() : Requires v1.5.0+
* Provides the dictID of the dictionary loaded into `cdict`.
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict);
/*! ZSTD_getDictID_fromDDict() : Requires v1.4.0+
* Provides the dictID of the dictionary loaded into `ddict`.
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
/*! ZSTD_getDictID_fromFrame() :
/*! ZSTD_getDictID_fromFrame() : Requires v1.4.0+
* Provides the dictID required to decompressed the frame stored within `src`.
* If @return == 0, the dictID could not be decoded.
* This could for one of the following reasons :
@@ -920,7 +941,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
/*******************************************************************************
* Advanced dictionary and prefix API
* Advanced dictionary and prefix API (Requires v1.4.0+)
*
* This API allows dictionaries to be used with ZSTD_compress2(),
* ZSTD_compressStream2(), and ZSTD_decompress(). Dictionaries are sticky, and
@@ -929,7 +950,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
******************************************************************************/
/*! ZSTD_CCtx_loadDictionary() :
/*! ZSTD_CCtx_loadDictionary() : Requires v1.4.0+
* Create an internal CDict from `dict` buffer.
* Decompression will have to use same dictionary.
* @result : 0, or an error code (which can be tested with ZSTD_isError()).
@@ -948,7 +969,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
* to precisely select how dictionary content must be interpreted. */
ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize);
/*! ZSTD_CCtx_refCDict() :
/*! ZSTD_CCtx_refCDict() : Requires v1.4.0+
* Reference a prepared dictionary, to be used for all next compressed frames.
* Note that compression parameters are enforced from within CDict,
* and supersede any compression parameter previously set within CCtx.
@@ -962,7 +983,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, s
* Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */
ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
/*! ZSTD_CCtx_refPrefix() :
/*! ZSTD_CCtx_refPrefix() : Requires v1.4.0+
* Reference a prefix (single-usage dictionary) for next compressed frame.
* A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end).
* Decompression will need same prefix to properly regenerate data.
@@ -983,7 +1004,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict);
ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
const void* prefix, size_t prefixSize);
/*! ZSTD_DCtx_loadDictionary() :
/*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+
* Create an internal DDict from dict buffer,
* to be used to decompress next frames.
* The dictionary remains valid for all future frames, until explicitly invalidated.
@@ -1000,7 +1021,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx,
*/
ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
/*! ZSTD_DCtx_refDDict() :
/*! ZSTD_DCtx_refDDict() : Requires v1.4.0+
* Reference a prepared dictionary, to be used to decompress next frames.
* The dictionary remains active for decompression of future frames using same DCtx.
*
@@ -1018,7 +1039,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, s
*/
ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
/*! ZSTD_DCtx_refPrefix() :
/*! ZSTD_DCtx_refPrefix() : Requires v1.4.0+
* Reference a prefix (single-usage dictionary) to decompress next frame.
* This is the reverse operation of ZSTD_CCtx_refPrefix(),
* and must use the same prefix as the one used during compression.
@@ -1039,7 +1060,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx,
/* === Memory management === */
/*! ZSTD_sizeof_*() :
/*! ZSTD_sizeof_*() : Requires v1.4.0+
* These functions give the _current_ memory usage of selected object.
* Note that object memory usage can evolve (increase or decrease) over time. */
ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
@@ -1262,14 +1283,10 @@ typedef enum {
} ZSTD_dictAttachPref_e;
typedef enum {
ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level.
* Negative compression levels will be uncompressed, and positive compression
* levels will be compressed. */
ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be
* emitted if Huffman compression is not profitable. */
ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */
} ZSTD_literalCompressionMode_e;
ZSTD_urm_auto = 0, /* Automatically determine whether or not we use row matchfinder */
ZSTD_urm_disableRowMatchFinder = 1, /* Never use row matchfinder */
ZSTD_urm_enableRowMatchFinder = 2 /* Always use row matchfinder when applicable */
} ZSTD_useRowMatchFinderMode_e;
/***************************************
* Frame size functions
@@ -1577,12 +1594,6 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(
* note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
/*! ZSTD_getDictID_fromCDict() :
* Provides the dictID of the dictionary loaded into `cdict`.
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict);
/*! ZSTD_getCParams() :
* @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize.
* `estimatedSrcSize` value is optional, select 0 if not known */
@@ -1680,12 +1691,6 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
* See the comments on that enum for an explanation of the feature. */
#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4
/* Controls how the literals are compressed (default is auto).
* The value must be of type ZSTD_literalCompressionMode_e.
* See ZSTD_literalCompressionMode_t enum definition for details.
*/
#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5
/* Tries to fit compressed block size to be around targetCBlockSize.
* No target when targetCBlockSize == 0.
* There is no guarantee on compressed block size (default:0) */
@@ -1834,6 +1839,26 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre
*/
#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12
/* ZSTD_c_splitBlocks
* Default is 0 == disabled. Set to 1 to enable block splitting.
*
* Will attempt to split blocks in order to improve compression ratio at the cost of speed.
*/
#define ZSTD_c_splitBlocks ZSTD_c_experimentalParam13
/* ZSTD_c_useRowMatchFinder
* Default is ZSTD_urm_auto.
* Controlled with ZSTD_useRowMatchFinderMode_e enum.
*
* By default, in ZSTD_urm_auto, when finalizing the compression parameters, the library
* will decide at runtime whether to use the row-based matchfinder based on support for SIMD
* instructions as well as the windowLog.
*
* Set to ZSTD_urm_disableRowMatchFinder to never use row-based matchfinder.
* Set to ZSTD_urm_enableRowMatchFinder to force usage of row-based matchfinder.
*/
#define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14
/*! ZSTD_CCtx_getParameter() :
* Get the requested compression parameter value, selected by enum ZSTD_cParameter,
* and store it into int* value.
@@ -1879,7 +1904,7 @@ ZSTDLIB_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compre
*/
ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params);
/*! ZSTD_CCtxParams_setParameter() :
/*! ZSTD_CCtxParams_setParameter() : Requires v1.4.0+
* Similar to ZSTD_CCtx_setParameter.
* Set one compression parameter, selected by enum ZSTD_cParameter.
* Parameters must be applied to a ZSTD_CCtx using
lib/common/zstd_errors.h → lib/zstd_errors.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
programs/Makefile
+6 -3
View File
@@ -1,5 +1,5 @@
# ################################################################
# Copyright (c) 2015-2021, Yann Collet, Facebook, Inc.
# Copyright (c) Yann Collet, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
@@ -100,8 +100,12 @@ ZSTD_ALL_SRC = $(ZSTDLIB_LOCAL_SRC) $(ZSTD_CLI_SRC)
ZSTD_ALL_OBJ := $(ZSTD_ALL_SRC:.c=.o)
UNAME := $(shell uname)
ifndef BUILD_DIR
ifeq ($(UNAME), Darwin)
HASH ?= md5
ifeq ($(shell md5 < /dev/null > /dev/null; echo $$?), 0)
HASH ?= md5
endif
else ifeq ($(UNAME), FreeBSD)
HASH ?= gmd5sum
else ifeq ($(UNAME), NetBSD)
@@ -112,7 +116,6 @@ endif
HASH ?= md5sum
HAVE_HASH :=$(shell echo 1 | $(HASH) > /dev/null && echo 1 || echo 0)
ifndef BUILD_DIR
HASH_DIR = conf_$(shell echo $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(LDLIBS) $(ZSTD_FILES) | $(HASH) | cut -f 1 -d " ")
ifeq ($(HAVE_HASH),0)
$(info warning : could not find HASH ($(HASH)), needed to differentiate builds using different flags)
programs/benchfn.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
programs/benchfn.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
programs/benchzstd.c
+5 -3
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -36,7 +36,7 @@
#include "datagen.h" /* RDG_genBuffer */
#include "../lib/common/xxhash.h"
#include "benchzstd.h"
#include "../lib/common/zstd_errors.h"
#include "../lib/zstd_errors.h"
/* *************************************
@@ -137,7 +137,8 @@ BMK_advancedParams_t BMK_initAdvancedParams(void) {
0, /* ldmHashLog */
0, /* ldmBuckSizeLog */
0, /* ldmHashRateLog */
ZSTD_lcm_auto /* literalCompressionMode */
ZSTD_lcm_auto, /* literalCompressionMode */
0 /* useRowMatchFinder */
};
return res;
}
@@ -175,6 +176,7 @@ BMK_initCCtx(ZSTD_CCtx* ctx,
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_nbWorkers, adv->nbWorkers));
}
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, cLevel));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_useRowMatchFinder, adv->useRowMatchFinder));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_enableLongDistanceMatching, adv->ldmFlag));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmMinMatch, adv->ldmMinMatch));
CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashLog, adv->ldmHashLog));
programs/benchzstd.h
+2 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -117,6 +117,7 @@ typedef struct {
int ldmBucketSizeLog;
int ldmHashRateLog;
ZSTD_literalCompressionMode_e literalCompressionMode;
int useRowMatchFinder; /* use row-based matchfinder if possible */
} BMK_advancedParams_t;
/* returns default parameters used by nonAdvanced functions */
programs/datagen.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
programs/datagen.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
programs/dibio.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
programs/dibio.h
+2 -2
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -19,7 +19,7 @@
* Dependencies
***************************************/
#define ZDICT_STATIC_LINKING_ONLY
#include "../lib/dictBuilder/zdict.h" /* ZDICT_params_t */
#include "../lib/zdict.h" /* ZDICT_params_t */
/*-*************************************
programs/fileio.c
+9 -3
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -44,7 +44,7 @@
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_magicNumber, ZSTD_frameHeaderSize_max */
#include "../lib/zstd.h"
#include "../lib/common/zstd_errors.h" /* ZSTD_error_frameParameter_windowTooLarge */
#include "../lib/zstd_errors.h" /* ZSTD_error_frameParameter_windowTooLarge */
#if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS)
# include <zlib.h>
@@ -298,6 +298,7 @@ struct FIO_prefs_s {
int blockSize;
int overlapLog;
U32 adaptiveMode;
U32 useRowMatchFinder;
int rsyncable;
int minAdaptLevel;
int maxAdaptLevel;
@@ -468,6 +469,10 @@ void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt) {
prefs->adaptiveMode = adapt;
}
void FIO_setUseRowMatchFinder(FIO_prefs_t* const prefs, int useRowMatchFinder) {
prefs->useRowMatchFinder = useRowMatchFinder;
}
void FIO_setRsyncable(FIO_prefs_t* const prefs, int rsyncable) {
if ((rsyncable>0) && (prefs->nbWorkers==0))
EXM_THROW(1, "Rsyncable mode is not compatible with single thread mode \n");
@@ -986,6 +991,7 @@ static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
if (prefs->ldmHashRateLog != FIO_LDM_PARAM_NOTSET) {
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, prefs->ldmHashRateLog) );
}
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_useRowMatchFinder, prefs->useRowMatchFinder));
/* compression parameters */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_windowLog, (int)comprParams.windowLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_chainLog, (int)comprParams.chainLog) );
@@ -1509,7 +1515,7 @@ FIO_compressFilename_internal(FIO_ctx_t* const fCtx,
U64 readsize = 0;
U64 compressedfilesize = 0;
U64 const fileSize = UTIL_getFileSize(srcFileName);
DISPLAYLEVEL(5, "%s: %u bytes \n", srcFileName, (unsigned)fileSize);
DISPLAYLEVEL(5, "%s: %llu bytes \n", srcFileName, (unsigned long long)fileSize);
/* compression format selection */
switch (prefs->compressionType) {
programs/fileio.h
+2 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -77,6 +77,7 @@ void FIO_overwriteMode(FIO_prefs_t* const prefs);
void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt);
void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel);
void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel);
void FIO_setUseRowMatchFinder(FIO_prefs_t* const prefs, int useRowMatchFinder);
void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize);
void FIO_setChecksumFlag(FIO_prefs_t* const prefs, int checksumFlag);
void FIO_setDictIDFlag(FIO_prefs_t* const prefs, int dictIDFlag);
programs/platform.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
programs/timefn.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2019-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
programs/timefn.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
programs/util.c
+2 -2
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -983,7 +983,7 @@ void UTIL_mirrorSourceFilesDirectories(const char** inFileNames, unsigned int nb
}
FileNamesTable*
UTIL_createExpandedFNT(const char** inputNames, size_t nbIfns, int followLinks)
UTIL_createExpandedFNT(const char* const* inputNames, size_t nbIfns, int followLinks)
{
unsigned nbFiles;
char* buf = (char*)malloc(LIST_SIZE_INCREASE);
programs/util.h
+2 -2
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -277,7 +277,7 @@ void UTIL_refFilename(FileNamesTable* fnt, const char* filename);
* or NULL in case of error
*/
FileNamesTable*
UTIL_createExpandedFNT(const char** filenames, size_t nbFilenames, int followLinks);
UTIL_createExpandedFNT(const char* const* filenames, size_t nbFilenames, int followLinks);
/*-****************************************
programs/windres/verrsrc.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
programs/zstd.1
+6 -6
View File
@@ -1,5 +1,5 @@
.
.TH "ZSTD" "1" "December 2020" "zstd 1.4.8" "User Commands"
.TH "ZSTD" "1" "May 2021" "zstd 1.4.10" "User Commands"
.
.SH "NAME"
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
@@ -156,7 +156,7 @@ This is also used during compression when using with \-\-patch\-from=\. In this
\fB\-o FILE\fR: save result into \fBFILE\fR
.
.IP "\(bu" 4
\fB\-f\fR, \fB\-\-force\fR: overwrite output without prompting, and (de)compress symbolic links
\fB\-f\fR, \fB\-\-force\fR: disable input and output checks\. Allows overwriting existing files, input from console, output to stdout, operating on links, etc\.
.
.IP "\(bu" 4
\fB\-c\fR, \fB\-\-stdout\fR: force write to standard output, even if it is the console
@@ -218,7 +218,7 @@ Using environment variables to set parameters has security implications\. Theref
\fBZSTD_CLEVEL\fR can be used to set the level between 1 and 19 (the "normal" range)\. If the value of \fBZSTD_CLEVEL\fR is not a valid integer, it will be ignored with a warning message\. \fBZSTD_CLEVEL\fR just replaces the default compression level (\fB3\fR)\.
.
.P
\fBZSTD_NBTHREADS\fR can be used to set the number of threads \fBzstd\fR will attempt to use during compression\. If the value of \fBZSTD_NBTHREADS\fR is not a valid unsigned integer, it will be ignored with a warning message\. \'ZSTD_NBTHREADS\fBhas a default value of (\fR1\fB), and is capped at ZSTDMT_NBWORKERS_MAX==200\.\fRzstd` must be compiled with multithread support for this to have any effect\.
\fBZSTD_NBTHREADS\fR can be used to set the number of threads \fBzstd\fR will attempt to use during compression\. If the value of \fBZSTD_NBTHREADS\fR is not a valid unsigned integer, it will be ignored with a warning message\. \fBZSTD_NBTHREADS\fR has a default value of (\fB1\fR), and is capped at ZSTDMT_NBWORKERS_MAX==200\. \fBzstd\fR must be compiled with multithread support for this to have any effect\.
.
.P
They can both be overridden by corresponding command line arguments: \fB\-#\fR for compression level and \fB\-T#\fR for number of compression threads\.
@@ -343,6 +343,9 @@ set process priority to real\-time
.
.SH "ADVANCED COMPRESSION OPTIONS"
.
.SS "\-B#:"
Select the size of each compression job\. This parameter is only available when multi\-threading is enabled\. Each compression job is run in parallel, so this value indirectly impacts the nb of active threads\. Default job size varies depending on compression level (generally \fB4 * windowSize\fR)\. \fB\-B#\fR makes it possible to manually select a custom size\. Note that job size must respect a minimum value which is enforced transparently\. This minimum is either 512 KB, or \fBoverlapSize\fR, whichever is largest\. Different job sizes will lead to (slightly) different compressed frames\.
.
.SS "\-\-zstd[=options]:"
\fBzstd\fR provides 22 predefined compression levels\. The selected or default predefined compression level can be changed with advanced compression options\. The \fIoptions\fR are provided as a comma\-separated list\. You may specify only the options you want to change and the rest will be taken from the selected or default compression level\. The list of available \fIoptions\fR:
.
@@ -481,9 +484,6 @@ The following parameters sets advanced compression options to something similar
.P
\fB\-\-zstd\fR=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
.
.SS "\-B#:"
Select the size of each compression job\. This parameter is available only when multi\-threading is enabled\. Default value is \fB4 * windowSize\fR, which means it varies depending on compression level\. \fB\-B#\fR makes it possible to select a custom value\. Note that job size must respect a minimum value which is enforced transparently\. This minimum is either 1 MB, or \fBoverlapSize\fR, whichever is largest\.
.
.SH "BUGS"
Report bugs at: https://github\.com/facebook/zstd/issues
.
programs/zstd.1.md
+13 -10
View File
@@ -215,7 +215,7 @@ the last one takes effect.
This setting overrides default and can force sparse mode over stdout.
* `--rm`:
remove source file(s) after successful compression or decompression. If used in combination with
-o, will trigger a confirmation prompt (which can be silenced with -f), as this is a destructive operation.
-o, will trigger a confirmation prompt (which can be silenced with -f), as this is a destructive operation.
* `-k`, `--keep`:
keep source file(s) after successful compression or decompression.
This is the default behavior.
@@ -281,11 +281,11 @@ If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a
`ZSTD_NBTHREADS` can be used to set the number of threads `zstd` will attempt to use during compression.
If the value of `ZSTD_NBTHREADS` is not a valid unsigned integer, it will be ignored with a warning message.
'ZSTD_NBTHREADS` has a default value of (`1`), and is capped at ZSTDMT_NBWORKERS_MAX==200. `zstd` must be
`ZSTD_NBTHREADS` has a default value of (`1`), and is capped at ZSTDMT_NBWORKERS_MAX==200. `zstd` must be
compiled with multithread support for this to have any effect.
They can both be overridden by corresponding command line arguments:
`-#` for compression level and `-T#` for number of compression threads.
`-#` for compression level and `-T#` for number of compression threads.
DICTIONARY BUILDER
@@ -423,6 +423,16 @@ BENCHMARK
ADVANCED COMPRESSION OPTIONS
----------------------------
### -B#:
Select the size of each compression job.
This parameter is only available when multi-threading is enabled.
Each compression job is run in parallel, so this value indirectly impacts the nb of active threads.
Default job size varies depending on compression level (generally `4 * windowSize`).
`-B#` makes it possible to manually select a custom size.
Note that job size must respect a minimum value which is enforced transparently.
This minimum is either 512 KB, or `overlapSize`, whichever is largest.
Different job sizes will lead to (slightly) different compressed frames.
### --zstd[=options]:
`zstd` provides 22 predefined compression levels.
The selected or default predefined compression level can be changed with
@@ -566,13 +576,6 @@ similar to predefined level 19 for files bigger than 256 KB:
`--zstd`=wlog=23,clog=23,hlog=22,slog=6,mml=3,tlen=48,strat=6
### -B#:
Select the size of each compression job.
This parameter is available only when multi-threading is enabled.
Default value is `4 * windowSize`, which means it varies depending on compression level.
`-B#` makes it possible to select a custom value.
Note that job size must respect a minimum value which is enforced transparently.
This minimum is either 1 MB, or `overlapSize`, whichever is largest.
BUGS
----
programs/zstdcli.c
+8 -2
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -205,6 +205,7 @@ static void usage_advanced(const char* programName)
DISPLAYOUT( "--long[=#]: enable long distance matching with given window log (default: %u) \n", g_defaultMaxWindowLog);
DISPLAYOUT( "--fast[=#]: switch to very fast compression levels (default: %u) \n", 1);
DISPLAYOUT( "--adapt : dynamically adapt compression level to I/O conditions \n");
DISPLAYOUT( "--[no-]row-match-finder : force enable/disable usage of fast row-based matchfinder for greedy, lazy, and lazy2 strategies \n");
# ifdef ZSTD_MULTITHREAD
DISPLAYOUT( " -T# : spawns # compression threads (default: 1, 0==# cores) \n");
DISPLAYOUT( " -B# : select size of each job (default: 0==automatic) \n");
@@ -730,6 +731,7 @@ int main(int const argCount, const char* argv[])
main_pause = 0,
nbWorkers = 0,
adapt = 0,
useRowMatchFinder = 0,
adaptMin = MINCLEVEL,
adaptMax = MAXCLEVEL,
rsyncable = 0,
@@ -857,6 +859,8 @@ int main(int const argCount, const char* argv[])
if (!strcmp(argument, "--content-size")) { contentSize = 1; continue; }
if (!strcmp(argument, "--no-content-size")) { contentSize = 0; continue; }
if (!strcmp(argument, "--adapt")) { adapt = 1; continue; }
if (!strcmp(argument, "--no-row-match-finder")) { useRowMatchFinder = 1; continue; }
if (!strcmp(argument, "--row-match-finder")) { useRowMatchFinder = 2; continue; }
if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) { badusage(programName); CLEAN_RETURN(1); } continue; }
if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; }
if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; FIO_setCompressionType(prefs, FIO_zstdCompression); continue; }
@@ -1196,6 +1200,7 @@ int main(int const argCount, const char* argv[])
benchParams.ldmFlag = ldmFlag;
benchParams.ldmMinMatch = (int)g_ldmMinMatch;
benchParams.ldmHashLog = (int)g_ldmHashLog;
benchParams.useRowMatchFinder = useRowMatchFinder;
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
benchParams.ldmBucketSizeLog = (int)g_ldmBucketSizeLog;
}
@@ -1348,6 +1353,7 @@ int main(int const argCount, const char* argv[])
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, (int)g_ldmBucketSizeLog);
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, (int)g_ldmHashRateLog);
FIO_setAdaptiveMode(prefs, (unsigned)adapt);
FIO_setUseRowMatchFinder(prefs, useRowMatchFinder);
FIO_setAdaptMin(prefs, adaptMin);
FIO_setAdaptMax(prefs, adaptMax);
FIO_setRsyncable(prefs, rsyncable);
@@ -1387,7 +1393,7 @@ int main(int const argCount, const char* argv[])
else
operationResult = FIO_compressMultipleFilenames(fCtx, prefs, filenames->fileNames, outMirroredDirName, outDirName, outFileName, suffix, dictFileName, cLevel, compressionParams);
#else
(void)contentSize; (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; (void)ZSTD_strategyMap; /* not used when ZSTD_NOCOMPRESS set */
(void)contentSize; (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; (void)literalCompressionMode; (void)targetCBlockSize; (void)streamSrcSize; (void)srcSizeHint; (void)ZSTD_strategyMap; (void)useRowMatchFinder; /* not used when ZSTD_NOCOMPRESS set */
DISPLAY("Compression not supported \n");
#endif
} else { /* decompression or test */
programs/zstdcli_trace.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
programs/zstdcli_trace.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
programs/zstdgrep.1
+1 -1
View File
@@ -1,5 +1,5 @@
.
.TH "ZSTDGREP" "1" "December 2020" "zstd 1.4.8" "User Commands"
.TH "ZSTDGREP" "1" "May 2021" "zstd 1.4.10" "User Commands"
.
.SH "NAME"
\fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files
programs/zstdless.1
+1 -1
View File
@@ -1,5 +1,5 @@
.
.TH "ZSTDLESS" "1" "December 2020" "zstd 1.4.8" "User Commands"
.TH "ZSTDLESS" "1" "May 2021" "zstd 1.4.10" "User Commands"
.
.SH "NAME"
\fBzstdless\fR \- view zstandard\-compressed files
tests/DEPRECATED-test-zstd-speed.py
+1 -1
View File
@@ -2,7 +2,7 @@
# THIS BENCHMARK IS BEING REPLACED BY automated-bencmarking.py
# ################################################################
# Copyright (c) 2016-2021, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
# Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
tests/Makefile
+5 -20
View File
@@ -1,5 +1,5 @@
# ################################################################
# Copyright (c) 2015-2021, Yann Collet, Facebook, Inc.
# Copyright (c) Yann Collet, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
@@ -28,7 +28,8 @@ DEBUGLEVEL ?= 1
export DEBUGLEVEL # transmit value to sub-makefiles
DEBUGFLAGS = -g -DDEBUGLEVEL=$(DEBUGLEVEL)
CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
-I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) \
-DZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY=1
ifeq ($(OS),Windows_NT) # MinGW assumed
CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting
endif
@@ -46,7 +47,6 @@ ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c
ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c
ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
ZBUFF_FILES := $(ZSTDDIR)/deprecated/*.c
ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
ZSTD_F1 := $(wildcard $(ZSTD_FILES))
@@ -160,15 +160,6 @@ fuzzer32 : $(ZSTD_FILES)
fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c
$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
zbufftest zbufftest32 zbufftest-dll : CPPFLAGS += -I$(ZSTDDIR)/deprecated
zbufftest zbufftest32 zbufftest-dll : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings
zbufftest32 : CFLAGS += -m32
zbufftest zbufftest32 : $(ZSTD_OBJECTS) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c zbufftest.c
$(LINK.c) $^ -o $@$(EXT)
zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c zbufftest.c
$(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT)
ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c seqgen.c zstreamtest.c
ZSTREAM_PROPER_FILES := $(ZDICT_FILES) $(ZSTREAM_LOCAL_FILES)
ZSTREAMFILES := $(ZSTD_FILES) $(ZSTREAM_PROPER_FILES)
@@ -240,8 +231,8 @@ clean:
$(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \
fullbench$(EXT) fullbench32$(EXT) \
fullbench-lib$(EXT) fullbench-dll$(EXT) \
fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT) \
fuzzer$(EXT) fuzzer32$(EXT) \
fuzzer-dll$(EXT) zstreamtest-dll$(EXT) \
zstreamtest$(EXT) zstreamtest32$(EXT) \
datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \
symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) poolTests$(EXT) \
@@ -345,12 +336,6 @@ test-fuzzer-stackmode: test-fuzzer
test-fuzzer32: fuzzer32
$(QEMU_SYS) ./fuzzer32 -v $(FUZZERTEST) $(FUZZER_FLAGS)
test-zbuff: zbufftest
$(QEMU_SYS) ./zbufftest $(ZSTREAM_TESTTIME)
test-zbuff32: zbufftest32
$(QEMU_SYS) ./zbufftest32 $(ZSTREAM_TESTTIME)
test-zstream: zstreamtest
$(QEMU_SYS) ./zstreamtest -v $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
$(QEMU_SYS) ./zstreamtest --newapi -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)
tests/automated_benchmarking.py
+1 -1
View File
@@ -1,5 +1,5 @@
# ################################################################
# Copyright (c) 2021-2021, Facebook, Inc.
# Copyright (c) Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
tests/bigdict.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2017-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/checkTag.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2018-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/datagencli.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/decodecorpus.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2017-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fullbench.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzz/.gitignore
+1
View File
@@ -19,6 +19,7 @@ sequence_compression_api
fuzz-*.log
rt_lib_*
d_lib_*
crash-*
# misc
trace
tests/fuzz/Makefile
+1 -1
View File
@@ -1,5 +1,5 @@
# ################################################################
# Copyright (c) 2016-2021, Facebook, Inc.
# Copyright (c) Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
tests/fuzz/block_decompress.c
+1 -1
View File
@@ -1,5 +1,5 @@
/**
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzz/block_round_trip.c
+1 -1
View File
@@ -1,5 +1,5 @@
/**
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzz/decompress_dstSize_tooSmall.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzz/dictionary_decompress.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzz/dictionary_loader.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzz/dictionary_round_trip.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzz/dictionary_stream_round_trip.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzz/fse_read_ncount.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzz/fuzz.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzz/fuzz.py
+1 -1
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# ################################################################
# Copyright (c) 2016-2021, Facebook, Inc.
# Copyright (c) Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
tests/fuzz/fuzz_data_producer.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzz/fuzz_data_producer.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzz/fuzz_helpers.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzz/fuzz_helpers.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzz/raw_dictionary_round_trip.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzz/regression_driver.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzz/sequence_compression_api.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzz/simple_compress.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzz/simple_decompress.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzz/simple_round_trip.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzz/stream_decompress.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzz/stream_round_trip.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzz/zstd_frame_info.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzz/zstd_helpers.c
+4 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -91,9 +91,12 @@ void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer
/* Set misc parameters */
setRand(cctx, ZSTD_c_nbWorkers, 0, 2, producer);
setRand(cctx, ZSTD_c_rsyncable, 0, 1, producer);
setRand(cctx, ZSTD_c_useRowMatchFinder, 0, 2, producer);
setRand(cctx, ZSTD_c_enableDedicatedDictSearch, 0, 1, producer);
setRand(cctx, ZSTD_c_forceMaxWindow, 0, 1, producer);
setRand(cctx, ZSTD_c_literalCompressionMode, 0, 2, producer);
setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, producer);
setRand(cctx, ZSTD_c_splitBlocks, 0, 1, producer);
if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) {
setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer);
}
tests/fuzz/zstd_helpers.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/fuzzer.c
+80 -6
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -374,6 +374,12 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "%i (OK) \n", mcl);
}
DISPLAYLEVEL(3, "test%3u : default compression level : ", testNb++);
{ int const defaultCLevel = ZSTD_defaultCLevel();
if (defaultCLevel != ZSTD_CLEVEL_DEFAULT) goto _output_error;
DISPLAYLEVEL(3, "%i (OK) \n", defaultCLevel);
}
DISPLAYLEVEL(3, "test%3u : ZSTD_versionNumber : ", testNb++);
{ unsigned const vn = ZSTD_versionNumber();
DISPLAYLEVEL(3, "%u (OK) \n", vn);
@@ -1544,6 +1550,15 @@ static int basicUnitTests(U32 const seed, double compressibility)
ZSTD_freeCCtx(cctx);
}
DISPLAYLEVEL(3, "test%3i : compress with block splitting : ", testNb++)
{ ZSTD_CCtx* cctx = ZSTD_createCCtx();
CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_splitBlocks, 1) );
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
CHECK(cSize);
ZSTD_freeCCtx(cctx);
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb++)
{ ZSTD_CCtx* cctx = ZSTD_createCCtx();
size_t cSize1, cSize2;
@@ -1574,6 +1589,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(3, "test%3i : setting multithreaded parameters : ", testNb++)
{ ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
int const jobSize = 512 KB;
int value;
/* Check that the overlap log and job size are unset. */
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
@@ -1582,19 +1598,18 @@ static int basicUnitTests(U32 const seed, double compressibility)
CHECK_EQ(value, 0);
/* Set and check the overlap log and job size. */
CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) );
CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, 2 MB) );
CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, jobSize) );
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
CHECK_EQ(value, 5);
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
CHECK_EQ(value, 2 MB);
CHECK_EQ(value, jobSize);
/* Set the number of workers and check the overlap log and job size. */
CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) );
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
CHECK_EQ(value, 5);
CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
CHECK_EQ(value, 2 MB);
CHECK_EQ(value, jobSize);
ZSTD_freeCCtxParams(params);
}
DISPLAYLEVEL(3, "OK \n");
@@ -1742,13 +1757,14 @@ static int basicUnitTests(U32 const seed, double compressibility)
size_t const contentSize = 9 KB;
const void* const dict = (const char*)CNBuffer;
const void* const contentStart = (const char*)dict + flatdictSize;
/* These upper bounds are generally within a few bytes of the compressed size */
size_t const target_nodict_cSize[22+1] = { 3840, 3770, 3870, 3830, 3770,
3770, 3770, 3770, 3750, 3750,
3742, 3670, 3670, 3660, 3660,
3660, 3660, 3660, 3660, 3660,
3660, 3660, 3660 };
size_t const target_wdict_cSize[22+1] = { 2830, 2890, 2890, 2820, 2940,
2950, 2950, 2921, 2900, 2891,
2950, 2950, 2925, 2900, 2891,
2910, 2910, 2910, 2770, 2760,
2750, 2750, 2750, 2750, 2750,
2750, 2750, 2750 };
@@ -1785,6 +1801,22 @@ static int basicUnitTests(U32 const seed, double compressibility)
DISPLAYLEVEL(4, "level %i with dictionary : max expected %u >= reached %u \n",
l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize);
}
/* Dict compression with DMS */
for ( l=1 ; l <= maxLevel; l++) {
size_t wdict_cSize;
CHECK_Z( ZSTD_CCtx_loadDictionary(ctxOrig, dict, flatdictSize) );
CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_compressionLevel, l) );
CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_enableDedicatedDictSearch, 0) );
CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach) );
wdict_cSize = ZSTD_compress2(ctxOrig, compressedBuffer, compressedBufferSize, contentStart, contentSize);
if (wdict_cSize > target_wdict_cSize[l]) {
DISPLAYLEVEL(1, "error : compression with dictionary and compress2 at level %i worse than expected (%u > %u) \n",
l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]);
goto _output_error;
}
DISPLAYLEVEL(4, "level %i with dictionary and compress2 : max expected %u >= reached %u \n",
l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize);
}
DISPLAYLEVEL(4, "compression efficiency tests OK \n");
}
@@ -3181,6 +3213,48 @@ static int basicUnitTests(U32 const seed, double compressibility)
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : check compression mem usage monotonicity over levels for estimateCCtxSize() : ", testNb++);
{
int level = 1;
size_t prevSize = 0;
for (; level < ZSTD_maxCLevel(); ++level) {
size_t const currSize = ZSTD_estimateCCtxSize(level);
if (prevSize > currSize) {
DISPLAYLEVEL(3, "Error! previous cctx size: %zu at level: %d is larger than current cctx size: %zu at level: %d",
prevSize, level-1, currSize, level);
goto _output_error;
}
prevSize = currSize;
}
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : check estimateCCtxSize() always larger or equal to ZSTD_estimateCCtxSize_usingCParams() : ", testNb++);
{
size_t const kSizeIncrement = 2 KB;
int level = -3;
for (; level <= ZSTD_maxCLevel(); ++level) {
size_t dictSize = 0;
for (; dictSize <= 256 KB; dictSize += 8 * kSizeIncrement) {
size_t srcSize = 2 KB;
for (; srcSize < 300 KB; srcSize += kSizeIncrement) {
ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, srcSize, dictSize);
size_t const cctxSizeUsingCParams = ZSTD_estimateCCtxSize_usingCParams(cParams);
size_t const cctxSizeUsingLevel = ZSTD_estimateCCtxSize(level);
if (cctxSizeUsingLevel < cctxSizeUsingCParams
|| ZSTD_isError(cctxSizeUsingCParams)
|| ZSTD_isError(cctxSizeUsingLevel)) {
DISPLAYLEVEL(3, "error! l: %d dict: %zu srcSize: %zu cctx size cpar: %zu, cctx size level: %zu\n",
level, dictSize, srcSize, cctxSizeUsingCParams, cctxSizeUsingLevel);
goto _output_error;
}
}
}
}
}
DISPLAYLEVEL(3, "OK \n");
#endif
_end:
tests/gzip/Makefile
+1 -1
View File
@@ -1,5 +1,5 @@
# ################################################################
# Copyright (c) 2017-2021, Facebook, Inc.
# Copyright (c) Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
tests/invalidDictionaries.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/legacy.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/libzstd_partial_builds.sh
+16 -5
View File
@@ -21,7 +21,7 @@ mustBeAbsent() {
$ECHO "$@ correctly not present" # for some reason, this $ECHO must exist, otherwise mustBeAbsent() always fails (??)
}
# default compilation : all features enabled
# default compilation : all features enabled - no zbuff
$ECHO "testing default library compilation"
CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
@@ -29,10 +29,10 @@ isPresent "zstd_compress.o"
isPresent "zstd_decompress.o"
isPresent "zdict.o"
isPresent "zstd_v07.o"
isPresent "zbuff_compress.o"
mustBeAbsent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog
# compression disabled => also disable zdict and zbuff
# compression disabled => also disable zdict
$ECHO "testing with compression disabled"
ZSTD_LIB_COMPRESSION=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
@@ -43,7 +43,7 @@ isPresent "zstd_v07.o"
mustBeAbsent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog
# decompression disabled => also disable legacy and zbuff
# decompression disabled => also disable legacy
$ECHO "testing with decompression disabled"
ZSTD_LIB_DECOMPRESSION=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
@@ -65,6 +65,17 @@ isPresent "zstd_v07.o"
mustBeAbsent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog
# deprecated function enabled => zbuff present
$ECHO "testing with deprecated functions enabled"
ZSTD_LIB_DEPRECATED=1 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
isPresent "zstd_compress.o"
isPresent "zstd_decompress.o"
isPresent "zdict.o"
isPresent "zstd_v07.o"
isPresent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog
# dictionary builder disabled => only remove zdict
$ECHO "testing with dictionary builder disabled"
ZSTD_LIB_DICTBUILDER=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
@@ -73,7 +84,7 @@ isPresent "zstd_compress.o"
isPresent "zstd_decompress.o"
mustBeAbsent "zdict.o"
isPresent "zstd_v07.o"
isPresent "zbuff_compress.o"
mustBeAbsent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog
# both decompression and dictionary builder disabled => only compression remains
tests/longmatch.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2017-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/paramgrill.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/playTests.sh
+9 -1
View File
@@ -1225,7 +1225,7 @@ then
println "\n===> zstdmt round-trip tests "
roundTripTest -g4M "1 -T0"
roundTripTest -g8M "3 -T2"
roundTripTest -g8M "19 -T0 --long"
roundTripTest -g8M "19 --long"
roundTripTest -g8000K "2 --threads=2"
fileRoundTripTest -g4M "19 -T2 -B1M"
@@ -1430,6 +1430,14 @@ datagen -g5000000 > tmp_patch
zstd -15 --patch-from=tmp_dict tmp_patch 2>&1 | grep "long mode automatically triggered"
rm -rf tmp*
println "\n===> patch-from very large dictionary and file test"
datagen -g550000000 -P0 > tmp_dict
datagen -g100000000 -P1 > tmp_patch
zstd --long=30 -1f --patch-from tmp_dict tmp_patch
zstd --long=30 -df --patch-from tmp_dict tmp_patch.zst -o tmp_patch_recon
$DIFF -s tmp_patch_recon tmp_patch
rm -rf tmp*
println "\n===> patch-from --stream-size test"
datagen -g1000 -P50 > tmp_dict
datagen -g1000 -P10 > tmp_patch
tests/poolTests.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/rateLimiter.py
+1 -1
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
# ################################################################
# Copyright (c) 2018-2021, Facebook, Inc.
# Copyright (c) Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
tests/regression/Makefile
+1 -1
View File
@@ -1,5 +1,5 @@
# ################################################################
# Copyright (c) 2015-2021, Facebook, Inc.
# Copyright (c) Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
tests/regression/config.c
+133 -16
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -28,20 +28,134 @@
};
/* Define a config for each level we want to test with. */
#define LEVEL(x) \
param_value_t const level_##x##_param_values[] = { \
{.param = ZSTD_c_compressionLevel, .value = x}, \
}; \
config_t const level_##x = { \
.name = "level " #x, \
.cli_args = "-" #x, \
.param_values = PARAM_VALUES(level_##x##_param_values), \
}; \
config_t const level_##x##_dict = { \
.name = "level " #x " with dict", \
.cli_args = "-" #x, \
.param_values = PARAM_VALUES(level_##x##_param_values), \
.use_dictionary = 1, \
#define LEVEL(x) \
param_value_t const level_##x##_param_values[] = { \
{.param = ZSTD_c_compressionLevel, .value = x}, \
}; \
param_value_t const level_##x##_param_values_dms[] = { \
{.param = ZSTD_c_compressionLevel, .value = x}, \
{.param = ZSTD_c_enableDedicatedDictSearch, .value = 0}, \
{.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceAttach}, \
}; \
param_value_t const level_##x##_param_values_dds[] = { \
{.param = ZSTD_c_compressionLevel, .value = x}, \
{.param = ZSTD_c_enableDedicatedDictSearch, .value = 1}, \
{.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceAttach}, \
}; \
param_value_t const level_##x##_param_values_dictcopy[] = { \
{.param = ZSTD_c_compressionLevel, .value = x}, \
{.param = ZSTD_c_enableDedicatedDictSearch, .value = 0}, \
{.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceCopy}, \
}; \
param_value_t const level_##x##_param_values_dictload[] = { \
{.param = ZSTD_c_compressionLevel, .value = x}, \
{.param = ZSTD_c_enableDedicatedDictSearch, .value = 0}, \
{.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceLoad}, \
}; \
config_t const level_##x = { \
.name = "level " #x, \
.cli_args = "-" #x, \
.param_values = PARAM_VALUES(level_##x##_param_values), \
}; \
config_t const level_##x##_dict = { \
.name = "level " #x " with dict", \
.cli_args = "-" #x, \
.param_values = PARAM_VALUES(level_##x##_param_values), \
.use_dictionary = 1, \
}; \
config_t const level_##x##_dict_dms = { \
.name = "level " #x " with dict dms", \
.cli_args = "-" #x, \
.param_values = PARAM_VALUES(level_##x##_param_values_dms), \
.use_dictionary = 1, \
.advanced_api_only = 1, \
}; \
config_t const level_##x##_dict_dds = { \
.name = "level " #x " with dict dds", \
.cli_args = "-" #x, \
.param_values = PARAM_VALUES(level_##x##_param_values_dds), \
.use_dictionary = 1, \
.advanced_api_only = 1, \
}; \
config_t const level_##x##_dict_copy = { \
.name = "level " #x " with dict copy", \
.cli_args = "-" #x, \
.param_values = PARAM_VALUES(level_##x##_param_values_dictcopy), \
.use_dictionary = 1, \
.advanced_api_only = 1, \
}; \
config_t const level_##x##_dict_load = { \
.name = "level " #x " with dict load", \
.cli_args = "-" #x, \
.param_values = PARAM_VALUES(level_##x##_param_values_dictload), \
.use_dictionary = 1, \
.advanced_api_only = 1, \
};
/* Define a config specifically to test row hash based levels and settings.
*/
#define ROW_LEVEL(x, y) \
param_value_t const row_##y##_level_##x##_param_values[] = { \
{.param = ZSTD_c_useRowMatchFinder, .value = y}, \
{.param = ZSTD_c_compressionLevel, .value = x}, \
}; \
param_value_t const row_##y##_level_##x##_param_values_dms[] = { \
{.param = ZSTD_c_useRowMatchFinder, .value = y}, \
{.param = ZSTD_c_compressionLevel, .value = x}, \
{.param = ZSTD_c_enableDedicatedDictSearch, .value = 0}, \
{.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceAttach}, \
}; \
param_value_t const row_##y##_level_##x##_param_values_dds[] = { \
{.param = ZSTD_c_useRowMatchFinder, .value = y}, \
{.param = ZSTD_c_compressionLevel, .value = x}, \
{.param = ZSTD_c_enableDedicatedDictSearch, .value = 1}, \
{.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceAttach}, \
}; \
param_value_t const row_##y##_level_##x##_param_values_dictcopy[] = { \
{.param = ZSTD_c_useRowMatchFinder, .value = y}, \
{.param = ZSTD_c_compressionLevel, .value = x}, \
{.param = ZSTD_c_enableDedicatedDictSearch, .value = 0}, \
{.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceCopy}, \
}; \
param_value_t const row_##y##_level_##x##_param_values_dictload[] = { \
{.param = ZSTD_c_useRowMatchFinder, .value = y}, \
{.param = ZSTD_c_compressionLevel, .value = x}, \
{.param = ZSTD_c_enableDedicatedDictSearch, .value = 0}, \
{.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceLoad}, \
}; \
config_t const row_##y##_level_##x = { \
.name = "level " #x " row " #y, \
.cli_args = "-" #x, \
.param_values = PARAM_VALUES(row_##y##_level_##x##_param_values), \
.advanced_api_only = 1, \
}; \
config_t const row_##y##_level_##x##_dict_dms = { \
.name = "level " #x " row " #y " with dict dms", \
.cli_args = "-" #x, \
.param_values = PARAM_VALUES(row_##y##_level_##x##_param_values_dms), \
.use_dictionary = 1, \
.advanced_api_only = 1, \
}; \
config_t const row_##y##_level_##x##_dict_dds = { \
.name = "level " #x " row " #y " with dict dds", \
.cli_args = "-" #x, \
.param_values = PARAM_VALUES(row_##y##_level_##x##_param_values_dds), \
.use_dictionary = 1, \
.advanced_api_only = 1, \
}; \
config_t const row_##y##_level_##x##_dict_copy = { \
.name = "level " #x " row " #y" with dict copy", \
.cli_args = "-" #x, \
.param_values = PARAM_VALUES(row_##y##_level_##x##_param_values_dictcopy), \
.use_dictionary = 1, \
.advanced_api_only = 1, \
}; \
config_t const row_##y##_level_##x##_dict_load = { \
.name = "level " #x " row " #y " with dict load", \
.cli_args = "-" #x, \
.param_values = PARAM_VALUES(row_##y##_level_##x##_param_values_dictload), \
.use_dictionary = 1, \
.advanced_api_only = 1, \
};
#define PARAM_VALUES(pv) \
@@ -51,6 +165,7 @@
#undef LEVEL
#undef FAST_LEVEL
#undef ROW_LEVEL
static config_t no_pledged_src_size = {
.name = "no source size",
@@ -194,8 +309,10 @@ static config_t explicit_params = {
static config_t const* g_configs[] = {
#define FAST_LEVEL(x) &level_fast##x, &level_fast##x##_dict,
#define LEVEL(x) &level_##x, &level_##x##_dict,
#define LEVEL(x) &level_##x, &level_##x##_dict, &level_##x##_dict_dms, &level_##x##_dict_dds, &level_##x##_dict_copy, &level_##x##_dict_load,
#define ROW_LEVEL(x, y) &row_##y##_level_##x, &row_##y##_level_##x##_dict_dms, &row_##y##_level_##x##_dict_dds, &row_##y##_level_##x##_dict_copy, &row_##y##_level_##x##_dict_load,
#include "levels.h"
#undef ROW_LEVEL
#undef LEVEL
#undef FAST_LEVEL
tests/regression/config.h
+6 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -53,6 +53,11 @@ typedef struct {
* when the method allows it. Defaults to yes.
*/
int no_pledged_src_size;
/**
* Boolean parameter that says that this config should only be used
* for methods that use the advanced compression API
*/
int advanced_api_only;
} config_t;
/**
tests/regression/data.c
+2 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -14,6 +14,7 @@
#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h> /* free() */
#include <sys/stat.h>
tests/regression/data.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/regression/levels.h
+14 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -14,6 +14,9 @@
#ifndef FAST_LEVEL
# error FAST_LEVEL(x) must be defined
#endif
#ifndef ROW_LEVEL
# error ROW_LEVEL(x, y) must be defined
#endif
/**
* The levels are chosen to trigger every strategy in every source size,
@@ -31,12 +34,22 @@ LEVEL(1)
LEVEL(3)
LEVEL(4)
/* ROW_LEVEL triggers the row hash (force enabled and disabled) with different
* dictionary strategies, and 16/32 row entries based on the level/searchLog.
* 1 == disabled, 2 == enabled.
*/
ROW_LEVEL(5, 1)
ROW_LEVEL(5, 2)
LEVEL(5)
LEVEL(6)
ROW_LEVEL(7, 1)
ROW_LEVEL(7, 2)
LEVEL(7)
LEVEL(9)
ROW_LEVEL(12, 1)
ROW_LEVEL(12, 2)
LEVEL(13)
LEVEL(16)
tests/regression/method.c
+15 -2
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -102,6 +102,9 @@ static result_t simple_compress(method_state_t* base, config_t const* config) {
*/
if (base->data->type != data_type_file)
return result_error(result_error_skip);
if (config->advanced_api_only)
return result_error(result_error_skip);
if (config->use_dictionary || config->no_pledged_src_size)
return result_error(result_error_skip);
@@ -151,6 +154,9 @@ static result_t compress_cctx_compress(
if (base->data->type != data_type_dir)
return result_error(result_error_skip);
if (config->advanced_api_only)
return result_error(result_error_skip);
int const level = config_get_level(config);
@@ -254,6 +260,9 @@ static result_t cli_compress(method_state_t* state, config_t const* config) {
if (config->cli_args == NULL)
return result_error(result_error_skip);
if (config->advanced_api_only)
return result_error(result_error_skip);
/* We don't support no pledged source size with directories. Too slow. */
if (state->data->type == data_type_dir && config->no_pledged_src_size)
return result_error(result_error_skip);
@@ -523,6 +532,10 @@ static result_t old_streaming_compress_internal(
result = result_error(result_error_skip);
goto out;
}
if (config->advanced_api_only) {
result = result_error(result_error_skip);
goto out;
}
if (init_cstream(state, zcs, config, advanced, cdict ? &cd : NULL)) {
result = result_error(result_error_compression_error);
goto out;
@@ -651,7 +664,7 @@ method_t const old_streaming_advanced = {
};
method_t const old_streaming_cdict = {
.name = "old streaming cdcit",
.name = "old streaming cdict",
.create = buffer_state_create,
.compress = old_streaming_compress_cdict,
.destroy = buffer_state_destroy,
tests/regression/method.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/regression/result.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/regression/result.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/regression/results.csv
+746 -266
View File
@@ -6,10 +6,10 @@ silesia.tar, level 0, compress
silesia.tar, level 1, compress simple, 5334885
silesia.tar, level 3, compress simple, 4861425
silesia.tar, level 4, compress simple, 4799630
silesia.tar, level 5, compress simple, 4722324
silesia.tar, level 6, compress simple, 4672279
silesia.tar, level 7, compress simple, 4606715
silesia.tar, level 9, compress simple, 4554147
silesia.tar, level 5, compress simple, 4719256
silesia.tar, level 6, compress simple, 4677721
silesia.tar, level 7, compress simple, 4613541
silesia.tar, level 9, compress simple, 4555426
silesia.tar, level 13, compress simple, 4491764
silesia.tar, level 16, compress simple, 4381332
silesia.tar, level 19, compress simple, 4281605
@@ -23,10 +23,10 @@ github.tar, level 0, compress
github.tar, level 1, compress simple, 39265
github.tar, level 3, compress simple, 38441
github.tar, level 4, compress simple, 38467
github.tar, level 5, compress simple, 39788
github.tar, level 6, compress simple, 39603
github.tar, level 7, compress simple, 39206
github.tar, level 9, compress simple, 36717
github.tar, level 5, compress simple, 39693
github.tar, level 6, compress simple, 39621
github.tar, level 7, compress simple, 39213
github.tar, level 9, compress simple, 36758
github.tar, level 13, compress simple, 35621
github.tar, level 16, compress simple, 40255
github.tar, level 19, compress simple, 32837
@@ -40,10 +40,10 @@ silesia, level 0, compress
silesia, level 1, compress cctx, 5313204
silesia, level 3, compress cctx, 4849552
silesia, level 4, compress cctx, 4786970
silesia, level 5, compress cctx, 4710236
silesia, level 6, compress cctx, 4660056
silesia, level 7, compress cctx, 4596296
silesia, level 9, compress cctx, 4543925
silesia, level 5, compress cctx, 4707794
silesia, level 6, compress cctx, 4666383
silesia, level 7, compress cctx, 4603381
silesia, level 9, compress cctx, 4546001
silesia, level 13, compress cctx, 4482135
silesia, level 16, compress cctx, 4377465
silesia, level 19, compress cctx, 4293330
@@ -53,7 +53,7 @@ silesia, multithreaded long distance mode, compress
silesia, small window log, compress cctx, 7084179
silesia, small hash log, compress cctx, 6555021
silesia, small chain log, compress cctx, 4931148
silesia, explicit params, compress cctx, 4794677
silesia, explicit params, compress cctx, 4794479
silesia, uncompressed literals, compress cctx, 4849552
silesia, uncompressed literals optimal, compress cctx, 4293330
silesia, huffman literals, compress cctx, 6178460
@@ -73,13 +73,13 @@ github, level 3 with dict, compress
github, level 4, compress cctx, 136199
github, level 4 with dict, compress cctx, 41725
github, level 5, compress cctx, 135121
github, level 5 with dict, compress cctx, 38934
github, level 5 with dict, compress cctx, 38759
github, level 6, compress cctx, 135122
github, level 6 with dict, compress cctx, 38628
github, level 6 with dict, compress cctx, 38669
github, level 7, compress cctx, 135122
github, level 7 with dict, compress cctx, 38745
github, level 7 with dict, compress cctx, 38755
github, level 9, compress cctx, 135122
github, level 9 with dict, compress cctx, 39341
github, level 9 with dict, compress cctx, 39398
github, level 13, compress cctx, 134064
github, level 13 with dict, compress cctx, 39948
github, level 16, compress cctx, 134064
@@ -104,22 +104,22 @@ silesia, level 0, zstdcli,
silesia, level 1, zstdcli, 5314210
silesia, level 3, zstdcli, 4849600
silesia, level 4, zstdcli, 4787018
silesia, level 5, zstdcli, 4710284
silesia, level 6, zstdcli, 4660104
silesia, level 7, zstdcli, 4596344
silesia, level 9, zstdcli, 4543973
silesia, level 5, zstdcli, 4707842
silesia, level 6, zstdcli, 4666431
silesia, level 7, zstdcli, 4603429
silesia, level 9, zstdcli, 4546049
silesia, level 13, zstdcli, 4482183
silesia, level 16, zstdcli, 4377513
silesia, level 19, zstdcli, 4293378
silesia, level 16, zstdcli, 4360299
silesia, level 19, zstdcli, 4283285
silesia, long distance mode, zstdcli, 4840792
silesia, multithreaded, zstdcli, 4849600
silesia, multithreaded long distance mode, zstdcli, 4840792
silesia, small window log, zstdcli, 7111012
silesia, small hash log, zstdcli, 6555069
silesia, small chain log, zstdcli, 4931196
silesia, explicit params, zstdcli, 4797112
silesia, small hash log, zstdcli, 6526189
silesia, small chain log, zstdcli, 4912245
silesia, explicit params, zstdcli, 4795887
silesia, uncompressed literals, zstdcli, 5128030
silesia, uncompressed literals optimal, zstdcli, 4325520
silesia, uncompressed literals optimal, zstdcli, 4317944
silesia, huffman literals, zstdcli, 5331216
silesia, multithreaded with advanced params, zstdcli, 5128030
silesia.tar, level -5, zstdcli, 6738934
@@ -129,23 +129,23 @@ silesia.tar, level 0, zstdcli,
silesia.tar, level 1, zstdcli, 5336318
silesia.tar, level 3, zstdcli, 4861512
silesia.tar, level 4, zstdcli, 4800529
silesia.tar, level 5, zstdcli, 4723364
silesia.tar, level 6, zstdcli, 4673663
silesia.tar, level 7, zstdcli, 4608403
silesia.tar, level 9, zstdcli, 4554751
silesia.tar, level 5, zstdcli, 4720121
silesia.tar, level 6, zstdcli, 4678661
silesia.tar, level 7, zstdcli, 4614424
silesia.tar, level 9, zstdcli, 4556062
silesia.tar, level 13, zstdcli, 4491768
silesia.tar, level 16, zstdcli, 4381336
silesia.tar, level 19, zstdcli, 4281609
silesia.tar, level 16, zstdcli, 4356831
silesia.tar, level 19, zstdcli, 4264491
silesia.tar, no source size, zstdcli, 4861508
silesia.tar, long distance mode, zstdcli, 4853153
silesia.tar, multithreaded, zstdcli, 4861512
silesia.tar, multithreaded long distance mode, zstdcli, 4853153
silesia.tar, small window log, zstdcli, 7101576
silesia.tar, small hash log, zstdcli, 6587959
silesia.tar, small chain log, zstdcli, 4943310
silesia.tar, explicit params, zstdcli, 4822362
silesia.tar, small hash log, zstdcli, 6529290
silesia.tar, small chain log, zstdcli, 4917022
silesia.tar, explicit params, zstdcli, 4821274
silesia.tar, uncompressed literals, zstdcli, 5129559
silesia.tar, uncompressed literals optimal, zstdcli, 4320931
silesia.tar, uncompressed literals optimal, zstdcli, 4307457
silesia.tar, huffman literals, zstdcli, 5347610
silesia.tar, multithreaded with advanced params, zstdcli, 5129559
github, level -5, zstdcli, 207285
@@ -163,13 +163,13 @@ github, level 3 with dict, zstdcli,
github, level 4, zstdcli, 138199
github, level 4 with dict, zstdcli, 43251
github, level 5, zstdcli, 137121
github, level 5 with dict, zstdcli, 40741
github, level 5 with dict, zstdcli, 40728
github, level 6, zstdcli, 137122
github, level 6 with dict, zstdcli, 40632
github, level 6 with dict, zstdcli, 40630
github, level 7, zstdcli, 137122
github, level 7 with dict, zstdcli, 40771
github, level 7 with dict, zstdcli, 40747
github, level 9, zstdcli, 137122
github, level 9 with dict, zstdcli, 41332
github, level 9 with dict, zstdcli, 41338
github, level 13, zstdcli, 136064
github, level 13 with dict, zstdcli, 41743
github, level 16, zstdcli, 136064
@@ -188,27 +188,27 @@ github, uncompressed literals optimal, zstdcli,
github, huffman literals, zstdcli, 144465
github, multithreaded with advanced params, zstdcli, 167915
github.tar, level -5, zstdcli, 46751
github.tar, level -5 with dict, zstdcli, 43975
github.tar, level -5 with dict, zstdcli, 44444
github.tar, level -3, zstdcli, 43541
github.tar, level -3 with dict, zstdcli, 40809
github.tar, level -3 with dict, zstdcli, 41116
github.tar, level -1, zstdcli, 42469
github.tar, level -1 with dict, zstdcli, 41126
github.tar, level -1 with dict, zstdcli, 41200
github.tar, level 0, zstdcli, 38445
github.tar, level 0 with dict, zstdcli, 37999
github.tar, level 1, zstdcli, 39346
github.tar, level 1 with dict, zstdcli, 38313
github.tar, level 1 with dict, zstdcli, 38297
github.tar, level 3, zstdcli, 38445
github.tar, level 3 with dict, zstdcli, 37999
github.tar, level 4, zstdcli, 38471
github.tar, level 4 with dict, zstdcli, 37952
github.tar, level 5, zstdcli, 39792
github.tar, level 5 with dict, zstdcli, 39231
github.tar, level 6, zstdcli, 39607
github.tar, level 6 with dict, zstdcli, 38669
github.tar, level 7, zstdcli, 39210
github.tar, level 7 with dict, zstdcli, 37958
github.tar, level 9, zstdcli, 36721
github.tar, level 9 with dict, zstdcli, 36886
github.tar, level 5, zstdcli, 39697
github.tar, level 5 with dict, zstdcli, 39032
github.tar, level 6, zstdcli, 39625
github.tar, level 6 with dict, zstdcli, 38614
github.tar, level 7, zstdcli, 39217
github.tar, level 7 with dict, zstdcli, 37871
github.tar, level 9, zstdcli, 36762
github.tar, level 9 with dict, zstdcli, 36641
github.tar, level 13, zstdcli, 35625
github.tar, level 13 with dict, zstdcli, 38730
github.tar, level 16, zstdcli, 40259
@@ -223,7 +223,7 @@ github.tar, multithreaded long distance mode, zstdcli,
github.tar, small window log, zstdcli, 199432
github.tar, small hash log, zstdcli, 129874
github.tar, small chain log, zstdcli, 41673
github.tar, explicit params, zstdcli, 41199
github.tar, explicit params, zstdcli, 41227
github.tar, uncompressed literals, zstdcli, 41126
github.tar, uncompressed literals optimal, zstdcli, 35392
github.tar, huffman literals, zstdcli, 38804
@@ -235,23 +235,29 @@ silesia, level 0, advanced
silesia, level 1, advanced one pass, 5313204
silesia, level 3, advanced one pass, 4849552
silesia, level 4, advanced one pass, 4786970
silesia, level 5, advanced one pass, 4710236
silesia, level 6, advanced one pass, 4660056
silesia, level 7, advanced one pass, 4596296
silesia, level 9, advanced one pass, 4543925
silesia, level 5 row 1, advanced one pass, 4710236
silesia, level 5 row 2, advanced one pass, 4707794
silesia, level 5, advanced one pass, 4707794
silesia, level 6, advanced one pass, 4666383
silesia, level 7 row 1, advanced one pass, 4596296
silesia, level 7 row 2, advanced one pass, 4603381
silesia, level 7, advanced one pass, 4603381
silesia, level 9, advanced one pass, 4546001
silesia, level 12 row 1, advanced one pass, 4519288
silesia, level 12 row 2, advanced one pass, 4521397
silesia, level 13, advanced one pass, 4482135
silesia, level 16, advanced one pass, 4377465
silesia, level 19, advanced one pass, 4293330
silesia, level 16, advanced one pass, 4360251
silesia, level 19, advanced one pass, 4283237
silesia, no source size, advanced one pass, 4849552
silesia, long distance mode, advanced one pass, 4840744
silesia, multithreaded, advanced one pass, 4849552
silesia, multithreaded long distance mode, advanced one pass, 4840744
silesia, small window log, advanced one pass, 7095919
silesia, small hash log, advanced one pass, 6555021
silesia, small chain log, advanced one pass, 4931148
silesia, explicit params, advanced one pass, 4797095
silesia, small hash log, advanced one pass, 6526141
silesia, small chain log, advanced one pass, 4912197
silesia, explicit params, advanced one pass, 4795856
silesia, uncompressed literals, advanced one pass, 5127982
silesia, uncompressed literals optimal, advanced one pass, 4325472
silesia, uncompressed literals optimal, advanced one pass, 4317896
silesia, huffman literals, advanced one pass, 5326268
silesia, multithreaded with advanced params, advanced one pass, 5127982
silesia.tar, level -5, advanced one pass, 6738593
@@ -261,23 +267,29 @@ silesia.tar, level 0, advanced
silesia.tar, level 1, advanced one pass, 5334885
silesia.tar, level 3, advanced one pass, 4861425
silesia.tar, level 4, advanced one pass, 4799630
silesia.tar, level 5, advanced one pass, 4722324
silesia.tar, level 6, advanced one pass, 4672279
silesia.tar, level 7, advanced one pass, 4606715
silesia.tar, level 9, advanced one pass, 4554147
silesia.tar, level 5 row 1, advanced one pass, 4722324
silesia.tar, level 5 row 2, advanced one pass, 4719256
silesia.tar, level 5, advanced one pass, 4719256
silesia.tar, level 6, advanced one pass, 4677721
silesia.tar, level 7 row 1, advanced one pass, 4606715
silesia.tar, level 7 row 2, advanced one pass, 4613541
silesia.tar, level 7, advanced one pass, 4613541
silesia.tar, level 9, advanced one pass, 4555426
silesia.tar, level 12 row 1, advanced one pass, 4529459
silesia.tar, level 12 row 2, advanced one pass, 4530256
silesia.tar, level 13, advanced one pass, 4491764
silesia.tar, level 16, advanced one pass, 4381332
silesia.tar, level 19, advanced one pass, 4281605
silesia.tar, level 16, advanced one pass, 4356827
silesia.tar, level 19, advanced one pass, 4264487
silesia.tar, no source size, advanced one pass, 4861425
silesia.tar, long distance mode, advanced one pass, 4847735
silesia.tar, multithreaded, advanced one pass, 4861508
silesia.tar, multithreaded long distance mode, advanced one pass, 4853149
silesia.tar, small window log, advanced one pass, 7101530
silesia.tar, small hash log, advanced one pass, 6587951
silesia.tar, small chain log, advanced one pass, 4943307
silesia.tar, explicit params, advanced one pass, 4808589
silesia.tar, small hash log, advanced one pass, 6529232
silesia.tar, small chain log, advanced one pass, 4917041
silesia.tar, explicit params, advanced one pass, 4807380
silesia.tar, uncompressed literals, advanced one pass, 5129458
silesia.tar, uncompressed literals optimal, advanced one pass, 4320927
silesia.tar, uncompressed literals optimal, advanced one pass, 4307453
silesia.tar, huffman literals, advanced one pass, 5347335
silesia.tar, multithreaded with advanced params, advanced one pass, 5129555
github, level -5, advanced one pass, 205285
@@ -288,26 +300,100 @@ github, level -1, advanced
github, level -1 with dict, advanced one pass, 43170
github, level 0, advanced one pass, 136335
github, level 0 with dict, advanced one pass, 41148
github, level 0 with dict dms, advanced one pass, 41148
github, level 0 with dict dds, advanced one pass, 41148
github, level 0 with dict copy, advanced one pass, 41124
github, level 0 with dict load, advanced one pass, 42252
github, level 1, advanced one pass, 142465
github, level 1 with dict, advanced one pass, 41682
github, level 1 with dict dms, advanced one pass, 41682
github, level 1 with dict dds, advanced one pass, 41682
github, level 1 with dict copy, advanced one pass, 41674
github, level 1 with dict load, advanced one pass, 43755
github, level 3, advanced one pass, 136335
github, level 3 with dict, advanced one pass, 41148
github, level 3 with dict dms, advanced one pass, 41148
github, level 3 with dict dds, advanced one pass, 41148
github, level 3 with dict copy, advanced one pass, 41124
github, level 3 with dict load, advanced one pass, 42252
github, level 4, advanced one pass, 136199
github, level 4 with dict, advanced one pass, 41251
github, level 4 with dict dms, advanced one pass, 41251
github, level 4 with dict dds, advanced one pass, 41251
github, level 4 with dict copy, advanced one pass, 41216
github, level 4 with dict load, advanced one pass, 41159
github, level 5 row 1, advanced one pass, 135121
github, level 5 row 1 with dict dms, advanced one pass, 38938
github, level 5 row 1 with dict dds, advanced one pass, 38732
github, level 5 row 1 with dict copy, advanced one pass, 38934
github, level 5 row 1 with dict load, advanced one pass, 40725
github, level 5 row 2, advanced one pass, 134584
github, level 5 row 2 with dict dms, advanced one pass, 38758
github, level 5 row 2 with dict dds, advanced one pass, 38728
github, level 5 row 2 with dict copy, advanced one pass, 38759
github, level 5 row 2 with dict load, advanced one pass, 41518
github, level 5, advanced one pass, 135121
github, level 5 with dict, advanced one pass, 38938
github, level 5 with dict, advanced one pass, 38758
github, level 5 with dict dms, advanced one pass, 38758
github, level 5 with dict dds, advanced one pass, 38728
github, level 5 with dict copy, advanced one pass, 38759
github, level 5 with dict load, advanced one pass, 40725
github, level 6, advanced one pass, 135122
github, level 6 with dict, advanced one pass, 38632
github, level 6 with dict, advanced one pass, 38671
github, level 6 with dict dms, advanced one pass, 38671
github, level 6 with dict dds, advanced one pass, 38630
github, level 6 with dict copy, advanced one pass, 38669
github, level 6 with dict load, advanced one pass, 40695
github, level 7 row 1, advanced one pass, 135122
github, level 7 row 1 with dict dms, advanced one pass, 38771
github, level 7 row 1 with dict dds, advanced one pass, 38771
github, level 7 row 1 with dict copy, advanced one pass, 38745
github, level 7 row 1 with dict load, advanced one pass, 40695
github, level 7 row 2, advanced one pass, 134584
github, level 7 row 2 with dict dms, advanced one pass, 38758
github, level 7 row 2 with dict dds, advanced one pass, 38747
github, level 7 row 2 with dict copy, advanced one pass, 38755
github, level 7 row 2 with dict load, advanced one pass, 41030
github, level 7, advanced one pass, 135122
github, level 7 with dict, advanced one pass, 38771
github, level 7 with dict, advanced one pass, 38758
github, level 7 with dict dms, advanced one pass, 38758
github, level 7 with dict dds, advanced one pass, 38747
github, level 7 with dict copy, advanced one pass, 38755
github, level 7 with dict load, advanced one pass, 40695
github, level 9, advanced one pass, 135122
github, level 9 with dict, advanced one pass, 39332
github, level 9 with dict, advanced one pass, 39437
github, level 9 with dict dms, advanced one pass, 39437
github, level 9 with dict dds, advanced one pass, 39338
github, level 9 with dict copy, advanced one pass, 39398
github, level 9 with dict load, advanced one pass, 41710
github, level 12 row 1, advanced one pass, 134180
github, level 12 row 1 with dict dms, advanced one pass, 39677
github, level 12 row 1 with dict dds, advanced one pass, 39677
github, level 12 row 1 with dict copy, advanced one pass, 39677
github, level 12 row 1 with dict load, advanced one pass, 41166
github, level 12 row 2, advanced one pass, 134180
github, level 12 row 2 with dict dms, advanced one pass, 39677
github, level 12 row 2 with dict dds, advanced one pass, 39677
github, level 12 row 2 with dict copy, advanced one pass, 39677
github, level 12 row 2 with dict load, advanced one pass, 41166
github, level 13, advanced one pass, 134064
github, level 13 with dict, advanced one pass, 39743
github, level 13 with dict dms, advanced one pass, 39743
github, level 13 with dict dds, advanced one pass, 39743
github, level 13 with dict copy, advanced one pass, 39948
github, level 13 with dict load, advanced one pass, 42626
github, level 16, advanced one pass, 134064
github, level 16 with dict, advanced one pass, 37577
github, level 16 with dict dms, advanced one pass, 37577
github, level 16 with dict dds, advanced one pass, 37577
github, level 16 with dict copy, advanced one pass, 37568
github, level 16 with dict load, advanced one pass, 42340
github, level 19, advanced one pass, 134064
github, level 19 with dict, advanced one pass, 37576
github, level 19 with dict dms, advanced one pass, 37576
github, level 19 with dict dds, advanced one pass, 37576
github, level 19 with dict copy, advanced one pass, 37567
github, level 19 with dict load, advanced one pass, 39613
github, no source size, advanced one pass, 136335
github, no source size with dict, advanced one pass, 41148
github, long distance mode, advanced one pass, 136335
@@ -322,33 +408,107 @@ github, uncompressed literals optimal, advanced
github, huffman literals, advanced one pass, 142465
github, multithreaded with advanced params, advanced one pass, 165915
github.tar, level -5, advanced one pass, 46856
github.tar, level -5 with dict, advanced one pass, 43971
github.tar, level -5 with dict, advanced one pass, 44571
github.tar, level -3, advanced one pass, 43754
github.tar, level -3 with dict, advanced one pass, 40805
github.tar, level -3 with dict, advanced one pass, 41447
github.tar, level -1, advanced one pass, 42490
github.tar, level -1 with dict, advanced one pass, 41122
github.tar, level -1 with dict, advanced one pass, 41131
github.tar, level 0, advanced one pass, 38441
github.tar, level 0 with dict, advanced one pass, 37995
github.tar, level 0 with dict dms, advanced one pass, 38003
github.tar, level 0 with dict dds, advanced one pass, 38003
github.tar, level 0 with dict copy, advanced one pass, 37995
github.tar, level 0 with dict load, advanced one pass, 37956
github.tar, level 1, advanced one pass, 39265
github.tar, level 1 with dict, advanced one pass, 38309
github.tar, level 1 with dict, advanced one pass, 38280
github.tar, level 1 with dict dms, advanced one pass, 38290
github.tar, level 1 with dict dds, advanced one pass, 38290
github.tar, level 1 with dict copy, advanced one pass, 38280
github.tar, level 1 with dict load, advanced one pass, 38729
github.tar, level 3, advanced one pass, 38441
github.tar, level 3 with dict, advanced one pass, 37995
github.tar, level 3 with dict dms, advanced one pass, 38003
github.tar, level 3 with dict dds, advanced one pass, 38003
github.tar, level 3 with dict copy, advanced one pass, 37995
github.tar, level 3 with dict load, advanced one pass, 37956
github.tar, level 4, advanced one pass, 38467
github.tar, level 4 with dict, advanced one pass, 37948
github.tar, level 5, advanced one pass, 39788
github.tar, level 5 with dict, advanced one pass, 39715
github.tar, level 6, advanced one pass, 39603
github.tar, level 6 with dict, advanced one pass, 38800
github.tar, level 7, advanced one pass, 39206
github.tar, level 7 with dict, advanced one pass, 38071
github.tar, level 9, advanced one pass, 36717
github.tar, level 9 with dict, advanced one pass, 36898
github.tar, level 4 with dict dms, advanced one pass, 37954
github.tar, level 4 with dict dds, advanced one pass, 37954
github.tar, level 4 with dict copy, advanced one pass, 37948
github.tar, level 4 with dict load, advanced one pass, 37927
github.tar, level 5 row 1, advanced one pass, 39788
github.tar, level 5 row 1 with dict dms, advanced one pass, 39365
github.tar, level 5 row 1 with dict dds, advanced one pass, 39233
github.tar, level 5 row 1 with dict copy, advanced one pass, 39715
github.tar, level 5 row 1 with dict load, advanced one pass, 39209
github.tar, level 5 row 2, advanced one pass, 39693
github.tar, level 5 row 2 with dict dms, advanced one pass, 39024
github.tar, level 5 row 2 with dict dds, advanced one pass, 39028
github.tar, level 5 row 2 with dict copy, advanced one pass, 39040
github.tar, level 5 row 2 with dict load, advanced one pass, 39037
github.tar, level 5, advanced one pass, 39693
github.tar, level 5 with dict, advanced one pass, 39040
github.tar, level 5 with dict dms, advanced one pass, 39024
github.tar, level 5 with dict dds, advanced one pass, 39028
github.tar, level 5 with dict copy, advanced one pass, 39040
github.tar, level 5 with dict load, advanced one pass, 39037
github.tar, level 6, advanced one pass, 39621
github.tar, level 6 with dict, advanced one pass, 38622
github.tar, level 6 with dict dms, advanced one pass, 38608
github.tar, level 6 with dict dds, advanced one pass, 38610
github.tar, level 6 with dict copy, advanced one pass, 38622
github.tar, level 6 with dict load, advanced one pass, 38962
github.tar, level 7 row 1, advanced one pass, 39206
github.tar, level 7 row 1 with dict dms, advanced one pass, 37954
github.tar, level 7 row 1 with dict dds, advanced one pass, 37954
github.tar, level 7 row 1 with dict copy, advanced one pass, 38071
github.tar, level 7 row 1 with dict load, advanced one pass, 38584
github.tar, level 7 row 2, advanced one pass, 39213
github.tar, level 7 row 2 with dict dms, advanced one pass, 37848
github.tar, level 7 row 2 with dict dds, advanced one pass, 37867
github.tar, level 7 row 2 with dict copy, advanced one pass, 37848
github.tar, level 7 row 2 with dict load, advanced one pass, 38582
github.tar, level 7, advanced one pass, 39213
github.tar, level 7 with dict, advanced one pass, 37848
github.tar, level 7 with dict dms, advanced one pass, 37848
github.tar, level 7 with dict dds, advanced one pass, 37867
github.tar, level 7 with dict copy, advanced one pass, 37848
github.tar, level 7 with dict load, advanced one pass, 38582
github.tar, level 9, advanced one pass, 36758
github.tar, level 9 with dict, advanced one pass, 36457
github.tar, level 9 with dict dms, advanced one pass, 36549
github.tar, level 9 with dict dds, advanced one pass, 36637
github.tar, level 9 with dict copy, advanced one pass, 36457
github.tar, level 9 with dict load, advanced one pass, 36350
github.tar, level 12 row 1, advanced one pass, 36435
github.tar, level 12 row 1 with dict dms, advanced one pass, 36986
github.tar, level 12 row 1 with dict dds, advanced one pass, 36986
github.tar, level 12 row 1 with dict copy, advanced one pass, 36609
github.tar, level 12 row 1 with dict load, advanced one pass, 36419
github.tar, level 12 row 2, advanced one pass, 36435
github.tar, level 12 row 2 with dict dms, advanced one pass, 36986
github.tar, level 12 row 2 with dict dds, advanced one pass, 36986
github.tar, level 12 row 2 with dict copy, advanced one pass, 36609
github.tar, level 12 row 2 with dict load, advanced one pass, 36424
github.tar, level 13, advanced one pass, 35621
github.tar, level 13 with dict, advanced one pass, 38726
github.tar, level 13 with dict dms, advanced one pass, 38903
github.tar, level 13 with dict dds, advanced one pass, 38903
github.tar, level 13 with dict copy, advanced one pass, 38726
github.tar, level 13 with dict load, advanced one pass, 36372
github.tar, level 16, advanced one pass, 40255
github.tar, level 16 with dict, advanced one pass, 33639
github.tar, level 16 with dict dms, advanced one pass, 33544
github.tar, level 16 with dict dds, advanced one pass, 33544
github.tar, level 16 with dict copy, advanced one pass, 33639
github.tar, level 16 with dict load, advanced one pass, 39353
github.tar, level 19, advanced one pass, 32837
github.tar, level 19 with dict, advanced one pass, 32895
github.tar, level 19 with dict dms, advanced one pass, 32672
github.tar, level 19 with dict dds, advanced one pass, 32672
github.tar, level 19 with dict copy, advanced one pass, 32895
github.tar, level 19 with dict load, advanced one pass, 32676
github.tar, no source size, advanced one pass, 38441
github.tar, no source size with dict, advanced one pass, 37995
github.tar, long distance mode, advanced one pass, 39722
@@ -357,7 +517,7 @@ github.tar, multithreaded long distance mode, advanced
github.tar, small window log, advanced one pass, 198540
github.tar, small hash log, advanced one pass, 129870
github.tar, small chain log, advanced one pass, 41669
github.tar, explicit params, advanced one pass, 41199
github.tar, explicit params, advanced one pass, 41227
github.tar, uncompressed literals, advanced one pass, 41122
github.tar, uncompressed literals optimal, advanced one pass, 35388
github.tar, huffman literals, advanced one pass, 38777
@@ -369,23 +529,29 @@ silesia, level 0, advanced
silesia, level 1, advanced one pass small out, 5313204
silesia, level 3, advanced one pass small out, 4849552
silesia, level 4, advanced one pass small out, 4786970
silesia, level 5, advanced one pass small out, 4710236
silesia, level 6, advanced one pass small out, 4660056
silesia, level 7, advanced one pass small out, 4596296
silesia, level 9, advanced one pass small out, 4543925
silesia, level 5 row 1, advanced one pass small out, 4710236
silesia, level 5 row 2, advanced one pass small out, 4707794
silesia, level 5, advanced one pass small out, 4707794
silesia, level 6, advanced one pass small out, 4666383
silesia, level 7 row 1, advanced one pass small out, 4596296
silesia, level 7 row 2, advanced one pass small out, 4603381
silesia, level 7, advanced one pass small out, 4603381
silesia, level 9, advanced one pass small out, 4546001
silesia, level 12 row 1, advanced one pass small out, 4519288
silesia, level 12 row 2, advanced one pass small out, 4521397
silesia, level 13, advanced one pass small out, 4482135
silesia, level 16, advanced one pass small out, 4377465
silesia, level 19, advanced one pass small out, 4293330
silesia, level 16, advanced one pass small out, 4360251
silesia, level 19, advanced one pass small out, 4283237
silesia, no source size, advanced one pass small out, 4849552
silesia, long distance mode, advanced one pass small out, 4840744
silesia, multithreaded, advanced one pass small out, 4849552
silesia, multithreaded long distance mode, advanced one pass small out, 4840744
silesia, small window log, advanced one pass small out, 7095919
silesia, small hash log, advanced one pass small out, 6555021
silesia, small chain log, advanced one pass small out, 4931148
silesia, explicit params, advanced one pass small out, 4797095
silesia, small hash log, advanced one pass small out, 6526141
silesia, small chain log, advanced one pass small out, 4912197
silesia, explicit params, advanced one pass small out, 4795856
silesia, uncompressed literals, advanced one pass small out, 5127982
silesia, uncompressed literals optimal, advanced one pass small out, 4325472
silesia, uncompressed literals optimal, advanced one pass small out, 4317896
silesia, huffman literals, advanced one pass small out, 5326268
silesia, multithreaded with advanced params, advanced one pass small out, 5127982
silesia.tar, level -5, advanced one pass small out, 6738593
@@ -395,23 +561,29 @@ silesia.tar, level 0, advanced
silesia.tar, level 1, advanced one pass small out, 5334885
silesia.tar, level 3, advanced one pass small out, 4861425
silesia.tar, level 4, advanced one pass small out, 4799630
silesia.tar, level 5, advanced one pass small out, 4722324
silesia.tar, level 6, advanced one pass small out, 4672279
silesia.tar, level 7, advanced one pass small out, 4606715
silesia.tar, level 9, advanced one pass small out, 4554147
silesia.tar, level 5 row 1, advanced one pass small out, 4722324
silesia.tar, level 5 row 2, advanced one pass small out, 4719256
silesia.tar, level 5, advanced one pass small out, 4719256
silesia.tar, level 6, advanced one pass small out, 4677721
silesia.tar, level 7 row 1, advanced one pass small out, 4606715
silesia.tar, level 7 row 2, advanced one pass small out, 4613541
silesia.tar, level 7, advanced one pass small out, 4613541
silesia.tar, level 9, advanced one pass small out, 4555426
silesia.tar, level 12 row 1, advanced one pass small out, 4529459
silesia.tar, level 12 row 2, advanced one pass small out, 4530256
silesia.tar, level 13, advanced one pass small out, 4491764
silesia.tar, level 16, advanced one pass small out, 4381332
silesia.tar, level 19, advanced one pass small out, 4281605
silesia.tar, level 16, advanced one pass small out, 4356827
silesia.tar, level 19, advanced one pass small out, 4264487
silesia.tar, no source size, advanced one pass small out, 4861425
silesia.tar, long distance mode, advanced one pass small out, 4847735
silesia.tar, multithreaded, advanced one pass small out, 4861508
silesia.tar, multithreaded long distance mode, advanced one pass small out, 4853149
silesia.tar, small window log, advanced one pass small out, 7101530
silesia.tar, small hash log, advanced one pass small out, 6587951
silesia.tar, small chain log, advanced one pass small out, 4943307
silesia.tar, explicit params, advanced one pass small out, 4808589
silesia.tar, small hash log, advanced one pass small out, 6529232
silesia.tar, small chain log, advanced one pass small out, 4917041
silesia.tar, explicit params, advanced one pass small out, 4807380
silesia.tar, uncompressed literals, advanced one pass small out, 5129458
silesia.tar, uncompressed literals optimal, advanced one pass small out, 4320927
silesia.tar, uncompressed literals optimal, advanced one pass small out, 4307453
silesia.tar, huffman literals, advanced one pass small out, 5347335
silesia.tar, multithreaded with advanced params, advanced one pass small out, 5129555
github, level -5, advanced one pass small out, 205285
@@ -422,26 +594,100 @@ github, level -1, advanced
github, level -1 with dict, advanced one pass small out, 43170
github, level 0, advanced one pass small out, 136335
github, level 0 with dict, advanced one pass small out, 41148
github, level 0 with dict dms, advanced one pass small out, 41148
github, level 0 with dict dds, advanced one pass small out, 41148
github, level 0 with dict copy, advanced one pass small out, 41124
github, level 0 with dict load, advanced one pass small out, 42252
github, level 1, advanced one pass small out, 142465
github, level 1 with dict, advanced one pass small out, 41682
github, level 1 with dict dms, advanced one pass small out, 41682
github, level 1 with dict dds, advanced one pass small out, 41682
github, level 1 with dict copy, advanced one pass small out, 41674
github, level 1 with dict load, advanced one pass small out, 43755
github, level 3, advanced one pass small out, 136335
github, level 3 with dict, advanced one pass small out, 41148
github, level 3 with dict dms, advanced one pass small out, 41148
github, level 3 with dict dds, advanced one pass small out, 41148
github, level 3 with dict copy, advanced one pass small out, 41124
github, level 3 with dict load, advanced one pass small out, 42252
github, level 4, advanced one pass small out, 136199
github, level 4 with dict, advanced one pass small out, 41251
github, level 4 with dict dms, advanced one pass small out, 41251
github, level 4 with dict dds, advanced one pass small out, 41251
github, level 4 with dict copy, advanced one pass small out, 41216
github, level 4 with dict load, advanced one pass small out, 41159
github, level 5 row 1, advanced one pass small out, 135121
github, level 5 row 1 with dict dms, advanced one pass small out, 38938
github, level 5 row 1 with dict dds, advanced one pass small out, 38732
github, level 5 row 1 with dict copy, advanced one pass small out, 38934
github, level 5 row 1 with dict load, advanced one pass small out, 40725
github, level 5 row 2, advanced one pass small out, 134584
github, level 5 row 2 with dict dms, advanced one pass small out, 38758
github, level 5 row 2 with dict dds, advanced one pass small out, 38728
github, level 5 row 2 with dict copy, advanced one pass small out, 38759
github, level 5 row 2 with dict load, advanced one pass small out, 41518
github, level 5, advanced one pass small out, 135121
github, level 5 with dict, advanced one pass small out, 38938
github, level 5 with dict, advanced one pass small out, 38758
github, level 5 with dict dms, advanced one pass small out, 38758
github, level 5 with dict dds, advanced one pass small out, 38728
github, level 5 with dict copy, advanced one pass small out, 38759
github, level 5 with dict load, advanced one pass small out, 40725
github, level 6, advanced one pass small out, 135122
github, level 6 with dict, advanced one pass small out, 38632
github, level 6 with dict, advanced one pass small out, 38671
github, level 6 with dict dms, advanced one pass small out, 38671
github, level 6 with dict dds, advanced one pass small out, 38630
github, level 6 with dict copy, advanced one pass small out, 38669
github, level 6 with dict load, advanced one pass small out, 40695
github, level 7 row 1, advanced one pass small out, 135122
github, level 7 row 1 with dict dms, advanced one pass small out, 38771
github, level 7 row 1 with dict dds, advanced one pass small out, 38771
github, level 7 row 1 with dict copy, advanced one pass small out, 38745
github, level 7 row 1 with dict load, advanced one pass small out, 40695
github, level 7 row 2, advanced one pass small out, 134584
github, level 7 row 2 with dict dms, advanced one pass small out, 38758
github, level 7 row 2 with dict dds, advanced one pass small out, 38747
github, level 7 row 2 with dict copy, advanced one pass small out, 38755
github, level 7 row 2 with dict load, advanced one pass small out, 41030
github, level 7, advanced one pass small out, 135122
github, level 7 with dict, advanced one pass small out, 38771
github, level 7 with dict, advanced one pass small out, 38758
github, level 7 with dict dms, advanced one pass small out, 38758
github, level 7 with dict dds, advanced one pass small out, 38747
github, level 7 with dict copy, advanced one pass small out, 38755
github, level 7 with dict load, advanced one pass small out, 40695
github, level 9, advanced one pass small out, 135122
github, level 9 with dict, advanced one pass small out, 39332
github, level 9 with dict, advanced one pass small out, 39437
github, level 9 with dict dms, advanced one pass small out, 39437
github, level 9 with dict dds, advanced one pass small out, 39338
github, level 9 with dict copy, advanced one pass small out, 39398
github, level 9 with dict load, advanced one pass small out, 41710
github, level 12 row 1, advanced one pass small out, 134180
github, level 12 row 1 with dict dms, advanced one pass small out, 39677
github, level 12 row 1 with dict dds, advanced one pass small out, 39677
github, level 12 row 1 with dict copy, advanced one pass small out, 39677
github, level 12 row 1 with dict load, advanced one pass small out, 41166
github, level 12 row 2, advanced one pass small out, 134180
github, level 12 row 2 with dict dms, advanced one pass small out, 39677
github, level 12 row 2 with dict dds, advanced one pass small out, 39677
github, level 12 row 2 with dict copy, advanced one pass small out, 39677
github, level 12 row 2 with dict load, advanced one pass small out, 41166
github, level 13, advanced one pass small out, 134064
github, level 13 with dict, advanced one pass small out, 39743
github, level 13 with dict dms, advanced one pass small out, 39743
github, level 13 with dict dds, advanced one pass small out, 39743
github, level 13 with dict copy, advanced one pass small out, 39948
github, level 13 with dict load, advanced one pass small out, 42626
github, level 16, advanced one pass small out, 134064
github, level 16 with dict, advanced one pass small out, 37577
github, level 16 with dict dms, advanced one pass small out, 37577
github, level 16 with dict dds, advanced one pass small out, 37577
github, level 16 with dict copy, advanced one pass small out, 37568
github, level 16 with dict load, advanced one pass small out, 42340
github, level 19, advanced one pass small out, 134064
github, level 19 with dict, advanced one pass small out, 37576
github, level 19 with dict dms, advanced one pass small out, 37576
github, level 19 with dict dds, advanced one pass small out, 37576
github, level 19 with dict copy, advanced one pass small out, 37567
github, level 19 with dict load, advanced one pass small out, 39613
github, no source size, advanced one pass small out, 136335
github, no source size with dict, advanced one pass small out, 41148
github, long distance mode, advanced one pass small out, 136335
@@ -456,33 +702,107 @@ github, uncompressed literals optimal, advanced
github, huffman literals, advanced one pass small out, 142465
github, multithreaded with advanced params, advanced one pass small out, 165915
github.tar, level -5, advanced one pass small out, 46856
github.tar, level -5 with dict, advanced one pass small out, 43971
github.tar, level -5 with dict, advanced one pass small out, 44571
github.tar, level -3, advanced one pass small out, 43754
github.tar, level -3 with dict, advanced one pass small out, 40805
github.tar, level -3 with dict, advanced one pass small out, 41447
github.tar, level -1, advanced one pass small out, 42490
github.tar, level -1 with dict, advanced one pass small out, 41122
github.tar, level -1 with dict, advanced one pass small out, 41131
github.tar, level 0, advanced one pass small out, 38441
github.tar, level 0 with dict, advanced one pass small out, 37995
github.tar, level 0 with dict dms, advanced one pass small out, 38003
github.tar, level 0 with dict dds, advanced one pass small out, 38003
github.tar, level 0 with dict copy, advanced one pass small out, 37995
github.tar, level 0 with dict load, advanced one pass small out, 37956
github.tar, level 1, advanced one pass small out, 39265
github.tar, level 1 with dict, advanced one pass small out, 38309
github.tar, level 1 with dict, advanced one pass small out, 38280
github.tar, level 1 with dict dms, advanced one pass small out, 38290
github.tar, level 1 with dict dds, advanced one pass small out, 38290
github.tar, level 1 with dict copy, advanced one pass small out, 38280
github.tar, level 1 with dict load, advanced one pass small out, 38729
github.tar, level 3, advanced one pass small out, 38441
github.tar, level 3 with dict, advanced one pass small out, 37995
github.tar, level 3 with dict dms, advanced one pass small out, 38003
github.tar, level 3 with dict dds, advanced one pass small out, 38003
github.tar, level 3 with dict copy, advanced one pass small out, 37995
github.tar, level 3 with dict load, advanced one pass small out, 37956
github.tar, level 4, advanced one pass small out, 38467
github.tar, level 4 with dict, advanced one pass small out, 37948
github.tar, level 5, advanced one pass small out, 39788
github.tar, level 5 with dict, advanced one pass small out, 39715
github.tar, level 6, advanced one pass small out, 39603
github.tar, level 6 with dict, advanced one pass small out, 38800
github.tar, level 7, advanced one pass small out, 39206
github.tar, level 7 with dict, advanced one pass small out, 38071
github.tar, level 9, advanced one pass small out, 36717
github.tar, level 9 with dict, advanced one pass small out, 36898
github.tar, level 4 with dict dms, advanced one pass small out, 37954
github.tar, level 4 with dict dds, advanced one pass small out, 37954
github.tar, level 4 with dict copy, advanced one pass small out, 37948
github.tar, level 4 with dict load, advanced one pass small out, 37927
github.tar, level 5 row 1, advanced one pass small out, 39788
github.tar, level 5 row 1 with dict dms, advanced one pass small out, 39365
github.tar, level 5 row 1 with dict dds, advanced one pass small out, 39233
github.tar, level 5 row 1 with dict copy, advanced one pass small out, 39715
github.tar, level 5 row 1 with dict load, advanced one pass small out, 39209
github.tar, level 5 row 2, advanced one pass small out, 39693
github.tar, level 5 row 2 with dict dms, advanced one pass small out, 39024
github.tar, level 5 row 2 with dict dds, advanced one pass small out, 39028
github.tar, level 5 row 2 with dict copy, advanced one pass small out, 39040
github.tar, level 5 row 2 with dict load, advanced one pass small out, 39037
github.tar, level 5, advanced one pass small out, 39693
github.tar, level 5 with dict, advanced one pass small out, 39040
github.tar, level 5 with dict dms, advanced one pass small out, 39024
github.tar, level 5 with dict dds, advanced one pass small out, 39028
github.tar, level 5 with dict copy, advanced one pass small out, 39040
github.tar, level 5 with dict load, advanced one pass small out, 39037
github.tar, level 6, advanced one pass small out, 39621
github.tar, level 6 with dict, advanced one pass small out, 38622
github.tar, level 6 with dict dms, advanced one pass small out, 38608
github.tar, level 6 with dict dds, advanced one pass small out, 38610
github.tar, level 6 with dict copy, advanced one pass small out, 38622
github.tar, level 6 with dict load, advanced one pass small out, 38962
github.tar, level 7 row 1, advanced one pass small out, 39206
github.tar, level 7 row 1 with dict dms, advanced one pass small out, 37954
github.tar, level 7 row 1 with dict dds, advanced one pass small out, 37954
github.tar, level 7 row 1 with dict copy, advanced one pass small out, 38071
github.tar, level 7 row 1 with dict load, advanced one pass small out, 38584
github.tar, level 7 row 2, advanced one pass small out, 39213
github.tar, level 7 row 2 with dict dms, advanced one pass small out, 37848
github.tar, level 7 row 2 with dict dds, advanced one pass small out, 37867
github.tar, level 7 row 2 with dict copy, advanced one pass small out, 37848
github.tar, level 7 row 2 with dict load, advanced one pass small out, 38582
github.tar, level 7, advanced one pass small out, 39213
github.tar, level 7 with dict, advanced one pass small out, 37848
github.tar, level 7 with dict dms, advanced one pass small out, 37848
github.tar, level 7 with dict dds, advanced one pass small out, 37867
github.tar, level 7 with dict copy, advanced one pass small out, 37848
github.tar, level 7 with dict load, advanced one pass small out, 38582
github.tar, level 9, advanced one pass small out, 36758
github.tar, level 9 with dict, advanced one pass small out, 36457
github.tar, level 9 with dict dms, advanced one pass small out, 36549
github.tar, level 9 with dict dds, advanced one pass small out, 36637
github.tar, level 9 with dict copy, advanced one pass small out, 36457
github.tar, level 9 with dict load, advanced one pass small out, 36350
github.tar, level 12 row 1, advanced one pass small out, 36435
github.tar, level 12 row 1 with dict dms, advanced one pass small out, 36986
github.tar, level 12 row 1 with dict dds, advanced one pass small out, 36986
github.tar, level 12 row 1 with dict copy, advanced one pass small out, 36609
github.tar, level 12 row 1 with dict load, advanced one pass small out, 36419
github.tar, level 12 row 2, advanced one pass small out, 36435
github.tar, level 12 row 2 with dict dms, advanced one pass small out, 36986
github.tar, level 12 row 2 with dict dds, advanced one pass small out, 36986
github.tar, level 12 row 2 with dict copy, advanced one pass small out, 36609
github.tar, level 12 row 2 with dict load, advanced one pass small out, 36424
github.tar, level 13, advanced one pass small out, 35621
github.tar, level 13 with dict, advanced one pass small out, 38726
github.tar, level 13 with dict dms, advanced one pass small out, 38903
github.tar, level 13 with dict dds, advanced one pass small out, 38903
github.tar, level 13 with dict copy, advanced one pass small out, 38726
github.tar, level 13 with dict load, advanced one pass small out, 36372
github.tar, level 16, advanced one pass small out, 40255
github.tar, level 16 with dict, advanced one pass small out, 33639
github.tar, level 16 with dict dms, advanced one pass small out, 33544
github.tar, level 16 with dict dds, advanced one pass small out, 33544
github.tar, level 16 with dict copy, advanced one pass small out, 33639
github.tar, level 16 with dict load, advanced one pass small out, 39353
github.tar, level 19, advanced one pass small out, 32837
github.tar, level 19 with dict, advanced one pass small out, 32895
github.tar, level 19 with dict dms, advanced one pass small out, 32672
github.tar, level 19 with dict dds, advanced one pass small out, 32672
github.tar, level 19 with dict copy, advanced one pass small out, 32895
github.tar, level 19 with dict load, advanced one pass small out, 32676
github.tar, no source size, advanced one pass small out, 38441
github.tar, no source size with dict, advanced one pass small out, 37995
github.tar, long distance mode, advanced one pass small out, 39722
@@ -491,7 +811,7 @@ github.tar, multithreaded long distance mode, advanced
github.tar, small window log, advanced one pass small out, 198540
github.tar, small hash log, advanced one pass small out, 129870
github.tar, small chain log, advanced one pass small out, 41669
github.tar, explicit params, advanced one pass small out, 41199
github.tar, explicit params, advanced one pass small out, 41227
github.tar, uncompressed literals, advanced one pass small out, 41122
github.tar, uncompressed literals optimal, advanced one pass small out, 35388
github.tar, huffman literals, advanced one pass small out, 38777
@@ -503,23 +823,29 @@ silesia, level 0, advanced
silesia, level 1, advanced streaming, 5314162
silesia, level 3, advanced streaming, 4849552
silesia, level 4, advanced streaming, 4786970
silesia, level 5, advanced streaming, 4710236
silesia, level 6, advanced streaming, 4660056
silesia, level 7, advanced streaming, 4596296
silesia, level 9, advanced streaming, 4543925
silesia, level 5 row 1, advanced streaming, 4710236
silesia, level 5 row 2, advanced streaming, 4707794
silesia, level 5, advanced streaming, 4707794
silesia, level 6, advanced streaming, 4666383
silesia, level 7 row 1, advanced streaming, 4596296
silesia, level 7 row 2, advanced streaming, 4603381
silesia, level 7, advanced streaming, 4603381
silesia, level 9, advanced streaming, 4546001
silesia, level 12 row 1, advanced streaming, 4519288
silesia, level 12 row 2, advanced streaming, 4521397
silesia, level 13, advanced streaming, 4482135
silesia, level 16, advanced streaming, 4377465
silesia, level 19, advanced streaming, 4293330
silesia, level 16, advanced streaming, 4360251
silesia, level 19, advanced streaming, 4283237
silesia, no source size, advanced streaming, 4849516
silesia, long distance mode, advanced streaming, 4840744
silesia, multithreaded, advanced streaming, 4849552
silesia, multithreaded long distance mode, advanced streaming, 4840744
silesia, small window log, advanced streaming, 7112062
silesia, small hash log, advanced streaming, 6555021
silesia, small chain log, advanced streaming, 4931148
silesia, explicit params, advanced streaming, 4797112
silesia, small hash log, advanced streaming, 6526141
silesia, small chain log, advanced streaming, 4912197
silesia, explicit params, advanced streaming, 4795887
silesia, uncompressed literals, advanced streaming, 5127982
silesia, uncompressed literals optimal, advanced streaming, 4325472
silesia, uncompressed literals optimal, advanced streaming, 4317896
silesia, huffman literals, advanced streaming, 5331168
silesia, multithreaded with advanced params, advanced streaming, 5127982
silesia.tar, level -5, advanced streaming, 6982759
@@ -529,23 +855,29 @@ silesia.tar, level 0, advanced
silesia.tar, level 1, advanced streaming, 5336939
silesia.tar, level 3, advanced streaming, 4861427
silesia.tar, level 4, advanced streaming, 4799630
silesia.tar, level 5, advanced streaming, 4722329
silesia.tar, level 6, advanced streaming, 4672288
silesia.tar, level 7, advanced streaming, 4606715
silesia.tar, level 9, advanced streaming, 4554154
silesia.tar, level 5 row 1, advanced streaming, 4722329
silesia.tar, level 5 row 2, advanced streaming, 4719261
silesia.tar, level 5, advanced streaming, 4719261
silesia.tar, level 6, advanced streaming, 4677729
silesia.tar, level 7 row 1, advanced streaming, 4606715
silesia.tar, level 7 row 2, advanced streaming, 4613544
silesia.tar, level 7, advanced streaming, 4613544
silesia.tar, level 9, advanced streaming, 4555432
silesia.tar, level 12 row 1, advanced streaming, 4529459
silesia.tar, level 12 row 2, advanced streaming, 4530258
silesia.tar, level 13, advanced streaming, 4491765
silesia.tar, level 16, advanced streaming, 4381350
silesia.tar, level 19, advanced streaming, 4281562
silesia.tar, level 16, advanced streaming, 4356834
silesia.tar, level 19, advanced streaming, 4264392
silesia.tar, no source size, advanced streaming, 4861423
silesia.tar, long distance mode, advanced streaming, 4847735
silesia.tar, multithreaded, advanced streaming, 4861508
silesia.tar, multithreaded long distance mode, advanced streaming, 4853149
silesia.tar, small window log, advanced streaming, 7118769
silesia.tar, small hash log, advanced streaming, 6587952
silesia.tar, small chain log, advanced streaming, 4943312
silesia.tar, explicit params, advanced streaming, 4808618
silesia.tar, small hash log, advanced streaming, 6529235
silesia.tar, small chain log, advanced streaming, 4917021
silesia.tar, explicit params, advanced streaming, 4807401
silesia.tar, uncompressed literals, advanced streaming, 5129461
silesia.tar, uncompressed literals optimal, advanced streaming, 4320858
silesia.tar, uncompressed literals optimal, advanced streaming, 4307400
silesia.tar, huffman literals, advanced streaming, 5352360
silesia.tar, multithreaded with advanced params, advanced streaming, 5129555
github, level -5, advanced streaming, 205285
@@ -556,26 +888,100 @@ github, level -1, advanced
github, level -1 with dict, advanced streaming, 43170
github, level 0, advanced streaming, 136335
github, level 0 with dict, advanced streaming, 41148
github, level 0 with dict dms, advanced streaming, 41148
github, level 0 with dict dds, advanced streaming, 41148
github, level 0 with dict copy, advanced streaming, 41124
github, level 0 with dict load, advanced streaming, 42252
github, level 1, advanced streaming, 142465
github, level 1 with dict, advanced streaming, 41682
github, level 1 with dict dms, advanced streaming, 41682
github, level 1 with dict dds, advanced streaming, 41682
github, level 1 with dict copy, advanced streaming, 41674
github, level 1 with dict load, advanced streaming, 43755
github, level 3, advanced streaming, 136335
github, level 3 with dict, advanced streaming, 41148
github, level 3 with dict dms, advanced streaming, 41148
github, level 3 with dict dds, advanced streaming, 41148
github, level 3 with dict copy, advanced streaming, 41124
github, level 3 with dict load, advanced streaming, 42252
github, level 4, advanced streaming, 136199
github, level 4 with dict, advanced streaming, 41251
github, level 4 with dict dms, advanced streaming, 41251
github, level 4 with dict dds, advanced streaming, 41251
github, level 4 with dict copy, advanced streaming, 41216
github, level 4 with dict load, advanced streaming, 41159
github, level 5 row 1, advanced streaming, 135121
github, level 5 row 1 with dict dms, advanced streaming, 38938
github, level 5 row 1 with dict dds, advanced streaming, 38732
github, level 5 row 1 with dict copy, advanced streaming, 38934
github, level 5 row 1 with dict load, advanced streaming, 40725
github, level 5 row 2, advanced streaming, 134584
github, level 5 row 2 with dict dms, advanced streaming, 38758
github, level 5 row 2 with dict dds, advanced streaming, 38728
github, level 5 row 2 with dict copy, advanced streaming, 38759
github, level 5 row 2 with dict load, advanced streaming, 41518
github, level 5, advanced streaming, 135121
github, level 5 with dict, advanced streaming, 38938
github, level 5 with dict, advanced streaming, 38758
github, level 5 with dict dms, advanced streaming, 38758
github, level 5 with dict dds, advanced streaming, 38728
github, level 5 with dict copy, advanced streaming, 38759
github, level 5 with dict load, advanced streaming, 40725
github, level 6, advanced streaming, 135122
github, level 6 with dict, advanced streaming, 38632
github, level 6 with dict, advanced streaming, 38671
github, level 6 with dict dms, advanced streaming, 38671
github, level 6 with dict dds, advanced streaming, 38630
github, level 6 with dict copy, advanced streaming, 38669
github, level 6 with dict load, advanced streaming, 40695
github, level 7 row 1, advanced streaming, 135122
github, level 7 row 1 with dict dms, advanced streaming, 38771
github, level 7 row 1 with dict dds, advanced streaming, 38771
github, level 7 row 1 with dict copy, advanced streaming, 38745
github, level 7 row 1 with dict load, advanced streaming, 40695
github, level 7 row 2, advanced streaming, 134584
github, level 7 row 2 with dict dms, advanced streaming, 38758
github, level 7 row 2 with dict dds, advanced streaming, 38747
github, level 7 row 2 with dict copy, advanced streaming, 38755
github, level 7 row 2 with dict load, advanced streaming, 41030
github, level 7, advanced streaming, 135122
github, level 7 with dict, advanced streaming, 38771
github, level 7 with dict, advanced streaming, 38758
github, level 7 with dict dms, advanced streaming, 38758
github, level 7 with dict dds, advanced streaming, 38747
github, level 7 with dict copy, advanced streaming, 38755
github, level 7 with dict load, advanced streaming, 40695
github, level 9, advanced streaming, 135122
github, level 9 with dict, advanced streaming, 39332
github, level 9 with dict, advanced streaming, 39437
github, level 9 with dict dms, advanced streaming, 39437
github, level 9 with dict dds, advanced streaming, 39338
github, level 9 with dict copy, advanced streaming, 39398
github, level 9 with dict load, advanced streaming, 41710
github, level 12 row 1, advanced streaming, 134180
github, level 12 row 1 with dict dms, advanced streaming, 39677
github, level 12 row 1 with dict dds, advanced streaming, 39677
github, level 12 row 1 with dict copy, advanced streaming, 39677
github, level 12 row 1 with dict load, advanced streaming, 41166
github, level 12 row 2, advanced streaming, 134180
github, level 12 row 2 with dict dms, advanced streaming, 39677
github, level 12 row 2 with dict dds, advanced streaming, 39677
github, level 12 row 2 with dict copy, advanced streaming, 39677
github, level 12 row 2 with dict load, advanced streaming, 41166
github, level 13, advanced streaming, 134064
github, level 13 with dict, advanced streaming, 39743
github, level 13 with dict dms, advanced streaming, 39743
github, level 13 with dict dds, advanced streaming, 39743
github, level 13 with dict copy, advanced streaming, 39948
github, level 13 with dict load, advanced streaming, 42626
github, level 16, advanced streaming, 134064
github, level 16 with dict, advanced streaming, 37577
github, level 16 with dict dms, advanced streaming, 37577
github, level 16 with dict dds, advanced streaming, 37577
github, level 16 with dict copy, advanced streaming, 37568
github, level 16 with dict load, advanced streaming, 42340
github, level 19, advanced streaming, 134064
github, level 19 with dict, advanced streaming, 37576
github, level 19 with dict dms, advanced streaming, 37576
github, level 19 with dict dds, advanced streaming, 37576
github, level 19 with dict copy, advanced streaming, 37567
github, level 19 with dict load, advanced streaming, 39613
github, no source size, advanced streaming, 136335
github, no source size with dict, advanced streaming, 41148
github, long distance mode, advanced streaming, 136335
@@ -590,33 +996,107 @@ github, uncompressed literals optimal, advanced
github, huffman literals, advanced streaming, 142465
github, multithreaded with advanced params, advanced streaming, 165915
github.tar, level -5, advanced streaming, 46747
github.tar, level -5 with dict, advanced streaming, 43971
github.tar, level -5 with dict, advanced streaming, 44440
github.tar, level -3, advanced streaming, 43537
github.tar, level -3 with dict, advanced streaming, 40805
github.tar, level -3 with dict, advanced streaming, 41112
github.tar, level -1, advanced streaming, 42465
github.tar, level -1 with dict, advanced streaming, 41122
github.tar, level -1 with dict, advanced streaming, 41196
github.tar, level 0, advanced streaming, 38441
github.tar, level 0 with dict, advanced streaming, 37995
github.tar, level 0 with dict dms, advanced streaming, 38003
github.tar, level 0 with dict dds, advanced streaming, 38003
github.tar, level 0 with dict copy, advanced streaming, 37995
github.tar, level 0 with dict load, advanced streaming, 37956
github.tar, level 1, advanced streaming, 39342
github.tar, level 1 with dict, advanced streaming, 38309
github.tar, level 1 with dict, advanced streaming, 38293
github.tar, level 1 with dict dms, advanced streaming, 38303
github.tar, level 1 with dict dds, advanced streaming, 38303
github.tar, level 1 with dict copy, advanced streaming, 38293
github.tar, level 1 with dict load, advanced streaming, 38766
github.tar, level 3, advanced streaming, 38441
github.tar, level 3 with dict, advanced streaming, 37995
github.tar, level 3 with dict dms, advanced streaming, 38003
github.tar, level 3 with dict dds, advanced streaming, 38003
github.tar, level 3 with dict copy, advanced streaming, 37995
github.tar, level 3 with dict load, advanced streaming, 37956
github.tar, level 4, advanced streaming, 38467
github.tar, level 4 with dict, advanced streaming, 37948
github.tar, level 5, advanced streaming, 39788
github.tar, level 5 with dict, advanced streaming, 39715
github.tar, level 6, advanced streaming, 39603
github.tar, level 6 with dict, advanced streaming, 38800
github.tar, level 7, advanced streaming, 39206
github.tar, level 7 with dict, advanced streaming, 38071
github.tar, level 9, advanced streaming, 36717
github.tar, level 9 with dict, advanced streaming, 36898
github.tar, level 4 with dict dms, advanced streaming, 37954
github.tar, level 4 with dict dds, advanced streaming, 37954
github.tar, level 4 with dict copy, advanced streaming, 37948
github.tar, level 4 with dict load, advanced streaming, 37927
github.tar, level 5 row 1, advanced streaming, 39788
github.tar, level 5 row 1 with dict dms, advanced streaming, 39365
github.tar, level 5 row 1 with dict dds, advanced streaming, 39233
github.tar, level 5 row 1 with dict copy, advanced streaming, 39715
github.tar, level 5 row 1 with dict load, advanced streaming, 39209
github.tar, level 5 row 2, advanced streaming, 39693
github.tar, level 5 row 2 with dict dms, advanced streaming, 39024
github.tar, level 5 row 2 with dict dds, advanced streaming, 39028
github.tar, level 5 row 2 with dict copy, advanced streaming, 39040
github.tar, level 5 row 2 with dict load, advanced streaming, 39037
github.tar, level 5, advanced streaming, 39693
github.tar, level 5 with dict, advanced streaming, 39040
github.tar, level 5 with dict dms, advanced streaming, 39024
github.tar, level 5 with dict dds, advanced streaming, 39028
github.tar, level 5 with dict copy, advanced streaming, 39040
github.tar, level 5 with dict load, advanced streaming, 39037
github.tar, level 6, advanced streaming, 39621
github.tar, level 6 with dict, advanced streaming, 38622
github.tar, level 6 with dict dms, advanced streaming, 38608
github.tar, level 6 with dict dds, advanced streaming, 38610
github.tar, level 6 with dict copy, advanced streaming, 38622
github.tar, level 6 with dict load, advanced streaming, 38962
github.tar, level 7 row 1, advanced streaming, 39206
github.tar, level 7 row 1 with dict dms, advanced streaming, 37954
github.tar, level 7 row 1 with dict dds, advanced streaming, 37954
github.tar, level 7 row 1 with dict copy, advanced streaming, 38071
github.tar, level 7 row 1 with dict load, advanced streaming, 38584
github.tar, level 7 row 2, advanced streaming, 39213
github.tar, level 7 row 2 with dict dms, advanced streaming, 37848
github.tar, level 7 row 2 with dict dds, advanced streaming, 37867
github.tar, level 7 row 2 with dict copy, advanced streaming, 37848
github.tar, level 7 row 2 with dict load, advanced streaming, 38582
github.tar, level 7, advanced streaming, 39213
github.tar, level 7 with dict, advanced streaming, 37848
github.tar, level 7 with dict dms, advanced streaming, 37848
github.tar, level 7 with dict dds, advanced streaming, 37867
github.tar, level 7 with dict copy, advanced streaming, 37848
github.tar, level 7 with dict load, advanced streaming, 38582
github.tar, level 9, advanced streaming, 36758
github.tar, level 9 with dict, advanced streaming, 36457
github.tar, level 9 with dict dms, advanced streaming, 36549
github.tar, level 9 with dict dds, advanced streaming, 36637
github.tar, level 9 with dict copy, advanced streaming, 36457
github.tar, level 9 with dict load, advanced streaming, 36350
github.tar, level 12 row 1, advanced streaming, 36435
github.tar, level 12 row 1 with dict dms, advanced streaming, 36986
github.tar, level 12 row 1 with dict dds, advanced streaming, 36986
github.tar, level 12 row 1 with dict copy, advanced streaming, 36609
github.tar, level 12 row 1 with dict load, advanced streaming, 36419
github.tar, level 12 row 2, advanced streaming, 36435
github.tar, level 12 row 2 with dict dms, advanced streaming, 36986
github.tar, level 12 row 2 with dict dds, advanced streaming, 36986
github.tar, level 12 row 2 with dict copy, advanced streaming, 36609
github.tar, level 12 row 2 with dict load, advanced streaming, 36424
github.tar, level 13, advanced streaming, 35621
github.tar, level 13 with dict, advanced streaming, 38726
github.tar, level 13 with dict dms, advanced streaming, 38903
github.tar, level 13 with dict dds, advanced streaming, 38903
github.tar, level 13 with dict copy, advanced streaming, 38726
github.tar, level 13 with dict load, advanced streaming, 36372
github.tar, level 16, advanced streaming, 40255
github.tar, level 16 with dict, advanced streaming, 33639
github.tar, level 16 with dict dms, advanced streaming, 33544
github.tar, level 16 with dict dds, advanced streaming, 33544
github.tar, level 16 with dict copy, advanced streaming, 33639
github.tar, level 16 with dict load, advanced streaming, 39353
github.tar, level 19, advanced streaming, 32837
github.tar, level 19 with dict, advanced streaming, 32895
github.tar, level 19 with dict dms, advanced streaming, 32672
github.tar, level 19 with dict dds, advanced streaming, 32672
github.tar, level 19 with dict copy, advanced streaming, 32895
github.tar, level 19 with dict load, advanced streaming, 32676
github.tar, no source size, advanced streaming, 38438
github.tar, no source size with dict, advanced streaming, 38000
github.tar, long distance mode, advanced streaming, 39722
@@ -625,7 +1105,7 @@ github.tar, multithreaded long distance mode, advanced
github.tar, small window log, advanced streaming, 199558
github.tar, small hash log, advanced streaming, 129870
github.tar, small chain log, advanced streaming, 41669
github.tar, explicit params, advanced streaming, 41199
github.tar, explicit params, advanced streaming, 41227
github.tar, uncompressed literals, advanced streaming, 41122
github.tar, uncompressed literals optimal, advanced streaming, 35388
github.tar, huffman literals, advanced streaming, 38800
@@ -637,16 +1117,16 @@ silesia, level 0, old stre
silesia, level 1, old streaming, 5314162
silesia, level 3, old streaming, 4849552
silesia, level 4, old streaming, 4786970
silesia, level 5, old streaming, 4710236
silesia, level 6, old streaming, 4660056
silesia, level 7, old streaming, 4596296
silesia, level 9, old streaming, 4543925
silesia, level 5, old streaming, 4707794
silesia, level 6, old streaming, 4666383
silesia, level 7, old streaming, 4603381
silesia, level 9, old streaming, 4546001
silesia, level 13, old streaming, 4482135
silesia, level 16, old streaming, 4377465
silesia, level 19, old streaming, 4293330
silesia, level 16, old streaming, 4360251
silesia, level 19, old streaming, 4283237
silesia, no source size, old streaming, 4849516
silesia, uncompressed literals, old streaming, 4849552
silesia, uncompressed literals optimal, old streaming, 4293330
silesia, uncompressed literals optimal, old streaming, 4283237
silesia, huffman literals, old streaming, 6183403
silesia.tar, level -5, old streaming, 6982759
silesia.tar, level -3, old streaming, 6641283
@@ -655,16 +1135,16 @@ silesia.tar, level 0, old stre
silesia.tar, level 1, old streaming, 5336939
silesia.tar, level 3, old streaming, 4861427
silesia.tar, level 4, old streaming, 4799630
silesia.tar, level 5, old streaming, 4722329
silesia.tar, level 6, old streaming, 4672288
silesia.tar, level 7, old streaming, 4606715
silesia.tar, level 9, old streaming, 4554154
silesia.tar, level 5, old streaming, 4719261
silesia.tar, level 6, old streaming, 4677729
silesia.tar, level 7, old streaming, 4613544
silesia.tar, level 9, old streaming, 4555432
silesia.tar, level 13, old streaming, 4491765
silesia.tar, level 16, old streaming, 4381350
silesia.tar, level 19, old streaming, 4281562
silesia.tar, level 16, old streaming, 4356834
silesia.tar, level 19, old streaming, 4264392
silesia.tar, no source size, old streaming, 4861423
silesia.tar, uncompressed literals, old streaming, 4861427
silesia.tar, uncompressed literals optimal, old streaming, 4281562
silesia.tar, uncompressed literals optimal, old streaming, 4264392
silesia.tar, huffman literals, old streaming, 6190795
github, level -5, old streaming, 205285
github, level -5 with dict, old streaming, 46718
@@ -681,13 +1161,13 @@ github, level 3 with dict, old stre
github, level 4, old streaming, 136199
github, level 4 with dict, old streaming, 41251
github, level 5, old streaming, 135121
github, level 5 with dict, old streaming, 38938
github, level 5 with dict, old streaming, 38758
github, level 6, old streaming, 135122
github, level 6 with dict, old streaming, 38632
github, level 6 with dict, old streaming, 38671
github, level 7, old streaming, 135122
github, level 7 with dict, old streaming, 38771
github, level 7 with dict, old streaming, 38758
github, level 9, old streaming, 135122
github, level 9 with dict, old streaming, 39332
github, level 9 with dict, old streaming, 39437
github, level 13, old streaming, 134064
github, level 13 with dict, old streaming, 39743
github, level 16, old streaming, 134064
@@ -700,27 +1180,27 @@ github, uncompressed literals, old stre
github, uncompressed literals optimal, old streaming, 134064
github, huffman literals, old streaming, 175568
github.tar, level -5, old streaming, 46747
github.tar, level -5 with dict, old streaming, 43971
github.tar, level -5 with dict, old streaming, 44440
github.tar, level -3, old streaming, 43537
github.tar, level -3 with dict, old streaming, 40805
github.tar, level -3 with dict, old streaming, 41112
github.tar, level -1, old streaming, 42465
github.tar, level -1 with dict, old streaming, 41122
github.tar, level -1 with dict, old streaming, 41196
github.tar, level 0, old streaming, 38441
github.tar, level 0 with dict, old streaming, 37995
github.tar, level 1, old streaming, 39342
github.tar, level 1 with dict, old streaming, 38309
github.tar, level 1 with dict, old streaming, 38293
github.tar, level 3, old streaming, 38441
github.tar, level 3 with dict, old streaming, 37995
github.tar, level 4, old streaming, 38467
github.tar, level 4 with dict, old streaming, 37948
github.tar, level 5, old streaming, 39788
github.tar, level 5 with dict, old streaming, 39715
github.tar, level 6, old streaming, 39603
github.tar, level 6 with dict, old streaming, 38800
github.tar, level 7, old streaming, 39206
github.tar, level 7 with dict, old streaming, 38071
github.tar, level 9, old streaming, 36717
github.tar, level 9 with dict, old streaming, 36898
github.tar, level 5, old streaming, 39693
github.tar, level 5 with dict, old streaming, 39040
github.tar, level 6, old streaming, 39621
github.tar, level 6 with dict, old streaming, 38622
github.tar, level 7, old streaming, 39213
github.tar, level 7 with dict, old streaming, 37848
github.tar, level 9, old streaming, 36758
github.tar, level 9 with dict, old streaming, 36457
github.tar, level 13, old streaming, 35621
github.tar, level 13 with dict, old streaming, 38726
github.tar, level 16, old streaming, 40255
@@ -739,23 +1219,23 @@ silesia, level 0, old stre
silesia, level 1, old streaming advanced, 5314162
silesia, level 3, old streaming advanced, 4849552
silesia, level 4, old streaming advanced, 4786970
silesia, level 5, old streaming advanced, 4710236
silesia, level 6, old streaming advanced, 4660056
silesia, level 7, old streaming advanced, 4596296
silesia, level 9, old streaming advanced, 4543925
silesia, level 5, old streaming advanced, 4707794
silesia, level 6, old streaming advanced, 4666383
silesia, level 7, old streaming advanced, 4603381
silesia, level 9, old streaming advanced, 4546001
silesia, level 13, old streaming advanced, 4482135
silesia, level 16, old streaming advanced, 4377465
silesia, level 19, old streaming advanced, 4293330
silesia, level 16, old streaming advanced, 4360251
silesia, level 19, old streaming advanced, 4283237
silesia, no source size, old streaming advanced, 4849516
silesia, long distance mode, old streaming advanced, 4849552
silesia, multithreaded, old streaming advanced, 4849552
silesia, multithreaded long distance mode, old streaming advanced, 4849552
silesia, small window log, old streaming advanced, 7112062
silesia, small hash log, old streaming advanced, 6555021
silesia, small chain log, old streaming advanced, 4931148
silesia, explicit params, old streaming advanced, 4797112
silesia, small hash log, old streaming advanced, 6526141
silesia, small chain log, old streaming advanced, 4912197
silesia, explicit params, old streaming advanced, 4795887
silesia, uncompressed literals, old streaming advanced, 4849552
silesia, uncompressed literals optimal, old streaming advanced, 4293330
silesia, uncompressed literals optimal, old streaming advanced, 4283237
silesia, huffman literals, old streaming advanced, 6183403
silesia, multithreaded with advanced params, old streaming advanced, 4849552
silesia.tar, level -5, old streaming advanced, 6982759
@@ -765,23 +1245,23 @@ silesia.tar, level 0, old stre
silesia.tar, level 1, old streaming advanced, 5336939
silesia.tar, level 3, old streaming advanced, 4861427
silesia.tar, level 4, old streaming advanced, 4799630
silesia.tar, level 5, old streaming advanced, 4722329
silesia.tar, level 6, old streaming advanced, 4672288
silesia.tar, level 7, old streaming advanced, 4606715
silesia.tar, level 9, old streaming advanced, 4554154
silesia.tar, level 5, old streaming advanced, 4719261
silesia.tar, level 6, old streaming advanced, 4677729
silesia.tar, level 7, old streaming advanced, 4613544
silesia.tar, level 9, old streaming advanced, 4555432
silesia.tar, level 13, old streaming advanced, 4491765
silesia.tar, level 16, old streaming advanced, 4381350
silesia.tar, level 19, old streaming advanced, 4281562
silesia.tar, level 16, old streaming advanced, 4356834
silesia.tar, level 19, old streaming advanced, 4264392
silesia.tar, no source size, old streaming advanced, 4861423
silesia.tar, long distance mode, old streaming advanced, 4861427
silesia.tar, multithreaded, old streaming advanced, 4861427
silesia.tar, multithreaded long distance mode, old streaming advanced, 4861427
silesia.tar, small window log, old streaming advanced, 7118772
silesia.tar, small hash log, old streaming advanced, 6587952
silesia.tar, small chain log, old streaming advanced, 4943312
silesia.tar, explicit params, old streaming advanced, 4808618
silesia.tar, small hash log, old streaming advanced, 6529235
silesia.tar, small chain log, old streaming advanced, 4917021
silesia.tar, explicit params, old streaming advanced, 4807401
silesia.tar, uncompressed literals, old streaming advanced, 4861427
silesia.tar, uncompressed literals optimal, old streaming advanced, 4281562
silesia.tar, uncompressed literals optimal, old streaming advanced, 4264392
silesia.tar, huffman literals, old streaming advanced, 6190795
silesia.tar, multithreaded with advanced params, old streaming advanced, 4861427
github, level -5, old streaming advanced, 216734
@@ -799,13 +1279,13 @@ github, level 3 with dict, old stre
github, level 4, old streaming advanced, 141104
github, level 4 with dict, old streaming advanced, 41084
github, level 5, old streaming advanced, 139399
github, level 5 with dict, old streaming advanced, 39159
github, level 5 with dict, old streaming advanced, 38633
github, level 6, old streaming advanced, 139402
github, level 6 with dict, old streaming advanced, 38749
github, level 6 with dict, old streaming advanced, 38723
github, level 7, old streaming advanced, 138676
github, level 7 with dict, old streaming advanced, 38746
github, level 7 with dict, old streaming advanced, 38744
github, level 9, old streaming advanced, 138676
github, level 9 with dict, old streaming advanced, 38993
github, level 9 with dict, old streaming advanced, 38981
github, level 13, old streaming advanced, 138676
github, level 13 with dict, old streaming advanced, 39731
github, level 16, old streaming advanced, 138676
@@ -839,14 +1319,14 @@ github.tar, level 3, old stre
github.tar, level 3 with dict, old streaming advanced, 38013
github.tar, level 4, old streaming advanced, 38467
github.tar, level 4 with dict, old streaming advanced, 38063
github.tar, level 5, old streaming advanced, 39788
github.tar, level 5 with dict, old streaming advanced, 39310
github.tar, level 6, old streaming advanced, 39603
github.tar, level 6 with dict, old streaming advanced, 39279
github.tar, level 7, old streaming advanced, 39206
github.tar, level 7 with dict, old streaming advanced, 38728
github.tar, level 9, old streaming advanced, 36717
github.tar, level 9 with dict, old streaming advanced, 36504
github.tar, level 5, old streaming advanced, 39693
github.tar, level 5 with dict, old streaming advanced, 39049
github.tar, level 6, old streaming advanced, 39621
github.tar, level 6 with dict, old streaming advanced, 38959
github.tar, level 7, old streaming advanced, 39213
github.tar, level 7 with dict, old streaming advanced, 38573
github.tar, level 9, old streaming advanced, 36758
github.tar, level 9 with dict, old streaming advanced, 36233
github.tar, level 13, old streaming advanced, 35621
github.tar, level 13 with dict, old streaming advanced, 36035
github.tar, level 16, old streaming advanced, 40255
@@ -861,41 +1341,41 @@ github.tar, multithreaded long distance mode, old stre
github.tar, small window log, old streaming advanced, 199561
github.tar, small hash log, old streaming advanced, 129870
github.tar, small chain log, old streaming advanced, 41669
github.tar, explicit params, old streaming advanced, 41199
github.tar, explicit params, old streaming advanced, 41227
github.tar, uncompressed literals, old streaming advanced, 38441
github.tar, uncompressed literals optimal, old streaming advanced, 32837
github.tar, huffman literals, old streaming advanced, 42465
github.tar, multithreaded with advanced params, old streaming advanced, 38441
github, level -5 with dict, old streaming cdcit, 46718
github, level -3 with dict, old streaming cdcit, 45395
github, level -1 with dict, old streaming cdcit, 43170
github, level 0 with dict, old streaming cdcit, 41148
github, level 1 with dict, old streaming cdcit, 41682
github, level 3 with dict, old streaming cdcit, 41148
github, level 4 with dict, old streaming cdcit, 41251
github, level 5 with dict, old streaming cdcit, 38938
github, level 6 with dict, old streaming cdcit, 38632
github, level 7 with dict, old streaming cdcit, 38771
github, level 9 with dict, old streaming cdcit, 39332
github, level 13 with dict, old streaming cdcit, 39743
github, level 16 with dict, old streaming cdcit, 37577
github, level 19 with dict, old streaming cdcit, 37576
github, no source size with dict, old streaming cdcit, 40654
github.tar, level -5 with dict, old streaming cdcit, 45018
github.tar, level -3 with dict, old streaming cdcit, 41886
github.tar, level -1 with dict, old streaming cdcit, 41636
github.tar, level 0 with dict, old streaming cdcit, 37956
github.tar, level 1 with dict, old streaming cdcit, 38766
github.tar, level 3 with dict, old streaming cdcit, 37956
github.tar, level 4 with dict, old streaming cdcit, 37927
github.tar, level 5 with dict, old streaming cdcit, 39209
github.tar, level 6 with dict, old streaming cdcit, 38983
github.tar, level 7 with dict, old streaming cdcit, 38584
github.tar, level 9 with dict, old streaming cdcit, 36363
github.tar, level 13 with dict, old streaming cdcit, 36372
github.tar, level 16 with dict, old streaming cdcit, 39353
github.tar, level 19 with dict, old streaming cdcit, 32676
github.tar, no source size with dict, old streaming cdcit, 38000
github, level -5 with dict, old streaming cdict, 46718
github, level -3 with dict, old streaming cdict, 45395
github, level -1 with dict, old streaming cdict, 43170
github, level 0 with dict, old streaming cdict, 41148
github, level 1 with dict, old streaming cdict, 41682
github, level 3 with dict, old streaming cdict, 41148
github, level 4 with dict, old streaming cdict, 41251
github, level 5 with dict, old streaming cdict, 38758
github, level 6 with dict, old streaming cdict, 38671
github, level 7 with dict, old streaming cdict, 38758
github, level 9 with dict, old streaming cdict, 39437
github, level 13 with dict, old streaming cdict, 39743
github, level 16 with dict, old streaming cdict, 37577
github, level 19 with dict, old streaming cdict, 37576
github, no source size with dict, old streaming cdict, 40654
github.tar, level -5 with dict, old streaming cdict, 45018
github.tar, level -3 with dict, old streaming cdict, 41886
github.tar, level -1 with dict, old streaming cdict, 41636
github.tar, level 0 with dict, old streaming cdict, 37956
github.tar, level 1 with dict, old streaming cdict, 38766
github.tar, level 3 with dict, old streaming cdict, 37956
github.tar, level 4 with dict, old streaming cdict, 37927
github.tar, level 5 with dict, old streaming cdict, 39037
github.tar, level 6 with dict, old streaming cdict, 38962
github.tar, level 7 with dict, old streaming cdict, 38582
github.tar, level 9 with dict, old streaming cdict, 36350
github.tar, level 13 with dict, old streaming cdict, 36372
github.tar, level 16 with dict, old streaming cdict, 39353
github.tar, level 19 with dict, old streaming cdict, 32676
github.tar, no source size with dict, old streaming cdict, 38000
github, level -5 with dict, old streaming advanced cdict, 49562
github, level -3 with dict, old streaming advanced cdict, 44956
github, level -1 with dict, old streaming advanced cdict, 42383
@@ -903,10 +1383,10 @@ github, level 0 with dict, old stre
github, level 1 with dict, old streaming advanced cdict, 42430
github, level 3 with dict, old streaming advanced cdict, 41113
github, level 4 with dict, old streaming advanced cdict, 41084
github, level 5 with dict, old streaming advanced cdict, 39159
github, level 6 with dict, old streaming advanced cdict, 38749
github, level 7 with dict, old streaming advanced cdict, 38746
github, level 9 with dict, old streaming advanced cdict, 38993
github, level 5 with dict, old streaming advanced cdict, 38633
github, level 6 with dict, old streaming advanced cdict, 38723
github, level 7 with dict, old streaming advanced cdict, 38744
github, level 9 with dict, old streaming advanced cdict, 38981
github, level 13 with dict, old streaming advanced cdict, 39731
github, level 16 with dict, old streaming advanced cdict, 40789
github, level 19 with dict, old streaming advanced cdict, 37576
@@ -918,10 +1398,10 @@ github.tar, level 0 with dict, old stre
github.tar, level 1 with dict, old streaming advanced cdict, 39002
github.tar, level 3 with dict, old streaming advanced cdict, 38013
github.tar, level 4 with dict, old streaming advanced cdict, 38063
github.tar, level 5 with dict, old streaming advanced cdict, 39310
github.tar, level 6 with dict, old streaming advanced cdict, 39279
github.tar, level 7 with dict, old streaming advanced cdict, 38728
github.tar, level 9 with dict, old streaming advanced cdict, 36504
github.tar, level 5 with dict, old streaming advanced cdict, 39049
github.tar, level 6 with dict, old streaming advanced cdict, 38959
github.tar, level 7 with dict, old streaming advanced cdict, 38573
github.tar, level 9 with dict, old streaming advanced cdict, 36233
github.tar, level 13 with dict, old streaming advanced cdict, 36035
github.tar, level 16 with dict, old streaming advanced cdict, 38736
github.tar, level 19 with dict, old streaming advanced cdict, 32876
1 Data Config Method Total compressed size
6 silesia.tar level 1 compress simple 5334885
7 silesia.tar level 3 compress simple 4861425
8 silesia.tar level 4 compress simple 4799630
9 silesia.tar level 5 compress simple 4722324 4719256
10 silesia.tar level 6 compress simple 4672279 4677721
11 silesia.tar level 7 compress simple 4606715 4613541
12 silesia.tar level 9 compress simple 4554147 4555426
13 silesia.tar level 13 compress simple 4491764
14 silesia.tar level 16 compress simple 4381332
15 silesia.tar level 19 compress simple 4281605
23 github.tar level 1 compress simple 39265
24 github.tar level 3 compress simple 38441
25 github.tar level 4 compress simple 38467
26 github.tar level 5 compress simple 39788 39693
27 github.tar level 6 compress simple 39603 39621
28 github.tar level 7 compress simple 39206 39213
29 github.tar level 9 compress simple 36717 36758
30 github.tar level 13 compress simple 35621
31 github.tar level 16 compress simple 40255
32 github.tar level 19 compress simple 32837
40 silesia level 1 compress cctx 5313204
41 silesia level 3 compress cctx 4849552
42 silesia level 4 compress cctx 4786970
43 silesia level 5 compress cctx 4710236 4707794
44 silesia level 6 compress cctx 4660056 4666383
45 silesia level 7 compress cctx 4596296 4603381
46 silesia level 9 compress cctx 4543925 4546001
47 silesia level 13 compress cctx 4482135
48 silesia level 16 compress cctx 4377465
49 silesia level 19 compress cctx 4293330
53 silesia small window log compress cctx 7084179
54 silesia small hash log compress cctx 6555021
55 silesia small chain log compress cctx 4931148
56 silesia explicit params compress cctx 4794677 4794479
57 silesia uncompressed literals compress cctx 4849552
58 silesia uncompressed literals optimal compress cctx 4293330
59 silesia huffman literals compress cctx 6178460
73 github level 4 compress cctx 136199
74 github level 4 with dict compress cctx 41725
75 github level 5 compress cctx 135121
76 github level 5 with dict compress cctx 38934 38759
77 github level 6 compress cctx 135122
78 github level 6 with dict compress cctx 38628 38669
79 github level 7 compress cctx 135122
80 github level 7 with dict compress cctx 38745 38755
81 github level 9 compress cctx 135122
82 github level 9 with dict compress cctx 39341 39398
83 github level 13 compress cctx 134064
84 github level 13 with dict compress cctx 39948
85 github level 16 compress cctx 134064
104 silesia level 1 zstdcli 5314210
105 silesia level 3 zstdcli 4849600
106 silesia level 4 zstdcli 4787018
107 silesia level 5 zstdcli 4710284 4707842
108 silesia level 6 zstdcli 4660104 4666431
109 silesia level 7 zstdcli 4596344 4603429
110 silesia level 9 zstdcli 4543973 4546049
111 silesia level 13 zstdcli 4482183
112 silesia level 16 zstdcli 4377513 4360299
113 silesia level 19 zstdcli 4293378 4283285
114 silesia long distance mode zstdcli 4840792
115 silesia multithreaded zstdcli 4849600
116 silesia multithreaded long distance mode zstdcli 4840792
117 silesia small window log zstdcli 7111012
118 silesia small hash log zstdcli 6555069 6526189
119 silesia small chain log zstdcli 4931196 4912245
120 silesia explicit params zstdcli 4797112 4795887
121 silesia uncompressed literals zstdcli 5128030
122 silesia uncompressed literals optimal zstdcli 4325520 4317944
123 silesia huffman literals zstdcli 5331216
124 silesia multithreaded with advanced params zstdcli 5128030
125 silesia.tar level -5 zstdcli 6738934
129 silesia.tar level 1 zstdcli 5336318
130 silesia.tar level 3 zstdcli 4861512
131 silesia.tar level 4 zstdcli 4800529
132 silesia.tar level 5 zstdcli 4723364 4720121
133 silesia.tar level 6 zstdcli 4673663 4678661
134 silesia.tar level 7 zstdcli 4608403 4614424
135 silesia.tar level 9 zstdcli 4554751 4556062
136 silesia.tar level 13 zstdcli 4491768
137 silesia.tar level 16 zstdcli 4381336 4356831
138 silesia.tar level 19 zstdcli 4281609 4264491
139 silesia.tar no source size zstdcli 4861508
140 silesia.tar long distance mode zstdcli 4853153
141 silesia.tar multithreaded zstdcli 4861512
142 silesia.tar multithreaded long distance mode zstdcli 4853153
143 silesia.tar small window log zstdcli 7101576
144 silesia.tar small hash log zstdcli 6587959 6529290
145 silesia.tar small chain log zstdcli 4943310 4917022
146 silesia.tar explicit params zstdcli 4822362 4821274
147 silesia.tar uncompressed literals zstdcli 5129559
148 silesia.tar uncompressed literals optimal zstdcli 4320931 4307457
149 silesia.tar huffman literals zstdcli 5347610
150 silesia.tar multithreaded with advanced params zstdcli 5129559
151 github level -5 zstdcli 207285
163 github level 4 zstdcli 138199
164 github level 4 with dict zstdcli 43251
165 github level 5 zstdcli 137121
166 github level 5 with dict zstdcli 40741 40728
167 github level 6 zstdcli 137122
168 github level 6 with dict zstdcli 40632 40630
169 github level 7 zstdcli 137122
170 github level 7 with dict zstdcli 40771 40747
171 github level 9 zstdcli 137122
172 github level 9 with dict zstdcli 41332 41338
173 github level 13 zstdcli 136064
174 github level 13 with dict zstdcli 41743
175 github level 16 zstdcli 136064
188 github huffman literals zstdcli 144465
189 github multithreaded with advanced params zstdcli 167915
190 github.tar level -5 zstdcli 46751
191 github.tar level -5 with dict zstdcli 43975 44444
192 github.tar level -3 zstdcli 43541
193 github.tar level -3 with dict zstdcli 40809 41116
194 github.tar level -1 zstdcli 42469
195 github.tar level -1 with dict zstdcli 41126 41200
196 github.tar level 0 zstdcli 38445
197 github.tar level 0 with dict zstdcli 37999
198 github.tar level 1 zstdcli 39346
199 github.tar level 1 with dict zstdcli 38313 38297
200 github.tar level 3 zstdcli 38445
201 github.tar level 3 with dict zstdcli 37999
202 github.tar level 4 zstdcli 38471
203 github.tar level 4 with dict zstdcli 37952
204 github.tar level 5 zstdcli 39792 39697
205 github.tar level 5 with dict zstdcli 39231 39032
206 github.tar level 6 zstdcli 39607 39625
207 github.tar level 6 with dict zstdcli 38669 38614
208 github.tar level 7 zstdcli 39210 39217
209 github.tar level 7 with dict zstdcli 37958 37871
210 github.tar level 9 zstdcli 36721 36762
211 github.tar level 9 with dict zstdcli 36886 36641
212 github.tar level 13 zstdcli 35625
213 github.tar level 13 with dict zstdcli 38730
214 github.tar level 16 zstdcli 40259
223 github.tar small window log zstdcli 199432
224 github.tar small hash log zstdcli 129874
225 github.tar small chain log zstdcli 41673
226 github.tar explicit params zstdcli 41199 41227
227 github.tar uncompressed literals zstdcli 41126
228 github.tar uncompressed literals optimal zstdcli 35392
229 github.tar huffman literals zstdcli 38804
235 silesia level 1 advanced one pass 5313204
236 silesia level 3 advanced one pass 4849552
237 silesia level 4 advanced one pass 4786970
238 silesia level 5 level 5 row 1 advanced one pass 4710236
239 silesia level 6 level 5 row 2 advanced one pass 4660056 4707794
240 silesia level 7 level 5 advanced one pass 4596296 4707794
241 silesia level 9 level 6 advanced one pass 4543925 4666383
242 silesia level 7 row 1 advanced one pass 4596296
243 silesia level 7 row 2 advanced one pass 4603381
244 silesia level 7 advanced one pass 4603381
245 silesia level 9 advanced one pass 4546001
246 silesia level 12 row 1 advanced one pass 4519288
247 silesia level 12 row 2 advanced one pass 4521397
248 silesia level 13 advanced one pass 4482135
249 silesia level 16 advanced one pass 4377465 4360251
250 silesia level 19 advanced one pass 4293330 4283237
251 silesia no source size advanced one pass 4849552
252 silesia long distance mode advanced one pass 4840744
253 silesia multithreaded advanced one pass 4849552
254 silesia multithreaded long distance mode advanced one pass 4840744
255 silesia small window log advanced one pass 7095919
256 silesia small hash log advanced one pass 6555021 6526141
257 silesia small chain log advanced one pass 4931148 4912197
258 silesia explicit params advanced one pass 4797095 4795856
259 silesia uncompressed literals advanced one pass 5127982
260 silesia uncompressed literals optimal advanced one pass 4325472 4317896
261 silesia huffman literals advanced one pass 5326268
262 silesia multithreaded with advanced params advanced one pass 5127982
263 silesia.tar level -5 advanced one pass 6738593
267 silesia.tar level 1 advanced one pass 5334885
268 silesia.tar level 3 advanced one pass 4861425
269 silesia.tar level 4 advanced one pass 4799630
270 silesia.tar level 5 level 5 row 1 advanced one pass 4722324
271 silesia.tar level 6 level 5 row 2 advanced one pass 4672279 4719256
272 silesia.tar level 7 level 5 advanced one pass 4606715 4719256
273 silesia.tar level 9 level 6 advanced one pass 4554147 4677721
274 silesia.tar level 7 row 1 advanced one pass 4606715
275 silesia.tar level 7 row 2 advanced one pass 4613541
276 silesia.tar level 7 advanced one pass 4613541
277 silesia.tar level 9 advanced one pass 4555426
278 silesia.tar level 12 row 1 advanced one pass 4529459
279 silesia.tar level 12 row 2 advanced one pass 4530256
280 silesia.tar level 13 advanced one pass 4491764
281 silesia.tar level 16 advanced one pass 4381332 4356827
282 silesia.tar level 19 advanced one pass 4281605 4264487
283 silesia.tar no source size advanced one pass 4861425
284 silesia.tar long distance mode advanced one pass 4847735
285 silesia.tar multithreaded advanced one pass 4861508
286 silesia.tar multithreaded long distance mode advanced one pass 4853149
287 silesia.tar small window log advanced one pass 7101530
288 silesia.tar small hash log advanced one pass 6587951 6529232
289 silesia.tar small chain log advanced one pass 4943307 4917041
290 silesia.tar explicit params advanced one pass 4808589 4807380
291 silesia.tar uncompressed literals advanced one pass 5129458
292 silesia.tar uncompressed literals optimal advanced one pass 4320927 4307453
293 silesia.tar huffman literals advanced one pass 5347335
294 silesia.tar multithreaded with advanced params advanced one pass 5129555
295 github level -5 advanced one pass 205285
300 github level -1 with dict advanced one pass 43170
301 github level 0 advanced one pass 136335
302 github level 0 with dict advanced one pass 41148
303 github level 0 with dict dms advanced one pass 41148
304 github level 0 with dict dds advanced one pass 41148
305 github level 0 with dict copy advanced one pass 41124
306 github level 0 with dict load advanced one pass 42252
307 github level 1 advanced one pass 142465
308 github level 1 with dict advanced one pass 41682
309 github level 1 with dict dms advanced one pass 41682
310 github level 1 with dict dds advanced one pass 41682
311 github level 1 with dict copy advanced one pass 41674
312 github level 1 with dict load advanced one pass 43755
313 github level 3 advanced one pass 136335
314 github level 3 with dict advanced one pass 41148
315 github level 3 with dict dms advanced one pass 41148
316 github level 3 with dict dds advanced one pass 41148
317 github level 3 with dict copy advanced one pass 41124
318 github level 3 with dict load advanced one pass 42252
319 github level 4 advanced one pass 136199
320 github level 4 with dict advanced one pass 41251
321 github level 4 with dict dms advanced one pass 41251
322 github level 4 with dict dds advanced one pass 41251
323 github level 4 with dict copy advanced one pass 41216
324 github level 4 with dict load advanced one pass 41159
325 github level 5 row 1 advanced one pass 135121
326 github level 5 row 1 with dict dms advanced one pass 38938
327 github level 5 row 1 with dict dds advanced one pass 38732
328 github level 5 row 1 with dict copy advanced one pass 38934
329 github level 5 row 1 with dict load advanced one pass 40725
330 github level 5 row 2 advanced one pass 134584
331 github level 5 row 2 with dict dms advanced one pass 38758
332 github level 5 row 2 with dict dds advanced one pass 38728
333 github level 5 row 2 with dict copy advanced one pass 38759
334 github level 5 row 2 with dict load advanced one pass 41518
335 github level 5 advanced one pass 135121
336 github level 5 with dict advanced one pass 38938 38758
337 github level 5 with dict dms advanced one pass 38758
338 github level 5 with dict dds advanced one pass 38728
339 github level 5 with dict copy advanced one pass 38759
340 github level 5 with dict load advanced one pass 40725
341 github level 6 advanced one pass 135122
342 github level 6 with dict advanced one pass 38632 38671
343 github level 6 with dict dms advanced one pass 38671
344 github level 6 with dict dds advanced one pass 38630
345 github level 6 with dict copy advanced one pass 38669
346 github level 6 with dict load advanced one pass 40695
347 github level 7 row 1 advanced one pass 135122
348 github level 7 row 1 with dict dms advanced one pass 38771
349 github level 7 row 1 with dict dds advanced one pass 38771
350 github level 7 row 1 with dict copy advanced one pass 38745
351 github level 7 row 1 with dict load advanced one pass 40695
352 github level 7 row 2 advanced one pass 134584
353 github level 7 row 2 with dict dms advanced one pass 38758
354 github level 7 row 2 with dict dds advanced one pass 38747
355 github level 7 row 2 with dict copy advanced one pass 38755
356 github level 7 row 2 with dict load advanced one pass 41030
357 github level 7 advanced one pass 135122
358 github level 7 with dict advanced one pass 38771 38758
359 github level 7 with dict dms advanced one pass 38758
360 github level 7 with dict dds advanced one pass 38747
361 github level 7 with dict copy advanced one pass 38755
362 github level 7 with dict load advanced one pass 40695
363 github level 9 advanced one pass 135122
364 github level 9 with dict advanced one pass 39332 39437
365 github level 9 with dict dms advanced one pass 39437
366 github level 9 with dict dds advanced one pass 39338
367 github level 9 with dict copy advanced one pass 39398
368 github level 9 with dict load advanced one pass 41710
369 github level 12 row 1 advanced one pass 134180
370 github level 12 row 1 with dict dms advanced one pass 39677
371 github level 12 row 1 with dict dds advanced one pass 39677
372 github level 12 row 1 with dict copy advanced one pass 39677
373 github level 12 row 1 with dict load advanced one pass 41166
374 github level 12 row 2 advanced one pass 134180
375 github level 12 row 2 with dict dms advanced one pass 39677
376 github level 12 row 2 with dict dds advanced one pass 39677
377 github level 12 row 2 with dict copy advanced one pass 39677
378 github level 12 row 2 with dict load advanced one pass 41166
379 github level 13 advanced one pass 134064
380 github level 13 with dict advanced one pass 39743
381 github level 13 with dict dms advanced one pass 39743
382 github level 13 with dict dds advanced one pass 39743
383 github level 13 with dict copy advanced one pass 39948
384 github level 13 with dict load advanced one pass 42626
385 github level 16 advanced one pass 134064
386 github level 16 with dict advanced one pass 37577
387 github level 16 with dict dms advanced one pass 37577
388 github level 16 with dict dds advanced one pass 37577
389 github level 16 with dict copy advanced one pass 37568
390 github level 16 with dict load advanced one pass 42340
391 github level 19 advanced one pass 134064
392 github level 19 with dict advanced one pass 37576
393 github level 19 with dict dms advanced one pass 37576
394 github level 19 with dict dds advanced one pass 37576
395 github level 19 with dict copy advanced one pass 37567
396 github level 19 with dict load advanced one pass 39613
397 github no source size advanced one pass 136335
398 github no source size with dict advanced one pass 41148
399 github long distance mode advanced one pass 136335
408 github huffman literals advanced one pass 142465
409 github multithreaded with advanced params advanced one pass 165915
410 github.tar level -5 advanced one pass 46856
411 github.tar level -5 with dict advanced one pass 43971 44571
412 github.tar level -3 advanced one pass 43754
413 github.tar level -3 with dict advanced one pass 40805 41447
414 github.tar level -1 advanced one pass 42490
415 github.tar level -1 with dict advanced one pass 41122 41131
416 github.tar level 0 advanced one pass 38441
417 github.tar level 0 with dict advanced one pass 37995
418 github.tar level 0 with dict dms advanced one pass 38003
419 github.tar level 0 with dict dds advanced one pass 38003
420 github.tar level 0 with dict copy advanced one pass 37995
421 github.tar level 0 with dict load advanced one pass 37956
422 github.tar level 1 advanced one pass 39265
423 github.tar level 1 with dict advanced one pass 38309 38280
424 github.tar level 1 with dict dms advanced one pass 38290
425 github.tar level 1 with dict dds advanced one pass 38290
426 github.tar level 1 with dict copy advanced one pass 38280
427 github.tar level 1 with dict load advanced one pass 38729
428 github.tar level 3 advanced one pass 38441
429 github.tar level 3 with dict advanced one pass 37995
430 github.tar level 3 with dict dms advanced one pass 38003
431 github.tar level 3 with dict dds advanced one pass 38003
432 github.tar level 3 with dict copy advanced one pass 37995
433 github.tar level 3 with dict load advanced one pass 37956
434 github.tar level 4 advanced one pass 38467
435 github.tar level 4 with dict advanced one pass 37948
436 github.tar level 5 level 4 with dict dms advanced one pass 39788 37954
437 github.tar level 5 with dict level 4 with dict dds advanced one pass 39715 37954
438 github.tar level 6 level 4 with dict copy advanced one pass 39603 37948
439 github.tar level 6 with dict level 4 with dict load advanced one pass 38800 37927
440 github.tar level 7 level 5 row 1 advanced one pass 39206 39788
441 github.tar level 7 with dict level 5 row 1 with dict dms advanced one pass 38071 39365
442 github.tar level 9 level 5 row 1 with dict dds advanced one pass 36717 39233
443 github.tar level 9 with dict level 5 row 1 with dict copy advanced one pass 36898 39715
444 github.tar level 5 row 1 with dict load advanced one pass 39209
445 github.tar level 5 row 2 advanced one pass 39693
446 github.tar level 5 row 2 with dict dms advanced one pass 39024
447 github.tar level 5 row 2 with dict dds advanced one pass 39028
448 github.tar level 5 row 2 with dict copy advanced one pass 39040
449 github.tar level 5 row 2 with dict load advanced one pass 39037
450 github.tar level 5 advanced one pass 39693
451 github.tar level 5 with dict advanced one pass 39040
452 github.tar level 5 with dict dms advanced one pass 39024
453 github.tar level 5 with dict dds advanced one pass 39028
454 github.tar level 5 with dict copy advanced one pass 39040
455 github.tar level 5 with dict load advanced one pass 39037
456 github.tar level 6 advanced one pass 39621
457 github.tar level 6 with dict advanced one pass 38622
458 github.tar level 6 with dict dms advanced one pass 38608
459 github.tar level 6 with dict dds advanced one pass 38610
460 github.tar level 6 with dict copy advanced one pass 38622
461 github.tar level 6 with dict load advanced one pass 38962
462 github.tar level 7 row 1 advanced one pass 39206
463 github.tar level 7 row 1 with dict dms advanced one pass 37954
464 github.tar level 7 row 1 with dict dds advanced one pass 37954
465 github.tar level 7 row 1 with dict copy advanced one pass 38071
466 github.tar level 7 row 1 with dict load advanced one pass 38584
467 github.tar level 7 row 2 advanced one pass 39213
468 github.tar level 7 row 2 with dict dms advanced one pass 37848
469 github.tar level 7 row 2 with dict dds advanced one pass 37867
470 github.tar level 7 row 2 with dict copy advanced one pass 37848
471 github.tar level 7 row 2 with dict load advanced one pass 38582
472 github.tar level 7 advanced one pass 39213
473 github.tar level 7 with dict advanced one pass 37848
474 github.tar level 7 with dict dms advanced one pass 37848
475 github.tar level 7 with dict dds advanced one pass 37867
476 github.tar level 7 with dict copy advanced one pass 37848
477 github.tar level 7 with dict load advanced one pass 38582
478 github.tar level 9 advanced one pass 36758
479 github.tar level 9 with dict advanced one pass 36457
480 github.tar level 9 with dict dms advanced one pass 36549
481 github.tar level 9 with dict dds advanced one pass 36637
482 github.tar level 9 with dict copy advanced one pass 36457
483 github.tar level 9 with dict load advanced one pass 36350
484 github.tar level 12 row 1 advanced one pass 36435
485 github.tar level 12 row 1 with dict dms advanced one pass 36986
486 github.tar level 12 row 1 with dict dds advanced one pass 36986
487 github.tar level 12 row 1 with dict copy advanced one pass 36609
488 github.tar level 12 row 1 with dict load advanced one pass 36419
489 github.tar level 12 row 2 advanced one pass 36435
490 github.tar level 12 row 2 with dict dms advanced one pass 36986
491 github.tar level 12 row 2 with dict dds advanced one pass 36986
492 github.tar level 12 row 2 with dict copy advanced one pass 36609
493 github.tar level 12 row 2 with dict load advanced one pass 36424
494 github.tar level 13 advanced one pass 35621
495 github.tar level 13 with dict advanced one pass 38726
496 github.tar level 13 with dict dms advanced one pass 38903
497 github.tar level 13 with dict dds advanced one pass 38903
498 github.tar level 13 with dict copy advanced one pass 38726
499 github.tar level 13 with dict load advanced one pass 36372
500 github.tar level 16 advanced one pass 40255
501 github.tar level 16 with dict advanced one pass 33639
502 github.tar level 16 with dict dms advanced one pass 33544
503 github.tar level 16 with dict dds advanced one pass 33544
504 github.tar level 16 with dict copy advanced one pass 33639
505 github.tar level 16 with dict load advanced one pass 39353
506 github.tar level 19 advanced one pass 32837
507 github.tar level 19 with dict advanced one pass 32895
508 github.tar level 19 with dict dms advanced one pass 32672
509 github.tar level 19 with dict dds advanced one pass 32672
510 github.tar level 19 with dict copy advanced one pass 32895
511 github.tar level 19 with dict load advanced one pass 32676
512 github.tar no source size advanced one pass 38441
513 github.tar no source size with dict advanced one pass 37995
514 github.tar long distance mode advanced one pass 39722
517 github.tar small window log advanced one pass 198540
518 github.tar small hash log advanced one pass 129870
519 github.tar small chain log advanced one pass 41669
520 github.tar explicit params advanced one pass 41199 41227
521 github.tar uncompressed literals advanced one pass 41122
522 github.tar uncompressed literals optimal advanced one pass 35388
523 github.tar huffman literals advanced one pass 38777
529 silesia level 1 advanced one pass small out 5313204
530 silesia level 3 advanced one pass small out 4849552
531 silesia level 4 advanced one pass small out 4786970
532 silesia level 5 level 5 row 1 advanced one pass small out 4710236
533 silesia level 6 level 5 row 2 advanced one pass small out 4660056 4707794
534 silesia level 7 level 5 advanced one pass small out 4596296 4707794
535 silesia level 9 level 6 advanced one pass small out 4543925 4666383
536 silesia level 7 row 1 advanced one pass small out 4596296
537 silesia level 7 row 2 advanced one pass small out 4603381
538 silesia level 7 advanced one pass small out 4603381
539 silesia level 9 advanced one pass small out 4546001
540 silesia level 12 row 1 advanced one pass small out 4519288
541 silesia level 12 row 2 advanced one pass small out 4521397
542 silesia level 13 advanced one pass small out 4482135
543 silesia level 16 advanced one pass small out 4377465 4360251
544 silesia level 19 advanced one pass small out 4293330 4283237
545 silesia no source size advanced one pass small out 4849552
546 silesia long distance mode advanced one pass small out 4840744
547 silesia multithreaded advanced one pass small out 4849552
548 silesia multithreaded long distance mode advanced one pass small out 4840744
549 silesia small window log advanced one pass small out 7095919
550 silesia small hash log advanced one pass small out 6555021 6526141
551 silesia small chain log advanced one pass small out 4931148 4912197
552 silesia explicit params advanced one pass small out 4797095 4795856
553 silesia uncompressed literals advanced one pass small out 5127982
554 silesia uncompressed literals optimal advanced one pass small out 4325472 4317896
555 silesia huffman literals advanced one pass small out 5326268
556 silesia multithreaded with advanced params advanced one pass small out 5127982
557 silesia.tar level -5 advanced one pass small out 6738593
561 silesia.tar level 1 advanced one pass small out 5334885
562 silesia.tar level 3 advanced one pass small out 4861425
563 silesia.tar level 4 advanced one pass small out 4799630
564 silesia.tar level 5 level 5 row 1 advanced one pass small out 4722324
565 silesia.tar level 6 level 5 row 2 advanced one pass small out 4672279 4719256
566 silesia.tar level 7 level 5 advanced one pass small out 4606715 4719256
567 silesia.tar level 9 level 6 advanced one pass small out 4554147 4677721
568 silesia.tar level 7 row 1 advanced one pass small out 4606715
569 silesia.tar level 7 row 2 advanced one pass small out 4613541
570 silesia.tar level 7 advanced one pass small out 4613541
571 silesia.tar level 9 advanced one pass small out 4555426
572 silesia.tar level 12 row 1 advanced one pass small out 4529459
573 silesia.tar level 12 row 2 advanced one pass small out 4530256
574 silesia.tar level 13 advanced one pass small out 4491764
575 silesia.tar level 16 advanced one pass small out 4381332 4356827
576 silesia.tar level 19 advanced one pass small out 4281605 4264487
577 silesia.tar no source size advanced one pass small out 4861425
578 silesia.tar long distance mode advanced one pass small out 4847735
579 silesia.tar multithreaded advanced one pass small out 4861508
580 silesia.tar multithreaded long distance mode advanced one pass small out 4853149
581 silesia.tar small window log advanced one pass small out 7101530
582 silesia.tar small hash log advanced one pass small out 6587951 6529232
583 silesia.tar small chain log advanced one pass small out 4943307 4917041
584 silesia.tar explicit params advanced one pass small out 4808589 4807380
585 silesia.tar uncompressed literals advanced one pass small out 5129458
586 silesia.tar uncompressed literals optimal advanced one pass small out 4320927 4307453
587 silesia.tar huffman literals advanced one pass small out 5347335
588 silesia.tar multithreaded with advanced params advanced one pass small out 5129555
589 github level -5 advanced one pass small out 205285
594 github level -1 with dict advanced one pass small out 43170
595 github level 0 advanced one pass small out 136335
596 github level 0 with dict advanced one pass small out 41148
597 github level 0 with dict dms advanced one pass small out 41148
598 github level 0 with dict dds advanced one pass small out 41148
599 github level 0 with dict copy advanced one pass small out 41124
600 github level 0 with dict load advanced one pass small out 42252
601 github level 1 advanced one pass small out 142465
602 github level 1 with dict advanced one pass small out 41682
603 github level 1 with dict dms advanced one pass small out 41682
604 github level 1 with dict dds advanced one pass small out 41682
605 github level 1 with dict copy advanced one pass small out 41674
606 github level 1 with dict load advanced one pass small out 43755
607 github level 3 advanced one pass small out 136335
608 github level 3 with dict advanced one pass small out 41148
609 github level 3 with dict dms advanced one pass small out 41148
610 github level 3 with dict dds advanced one pass small out 41148
611 github level 3 with dict copy advanced one pass small out 41124
612 github level 3 with dict load advanced one pass small out 42252
613 github level 4 advanced one pass small out 136199
614 github level 4 with dict advanced one pass small out 41251
615 github level 4 with dict dms advanced one pass small out 41251
616 github level 4 with dict dds advanced one pass small out 41251
617 github level 4 with dict copy advanced one pass small out 41216
618 github level 4 with dict load advanced one pass small out 41159
619 github level 5 row 1 advanced one pass small out 135121
620 github level 5 row 1 with dict dms advanced one pass small out 38938
621 github level 5 row 1 with dict dds advanced one pass small out 38732
622 github level 5 row 1 with dict copy advanced one pass small out 38934
623 github level 5 row 1 with dict load advanced one pass small out 40725
624 github level 5 row 2 advanced one pass small out 134584
625 github level 5 row 2 with dict dms advanced one pass small out 38758
626 github level 5 row 2 with dict dds advanced one pass small out 38728
627 github level 5 row 2 with dict copy advanced one pass small out 38759
628 github level 5 row 2 with dict load advanced one pass small out 41518
629 github level 5 advanced one pass small out 135121
630 github level 5 with dict advanced one pass small out 38938 38758
631 github level 5 with dict dms advanced one pass small out 38758
632 github level 5 with dict dds advanced one pass small out 38728
633 github level 5 with dict copy advanced one pass small out 38759
634 github level 5 with dict load advanced one pass small out 40725
635 github level 6 advanced one pass small out 135122
636 github level 6 with dict advanced one pass small out 38632 38671
637 github level 6 with dict dms advanced one pass small out 38671
638 github level 6 with dict dds advanced one pass small out 38630
639 github level 6 with dict copy advanced one pass small out 38669
640 github level 6 with dict load advanced one pass small out 40695
641 github level 7 row 1 advanced one pass small out 135122
642 github level 7 row 1 with dict dms advanced one pass small out 38771
643 github level 7 row 1 with dict dds advanced one pass small out 38771
644 github level 7 row 1 with dict copy advanced one pass small out 38745
645 github level 7 row 1 with dict load advanced one pass small out 40695
646 github level 7 row 2 advanced one pass small out 134584
647 github level 7 row 2 with dict dms advanced one pass small out 38758
648 github level 7 row 2 with dict dds advanced one pass small out 38747
649 github level 7 row 2 with dict copy advanced one pass small out 38755
650 github level 7 row 2 with dict load advanced one pass small out 41030
651 github level 7 advanced one pass small out 135122
652 github level 7 with dict advanced one pass small out 38771 38758
653 github level 7 with dict dms advanced one pass small out 38758
654 github level 7 with dict dds advanced one pass small out 38747
655 github level 7 with dict copy advanced one pass small out 38755
656 github level 7 with dict load advanced one pass small out 40695
657 github level 9 advanced one pass small out 135122
658 github level 9 with dict advanced one pass small out 39332 39437
659 github level 9 with dict dms advanced one pass small out 39437
660 github level 9 with dict dds advanced one pass small out 39338
661 github level 9 with dict copy advanced one pass small out 39398
662 github level 9 with dict load advanced one pass small out 41710
663 github level 12 row 1 advanced one pass small out 134180
664 github level 12 row 1 with dict dms advanced one pass small out 39677
665 github level 12 row 1 with dict dds advanced one pass small out 39677
666 github level 12 row 1 with dict copy advanced one pass small out 39677
667 github level 12 row 1 with dict load advanced one pass small out 41166
668 github level 12 row 2 advanced one pass small out 134180
669 github level 12 row 2 with dict dms advanced one pass small out 39677
670 github level 12 row 2 with dict dds advanced one pass small out 39677
671 github level 12 row 2 with dict copy advanced one pass small out 39677
672 github level 12 row 2 with dict load advanced one pass small out 41166
673 github level 13 advanced one pass small out 134064
674 github level 13 with dict advanced one pass small out 39743
675 github level 13 with dict dms advanced one pass small out 39743
676 github level 13 with dict dds advanced one pass small out 39743
677 github level 13 with dict copy advanced one pass small out 39948
678 github level 13 with dict load advanced one pass small out 42626
679 github level 16 advanced one pass small out 134064
680 github level 16 with dict advanced one pass small out 37577
681 github level 16 with dict dms advanced one pass small out 37577
682 github level 16 with dict dds advanced one pass small out 37577
683 github level 16 with dict copy advanced one pass small out 37568
684 github level 16 with dict load advanced one pass small out 42340
685 github level 19 advanced one pass small out 134064
686 github level 19 with dict advanced one pass small out 37576
687 github level 19 with dict dms advanced one pass small out 37576
688 github level 19 with dict dds advanced one pass small out 37576
689 github level 19 with dict copy advanced one pass small out 37567
690 github level 19 with dict load advanced one pass small out 39613
691 github no source size advanced one pass small out 136335
692 github no source size with dict advanced one pass small out 41148
693 github long distance mode advanced one pass small out 136335
702 github huffman literals advanced one pass small out 142465
703 github multithreaded with advanced params advanced one pass small out 165915
704 github.tar level -5 advanced one pass small out 46856
705 github.tar level -5 with dict advanced one pass small out 43971 44571
706 github.tar level -3 advanced one pass small out 43754
707 github.tar level -3 with dict advanced one pass small out 40805 41447
708 github.tar level -1 advanced one pass small out 42490
709 github.tar level -1 with dict advanced one pass small out 41122 41131
710 github.tar level 0 advanced one pass small out 38441
711 github.tar level 0 with dict advanced one pass small out 37995
712 github.tar level 0 with dict dms advanced one pass small out 38003
713 github.tar level 0 with dict dds advanced one pass small out 38003
714 github.tar level 0 with dict copy advanced one pass small out 37995
715 github.tar level 0 with dict load advanced one pass small out 37956
716 github.tar level 1 advanced one pass small out 39265
717 github.tar level 1 with dict advanced one pass small out 38309 38280
718 github.tar level 1 with dict dms advanced one pass small out 38290
719 github.tar level 1 with dict dds advanced one pass small out 38290
720 github.tar level 1 with dict copy advanced one pass small out 38280
721 github.tar level 1 with dict load advanced one pass small out 38729
722 github.tar level 3 advanced one pass small out 38441
723 github.tar level 3 with dict advanced one pass small out 37995
724 github.tar level 3 with dict dms advanced one pass small out 38003
725 github.tar level 3 with dict dds advanced one pass small out 38003
726 github.tar level 3 with dict copy advanced one pass small out 37995
727 github.tar level 3 with dict load advanced one pass small out 37956
728 github.tar level 4 advanced one pass small out 38467
729 github.tar level 4 with dict advanced one pass small out 37948
730 github.tar level 5 level 4 with dict dms advanced one pass small out 39788 37954
731 github.tar level 5 with dict level 4 with dict dds advanced one pass small out 39715 37954
732 github.tar level 6 level 4 with dict copy advanced one pass small out 39603 37948
733 github.tar level 6 with dict level 4 with dict load advanced one pass small out 38800 37927
734 github.tar level 7 level 5 row 1 advanced one pass small out 39206 39788
735 github.tar level 7 with dict level 5 row 1 with dict dms advanced one pass small out 38071 39365
736 github.tar level 9 level 5 row 1 with dict dds advanced one pass small out 36717 39233
737 github.tar level 9 with dict level 5 row 1 with dict copy advanced one pass small out 36898 39715
738 github.tar level 5 row 1 with dict load advanced one pass small out 39209
739 github.tar level 5 row 2 advanced one pass small out 39693
740 github.tar level 5 row 2 with dict dms advanced one pass small out 39024
741 github.tar level 5 row 2 with dict dds advanced one pass small out 39028
742 github.tar level 5 row 2 with dict copy advanced one pass small out 39040
743 github.tar level 5 row 2 with dict load advanced one pass small out 39037
744 github.tar level 5 advanced one pass small out 39693
745 github.tar level 5 with dict advanced one pass small out 39040
746 github.tar level 5 with dict dms advanced one pass small out 39024
747 github.tar level 5 with dict dds advanced one pass small out 39028
748 github.tar level 5 with dict copy advanced one pass small out 39040
749 github.tar level 5 with dict load advanced one pass small out 39037
750 github.tar level 6 advanced one pass small out 39621
751 github.tar level 6 with dict advanced one pass small out 38622
752 github.tar level 6 with dict dms advanced one pass small out 38608
753 github.tar level 6 with dict dds advanced one pass small out 38610
754 github.tar level 6 with dict copy advanced one pass small out 38622
755 github.tar level 6 with dict load advanced one pass small out 38962
756 github.tar level 7 row 1 advanced one pass small out 39206
757 github.tar level 7 row 1 with dict dms advanced one pass small out 37954
758 github.tar level 7 row 1 with dict dds advanced one pass small out 37954
759 github.tar level 7 row 1 with dict copy advanced one pass small out 38071
760 github.tar level 7 row 1 with dict load advanced one pass small out 38584
761 github.tar level 7 row 2 advanced one pass small out 39213
762 github.tar level 7 row 2 with dict dms advanced one pass small out 37848
763 github.tar level 7 row 2 with dict dds advanced one pass small out 37867
764 github.tar level 7 row 2 with dict copy advanced one pass small out 37848
765 github.tar level 7 row 2 with dict load advanced one pass small out 38582
766 github.tar level 7 advanced one pass small out 39213
767 github.tar level 7 with dict advanced one pass small out 37848
768 github.tar level 7 with dict dms advanced one pass small out 37848
769 github.tar level 7 with dict dds advanced one pass small out 37867
770 github.tar level 7 with dict copy advanced one pass small out 37848
771 github.tar level 7 with dict load advanced one pass small out 38582
772 github.tar level 9 advanced one pass small out 36758
773 github.tar level 9 with dict advanced one pass small out 36457
774 github.tar level 9 with dict dms advanced one pass small out 36549
775 github.tar level 9 with dict dds advanced one pass small out 36637
776 github.tar level 9 with dict copy advanced one pass small out 36457
777 github.tar level 9 with dict load advanced one pass small out 36350
778 github.tar level 12 row 1 advanced one pass small out 36435
779 github.tar level 12 row 1 with dict dms advanced one pass small out 36986
780 github.tar level 12 row 1 with dict dds advanced one pass small out 36986
781 github.tar level 12 row 1 with dict copy advanced one pass small out 36609
782 github.tar level 12 row 1 with dict load advanced one pass small out 36419
783 github.tar level 12 row 2 advanced one pass small out 36435
784 github.tar level 12 row 2 with dict dms advanced one pass small out 36986
785 github.tar level 12 row 2 with dict dds advanced one pass small out 36986
786 github.tar level 12 row 2 with dict copy advanced one pass small out 36609
787 github.tar level 12 row 2 with dict load advanced one pass small out 36424
788 github.tar level 13 advanced one pass small out 35621
789 github.tar level 13 with dict advanced one pass small out 38726
790 github.tar level 13 with dict dms advanced one pass small out 38903
791 github.tar level 13 with dict dds advanced one pass small out 38903
792 github.tar level 13 with dict copy advanced one pass small out 38726
793 github.tar level 13 with dict load advanced one pass small out 36372
794 github.tar level 16 advanced one pass small out 40255
795 github.tar level 16 with dict advanced one pass small out 33639
796 github.tar level 16 with dict dms advanced one pass small out 33544
797 github.tar level 16 with dict dds advanced one pass small out 33544
798 github.tar level 16 with dict copy advanced one pass small out 33639
799 github.tar level 16 with dict load advanced one pass small out 39353
800 github.tar level 19 advanced one pass small out 32837
801 github.tar level 19 with dict advanced one pass small out 32895
802 github.tar level 19 with dict dms advanced one pass small out 32672
803 github.tar level 19 with dict dds advanced one pass small out 32672
804 github.tar level 19 with dict copy advanced one pass small out 32895
805 github.tar level 19 with dict load advanced one pass small out 32676
806 github.tar no source size advanced one pass small out 38441
807 github.tar no source size with dict advanced one pass small out 37995
808 github.tar long distance mode advanced one pass small out 39722
811 github.tar small window log advanced one pass small out 198540
812 github.tar small hash log advanced one pass small out 129870
813 github.tar small chain log advanced one pass small out 41669
814 github.tar explicit params advanced one pass small out 41199 41227
815 github.tar uncompressed literals advanced one pass small out 41122
816 github.tar uncompressed literals optimal advanced one pass small out 35388
817 github.tar huffman literals advanced one pass small out 38777
823 silesia level 1 advanced streaming 5314162
824 silesia level 3 advanced streaming 4849552
825 silesia level 4 advanced streaming 4786970
826 silesia level 5 level 5 row 1 advanced streaming 4710236
827 silesia level 6 level 5 row 2 advanced streaming 4660056 4707794
828 silesia level 7 level 5 advanced streaming 4596296 4707794
829 silesia level 9 level 6 advanced streaming 4543925 4666383
830 silesia level 7 row 1 advanced streaming 4596296
831 silesia level 7 row 2 advanced streaming 4603381
832 silesia level 7 advanced streaming 4603381
833 silesia level 9 advanced streaming 4546001
834 silesia level 12 row 1 advanced streaming 4519288
835 silesia level 12 row 2 advanced streaming 4521397
836 silesia level 13 advanced streaming 4482135
837 silesia level 16 advanced streaming 4377465 4360251
838 silesia level 19 advanced streaming 4293330 4283237
839 silesia no source size advanced streaming 4849516
840 silesia long distance mode advanced streaming 4840744
841 silesia multithreaded advanced streaming 4849552
842 silesia multithreaded long distance mode advanced streaming 4840744
843 silesia small window log advanced streaming 7112062
844 silesia small hash log advanced streaming 6555021 6526141
845 silesia small chain log advanced streaming 4931148 4912197
846 silesia explicit params advanced streaming 4797112 4795887
847 silesia uncompressed literals advanced streaming 5127982
848 silesia uncompressed literals optimal advanced streaming 4325472 4317896
849 silesia huffman literals advanced streaming 5331168
850 silesia multithreaded with advanced params advanced streaming 5127982
851 silesia.tar level -5 advanced streaming 6982759
855 silesia.tar level 1 advanced streaming 5336939
856 silesia.tar level 3 advanced streaming 4861427
857 silesia.tar level 4 advanced streaming 4799630
858 silesia.tar level 5 level 5 row 1 advanced streaming 4722329
859 silesia.tar level 6 level 5 row 2 advanced streaming 4672288 4719261
860 silesia.tar level 7 level 5 advanced streaming 4606715 4719261
861 silesia.tar level 9 level 6 advanced streaming 4554154 4677729
862 silesia.tar level 7 row 1 advanced streaming 4606715
863 silesia.tar level 7 row 2 advanced streaming 4613544
864 silesia.tar level 7 advanced streaming 4613544
865 silesia.tar level 9 advanced streaming 4555432
866 silesia.tar level 12 row 1 advanced streaming 4529459
867 silesia.tar level 12 row 2 advanced streaming 4530258
868 silesia.tar level 13 advanced streaming 4491765
869 silesia.tar level 16 advanced streaming 4381350 4356834
870 silesia.tar level 19 advanced streaming 4281562 4264392
871 silesia.tar no source size advanced streaming 4861423
872 silesia.tar long distance mode advanced streaming 4847735
873 silesia.tar multithreaded advanced streaming 4861508
874 silesia.tar multithreaded long distance mode advanced streaming 4853149
875 silesia.tar small window log advanced streaming 7118769
876 silesia.tar small hash log advanced streaming 6587952 6529235
877 silesia.tar small chain log advanced streaming 4943312 4917021
878 silesia.tar explicit params advanced streaming 4808618 4807401
879 silesia.tar uncompressed literals advanced streaming 5129461
880 silesia.tar uncompressed literals optimal advanced streaming 4320858 4307400
881 silesia.tar huffman literals advanced streaming 5352360
882 silesia.tar multithreaded with advanced params advanced streaming 5129555
883 github level -5 advanced streaming 205285
888 github level -1 with dict advanced streaming 43170
889 github level 0 advanced streaming 136335
890 github level 0 with dict advanced streaming 41148
891 github level 0 with dict dms advanced streaming 41148
892 github level 0 with dict dds advanced streaming 41148
893 github level 0 with dict copy advanced streaming 41124
894 github level 0 with dict load advanced streaming 42252
895 github level 1 advanced streaming 142465
896 github level 1 with dict advanced streaming 41682
897 github level 1 with dict dms advanced streaming 41682
898 github level 1 with dict dds advanced streaming 41682
899 github level 1 with dict copy advanced streaming 41674
900 github level 1 with dict load advanced streaming 43755
901 github level 3 advanced streaming 136335
902 github level 3 with dict advanced streaming 41148
903 github level 3 with dict dms advanced streaming 41148
904 github level 3 with dict dds advanced streaming 41148
905 github level 3 with dict copy advanced streaming 41124
906 github level 3 with dict load advanced streaming 42252
907 github level 4 advanced streaming 136199
908 github level 4 with dict advanced streaming 41251
909 github level 4 with dict dms advanced streaming 41251
910 github level 4 with dict dds advanced streaming 41251
911 github level 4 with dict copy advanced streaming 41216
912 github level 4 with dict load advanced streaming 41159
913 github level 5 row 1 advanced streaming 135121
914 github level 5 row 1 with dict dms advanced streaming 38938
915 github level 5 row 1 with dict dds advanced streaming 38732
916 github level 5 row 1 with dict copy advanced streaming 38934
917 github level 5 row 1 with dict load advanced streaming 40725
918 github level 5 row 2 advanced streaming 134584
919 github level 5 row 2 with dict dms advanced streaming 38758
920 github level 5 row 2 with dict dds advanced streaming 38728
921 github level 5 row 2 with dict copy advanced streaming 38759
922 github level 5 row 2 with dict load advanced streaming 41518
923 github level 5 advanced streaming 135121
924 github level 5 with dict advanced streaming 38938 38758
925 github level 5 with dict dms advanced streaming 38758
926 github level 5 with dict dds advanced streaming 38728
927 github level 5 with dict copy advanced streaming 38759
928 github level 5 with dict load advanced streaming 40725
929 github level 6 advanced streaming 135122
930 github level 6 with dict advanced streaming 38632 38671
931 github level 6 with dict dms advanced streaming 38671
932 github level 6 with dict dds advanced streaming 38630
933 github level 6 with dict copy advanced streaming 38669
934 github level 6 with dict load advanced streaming 40695
935 github level 7 row 1 advanced streaming 135122
936 github level 7 row 1 with dict dms advanced streaming 38771
937 github level 7 row 1 with dict dds advanced streaming 38771
938 github level 7 row 1 with dict copy advanced streaming 38745
939 github level 7 row 1 with dict load advanced streaming 40695
940 github level 7 row 2 advanced streaming 134584
941 github level 7 row 2 with dict dms advanced streaming 38758
942 github level 7 row 2 with dict dds advanced streaming 38747
943 github level 7 row 2 with dict copy advanced streaming 38755
944 github level 7 row 2 with dict load advanced streaming 41030
945 github level 7 advanced streaming 135122
946 github level 7 with dict advanced streaming 38771 38758
947 github level 7 with dict dms advanced streaming 38758
948 github level 7 with dict dds advanced streaming 38747
949 github level 7 with dict copy advanced streaming 38755
950 github level 7 with dict load advanced streaming 40695
951 github level 9 advanced streaming 135122
952 github level 9 with dict advanced streaming 39332 39437
953 github level 9 with dict dms advanced streaming 39437
954 github level 9 with dict dds advanced streaming 39338
955 github level 9 with dict copy advanced streaming 39398
956 github level 9 with dict load advanced streaming 41710
957 github level 12 row 1 advanced streaming 134180
958 github level 12 row 1 with dict dms advanced streaming 39677
959 github level 12 row 1 with dict dds advanced streaming 39677
960 github level 12 row 1 with dict copy advanced streaming 39677
961 github level 12 row 1 with dict load advanced streaming 41166
962 github level 12 row 2 advanced streaming 134180
963 github level 12 row 2 with dict dms advanced streaming 39677
964 github level 12 row 2 with dict dds advanced streaming 39677
965 github level 12 row 2 with dict copy advanced streaming 39677
966 github level 12 row 2 with dict load advanced streaming 41166
967 github level 13 advanced streaming 134064
968 github level 13 with dict advanced streaming 39743
969 github level 13 with dict dms advanced streaming 39743
970 github level 13 with dict dds advanced streaming 39743
971 github level 13 with dict copy advanced streaming 39948
972 github level 13 with dict load advanced streaming 42626
973 github level 16 advanced streaming 134064
974 github level 16 with dict advanced streaming 37577
975 github level 16 with dict dms advanced streaming 37577
976 github level 16 with dict dds advanced streaming 37577
977 github level 16 with dict copy advanced streaming 37568
978 github level 16 with dict load advanced streaming 42340
979 github level 19 advanced streaming 134064
980 github level 19 with dict advanced streaming 37576
981 github level 19 with dict dms advanced streaming 37576
982 github level 19 with dict dds advanced streaming 37576
983 github level 19 with dict copy advanced streaming 37567
984 github level 19 with dict load advanced streaming 39613
985 github no source size advanced streaming 136335
986 github no source size with dict advanced streaming 41148
987 github long distance mode advanced streaming 136335
996 github huffman literals advanced streaming 142465
997 github multithreaded with advanced params advanced streaming 165915
998 github.tar level -5 advanced streaming 46747
999 github.tar level -5 with dict advanced streaming 43971 44440
1000 github.tar level -3 advanced streaming 43537
1001 github.tar level -3 with dict advanced streaming 40805 41112
1002 github.tar level -1 advanced streaming 42465
1003 github.tar level -1 with dict advanced streaming 41122 41196
1004 github.tar level 0 advanced streaming 38441
1005 github.tar level 0 with dict advanced streaming 37995
1006 github.tar level 0 with dict dms advanced streaming 38003
1007 github.tar level 0 with dict dds advanced streaming 38003
1008 github.tar level 0 with dict copy advanced streaming 37995
1009 github.tar level 0 with dict load advanced streaming 37956
1010 github.tar level 1 advanced streaming 39342
1011 github.tar level 1 with dict advanced streaming 38309 38293
1012 github.tar level 1 with dict dms advanced streaming 38303
1013 github.tar level 1 with dict dds advanced streaming 38303
1014 github.tar level 1 with dict copy advanced streaming 38293
1015 github.tar level 1 with dict load advanced streaming 38766
1016 github.tar level 3 advanced streaming 38441
1017 github.tar level 3 with dict advanced streaming 37995
1018 github.tar level 3 with dict dms advanced streaming 38003
1019 github.tar level 3 with dict dds advanced streaming 38003
1020 github.tar level 3 with dict copy advanced streaming 37995
1021 github.tar level 3 with dict load advanced streaming 37956
1022 github.tar level 4 advanced streaming 38467
1023 github.tar level 4 with dict advanced streaming 37948
1024 github.tar level 5 level 4 with dict dms advanced streaming 39788 37954
1025 github.tar level 5 with dict level 4 with dict dds advanced streaming 39715 37954
1026 github.tar level 6 level 4 with dict copy advanced streaming 39603 37948
1027 github.tar level 6 with dict level 4 with dict load advanced streaming 38800 37927
1028 github.tar level 7 level 5 row 1 advanced streaming 39206 39788
1029 github.tar level 7 with dict level 5 row 1 with dict dms advanced streaming 38071 39365
1030 github.tar level 9 level 5 row 1 with dict dds advanced streaming 36717 39233
1031 github.tar level 9 with dict level 5 row 1 with dict copy advanced streaming 36898 39715
1032 github.tar level 5 row 1 with dict load advanced streaming 39209
1033 github.tar level 5 row 2 advanced streaming 39693
1034 github.tar level 5 row 2 with dict dms advanced streaming 39024
1035 github.tar level 5 row 2 with dict dds advanced streaming 39028
1036 github.tar level 5 row 2 with dict copy advanced streaming 39040
1037 github.tar level 5 row 2 with dict load advanced streaming 39037
1038 github.tar level 5 advanced streaming 39693
1039 github.tar level 5 with dict advanced streaming 39040
1040 github.tar level 5 with dict dms advanced streaming 39024
1041 github.tar level 5 with dict dds advanced streaming 39028
1042 github.tar level 5 with dict copy advanced streaming 39040
1043 github.tar level 5 with dict load advanced streaming 39037
1044 github.tar level 6 advanced streaming 39621
1045 github.tar level 6 with dict advanced streaming 38622
1046 github.tar level 6 with dict dms advanced streaming 38608
1047 github.tar level 6 with dict dds advanced streaming 38610
1048 github.tar level 6 with dict copy advanced streaming 38622
1049 github.tar level 6 with dict load advanced streaming 38962
1050 github.tar level 7 row 1 advanced streaming 39206
1051 github.tar level 7 row 1 with dict dms advanced streaming 37954
1052 github.tar level 7 row 1 with dict dds advanced streaming 37954
1053 github.tar level 7 row 1 with dict copy advanced streaming 38071
1054 github.tar level 7 row 1 with dict load advanced streaming 38584
1055 github.tar level 7 row 2 advanced streaming 39213
1056 github.tar level 7 row 2 with dict dms advanced streaming 37848
1057 github.tar level 7 row 2 with dict dds advanced streaming 37867
1058 github.tar level 7 row 2 with dict copy advanced streaming 37848
1059 github.tar level 7 row 2 with dict load advanced streaming 38582
1060 github.tar level 7 advanced streaming 39213
1061 github.tar level 7 with dict advanced streaming 37848
1062 github.tar level 7 with dict dms advanced streaming 37848
1063 github.tar level 7 with dict dds advanced streaming 37867
1064 github.tar level 7 with dict copy advanced streaming 37848
1065 github.tar level 7 with dict load advanced streaming 38582
1066 github.tar level 9 advanced streaming 36758
1067 github.tar level 9 with dict advanced streaming 36457
1068 github.tar level 9 with dict dms advanced streaming 36549
1069 github.tar level 9 with dict dds advanced streaming 36637
1070 github.tar level 9 with dict copy advanced streaming 36457
1071 github.tar level 9 with dict load advanced streaming 36350
1072 github.tar level 12 row 1 advanced streaming 36435
1073 github.tar level 12 row 1 with dict dms advanced streaming 36986
1074 github.tar level 12 row 1 with dict dds advanced streaming 36986
1075 github.tar level 12 row 1 with dict copy advanced streaming 36609
1076 github.tar level 12 row 1 with dict load advanced streaming 36419
1077 github.tar level 12 row 2 advanced streaming 36435
1078 github.tar level 12 row 2 with dict dms advanced streaming 36986
1079 github.tar level 12 row 2 with dict dds advanced streaming 36986
1080 github.tar level 12 row 2 with dict copy advanced streaming 36609
1081 github.tar level 12 row 2 with dict load advanced streaming 36424
1082 github.tar level 13 advanced streaming 35621
1083 github.tar level 13 with dict advanced streaming 38726
1084 github.tar level 13 with dict dms advanced streaming 38903
1085 github.tar level 13 with dict dds advanced streaming 38903
1086 github.tar level 13 with dict copy advanced streaming 38726
1087 github.tar level 13 with dict load advanced streaming 36372
1088 github.tar level 16 advanced streaming 40255
1089 github.tar level 16 with dict advanced streaming 33639
1090 github.tar level 16 with dict dms advanced streaming 33544
1091 github.tar level 16 with dict dds advanced streaming 33544
1092 github.tar level 16 with dict copy advanced streaming 33639
1093 github.tar level 16 with dict load advanced streaming 39353
1094 github.tar level 19 advanced streaming 32837
1095 github.tar level 19 with dict advanced streaming 32895
1096 github.tar level 19 with dict dms advanced streaming 32672
1097 github.tar level 19 with dict dds advanced streaming 32672
1098 github.tar level 19 with dict copy advanced streaming 32895
1099 github.tar level 19 with dict load advanced streaming 32676
1100 github.tar no source size advanced streaming 38438
1101 github.tar no source size with dict advanced streaming 38000
1102 github.tar long distance mode advanced streaming 39722
1105 github.tar small window log advanced streaming 199558
1106 github.tar small hash log advanced streaming 129870
1107 github.tar small chain log advanced streaming 41669
1108 github.tar explicit params advanced streaming 41199 41227
1109 github.tar uncompressed literals advanced streaming 41122
1110 github.tar uncompressed literals optimal advanced streaming 35388
1111 github.tar huffman literals advanced streaming 38800
1117 silesia level 1 old streaming 5314162
1118 silesia level 3 old streaming 4849552
1119 silesia level 4 old streaming 4786970
1120 silesia level 5 old streaming 4710236 4707794
1121 silesia level 6 old streaming 4660056 4666383
1122 silesia level 7 old streaming 4596296 4603381
1123 silesia level 9 old streaming 4543925 4546001
1124 silesia level 13 old streaming 4482135
1125 silesia level 16 old streaming 4377465 4360251
1126 silesia level 19 old streaming 4293330 4283237
1127 silesia no source size old streaming 4849516
1128 silesia uncompressed literals old streaming 4849552
1129 silesia uncompressed literals optimal old streaming 4293330 4283237
1130 silesia huffman literals old streaming 6183403
1131 silesia.tar level -5 old streaming 6982759
1132 silesia.tar level -3 old streaming 6641283
1135 silesia.tar level 1 old streaming 5336939
1136 silesia.tar level 3 old streaming 4861427
1137 silesia.tar level 4 old streaming 4799630
1138 silesia.tar level 5 old streaming 4722329 4719261
1139 silesia.tar level 6 old streaming 4672288 4677729
1140 silesia.tar level 7 old streaming 4606715 4613544
1141 silesia.tar level 9 old streaming 4554154 4555432
1142 silesia.tar level 13 old streaming 4491765
1143 silesia.tar level 16 old streaming 4381350 4356834
1144 silesia.tar level 19 old streaming 4281562 4264392
1145 silesia.tar no source size old streaming 4861423
1146 silesia.tar uncompressed literals old streaming 4861427
1147 silesia.tar uncompressed literals optimal old streaming 4281562 4264392
1148 silesia.tar huffman literals old streaming 6190795
1149 github level -5 old streaming 205285
1150 github level -5 with dict old streaming 46718
1161 github level 4 old streaming 136199
1162 github level 4 with dict old streaming 41251
1163 github level 5 old streaming 135121
1164 github level 5 with dict old streaming 38938 38758
1165 github level 6 old streaming 135122
1166 github level 6 with dict old streaming 38632 38671
1167 github level 7 old streaming 135122
1168 github level 7 with dict old streaming 38771 38758
1169 github level 9 old streaming 135122
1170 github level 9 with dict old streaming 39332 39437
1171 github level 13 old streaming 134064
1172 github level 13 with dict old streaming 39743
1173 github level 16 old streaming 134064
1180 github uncompressed literals optimal old streaming 134064
1181 github huffman literals old streaming 175568
1182 github.tar level -5 old streaming 46747
1183 github.tar level -5 with dict old streaming 43971 44440
1184 github.tar level -3 old streaming 43537
1185 github.tar level -3 with dict old streaming 40805 41112
1186 github.tar level -1 old streaming 42465
1187 github.tar level -1 with dict old streaming 41122 41196
1188 github.tar level 0 old streaming 38441
1189 github.tar level 0 with dict old streaming 37995
1190 github.tar level 1 old streaming 39342
1191 github.tar level 1 with dict old streaming 38309 38293
1192 github.tar level 3 old streaming 38441
1193 github.tar level 3 with dict old streaming 37995
1194 github.tar level 4 old streaming 38467
1195 github.tar level 4 with dict old streaming 37948
1196 github.tar level 5 old streaming 39788 39693
1197 github.tar level 5 with dict old streaming 39715 39040
1198 github.tar level 6 old streaming 39603 39621
1199 github.tar level 6 with dict old streaming 38800 38622
1200 github.tar level 7 old streaming 39206 39213
1201 github.tar level 7 with dict old streaming 38071 37848
1202 github.tar level 9 old streaming 36717 36758
1203 github.tar level 9 with dict old streaming 36898 36457
1204 github.tar level 13 old streaming 35621
1205 github.tar level 13 with dict old streaming 38726
1206 github.tar level 16 old streaming 40255
1219 silesia level 1 old streaming advanced 5314162
1220 silesia level 3 old streaming advanced 4849552
1221 silesia level 4 old streaming advanced 4786970
1222 silesia level 5 old streaming advanced 4710236 4707794
1223 silesia level 6 old streaming advanced 4660056 4666383
1224 silesia level 7 old streaming advanced 4596296 4603381
1225 silesia level 9 old streaming advanced 4543925 4546001
1226 silesia level 13 old streaming advanced 4482135
1227 silesia level 16 old streaming advanced 4377465 4360251
1228 silesia level 19 old streaming advanced 4293330 4283237
1229 silesia no source size old streaming advanced 4849516
1230 silesia long distance mode old streaming advanced 4849552
1231 silesia multithreaded old streaming advanced 4849552
1232 silesia multithreaded long distance mode old streaming advanced 4849552
1233 silesia small window log old streaming advanced 7112062
1234 silesia small hash log old streaming advanced 6555021 6526141
1235 silesia small chain log old streaming advanced 4931148 4912197
1236 silesia explicit params old streaming advanced 4797112 4795887
1237 silesia uncompressed literals old streaming advanced 4849552
1238 silesia uncompressed literals optimal old streaming advanced 4293330 4283237
1239 silesia huffman literals old streaming advanced 6183403
1240 silesia multithreaded with advanced params old streaming advanced 4849552
1241 silesia.tar level -5 old streaming advanced 6982759
1245 silesia.tar level 1 old streaming advanced 5336939
1246 silesia.tar level 3 old streaming advanced 4861427
1247 silesia.tar level 4 old streaming advanced 4799630
1248 silesia.tar level 5 old streaming advanced 4722329 4719261
1249 silesia.tar level 6 old streaming advanced 4672288 4677729
1250 silesia.tar level 7 old streaming advanced 4606715 4613544
1251 silesia.tar level 9 old streaming advanced 4554154 4555432
1252 silesia.tar level 13 old streaming advanced 4491765
1253 silesia.tar level 16 old streaming advanced 4381350 4356834
1254 silesia.tar level 19 old streaming advanced 4281562 4264392
1255 silesia.tar no source size old streaming advanced 4861423
1256 silesia.tar long distance mode old streaming advanced 4861427
1257 silesia.tar multithreaded old streaming advanced 4861427
1258 silesia.tar multithreaded long distance mode old streaming advanced 4861427
1259 silesia.tar small window log old streaming advanced 7118772
1260 silesia.tar small hash log old streaming advanced 6587952 6529235
1261 silesia.tar small chain log old streaming advanced 4943312 4917021
1262 silesia.tar explicit params old streaming advanced 4808618 4807401
1263 silesia.tar uncompressed literals old streaming advanced 4861427
1264 silesia.tar uncompressed literals optimal old streaming advanced 4281562 4264392
1265 silesia.tar huffman literals old streaming advanced 6190795
1266 silesia.tar multithreaded with advanced params old streaming advanced 4861427
1267 github level -5 old streaming advanced 216734
1279 github level 4 old streaming advanced 141104
1280 github level 4 with dict old streaming advanced 41084
1281 github level 5 old streaming advanced 139399
1282 github level 5 with dict old streaming advanced 39159 38633
1283 github level 6 old streaming advanced 139402
1284 github level 6 with dict old streaming advanced 38749 38723
1285 github level 7 old streaming advanced 138676
1286 github level 7 with dict old streaming advanced 38746 38744
1287 github level 9 old streaming advanced 138676
1288 github level 9 with dict old streaming advanced 38993 38981
1289 github level 13 old streaming advanced 138676
1290 github level 13 with dict old streaming advanced 39731
1291 github level 16 old streaming advanced 138676
1319 github.tar level 3 with dict old streaming advanced 38013
1320 github.tar level 4 old streaming advanced 38467
1321 github.tar level 4 with dict old streaming advanced 38063
1322 github.tar level 5 old streaming advanced 39788 39693
1323 github.tar level 5 with dict old streaming advanced 39310 39049
1324 github.tar level 6 old streaming advanced 39603 39621
1325 github.tar level 6 with dict old streaming advanced 39279 38959
1326 github.tar level 7 old streaming advanced 39206 39213
1327 github.tar level 7 with dict old streaming advanced 38728 38573
1328 github.tar level 9 old streaming advanced 36717 36758
1329 github.tar level 9 with dict old streaming advanced 36504 36233
1330 github.tar level 13 old streaming advanced 35621
1331 github.tar level 13 with dict old streaming advanced 36035
1332 github.tar level 16 old streaming advanced 40255
1341 github.tar small window log old streaming advanced 199561
1342 github.tar small hash log old streaming advanced 129870
1343 github.tar small chain log old streaming advanced 41669
1344 github.tar explicit params old streaming advanced 41199 41227
1345 github.tar uncompressed literals old streaming advanced 38441
1346 github.tar uncompressed literals optimal old streaming advanced 32837
1347 github.tar huffman literals old streaming advanced 42465
1348 github.tar multithreaded with advanced params old streaming advanced 38441
1349 github level -5 with dict old streaming cdcit old streaming cdict 46718
1350 github level -3 with dict old streaming cdcit old streaming cdict 45395
1351 github level -1 with dict old streaming cdcit old streaming cdict 43170
1352 github level 0 with dict old streaming cdcit old streaming cdict 41148
1353 github level 1 with dict old streaming cdcit old streaming cdict 41682
1354 github level 3 with dict old streaming cdcit old streaming cdict 41148
1355 github level 4 with dict old streaming cdcit old streaming cdict 41251
1356 github level 5 with dict old streaming cdcit old streaming cdict 38938 38758
1357 github level 6 with dict old streaming cdcit old streaming cdict 38632 38671
1358 github level 7 with dict old streaming cdcit old streaming cdict 38771 38758
1359 github level 9 with dict old streaming cdcit old streaming cdict 39332 39437
1360 github level 13 with dict old streaming cdcit old streaming cdict 39743
1361 github level 16 with dict old streaming cdcit old streaming cdict 37577
1362 github level 19 with dict old streaming cdcit old streaming cdict 37576
1363 github no source size with dict old streaming cdcit old streaming cdict 40654
1364 github.tar level -5 with dict old streaming cdcit old streaming cdict 45018
1365 github.tar level -3 with dict old streaming cdcit old streaming cdict 41886
1366 github.tar level -1 with dict old streaming cdcit old streaming cdict 41636
1367 github.tar level 0 with dict old streaming cdcit old streaming cdict 37956
1368 github.tar level 1 with dict old streaming cdcit old streaming cdict 38766
1369 github.tar level 3 with dict old streaming cdcit old streaming cdict 37956
1370 github.tar level 4 with dict old streaming cdcit old streaming cdict 37927
1371 github.tar level 5 with dict old streaming cdcit old streaming cdict 39209 39037
1372 github.tar level 6 with dict old streaming cdcit old streaming cdict 38983 38962
1373 github.tar level 7 with dict old streaming cdcit old streaming cdict 38584 38582
1374 github.tar level 9 with dict old streaming cdcit old streaming cdict 36363 36350
1375 github.tar level 13 with dict old streaming cdcit old streaming cdict 36372
1376 github.tar level 16 with dict old streaming cdcit old streaming cdict 39353
1377 github.tar level 19 with dict old streaming cdcit old streaming cdict 32676
1378 github.tar no source size with dict old streaming cdcit old streaming cdict 38000
1379 github level -5 with dict old streaming advanced cdict 49562
1380 github level -3 with dict old streaming advanced cdict 44956
1381 github level -1 with dict old streaming advanced cdict 42383
1383 github level 1 with dict old streaming advanced cdict 42430
1384 github level 3 with dict old streaming advanced cdict 41113
1385 github level 4 with dict old streaming advanced cdict 41084
1386 github level 5 with dict old streaming advanced cdict 39159 38633
1387 github level 6 with dict old streaming advanced cdict 38749 38723
1388 github level 7 with dict old streaming advanced cdict 38746 38744
1389 github level 9 with dict old streaming advanced cdict 38993 38981
1390 github level 13 with dict old streaming advanced cdict 39731
1391 github level 16 with dict old streaming advanced cdict 40789
1392 github level 19 with dict old streaming advanced cdict 37576
1398 github.tar level 1 with dict old streaming advanced cdict 39002
1399 github.tar level 3 with dict old streaming advanced cdict 38013
1400 github.tar level 4 with dict old streaming advanced cdict 38063
1401 github.tar level 5 with dict old streaming advanced cdict 39310 39049
1402 github.tar level 6 with dict old streaming advanced cdict 39279 38959
1403 github.tar level 7 with dict old streaming advanced cdict 38728 38573
1404 github.tar level 9 with dict old streaming advanced cdict 36504 36233
1405 github.tar level 13 with dict old streaming advanced cdict 36035
1406 github.tar level 16 with dict old streaming advanced cdict 38736
1407 github.tar level 19 with dict old streaming advanced cdict 32876
tests/regression/test.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/roundTripCrash.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/seqgen.c
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2017-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/seqgen.h
+1 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2017-2021, Facebook, Inc.
* Copyright (c) Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
tests/test-license.py
+43 -29
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
# ################################################################
# Copyright (c) 2016-2021, Facebook, Inc.
# Copyright (c) Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
@@ -10,16 +10,12 @@
# You may select, at your option, one of the above-listed licenses.
# ################################################################
import datetime
import enum
import glob
import os
import re
import sys
YEAR = datetime.datetime.now().year
YEAR_STR = str(YEAR)
ROOT = os.path.join(os.path.dirname(__file__), "..")
RELDIRS = [
@@ -28,22 +24,26 @@ RELDIRS = [
"lib",
"programs",
"tests",
"contrib/linux-kernel",
]
DIRS = [os.path.join(ROOT, d) for d in RELDIRS]
REL_EXCLUDES = [
"contrib/linux-kernel/test/include",
]
class File(enum.Enum):
C = 1
H = 2
MAKE = 3
PY = 4
def to_abs(d):
return os.path.normpath(os.path.join(ROOT, d)) + "/"
SUFFIX = {
File.C: ".c",
File.H: ".h",
File.MAKE: "Makefile",
File.PY: ".py",
}
DIRS = [to_abs(d) for d in RELDIRS]
EXCLUDES = [to_abs(d) for d in REL_EXCLUDES]
SUFFIXES = [
".c",
".h",
"Makefile",
".mk",
".py",
]
# License should certainly be in the first 10 KB.
MAX_BYTES = 10000
@@ -69,10 +69,13 @@ LICENSE_EXCEPTIONS = {
# From divsufsort
"divsufsort.c",
"divsufsort.h",
# License is slightly different because it references GitHub
"linux_zstd.h",
}
def valid_copyright(lines):
YEAR_REGEX = re.compile("\d\d\d\d|present")
for line in lines:
line = line.strip()
if "Copyright" not in line:
@@ -81,8 +84,9 @@ def valid_copyright(lines):
return (False, f"Copyright line '{line}' contains 'present'!")
if "Facebook, Inc" not in line:
return (False, f"Copyright line '{line}' does not contain 'Facebook, Inc'")
if YEAR_STR not in line:
return (False, f"Copyright line '{line}' does not contain {YEAR}")
year = YEAR_REGEX.search(line)
if year is not None:
return (False, f"Copyright line '{line}' contains {year.group(0)}; it should be yearless")
if " (c) " not in line:
return (False, f"Copyright line '{line}' does not contain ' (c) '!")
return (True, "")
@@ -107,35 +111,45 @@ def valid_file(filename):
with open(filename, "r") as f:
lines = f.readlines(MAX_BYTES)
lines = lines[:min(len(lines), MAX_LINES)]
ok = True
if os.path.basename(filename) not in COPYRIGHT_EXCEPTIONS:
c_ok, c_msg = valid_copyright(lines)
if not c_ok:
print(f"{filename}: {c_msg}")
print(f"{filename}: {c_msg}", file=sys.stderr)
ok = False
if os.path.basename(filename) not in LICENSE_EXCEPTIONS:
l_ok, l_msg = valid_license(lines)
if not l_ok:
print(f"{filename}: {l_msg}")
print(f"{filename}: {l_msg}", file=sys.stderr)
ok = False
return ok
def exclude(filename):
for x in EXCLUDES:
if filename.startswith(x):
return True
return False
def main():
invalid_files = []
for directory in DIRS:
for suffix in SUFFIX.values():
files = set(glob.glob(f"{directory}/*{suffix}"))
files |= set(glob.glob(f"{directory}/**/*{suffix}"))
for suffix in SUFFIXES:
files = set(glob.glob(f"{directory}/**/*{suffix}", recursive=True))
for filename in files:
if exclude(filename):
continue
if not valid_file(filename):
invalid_files.append(filename)
if len(invalid_files) > 0:
print(f"Invalid files: {invalid_files}")
print("Fail!", file=sys.stderr)
for f in invalid_files:
print(f)
return 1
else:
print("Pass!")
return len(invalid_files)
print("Pass!", file=sys.stderr)
return 0
if __name__ == "__main__":
sys.exit(main())
tests/test-zstd-versions.py
+1 -1
View File
@@ -2,7 +2,7 @@
"""Test zstd interoperability between versions"""
# ################################################################
# Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
# Copyright (c) Yann Collet, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
tests/zbufftest.c
-625
View File
@@ -1,625 +0,0 @@
/*
* Copyright (c) 2015-2021, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/*-************************************
* Compiler specific
**************************************/
#ifdef _MSC_VER /* Visual Studio */
# define _CRT_SECURE_NO_WARNINGS /* fgets */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
#endif
/*-************************************
* Includes
**************************************/
#include <stdlib.h> /* free */
#include <stdio.h> /* fgets, sscanf */
#include <string.h> /* strcmp */
#include "timefn.h" /* UTIL_time_t */
#include "mem.h"
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */
#include "zstd.h" /* ZSTD_compressBound */
#define ZBUFF_STATIC_LINKING_ONLY /* ZBUFF_createCCtx_advanced */
#include "zbuff.h" /* ZBUFF_isError */
#include "datagen.h" /* RDG_genBuffer */
#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h" /* XXH64_* */
#include "util.h"
#include "assert.h"
/*-************************************
* Constants
**************************************/
#define KB *(1U<<10)
#define MB *(1U<<20)
#define GB *(1U<<30)
static const U32 nbTestsDefault = 10000;
#define COMPRESSIBLE_NOISE_LENGTH (10 MB)
#define FUZ_COMPRESSIBILITY_DEFAULT 50
static const U32 prime1 = 2654435761U;
static const U32 prime2 = 2246822519U;
/*-************************************
* Display Macros
**************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static U32 g_displayLevel = 2;
static const U64 g_refreshRate = SEC_TO_MICRO / 6;
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
{ g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
if (g_displayLevel>=4) fflush(stderr); } }
static U64 g_clockTime = 0;
/*-*******************************************************
* Fuzzer functions
*********************************************************/
#undef MIN
#undef MAX
#define MIN(a,b) ((a)<(b)?(a):(b))
#define MAX(a,b) ((a)>(b)?(a):(b))
/*! FUZ_rand() :
@return : a 27 bits random value, from a 32-bits `seed`.
`seed` is also modified */
# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
static unsigned int FUZ_rand(unsigned int* seedPtr)
{
U32 rand32 = *seedPtr;
rand32 *= prime1;
rand32 += prime2;
rand32 = FUZ_rotl32(rand32, 13);
*seedPtr = rand32;
return rand32 >> 5;
}
/*
static unsigned FUZ_highbit32(U32 v32)
{
unsigned nbBits = 0;
if (v32==0) return 0;
for ( ; v32 ; v32>>=1) nbBits++;
return nbBits;
}
*/
static void* ZBUFF_allocFunction(void* opaque, size_t size)
{
void* address = malloc(size);
(void)opaque;
/* DISPLAYLEVEL(4, "alloc %p, %d opaque=%p \n", address, (int)size, opaque); */
return address;
}
static void ZBUFF_freeFunction(void* opaque, void* address)
{
(void)opaque;
/* if (address) DISPLAYLEVEL(4, "free %p opaque=%p \n", address, opaque); */
free(address);
}
static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem)
{
int testResult = 0;
size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
void* CNBuffer = malloc(CNBufferSize);
size_t const skippableFrameSize = 11;
size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
void* compressedBuffer = malloc(compressedBufferSize);
size_t const decodedBufferSize = CNBufferSize;
void* decodedBuffer = malloc(decodedBufferSize);
size_t cSize, readSize, readSkipSize, genSize;
U32 testNb=0;
ZBUFF_CCtx* zc = ZBUFF_createCCtx_advanced(customMem);
ZBUFF_DCtx* zd = ZBUFF_createDCtx_advanced(customMem);
/* Create compressible test buffer */
if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) {
DISPLAY("Not enough memory, aborting\n");
goto _output_error;
}
RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);
/* generate skippable frame */
MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
cSize = skippableFrameSize + 8;
/* Basic compression test */
DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1);
readSize = CNBufferSize;
genSize = compressedBufferSize;
{ size_t const r = ZBUFF_compressContinue(zc, ((char*)compressedBuffer)+cSize, &genSize, CNBuffer, &readSize);
if (ZBUFF_isError(r)) goto _output_error; }
if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */
cSize += genSize;
genSize = compressedBufferSize - cSize;
{ size_t const r = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize);
if (r != 0) goto _output_error; } /* error, or some data not flushed */
cSize += genSize;
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
/* skippable frame test */
DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++);
ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
readSkipSize = cSize;
genSize = CNBufferSize;
{ size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSkipSize);
if (r != 0) goto _output_error; }
if (genSize != 0) goto _output_error; /* skippable frame len is 0 */
DISPLAYLEVEL(4, "OK \n");
/* Basic decompression test */
DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
readSize = cSize - readSkipSize;
genSize = CNBufferSize;
{ size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, ((char*)compressedBuffer)+readSkipSize, &readSize);
if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */
if (readSize+readSkipSize != cSize) goto _output_error; /* should have read the entire frame */
DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : ZBUFF_recommendedCInSize : ", testNb++); { assert(ZBUFF_recommendedCInSize() != 0); } DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : ZBUFF_recommendedCOutSize : ", testNb++); { assert(ZBUFF_recommendedCOutSize() != 0); } DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : ZBUFF_recommendedDInSize : ", testNb++); { assert(ZBUFF_recommendedDInSize() != 0); } DISPLAYLEVEL(4, "OK \n");
DISPLAYLEVEL(4, "test%3i : ZBUFF_recommendedDOutSize : ", testNb++); { assert(ZBUFF_recommendedDOutSize() != 0); } DISPLAYLEVEL(4, "OK \n");
/* check regenerated data is byte exact */
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
{ size_t i;
for (i=0; i<CNBufferSize; i++) {
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;
} }
DISPLAYLEVEL(4, "OK \n");
/* Byte-by-byte decompression test */
DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++);
{ size_t r, pIn=0, pOut=0;
do
{ ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB);
r = 1;
while (r) {
size_t inS = 1;
size_t outS = 1;
r = ZBUFF_decompressContinue(zd, ((BYTE*)decodedBuffer)+pOut, &outS, ((BYTE*)compressedBuffer)+pIn, &inS);
pIn += inS;
pOut += outS;
}
readSize = pIn;
genSize = pOut;
} while (genSize==0);
}
if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */
if (readSize != cSize) goto _output_error; /* should have read the entire frame */
DISPLAYLEVEL(4, "OK \n");
/* check regenerated data is byte exact */
DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++);
{ size_t i;
for (i=0; i<CNBufferSize; i++) {
if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;
} }
DISPLAYLEVEL(4, "OK \n");
_end:
ZBUFF_freeCCtx(zc);
ZBUFF_freeDCtx(zd);
free(CNBuffer);
free(compressedBuffer);
free(decodedBuffer);
return testResult;
_output_error:
testResult = 1;
DISPLAY("Error detected in Unit tests ! \n");
goto _end;
}
static size_t findDiff(const void* buf1, const void* buf2, size_t max)
{
const BYTE* b1 = (const BYTE*)buf1;
const BYTE* b2 = (const BYTE*)buf2;
size_t u;
for (u=0; u<max; u++) {
if (b1[u] != b2[u]) break;
}
return u;
}
static size_t FUZ_rLogLength(U32* seed, U32 logLength)
{
size_t const lengthMask = ((size_t)1 << logLength) - 1;
return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
}
static size_t FUZ_randomLength(U32* seed, U32 maxLog)
{
U32 const logLength = FUZ_rand(seed) % maxLog;
return FUZ_rLogLength(seed, logLength);
}
#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \
DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; }
static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility)
{
static const U32 maxSrcLog = 24;
static const U32 maxSampleLog = 19;
BYTE* cNoiseBuffer[5];
size_t const srcBufferSize = (size_t)1<<maxSrcLog;
BYTE* copyBuffer;
size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog);
BYTE* cBuffer;
size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
BYTE* dstBuffer;
size_t dstBufferSize = srcBufferSize;
U32 result = 0;
U32 testNb = 0;
U32 coreSeed = seed;
ZBUFF_CCtx* zc;
ZBUFF_DCtx* zd;
UTIL_time_t startClock = UTIL_getTime();
/* allocations */
zc = ZBUFF_createCCtx();
zd = ZBUFF_createDCtx();
cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
copyBuffer= (BYTE*)malloc (copyBufferSize);
dstBuffer = (BYTE*)malloc (dstBufferSize);
cBuffer = (BYTE*)malloc (cBufferSize);
CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
!copyBuffer || !dstBuffer || !cBuffer || !zc || !zd,
"Not enough memory, fuzzer tests cancelled");
/* Create initial samples */
RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
/* catch up testNb */
for (testNb=1; testNb < startTest; testNb++)
FUZ_rand(&coreSeed);
/* test loop */
for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
U32 lseed;
const BYTE* srcBuffer;
const BYTE* dict;
size_t maxTestSize, dictSize;
size_t cSize, totalTestSize, totalCSize, totalGenSize;
size_t errorCode;
U32 n, nbChunks;
XXH64_state_t xxhState;
U64 crcOrig;
/* init */
DISPLAYUPDATE(2, "\r%6u", testNb);
if (nbTests >= testNb) DISPLAYUPDATE(2, "/%6u ", nbTests);
FUZ_rand(&coreSeed);
lseed = coreSeed ^ prime1;
/* states full reset (unsynchronized) */
/* some issues only happen when reusing states in a specific sequence of parameters */
if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZBUFF_freeCCtx(zc); zc = ZBUFF_createCCtx(); }
if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZBUFF_freeDCtx(zd); zd = ZBUFF_createDCtx(); }
/* srcBuffer selection [0-4] */
{ U32 buffNb = FUZ_rand(&lseed) & 0x7F;
if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
else {
buffNb >>= 3;
if (buffNb & 7) {
const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
buffNb = tnb[buffNb >> 3];
} else {
const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
buffNb = tnb[buffNb >> 3];
} }
srcBuffer = cNoiseBuffer[buffNb];
}
/* compression init */
{ U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1;
maxTestSize = FUZ_rLogLength(&lseed, testLog);
dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0;
/* random dictionary selection */
{ size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
dict = srcBuffer + dictStart;
}
{ ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictSize);
params.fParams.checksumFlag = FUZ_rand(&lseed) & 1;
params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1;
{ size_t const initError = ZBUFF_compressInit_advanced(zc, dict, dictSize, params, ZSTD_CONTENTSIZE_UNKNOWN);
CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError));
} } }
/* multi-segments compression test */
XXH64_reset(&xxhState, 0);
nbChunks = (FUZ_rand(&lseed) & 127) + 2;
for (n=0, cSize=0, totalTestSize=0 ; (n<nbChunks) && (totalTestSize < maxTestSize) ; n++) {
/* compress random chunk into random size dst buffer */
{ size_t readChunkSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - readChunkSize);
size_t const compressionError = ZBUFF_compressContinue(zc, cBuffer+cSize, &dstBuffSize, srcBuffer+srcStart, &readChunkSize);
CHECK (ZBUFF_isError(compressionError), "compression error : %s", ZBUFF_getErrorName(compressionError));
XXH64_update(&xxhState, srcBuffer+srcStart, readChunkSize);
memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, readChunkSize);
cSize += dstBuffSize;
totalTestSize += readChunkSize;
}
/* random flush operation, to mess around */
if ((FUZ_rand(&lseed) & 15) == 0) {
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
size_t const flushError = ZBUFF_compressFlush(zc, cBuffer+cSize, &dstBuffSize);
CHECK (ZBUFF_isError(flushError), "flush error : %s", ZBUFF_getErrorName(flushError));
cSize += dstBuffSize;
} }
/* final frame epilogue */
{ size_t remainingToFlush = (size_t)(-1);
while (remainingToFlush) {
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
U32 const enoughDstSize = dstBuffSize >= remainingToFlush;
remainingToFlush = ZBUFF_compressEnd(zc, cBuffer+cSize, &dstBuffSize);
CHECK (ZBUFF_isError(remainingToFlush), "flush error : %s", ZBUFF_getErrorName(remainingToFlush));
CHECK (enoughDstSize && remainingToFlush, "ZBUFF_compressEnd() not fully flushed (%u remaining), but enough space available", (U32)remainingToFlush);
cSize += dstBuffSize;
} }
crcOrig = XXH64_digest(&xxhState);
/* multi - fragments decompression test */
ZBUFF_decompressInitDictionary(zd, dict, dictSize);
errorCode = 1;
for (totalCSize = 0, totalGenSize = 0 ; errorCode ; ) {
size_t readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &dstBuffSize, cBuffer+totalCSize, &readCSrcSize);
CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode));
totalGenSize += dstBuffSize;
totalCSize += readCSrcSize;
}
CHECK (errorCode != 0, "frame not fully decoded");
CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size")
CHECK (totalCSize != cSize, "compressed data should be fully read")
{ U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
CHECK (crcDest!=crcOrig, "decompressed data corrupted"); }
/*===== noisy/erroneous src decompression test =====*/
/* add some noise */
{ U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
} }
/* try decompression on noisy data */
ZBUFF_decompressInit(zd);
totalCSize = 0;
totalGenSize = 0;
while ( (totalCSize < cSize) && (totalGenSize < dstBufferSize) ) {
size_t readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
size_t dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
size_t const decompressError = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &dstBuffSize, cBuffer+totalCSize, &readCSrcSize);
if (ZBUFF_isError(decompressError)) break; /* error correctly detected */
totalGenSize += dstBuffSize;
totalCSize += readCSrcSize;
} }
DISPLAY("\r%u fuzzer tests completed \n", testNb);
_cleanup:
ZBUFF_freeCCtx(zc);
ZBUFF_freeDCtx(zd);
free(cNoiseBuffer[0]);
free(cNoiseBuffer[1]);
free(cNoiseBuffer[2]);
free(cNoiseBuffer[3]);
free(cNoiseBuffer[4]);
free(copyBuffer);
free(cBuffer);
free(dstBuffer);
return result;
_output_error:
result = 1;
goto _cleanup;
}
/*-*******************************************************
* Command line
*********************************************************/
static int FUZ_usage(const char* programName)
{
DISPLAY( "Usage :\n");
DISPLAY( " %s [args]\n", programName);
DISPLAY( "\n");
DISPLAY( "Arguments :\n");
DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault);
DISPLAY( " -s# : Select seed (default:prompt user)\n");
DISPLAY( " -t# : Select starting test number (default:0)\n");
DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
DISPLAY( " -v : verbose\n");
DISPLAY( " -p : pause at the end\n");
DISPLAY( " -h : display help and exit\n");
return 0;
}
int main(int argc, const char** argv)
{
U32 seed=0;
int seedset=0;
int argNb;
int nbTests = nbTestsDefault;
int testNb = 0;
int proba = FUZ_COMPRESSIBILITY_DEFAULT;
int result=0;
U32 mainPause = 0;
const char* programName = argv[0];
ZSTD_customMem customMem = { ZBUFF_allocFunction, ZBUFF_freeFunction, NULL };
ZSTD_customMem customNULL = { NULL, NULL, NULL };
/* Check command line */
for(argNb=1; argNb<argc; argNb++) {
const char* argument = argv[argNb];
if(!argument) continue; /* Protection if argument empty */
/* Parsing commands. Aggregated commands are allowed */
if (argument[0]=='-') {
argument++;
while (*argument!=0) {
switch(*argument)
{
case 'h':
return FUZ_usage(programName);
case 'v':
argument++;
g_displayLevel=4;
break;
case 'q':
argument++;
g_displayLevel--;
break;
case 'p': /* pause at the end */
argument++;
mainPause = 1;
break;
case 'i':
argument++;
nbTests=0; g_clockTime=0;
while ((*argument>='0') && (*argument<='9')) {
nbTests *= 10;
nbTests += *argument - '0';
argument++;
}
break;
case 'T':
argument++;
nbTests=0; g_clockTime=0;
while ((*argument>='0') && (*argument<='9')) {
g_clockTime *= 10;
g_clockTime += *argument - '0';
argument++;
}
if (*argument=='m') g_clockTime *=60, argument++;
if (*argument=='n') argument++;
g_clockTime *= SEC_TO_MICRO;
break;
case 's':
argument++;
seed=0;
seedset=1;
while ((*argument>='0') && (*argument<='9')) {
seed *= 10;
seed += *argument - '0';
argument++;
}
break;
case 't':
argument++;
testNb=0;
while ((*argument>='0') && (*argument<='9')) {
testNb *= 10;
testNb += *argument - '0';
argument++;
}
break;
case 'P': /* compressibility % */
argument++;
proba=0;
while ((*argument>='0') && (*argument<='9')) {
proba *= 10;
proba += *argument - '0';
argument++;
}
if (proba<0) proba=0;
if (proba>100) proba=100;
break;
default:
return FUZ_usage(programName);
}
} } } /* for(argNb=1; argNb<argc; argNb++) */
/* Get Seed */
DISPLAY("Starting zstd_buffered tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
if (!seedset) {
time_t const t = time(NULL);
U32 const h = XXH32(&t, sizeof(t), 1);
seed = h % 10000;
}
DISPLAY("Seed = %u\n", seed);
if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
if (nbTests<=0) nbTests=1;
if (testNb==0) {
result = basicUnitTests(0, ((double)proba) / 100, customNULL); /* constant seed for predictability */
if (!result) {
DISPLAYLEVEL(4, "Unit tests using customMem :\n")
result = basicUnitTests(0, ((double)proba) / 100, customMem); /* use custom memory allocation functions */
} }
if (!result)
result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100);
if (mainPause) {
int unused;
DISPLAY("Press Enter \n");
unused = getchar();
(void)unused;
}
return result;
}
tests/zstreamtest.c
+3 -3
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2021, Yann Collet, Facebook, Inc.
* Copyright (c) Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@@ -672,7 +672,7 @@ static int basicUnitTests(U32 seed, double compressibility)
cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize, 1);
CHECK_Z(cSize);
{ ZSTD_DCtx* dctx = ZSTD_createDCtx();
size_t const dctxSize0 = ZSTD_sizeof_DCtx(dctx);
size_t const dctxSize0 = ZSTD_sizeof_DCtx(dctx);
size_t dctxSize1;
CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
@@ -735,7 +735,7 @@ static int basicUnitTests(U32 seed, double compressibility)
CHECK(ZSTD_getErrorCode(r) != ZSTD_error_dstBuffer_wrong, "Must error but got %s", ZSTD_getErrorName(r));
}
DISPLAYLEVEL(3, "OK \n");
DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() buffered output : ", testNb++);
ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 0));