Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
97a3da1df0 | ||
|
|
7c495e8ea2 | ||
|
|
ceadfa2444 | ||
|
|
a7cb4af573 | ||
|
|
87dcf71115 | ||
|
|
7f09cdc8a8 | ||
|
|
54cc01c5be | ||
|
|
456ca2aa06 | ||
|
|
792fdf84f8 | ||
|
|
3536e9d5ff | ||
|
|
d4d1e45bcb | ||
|
|
d5eb7d1569 | ||
|
|
c11db9c8b5 | ||
|
|
4680d817c0 | ||
|
|
ae85676d44 | ||
|
|
ce34dc39a0 | ||
|
|
0b39531d75 |
@@ -143,8 +143,9 @@ workflows:
|
|||||||
filters:
|
filters:
|
||||||
branches:
|
branches:
|
||||||
only:
|
only:
|
||||||
- master
|
- release
|
||||||
- dev
|
- dev
|
||||||
|
- master
|
||||||
jobs:
|
jobs:
|
||||||
# Run daily long regression tests
|
# Run daily long regression tests
|
||||||
- regression-test
|
- regression-test
|
||||||
|
|||||||
@@ -2,14 +2,13 @@ name: generic-dev
|
|||||||
|
|
||||||
on:
|
on:
|
||||||
pull_request:
|
pull_request:
|
||||||
branches: [ dev, master, actionsTest ]
|
branches: [ dev, release, actionsTest ]
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
|
|
||||||
# Dev PR jobs that still have to be migrated from travis
|
# Dev PR jobs that still have to be migrated from travis
|
||||||
#
|
#
|
||||||
# icc (need self-hosted)
|
# versionTag (only on release tags)
|
||||||
# versionTag
|
|
||||||
# valgrindTest (keeps failing for some reason. need investigation)
|
# valgrindTest (keeps failing for some reason. need investigation)
|
||||||
# staticAnalyze (need trusty so need self-hosted)
|
# staticAnalyze (need trusty so need self-hosted)
|
||||||
# pcc-fuzz: (need trusty so need self-hosted)
|
# pcc-fuzz: (need trusty so need self-hosted)
|
||||||
@@ -19,7 +18,7 @@ jobs:
|
|||||||
# I need admins permissions to the repo for that it looks like
|
# I need admins permissions to the repo for that it looks like
|
||||||
# So I'm tabling that for now
|
# So I'm tabling that for now
|
||||||
#
|
#
|
||||||
# The master branch exclusive jobs will be in a separate
|
# The release branch exclusive jobs will be in a separate
|
||||||
# workflow file (the osx tests and meson build that is)
|
# workflow file (the osx tests and meson build that is)
|
||||||
|
|
||||||
benchmarking:
|
benchmarking:
|
||||||
@@ -36,6 +35,15 @@ jobs:
|
|||||||
- name: make test
|
- name: make test
|
||||||
run: make test
|
run: make test
|
||||||
|
|
||||||
|
check-32bit: # designed to catch https://github.com/facebook/zstd/issues/2428
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v2
|
||||||
|
- name: make check on 32-bit
|
||||||
|
run: |
|
||||||
|
make libc6install
|
||||||
|
CFLAGS="-m32 -O1 -fstack-protector" make check V=1
|
||||||
|
|
||||||
gcc-6-7-libzstd:
|
gcc-6-7-libzstd:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
@@ -50,6 +58,10 @@ jobs:
|
|||||||
LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt
|
LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt
|
||||||
make -C tests zbufftest-dll
|
make -C tests zbufftest-dll
|
||||||
|
|
||||||
|
# candidate test (to check) : underlink test
|
||||||
|
# LDFLAGS=-Wl,--no-undefined : will make the linker fail if dll is underlinked
|
||||||
|
# zbufftest-dll : test that a user program can link to multi-threaded libzstd without specifying -pthread
|
||||||
|
|
||||||
gcc-8-asan-ubsan-testzstd:
|
gcc-8-asan-ubsan-testzstd:
|
||||||
runs-on: ubuntu-16.04 # fails on 18.04
|
runs-on: ubuntu-16.04 # fails on 18.04
|
||||||
steps:
|
steps:
|
||||||
@@ -84,6 +96,10 @@ jobs:
|
|||||||
sudo apt-get install clang-3.8
|
sudo apt-get install clang-3.8
|
||||||
CC=clang-3.8 make clean msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0
|
CC=clang-3.8 make clean msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0
|
||||||
|
|
||||||
|
# Note : external libraries must be turned off when using MSAN tests,
|
||||||
|
# because they are not msan-instrumented,
|
||||||
|
# so any data coming from these libraries is always considered "uninitialized"
|
||||||
|
|
||||||
cmake-build-and-test-check:
|
cmake-build-and-test-check:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
@@ -147,7 +163,7 @@ jobs:
|
|||||||
make clean
|
make clean
|
||||||
make c99build
|
make c99build
|
||||||
make clean
|
make clean
|
||||||
make travis-install
|
make travis-install # just ensures `make install` works
|
||||||
|
|
||||||
mingw-cross-compilation:
|
mingw-cross-compilation:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|||||||
@@ -2,10 +2,10 @@ name: generic-release
|
|||||||
|
|
||||||
on:
|
on:
|
||||||
pull_request:
|
pull_request:
|
||||||
# This will eventually only be for pushes to master
|
# This will eventually only be for pushes to release
|
||||||
# but for dogfooding purposes, I'm running it even
|
# but for dogfooding purposes, I'm running it even
|
||||||
# on dev pushes
|
# on dev pushes
|
||||||
branches: [ dev, master, actionsTest ]
|
branches: [ dev, release, actionsTest ]
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
# missing jobs
|
# missing jobs
|
||||||
@@ -44,6 +44,7 @@ jobs:
|
|||||||
sudo apt-get install clang-3.8
|
sudo apt-get install clang-3.8
|
||||||
CC=clang-3.8 make tsan-test-zstream
|
CC=clang-3.8 make tsan-test-zstream
|
||||||
CC=clang-3.8 make tsan-fuzztest
|
CC=clang-3.8 make tsan-fuzztest
|
||||||
|
|
||||||
zlib-wrapper:
|
zlib-wrapper:
|
||||||
runs-on: ubuntu-16.04
|
runs-on: ubuntu-16.04
|
||||||
steps:
|
steps:
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ name: linux-kernel
|
|||||||
|
|
||||||
on:
|
on:
|
||||||
pull_request:
|
pull_request:
|
||||||
branches: [ dev, master, actionsTest ]
|
branches: [ dev, release, actionsTest ]
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
test:
|
test:
|
||||||
|
|||||||
+9
-109
@@ -8,6 +8,7 @@ git:
|
|||||||
branches:
|
branches:
|
||||||
only:
|
only:
|
||||||
- dev
|
- dev
|
||||||
|
- release
|
||||||
- master
|
- master
|
||||||
- travisTest
|
- travisTest
|
||||||
|
|
||||||
@@ -31,49 +32,11 @@ matrix:
|
|||||||
script:
|
script:
|
||||||
- make check
|
- make check
|
||||||
|
|
||||||
- name: make benchmarking
|
|
||||||
script:
|
|
||||||
- make benchmarking
|
|
||||||
|
|
||||||
- name: make test (complete)
|
- name: make test (complete)
|
||||||
script:
|
script:
|
||||||
# DEVNULLRIGHTS : will request sudo rights to test permissions on /dev/null
|
# DEVNULLRIGHTS : will request sudo rights to test permissions on /dev/null
|
||||||
- DEVNULLRIGHTS=test make test
|
- DEVNULLRIGHTS=test make test
|
||||||
|
|
||||||
- name: gcc-6 + gcc-7 + libzstdmt compilation # ~ 6mn
|
|
||||||
script:
|
|
||||||
- make gcc6install gcc7install
|
|
||||||
- CC=gcc-6 CFLAGS=-Werror make -j all
|
|
||||||
- make clean
|
|
||||||
- CC=gcc-7 CFLAGS=-Werror make -j all
|
|
||||||
- make clean
|
|
||||||
- LDFLAGS=-Wl,--no-undefined make -C lib libzstd-mt
|
|
||||||
- make -C tests zbufftest-dll
|
|
||||||
# LDFLAGS=-Wl,--no-undefined : will make the linker fail if dll is underlinked
|
|
||||||
# zbufftest-dll : test that a user program can link to multi-threaded libzstd without specifying -pthread
|
|
||||||
|
|
||||||
- name: gcc-8 + ASan + UBSan + Test Zstd # ~6.5mn
|
|
||||||
script:
|
|
||||||
- make gcc8install
|
|
||||||
- CC=gcc-8 CFLAGS="-Werror" make -j all
|
|
||||||
- make clean
|
|
||||||
- CC=gcc-8 make -j uasan-test-zstd </dev/null # test when stdin is not a tty
|
|
||||||
|
|
||||||
- name: gcc-6 + ASan + UBSan + Test Zstd, 32bit mode # ~4mn
|
|
||||||
script:
|
|
||||||
- make gcc6install libc6install
|
|
||||||
- CC=gcc-6 CFLAGS="-Werror -m32" make -j all32
|
|
||||||
- make clean
|
|
||||||
- CC=gcc-6 make -j uasan-test-zstd32 # note : can complain about pointer overflow
|
|
||||||
|
|
||||||
- name: clang-3.8 + MSan + Test Zstd # ~3.5mn
|
|
||||||
script:
|
|
||||||
- make clang38install
|
|
||||||
# External libraries must be turned off when using MSAN tests,
|
|
||||||
# because they are not msan-instrumented,
|
|
||||||
# so any data coming from these libraries is always considered "uninitialized"
|
|
||||||
- CC=clang-3.8 make clean msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0
|
|
||||||
|
|
||||||
- name: Minimal Decompressor Macros # ~5mn
|
- name: Minimal Decompressor Macros # ~5mn
|
||||||
script:
|
script:
|
||||||
- make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
|
- make clean && make -j all ZSTD_LIB_MINIFY=1 MOREFLAGS="-Werror"
|
||||||
@@ -85,51 +48,11 @@ matrix:
|
|||||||
- make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
|
- make clean && make -j all MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
|
||||||
- make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
|
- make clean && make check MOREFLAGS="-Werror -DZSTD_NO_INLINE -DZSTD_STRIP_ERROR_STRINGS"
|
||||||
|
|
||||||
- name: cmake build and test check # ~6mn
|
|
||||||
script:
|
|
||||||
- make cmakebuild
|
|
||||||
|
|
||||||
- name: static analyzer scanbuild # ~26mn
|
- name: static analyzer scanbuild # ~26mn
|
||||||
dist: trusty # note : it's important to pin down a version of static analyzer, since different versions report different false positives
|
dist: trusty # note : it's important to pin down a version of static analyzer, since different versions report different false positives
|
||||||
script:
|
script:
|
||||||
- make staticAnalyze
|
- make staticAnalyze
|
||||||
|
|
||||||
- name: gcc-8 + ASan + UBSan + Fuzz Test # ~19mn
|
|
||||||
script:
|
|
||||||
- make gcc8install
|
|
||||||
- CC=gcc-8 make clean uasan-fuzztest
|
|
||||||
|
|
||||||
- name: gcc-6 + ASan + UBSan + Fuzz Test 32bit # ~15.5mn
|
|
||||||
script:
|
|
||||||
- make gcc6install libc6install
|
|
||||||
- CC=gcc-6 CFLAGS="-O2 -m32" make uasan-fuzztest # can complain about pointer overflow
|
|
||||||
|
|
||||||
- name: clang-3.8 + MSan + Fuzz Test # ~14.5mn
|
|
||||||
script:
|
|
||||||
- make clang38install
|
|
||||||
- CC=clang-3.8 make clean msan-fuzztest
|
|
||||||
|
|
||||||
- name: ASan + UBSan + MSan + Regression Test # ~ 4.5mn
|
|
||||||
script:
|
|
||||||
- make -j uasanregressiontest
|
|
||||||
- make clean
|
|
||||||
- make -j msanregressiontest
|
|
||||||
|
|
||||||
- name: C++, gnu90 and c99 compatibility # ~3mn
|
|
||||||
script:
|
|
||||||
- make cxxtest
|
|
||||||
- make clean
|
|
||||||
- make gnu90build
|
|
||||||
- make clean
|
|
||||||
- make c99build
|
|
||||||
- make clean
|
|
||||||
- make travis-install # just ensures `make install` works
|
|
||||||
|
|
||||||
- name: mingw cross-compilation
|
|
||||||
script :
|
|
||||||
- sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix;
|
|
||||||
- CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ CFLAGS="-Werror -O1" make zstd
|
|
||||||
|
|
||||||
- name: Valgrind + Fuzz Test Stack Mode # ~ 7mn
|
- name: Valgrind + Fuzz Test Stack Mode # ~ 7mn
|
||||||
script:
|
script:
|
||||||
- make valgrindinstall
|
- make valgrindinstall
|
||||||
@@ -162,33 +85,28 @@ matrix:
|
|||||||
- make -C tests checkTag
|
- make -C tests checkTag
|
||||||
- tests/checkTag "$TRAVIS_BRANCH"
|
- tests/checkTag "$TRAVIS_BRANCH"
|
||||||
|
|
||||||
# tests for master branch and cron job only
|
# tests for release branch and cron job only
|
||||||
- name: OS-X # ~13mn
|
- name: OS-X # ~13mn
|
||||||
if: branch = master
|
if: branch = release
|
||||||
os: osx
|
os: osx
|
||||||
script:
|
script:
|
||||||
- make test
|
- make test
|
||||||
- make -C lib all
|
- make -C lib all
|
||||||
|
|
||||||
- name: zbuff test
|
|
||||||
if: branch = master
|
|
||||||
script:
|
|
||||||
- make -C tests test-zbuff
|
|
||||||
|
|
||||||
- name: Versions Compatibility Test # 11.5mn
|
- name: Versions Compatibility Test # 11.5mn
|
||||||
if: branch = master
|
if: branch = release
|
||||||
script:
|
script:
|
||||||
- make -C tests versionsTest
|
- make -C tests versionsTest
|
||||||
|
|
||||||
- name: thread sanitizer # ~29mn
|
- name: thread sanitizer # ~29mn
|
||||||
if: branch = master
|
if: branch = release
|
||||||
script:
|
script:
|
||||||
- make clang38install
|
- make clang38install
|
||||||
- CC=clang-3.8 make tsan-test-zstream
|
- CC=clang-3.8 make tsan-test-zstream
|
||||||
- CC=clang-3.8 make tsan-fuzztest
|
- CC=clang-3.8 make tsan-fuzztest
|
||||||
|
|
||||||
- name: PPC64LE + Fuzz test # ~13mn
|
- name: PPC64LE + Fuzz test # ~13mn
|
||||||
if: branch = master
|
if: branch = release
|
||||||
arch: ppc64le
|
arch: ppc64le
|
||||||
script:
|
script:
|
||||||
- cat /proc/cpuinfo
|
- cat /proc/cpuinfo
|
||||||
@@ -196,40 +114,22 @@ matrix:
|
|||||||
|
|
||||||
- name: Qemu PPC64 + Fuzz test # ~13mn, presumed Big-Endian (?)
|
- name: Qemu PPC64 + Fuzz test # ~13mn, presumed Big-Endian (?)
|
||||||
dist: trusty # note : PPC64 cross-compilation for Qemu tests seems broken on Xenial
|
dist: trusty # note : PPC64 cross-compilation for Qemu tests seems broken on Xenial
|
||||||
if: branch = master
|
if: branch = release
|
||||||
script:
|
script:
|
||||||
- make ppcinstall
|
- make ppcinstall
|
||||||
- make ppc64fuzz
|
- make ppc64fuzz
|
||||||
|
|
||||||
# note : we already have aarch64 tests on hardware
|
# note : we already have aarch64 tests on hardware
|
||||||
- name: Qemu aarch64 + Fuzz Test (on Xenial) # ~14mn
|
- name: Qemu aarch64 + Fuzz Test (on Xenial) # ~14mn
|
||||||
if: branch = master
|
if: branch = release
|
||||||
dist: xenial
|
dist: xenial
|
||||||
script:
|
script:
|
||||||
- make arminstall
|
- make arminstall
|
||||||
- make aarch64fuzz
|
- make aarch64fuzz
|
||||||
|
|
||||||
- name: zlib wrapper test # ~7.5mn
|
|
||||||
if: branch = master
|
|
||||||
script:
|
|
||||||
- make gpp6install valgrindinstall
|
|
||||||
- make -C zlibWrapper test
|
|
||||||
- make -C zlibWrapper valgrindTest
|
|
||||||
|
|
||||||
- name: LZ4, thread pool, and partial libs tests # ~4mn
|
|
||||||
if: branch = master
|
|
||||||
script:
|
|
||||||
- make lz4install
|
|
||||||
- make -C tests test-lz4
|
|
||||||
- make check < /dev/null | tee # mess with lz4 console detection
|
|
||||||
- make clean
|
|
||||||
- make -C tests test-pool
|
|
||||||
- make clean
|
|
||||||
- bash tests/libzstd_partial_builds.sh
|
|
||||||
|
|
||||||
# meson dedicated test
|
# meson dedicated test
|
||||||
- name: Xenial (Meson + clang) # ~15mn
|
- name: Xenial (Meson + clang) # ~15mn
|
||||||
if: branch = master
|
if: branch = release
|
||||||
dist: xenial
|
dist: xenial
|
||||||
language: cpp
|
language: cpp
|
||||||
compiler: clang
|
compiler: clang
|
||||||
|
|||||||
@@ -1,4 +1,7 @@
|
|||||||
v1.4.7
|
v1.4.8 (Dec 18, 2020)
|
||||||
|
hotfix: wrong alignment of an internal buffer
|
||||||
|
|
||||||
|
v1.4.7 (Dec 16, 2020)
|
||||||
perf: stronger --long mode at high compression levels, by @senhuang42
|
perf: stronger --long mode at high compression levels, by @senhuang42
|
||||||
perf: stronger --patch-from at high compression levels, thanks to --long improvements
|
perf: stronger --patch-from at high compression levels, thanks to --long improvements
|
||||||
perf: faster dictionary compression at medium compression levels, by @felixhandte
|
perf: faster dictionary compression at medium compression levels, by @felixhandte
|
||||||
|
|||||||
+2
-2
@@ -5,7 +5,7 @@ possible.
|
|||||||
## Our Development Process
|
## Our Development Process
|
||||||
New versions are being developed in the "dev" branch,
|
New versions are being developed in the "dev" branch,
|
||||||
or in their own feature branch.
|
or in their own feature branch.
|
||||||
When they are deemed ready for a release, they are merged into "master".
|
When they are deemed ready for a release, they are merged into "release".
|
||||||
|
|
||||||
As a consequences, all contributions must stage first through "dev"
|
As a consequences, all contributions must stage first through "dev"
|
||||||
or their own feature branch.
|
or their own feature branch.
|
||||||
@@ -383,7 +383,7 @@ CI tests run every time a pull request (PR) is created or updated. The exact tes
|
|||||||
that get run will depend on the destination branch you specify. Some tests take
|
that get run will depend on the destination branch you specify. Some tests take
|
||||||
longer to run than others. Currently, our CI is set up to run a short
|
longer to run than others. Currently, our CI is set up to run a short
|
||||||
series of tests when creating a PR to the dev branch and a longer series of tests
|
series of tests when creating a PR to the dev branch and a longer series of tests
|
||||||
when creating a PR to the master branch. You can look in the configuration files
|
when creating a PR to the release branch. You can look in the configuration files
|
||||||
of the respective CI platform for more information on what gets run when.
|
of the respective CI platform for more information on what gets run when.
|
||||||
|
|
||||||
Most people will just want to create a PR with the destination set to their local dev
|
Most people will just want to create a PR with the destination set to their local dev
|
||||||
|
|||||||
@@ -193,7 +193,7 @@ Zstandard is dual-licensed under [BSD](LICENSE) and [GPLv2](COPYING).
|
|||||||
|
|
||||||
## Contributing
|
## Contributing
|
||||||
|
|
||||||
The "dev" branch is the one where all contributions are merged before reaching "master".
|
The `dev` branch is the one where all contributions are merged before reaching `release`.
|
||||||
If you plan to propose a patch, please commit into the "dev" branch, or its own feature branch.
|
If you plan to propose a patch, please commit into the `dev` branch, or its own feature branch.
|
||||||
Direct commit to "master" are not permitted.
|
Direct commit to `release` are not permitted.
|
||||||
For more information, please read [CONTRIBUTING](CONTRIBUTING.md).
|
For more information, please read [CONTRIBUTING](CONTRIBUTING.md).
|
||||||
|
|||||||
+1
-1
@@ -27,7 +27,7 @@ They consist of the following tests:
|
|||||||
|
|
||||||
Long Tests
|
Long Tests
|
||||||
----------
|
----------
|
||||||
Long tests run on all commits to `master` branch,
|
Long tests run on all commits to `release` branch,
|
||||||
and once a day on the current version of `dev` branch,
|
and once a day on the current version of `dev` branch,
|
||||||
on TravisCI.
|
on TravisCI.
|
||||||
They consist of the following tests:
|
They consist of the following tests:
|
||||||
|
|||||||
+7
-7
@@ -1,20 +1,20 @@
|
|||||||
# Following tests are run _only_ on master branch
|
# Following tests are run _only_ on `release` branch
|
||||||
# To reproduce these tests, it's possible to push into a branch `appveyorTest`
|
# and on selected feature branch named `appveyorTest` or `visual*`
|
||||||
# or a branch `visual*`, they will intentionnally trigger `master` tests
|
|
||||||
|
|
||||||
-
|
-
|
||||||
version: 1.0.{build}
|
version: 1.0.{build}
|
||||||
branches:
|
branches:
|
||||||
only:
|
only:
|
||||||
|
- release
|
||||||
- master
|
- master
|
||||||
- appveyorTest
|
- /appveyor*/
|
||||||
- /visual*/
|
- /visual*/
|
||||||
environment:
|
environment:
|
||||||
matrix:
|
matrix:
|
||||||
- COMPILER: "gcc"
|
- COMPILER: "gcc"
|
||||||
HOST: "mingw"
|
HOST: "mingw"
|
||||||
PLATFORM: "x64"
|
PLATFORM: "x64"
|
||||||
SCRIPT: "make allzstd MOREFLAGS=-static && make -C tests fullbench-lib"
|
SCRIPT: "make allzstd MOREFLAGS=-static"
|
||||||
ARTIFACT: "true"
|
ARTIFACT: "true"
|
||||||
BUILD: "true"
|
BUILD: "true"
|
||||||
- COMPILER: "gcc"
|
- COMPILER: "gcc"
|
||||||
@@ -92,9 +92,9 @@
|
|||||||
cd programs\ && 7z a -tzip -mx9 zstd-win-binary-%PLATFORM%.zip zstd.exe &&
|
cd programs\ && 7z a -tzip -mx9 zstd-win-binary-%PLATFORM%.zip zstd.exe &&
|
||||||
appveyor PushArtifact zstd-win-binary-%PLATFORM%.zip &&
|
appveyor PushArtifact zstd-win-binary-%PLATFORM%.zip &&
|
||||||
cp zstd.exe ..\bin\zstd.exe &&
|
cp zstd.exe ..\bin\zstd.exe &&
|
||||||
git clone --depth 1 --branch master https://github.com/facebook/zstd &&
|
git clone --depth 1 --branch release https://github.com/facebook/zstd &&
|
||||||
cd zstd &&
|
cd zstd &&
|
||||||
git archive --format=tar master -o zstd-src.tar &&
|
git archive --format=tar release -o zstd-src.tar &&
|
||||||
..\zstd -19 zstd-src.tar &&
|
..\zstd -19 zstd-src.tar &&
|
||||||
appveyor PushArtifact zstd-src.tar.zst &&
|
appveyor PushArtifact zstd-src.tar.zst &&
|
||||||
certUtil -hashfile zstd-src.tar.zst SHA256 > zstd-src.tar.zst.sha256.sig &&
|
certUtil -hashfile zstd-src.tar.zst SHA256 > zstd-src.tar.zst.sha256.sig &&
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ __`Frame_Size`__
|
|||||||
The total size of the skippable frame, not including the `Skippable_Magic_Number` or `Frame_Size`.
|
The total size of the skippable frame, not including the `Skippable_Magic_Number` or `Frame_Size`.
|
||||||
This is for compatibility with [Zstandard skippable frames].
|
This is for compatibility with [Zstandard skippable frames].
|
||||||
|
|
||||||
[Zstandard skippable frames]: https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#skippable-frames
|
[Zstandard skippable frames]: https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#skippable-frames
|
||||||
|
|
||||||
#### `Seek_Table_Footer`
|
#### `Seek_Table_Footer`
|
||||||
The seek table footer format is as follows:
|
The seek table footer format is as follows:
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
<html>
|
<html>
|
||||||
<head>
|
<head>
|
||||||
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
|
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
|
||||||
<title>zstd 1.4.7 Manual</title>
|
<title>zstd 1.4.8 Manual</title>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<h1>zstd 1.4.7 Manual</h1>
|
<h1>zstd 1.4.8 Manual</h1>
|
||||||
<hr>
|
<hr>
|
||||||
<a name="Contents"></a><h2>Contents</h2>
|
<a name="Contents"></a><h2>Contents</h2>
|
||||||
<ol>
|
<ol>
|
||||||
|
|||||||
+3
-2
@@ -335,9 +335,10 @@ size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
|
|||||||
|
|
||||||
/* FSE_buildCTable_wksp() :
|
/* FSE_buildCTable_wksp() :
|
||||||
* Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
|
* Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`).
|
||||||
* `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog)`.
|
* `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)` of `unsigned`.
|
||||||
*/
|
*/
|
||||||
#define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * (maxSymbolValue + 2) + (1ull << tableLog))
|
#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (maxSymbolValue + 2 + (1ull << (tableLog - 2)))
|
||||||
|
#define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog))
|
||||||
size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
|
size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);
|
||||||
|
|
||||||
#define FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) (sizeof(short) * (maxSymbolValue + 1) + (1ULL << maxTableLog) + 8)
|
#define FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) (sizeof(short) * (maxSymbolValue + 1) + (1ULL << maxTableLog) + 8)
|
||||||
|
|||||||
@@ -69,7 +69,7 @@ static size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weight
|
|||||||
U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER;
|
U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER;
|
||||||
|
|
||||||
FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)];
|
FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)];
|
||||||
BYTE scratchBuffer[FSE_BUILD_CTABLE_WORKSPACE_SIZE(HUF_TABLELOG_MAX, MAX_FSE_TABLELOG_FOR_HUFF_HEADER)];
|
U32 scratchBuffer[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(HUF_TABLELOG_MAX, MAX_FSE_TABLELOG_FOR_HUFF_HEADER)];
|
||||||
|
|
||||||
unsigned count[HUF_TABLELOG_MAX+1];
|
unsigned count[HUF_TABLELOG_MAX+1];
|
||||||
S16 norm[HUF_TABLELOG_MAX+1];
|
S16 norm[HUF_TABLELOG_MAX+1];
|
||||||
@@ -736,25 +736,26 @@ typedef struct {
|
|||||||
} HUF_compress_tables_t;
|
} HUF_compress_tables_t;
|
||||||
|
|
||||||
/* HUF_compress_internal() :
|
/* HUF_compress_internal() :
|
||||||
* `workSpace` must a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
|
* `workSpace_align4` must be aligned on 4-bytes boundaries,
|
||||||
|
* and occupies the same space as a table of HUF_WORKSPACE_SIZE_U32 unsigned */
|
||||||
static size_t
|
static size_t
|
||||||
HUF_compress_internal (void* dst, size_t dstSize,
|
HUF_compress_internal (void* dst, size_t dstSize,
|
||||||
const void* src, size_t srcSize,
|
const void* src, size_t srcSize,
|
||||||
unsigned maxSymbolValue, unsigned huffLog,
|
unsigned maxSymbolValue, unsigned huffLog,
|
||||||
HUF_nbStreams_e nbStreams,
|
HUF_nbStreams_e nbStreams,
|
||||||
void* workSpace, size_t wkspSize,
|
void* workSpace_align4, size_t wkspSize,
|
||||||
HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat,
|
HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat,
|
||||||
const int bmi2)
|
const int bmi2)
|
||||||
{
|
{
|
||||||
HUF_compress_tables_t* const table = (HUF_compress_tables_t*)workSpace;
|
HUF_compress_tables_t* const table = (HUF_compress_tables_t*)workSpace_align4;
|
||||||
BYTE* const ostart = (BYTE*)dst;
|
BYTE* const ostart = (BYTE*)dst;
|
||||||
BYTE* const oend = ostart + dstSize;
|
BYTE* const oend = ostart + dstSize;
|
||||||
BYTE* op = ostart;
|
BYTE* op = ostart;
|
||||||
|
|
||||||
HUF_STATIC_ASSERT(sizeof(*table) <= HUF_WORKSPACE_SIZE);
|
HUF_STATIC_ASSERT(sizeof(*table) <= HUF_WORKSPACE_SIZE);
|
||||||
|
assert(((size_t)workSpace_align4 & 3) == 0); /* must be aligned on 4-bytes boundaries */
|
||||||
|
|
||||||
/* checks & inits */
|
/* checks & inits */
|
||||||
if (((size_t)workSpace & 3) != 0) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */
|
|
||||||
if (wkspSize < HUF_WORKSPACE_SIZE) return ERROR(workSpace_tooSmall);
|
if (wkspSize < HUF_WORKSPACE_SIZE) return ERROR(workSpace_tooSmall);
|
||||||
if (!srcSize) return 0; /* Uncompressed */
|
if (!srcSize) return 0; /* Uncompressed */
|
||||||
if (!dstSize) return 0; /* cannot fit anything within dst budget */
|
if (!dstSize) return 0; /* cannot fit anything within dst budget */
|
||||||
@@ -772,7 +773,7 @@ HUF_compress_internal (void* dst, size_t dstSize,
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* Scan input and build symbol stats */
|
/* Scan input and build symbol stats */
|
||||||
{ CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, workSpace, wkspSize) );
|
{ CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, workSpace_align4, wkspSize) );
|
||||||
if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */
|
if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */
|
||||||
if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */
|
if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3258,7 +3258,7 @@ size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace,
|
|||||||
|
|
||||||
/* Dictionary format :
|
/* Dictionary format :
|
||||||
* See :
|
* See :
|
||||||
* https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format
|
* https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#dictionary-format
|
||||||
*/
|
*/
|
||||||
/*! ZSTD_loadZstdDictionary() :
|
/*! ZSTD_loadZstdDictionary() :
|
||||||
* @return : dictID, or an error code
|
* @return : dictID, or an error code
|
||||||
|
|||||||
@@ -236,7 +236,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
|
|||||||
|
|
||||||
/* Default FSE distribution tables.
|
/* Default FSE distribution tables.
|
||||||
* These are pre-calculated FSE decoding tables using default distributions as defined in specification :
|
* These are pre-calculated FSE decoding tables using default distributions as defined in specification :
|
||||||
* https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#default-distributions
|
* https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributions
|
||||||
* They were generated programmatically with following method :
|
* They were generated programmatically with following method :
|
||||||
* - start from default distributions, present in /lib/common/zstd_internal.h
|
* - start from default distributions, present in /lib/common/zstd_internal.h
|
||||||
* - generate tables normally, using ZSTD_buildFSETable()
|
* - generate tables normally, using ZSTD_buildFSETable()
|
||||||
|
|||||||
+1
-1
@@ -72,7 +72,7 @@ extern "C" {
|
|||||||
/*------ Version ------*/
|
/*------ Version ------*/
|
||||||
#define ZSTD_VERSION_MAJOR 1
|
#define ZSTD_VERSION_MAJOR 1
|
||||||
#define ZSTD_VERSION_MINOR 4
|
#define ZSTD_VERSION_MINOR 4
|
||||||
#define ZSTD_VERSION_RELEASE 7
|
#define ZSTD_VERSION_RELEASE 8
|
||||||
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
|
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
|
||||||
|
|
||||||
/*! ZSTD_versionNumber() :
|
/*! ZSTD_versionNumber() :
|
||||||
|
|||||||
+1
-1
@@ -1,5 +1,5 @@
|
|||||||
.
|
.
|
||||||
.TH "ZSTD" "1" "December 2020" "zstd 1.4.7" "User Commands"
|
.TH "ZSTD" "1" "December 2020" "zstd 1.4.8" "User Commands"
|
||||||
.
|
.
|
||||||
.SH "NAME"
|
.SH "NAME"
|
||||||
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
|
\fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files
|
||||||
|
|||||||
+1
-1
@@ -1,5 +1,5 @@
|
|||||||
.
|
.
|
||||||
.TH "ZSTDGREP" "1" "December 2020" "zstd 1.4.7" "User Commands"
|
.TH "ZSTDGREP" "1" "December 2020" "zstd 1.4.8" "User Commands"
|
||||||
.
|
.
|
||||||
.SH "NAME"
|
.SH "NAME"
|
||||||
\fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files
|
\fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files
|
||||||
|
|||||||
+1
-1
@@ -1,5 +1,5 @@
|
|||||||
.
|
.
|
||||||
.TH "ZSTDLESS" "1" "December 2020" "zstd 1.4.7" "User Commands"
|
.TH "ZSTDLESS" "1" "December 2020" "zstd 1.4.8" "User Commands"
|
||||||
.
|
.
|
||||||
.SH "NAME"
|
.SH "NAME"
|
||||||
\fBzstdless\fR \- view zstandard\-compressed files
|
\fBzstdless\fR \- view zstandard\-compressed files
|
||||||
|
|||||||
+1
-1
@@ -28,7 +28,7 @@ desktop machine for every pull request that is made to the zstd repo but can als
|
|||||||
be run on any machine via the command line interface.
|
be run on any machine via the command line interface.
|
||||||
|
|
||||||
There are three modes of usage for this script: fastmode will just run a minimal single
|
There are three modes of usage for this script: fastmode will just run a minimal single
|
||||||
build comparison (between facebook:dev and facebook:master), onetime will pull all the current
|
build comparison (between facebook:dev and facebook:release), onetime will pull all the current
|
||||||
pull requests from the zstd repo and compare facebook:dev to all of them once, continuous
|
pull requests from the zstd repo and compare facebook:dev to all of them once, continuous
|
||||||
will continuously get pull requests from the zstd repo and run benchmarks against facebook:dev.
|
will continuously get pull requests from the zstd repo and run benchmarks against facebook:dev.
|
||||||
|
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ import urllib.request
|
|||||||
|
|
||||||
GITHUB_API_PR_URL = "https://api.github.com/repos/facebook/zstd/pulls?state=open"
|
GITHUB_API_PR_URL = "https://api.github.com/repos/facebook/zstd/pulls?state=open"
|
||||||
GITHUB_URL_TEMPLATE = "https://github.com/{}/zstd"
|
GITHUB_URL_TEMPLATE = "https://github.com/{}/zstd"
|
||||||
MASTER_BUILD = {"user": "facebook", "branch": "dev", "hash": None}
|
RELEASE_BUILD = {"user": "facebook", "branch": "dev", "hash": None}
|
||||||
|
|
||||||
# check to see if there are any new PRs every minute
|
# check to see if there are any new PRs every minute
|
||||||
DEFAULT_MAX_API_CALL_FREQUENCY_SEC = 60
|
DEFAULT_MAX_API_CALL_FREQUENCY_SEC = 60
|
||||||
@@ -264,11 +264,11 @@ def main(filenames, levels, iterations, builds=None, emails=None, continuous=Fal
|
|||||||
for test_build in builds:
|
for test_build in builds:
|
||||||
if dictionary_filename == None:
|
if dictionary_filename == None:
|
||||||
regressions = get_regressions(
|
regressions = get_regressions(
|
||||||
MASTER_BUILD, test_build, iterations, filenames, levels
|
RELEASE_BUILD, test_build, iterations, filenames, levels
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
regressions = get_regressions_dictionary(
|
regressions = get_regressions_dictionary(
|
||||||
MASTER_BUILD, test_build, filenames, dictionary_filename, levels, iterations
|
RELEASE_BUILD, test_build, filenames, dictionary_filename, levels, iterations
|
||||||
)
|
)
|
||||||
body = "\n".join(regressions)
|
body = "\n".join(regressions)
|
||||||
if len(regressions) > 0:
|
if len(regressions) > 0:
|
||||||
@@ -320,7 +320,7 @@ if __name__ == "__main__":
|
|||||||
builds = [{"user": None, "branch": "None", "hash": None}]
|
builds = [{"user": None, "branch": "None", "hash": None}]
|
||||||
main(filenames, levels, iterations, builds, frequency=frequency, dictionary_filename=dictionary_filename)
|
main(filenames, levels, iterations, builds, frequency=frequency, dictionary_filename=dictionary_filename)
|
||||||
elif mode == "fastmode":
|
elif mode == "fastmode":
|
||||||
builds = [{"user": "facebook", "branch": "master", "hash": None}]
|
builds = [{"user": "facebook", "branch": "release", "hash": None}]
|
||||||
main(filenames, levels, iterations, builds, frequency=frequency, dictionary_filename=dictionary_filename)
|
main(filenames, levels, iterations, builds, frequency=frequency, dictionary_filename=dictionary_filename)
|
||||||
else:
|
else:
|
||||||
main(filenames, levels, iterations, None, emails, True, frequency=frequency, dictionary_filename=dictionary_filename)
|
main(filenames, levels, iterations, None, emails, True, frequency=frequency, dictionary_filename=dictionary_filename)
|
||||||
|
|||||||
@@ -1342,8 +1342,6 @@ optCSize19=$(datagen -g2M | zstd -19 -c | wc -c)
|
|||||||
longCSize19=$(datagen -g2M | zstd -19 --long -c | wc -c)
|
longCSize19=$(datagen -g2M | zstd -19 --long -c | wc -c)
|
||||||
optCSize19wlog23=$(datagen -g2M | zstd -19 -c --zstd=wlog=23 | wc -c)
|
optCSize19wlog23=$(datagen -g2M | zstd -19 -c --zstd=wlog=23 | wc -c)
|
||||||
longCSize19wlog23=$(datagen -g2M | zstd -19 -c --long=23 | wc -c)
|
longCSize19wlog23=$(datagen -g2M | zstd -19 -c --long=23 | wc -c)
|
||||||
optCSize22=$(datagen -g900K | zstd -22 --ultra -c | wc -c)
|
|
||||||
longCSize22=$(datagen -g900K | zstd -22 --ultra --long -c | wc -c)
|
|
||||||
if [ "$longCSize16" -gt "$optCSize16" ]; then
|
if [ "$longCSize16" -gt "$optCSize16" ]; then
|
||||||
echo using --long on compression level 16 should not cause compressed size regression
|
echo using --long on compression level 16 should not cause compressed size regression
|
||||||
exit 1
|
exit 1
|
||||||
@@ -1353,9 +1351,6 @@ elif [ "$longCSize19" -gt "$optCSize19" ]; then
|
|||||||
elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then
|
elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then
|
||||||
echo using --long on compression level 19 with wLog=23 should not cause compressed size regression
|
echo using --long on compression level 19 with wLog=23 should not cause compressed size regression
|
||||||
exit 1
|
exit 1
|
||||||
elif [ "$longCSize22" -gt "$optCSize22" ]; then
|
|
||||||
echo using --long on compression level 22 should not cause compressed size regression
|
|
||||||
exit 1
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user