Merge branch 'dev' into http-to-https

tests/cli-tests/compression/adapt.sh

@@ -4,3 +4,11 @@ set -e
 
 # Test --adapt
 zstd -f file --adapt -c | zstd -t
+
+datagen -g100M > file100M
+
+# Pick parameters to force fast adaptation, even on slow systems
+zstd --adapt -vvvv -19 --zstd=wlog=10 file100M -o /dev/null 2>&1 | grep -q "faster speed , lighter compression"
+
+# Adaption still happens with --no-progress
+zstd --no-progress --adapt -vvvv -19 --zstd=wlog=10 file100M -o /dev/null 2>&1 | grep -q "faster speed , lighter compression"

tests/cli-tests/compression/window-resize.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+datagen -g1G > file
+zstd --long=31 -1 --single-thread --no-content-size -f file
+zstd -l -v file.zst
+
+# We want to ignore stderr (its outputting "*** zstd command line interface
+# 64-bits v1.5.3, by Yann Collet ***")
+
+rm file file.zst

tests/cli-tests/compression/window-resize.sh.stdout.glob

@@ -0,0 +1,3 @@
+...
+Window Size: 1.000 GiB (1073741824 B)
+...

tests/cli-tests/progress/no-progress.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+
+#!/bin/sh
+
+. "$COMMON/platform.sh"
+
+set -e
+
+echo hello > hello
+echo world > world
+
+zstd -q hello world
+
+println >&2 "Tests cases where progress information should not be printed"
+
+for args in \
+	"" \
+	"--fake-stderr-is-console -q" \
+	"--fake-stderr-is-console -qq --progress" \
+	"--no-progress --fake-stderr-is-console" \
+	"--no-progress --fake-stderr-is-console -v"
+do
+	println >&2 "args = $args"
+	println >&2 "compress file to file"
+	zstd $args -f hello
+	println >&2 "compress pipe to pipe"
+	zstd $args < hello > $INTOVOID
+	println >&2 "compress pipe to file"
+	zstd $args < hello -fo hello.zst
+	println >&2 "compress file to pipe"
+	zstd $args hello -c > $INTOVOID
+	println >&2 "compress 2 files"
+	zstd $args -f hello world
+
+	println >&2 "decompress file to file"
+	zstd $args -d -f hello.zst
+	println >&2 "decompress pipe to pipe"
+	zstd $args -d < hello.zst > $INTOVOID
+	println >&2 "decompress pipe to file"
+	zstd $args -d < hello.zst -fo hello
+	println >&2 "decompress file to pipe"
+	zstd $args -d hello.zst -c > $INTOVOID
+	println >&2 "decompress 2 files"
+	zstd $args -d -f hello.zst world.zst
+	println >&2 ""
+done

tests/cli-tests/progress/no-progress.sh.stderr.glob

@@ -0,0 +1,90 @@
+Tests cases where progress information should not be printed
+args =
+compress file to file
+compress pipe to pipe
+compress pipe to file
+compress file to pipe
+compress 2 files
+decompress file to file
+decompress pipe to pipe
+decompress pipe to file
+decompress file to pipe
+decompress 2 files
+
+args = --fake-stderr-is-console -q
+compress file to file
+compress pipe to pipe
+compress pipe to file
+compress file to pipe
+compress 2 files
+decompress file to file
+decompress pipe to pipe
+decompress pipe to file
+decompress file to pipe
+decompress 2 files
+
+args = --fake-stderr-is-console -qq --progress
+compress file to file
+compress pipe to pipe
+compress pipe to file
+compress file to pipe
+compress 2 files
+decompress file to file
+decompress pipe to pipe
+decompress pipe to file
+decompress file to pipe
+decompress 2 files
+
+args = --no-progress --fake-stderr-is-console
+compress file to file
+hello*hello.zst*
+compress pipe to pipe
+compress pipe to file
+*stdin*hello.zst*
+compress file to pipe
+compress 2 files
+2 files compressed*
+decompress file to file
+hello.zst*
+decompress pipe to pipe
+decompress pipe to file
+*stdin*
+decompress file to pipe
+decompress 2 files
+2 files decompressed*
+
+args = --no-progress --fake-stderr-is-console -v
+compress file to file
+*zstd*
+hello*hello.zst*
+compress pipe to pipe
+*zstd*
+*stdin*stdout*
+compress pipe to file
+*zstd*
+*stdin*hello.zst*
+compress file to pipe
+*zstd*
+*hello*stdout*
+compress 2 files
+*zstd*
+*hello*hello.zst*
+*world*world.zst*
+2 files compressed*
+decompress file to file
+*zstd*
+hello.zst*
+decompress pipe to pipe
+*zstd*
+*stdin*
+decompress pipe to file
+*zstd*
+*stdin*
+decompress file to pipe
+*zstd*
+hello.zst*
+decompress 2 files
+*zstd*
+hello.zst*
+world.zst*
+2 files decompressed*

tests/cli-tests/progress/progress.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+
+. "$COMMON/platform.sh"
+
+set -e
+
+println >&2 "Tests cases where progress information should be printed"
+
+echo hello > hello
+echo world > world
+
+zstd -q hello world
+
+for args in \
+	"--progress" \
+	"--fake-stderr-is-console" \
+	"--progress --fake-stderr-is-console -q"; do
+	println >&2 "args = $args"
+	println >&2 "compress file to file"
+	zstd $args -f hello
+	println >&2 "compress pipe to pipe"
+	zstd $args < hello > $INTOVOID
+	println >&2 "compress pipe to file"
+	zstd $args < hello -fo hello.zst
+	println >&2 "compress file to pipe"
+	zstd $args hello -c > $INTOVOID
+	println >&2 "compress 2 files"
+	zstd $args -f hello world
+
+	println >&2 "decompress file to file"
+	zstd $args -d -f hello.zst
+	println >&2 "decompress pipe to pipe"
+	zstd $args -d < hello.zst > $INTOVOID
+	println >&2 "decompress pipe to file"
+	zstd $args -d < hello.zst -fo hello
+	println >&2 "decompress file to pipe"
+	zstd $args -d hello.zst -c > $INTOVOID
+	println >&2 "decompress 2 files"
+	zstd $args -d -f hello.zst world.zst
+	println >&2 ""
+done

tests/cli-tests/progress/progress.sh.stderr.glob

@@ -0,0 +1,62 @@
+Tests cases where progress information should be printed
+args = --progress
+compress file to file
+*Read:*hello*hello.zst*
+compress pipe to pipe
+*Read:*stdin*stdout*
+compress pipe to file
+*Read:*stdin*hello.zst*
+compress file to pipe
+*Read:*hello*stdout*
+compress 2 files
+*Read*2 files compressed*
+decompress file to file
+*hello.zst*hello.zst*
+decompress pipe to pipe
+*stdin*stdin*
+decompress pipe to file
+*stdin*stdin*
+decompress file to pipe
+*hello.zst*hello.zst*
+decompress 2 files
+*hello.zst*2 files decompressed*
+
+args = --fake-stderr-is-console
+compress file to file
+*Read:*hello*hello.zst*
+compress pipe to pipe
+compress pipe to file
+*Read:*stdin*hello.zst*
+compress file to pipe
+compress 2 files
+*Read*2 files compressed*
+decompress file to file
+*hello.zst*hello.zst*
+decompress pipe to pipe
+decompress pipe to file
+*stdin*stdin*
+decompress file to pipe
+decompress 2 files
+*hello.zst*2 files decompressed*
+
+args = --progress --fake-stderr-is-console -q
+compress file to file
+*Read:*hello*hello.zst*
+compress pipe to pipe
+*Read:*stdin*stdout*
+compress pipe to file
+*Read:*stdin*hello.zst*
+compress file to pipe
+*Read:*hello*stdout*
+compress 2 files
+*Read*2 files compressed*
+decompress file to file
+*hello.zst*hello.zst*
+decompress pipe to pipe
+*stdin*stdin*
+decompress pipe to file
+*stdin*stdin*
+decompress file to pipe
+*hello.zst*hello.zst*
+decompress 2 files
+*hello.zst*2 files decompressed*

tests/fuzz/fuzz.h

@@ -26,8 +26,7 @@
  * @param MEM_FORCE_MEMORY_ACCESS:
  *        This flag controls how the zstd library accesses unaligned memory.
  *        It can be undefined, or 0 through 2. If it is undefined, it selects
- *        the method to use based on the compiler. If testing with UBSAN set
- *        MEM_FORCE_MEMORY_ACCESS=0 to use the standard compliant method.
+ *        the method to use based on the compiler.
  * @param FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
  *        This is the canonical flag to enable deterministic builds for fuzzing.
  *        Changes to zstd for fuzzing are gated behind this define.

tests/fuzz/stream_decompress.c

@@ -99,14 +99,14 @@ int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size)
 
     while (size > 0) {
         ZSTD_inBuffer in = makeInBuffer(&src, &size, producer);
-        while (in.pos != in.size) {
+        do {
+            size_t const rc = ZSTD_decompressStream(dstream, &out, &in);
+            if (ZSTD_isError(rc)) goto error;
             if (out.pos == out.size) {
                 if (stableOutBuffer) goto error;
                 out = makeOutBuffer(producer, buf, bufSize);
             }
-            size_t const rc = ZSTD_decompressStream(dstream, &out, &in);
-            if (ZSTD_isError(rc)) goto error;
-        }
+        } while (in.pos != in.size);
     }
 
 error: