ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

initial commit

Browse Source
This commit is contained in:
Danielle Rozenblit
2023-01-04 13:01:54 -08:00
parent ef566c8d68
commit 908e812733
328 changed files with 3340 additions and 1426 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/cli-tests/README.md
+10
View File
@@ -45,6 +45,16 @@ Examples:
./run.py --preserve --verbose basic/help.sh
```
### Updating exact output
If a test is failing because a `.stderr.exact` or `.stdout.exact` no longer matches, you can re-run the tests with `--set-exact-output` and the correct output will be written.
Example:
```
./run.py --set-exact-output
./run.py basic/help.sh --set-exact-output
```
## Writing a test
Test cases are arbitrary executables, and can be written in any language, but are generally shell scripts.
tests/cli-tests/basic/help.sh.stdout.glob
+25 -16
View File
@@ -1,25 +1,34 @@
+ zstd -h
Usage: zstd *OPTION*... *FILE*... *-o file*
Compress or uncompress FILEs (with no FILE or when FILE is `-`, read from standard input).
Compress or decompress the INPUT file(s); reads from STDIN if INPUT is `-` or not provided.
-o file result stored into `file` (only 1 output file)
-1 .. -19 compression level (faster .. better; default: 3)
-d, --decompress decompression
-f, --force disable input and output checks. Allows overwriting existing files,
input from console, output to stdout, operating on links,
block devices, etc. During decompression and when the output
destination is stdout, pass-through unrecognized formats as-is.
--rm remove source file(s) after successful de/compression
-k, --keep preserve source file(s) (default)
-D DICT use DICT as Dictionary for compression or decompression
-h display usage and exit
-H,--help display long help and exit
Usage: zstd *OPTIONS...* *INPUT... | -* *-o OUTPUT*
Options:
-o OUTPUT Write output to a single file, OUTPUT.
-k, --keep Preserve INPUT file(s). *Default*
--rm Remove INPUT file(s) after successful (de)compression.
-# Desired compression level, where `#` is a number between 1 and 19;
lower numbers provide faster compression, higher numbers yield
better compression ratios. *Default: 3*
-d, --decompress Perform decompression.
-D DICT Use DICT as the dictionary for compression or decompression.
-f, --force Disable input and output checks. Allows overwriting existing files,
receiving input from the console, printing ouput to STDOUT, and
operating on links, block devices, etc. Unrecognized formats will be
passed-through through as-is.
-h Display short usage and exit.
-H, --help Display full help and exit.
-V, --version Display the program version and exit.
+ zstd -H
...
Advanced options :
Advanced options:
...
+ zstd --help
...
Advanced options :
Advanced options:
...
tests/cli-tests/basic/version.sh.stdout.glob
+2 -2
View File
@@ -1,2 +1,2 @@
*** zstd command line interface *-bits v1.*.*, by Yann Collet ***
*** zstd command line interface *-bits v1.*.*, by Yann Collet ***
*** Zstandard CLI (*-bit) v1.*.*, by Yann Collet ***
*** Zstandard CLI (*-bit) v1.*.*, by Yann Collet ***
tests/cli-tests/compression/levels.sh
+1 -3
View File
@@ -10,11 +10,9 @@ zstd --fast=10 file -o file-f10.zst
zstd --fast=1 file -o file-f1.zst
zstd -1 file -o file-1.zst
zstd -19 file -o file-19.zst
zstd -22 --ultra file -o file-22.zst
zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-22.zst
zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst
cmp_size -ne file-19.zst file-22.zst
cmp_size -lt file-19.zst file-1.zst
cmp_size -lt file-1.zst file-f1.zst
cmp_size -lt file-f1.zst file-f10.zst
tests/cli-tests/compression/levels.sh.stderr.exact
+1 -3
View File
@@ -6,11 +6,9 @@ zstd --fast=10 file -o file-f10.zst
zstd --fast=1 file -o file-f1.zst
zstd -1 file -o file-1.zst
zstd -19 file -o file-19.zst
zstd -22 --ultra file -o file-22.zst
zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-22.zst
zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst
cmp_size -ne file-19.zst file-22.zst
cmp_size -lt file-19.zst file-1.zst
cmp_size -lt file-1.zst file-f1.zst
cmp_size -lt file-f1.zst file-f10.zst
tests/cli-tests/file-stat/compress-file-to-file.sh
Executable
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -e
datagen > file
zstd file -q --trace-file-stat -o file.zst
zstd -tq file.zst
tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact
+36
View File
@@ -0,0 +1,36 @@
Trace:FileStat: > UTIL_isLink(file)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(2)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_getFileSize(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 1
Trace:FileStat: < 65537
Trace:FileStat: > UTIL_isDirectory(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 1
Trace:FileStat: < 0
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isSameFile(file, file.zst)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file.zst)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file.zst)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: < 1
Trace:FileStat: < 1
Trace:FileStat: > UTIL_getFileSize(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 1
Trace:FileStat: < 65537
Trace:FileStat: > UTIL_utime(file.zst)
Trace:FileStat: < 0
tests/cli-tests/file-stat/compress-file-to-stdout.sh
Executable
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -e
datagen > file
zstd file -cq --trace-file-stat > file.zst
zstd -tq file.zst
tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact
+22
View File
@@ -0,0 +1,22 @@
Trace:FileStat: > UTIL_isLink(file)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(1)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_getFileSize(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 1
Trace:FileStat: < 65537
Trace:FileStat: > UTIL_isDirectory(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 1
Trace:FileStat: < 0
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
Trace:FileStat: > UTIL_stat(/*stdout*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_getFileSize(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 1
Trace:FileStat: < 65537
tests/cli-tests/file-stat/compress-stdin-to-file.sh
Executable
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -e
datagen > file
zstd < file -q --trace-file-stat -o file.zst
zstd -tq file.zst
tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact
+28
View File
@@ -0,0 +1,28 @@
Trace:FileStat: > UTIL_isConsole(0)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(2)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < -1
Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file.zst)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file.zst)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file.zst)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: < 1
Trace:FileStat: < 1
Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < -1
tests/cli-tests/file-stat/compress-stdin-to-stdout.sh
Executable
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -e
datagen > file
zstd < file -cq --trace-file-stat > file.zst
zstd -tq file.zst
tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact
+20
View File
@@ -0,0 +1,20 @@
Trace:FileStat: > UTIL_isConsole(0)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(1)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < -1
Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
Trace:FileStat: > UTIL_stat(/*stdout*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_getFileSize(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < -1
tests/cli-tests/file-stat/decompress-file-to-file.sh
Executable
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -e
datagen | zstd -q > file.zst
zstd -dq --trace-file-stat file.zst
tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact
+30
View File
@@ -0,0 +1,30 @@
Trace:FileStat: > UTIL_isLink(file.zst)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(1)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(2)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isDirectory(file.zst)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: < 1
Trace:FileStat: < 0
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isSameFile(file.zst, file)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 1
Trace:FileStat: < 1
Trace:FileStat: > UTIL_utime(file)
Trace:FileStat: < 0
tests/cli-tests/file-stat/decompress-file-to-stdout.sh
Executable
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -e
datagen | zstd -q > file.zst
zstd -dcq --trace-file-stat file.zst > file
tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact
+14
View File
@@ -0,0 +1,14 @@
Trace:FileStat: > UTIL_isLink(file.zst)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(1)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isDirectory(file.zst)
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: < 1
Trace:FileStat: < 0
Trace:FileStat: > UTIL_stat(file.zst)
Trace:FileStat: < 1
Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
Trace:FileStat: > UTIL_stat(/*stdout*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
tests/cli-tests/file-stat/decompress-stdin-to-file.sh
Executable
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -e
datagen | zstd -q > file.zst
zstd -dcq --trace-file-stat < file.zst -o file
tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact
+20
View File
@@ -0,0 +1,20 @@
Trace:FileStat: > UTIL_isConsole(0)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(2)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(file)
Trace:FileStat: > UTIL_stat(file)
Trace:FileStat: < 1
Trace:FileStat: < 1
tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh
Executable
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -e
datagen | zstd -q > file.zst
zstd -dcq --trace-file-stat < file.zst > file
tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact
+12
View File
@@ -0,0 +1,12 @@
Trace:FileStat: > UTIL_isConsole(0)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isConsole(1)
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isDirectory(/*stdin*\)
Trace:FileStat: > UTIL_stat(/*stdin*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
Trace:FileStat: > UTIL_isRegularFile(/*stdout*\)
Trace:FileStat: > UTIL_stat(/*stdout*\)
Trace:FileStat: < 0
Trace:FileStat: < 0
tests/cli-tests/progress/no-progress.sh.stderr.glob
+10 -10
View File
@@ -55,36 +55,36 @@ decompress 2 files
args = --no-progress --fake-stderr-is-console -v
compress file to file
*zstd*
*Zstandard CLI*
hello*hello.zst*
compress pipe to pipe
*zstd*
*Zstandard CLI*
*stdin*stdout*
compress pipe to file
*zstd*
*Zstandard CLI*
*stdin*hello.zst*
compress file to pipe
*zstd*
*Zstandard CLI*
*hello*stdout*
compress 2 files
*zstd*
*Zstandard CLI*
*hello*hello.zst*
*world*world.zst*
2 files compressed*
decompress file to file
*zstd*
*Zstandard CLI*
hello.zst*
decompress pipe to pipe
*zstd*
*Zstandard CLI*
*stdin*
decompress pipe to file
*zstd*
*Zstandard CLI*
*stdin*
decompress file to pipe
*zstd*
*Zstandard CLI*
hello.zst*
decompress 2 files
*zstd*
*Zstandard CLI*
hello.zst*
world.zst*
2 files decompressed*
tests/cli-tests/run.py
+16 -4
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
# ################################################################
# Copyright (c) Facebook, Inc.
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under both the BSD-style license (found in the
@@ -209,6 +209,7 @@ class Options:
preserve: bool,
scratch_dir: str,
test_dir: str,
set_exact_output: bool,
) -> None:
self.env = env
self.timeout = timeout
@@ -216,6 +217,7 @@ class Options:
self.preserve = preserve
self.scratch_dir = scratch_dir
self.test_dir = test_dir
self.set_exact_output = set_exact_output
class TestCase:
@@ -335,7 +337,7 @@ class TestCase:
self._test_stdin.close()
self._test_stdin = None
def _check_output_exact(self, out_name: str, expected: bytes) -> None:
def _check_output_exact(self, out_name: str, expected: bytes, exact_name: str) -> None:
"""
Check the output named :out_name: for an exact match against the :expected: content.
Saves the success and message.
@@ -349,6 +351,10 @@ class TestCase:
self._success[check_name] = False
self._message[check_name] = f"{out_name} does not match!\n> diff expected actual\n{diff(expected, actual)}"
if self._opts.set_exact_output:
with open(exact_name, "wb") as f:
f.write(actual)
def _check_output_glob(self, out_name: str, expected: bytes) -> None:
"""
Check the output named :out_name: for a glob match against the :expected: glob.
@@ -386,7 +392,7 @@ class TestCase:
ignore_name = f"{self._test_file}.{out_name}.ignore"
if os.path.exists(exact_name):
return self._check_output_exact(out_name, read_file(exact_name))
return self._check_output_exact(out_name, read_file(exact_name), exact_name)
elif os.path.exists(glob_name):
return self._check_output_glob(out_name, read_file(glob_name))
elif os.path.exists(ignore_name):
@@ -394,7 +400,7 @@ class TestCase:
self._success[check_name] = True
self._message[check_name] = f"{out_name} ignored!"
else:
return self._check_output_exact(out_name, bytes())
return self._check_output_exact(out_name, bytes(), exact_name)
def _check_stderr(self) -> None:
"""Checks the stderr output against the expectation."""
@@ -678,6 +684,11 @@ if __name__ == "__main__":
"Scratch directory located in TEST_DIR/scratch/."
)
)
parser.add_argument(
"--set-exact-output",
action="store_true",
help="Set stderr.exact and stdout.exact for all failing tests, unless .ignore or .glob already exists"
)
parser.add_argument(
"tests",
nargs="*",
@@ -714,6 +725,7 @@ if __name__ == "__main__":
preserve=args.preserve,
test_dir=args.test_dir,
scratch_dir=scratch_dir,
set_exact_output=args.set_exact_output,
)
if len(args.tests) == 0: