ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/longRangeMatcher' into ldm-integrate

Browse Source
This commit is contained in:
Stella Lau
2017-09-01 10:19:38 -07:00
parent 8081becadc 9ab7614d23
commit 17d8e0bdcc
199 changed files with 6486 additions and 1738 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/.gitignore
+2
View File
@@ -21,6 +21,7 @@ symbols
legacy
decodecorpus
pool
poolTests
invalidDictionaries
# Tmp test directory
@@ -48,6 +49,7 @@ grillResults.txt
_*
tmp*
*.zst
*.gz
result
out
tests/Makefile
+17 -22
View File
@@ -1,13 +1,11 @@
# ##########################################################################
# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
# ################################################################
# Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
# All rights reserved.
#
# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.
# ##########################################################################
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# ################################################################
# datagen : Synthetic and parametrable data generator, for tests
# fullbench : Precisely measure speed for each zstd inner functions
# fullbench32: Same as fullbench, but forced to compile in 32-bits mode
@@ -46,10 +44,6 @@ ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
ZBUFF_FILES := $(ZSTDDIR)/deprecated/*.c
ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
ZSTD_OBJ := $(patsubst %.c,%.o, $(wildcard $(ZSTD_FILES)) )
ZBUFF_OBJ := $(patsubst %.c,%.o, $(wildcard $(ZBUFF_FILES)) )
ZDICT_OBJ := $(patsubst %.c,%.o, $(wildcard $(ZDICT_FILES)) )
# Define *.exe as extension for Windows systems
ifneq (,$(filter Windows%,$(OS)))
@@ -77,7 +71,7 @@ all: fullbench fuzzer zstreamtest paramgrill datagen decodecorpus
all32: fullbench32 fuzzer32 zstreamtest32
allnothread: fullbench fuzzer paramgrill datagen decodecorpus
allnothread: fullbench fuzzer paramgrill datagen decodecorpus
dll: fuzzer-dll zstreamtest-dll
@@ -91,7 +85,7 @@ zstd-nolegacy:
$(MAKE) -C $(PRGDIR) $@
gzstd:
$(MAKE) -C $(PRGDIR) $@
$(MAKE) -C $(PRGDIR) zstd HAVE_ZLIB=1
fullbench32: CPPFLAGS += -m32
fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP)
@@ -192,8 +186,8 @@ else
$(CC) $(FLAGS) $^ -o $@$(EXT) -Wl,-rpath=$(ZSTDDIR) $(ZSTDDIR)/libzstd.so
endif
poolTests : poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c
$(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
poolTests : poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c
$(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
namespaceTest:
if $(CC) namespaceTest.c ../lib/common/xxhash.c -o $@ ; then echo compilation should fail; exit 1 ; fi
@@ -280,17 +274,17 @@ test-zstd: zstd zstd-playTests
test-zstd32: ZSTD = $(PRGDIR)/zstd32
test-zstd32: zstd32 zstd-playTests
test-zstd-nolegacy: ZSTD = $(PRGDIR)/zstd
test-zstd-nolegacy: ZSTD = $(PRGDIR)/zstd-nolegacy
test-zstd-nolegacy: zstd-nolegacy zstd-playTests
test-gzstd: gzstd
$(PRGDIR)/zstd README.md test-zstd-speed.py
gzip README.md test-zstd-speed.py
$(PRGDIR)/zstd -f README.md test-zstd-speed.py
gzip -f README.md test-zstd-speed.py
cat README.md.zst test-zstd-speed.py.gz >zstd_gz.zst
cat README.md.gz test-zstd-speed.py.zst >gz_zstd.gz
$(PRGDIR)/zstd -d README.md.gz -o README2.md
$(PRGDIR)/zstd -d README.md.gz test-zstd-speed.py.gz
$(PRGDIR)/zstd -d zstd_gz.zst gz_zstd.gz
$(PRGDIR)/zstd -df README.md.gz -o README2.md
$(PRGDIR)/zstd -df README.md.gz test-zstd-speed.py.gz
$(PRGDIR)/zstd -df zstd_gz.zst gz_zstd.gz
$(DIFF) -q zstd_gz gz_zstd
echo Hello World ZSTD | $(PRGDIR)/zstd -c - >hello.zst
echo Hello World GZIP | gzip -c - >hello.gz
@@ -298,6 +292,7 @@ test-gzstd: gzstd
cat hello.zst hello.gz hello.txt >hello_zst_gz_txt.gz
$(PRGDIR)/zstd -dcf hello.*
$(PRGDIR)/zstd -dcf - <hello_zst_gz_txt.gz
$(RM) *.gz *.zst README2.md gz_zstd zstd_gz hello.txt
test-fullbench: fullbench datagen
$(QEMU_SYS) ./fullbench -i1
tests/datagencli.c
+4 -4
View File
@@ -1,10 +1,10 @@
/**
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
tests/decodecorpus.c
+5 -5
View File
@@ -1,10 +1,10 @@
/**
* Copyright (c) 2017-present, Facebook, Inc.
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
#include <limits.h>
tests/files/huffman-compressed-larger
BIN
View File
Binary file not shown.
tests/fullbench.c
+4 -4
View File
@@ -1,10 +1,10 @@
/**
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
tests/fuzz/Makefile
+5 -7
View File
@@ -1,13 +1,11 @@
# ##########################################################################
# ################################################################
# Copyright (c) 2016-present, Facebook, Inc.
# All rights reserved.
#
# This Makefile is validated for Linux, and macOS targets
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.
# ##########################################################################
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# ################################################################
CFLAGS ?= -O3
CXXFLAGS ?= -O3
tests/fuzz/fuzz.h
+4 -4
View File
@@ -1,10 +1,10 @@
/**
/*
* Copyright (c) 2016-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
/**
tests/fuzz/fuzz_helpers.h
+4 -4
View File
@@ -1,10 +1,10 @@
/**
/*
* Copyright (c) 2016-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
/**
tests/fuzz/regression_driver.c
+4 -4
View File
@@ -1,10 +1,10 @@
/**
/*
* Copyright (c) 2016-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
#include "fuzz.h"
tests/fuzz/simple_decompress.c
+5 -5
View File
@@ -1,10 +1,10 @@
/**
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
/*
* Copyright (c) 2016-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
/**
tests/fuzz/simple_round_trip.c
+4 -4
View File
@@ -1,10 +1,10 @@
/**
/*
* Copyright (c) 2016-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
/**
tests/fuzz/stream_decompress.c
+5 -5
View File
@@ -1,10 +1,10 @@
/**
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
/*
* Copyright (c) 2016-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
/**
tests/fuzz/stream_round_trip.c
+4 -4
View File
@@ -1,10 +1,10 @@
/**
/*
* Copyright (c) 2016-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
/**
tests/fuzzer.c
+4 -4
View File
@@ -1,10 +1,10 @@
/**
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
tests/gzip/Makefile
+5 -5
View File
@@ -1,10 +1,10 @@
# ################################################################
# Copyright (c) 2017-present, Yann Collet, Facebook, Inc.
# Copyright (c) 2017-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# ################################################################
PRGDIR = ../../programs
@@ -12,7 +12,7 @@ VOID = /dev/null
export PATH := .:$(PATH)
.PHONY: all
#all: test-gzip-env
#all: test-gzip-env
all: test-helin-segv test-hufts test-keep test-list test-memcpy-abuse test-mixed
all: test-null-suffix-clobber test-stdin test-trailing-nul test-unpack-invalid
all: test-zdiff test-zgrep-context test-zgrep-f test-zgrep-signal test-znew-k test-z-suffix
tests/invalidDictionaries.c
+9
View File
@@ -1,3 +1,12 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
#include <stddef.h>
#include "zstd.h"
tests/legacy.c
+5 -6
View File
@@ -1,10 +1,10 @@
/**
* Copyright (c) 2017-present, Facebook, Inc.
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
/*
@@ -226,4 +226,3 @@ const char* const EXPECTED =
"snowden is snowed in / he's now then in his snow den / when does the snow end?\n"
"goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n"
"when life shuts a door, / just open it. it’s a door. / that is how doors work.\n";
tests/longmatch.c
+10
View File
@@ -1,3 +1,13 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
#include <stdio.h>
#include <stddef.h>
#include <stdlib.h>
tests/namespaceTest.c
+4 -4
View File
@@ -1,10 +1,10 @@
/**
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
tests/paramgrill.c
+4 -4
View File
@@ -1,10 +1,10 @@
/**
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
tests/playTests.sh
+10 -1
View File
@@ -291,8 +291,10 @@ $ECHO "- Create dictionary with wrong dictID parameter order (must fail)"
$ZSTD --train *.c ../programs/*.c --dictID -o 1 tmpDict1 && die "wrong order : --dictID must be followed by argument "
$ECHO "- Create dictionary with size limit"
$ZSTD --train *.c ../programs/*.c -o tmpDict2 --maxdict=4K -v
$ECHO "- Create dictionary with small size limit"
$ZSTD --train *.c ../programs/*.c -o tmpDict3 --maxdict=1K -v
$ECHO "- Create dictionary with wrong parameter order (must fail)"
$ZSTD --train *.c ../programs/*.c -o tmpDict2 --maxdict -v 4K && die "wrong order : --maxdict must be followed by argument "
$ZSTD --train *.c ../programs/*.c -o tmpDict3 --maxdict -v 4K && die "wrong order : --maxdict must be followed by argument "
$ECHO "- Compress without dictID"
$ZSTD -f tmp -D tmpDict1 --no-dictID
$ZSTD -d tmp.zst -D tmpDict -fo result
@@ -386,6 +388,13 @@ $ZSTD -t tmpSplit.* && die "bad file not detected !"
./datagen | $ZSTD -c | $ZSTD -t
$ECHO "\n**** golden files tests **** "
$ZSTD -t -r files
$ZSTD -c -r files | $ZSTD -t
$ECHO "\n**** benchmark mode tests **** "
$ECHO "bench one file"
tests/poolTests.c
+38 -3
View File
@@ -1,5 +1,16 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
#include "pool.h"
#include "threading.h"
#include "util.h"
#include <stddef.h>
#include <stdio.h>
@@ -50,21 +61,45 @@ int testOrder(size_t numThreads, size_t queueSize) {
return 0;
}
void waitFn(void *opaque) {
(void)opaque;
UTIL_sleepMilli(1);
}
/* Tests for deadlock */
int testWait(size_t numThreads, size_t queueSize) {
struct data data;
POOL_ctx *ctx = POOL_create(numThreads, queueSize);
ASSERT_TRUE(ctx);
{
size_t i;
for (i = 0; i < 16; ++i) {
POOL_add(ctx, &waitFn, &data);
}
}
POOL_free(ctx);
return 0;
}
int main(int argc, const char **argv) {
size_t numThreads;
for (numThreads = 1; numThreads <= 4; ++numThreads) {
size_t queueSize;
for (queueSize = 1; queueSize <= 2; ++queueSize) {
for (queueSize = 0; queueSize <= 2; ++queueSize) {
if (testOrder(numThreads, queueSize)) {
printf("FAIL: testOrder\n");
return 1;
}
if (testWait(numThreads, queueSize)) {
printf("FAIL: testWait\n");
return 1;
}
}
}
printf("PASS: testOrder\n");
(void)argc;
(void)argv;
return (POOL_create(0, 1) || POOL_create(1, 0)) ? printf("FAIL: testInvalid\n"), 1
: printf("PASS: testInvalid\n"), 0;
return (POOL_create(0, 1)) ? printf("FAIL: testInvalid\n"), 1
: printf("PASS: testInvalid\n"), 0;
return 0;
}
tests/roundTripCrash.c
+4 -4
View File
@@ -1,10 +1,10 @@
/**
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
/*
tests/symbols.c
+10
View File
@@ -1,3 +1,13 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
#include <stdio.h>
#include "zstd_errors.h"
#define ZSTD_STATIC_LINKING_ONLY
tests/test-zstd-speed.py
+5 -5
View File
@@ -1,13 +1,13 @@
#! /usr/bin/env python3
#
# ################################################################
# Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# ##########################################################################
# Limitations:
# - doesn't support filenames with spaces
tests/test-zstd-versions.py
+5 -5
View File
@@ -1,14 +1,14 @@
#!/usr/bin/env python3
"""Test zstd interoperability between versions"""
#
# ################################################################
# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.
#
# This source code is licensed under both the BSD-style license (found in the
# LICENSE file in the root directory of this source tree) and the GPLv2 (found
# in the COPYING file in the root directory of this source tree).
# ################################################################
import filecmp
import glob
tests/zbufftest.c
+4 -4
View File
@@ -1,10 +1,10 @@
/**
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/
tests/zstreamtest.c
+4 -4
View File
@@ -1,10 +1,10 @@
/**
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*/