ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CI] Re-enable versions-test

Browse Source 
It seems like with the deletion of Travis CI we didn't successfully transfer the
version compatibility test. Attempt to enable the version compatibility test.
This commit is contained in:
Nick Terrell
2022-12-16 12:04:21 -08:00
committed by Nick Terrell
parent ef566c8d68
commit 58508398f4
2 changed files with 48 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/dev-short-tests.yml
+7 -12
View File
@@ -431,18 +431,13 @@ jobs:
cc -Wall -Wextra -Wpedantic -Werror -o simple examples/simple_compression.c $(pkg-config --cflags --libs libzstd) cc -Wall -Wextra -Wpedantic -Werror -o simple examples/simple_compression.c $(pkg-config --cflags --libs libzstd)
./simple LICENSE ./simple LICENSE
versions-compatibility:
# This test currently fails on Github Actions specifically. runs-on: ubuntu-latest
# Possible reason : TTY emulation. steps:
# Note that the same test works fine locally and on travisCI. - uses: actions/checkout@v3
# This will have to be fixed before transferring the test to GA. - name: Versions Compatibility Test
# versions-compatibility: run: |
# runs-on: ubuntu-latest make -C tests versionsTest
# steps:
# - uses: actions/checkout@v3
# - name: Versions Compatibility Test
# run: |
# make -C tests versionsTest
# For reference : icc tests # For reference : icc tests
tests/test-zstd-versions.py
+41 -61
View File
@@ -23,7 +23,7 @@ from subprocess import Popen, PIPE
repo_url = 'https://github.com/facebook/zstd.git' repo_url = 'https://github.com/facebook/zstd.git'
tmp_dir_name = 'tests/versionsTest' tmp_dir_name = 'tests/versionsTest'
make_cmd = 'make' make_cmd = 'make'
make_args = ['-j','CFLAGS=-O1'] make_args = ['-j','CFLAGS=-O0']
git_cmd = 'git' git_cmd = 'git'
test_dat_src = 'README.md' test_dat_src = 'README.md'
test_dat = 'test_dat' test_dat = 'test_dat'
@@ -86,41 +86,46 @@ def create_dict(tag, dict_source_path):
if result == 0: if result == 0:
print(dict_name + ' created') print(dict_name + ' created')
else: else:
print('ERROR: creating of ' + dict_name + ' failed') raise RuntimeError('ERROR: creating of ' + dict_name + ' failed')
else: else:
print(dict_name + ' already exists') print(dict_name + ' already exists')
def zstd(tag, args, input_file, output_file):
"""
Zstd compress input_file to output_file.
Need this helper because 0.5.0 is broken when stdout is not a TTY.
Throws an exception if the command returns non-zero.
"""
with open(input_file, "rb") as i:
with open(output_file, "wb") as o:
cmd = ['./zstd.' + tag] + args
print("Running: '{}', input={}, output={}" .format(
' '.join(cmd), input_file, output_file
))
subprocess.check_call(cmd, stdin=i, stdout=o)
def dict_compress_sample(tag, sample): def dict_compress_sample(tag, sample):
dict_name = 'dict.' + tag dict_name = 'dict.' + tag
DEVNULL = open(os.devnull, 'wb') zstd(tag, ['-D', dict_name, '-1'], sample, sample + '_01_64_' + tag + '_dictio.zst')
if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-f', sample], stderr=DEVNULL) == 0: zstd(tag, ['-D', dict_name, '-3'], sample, sample + '_03_64_' + tag + '_dictio.zst')
os.rename(sample + '.zst', sample + '_01_64_' + tag + '_dictio.zst') zstd(tag, ['-D', dict_name, '-5'], sample, sample + '_05_64_' + tag + '_dictio.zst')
if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-5f', sample], stderr=DEVNULL) == 0: zstd(tag, ['-D', dict_name, '-9'], sample, sample + '_09_64_' + tag + '_dictio.zst')
os.rename(sample + '.zst', sample + '_05_64_' + tag + '_dictio.zst') zstd(tag, ['-D', dict_name, '-15'], sample, sample + '_15_64_' + tag + '_dictio.zst')
if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-9f', sample], stderr=DEVNULL) == 0: zstd(tag, ['-D', dict_name, '-18'], sample, sample + '_18_64_' + tag + '_dictio.zst')
os.rename(sample + '.zst', sample + '_09_64_' + tag + '_dictio.zst')
if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-15f', sample], stderr=DEVNULL) == 0:
os.rename(sample + '.zst', sample + '_15_64_' + tag + '_dictio.zst')
if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-18f', sample], stderr=DEVNULL) == 0:
os.rename(sample + '.zst', sample + '_18_64_' + tag + '_dictio.zst')
# zstdFiles = glob.glob("*.zst*") # zstdFiles = glob.glob("*.zst*")
# print(zstdFiles) # print(zstdFiles)
print(tag + " : dict compression completed") print(tag + " : dict compression completed")
def compress_sample(tag, sample): def compress_sample(tag, sample):
DEVNULL = open(os.devnull, 'wb') zstd(tag, ['-1'], sample, sample + '_01_64_' + tag + '_nodict.zst')
if subprocess.call(['./zstd.' + tag, '-f', sample], stderr=DEVNULL) == 0: zstd(tag, ['-3'], sample, sample + '_03_64_' + tag + '_nodict.zst')
os.rename(sample + '.zst', sample + '_01_64_' + tag + '_nodict.zst') zstd(tag, ['-5'], sample, sample + '_05_64_' + tag + '_nodict.zst')
if subprocess.call(['./zstd.' + tag, '-5f', sample], stderr=DEVNULL) == 0: zstd(tag, ['-9'], sample, sample + '_09_64_' + tag + '_nodict.zst')
os.rename(sample + '.zst', sample + '_05_64_' + tag + '_nodict.zst') zstd(tag, ['-15'], sample, sample + '_15_64_' + tag + '_nodict.zst')
if subprocess.call(['./zstd.' + tag, '-9f', sample], stderr=DEVNULL) == 0: zstd(tag, ['-18'], sample, sample + '_18_64_' + tag + '_nodict.zst')
os.rename(sample + '.zst', sample + '_09_64_' + tag + '_nodict.zst')
if subprocess.call(['./zstd.' + tag, '-15f', sample], stderr=DEVNULL) == 0:
os.rename(sample + '.zst', sample + '_15_64_' + tag + '_nodict.zst')
if subprocess.call(['./zstd.' + tag, '-18f', sample], stderr=DEVNULL) == 0:
os.rename(sample + '.zst', sample + '_18_64_' + tag + '_nodict.zst')
# zstdFiles = glob.glob("*.zst*") # zstdFiles = glob.glob("*.zst*")
# print(zstdFiles) # print(zstdFiles)
print(tag + " : compression completed") print(tag + " : compression completed")
@@ -150,23 +155,13 @@ def decompress_zst(tag):
dec_error = 0 dec_error = 0
list_zst = sorted(glob.glob('*_nodict.zst')) list_zst = sorted(glob.glob('*_nodict.zst'))
for file_zst in list_zst: for file_zst in list_zst:
print(file_zst, end=' ') print(file_zst + ' ' + tag)
print(tag, end=' ')
file_dec = file_zst + '_d64_' + tag + '.dec' file_dec = file_zst + '_d64_' + tag + '.dec'
if tag <= 'v0.5.0': zstd(tag, ['-d'], file_zst, file_dec)
params = ['./zstd.' + tag, '-df', file_zst, file_dec] if not filecmp.cmp(file_dec, test_dat):
raise RuntimeError('Decompression failed: tag={} file={}'.format(tag, file_zst))
else: else:
params = ['./zstd.' + tag, '-df', file_zst, '-o', file_dec] print('OK ')
if execute(params) == 0:
if not filecmp.cmp(file_dec, test_dat):
print('ERR !! ')
dec_error = 1
else:
print('OK ')
else:
print('command does not work')
dec_error = 1
return dec_error
def decompress_dict(tag): def decompress_dict(tag):
@@ -181,22 +176,13 @@ def decompress_dict(tag):
if tag == 'v0.6.0' and dict_tag < 'v0.6.0': if tag == 'v0.6.0' and dict_tag < 'v0.6.0':
continue continue
dict_name = 'dict.' + dict_tag dict_name = 'dict.' + dict_tag
print(file_zst + ' ' + tag + ' dict=' + dict_tag, end=' ') print(file_zst + ' ' + tag + ' dict=' + dict_tag)
file_dec = file_zst + '_d64_' + tag + '.dec' file_dec = file_zst + '_d64_' + tag + '.dec'
if tag <= 'v0.5.0': zstd(tag, ['-D', dict_name, '-d'], file_zst, file_dec)
params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, file_dec] if not filecmp.cmp(file_dec, test_dat):
raise RuntimeError('Decompression failed: tag={} file={}'.format(tag, file_zst))
else: else:
params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, '-o', file_dec] print('OK ')
if execute(params) == 0:
if not filecmp.cmp(file_dec, test_dat):
print('ERR !! ')
dec_error = 1
else:
print('OK ')
else:
print('command does not work')
dec_error = 1
return dec_error
if __name__ == '__main__': if __name__ == '__main__':
@@ -267,25 +253,19 @@ if __name__ == '__main__':
print('Compress test.dat by all released zstd') print('Compress test.dat by all released zstd')
print('-----------------------------------------------') print('-----------------------------------------------')
error_code = 0
for tag in tags: for tag in tags:
print(tag) print(tag)
if tag >= 'v0.5.0': if tag >= 'v0.5.0':
create_dict(tag, dict_source_path) create_dict(tag, dict_source_path)
dict_compress_sample(tag, test_dat) dict_compress_sample(tag, test_dat)
remove_duplicates() remove_duplicates()
error_code += decompress_dict(tag) decompress_dict(tag)
compress_sample(tag, test_dat) compress_sample(tag, test_dat)
remove_duplicates() remove_duplicates()
error_code += decompress_zst(tag) decompress_zst(tag)
print('') print('')
print('Enumerate different compressed files') print('Enumerate different compressed files')
zstds = sorted(glob.glob('*.zst')) zstds = sorted(glob.glob('*.zst'))
for zstd in zstds: for zstd in zstds:
print(zstd + ' : ' + repr(os.path.getsize(zstd)) + ', ' + sha1_of_file(zstd)) print(zstd + ' : ' + repr(os.path.getsize(zstd)) + ', ' + sha1_of_file(zstd))
if error_code != 0:
print('====== ERROR !!! =======')
sys.exit(error_code)