Files
wuffs/test/c/mimiclib/deflate-gzip-zlib.c
T
Nigel Tao be546c3587 Be able to test and bench the libdeflate library
name                                                               old speed      new speed       delta

wuffs_adler32_10k/clang9                                           11.4GB/s ± 2%   11.6GB/s ± 2%     +1.26%  (p=0.013 n=9+10)
wuffs_adler32_100k/clang9                                          13.7GB/s ± 1%   13.6GB/s ± 1%       ~     (p=0.277 n=8+9)

wuffs_adler32_10k/gcc10                                            12.2GB/s ± 0%   11.4GB/s ± 1%     -7.01%  (p=0.000 n=8+9)
wuffs_adler32_100k/gcc10                                           12.0GB/s ± 1%   11.7GB/s ± 1%     -2.80%  (p=0.000 n=9+9)

mimic_adler32_10k                                                  1.91GB/s ± 0%  25.24GB/s ± 1%  +1218.88%  (p=0.000 n=9+9)
mimic_adler32_100k                                                 1.87GB/s ± 0%  22.94GB/s ± 1%  +1126.66%  (p=0.000 n=10+10)

wuffs_crc32_ieee_10k/clang9                                        7.81GB/s ± 0%   7.70GB/s ± 0%     -1.41%  (p=0.000 n=8+8)
wuffs_crc32_ieee_100k/clang9                                       10.0GB/s ± 2%   10.6GB/s ± 1%     +5.25%  (p=0.000 n=7+8)

wuffs_crc32_ieee_10k/gcc10                                         9.80GB/s ± 1%  10.03GB/s ± 1%     +2.35%  (p=0.000 n=9+10)
wuffs_crc32_ieee_100k/gcc10                                        13.1GB/s ± 0%   12.9GB/s ± 0%     -1.52%  (p=0.000 n=10+9)

mimic_crc32_ieee_10k                                                842MB/s ± 1%  12952MB/s ± 0%  +1437.67%  (p=0.000 n=10+9)
mimic_crc32_ieee_100k                                               838MB/s ± 2%  13363MB/s ± 1%  +1493.79%  (p=0.000 n=10+9)

wuffs_deflate_decode_1k_full_init/clang9                            101MB/s ± 0%    106MB/s ± 1%     +4.22%  (p=0.000 n=8+10)
wuffs_deflate_decode_1k_part_init/clang9                            119MB/s ± 1%    121MB/s ± 1%     +1.39%  (p=0.000 n=10+10)
wuffs_deflate_decode_10k_full_init/clang9                           222MB/s ± 0%    215MB/s ± 0%     -3.24%  (p=0.000 n=9+9)
wuffs_deflate_decode_10k_part_init/clang9                           228MB/s ± 0%    218MB/s ± 3%     -4.47%  (p=0.000 n=10+9)
wuffs_deflate_decode_100k_just_one_read/clang9                      291MB/s ± 0%    286MB/s ± 3%     -1.85%  (p=0.000 n=10+10)
wuffs_deflate_decode_100k_many_big_reads/clang9                     189MB/s ± 1%    178MB/s ± 1%     -5.50%  (p=0.000 n=10+10)

wuffs_deflate_decode_1k_full_init/gcc10                            99.1MB/s ± 2%  102.1MB/s ± 1%     +3.12%  (p=0.000 n=10+9)
wuffs_deflate_decode_1k_part_init/gcc10                             117MB/s ± 1%    121MB/s ± 1%     +3.41%  (p=0.000 n=10+9)
wuffs_deflate_decode_10k_full_init/gcc10                            215MB/s ± 1%    218MB/s ± 1%     +1.55%  (p=0.000 n=10+10)
wuffs_deflate_decode_10k_part_init/gcc10                            220MB/s ± 1%    225MB/s ± 0%     +2.08%  (p=0.000 n=10+8)
wuffs_deflate_decode_100k_just_one_read/gcc10                       281MB/s ± 1%    279MB/s ± 1%       ~     (p=0.105 n=10+10)
wuffs_deflate_decode_100k_many_big_reads/gcc10                      182MB/s ± 1%    180MB/s ± 0%     -1.14%  (p=0.000 n=9+7)

mimic_deflate_decode_1k_full_init                                   138MB/s ± 1%    179MB/s ± 1%    +29.56%  (p=0.000 n=10+10)
mimic_deflate_decode_10k_full_init                                  155MB/s ± 2%    268MB/s ± 1%    +72.15%  (p=0.000 n=9+10)
mimic_deflate_decode_100k_just_one_read                             191MB/s ± 0%    269MB/s ± 1%    +40.38%  (p=0.000 n=10+10)
mimic_deflate_decode_100k_many_big_reads                            147MB/s ± 1%         skipped

wuffs_gzip_decode_10k/clang9                                        218MB/s ± 1%    211MB/s ± 1%     -3.18%  (p=0.000 n=8+9)
wuffs_gzip_decode_100k/clang9                                       282MB/s ± 0%    282MB/s ± 0%       ~     (p=0.113 n=9+9)

wuffs_gzip_decode_10k/gcc10                                         215MB/s ± 1%    231MB/s ± 1%     +7.34%  (p=0.000 n=10+10)
wuffs_gzip_decode_100k/gcc10                                        271MB/s ± 2%    293MB/s ± 0%     +7.84%  (p=0.000 n=10+9)

mimic_gzip_decode_10k                                               132MB/s ± 1%    263MB/s ± 0%    +99.23%  (p=0.000 n=10+9)
mimic_gzip_decode_100k                                              156MB/s ± 1%    262MB/s ± 3%    +67.93%  (p=0.000 n=9+10)

wuffs_zlib_decode_10k/clang9                                        223MB/s ± 0%    212MB/s ± 1%     -5.05%  (p=0.000 n=7+10)
wuffs_zlib_decode_100k/clang9                                       280MB/s ± 0%    280MB/s ± 0%       ~     (p=0.867 n=8+7)

wuffs_zlib_decode_10k/gcc10                                         232MB/s ± 2%    220MB/s ± 1%     -5.42%  (p=0.000 n=10+10)
wuffs_zlib_decode_100k/gcc10                                        286MB/s ± 5%    277MB/s ± 0%     -2.88%  (p=0.002 n=10+9)

mimic_zlib_decode_10k                                               145MB/s ± 0%    266MB/s ± 0%    +84.02%  (p=0.000 n=10+9)
mimic_zlib_decode_100k                                              173MB/s ± 1%    265MB/s ± 2%    +53.31%  (p=0.000 n=10+9)
2021-04-20 23:33:58 +10:00

470 lines
15 KiB
C

// Copyright 2017 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ----------------
// Uncomment one of these #define lines to test and bench alternative mimic
// libraries (libdeflate or miniz) instead of zlib-the-library.
//
// These are collectively referred to as
// WUFFS_MIMICLIB_USE_XXX_INSTEAD_OF_ZLIB.
//
// #define WUFFS_MIMICLIB_USE_LIBDEFLATE_INSTEAD_OF_ZLIB 1
// #define WUFFS_MIMICLIB_USE_MINIZ_INSTEAD_OF_ZLIB 1
// -------------------------------- WUFFS_MIMICLIB_USE_XXX_INSTEAD_OF_ZLIB
#if defined(WUFFS_MIMICLIB_USE_LIBDEFLATE_INSTEAD_OF_ZLIB)
#include "libdeflate.h"
#define WUFFS_MIMICLIB_DEFLATE_DOES_NOT_SUPPORT_STREAMING 1
#define WUFFS_MIMICLIB_ZLIB_DOES_NOT_SUPPORT_DICTIONARIES 1
uint32_t global_mimiclib_deflate_unused_u32;
typedef enum libdeflate_result (*libdeflate_decompress_func)(
struct libdeflate_decompressor* decompressor,
const void* in,
size_t in_nbytes,
void* out,
size_t out_nbytes_avail,
size_t* actual_in_nbytes_ret,
size_t* actual_out_nbytes_ret);
const char* //
libdeflate_result_as_const_char_star(enum libdeflate_result r) {
switch (r) {
case LIBDEFLATE_SUCCESS:
return NULL;
case LIBDEFLATE_BAD_DATA:
return "libdeflate: bad data";
case LIBDEFLATE_SHORT_OUTPUT:
return "libdeflate: short output";
case LIBDEFLATE_INSUFFICIENT_SPACE:
return "libdeflate: insufficient space";
}
return "libdeflate: unknown error";
}
const char* //
mimic_bench_adler32(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
uint32_t wuffs_initialize_flags,
uint64_t wlimit,
uint64_t rlimit) {
global_mimiclib_deflate_unused_u32 = 1;
while (src->meta.ri < src->meta.wi) {
uint8_t* ptr = src->data.ptr + src->meta.ri;
size_t len = src->meta.wi - src->meta.ri;
if (len > 0x7FFFFFFF) {
return "src length is too large";
} else if (len > rlimit) {
len = rlimit;
}
global_mimiclib_deflate_unused_u32 =
libdeflate_adler32(global_mimiclib_deflate_unused_u32, ptr, len);
src->meta.ri += len;
}
return NULL;
}
const char* //
mimic_bench_crc32_ieee(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
uint32_t wuffs_initialize_flags,
uint64_t wlimit,
uint64_t rlimit) {
global_mimiclib_deflate_unused_u32 = 0;
while (src->meta.ri < src->meta.wi) {
uint8_t* ptr = src->data.ptr + src->meta.ri;
size_t len = src->meta.wi - src->meta.ri;
if (len > 0x7FFFFFFF) {
return "src length is too large";
} else if (len > rlimit) {
len = rlimit;
}
global_mimiclib_deflate_unused_u32 =
libdeflate_crc32(global_mimiclib_deflate_unused_u32, ptr, len);
src->meta.ri += len;
}
return NULL;
}
const char* //
mimic_deflate_gzip_zlib_decode(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
uint32_t wuffs_initialize_flags,
uint64_t wlimit,
uint64_t rlimit,
libdeflate_decompress_func func) {
struct libdeflate_decompressor* dec = libdeflate_alloc_decompressor();
if (!dec) {
return "libdeflate: alloc failed";
}
size_t n_dst = 0;
size_t n_src = 0;
enum libdeflate_result res =
(*func)(dec, wuffs_base__io_buffer__reader_pointer(src),
((size_t)wuffs_base__u64__min(
rlimit, wuffs_base__io_buffer__reader_length(src))),
wuffs_base__io_buffer__writer_pointer(dst),
((size_t)wuffs_base__u64__min(
wlimit, wuffs_base__io_buffer__writer_length(dst))),
&n_src, &n_dst);
if (res == LIBDEFLATE_SUCCESS) {
dst->meta.wi += n_dst;
src->meta.ri += n_src;
}
libdeflate_free_decompressor(dec);
return libdeflate_result_as_const_char_star(res);
}
const char* //
mimic_deflate_decode(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
uint32_t wuffs_initialize_flags,
uint64_t wlimit,
uint64_t rlimit) {
return mimic_deflate_gzip_zlib_decode(dst, src, wuffs_initialize_flags,
wlimit, rlimit,
&libdeflate_deflate_decompress_ex);
}
const char* //
mimic_gzip_decode(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
uint32_t wuffs_initialize_flags,
uint64_t wlimit,
uint64_t rlimit) {
return mimic_deflate_gzip_zlib_decode(dst, src, wuffs_initialize_flags,
wlimit, rlimit,
&libdeflate_gzip_decompress_ex);
}
const char* //
mimic_zlib_decode(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
uint32_t wuffs_initialize_flags,
uint64_t wlimit,
uint64_t rlimit) {
return mimic_deflate_gzip_zlib_decode(dst, src, wuffs_initialize_flags,
wlimit, rlimit,
&libdeflate_zlib_decompress_ex);
}
const char* //
mimic_zlib_decode_with_dictionary(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
wuffs_base__slice_u8 dictionary) {
return "libdeflate does not implement zlib dictionaries";
}
// -------------------------------- WUFFS_MIMICLIB_USE_XXX_INSTEAD_OF_ZLIB
#elif defined(WUFFS_MIMICLIB_USE_MINIZ_INSTEAD_OF_ZLIB)
#include "/path/to/your/copy/of/github.com/richgel999/miniz/miniz_tinfl.c"
// We deliberately do not define the
// WUFFS_MIMICLIB_DEFLATE_DOES_NOT_SUPPORT_STREAMING macro.
#define WUFFS_MIMICLIB_ZLIB_DOES_NOT_SUPPORT_DICTIONARIES 1
const char* //
mimic_bench_adler32(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
uint32_t wuffs_initialize_flags,
uint64_t wlimit,
uint64_t rlimit) {
return "miniz does not independently compute Adler32";
}
const char* //
mimic_bench_crc32_ieee(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
uint32_t wuffs_initialize_flags,
uint64_t wlimit,
uint64_t rlimit) {
return "miniz does not implement CRC32/IEEE";
}
const char* //
mimic_deflate_zlib_decode(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
uint32_t wuffs_initialize_flags,
uint64_t wlimit,
uint64_t rlimit,
bool deflate_instead_of_zlib) {
if ((wlimit < UINT64_MAX) || (rlimit < UINT64_MAX)) {
// Supporting this would probably mean using tinfl_decompress instead of
// the simpler tinfl_decompress_mem_to_mem function.
return "unsupported I/O limit";
}
int flags = 0;
if (!deflate_instead_of_zlib) {
flags |= TINFL_FLAG_PARSE_ZLIB_HEADER;
}
size_t n = tinfl_decompress_mem_to_mem(
dst->data.ptr + dst->meta.wi, dst->data.len - dst->meta.wi,
src->data.ptr + src->meta.ri, src->meta.wi - src->meta.ri, flags);
if (n == TINFL_DECOMPRESS_MEM_TO_MEM_FAILED) {
return "TINFL_DECOMPRESS_MEM_TO_MEM_FAILED";
}
dst->meta.wi += n;
src->meta.ri = src->meta.wi;
return NULL;
}
const char* //
mimic_deflate_decode(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
uint32_t wuffs_initialize_flags,
uint64_t wlimit,
uint64_t rlimit) {
return mimic_deflate_zlib_decode(dst, src, wuffs_initialize_flags, wlimit,
rlimit, true);
}
const char* //
mimic_gzip_decode(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
uint32_t wuffs_initialize_flags,
uint64_t wlimit,
uint64_t rlimit) {
return "miniz does not implement gzip";
}
const char* //
mimic_zlib_decode(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
uint32_t wuffs_initialize_flags,
uint64_t wlimit,
uint64_t rlimit) {
return mimic_deflate_zlib_decode(dst, src, wuffs_initialize_flags, wlimit,
rlimit, false);
}
const char* //
mimic_zlib_decode_with_dictionary(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
wuffs_base__slice_u8 dictionary) {
return "miniz does not implement zlib dictionaries";
}
// -------------------------------- WUFFS_MIMICLIB_USE_XXX_INSTEAD_OF_ZLIB
#else
#include "zlib.h"
// We deliberately do not define the
// WUFFS_MIMICLIB_DEFLATE_DOES_NOT_SUPPORT_STREAMING macro.
// We deliberately do not define the
// WUFFS_MIMICLIB_ZLIB_DOES_NOT_SUPPORT_DICTIONARIES macro.
uint32_t global_mimiclib_deflate_unused_u32;
const char* //
mimic_bench_adler32(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
uint32_t wuffs_initialize_flags,
uint64_t wlimit,
uint64_t rlimit) {
global_mimiclib_deflate_unused_u32 = 0;
while (src->meta.ri < src->meta.wi) {
uint8_t* ptr = src->data.ptr + src->meta.ri;
size_t len = src->meta.wi - src->meta.ri;
if (len > 0x7FFFFFFF) {
return "src length is too large";
} else if (len > rlimit) {
len = rlimit;
}
global_mimiclib_deflate_unused_u32 =
adler32(global_mimiclib_deflate_unused_u32, ptr, len);
src->meta.ri += len;
}
return NULL;
}
const char* //
mimic_bench_crc32_ieee(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
uint32_t wuffs_initialize_flags,
uint64_t wlimit,
uint64_t rlimit) {
global_mimiclib_deflate_unused_u32 = 0;
while (src->meta.ri < src->meta.wi) {
uint8_t* ptr = src->data.ptr + src->meta.ri;
size_t len = src->meta.wi - src->meta.ri;
if (len > 0x7FFFFFFF) {
return "src length is too large";
} else if (len > rlimit) {
len = rlimit;
}
global_mimiclib_deflate_unused_u32 =
crc32(global_mimiclib_deflate_unused_u32, ptr, len);
src->meta.ri += len;
}
return NULL;
}
typedef enum {
zlib_flavor_raw,
zlib_flavor_gzip,
zlib_flavor_zlib,
} zlib_flavor;
const char* //
mimic_deflate_gzip_zlib_decode(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
wuffs_base__slice_u8* dictionary,
uint64_t wlimit,
uint64_t rlimit,
zlib_flavor flavor) {
const char* ret = NULL;
if (dst->data.len > UINT_MAX) {
ret = "dst length is too large";
goto cleanup0;
}
if (src->data.len > UINT_MAX) {
ret = "src length is too large";
goto cleanup0;
}
// See inflateInit2 in the zlib manual, or in zlib.h, for details about how
// the window_bits int also encodes the wire format wrapper.
int window_bits = 0;
switch (flavor) {
case zlib_flavor_raw:
window_bits = -15;
break;
case zlib_flavor_gzip:
window_bits = +15 | 16;
break;
case zlib_flavor_zlib:
window_bits = +15;
break;
default:
ret = "invalid zlib_flavor";
goto cleanup0;
}
z_stream z = {0};
int ii2_err = inflateInit2(&z, window_bits);
if (ii2_err != Z_OK) {
ret = "inflateInit2 failed";
goto cleanup0;
}
while (true) {
z.next_in = src->data.ptr + src->meta.ri;
z.avail_in = src->meta.wi - src->meta.ri;
if (z.avail_in > rlimit) {
z.avail_in = rlimit;
}
uInt initial_avail_in = z.avail_in;
z.next_out = dst->data.ptr + dst->meta.wi;
z.avail_out = dst->data.len - dst->meta.wi;
if (z.avail_out > wlimit) {
z.avail_out = wlimit;
}
uInt initial_avail_out = z.avail_out;
// TODO: s/Z_NO_FLUSH/Z_SYNC_FLUSH/ more closely matches Wuffs' behavior.
int i_err = inflate(&z, Z_NO_FLUSH);
if (initial_avail_in < z.avail_in) {
ret = "inconsistent avail_in";
goto cleanup1;
}
src->meta.ri += initial_avail_in - z.avail_in;
if (initial_avail_out < z.avail_out) {
ret = "inconsistent avail_out";
goto cleanup1;
}
dst->meta.wi += initial_avail_out - z.avail_out;
if (i_err == Z_STREAM_END) {
break;
} else if (i_err == Z_NEED_DICT) {
if (dictionary) {
int isd_err =
inflateSetDictionary(&z, dictionary->ptr, dictionary->len);
if (isd_err != Z_OK) {
ret = "inflateSetDictionary failed";
goto cleanup1;
}
dictionary = NULL;
continue;
}
ret = "inflate failed (need dict)";
goto cleanup1;
} else if (i_err == Z_DATA_ERROR) {
ret = "inflate failed (data error)";
goto cleanup1;
} else if (i_err != Z_OK) {
ret = "inflate failed";
goto cleanup1;
}
}
cleanup1:;
int ie_err = inflateEnd(&z);
if ((ie_err != Z_OK) && !ret) {
ret = "inflateEnd failed";
}
cleanup0:;
return ret;
}
const char* //
mimic_deflate_decode(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
uint32_t wuffs_initialize_flags,
uint64_t wlimit,
uint64_t rlimit) {
return mimic_deflate_gzip_zlib_decode(dst, src, NULL, wlimit, rlimit,
zlib_flavor_raw);
}
const char* //
mimic_gzip_decode(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
uint32_t wuffs_initialize_flags,
uint64_t wlimit,
uint64_t rlimit) {
return mimic_deflate_gzip_zlib_decode(dst, src, NULL, wlimit, rlimit,
zlib_flavor_gzip);
}
const char* //
mimic_zlib_decode(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
uint32_t wuffs_initialize_flags,
uint64_t wlimit,
uint64_t rlimit) {
return mimic_deflate_gzip_zlib_decode(dst, src, NULL, wlimit, rlimit,
zlib_flavor_zlib);
}
const char* //
mimic_zlib_decode_with_dictionary(wuffs_base__io_buffer* dst,
wuffs_base__io_buffer* src,
wuffs_base__slice_u8 dictionary) {
return mimic_deflate_gzip_zlib_decode(dst, src, &dictionary, UINT64_MAX,
UINT64_MAX, zlib_flavor_zlib);
}
#endif
// -------------------------------- WUFFS_MIMICLIB_USE_XXX_INSTEAD_OF_ZLIB