From 9700f925837d819896ee02bd7362b718e6f13916 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Mon, 30 Jan 2017 11:42:45 -0800 Subject: [PATCH 001/223] Add educational decoder to /contrib --- contrib/educational_decoder/README.md | 18 + contrib/educational_decoder/harness.c | 93 + contrib/educational_decoder/zstd_decompress.c | 2096 +++++++++++++++++ contrib/educational_decoder/zstd_decompress.h | 6 + 4 files changed, 2213 insertions(+) create mode 100644 contrib/educational_decoder/README.md create mode 100644 contrib/educational_decoder/harness.c create mode 100644 contrib/educational_decoder/zstd_decompress.c create mode 100644 contrib/educational_decoder/zstd_decompress.h diff --git a/contrib/educational_decoder/README.md b/contrib/educational_decoder/README.md new file mode 100644 index 000000000..a1f703f62 --- /dev/null +++ b/contrib/educational_decoder/README.md @@ -0,0 +1,18 @@ +Educational Decoder +=================== + +`zstd_decompress.c` is a self-contained implementation of a decoder according +to the Zstandard format specification written in C99. +While it does not implement as many features as the reference decoder, +such as the streaming API or content checksums, it is written to be easy to +follow and understand, to help understand how the Zstandard format works. +It's laid out to match the [format specification], +so it can be used to understand how confusing segments could be implemented. +It also contains implementations of Huffman and FSE table decoding. + +[format specification]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md + +`harness.c` provides a simple test harness around the decoder: + + harness [dictionary] + diff --git a/contrib/educational_decoder/harness.c b/contrib/educational_decoder/harness.c new file mode 100644 index 000000000..6f4765d9d --- /dev/null +++ b/contrib/educational_decoder/harness.c @@ -0,0 +1,93 @@ +#include +#include + +#include "zstd_decompress.h" + +typedef unsigned char u8; + +// There's no good way to determine output size without decompressing +// For this example assume we'll never decompress at a ratio larger than 16 +#define MAX_COMPRESSION_RATIO (16) + +u8 *input; +u8 *output; +u8 *dict; + +size_t read_file(const char *path, u8 **ptr) { + FILE *f = fopen(path, "rb"); + if (!f) { + fprintf(stderr, "failed to open file %s\n", path); + exit(1); + } + + fseek(f, 0L, SEEK_END); + size_t size = ftell(f); + rewind(f); + + *ptr = malloc(size); + if (!ptr) { + fprintf(stderr, "failed to allocate memory to hold %s\n", path); + exit(1); + } + + size_t pos = 0; + while (!feof(f)) { + size_t read = fread(&(*ptr)[pos], 1, size, f); + if (ferror(f)) { + fprintf(stderr, "error while reading file %s\n", path); + exit(1); + } + pos += read; + } + + fclose(f); + + return pos; +} + +void write_file(const char *path, const u8 *ptr, size_t size) { + FILE *f = fopen(path, "wb"); + + size_t written = 0; + while (written < size) { + written += fwrite(&ptr[written], 1, size, f); + if (ferror(f)) { + fprintf(stderr, "error while writing file %s\n", path); + exit(1); + } + } + + fclose(f); +} + +int main(int argc, char **argv) { + if (argc < 3) { + fprintf(stderr, "usage: %s [dictionary]\n", argv[0]); + + return 1; + } + + size_t input_size = read_file(argv[1], &input); + size_t dict_size = 0; + if (argc >= 4) { + dict_size = read_file(argv[3], &dict); + } + + output = malloc(MAX_COMPRESSION_RATIO * input_size); + if (!output) { + fprintf(stderr, "failed to allocate memory\n"); + return 1; + } + + size_t decompressed = + ZSTD_decompress_with_dict(output, input_size * MAX_COMPRESSION_RATIO, + input, input_size, dict, dict_size); + + write_file(argv[2], output, decompressed); + + free(input); + free(output); + free(dict); + input = output = dict = NULL; +} + diff --git a/contrib/educational_decoder/zstd_decompress.c b/contrib/educational_decoder/zstd_decompress.c new file mode 100644 index 000000000..8dc159008 --- /dev/null +++ b/contrib/educational_decoder/zstd_decompress.c @@ -0,0 +1,2096 @@ +/// Zstandard educational decoder implementation +/// See https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md + +#include +#include +#include +#include + +/// Zstandard decompression functions. +/// `dst` must point to a space at least as large as the reconstructed output. +size_t ZSTD_decompress(void *dst, size_t dst_len, const void *src, + size_t src_len); +/// If `dict != NULL` and `dict_len >= 8`, does the same thing as +/// `ZSTD_decompress` but uses the provided dict +size_t ZSTD_decompress_with_dict(void *dst, size_t dst_len, const void *src, + size_t src_len, const void *dict, + size_t dict_len); + +/******* UTILITY MACROS AND TYPES *********************************************/ +#define MAX_WINDOW_SIZE ((size_t)512 << 20) +// Max block size decompressed size is 128 KB and literal blocks must be smaller +// than that +#define MAX_LITERALS_SIZE ((size_t)(1024 * 128)) + +#define MAX(a, b) ((a) > (b) ? (a) : (b)) +#define MIN(a, b) ((a) < (b) ? (a) : (b)) + +#define ERROR(s) \ + do { \ + fprintf(stderr, "Error: %s\n", s); \ + exit(1); \ + } while (0) +#define INP_SIZE() \ + ERROR("Input buffer smaller than it should be or input is " \ + "corrupted") +#define OUT_SIZE() ERROR("Output buffer too small for output") +#define CORRUPTION() ERROR("Corruption detected while decompressing") +#define BAD_ALLOC() ERROR("Memory allocation error") + +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; + +typedef int8_t i8; +typedef int16_t i16; +typedef int32_t i32; +typedef int64_t i64; +/******* END UTILITY MACROS AND TYPES *****************************************/ + +/******* IMPLEMENTATION PRIMITIVE PROTOTYPES **********************************/ +/// The implementations for these functions can be found at the bottom of this +/// file. They implement low-level functionality needed for the higher level +/// decompression functions. + +/*** CIRCULAR BUFFER ******************/ +/// A standard circular buffer, used to facilitate back reference commands +typedef struct { + u8 *ptr; + size_t idx, last_flush, size; +} cbuf_t; + +/// Initialize a circular buffer +static void cbuf_init(cbuf_t *buf, size_t size); +static void cbuf_free(cbuf_t *buf); + +/// Copies up to `src_len` bytes from `src` into the buffer, stopping if it +/// would need to flush. +/// Returns the total amount of data copied. +static size_t cbuf_write_data(cbuf_t *buf, const u8 *src, size_t src_len); +/// Copies `len` bytes from `offset` back in the buffer, stopping if it would +/// need to flush. +/// Returns the number of bytes copied. +static size_t cbuf_copy_offset(cbuf_t *buf, size_t offset, size_t len); +/// Writes up to `len` copies of `byte`, stopping if would need to flush. +/// Returns the number of bytes copied. +static size_t cbuf_repeat_byte(cbuf_t *buf, u8 byte, size_t len); + +/// The `full` versions of the above functions write the full amount requested, +/// flushing to `out` when necessary. +/// They return the number of bytes flushed to `out`, if any. +static size_t cbuf_write_data_full(cbuf_t *buf, const u8 *src, size_t src_len, + u8 *out, size_t out_len); +static size_t cbuf_copy_offset_full(cbuf_t *buf, size_t offset, size_t len, + u8 *out, size_t out_len); +static size_t cbuf_repeat_byte_full(cbuf_t *buf, u8 byte, size_t len, u8 *out, + size_t out_len); + +/// Flushes any unflushed data to `dst` +static size_t cbuf_flush(cbuf_t *buf, u8 *dst, size_t dst_len); +/*** END CIRCULAR BUFFER **************/ + +/*** BITSTREAM OPERATIONS *************/ +/// Read `num` bits (up to 64) from `src + offset`, where `offset` is in bits +static inline u64 read_bits_LE(const u8 *src, int num, size_t offset); + +/// Read bits from the end of a HUF or FSE bitstream. `offset` is in bits, so +/// it updates `offset` to `offset - bits`, and then reads `bits` bits from +/// `src + offset`. If the offset becomes negative, the extra bits at the +/// bottom are filled in with `0` bits instead of reading from before `src`. +static inline u64 STREAM_read_bits(const u8 *src, int bits, i64 *offset); +/*** END BITSTREAM OPERATIONS *********/ + +/*** BIT COUNTING OPERATIONS **********/ +/// Returns `x`, where `2^x` is the smallest power of 2 greater than or equal to +/// `num`, or `-1` if `num > 2^63` +static inline int log2sup(u64 num); + +/// Returns `x`, where `2^x` is the largest power of 2 less than or equal to +/// `num`, or `-1` if `num == 0`. +static inline int log2inf(u64 num); +/*** END BIT COUNTING OPERATIONS ******/ + +/*** HUFFMAN PRIMITIVES ***************/ +// Table decode method uses exponential memory, so we need to limit depth +#define HUF_MAX_BITS (16) + +// Limit the maximum number of symbols to 256 so we can store a symbol in a byte +#define HUF_MAX_SYMBS (256) + +/// Structure containing all tables necessary for efficient Huffman decoding +typedef struct { + u8 *symbols; + u8 *num_bits; + int max_bits; +} HUF_dtable; + +/// Decode a single symbol and read in enough bits to refresh the state +static inline u8 HUF_decode_symbol(HUF_dtable *dtable, u16 *state, + const u8 *src, i64 *offset); +/// Read in a full state's worth of bits to initialize it +static inline void HUF_init_state(HUF_dtable *dtable, u16 *state, const u8 *src, + i64 *offset); + +/// Initialize a Huffman decoding table using the table of bit counts provided +static void HUF_init_dtable(HUF_dtable *table, u8 *bits, int num_symbs); +/// Initialize a Huffman decoding table using the table of weights provided +/// Weights follow the definition provided in the Zstandard specification +static void HUF_init_dtable_usingweights(HUF_dtable *table, u8 *weights, + int num_symbs); + +/// Decompresses a single Huffman stream, returns the number of bytes decoded. +/// `src_len` must be the exact length of the Huffman-coded block. +static size_t HUF_decompress_1stream(HUF_dtable *table, u8 *dst, size_t dst_len, + const u8 *src, size_t src_len); +/// Same as previous but decodes 4 streams, formatted as in the Zstandard +/// specification. +/// `src_len` must be the exact length of the Huffman-coded block. +static size_t HUF_decompress_4stream(HUF_dtable *dtable, u8 *dst, + size_t dst_len, const u8 *src, + size_t src_len); + +/// Free the malloc'ed parts of a decoding table +static void HUF_free_dtable(HUF_dtable *dtable); + +/// Deep copy a decoding table, so that it can be used and free'd without +/// impacting the source table. +static void HUF_copy_dtable(HUF_dtable *dst, const HUF_dtable *src); +/*** END HUFFMAN PRIMITIVES ***********/ + +/*** FSE PRIMITIVES *******************/ +/// For more description of FSE see +/// https://github.com/Cyan4973/FiniteStateEntropy/ + +// FSE table decoding uses exponential memory, so limit the maximum accuracy +#define FSE_MAX_ACCURACY_LOG (15) +// Limit the maximum number of symbols so they can be stored in a single byte +#define FSE_MAX_SYMBS (256) + +/// The tables needed to decode FSE encoded streams +typedef struct { + u8 *symbols; + u8 *num_bits; + u16 *new_state_base; + int accuracy_log; +} FSE_dtable; + +/// Return the symbol for the current state +static inline u8 FSE_peek_symbol(FSE_dtable *dtable, u16 state); +/// Read the number of bits necessary to update state, update, and shift offset +/// back to reflect the bits read +static inline void FSE_update_state(FSE_dtable *dtable, u16 *state, + const u8 *src, i64 *offset); + +/// Combine peek and update: decode a symbol and update the state +static inline u8 FSE_decode_symbol(FSE_dtable *dtable, u16 *state, + const u8 *src, i64 *offset); + +/// Read bits from the stream to initialize the state and shift offset back +static inline void FSE_init_state(FSE_dtable *dtable, u16 *state, const u8 *src, + i64 *offset); + +/// Decompress two interleaved bitstreams (e.g. compressed Huffman weights) +/// using an FSE decoding table. `src_len` must be the exact length of the +/// block. +static size_t FSE_decompress_interleaved2(FSE_dtable *dtable, u8 *dst, + size_t dst_len, const u8 *src, + size_t src_len); + +/// Initialize a decoding table using normalized frequencies. +static void FSE_init_dtable(FSE_dtable *dtable, const i16 *norm_freqs, + int num_symbs, int accuracy_log); + +/// Decode an FSE header as defined in the Zstandard format specification and +/// use the decoded frequencies to initialize a decoding table. +static size_t FSE_decode_header(FSE_dtable *dtable, const u8 *src, + size_t src_len, int max_accuracy_log); + +/// Initialize an FSE table that will always return the same symbol and consume +/// 0 bits per symbol, to be used for RLE mode in sequence commands +static void FSE_init_dtable_rle(FSE_dtable *dtable, u8 symb); + +/// Free the malloc'ed parts of a decoding table +static void FSE_free_dtable(FSE_dtable *dtable); + +/// Deep copy a decoding table, so that it can be used and free'd without +/// impacting the source table. +static void FSE_copy_dtable(FSE_dtable *dst, const FSE_dtable *src); +/*** END FSE PRIMITIVES ***************/ + +/******* END IMPLEMENTATION PRIMITIVE PROTOTYPES ******************************/ + +/******* ZSTD HELPER STRUCTS AND PROTOTYPES ***********************************/ + +/// Input and output pointers to allow them to be advanced by +/// functions that consume input/produce output +typedef struct { + u8 *dst; + size_t dst_len; + + const u8 *src; + size_t src_len; +} io_streams_t; + +/// The context needed to decode blocks in a frame +typedef struct { + size_t window_size; + size_t frame_content_size; + + // The total amount of data available for backreferences, to determine if an + // offset too large to be correct + size_t current_total_output; + + // A sliding window of the past `window_size` bytes decoded + cbuf_t window; + + // Entropy encoding tables so they can be repeated by future blocks instead + // of + // retransmitting + HUF_dtable literals_dtable; + FSE_dtable ll_dtable; + FSE_dtable ml_dtable; + FSE_dtable of_dtable; + + // The last 3 offsets for the special "repeat offsets". Array size is 4 so + // that previous_offsets[1] corresponds to the most recent offset + u64 previous_offsets[4]; + + // The dictionary id for this frame if one exists + u32 dictionary_id; + + int single_segment_flag; + int content_checksum_flag; +} frame_context_t; + +/// The decoded contents of a dictionary so that it doesn't have to be repeated +/// for each frame that uses it +typedef struct { + // Entropy tables + HUF_dtable literals_dtable; + FSE_dtable ll_dtable; + FSE_dtable ml_dtable; + FSE_dtable of_dtable; + + // Raw content for backreferences + u8 *content; + size_t content_size; + + // Offset history to prepopulate the frame's history + u64 previous_offsets[4]; + + u32 dictionary_id; +} dictionary_t; + +/// A tuple containing the parts necessary to decode and execute a ZSTD sequence +/// command +typedef struct { + u32 literal_length; + u32 match_length; + u32 offset; +} sequence_command_t; + +/// The decoder works top-down, starting at the high level like Zstd frames, and +/// working down to lower more technical levels such as blocks, literals, and +/// sequences. The high-level functions roughly follow the outline of the +/// format specification: +/// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md + +/// Before the implementation of each high-level function declared here, the +/// prototypes for their helper functions are defined and explained + +/// Decode a single Zstd frame, or error if the input is not a valid frame. +/// Accepts a dict argument, which may be NULL indicating no dictionary. +/// See +/// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#frame-concatenation +static void decode_frame(io_streams_t *streams, dictionary_t *dict); + +// Decode data in a compressed block +static void decompress_block(io_streams_t *streams, frame_context_t *ctx, + size_t block_len); + +// Decode the literals section of a block +static size_t decode_literals(io_streams_t *streams, frame_context_t *ctx, + u8 **literals); + +// Decode the sequences part of a block +static size_t decode_sequences(frame_context_t *ctx, const u8 *src, + size_t src_len, sequence_command_t **sequences); + +// Execute the decoded sequences on the literals block +static size_t execute_sequences(io_streams_t *streams, frame_context_t *ctx, + sequence_command_t *sequences, + size_t num_sequences, const u8 *literals, + size_t literals_len); + +// Parse a provided dictionary blob for use in decompression +static void parse_dictionary(dictionary_t *dict, const u8 *src, size_t src_len); +static void free_dictionary(dictionary_t *dict); +/******* END ZSTD HELPER STRUCTS AND PROTOTYPES *******************************/ + +size_t ZSTD_decompress(void *dst, size_t dst_len, const void *src, + size_t src_len) { + return ZSTD_decompress_with_dict(dst, dst_len, src, src_len, NULL, 0); +} + +size_t ZSTD_decompress_usingDict(void *_ctx, void *dst, size_t dst_len, + const void *src, size_t src_len, + const void *dict, size_t dict_len) { + // _ctx needed to match ZSTD lib signature + return ZSTD_decompress_with_dict(dst, dst_len, src, src_len, dict, + dict_len); +} + +size_t ZSTD_decompress_with_dict(void *dst, size_t dst_len, const void *src, + size_t src_len, const void *dict, + size_t dict_len) { + dictionary_t parsed_dict; + memset(&parsed_dict, 0, sizeof(dictionary_t)); + // dict_len < 8 is not a valid dictionary + if (dict && dict_len > 8) { + parse_dictionary(&parsed_dict, (const u8 *)dict, dict_len); + } + + io_streams_t streams = {(u8 *)dst, dst_len, (const u8 *)src, src_len}; + while (streams.src_len > 0) { + decode_frame(&streams, &parsed_dict); + } + + free_dictionary(&parsed_dict); + + return streams.dst - (u8 *)dst; +} + +/******* FRAME DECODING ******************************************************/ + +static void decode_data_frame(io_streams_t *streams, dictionary_t *dict); +static void init_frame_context(frame_context_t *context); +static void free_frame_context(frame_context_t *context); +static void parse_frame_header(io_streams_t *streams, frame_context_t *ctx, + dictionary_t *dict); +static void frame_context_apply_dict(frame_context_t *ctx, dictionary_t *dict); + +static void decompress_data(io_streams_t *streams, frame_context_t *ctx); + +static void decode_frame(io_streams_t *streams, dictionary_t *dict) { + if (streams->src_len < 4) { + INP_SIZE(); + } + u32 magic_number = read_bits_LE(streams->src, 32, 0); + + streams->src += 4; + streams->src_len -= 4; + if (magic_number >= 0x184D2A50U && magic_number <= 0x184D2A5F) { + // skippable frame + if (streams->src_len < 4) { + INP_SIZE(); + } + size_t frame_size = read_bits_LE(streams->src, 32, 32); + + if (streams->src_len < 4 + frame_size) { + INP_SIZE(); + } + + // skip over frame + streams->src += 4 + frame_size; + streams->src_len -= 4 + frame_size; + } else if (magic_number == 0xFD2FB528U) { + // ZSTD frame + decode_data_frame(streams, dict); + } else { + // not a real frame + ERROR("Invalid magic number"); + } +} + +/// Decode a frame that contains compressed data. Not all frames do as there +/// are skippable frames. +/// See +/// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#general-structure-of-zstandard-frame-format +static void decode_data_frame(io_streams_t *streams, dictionary_t *dict) { + frame_context_t ctx; + + // Initialize the context that needs to be carried from block to block + init_frame_context(&ctx); + parse_frame_header(streams, &ctx, dict); + frame_context_apply_dict(&ctx, dict); + + if (ctx.frame_content_size != 0 && + ctx.frame_content_size > streams->dst_len) { + OUT_SIZE(); + } + + decompress_data(streams, &ctx); + + free_frame_context(&ctx); +} + +static void init_frame_context(frame_context_t *context) { + memset(context, 0x00, sizeof(frame_context_t)); + + // Set up the offset history for the repeat offset commands + context->previous_offsets[1] = 1; + context->previous_offsets[2] = 4; + context->previous_offsets[3] = 8; +} + +static void free_frame_context(frame_context_t *context) { + HUF_free_dtable(&context->literals_dtable); + + FSE_free_dtable(&context->ll_dtable); + FSE_free_dtable(&context->ml_dtable); + FSE_free_dtable(&context->of_dtable); + + cbuf_free(&context->window); + + memset(context, 0, sizeof(frame_context_t)); +} + +static void parse_frame_header(io_streams_t *streams, frame_context_t *ctx, + dictionary_t *dict) { + if (streams->src_len < 1) { + INP_SIZE(); + } + + u8 descriptor = read_bits_LE(streams->src, 8, 0); + + // decode frame header descriptor into flags + u8 frame_content_size_flag = descriptor >> 6; + u8 single_segment_flag = (descriptor >> 5) & 1; + u8 reserved_bit = (descriptor >> 3) & 1; + u8 content_checksum_flag = (descriptor >> 2) & 1; + u8 dictionary_id_flag = descriptor & 3; + + if (reserved_bit != 0) { + CORRUPTION(); + } + + streams->src++; + streams->src_len--; + + ctx->single_segment_flag = single_segment_flag; + ctx->content_checksum_flag = content_checksum_flag; + + // decode window size + if (!single_segment_flag) { + if (streams->src_len < 1) { + INP_SIZE(); + } + + // Use the algorithm from the specification to compute window size + // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#window_descriptor + u8 window_descriptor = read_bits_LE(streams->src, 8, 0); + u8 exponent = window_descriptor >> 3; + u8 mantissa = window_descriptor & 7; + + size_t window_base = (size_t)1 << (10 + exponent); + size_t window_add = (window_base / 8) * mantissa; + ctx->window_size = window_base + window_add; + + streams->src++; + streams->src_len--; + } + + // decode dictionary id if it exists + if (dictionary_id_flag) { + const int bytes_array[] = {0, 1, 2, 4}; + const int bytes = bytes_array[dictionary_id_flag]; + + if (streams->src_len < bytes) { + INP_SIZE(); + } + + ctx->dictionary_id = read_bits_LE(streams->src, bytes * 8, 0); + streams->src += bytes; + streams->src_len -= bytes; + } else { + ctx->dictionary_id = 0; + } + + // decode frame content size if it exists + if (single_segment_flag || frame_content_size_flag) { + // if frame_content_size_flag == 0 but single_segment_flag is set, we + // still + // have a 1 byte field + const int bytes_array[] = {1, 2, 4, 8}; + const int bytes = bytes_array[frame_content_size_flag]; + + if (streams->src_len < bytes) { + INP_SIZE(); + } + + ctx->frame_content_size = read_bits_LE(streams->src, bytes * 8, 0); + if (bytes == 2) { + ctx->frame_content_size += 256; + } + + streams->src += bytes; + streams->src_len -= bytes; + } + + if (single_segment_flag) { + ctx->window_size = + ctx->frame_content_size + (dict ? dict->content_size : 0); + // We need to allocate a buffer to write to of size at least output + + // dict + // size + size_t size = ctx->frame_content_size + (dict ? dict->content_size : 0); + } + + // Allocate the window + if (ctx->window_size > MAX_WINDOW_SIZE) { + ERROR("Requested window size too large"); + } + cbuf_init(&ctx->window, ctx->window_size); +} + +/// A dictionary acts as initializing values for the frame context before +/// decompression, so we implement it by applying it's predetermined +/// tables and content to the context before beginning decompression +static void frame_context_apply_dict(frame_context_t *ctx, dictionary_t *dict) { + // If the content pointer is NULL then it must be an empty dict + if (!dict || !dict->content) + return; + + if (ctx->dictionary_id == 0 && dict->dictionary_id != 0) { + // The dictionary is unneeded, and shouldn't be used as it may interfere + // with the default offset history + return; + } + + // If the dictionary id is 0, it doesn't matter if we provide the wrong raw + // content dict, it won't change anything + if (ctx->dictionary_id != 0 && ctx->dictionary_id != dict->dictionary_id) { + ERROR("Wrong/no dictionary provided"); + } + + // Write the dict data in, and then flush to NULL so it's not sent to the + // output stream + cbuf_write_data_full(&ctx->window, dict->content, dict->content_size, NULL, + -1); + cbuf_flush(&ctx->window, NULL, -1); + ctx->current_total_output = dict->content_size; + + // If it's a formatted dict copy the precomputed tables in so they can + // be used in the table repeat modes + if (dict->dictionary_id != 0) { + // Deep copy the entropy tables so they can be freed independently of + // the + // dictionary struct + HUF_copy_dtable(&ctx->literals_dtable, &dict->literals_dtable); + FSE_copy_dtable(&ctx->ll_dtable, &dict->ll_dtable); + FSE_copy_dtable(&ctx->of_dtable, &dict->of_dtable); + FSE_copy_dtable(&ctx->ml_dtable, &dict->ml_dtable); + + memcpy(ctx->previous_offsets, dict->previous_offsets, + sizeof(ctx->previous_offsets)); + } +} + +/// Decompress the data from a frame block by block +static void decompress_data(io_streams_t *streams, frame_context_t *ctx) { + + u8 last_block = 0; + do { + if (streams->src_len < 3) { + INP_SIZE(); + } + // Parse the block header + last_block = streams->src[0] & 1; + u8 block_type = (streams->src[0] >> 1) & 3; + size_t block_len = read_bits_LE(streams->src, 21, 3); + + streams->src += 3; + streams->src_len -= 3; + + switch (block_type) { + case 0: { + // Raw, uncompressed block + if (streams->src_len < block_len) { + INP_SIZE(); + } + if (streams->dst_len < block_len) { + OUT_SIZE(); + } + + // Write the raw data into the window buffer + size_t written = + cbuf_write_data_full(&ctx->window, streams->src, block_len, + streams->dst, streams->dst_len); + streams->src += block_len; + streams->src_len -= block_len; + + streams->dst += written; + streams->dst_len -= written; + break; + } + case 1: { + // RLE block, repeat the first byte N times + if (streams->src_len < 1) { + INP_SIZE(); + } + if (streams->dst_len < block_len) { + OUT_SIZE(); + } + + // Write streams->src[0] into the buffer block_len times + size_t written = + cbuf_repeat_byte_full(&ctx->window, streams->src[0], block_len, + streams->dst, streams->dst_len); + streams->dst += written; + streams->dst_len -= written; + + streams->src += 1; + streams->src_len -= 1; + break; + } + case 2: + // Compressed block, this is mode complex + decompress_block(streams, ctx, block_len); + break; + } + } while (!last_block); + + // Flush out anything left in the window buffer to the destination stream + size_t written = cbuf_flush(&ctx->window, streams->dst, streams->dst_len); + streams->dst += written; + streams->dst_len -= written; + + if (ctx->content_checksum_flag) { + // This program does not support checking the checksum, so skip over it + // if + // it's present + if (streams->src_len < 4) { + INP_SIZE(); + } + streams->src += 4; + streams->src_len -= 4; + } +} +/******* END FRAME DECODING ***************************************************/ + +/******* BLOCK DECOMPRESSION **************************************************/ +static void decompress_block(io_streams_t *streams, frame_context_t *ctx, + size_t block_len) { + if (streams->src_len < block_len) { + INP_SIZE(); + } + // We need this to determine how long the compressed literals block was + const u8 *const end_of_block = streams->src + block_len; + + // Part 1: decode the literals block + u8 *literals = NULL; + size_t literals_size = decode_literals(streams, ctx, &literals); + + // Part 2: decode the sequences block + if (streams->src > end_of_block) { + INP_SIZE(); + } + size_t sequences_size = end_of_block - streams->src; + sequence_command_t *sequences = NULL; + size_t num_sequences = + decode_sequences(ctx, streams->src, sequences_size, &sequences); + + streams->src += sequences_size; + streams->src_len -= sequences_size; + + // Part 3: combine literals and sequence commands to generate output + execute_sequences(streams, ctx, sequences, num_sequences, literals, + literals_size); + free(literals); + free(sequences); +} +/******* END BLOCK DECOMPRESSION **********************************************/ + +/******* LITERALS DECODING ****************************************************/ +static size_t decode_literals_simple(io_streams_t *streams, u8 **literals, + int block_type, int size_format); +static size_t decode_literals_compressed(io_streams_t *streams, + frame_context_t *ctx, u8 **literals, + int block_type, int size_format); +static size_t decode_huf_table(const u8 *src, size_t src_len, + HUF_dtable *dtable); +static size_t fse_decode_hufweights(const u8 *src, size_t src_len, u8 *weights, + int *num_symbs, size_t compressed_size); + +static size_t decode_literals(io_streams_t *streams, frame_context_t *ctx, + u8 **literals) { + if (streams->src_len < 1) { + INP_SIZE(); + } + // Decode literals header + int block_type = streams->src[0] & 3; + int size_format = (streams->src[0] >> 2) & 3; + + if (block_type <= 1) { + // Raw or RLE literals block + return decode_literals_simple(streams, literals, block_type, + size_format); + } else { + // Huffman compressed literals + return decode_literals_compressed(streams, ctx, literals, block_type, + size_format); + } +} + +/// Decodes literals blocks in raw or RLE form +static size_t decode_literals_simple(io_streams_t *streams, u8 **literals, + int block_type, int size_format) { + size_t size; + switch (size_format) { + // These cases are in the form X0 + // In this case, the X bit is actually part of the size field + case 0: + case 2: + size = read_bits_LE(streams->src, 5, 3); + streams->src += 1; + streams->src_len -= 1; + break; + case 1: + if (streams->src_len < 2) { + INP_SIZE(); + } + size = read_bits_LE(streams->src, 12, 4); + streams->src += 2; + streams->src_len -= 2; + break; + case 3: + if (streams->src_len < 2) { + INP_SIZE(); + } + size = read_bits_LE(streams->src, 20, 4); + streams->src += 3; + streams->src_len -= 3; + break; + default: + // Impossible + size = -1; + } + + if (size > MAX_LITERALS_SIZE) { + CORRUPTION(); + } + + *literals = malloc(size); + if (!*literals) { + BAD_ALLOC(); + } + + switch (block_type) { + case 0: + // Raw data + if (size > streams->src_len) { + INP_SIZE(); + } + memcpy(*literals, streams->src, size); + streams->src += size; + streams->src_len -= size; + break; + case 1: + // Single repeated byte + if (1 > streams->src_len) { + INP_SIZE(); + } + memset(*literals, streams->src[0], size); + streams->src += 1; + streams->src_len -= 1; + break; + } + + return size; +} + +/// Decodes Huffman compressed literals +static size_t decode_literals_compressed(io_streams_t *streams, + frame_context_t *ctx, u8 **literals, + int block_type, int size_format) { + size_t regenerated_size, compressed_size; + // Only size_format=0 has 1 stream, so default to 4 + int num_streams = 4; + switch (size_format) { + case 0: + num_streams = 1; + // Fall through as it has the same size format + case 1: + if (streams->src_len < 3) { + INP_SIZE(); + } + regenerated_size = read_bits_LE(streams->src, 10, 4); + compressed_size = read_bits_LE(streams->src, 10, 14); + streams->src += 3; + streams->src_len -= 3; + break; + case 2: + if (streams->src_len < 4) { + INP_SIZE(); + } + regenerated_size = read_bits_LE(streams->src, 14, 4); + compressed_size = read_bits_LE(streams->src, 14, 18); + streams->src += 4; + streams->src_len -= 4; + break; + case 3: + if (streams->src_len < 5) { + INP_SIZE(); + } + regenerated_size = read_bits_LE(streams->src, 18, 4); + compressed_size = read_bits_LE(streams->src, 18, 22); + streams->src += 5; + streams->src_len -= 5; + break; + default: + // Impossible + compressed_size = regenerated_size = -1; + } + if (regenerated_size > MAX_LITERALS_SIZE || + compressed_size > regenerated_size) { + CORRUPTION(); + } + + if (compressed_size > streams->src_len) { + INP_SIZE(); + } + + *literals = malloc(regenerated_size); + if (!*literals) { + BAD_ALLOC(); + } + + if (block_type == 2) { + // Decode provided Huffman table + + HUF_free_dtable(&ctx->literals_dtable); + size_t size = decode_huf_table(streams->src, compressed_size, + &ctx->literals_dtable); + streams->src += size; + streams->src_len -= size; + compressed_size -= size; + } else { + // If we're to repeat the previous Huffman table, make sure it exists + if (!ctx->literals_dtable.symbols) { + CORRUPTION(); + } + } + + if (num_streams == 1) { + HUF_decompress_1stream(&ctx->literals_dtable, *literals, + regenerated_size, streams->src, compressed_size); + } else { + HUF_decompress_4stream(&ctx->literals_dtable, *literals, + regenerated_size, streams->src, compressed_size); + } + streams->src += compressed_size; + streams->src_len -= compressed_size; + + return regenerated_size; +} + +// Decode the Huffman table description +static size_t decode_huf_table(const u8 *src, size_t src_len, + HUF_dtable *dtable) { + if (src_len < 1) { + INP_SIZE(); + } + + const u8 *const osrc = src; + + u8 header = src[0]; + u8 weights[HUF_MAX_SYMBS]; + memset(weights, 0, sizeof(weights)); + + src++; + src_len--; + + int num_symbs; + + if (header >= 128) { + // Direct representation, read the weights out + num_symbs = header - 127; + size_t bytes = (num_symbs + 1) / 2; + + if (bytes > src_len) { + INP_SIZE(); + } + + for (int i = 0; i < num_symbs; i++) { + if (i % 2 == 0) { + weights[i] = src[i / 2] >> 4; + } else { + weights[i] = src[i / 2] & 0xf; + } + } + + src += bytes; + src_len -= bytes; + } else { + // The weights are FSE encoded, decode them before we can construct the + // table + size_t size = + fse_decode_hufweights(src, src_len, weights, &num_symbs, header); + src += size; + src_len -= size; + } + + // Construct the table using the decoded weights + HUF_init_dtable_usingweights(dtable, weights, num_symbs); + return src - osrc; +} + +static size_t fse_decode_hufweights(const u8 *src, size_t src_len, u8 *weights, + int *num_symbs, size_t compressed_size) { + const int MAX_ACCURACY_LOG = 7; + + FSE_dtable dtable; + + // Construct the FSE table + size_t read = FSE_decode_header(&dtable, src, src_len, MAX_ACCURACY_LOG); + + if (src_len < compressed_size) { + INP_SIZE(); + } + + // Decode the weights + *num_symbs = FSE_decompress_interleaved2( + &dtable, weights, HUF_MAX_SYMBS, src + read, compressed_size - read); + + FSE_free_dtable(&dtable); + + return compressed_size; +} +/******* END LITERALS DECODING ************************************************/ + +/******* SEQUENCE DECODING ****************************************************/ +/// The combination of FSE states needed to decode sequences +typedef struct { + u16 ll_state, of_state, ml_state; + FSE_dtable ll_table, of_table, ml_table; +} sequence_state_t; + +/// Different modes to signal to decode_seq_tables what to do +typedef enum { + seq_literal_length = 0, + seq_offset = 1, + seq_match_length = 2, +} seq_part_t; + +typedef enum { + seq_predefined = 0, + seq_rle = 1, + seq_fse = 2, + seq_repeat = 3, +} seq_mode_t; + +/// The predefined FSE distribution tables for `seq_predefined` mode +static const i16 SEQ_LITERAL_LENGTH_DEFAULT_DIST[36] = { + 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, -1, -1, -1, -1}; +static const i16 SEQ_OFFSET_DEFAULT_DIST[29] = { + 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1}; +static const i16 SEQ_MATCH_LENGTH_DEFAULT_DIST[53] = { + 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1, -1, -1}; + +/// The sequence decoding baseline and number of additional bits to read/add +/// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#the-codes-for-literals-lengths-match-lengths-and-offsets +static const u32 SEQ_LITERAL_LENGTH_BASELINES[36] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16, 18, 20, 22, 24, 28, 32, 40, + 48, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65538}; +static const u8 SEQ_LITERAL_LENGTH_EXTRA_BITS[36] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, + 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; + +static const u32 SEQ_MATCH_LENGTH_BASELINES[53] = { + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, + 31, 32, 33, 34, 35, 37, 39, 41, 43, 47, 51, 59, 67, 83, + 99, 131, 259, 515, 1027, 2051, 4099, 8195, 16387, 32771, 65539}; +static const u8 SEQ_MATCH_LENGTH_EXTRA_BITS[53] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, + 2, 2, 3, 3, 4, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; + +/// Offset decoding is simpler so we just need a maximum code value +static const u8 SEQ_MAX_CODES[3] = {35, -1, 52}; + +static void decompress_sequences(frame_context_t *ctx, const u8 *src, + size_t src_len, sequence_command_t *sequences, + size_t num_sequences); +static sequence_command_t decode_sequence(sequence_state_t *state, + const u8 *src, i64 *offset); +static size_t decode_seq_table(const u8 *src, size_t src_len, FSE_dtable *table, + seq_part_t type, seq_mode_t mode); + +static size_t decode_sequences(frame_context_t *ctx, const u8 *src, + size_t src_len, sequence_command_t **sequences) { + size_t num_sequences; + + // Decode the sequence header and allocate space for the output + if (src_len < 1) { + INP_SIZE(); + } + if (src[0] == 0) { + *sequences = NULL; + return 0; + } else if (src[0] < 128) { + num_sequences = src[0]; + src++; + src_len--; + } else if (src[0] < 255) { + if (src_len < 2) { + INP_SIZE(); + } + num_sequences = ((src[0] - 128) << 8) + src[1]; + src += 2; + src_len -= 2; + } else { + if (src_len < 3) { + INP_SIZE(); + } + num_sequences = src[1] + ((u64)src[2] << 8) + 0x7F00; + src += 3; + src_len -= 3; + } + + *sequences = malloc(num_sequences * sizeof(sequence_command_t)); + if (!*sequences) { + BAD_ALLOC(); + } + + decompress_sequences(ctx, src, src_len, *sequences, num_sequences); + return num_sequences; +} + +/// Decompress the FSE encoded sequence commands +static void decompress_sequences(frame_context_t *ctx, const u8 *src, + size_t src_len, sequence_command_t *sequences, + size_t num_sequences) { + if (src_len < 1) { + INP_SIZE(); + } + u8 compression_modes = src[0]; + src++; + src_len--; + + if ((compression_modes & 3) != 0) { + CORRUPTION(); + } + + sequence_state_t state; + size_t read; + // Update the tables we have stored in the context + read = decode_seq_table(src, src_len, &ctx->ll_dtable, seq_literal_length, + (compression_modes >> 6) & 3); + src += read; + src_len -= read; + read = decode_seq_table(src, src_len, &ctx->of_dtable, seq_offset, + (compression_modes >> 4) & 3); + src += read; + src_len -= read; + read = decode_seq_table(src, src_len, &ctx->ml_dtable, seq_match_length, + (compression_modes >> 2) & 3); + src += read; + src_len -= read; + + // Check to make sure none of the tables are uninitialized + if (!ctx->ll_dtable.symbols || !ctx->of_dtable.symbols || + !ctx->ml_dtable.symbols) { + CORRUPTION(); + } + + // Now use the context's tables + memcpy(&state.ll_table, &ctx->ll_dtable, sizeof(FSE_dtable)); + memcpy(&state.of_table, &ctx->of_dtable, sizeof(FSE_dtable)); + memcpy(&state.ml_table, &ctx->ml_dtable, sizeof(FSE_dtable)); + + int padding = 8 - log2inf(src[src_len - 1]); + i64 offset = src_len * 8 - padding; + + FSE_init_state(&state.ll_table, &state.ll_state, src, &offset); + FSE_init_state(&state.of_table, &state.of_state, src, &offset); + FSE_init_state(&state.ml_table, &state.ml_state, src, &offset); + + for (size_t i = 0; i < num_sequences; i++) { + // Decode sequences one by one + sequences[i] = decode_sequence(&state, src, &offset); + } + + if (offset != 0) { + CORRUPTION(); + } + + // Don't free our tables so they can be used in the next block +} + +// Decode a single sequence and update the state +static sequence_command_t decode_sequence(sequence_state_t *state, + const u8 *src, i64 *offset) { + // Decode symbols, but don't update states + u8 of_code = FSE_peek_symbol(&state->of_table, state->of_state); + u8 ll_code = FSE_peek_symbol(&state->ll_table, state->ll_state); + u8 ml_code = FSE_peek_symbol(&state->ml_table, state->ml_state); + + // Offset doesn't need a max value as it's not decoded using a table + if (ll_code > SEQ_MAX_CODES[seq_literal_length] || + ml_code > SEQ_MAX_CODES[seq_match_length]) { + CORRUPTION(); + } + + // Read the interleaved bits + sequence_command_t seq; + // Offset computation works differently + seq.offset = ((u32)1 << of_code) + STREAM_read_bits(src, of_code, offset); + seq.match_length = + SEQ_MATCH_LENGTH_BASELINES[ml_code] + + STREAM_read_bits(src, SEQ_MATCH_LENGTH_EXTRA_BITS[ml_code], offset); + seq.literal_length = + SEQ_LITERAL_LENGTH_BASELINES[ll_code] + + STREAM_read_bits(src, SEQ_LITERAL_LENGTH_EXTRA_BITS[ll_code], offset); + + // If the stream is complete don't read bits to update state + if (*offset != 0) { + // Update state in the order specified in the specification + FSE_update_state(&state->ll_table, &state->ll_state, src, offset); + FSE_update_state(&state->ml_table, &state->ml_state, src, offset); + FSE_update_state(&state->of_table, &state->of_state, src, offset); + } + + return seq; +} + +/// Given a sequence part and table mode, decode the FSE distribution +static size_t decode_seq_table(const u8 *src, size_t src_len, FSE_dtable *table, + seq_part_t type, seq_mode_t mode) { + + // Constant arrays indexed by seq_part_t + const i16 *const default_distributions[] = {SEQ_LITERAL_LENGTH_DEFAULT_DIST, + SEQ_OFFSET_DEFAULT_DIST, + SEQ_MATCH_LENGTH_DEFAULT_DIST}; + const size_t default_distribution_lengths[] = {36, 29, 53}; + const size_t default_distribution_accuracies[] = {6, 5, 6}; + + const size_t max_accuracies[] = {9, 8, 9}; + + if (mode != seq_repeat) { + // ree old one before overwriting + FSE_free_dtable(table); + } + + switch (mode) { + case seq_predefined: { + const i16 *distribution = default_distributions[type]; + const size_t symbs = default_distribution_lengths[type]; + const size_t accuracy_log = default_distribution_accuracies[type]; + + FSE_init_dtable(table, distribution, symbs, accuracy_log); + + return 0; + } + case seq_rle: { + if (src_len < 1) { + INP_SIZE(); + } + u8 symb = src[0]; + src++; + src_len--; + FSE_init_dtable_rle(table, symb); + + return 1; + } + case seq_fse: { + size_t read = + FSE_decode_header(table, src, src_len, max_accuracies[type]); + src += read; + src_len -= read; + + return read; + } + case seq_repeat: + // Don't have to do anything here as we're not changing the table + return 0; + default: + // Impossible, as mode is from 0-3 + return -1; + } +} +/******* END SEQUENCE DECODING ************************************************/ + +/******* SEQUENCE EXECUTION ***************************************************/ +static size_t execute_sequences(io_streams_t *streams, frame_context_t *ctx, + sequence_command_t *sequences, + size_t num_sequences, const u8 *literals, + size_t literals_len) { + u64 *offset_hist = ctx->previous_offsets; + size_t total_output = ctx->current_total_output; + + for (size_t i = 0; i < num_sequences; i++) { + sequence_command_t seq = sequences[i]; + + if (seq.literal_length > literals_len) { + CORRUPTION(); + } + + { + // Copy literals to the buffer + size_t written = + cbuf_write_data_full(&ctx->window, literals, seq.literal_length, + streams->dst, streams->dst_len); + + literals += seq.literal_length; + literals_len -= seq.literal_length; + + streams->dst += written; + streams->dst_len -= written; + + total_output += seq.literal_length; + } + + size_t offset; + + // Offsets are special, we need to handle the repeat offsets + if (seq.offset <= 3) { + u32 idx = seq.offset; + if (seq.literal_length == 0) { + // Special case when literal length is 0 + idx++; + } + + if (idx == 1) { + offset = offset_hist[1]; + } else { + // If idx == 4 then literal length was 0 and the offset was 3 + offset = idx < 4 ? offset_hist[idx] : offset_hist[1] - 1; + + // If idx == 2 we don't need to modify offset_hist[3] + if (idx > 2) { + offset_hist[3] = offset_hist[2]; + } + offset_hist[2] = offset_hist[1]; + offset_hist[1] = offset; + } + } else { + offset = seq.offset - 3; + + // Shift back history + offset_hist[3] = offset_hist[2]; + offset_hist[2] = offset_hist[1]; + offset_hist[1] = offset; + } + + if (offset > total_output) { + CORRUPTION(); + } + + { + // Do the offset copy operation + size_t written = + cbuf_copy_offset_full(&ctx->window, offset, seq.match_length, + streams->dst, streams->dst_len); + + streams->dst += written; + streams->dst_len -= written; + total_output += seq.match_length; + } + } + + { + // Copy any leftover literal bytes + size_t written = + cbuf_write_data_full(&ctx->window, literals, literals_len, + streams->dst, streams->dst_len); + streams->dst += written; + streams->dst_len -= written; + + total_output += literals_len; + } + + ctx->current_total_output = total_output; + + return total_output; +} +/******* END SEQUENCE EXECUTION ***********************************************/ + +/******* DICTIONARY PARSING ***************************************************/ +static void init_raw_content_dict(dictionary_t *dict, const u8 *src, + size_t src_len); + +static void parse_dictionary(dictionary_t *dict, const u8 *src, + size_t src_len) { + memset(dict, 0, sizeof(dictionary_t)); + if (src_len < 8) { + INP_SIZE(); + } + u32 magic_number = read_bits_LE(src, 32, 0); + if (magic_number != 0xEC30A437) { + // raw content dict + init_raw_content_dict(dict, src, src_len); + return; + } + dict->dictionary_id = read_bits_LE(src, 32, 32); + + src += 8; + src_len -= 8; + + // Parse the provided entropy tables in order + { + size_t read = decode_huf_table(src, src_len, &dict->literals_dtable); + src += read; + src_len -= read; + } + { + size_t read = decode_seq_table(src, src_len, &dict->of_dtable, + seq_offset, seq_fse); + src += read; + src_len -= read; + } + { + size_t read = decode_seq_table(src, src_len, &dict->ml_dtable, + seq_match_length, seq_fse); + src += read; + src_len -= read; + } + { + size_t read = decode_seq_table(src, src_len, &dict->ll_dtable, + seq_literal_length, seq_fse); + src += read; + src_len -= read; + } + + if (src_len < 12) { + INP_SIZE(); + } + // Read in the previous offset history + dict->previous_offsets[1] = read_bits_LE(src, 32, 0); + dict->previous_offsets[2] = read_bits_LE(src, 32, 32); + dict->previous_offsets[3] = read_bits_LE(src, 32, 64); + + src += 12; + src_len -= 12; + + // Ensure the provided offsets aren't too large + for (int i = 1; i <= 3; i++) { + if (dict->previous_offsets[i] > src_len) { + ERROR("Dictionary corrupted"); + } + } + // The rest is the content + dict->content = malloc(src_len); + if (!dict->content) { + BAD_ALLOC(); + } + + dict->content_size = src_len; + memcpy(dict->content, src, src_len); +} + +/// If parse_dictionary is given a raw content dictionary, it delegates here +static void init_raw_content_dict(dictionary_t *dict, const u8 *src, + size_t src_len) { + dict->dictionary_id = 0; + // Copy in the content + dict->content = malloc(src_len); + if (!dict->content) { + BAD_ALLOC(); + } + + dict->content_size = src_len; + memcpy(dict->content, src, src_len); +} + +/// Free an allocated dictionary +static void free_dictionary(dictionary_t *dict) { + HUF_free_dtable(&dict->literals_dtable); + FSE_free_dtable(&dict->ll_dtable); + FSE_free_dtable(&dict->of_dtable); + FSE_free_dtable(&dict->ml_dtable); + + free(dict->content); + + memset(dict, 0, sizeof(dictionary_t)); +} +/******* END DICTIONARY PARSING ***********************************************/ + +/******* CIRCULAR BUFFER ******************************************************/ +static void cbuf_init(cbuf_t *buf, size_t size) { + buf->ptr = malloc(size); + + if (!buf->ptr) { + BAD_ALLOC(); + } + + memset(buf->ptr, 0x3f, size); + + buf->size = size; + buf->idx = 0; + buf->last_flush = 0; +} + +static size_t cbuf_write_data(cbuf_t *buf, const u8 *src, size_t src_len) { + if (buf->size == 0 && src_len > 0) { + CORRUPTION(); + } + size_t max_len = buf->size - buf->idx; + size_t len = MIN(src_len, max_len); + + memcpy(buf->ptr + buf->idx, src, len); + + buf->idx += len; + + return len; +} + +static size_t cbuf_write_data_full(cbuf_t *buf, const u8 *src, size_t src_len, + u8 *out, size_t out_len) { + size_t written = 0; + size_t flushed = 0; + while (1) { + written += cbuf_write_data(buf, src + written, src_len - written); + if (written == src_len) { + break; + } else { + flushed += cbuf_flush(buf, out + flushed, out_len - flushed); + } + } + + return flushed; +} + +static size_t cbuf_copy_offset(cbuf_t *buf, size_t offset, size_t len) { + if (buf->size == 0 && len > 0) { + CORRUPTION(); + } + if (offset > buf->size) { + CORRUPTION(); + } + size_t max_len = buf->size - buf->idx; + len = MIN(len, max_len); + + size_t read_off = (buf->idx + buf->size - offset) % buf->size; + + for (size_t i = 0; i < len; i++) { + buf->ptr[buf->idx++] = buf->ptr[read_off++]; + if (read_off == buf->size) { + read_off = 0; + } + } + + return len; +} + +static size_t cbuf_copy_offset_full(cbuf_t *buf, size_t offset, size_t len, + u8 *out, size_t out_len) { + size_t written = 0; + size_t flushed = 0; + while (1) { + written += cbuf_copy_offset(buf, offset, len - written); + if (written == len) { + break; + } else { + flushed += cbuf_flush(buf, out + flushed, out_len - flushed); + } + } + + return flushed; +} + +static size_t cbuf_repeat_byte(cbuf_t *buf, u8 byte, size_t len) { + if (buf->size == 0 && len > 0) { + CORRUPTION(); + } + size_t max_len = buf->size - buf->idx; + len = MIN(len, max_len); + + memset(buf->ptr + buf->idx, byte, len); + + return len; +} + +static size_t cbuf_repeat_byte_full(cbuf_t *buf, u8 byte, size_t len, u8 *out, + size_t out_len) { + size_t written = 0; + size_t flushed = 0; + while (1) { + written += cbuf_repeat_byte(buf, byte, len - written); + if (written == len) { + break; + } else { + flushed += cbuf_flush(buf, out + flushed, out_len - flushed); + } + } + + return flushed; +} + +static size_t cbuf_flush(cbuf_t *buf, u8 *dst, size_t dst_len) { + if (buf->idx < buf->last_flush) { + CORRUPTION(); + } + + size_t len = buf->idx - buf->last_flush; + + if (dst && len > dst_len) { + OUT_SIZE(); + } + + // allow for NULL buffers to indicate flushing to nowhere + if (dst) { + memcpy(dst, buf->ptr + buf->last_flush, len); + } + + // we could have a 0 size buffer + if (buf->size) { + buf->idx = buf->idx % buf->size; + } + buf->last_flush = buf->idx; + + return len; +} + +static void cbuf_free(cbuf_t *buf) { + free(buf->ptr); + memset(buf, 0, sizeof(cbuf_t)); +} +/******* END CIRCULAR BUFFER **************************************************/ + +/******* BITSTREAM OPERATIONS *************************************************/ +static inline u64 read_bits_LE(const u8 *src, int num, size_t offset) { + if (num > 64) { + return -1; + } + + src += offset / 8; + offset %= 8; + u64 res = 0; + + int shift = 0; + int left = num; + while (left > 0) { + u64 mask = left >= 8 ? 0xff : (((u64)1 << left) - 1); + res += (((u64)*src++ >> offset) & mask) << shift; + shift += 8 - offset; + left -= 8 - offset; + offset = 0; + } + + return res; +} + +static inline u64 STREAM_read_bits(const u8 *src, int bits, i64 *offset) { + *offset = *offset - bits; + size_t actual_off = *offset; + if (*offset < 0) { + bits += *offset; + actual_off = 0; + } + u64 res = read_bits_LE(src, bits, actual_off); + + if (*offset < 0) { + // Fill in the bottom "overflowed" bits with 0's + res = -*offset >= 64 ? 0 : (res << -*offset); + } + return res; +} +/******* END BITSTREAM OPERATIONS *********************************************/ + +/******* BIT COUNTING OPERATIONS **********************************************/ +static inline int log2sup(u64 num) { + for (int i = 0; i < 64; i++) { + if (((u64)1 << i) >= num) { + return i; + } + } + return -1; +} + +static inline int log2inf(u64 num) { + for (int i = 63; i >= 0; i--) { + if (((u64)1 << i) <= num) { + return i; + } + } + return -1; +} +/******* END BIT COUNTING OPERATIONS ******************************************/ + +/******* HUFFMAN PRIMITIVES ***************************************************/ +static inline u8 HUF_decode_symbol(HUF_dtable *dtable, u16 *state, + const u8 *src, i64 *offset) { + // Look up the symbol and number of bits to read + const u8 symb = dtable->symbols[*state]; + const u8 bits = dtable->num_bits[*state]; + const u16 rest = STREAM_read_bits(src, bits, offset); + *state = ((*state << bits) + rest) & (((u16)1 << dtable->max_bits) - 1); + + return symb; +} + +static inline void HUF_init_state(HUF_dtable *dtable, u16 *state, const u8 *src, + i64 *offset) { + // Read in a full dtable->max_bits to initialize the state + const u8 bits = dtable->max_bits; + *state = STREAM_read_bits(src, bits, offset); +} + +static size_t HUF_decompress_1stream(HUF_dtable *dtable, u8 *dst, + size_t dst_len, const u8 *src, + size_t src_len) { + u8 *const dst_max = dst + dst_len; + u8 *const odst = dst; + + // To maintain similarity with FSE, start from the end + // Find the last 1 bit + int padding = 8 - log2inf(src[src_len - 1]); + + i64 offset = src_len * 8 - padding; + u16 state; + + HUF_init_state(dtable, &state, src, &offset); + + while (dst < dst_max && offset > -dtable->max_bits) { + *dst++ = HUF_decode_symbol(dtable, &state, src, &offset); + } + // If we stopped before consuming all the input, we didn't have enough space + if (dst == dst_max && offset > -dtable->max_bits) { + OUT_SIZE(); + } + + // The current state should be the `max_bits` preceding the start as + // everything from `src` onward should be consumed + if (offset != -dtable->max_bits) { + CORRUPTION(); + } + + return dst - odst; +} + +static size_t HUF_decompress_4stream(HUF_dtable *dtable, u8 *dst, + size_t dst_len, const u8 *src, + size_t src_len) { + // Decode each stream independently for simplicity + // If we wanted to we could decode all 4 at the same time for speed, + // utilizing + // more execution units + + const u8 *src1, *src2, *src3, *src4, *src_end; + u8 *dst1, *dst2, *dst3, *dst4, *dst_end; + + size_t total_out = 0; + + if (src_len < 6) { + INP_SIZE(); + } + + src1 = src + 6; + src2 = src1 + read_bits_LE(src, 16, 0); + src3 = src2 + read_bits_LE(src, 16, 16); + src4 = src3 + read_bits_LE(src, 16, 32); + src_end = src + src_len; + + // We can't test with all 4 sizes because the 4th size is a function of the + // other 3 and the provided length + if (src4 - src >= src_len) { + INP_SIZE(); + } + + size_t segment_size = (dst_len + 3) / 4; + dst1 = dst; + dst2 = dst1 + segment_size; + dst3 = dst2 + segment_size; + dst4 = dst3 + segment_size; + dst_end = dst + dst_len; + + total_out += + HUF_decompress_1stream(dtable, dst1, segment_size, src1, src2 - src1); + total_out += + HUF_decompress_1stream(dtable, dst2, segment_size, src2, src3 - src2); + total_out += + HUF_decompress_1stream(dtable, dst3, segment_size, src3, src4 - src3); + total_out += HUF_decompress_1stream(dtable, dst4, dst_end - dst4, src4, + src_end - src4); + + return total_out; +} + +static void HUF_init_dtable(HUF_dtable *table, u8 *bits, int num_symbs) { + memset(table, 0, sizeof(HUF_dtable)); + if (num_symbs > HUF_MAX_SYMBS) { + ERROR("Too many symbols for Huffman"); + } + + u8 max_bits = 0; + u16 rank_count[HUF_MAX_BITS + 1]; + memset(rank_count, 0, sizeof(rank_count)); + + // Count the number of symbols for each number of bits, and determine the + // depth of the tree + for (int i = 0; i < num_symbs; i++) { + if (bits[i] > HUF_MAX_BITS) { + ERROR("Huffman table depth too large"); + } + max_bits = MAX(max_bits, bits[i]); + rank_count[bits[i]]++; + } + + size_t table_size = 1 << max_bits; + table->max_bits = max_bits; + table->symbols = malloc(table_size); + table->num_bits = malloc(table_size); + + if (!table->symbols || !table->num_bits) { + free(table->symbols); + free(table->num_bits); + BAD_ALLOC(); + } + + u32 rank_idx[HUF_MAX_BITS + 1]; + // Initialize the starting codes for each rank (number of bits) + rank_idx[max_bits] = 0; + for (int i = max_bits; i >= 1; i--) { + rank_idx[i - 1] = rank_idx[i] + rank_count[i] * (1 << (max_bits - i)); + // The entire range takes the same number of bits so we can memset it + memset(&table->num_bits[rank_idx[i]], i, rank_idx[i - 1] - rank_idx[i]); + } + + if (rank_idx[0] != table_size) { + CORRUPTION(); + } + + // Allocate codes and fill in the table + for (int i = 0; i < num_symbs; i++) { + if (bits[i] != 0) { + // Allocate a code for this symbol and set its range in the table + const u16 code = rank_idx[bits[i]]; + const u16 len = 1 << (max_bits - bits[i]); + memset(&table->symbols[code], i, len); + rank_idx[bits[i]] += len; + } + } +} + +static void HUF_init_dtable_usingweights(HUF_dtable *table, u8 *weights, + int num_symbs) { + // +1 because the last weight is not transmitted in the header + if (num_symbs + 1 > HUF_MAX_SYMBS) { + ERROR("Too many symbols for Huffman"); + } + + u8 bits[HUF_MAX_SYMBS]; + + u64 weight_sum = 0; + for (int i = 0; i < num_symbs; i++) { + weight_sum += weights[i] > 0 ? (u64)1 << (weights[i] - 1) : 0; + } + + // Find the first power of 2 larger than the sum + int max_bits = log2inf(weight_sum) + 1; + u64 left_over = ((u64)1 << max_bits) - weight_sum; + // If the left over isn't a power of 2, the weights are invalid + if (left_over & (left_over - 1)) { + CORRUPTION(); + } + + int last_weight = log2inf(left_over) + 1; + + for (int i = 0; i < num_symbs; i++) { + bits[i] = weights[i] > 0 ? (max_bits + 1 - weights[i]) : 0; + } + bits[num_symbs] = + max_bits + 1 - last_weight; // last weight is always non-zero + + HUF_init_dtable(table, bits, num_symbs + 1); +} + +static void HUF_free_dtable(HUF_dtable *dtable) { + free(dtable->symbols); + free(dtable->num_bits); + memset(dtable, 0, sizeof(HUF_dtable)); +} + +static void HUF_copy_dtable(HUF_dtable *dst, const HUF_dtable *src) { + if (src->max_bits == 0) { + memset(dst, 0, sizeof(HUF_dtable)); + return; + } + + size_t size = (size_t)1 << src->max_bits; + dst->max_bits = src->max_bits; + + dst->symbols = malloc(size); + dst->num_bits = malloc(size); + if (!dst->symbols || !dst->num_bits) { + BAD_ALLOC(); + } + + memcpy(dst->symbols, src->symbols, size); + memcpy(dst->num_bits, src->num_bits, size); +} +/******* END HUFFMAN PRIMITIVES ***********************************************/ + +/******* FSE PRIMITIVES *******************************************************/ +static inline u8 FSE_peek_symbol(FSE_dtable *dtable, u16 state) { + return dtable->symbols[state]; +} + +static inline void FSE_update_state(FSE_dtable *dtable, u16 *state, + const u8 *src, i64 *offset) { + const u8 bits = dtable->num_bits[*state]; + const u16 rest = STREAM_read_bits(src, bits, offset); + *state = dtable->new_state_base[*state] + rest; +} + +// Decodes a single FSE symbol and updates the offset +static inline u8 FSE_decode_symbol(FSE_dtable *dtable, u16 *state, + const u8 *src, i64 *offset) { + const u8 symb = FSE_peek_symbol(dtable, *state); + FSE_update_state(dtable, state, src, offset); + return symb; +} + +static inline void FSE_init_state(FSE_dtable *dtable, u16 *state, const u8 *src, + i64 *offset) { + const u8 bits = dtable->accuracy_log; + *state = STREAM_read_bits(src, bits, offset); +} + +static size_t FSE_decompress_interleaved2(FSE_dtable *dtable, u8 *dst, + size_t dst_len, const u8 *src, + size_t src_len) { + if (src_len == 0) { + INP_SIZE(); + } + + u8 *dst_max = dst + dst_len; + u8 *const odst = dst; + + // Find the last 1 bit + int padding = 8 - log2inf(src[src_len - 1]); + + i64 offset = src_len * 8 - padding; + + u16 state1, state2; + FSE_init_state(dtable, &state1, src, &offset); + FSE_init_state(dtable, &state2, src, &offset); + + // Decode until we overflow the stream + // Since we decode in reverse order, overflowing the stream is offset going + // negative + while (1) { + if (dst > dst_max - 2) { + OUT_SIZE(); + } + *dst++ = FSE_decode_symbol(dtable, &state1, src, &offset); + if (offset < 0) { + // There's still a symbol to decode in state2 + *dst++ = FSE_decode_symbol(dtable, &state2, src, &offset); + break; + } + + if (dst > dst_max - 2) { + OUT_SIZE(); + } + *dst++ = FSE_decode_symbol(dtable, &state2, src, &offset); + if (offset < 0) { + // There's still a symbol to decode in state1 + *dst++ = FSE_decode_symbol(dtable, &state1, src, &offset); + break; + } + } + + // number of symbols read + return dst - odst; +} + +static void FSE_init_dtable(FSE_dtable *dtable, const i16 *norm_freqs, + int num_symbs, int accuracy_log) { + if (accuracy_log > FSE_MAX_ACCURACY_LOG) { + ERROR("FSE accuracy too large"); + } + if (num_symbs > FSE_MAX_SYMBS) { + ERROR("Too many symbols for FSE"); + } + + dtable->accuracy_log = accuracy_log; + + size_t size = (size_t)1 << accuracy_log; + dtable->symbols = malloc(size * sizeof(u8)); + dtable->num_bits = malloc(size * sizeof(u8)); + dtable->new_state_base = malloc(size * sizeof(u16)); + + // Used to determine how many bits need to be read for each state, + // and where the destination range should start + // Needs to be u16 because max value is 2 * max number of symbols, + // which can be larger than a byte can store + u16 state_desc[FSE_MAX_SYMBS]; + + int high_threshold = size; + for (int s = 0; s < num_symbs; s++) { + // Scan for low probability symbols to put at the top + if (norm_freqs[s] == -1) { + dtable->symbols[--high_threshold] = s; + state_desc[s] = 1; + } + } + + // Place the rest in the table + u16 step = (size >> 1) + (size >> 3) + 3; + u16 mask = size - 1; + u16 pos = 0; + for (int s = 0; s < num_symbs; s++) { + if (norm_freqs[s] <= 0) { + continue; + } + + state_desc[s] = norm_freqs[s]; + + for (int i = 0; i < norm_freqs[s]; i++) { + dtable->symbols[pos] = s; + do { + pos = (pos + step) & mask; + } while (pos >= + high_threshold); // Make sure we don't occupy a spot taken + // by the low prob symbols + // Note: no other collision checking is necessary as `step` is + // coprime to + // `size`, so the cycle will visit each position exactly once + } + } + if (pos != 0) { + CORRUPTION(); + } + + // Now we can fill baseline and num bits + for (int i = 0; i < size; i++) { + u8 symbol = dtable->symbols[i]; + u16 next_state_desc = state_desc[symbol]++; + // Fills in the table appropriately + // next_state_desc increases by symbol over time, decreasing number of + // bits + dtable->num_bits[i] = (u8)(accuracy_log - log2inf(next_state_desc)); + // baseline increases until the bit threshold is passed, at which point + // it + // resets to 0 + dtable->new_state_base[i] = + ((u16)next_state_desc << dtable->num_bits[i]) - size; + } +} + +static size_t FSE_decode_header(FSE_dtable *dtable, const u8 *src, + size_t src_len, int max_accuracy_log) { + if (max_accuracy_log > FSE_MAX_ACCURACY_LOG) { + ERROR("FSE accuracy too large"); + } + if (src_len < 1) { + INP_SIZE(); + } + + int accuracy_log = 5 + read_bits_LE(src, 4, 0); + if (accuracy_log > max_accuracy_log) { + ERROR("FSE accuracy too large"); + } + + // The +1 facilitates the `-1` probabilities + i32 remaining = (1 << accuracy_log) + 1; + i16 frequencies[FSE_MAX_SYMBS]; + + int symb = 0; + size_t offset = 4; + while (remaining > 1 && symb < FSE_MAX_SYMBS) { + int bits = log2sup(remaining + + 1); // the number of possible values we could read + u16 val = read_bits_LE(src, bits, offset); + offset += bits; + + // try to mask out the lower bits to see if it qualifies for the "small + // value" threshold + u16 lower_mask = ((u16)1 << (bits - 1)) - 1; + u16 threshold = ((u16)1 << bits) - 1 - remaining; + + if ((val & lower_mask) < threshold) { + offset--; + val = val & lower_mask; + } else if (val > lower_mask) { + val = val - threshold; + } + + i16 proba = (i16)val - 1; + // a value of -1 is possible, and has special meaning + remaining -= proba < 0 ? -proba : proba; + + frequencies[symb] = proba; + symb++; + + // Handle the special probability = 0 case + if (proba == 0) { + // read the next two bits to see how many more 0s + int repeat = read_bits_LE(src, 2, offset); + offset += 2; + + while (1) { + for (int i = 0; i < repeat && symb < FSE_MAX_SYMBS; i++) { + frequencies[symb++] = 0; + } + if (repeat == 3) { + repeat = read_bits_LE(src, 2, offset); + offset += 2; + } else { + break; + } + } + } + } + + if (remaining != 1 || symb >= FSE_MAX_SYMBS) { + CORRUPTION(); + } + + // Initialize the decoding table using the determined weights + FSE_init_dtable(dtable, frequencies, symb, accuracy_log); + + return (offset + 7) / 8; +} + +static void FSE_init_dtable_rle(FSE_dtable *dtable, u8 symb) { + dtable->symbols = malloc(sizeof(u8)); + dtable->num_bits = malloc(sizeof(u8)); + dtable->new_state_base = malloc(sizeof(u16)); + + // This setup will always have a state of 0, always return symbol `symb`, + // and + // never consume any bits + dtable->symbols[0] = symb; + dtable->num_bits[0] = 0; + dtable->new_state_base[0] = 0; + dtable->accuracy_log = 0; +} + +static void FSE_free_dtable(FSE_dtable *dtable) { + free(dtable->symbols); + free(dtable->num_bits); + free(dtable->new_state_base); + memset(dtable, 0, sizeof(FSE_dtable)); +} + +static void FSE_copy_dtable(FSE_dtable *dst, const FSE_dtable *src) { + if (src->accuracy_log == 0) { + memset(dst, 0, sizeof(FSE_dtable)); + return; + } + + size_t size = (size_t)1 << src->accuracy_log; + dst->accuracy_log = src->accuracy_log; + + dst->symbols = malloc(size); + dst->num_bits = malloc(size); + dst->new_state_base = malloc(size * sizeof(u16)); + if (!dst->symbols || !dst->num_bits || !dst->new_state_base) { + BAD_ALLOC(); + } + + memcpy(dst->symbols, src->symbols, size); + memcpy(dst->num_bits, src->num_bits, size); + memcpy(dst->new_state_base, src->new_state_base, size * sizeof(u16)); +} +/******* END FSE PRIMITIVES ***************************************************/ + diff --git a/contrib/educational_decoder/zstd_decompress.h b/contrib/educational_decoder/zstd_decompress.h new file mode 100644 index 000000000..3671678b1 --- /dev/null +++ b/contrib/educational_decoder/zstd_decompress.h @@ -0,0 +1,6 @@ +size_t ZSTD_decompress(void *dst, size_t dst_len, const void *src, + size_t src_len); +size_t ZSTD_decompress_with_dict(void *dst, size_t dst_len, const void *src, + size_t src_len, const void *dict, + size_t dict_len); + From 5657e0e07d4ecb23300390c0e95d4f9ec4ca1d66 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Mon, 30 Jan 2017 14:42:21 -0800 Subject: [PATCH 002/223] Added ZSTD_get_decompressed_size Since this implementation handles multiple concatenated frames, to determine decompressed size we must traverse the entire input, checking each frame's frame_content_size field --- contrib/educational_decoder/harness.c | 127 ++++---- contrib/educational_decoder/zstd_decompress.c | 298 +++++++++++++----- contrib/educational_decoder/zstd_decompress.h | 1 + 3 files changed, 293 insertions(+), 133 deletions(-) diff --git a/contrib/educational_decoder/harness.c b/contrib/educational_decoder/harness.c index 6f4765d9d..c44100fff 100644 --- a/contrib/educational_decoder/harness.c +++ b/contrib/educational_decoder/harness.c @@ -5,8 +5,8 @@ typedef unsigned char u8; -// There's no good way to determine output size without decompressing -// For this example assume we'll never decompress at a ratio larger than 16 +// If the data doesn't have decompressed size with it, fallback on assuming the +// compression ratio is at most 16 #define MAX_COMPRESSION_RATIO (16) u8 *input; @@ -14,80 +14,89 @@ u8 *output; u8 *dict; size_t read_file(const char *path, u8 **ptr) { - FILE *f = fopen(path, "rb"); - if (!f) { - fprintf(stderr, "failed to open file %s\n", path); - exit(1); - } - - fseek(f, 0L, SEEK_END); - size_t size = ftell(f); - rewind(f); - - *ptr = malloc(size); - if (!ptr) { - fprintf(stderr, "failed to allocate memory to hold %s\n", path); - exit(1); - } - - size_t pos = 0; - while (!feof(f)) { - size_t read = fread(&(*ptr)[pos], 1, size, f); - if (ferror(f)) { - fprintf(stderr, "error while reading file %s\n", path); - exit(1); + FILE *f = fopen(path, "rb"); + if (!f) { + fprintf(stderr, "failed to open file %s\n", path); + exit(1); } - pos += read; - } - fclose(f); + fseek(f, 0L, SEEK_END); + size_t size = ftell(f); + rewind(f); - return pos; + *ptr = malloc(size); + if (!ptr) { + fprintf(stderr, "failed to allocate memory to hold %s\n", path); + exit(1); + } + + size_t pos = 0; + while (!feof(f)) { + size_t read = fread(&(*ptr)[pos], 1, size, f); + if (ferror(f)) { + fprintf(stderr, "error while reading file %s\n", path); + exit(1); + } + pos += read; + } + + fclose(f); + + return pos; } void write_file(const char *path, const u8 *ptr, size_t size) { - FILE *f = fopen(path, "wb"); + FILE *f = fopen(path, "wb"); - size_t written = 0; - while (written < size) { - written += fwrite(&ptr[written], 1, size, f); - if (ferror(f)) { - fprintf(stderr, "error while writing file %s\n", path); - exit(1); + size_t written = 0; + while (written < size) { + written += fwrite(&ptr[written], 1, size, f); + if (ferror(f)) { + fprintf(stderr, "error while writing file %s\n", path); + exit(1); + } } - } - fclose(f); + fclose(f); } int main(int argc, char **argv) { - if (argc < 3) { - fprintf(stderr, "usage: %s [dictionary]\n", argv[0]); + if (argc < 3) { + fprintf(stderr, "usage: %s [dictionary]\n", + argv[0]); - return 1; - } + return 1; + } - size_t input_size = read_file(argv[1], &input); - size_t dict_size = 0; - if (argc >= 4) { - dict_size = read_file(argv[3], &dict); - } + size_t input_size = read_file(argv[1], &input); + size_t dict_size = 0; + if (argc >= 4) { + dict_size = read_file(argv[3], &dict); + } - output = malloc(MAX_COMPRESSION_RATIO * input_size); - if (!output) { - fprintf(stderr, "failed to allocate memory\n"); - return 1; - } + size_t decompressed_size = ZSTD_get_decompressed_size(input, input_size); + if (decompressed_size == -1) { + decompressed_size = MAX_COMPRESSION_RATIO * input_size; + fprintf(stderr, "WARNING: Compressed data does contain decompressed " + "size, going to assume the compression ratio is at " + "most %d (decompressed size of at most %lld\n", + MAX_COMPRESSION_RATIO, decompressed_size); + } + output = malloc(decompressed_size); + if (!output) { + fprintf(stderr, "failed to allocate memory\n"); + return 1; + } - size_t decompressed = - ZSTD_decompress_with_dict(output, input_size * MAX_COMPRESSION_RATIO, - input, input_size, dict, dict_size); + size_t decompressed = + ZSTD_decompress_with_dict(output, input_size * MAX_COMPRESSION_RATIO, + input, input_size, dict, dict_size); - write_file(argv[2], output, decompressed); + write_file(argv[2], output, decompressed); - free(input); - free(output); - free(dict); - input = output = dict = NULL; + free(input); + free(output); + free(dict); + input = output = dict = NULL; } diff --git a/contrib/educational_decoder/zstd_decompress.c b/contrib/educational_decoder/zstd_decompress.c index 8dc159008..7b04c4b29 100644 --- a/contrib/educational_decoder/zstd_decompress.c +++ b/contrib/educational_decoder/zstd_decompress.c @@ -16,6 +16,10 @@ size_t ZSTD_decompress_with_dict(void *dst, size_t dst_len, const void *src, size_t src_len, const void *dict, size_t dict_len); +/// Get the decompressed size of an input stream so memory can be allocated in +/// advance +size_t ZSTD_get_decompressed_size(const void *src, size_t src_len); + /******* UTILITY MACROS AND TYPES *********************************************/ #define MAX_WINDOW_SIZE ((size_t)512 << 20) // Max block size decompressed size is 128 KB and literal blocks must be smaller @@ -232,10 +236,30 @@ typedef struct { size_t src_len; } io_streams_t; +/// A small structure that can be reused in various places that need to access +/// frame header information +typedef struct { + // The size of window that we need to be able to contiguously store for + // references + size_t window_size; + // The total output size of this compressed frame + size_t frame_content_size; + + // The dictionary id if this frame uses one + u32 dictionary_id; + + // Whether or not the content of this frame has a checksum + int content_checksum_flag; + // Whether or not the output for this frame is in a single segment + int single_segment_flag; + + // The size in bytes of this header + int header_size; +} frame_header_t; + /// The context needed to decode blocks in a frame typedef struct { - size_t window_size; - size_t frame_content_size; + frame_header_t header; // The total amount of data available for backreferences, to determine if an // offset too large to be correct @@ -255,12 +279,6 @@ typedef struct { // The last 3 offsets for the special "repeat offsets". Array size is 4 so // that previous_offsets[1] corresponds to the most recent offset u64 previous_offsets[4]; - - // The dictionary id for this frame if one exists - u32 dictionary_id; - - int single_segment_flag; - int content_checksum_flag; } frame_context_t; /// The decoded contents of a dictionary so that it doesn't have to be repeated @@ -364,10 +382,11 @@ size_t ZSTD_decompress_with_dict(void *dst, size_t dst_len, const void *src, /******* FRAME DECODING ******************************************************/ static void decode_data_frame(io_streams_t *streams, dictionary_t *dict); -static void init_frame_context(frame_context_t *context); -static void free_frame_context(frame_context_t *context); -static void parse_frame_header(io_streams_t *streams, frame_context_t *ctx, +static void init_frame_context(io_streams_t *streams, frame_context_t *context, dictionary_t *dict); +static void free_frame_context(frame_context_t *context); +static void parse_frame_header(frame_header_t *header, const u8 *src, + size_t src_len); static void frame_context_apply_dict(frame_context_t *ctx, dictionary_t *dict); static void decompress_data(io_streams_t *streams, frame_context_t *ctx); @@ -411,12 +430,10 @@ static void decode_data_frame(io_streams_t *streams, dictionary_t *dict) { frame_context_t ctx; // Initialize the context that needs to be carried from block to block - init_frame_context(&ctx); - parse_frame_header(streams, &ctx, dict); - frame_context_apply_dict(&ctx, dict); + init_frame_context(streams, &ctx, dict); - if (ctx.frame_content_size != 0 && - ctx.frame_content_size > streams->dst_len) { + if (ctx.header.frame_content_size != 0 && + ctx.header.frame_content_size > streams->dst_len) { OUT_SIZE(); } @@ -425,13 +442,40 @@ static void decode_data_frame(io_streams_t *streams, dictionary_t *dict) { free_frame_context(&ctx); } -static void init_frame_context(frame_context_t *context) { +/// Takes the information provided in the header and dictionary, and initializes +/// the context for this frame +static void init_frame_context(io_streams_t *streams, frame_context_t *context, + dictionary_t *dict) { memset(context, 0x00, sizeof(frame_context_t)); + // Parse data from the frame header + parse_frame_header(&context->header, streams->src, streams->src_len); + streams->src += context->header.header_size; + streams->src_len -= context->header.header_size; + // Set up the offset history for the repeat offset commands context->previous_offsets[1] = 1; context->previous_offsets[2] = 4; context->previous_offsets[3] = 8; + + { + // Allocate the window buffer + size_t buffer_size; + if (context->header.single_segment_flag) { + buffer_size = context->header.frame_content_size + + (dict ? dict->content_size : 0); + } else { + buffer_size = context->header.window_size; + } + + if (buffer_size > MAX_WINDOW_SIZE) { + ERROR("Requested window size too large"); + } + cbuf_init(&context->window, buffer_size); + } + + // Apply details from the dict if it exists + frame_context_apply_dict(context, dict); } static void free_frame_context(frame_context_t *context) { @@ -446,13 +490,13 @@ static void free_frame_context(frame_context_t *context) { memset(context, 0, sizeof(frame_context_t)); } -static void parse_frame_header(io_streams_t *streams, frame_context_t *ctx, - dictionary_t *dict) { - if (streams->src_len < 1) { +static void parse_frame_header(frame_header_t *header, const u8 *src, + size_t src_len) { + if (src_len < 1) { INP_SIZE(); } - u8 descriptor = read_bits_LE(streams->src, 8, 0); + u8 descriptor = read_bits_LE(src, 8, 0); // decode frame header descriptor into flags u8 frame_content_size_flag = descriptor >> 6; @@ -465,30 +509,28 @@ static void parse_frame_header(io_streams_t *streams, frame_context_t *ctx, CORRUPTION(); } - streams->src++; - streams->src_len--; + int header_size = 1; - ctx->single_segment_flag = single_segment_flag; - ctx->content_checksum_flag = content_checksum_flag; + header->single_segment_flag = single_segment_flag; + header->content_checksum_flag = content_checksum_flag; // decode window size if (!single_segment_flag) { - if (streams->src_len < 1) { + if (src_len < header_size + 1) { INP_SIZE(); } // Use the algorithm from the specification to compute window size // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#window_descriptor - u8 window_descriptor = read_bits_LE(streams->src, 8, 0); + u8 window_descriptor = src[header_size]; u8 exponent = window_descriptor >> 3; u8 mantissa = window_descriptor & 7; size_t window_base = (size_t)1 << (10 + exponent); size_t window_add = (window_base / 8) * mantissa; - ctx->window_size = window_base + window_add; + header->window_size = window_base + window_add; - streams->src++; - streams->src_len--; + header_size++; } // decode dictionary id if it exists @@ -496,52 +538,40 @@ static void parse_frame_header(io_streams_t *streams, frame_context_t *ctx, const int bytes_array[] = {0, 1, 2, 4}; const int bytes = bytes_array[dictionary_id_flag]; - if (streams->src_len < bytes) { + if (src_len < header_size + bytes) { INP_SIZE(); } - ctx->dictionary_id = read_bits_LE(streams->src, bytes * 8, 0); - streams->src += bytes; - streams->src_len -= bytes; + header->dictionary_id = read_bits_LE(src + header_size, bytes * 8, 0); + + header_size += bytes; } else { - ctx->dictionary_id = 0; + header->dictionary_id = 0; } // decode frame content size if it exists if (single_segment_flag || frame_content_size_flag) { // if frame_content_size_flag == 0 but single_segment_flag is set, we - // still - // have a 1 byte field + // still have a 1 byte field const int bytes_array[] = {1, 2, 4, 8}; const int bytes = bytes_array[frame_content_size_flag]; - if (streams->src_len < bytes) { + if (src_len < header_size + bytes) { INP_SIZE(); } - ctx->frame_content_size = read_bits_LE(streams->src, bytes * 8, 0); + header->frame_content_size = + read_bits_LE(src + header_size, bytes * 8, 0); if (bytes == 2) { - ctx->frame_content_size += 256; + header->frame_content_size += 256; } - streams->src += bytes; - streams->src_len -= bytes; + header_size += bytes; + } else { + header->frame_content_size = 0; } - if (single_segment_flag) { - ctx->window_size = - ctx->frame_content_size + (dict ? dict->content_size : 0); - // We need to allocate a buffer to write to of size at least output + - // dict - // size - size_t size = ctx->frame_content_size + (dict ? dict->content_size : 0); - } - - // Allocate the window - if (ctx->window_size > MAX_WINDOW_SIZE) { - ERROR("Requested window size too large"); - } - cbuf_init(&ctx->window, ctx->window_size); + header->header_size = header_size; } /// A dictionary acts as initializing values for the frame context before @@ -552,7 +582,7 @@ static void frame_context_apply_dict(frame_context_t *ctx, dictionary_t *dict) { if (!dict || !dict->content) return; - if (ctx->dictionary_id == 0 && dict->dictionary_id != 0) { + if (ctx->header.dictionary_id == 0 && dict->dictionary_id != 0) { // The dictionary is unneeded, and shouldn't be used as it may interfere // with the default offset history return; @@ -560,7 +590,8 @@ static void frame_context_apply_dict(frame_context_t *ctx, dictionary_t *dict) { // If the dictionary id is 0, it doesn't matter if we provide the wrong raw // content dict, it won't change anything - if (ctx->dictionary_id != 0 && ctx->dictionary_id != dict->dictionary_id) { + if (ctx->header.dictionary_id != 0 && + ctx->header.dictionary_id != dict->dictionary_id) { ERROR("Wrong/no dictionary provided"); } @@ -575,8 +606,7 @@ static void frame_context_apply_dict(frame_context_t *ctx, dictionary_t *dict) { // be used in the table repeat modes if (dict->dictionary_id != 0) { // Deep copy the entropy tables so they can be freed independently of - // the - // dictionary struct + // the dictionary struct HUF_copy_dtable(&ctx->literals_dtable, &dict->literals_dtable); FSE_copy_dtable(&ctx->ll_dtable, &dict->ll_dtable); FSE_copy_dtable(&ctx->of_dtable, &dict->of_dtable); @@ -590,14 +620,14 @@ static void frame_context_apply_dict(frame_context_t *ctx, dictionary_t *dict) { /// Decompress the data from a frame block by block static void decompress_data(io_streams_t *streams, frame_context_t *ctx) { - u8 last_block = 0; + int last_block = 0; do { if (streams->src_len < 3) { INP_SIZE(); } // Parse the block header last_block = streams->src[0] & 1; - u8 block_type = (streams->src[0] >> 1) & 3; + int block_type = (streams->src[0] >> 1) & 3; size_t block_len = read_bits_LE(streams->src, 21, 3); streams->src += 3; @@ -648,6 +678,10 @@ static void decompress_data(io_streams_t *streams, frame_context_t *ctx) { // Compressed block, this is mode complex decompress_block(streams, ctx, block_len); break; + case 3: + // Reserved block type + CORRUPTION(); + break; } } while (!last_block); @@ -656,10 +690,9 @@ static void decompress_data(io_streams_t *streams, frame_context_t *ctx) { streams->dst += written; streams->dst_len -= written; - if (ctx->content_checksum_flag) { + if (ctx->header.content_checksum_flag) { // This program does not support checking the checksum, so skip over it - // if - // it's present + // if it's present if (streams->src_len < 4) { INP_SIZE(); } @@ -1312,6 +1345,126 @@ static size_t execute_sequences(io_streams_t *streams, frame_context_t *ctx, } /******* END SEQUENCE EXECUTION ***********************************************/ +/******* OUTPUT SIZE COUNTING *************************************************/ +size_t traverse_frame(frame_header_t *header, const u8 *src, size_t src_len); + +/// Get the decompressed size of an input stream so memory can be allocated in +/// advance. +/// This is more complex than the implementation in the reference +/// implementation, as this API allows for the decompression of multiple +/// concatenated frames. +size_t ZSTD_get_decompressed_size(const void *src, size_t src_len) { + const u8 *ip = (const u8 *) src; + size_t dst_size = 0; + + // Each frame header only gives us the size of its frame, so iterate over all + // frames + while (src_len > 0) { + if (src_len < 4) { + INP_SIZE(); + } + + u32 magic_number = read_bits_LE(ip, 32, 0); + + ip += 4; + src_len -= 4; + if (magic_number >= 0x184D2A50U && magic_number <= 0x184D2A5F) { + // skippable frame, this has no impact on output size + if (src_len < 4) { + INP_SIZE(); + } + size_t frame_size = read_bits_LE(ip, 32, 32); + + if (src_len < 4 + frame_size) { + INP_SIZE(); + } + + // skip over frame + ip += 4 + frame_size; + src_len -= 4 + frame_size; + } else if (magic_number == 0xFD2FB528U) { + // ZSTD frame + frame_header_t header; + parse_frame_header(&header, ip, src_len); + + if (header.frame_content_size == 0 && !header.single_segment_flag) { + // Content size not provided, we can't tell + return -1; + } + + dst_size += header.frame_content_size; + + // we need to traverse the frame to find when the next one starts + size_t traversed = traverse_frame(&header, ip, src_len); + ip += traversed; + src_len -= traversed; + } else { + // not a real frame + ERROR("Invalid magic number"); + } + } + + return dst_size; +} + +/// Iterate over each block in a frame to find the end of it, to get to the +/// start of the next frame +size_t traverse_frame(frame_header_t *header, const u8 *src, size_t src_len) { + const u8 *const src_beg = src; + const u8 *const src_end = src + src_len; + src += header->header_size; + src_len += header->header_size; + + int last_block = 0; + + do { + if (src + 3 > src_end) { + INP_SIZE(); + } + // Parse the block header + last_block = src[0] & 1; + int block_type = (src[0] >> 1) & 3; + size_t block_len = read_bits_LE(src, 21, 3); + + src += 3; + switch (block_type) { + case 0: // Raw block, block_len bytes + if (src + block_len > src_end) { + INP_SIZE(); + } + src += block_len; + break; + case 1: // RLE block, 1 byte + if (src + 1 > src_end) { + INP_SIZE(); + } + src++; + break; + case 2: // Compressed block, compressed size is block_len + if (src + block_len > src_end) { + INP_SIZE(); + } + src += block_len; + break; + case 3: + // Reserved block type + CORRUPTION(); + break; + } + } while (!last_block); + + if (header->content_checksum_flag) { + if (src + 4 > src_end) { + INP_SIZE(); + } + src += 4; + } + + return src - src_beg; +} + +/******* END OUTPUT SIZE COUNTING *********************************************/ + /******* DICTIONARY PARSING ***************************************************/ static void init_raw_content_dict(dictionary_t *dict, const u8 *src, size_t src_len); @@ -1952,8 +2105,8 @@ static void FSE_init_dtable(FSE_dtable *dtable, const i16 *norm_freqs, high_threshold); // Make sure we don't occupy a spot taken // by the low prob symbols // Note: no other collision checking is necessary as `step` is - // coprime to - // `size`, so the cycle will visit each position exactly once + // coprime to `size`, so the cycle will visit each position exactly + // once } } if (pos != 0) { @@ -1964,13 +2117,11 @@ static void FSE_init_dtable(FSE_dtable *dtable, const i16 *norm_freqs, for (int i = 0; i < size; i++) { u8 symbol = dtable->symbols[i]; u16 next_state_desc = state_desc[symbol]++; - // Fills in the table appropriately - // next_state_desc increases by symbol over time, decreasing number of - // bits + // Fills in the table appropriately next_state_desc increases by symbol + // over time, decreasing number of bits dtable->num_bits[i] = (u8)(accuracy_log - log2inf(next_state_desc)); // baseline increases until the bit threshold is passed, at which point - // it - // resets to 0 + // it resets to 0 dtable->new_state_base[i] = ((u16)next_state_desc << dtable->num_bits[i]) - size; } @@ -2057,8 +2208,7 @@ static void FSE_init_dtable_rle(FSE_dtable *dtable, u8 symb) { dtable->new_state_base = malloc(sizeof(u16)); // This setup will always have a state of 0, always return symbol `symb`, - // and - // never consume any bits + // and never consume any bits dtable->symbols[0] = symb; dtable->num_bits[0] = 0; dtable->new_state_base[0] = 0; diff --git a/contrib/educational_decoder/zstd_decompress.h b/contrib/educational_decoder/zstd_decompress.h index 3671678b1..3e1bc568f 100644 --- a/contrib/educational_decoder/zstd_decompress.h +++ b/contrib/educational_decoder/zstd_decompress.h @@ -3,4 +3,5 @@ size_t ZSTD_decompress(void *dst, size_t dst_len, const void *src, size_t ZSTD_decompress_with_dict(void *dst, size_t dst_len, const void *src, size_t src_len, const void *dict, size_t dict_len); +size_t ZSTD_get_decompressed_size(const void *src, size_t src_len); From f231626244f857bb18b3459f8d6ac143c6f65da6 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Mon, 30 Jan 2017 14:57:02 -0800 Subject: [PATCH 003/223] Minor fixes according to comments - Add Facebook copyright notice - Make max size macros more consistent - Fix some unchecked malloc's --- contrib/educational_decoder/README.md | 7 +++--- contrib/educational_decoder/harness.c | 9 +++++++ contrib/educational_decoder/zstd_decompress.c | 24 +++++++++++++++++-- contrib/educational_decoder/zstd_decompress.h | 9 +++++++ 4 files changed, 44 insertions(+), 5 deletions(-) diff --git a/contrib/educational_decoder/README.md b/contrib/educational_decoder/README.md index a1f703f62..2e2186e02 100644 --- a/contrib/educational_decoder/README.md +++ b/contrib/educational_decoder/README.md @@ -1,15 +1,16 @@ Educational Decoder =================== -`zstd_decompress.c` is a self-contained implementation of a decoder according -to the Zstandard format specification written in C99. +`zstd_decompress.c` is a self-contained implementation in C99 of a decoder, +according to the [Zstandard format specification]. While it does not implement as many features as the reference decoder, such as the streaming API or content checksums, it is written to be easy to follow and understand, to help understand how the Zstandard format works. It's laid out to match the [format specification], -so it can be used to understand how confusing segments could be implemented. +so it can be used to understand how complex segments could be implemented. It also contains implementations of Huffman and FSE table decoding. +[Zstandard format specification]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md [format specification]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md `harness.c` provides a simple test harness around the decoder: diff --git a/contrib/educational_decoder/harness.c b/contrib/educational_decoder/harness.c index c44100fff..42424d4bd 100644 --- a/contrib/educational_decoder/harness.c +++ b/contrib/educational_decoder/harness.c @@ -1,3 +1,12 @@ +/* + * Copyright (c) 2017-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + #include #include diff --git a/contrib/educational_decoder/zstd_decompress.c b/contrib/educational_decoder/zstd_decompress.c index 7b04c4b29..79fd26853 100644 --- a/contrib/educational_decoder/zstd_decompress.c +++ b/contrib/educational_decoder/zstd_decompress.c @@ -1,3 +1,12 @@ +/* + * Copyright (c) 2017-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + /// Zstandard educational decoder implementation /// See https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md @@ -21,10 +30,13 @@ size_t ZSTD_decompress_with_dict(void *dst, size_t dst_len, const void *src, size_t ZSTD_get_decompressed_size(const void *src, size_t src_len); /******* UTILITY MACROS AND TYPES *********************************************/ -#define MAX_WINDOW_SIZE ((size_t)512 << 20) +// Specification recommends supporting at least 8MB. The maximum possible value +// is 1.875TB, but this implementation limits it to 512MB to avoid allocating +// too much memory. +#define MAX_WINDOW_SIZE ((size_t)512 * 1024 * 1024) // Max block size decompressed size is 128 KB and literal blocks must be smaller // than that -#define MAX_LITERALS_SIZE ((size_t)(1024 * 128)) +#define MAX_LITERALS_SIZE ((size_t)128 * 1024) #define MAX(a, b) ((a) > (b) ? (a) : (b)) #define MIN(a, b) ((a) < (b) ? (a) : (b)) @@ -2071,6 +2083,10 @@ static void FSE_init_dtable(FSE_dtable *dtable, const i16 *norm_freqs, dtable->num_bits = malloc(size * sizeof(u8)); dtable->new_state_base = malloc(size * sizeof(u16)); + if (!dtable->symbols || !dtable->num_bits || !dtable->new_state_base) { + BAD_ALLOC(); + } + // Used to determine how many bits need to be read for each state, // and where the destination range should start // Needs to be u16 because max value is 2 * max number of symbols, @@ -2207,6 +2223,10 @@ static void FSE_init_dtable_rle(FSE_dtable *dtable, u8 symb) { dtable->num_bits = malloc(sizeof(u8)); dtable->new_state_base = malloc(sizeof(u16)); + if (!dtable->symbols || !dtable->num_bits || !dtable->new_state_base) { + BAD_ALLOC(); + } + // This setup will always have a state of 0, always return symbol `symb`, // and never consume any bits dtable->symbols[0] = symb; diff --git a/contrib/educational_decoder/zstd_decompress.h b/contrib/educational_decoder/zstd_decompress.h index 3e1bc568f..6e1736720 100644 --- a/contrib/educational_decoder/zstd_decompress.h +++ b/contrib/educational_decoder/zstd_decompress.h @@ -1,3 +1,12 @@ +/* + * Copyright (c) 2017-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + size_t ZSTD_decompress(void *dst, size_t dst_len, const void *src, size_t src_len); size_t ZSTD_decompress_with_dict(void *dst, size_t dst_len, const void *src, From f5d2f32d4dd34a88dcf68cc8929e3063d591654b Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Tue, 31 Jan 2017 15:54:02 -0800 Subject: [PATCH 004/223] Removed circular buffer, matches access destination buffer directly --- contrib/educational_decoder/zstd_decompress.c | 196 +++++++----------- 1 file changed, 75 insertions(+), 121 deletions(-) diff --git a/contrib/educational_decoder/zstd_decompress.c b/contrib/educational_decoder/zstd_decompress.c index 79fd26853..90d4a5229 100644 --- a/contrib/educational_decoder/zstd_decompress.c +++ b/contrib/educational_decoder/zstd_decompress.c @@ -30,10 +30,6 @@ size_t ZSTD_decompress_with_dict(void *dst, size_t dst_len, const void *src, size_t ZSTD_get_decompressed_size(const void *src, size_t src_len); /******* UTILITY MACROS AND TYPES *********************************************/ -// Specification recommends supporting at least 8MB. The maximum possible value -// is 1.875TB, but this implementation limits it to 512MB to avoid allocating -// too much memory. -#define MAX_WINDOW_SIZE ((size_t)512 * 1024 * 1024) // Max block size decompressed size is 128 KB and literal blocks must be smaller // than that #define MAX_LITERALS_SIZE ((size_t)128 * 1024) @@ -69,43 +65,6 @@ typedef int64_t i64; /// file. They implement low-level functionality needed for the higher level /// decompression functions. -/*** CIRCULAR BUFFER ******************/ -/// A standard circular buffer, used to facilitate back reference commands -typedef struct { - u8 *ptr; - size_t idx, last_flush, size; -} cbuf_t; - -/// Initialize a circular buffer -static void cbuf_init(cbuf_t *buf, size_t size); -static void cbuf_free(cbuf_t *buf); - -/// Copies up to `src_len` bytes from `src` into the buffer, stopping if it -/// would need to flush. -/// Returns the total amount of data copied. -static size_t cbuf_write_data(cbuf_t *buf, const u8 *src, size_t src_len); -/// Copies `len` bytes from `offset` back in the buffer, stopping if it would -/// need to flush. -/// Returns the number of bytes copied. -static size_t cbuf_copy_offset(cbuf_t *buf, size_t offset, size_t len); -/// Writes up to `len` copies of `byte`, stopping if would need to flush. -/// Returns the number of bytes copied. -static size_t cbuf_repeat_byte(cbuf_t *buf, u8 byte, size_t len); - -/// The `full` versions of the above functions write the full amount requested, -/// flushing to `out` when necessary. -/// They return the number of bytes flushed to `out`, if any. -static size_t cbuf_write_data_full(cbuf_t *buf, const u8 *src, size_t src_len, - u8 *out, size_t out_len); -static size_t cbuf_copy_offset_full(cbuf_t *buf, size_t offset, size_t len, - u8 *out, size_t out_len); -static size_t cbuf_repeat_byte_full(cbuf_t *buf, u8 byte, size_t len, u8 *out, - size_t out_len); - -/// Flushes any unflushed data to `dst` -static size_t cbuf_flush(cbuf_t *buf, u8 *dst, size_t dst_len); -/*** END CIRCULAR BUFFER **************/ - /*** BITSTREAM OPERATIONS *************/ /// Read `num` bits (up to 64) from `src + offset`, where `offset` is in bits static inline u64 read_bits_LE(const u8 *src, int num, size_t offset); @@ -277,12 +236,11 @@ typedef struct { // offset too large to be correct size_t current_total_output; - // A sliding window of the past `window_size` bytes decoded - cbuf_t window; + const u8 *dict_content; + size_t dict_content_len; // Entropy encoding tables so they can be repeated by future blocks instead - // of - // retransmitting + // of retransmitting HUF_dtable literals_dtable; FSE_dtable ll_dtable; FSE_dtable ml_dtable; @@ -470,22 +428,6 @@ static void init_frame_context(io_streams_t *streams, frame_context_t *context, context->previous_offsets[2] = 4; context->previous_offsets[3] = 8; - { - // Allocate the window buffer - size_t buffer_size; - if (context->header.single_segment_flag) { - buffer_size = context->header.frame_content_size + - (dict ? dict->content_size : 0); - } else { - buffer_size = context->header.window_size; - } - - if (buffer_size > MAX_WINDOW_SIZE) { - ERROR("Requested window size too large"); - } - cbuf_init(&context->window, buffer_size); - } - // Apply details from the dict if it exists frame_context_apply_dict(context, dict); } @@ -497,8 +439,6 @@ static void free_frame_context(frame_context_t *context) { FSE_free_dtable(&context->ml_dtable); FSE_free_dtable(&context->of_dtable); - cbuf_free(&context->window); - memset(context, 0, sizeof(frame_context_t)); } @@ -583,6 +523,13 @@ static void parse_frame_header(frame_header_t *header, const u8 *src, header->frame_content_size = 0; } + if (single_segment_flag) { + // in this case the effective window size is frame_content_size this + // impacts sequence decoding as we need to determine whether to fall + // back to the dictionary or not on large offsets + header->window_size = header->frame_content_size; + } + header->header_size = header_size; } @@ -607,12 +554,9 @@ static void frame_context_apply_dict(frame_context_t *ctx, dictionary_t *dict) { ERROR("Wrong/no dictionary provided"); } - // Write the dict data in, and then flush to NULL so it's not sent to the - // output stream - cbuf_write_data_full(&ctx->window, dict->content, dict->content_size, NULL, - -1); - cbuf_flush(&ctx->window, NULL, -1); - ctx->current_total_output = dict->content_size; + // Copy the pointer in so we can reference it in sequence execution + ctx->dict_content = dict->content; + ctx->dict_content_len = dict->content_size; // If it's a formatted dict copy the precomputed tables in so they can // be used in the table repeat modes @@ -655,15 +599,16 @@ static void decompress_data(io_streams_t *streams, frame_context_t *ctx) { OUT_SIZE(); } - // Write the raw data into the window buffer - size_t written = - cbuf_write_data_full(&ctx->window, streams->src, block_len, - streams->dst, streams->dst_len); + // Copy the raw data into the output + memcpy(streams->dst, streams->src, block_len); + streams->src += block_len; streams->src_len -= block_len; - streams->dst += written; - streams->dst_len -= written; + streams->dst += block_len; + streams->dst_len -= block_len; + + ctx->current_total_output += block_len; break; } case 1: { @@ -675,15 +620,16 @@ static void decompress_data(io_streams_t *streams, frame_context_t *ctx) { OUT_SIZE(); } - // Write streams->src[0] into the buffer block_len times - size_t written = - cbuf_repeat_byte_full(&ctx->window, streams->src[0], block_len, - streams->dst, streams->dst_len); - streams->dst += written; - streams->dst_len -= written; + // Copy `block_len` copies of `streams->src[0]` to the output + memset(streams->dst, streams->src[0], block_len); + + streams->dst += block_len; + streams->dst_len -= block_len; streams->src += 1; streams->src_len -= 1; + + ctx->current_total_output += block_len; break; } case 2: @@ -697,11 +643,6 @@ static void decompress_data(io_streams_t *streams, frame_context_t *ctx) { } } while (!last_block); - // Flush out anything left in the window buffer to the destination stream - size_t written = cbuf_flush(&ctx->window, streams->dst, streams->dst_len); - streams->dst += written; - streams->dst_len -= written; - if (ctx->header.content_checksum_flag) { // This program does not support checking the checksum, so skip over it // if it's present @@ -1277,20 +1218,19 @@ static size_t execute_sequences(io_streams_t *streams, frame_context_t *ctx, CORRUPTION(); } - { - // Copy literals to the buffer - size_t written = - cbuf_write_data_full(&ctx->window, literals, seq.literal_length, - streams->dst, streams->dst_len); - - literals += seq.literal_length; - literals_len -= seq.literal_length; - - streams->dst += written; - streams->dst_len -= written; - - total_output += seq.literal_length; + if (streams->dst_len < seq.literal_length + seq.match_length) { + OUT_SIZE(); } + // Copy literals to output + memcpy(streams->dst, literals, seq.literal_length); + + literals += seq.literal_length; + literals_len -= seq.literal_length; + + streams->dst += seq.literal_length; + streams->dst_len -= seq.literal_length; + + total_output += seq.literal_length; size_t offset; @@ -1324,36 +1264,50 @@ static size_t execute_sequences(io_streams_t *streams, frame_context_t *ctx, offset_hist[1] = offset; } - if (offset > total_output) { - CORRUPTION(); + size_t match_length = seq.match_length; + if (total_output <= ctx->header.window_size) { + // In this case offset might go back into the dictionary + if (offset > total_output + ctx->dict_content_len) { + // The offset goes beyond even the dictionary + CORRUPTION(); + } + + if (offset > total_output) { + const size_t dict_copy = + MIN(offset - total_output, match_length); + const size_t dict_offset = + ctx->dict_content_len - (offset - total_output); + for (size_t i = 0; i < dict_copy; i++) { + *streams->dst++ = ctx->dict_content[dict_offset + i]; + } + match_length -= dict_copy; + } } - { - // Do the offset copy operation - size_t written = - cbuf_copy_offset_full(&ctx->window, offset, seq.match_length, - streams->dst, streams->dst_len); - - streams->dst += written; - streams->dst_len -= written; - total_output += seq.match_length; + // We must copy byte by byte because the match length might be larger + // than the offset + // ex: if the output so far was "abc", a command with offset=3 and + // match_length=6 would produce "abcabcabc" as the new output + for (size_t i = 0; i < match_length; i++) { + *streams->dst = *(streams->dst - offset); + streams->dst++; } + + streams->dst_len -= seq.match_length; + total_output += seq.match_length; } - { - // Copy any leftover literal bytes - size_t written = - cbuf_write_data_full(&ctx->window, literals, literals_len, - streams->dst, streams->dst_len); - streams->dst += written; - streams->dst_len -= written; - - total_output += literals_len; + if (streams->dst_len < literals_len) { + OUT_SIZE(); } + // Copy any leftover literals + memcpy(streams->dst, literals, literals_len); + streams->dst += literals_len; + streams->dst_len -= literals_len; + + total_output += literals_len; ctx->current_total_output = total_output; - - return total_output; } /******* END SEQUENCE EXECUTION ***********************************************/ From 92ec2ea62f34870d0f9900af68eb816845a3494c Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Tue, 31 Jan 2017 15:57:18 -0800 Subject: [PATCH 005/223] More const's and readability improvements --- contrib/educational_decoder/harness.c | 2 +- contrib/educational_decoder/zstd_decompress.c | 860 ++++++++---------- contrib/educational_decoder/zstd_decompress.h | 12 +- 3 files changed, 410 insertions(+), 464 deletions(-) diff --git a/contrib/educational_decoder/harness.c b/contrib/educational_decoder/harness.c index 42424d4bd..107a16a22 100644 --- a/contrib/educational_decoder/harness.c +++ b/contrib/educational_decoder/harness.c @@ -88,7 +88,7 @@ int main(int argc, char **argv) { decompressed_size = MAX_COMPRESSION_RATIO * input_size; fprintf(stderr, "WARNING: Compressed data does contain decompressed " "size, going to assume the compression ratio is at " - "most %d (decompressed size of at most %lld\n", + "most %d (decompressed size of at most %zu)\n", MAX_COMPRESSION_RATIO, decompressed_size); } output = malloc(decompressed_size); diff --git a/contrib/educational_decoder/zstd_decompress.c b/contrib/educational_decoder/zstd_decompress.c index 90d4a5229..3c1c56730 100644 --- a/contrib/educational_decoder/zstd_decompress.c +++ b/contrib/educational_decoder/zstd_decompress.c @@ -17,17 +17,17 @@ /// Zstandard decompression functions. /// `dst` must point to a space at least as large as the reconstructed output. -size_t ZSTD_decompress(void *dst, size_t dst_len, const void *src, - size_t src_len); +size_t ZSTD_decompress(void *const dst, const size_t dst_len, + const void *const src, const size_t src_len); /// If `dict != NULL` and `dict_len >= 8`, does the same thing as /// `ZSTD_decompress` but uses the provided dict -size_t ZSTD_decompress_with_dict(void *dst, size_t dst_len, const void *src, - size_t src_len, const void *dict, - size_t dict_len); +size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len, + const void *const src, const size_t src_len, + const void *const dict, const size_t dict_len); /// Get the decompressed size of an input stream so memory can be allocated in /// advance -size_t ZSTD_get_decompressed_size(const void *src, size_t src_len); +size_t ZSTD_get_decompressed_size(const void *const src, const size_t src_len); /******* UTILITY MACROS AND TYPES *********************************************/ // Max block size decompressed size is 128 KB and literal blocks must be smaller @@ -67,23 +67,21 @@ typedef int64_t i64; /*** BITSTREAM OPERATIONS *************/ /// Read `num` bits (up to 64) from `src + offset`, where `offset` is in bits -static inline u64 read_bits_LE(const u8 *src, int num, size_t offset); +static inline u64 read_bits_LE(const u8 *src, const int num, + const size_t offset); /// Read bits from the end of a HUF or FSE bitstream. `offset` is in bits, so /// it updates `offset` to `offset - bits`, and then reads `bits` bits from /// `src + offset`. If the offset becomes negative, the extra bits at the /// bottom are filled in with `0` bits instead of reading from before `src`. -static inline u64 STREAM_read_bits(const u8 *src, int bits, i64 *offset); +static inline u64 STREAM_read_bits(const u8 *src, const int bits, + i64 *const offset); /*** END BITSTREAM OPERATIONS *********/ /*** BIT COUNTING OPERATIONS **********/ -/// Returns `x`, where `2^x` is the smallest power of 2 greater than or equal to -/// `num`, or `-1` if `num > 2^63` -static inline int log2sup(u64 num); - /// Returns `x`, where `2^x` is the largest power of 2 less than or equal to /// `num`, or `-1` if `num == 0`. -static inline int log2inf(u64 num); +static inline int log2inf(const u64 num); /*** END BIT COUNTING OPERATIONS ******/ /*** HUFFMAN PRIMITIVES ***************/ @@ -101,36 +99,41 @@ typedef struct { } HUF_dtable; /// Decode a single symbol and read in enough bits to refresh the state -static inline u8 HUF_decode_symbol(HUF_dtable *dtable, u16 *state, - const u8 *src, i64 *offset); +static inline u8 HUF_decode_symbol(const HUF_dtable *const dtable, + u16 *const state, const u8 *const src, + i64 *const offset); /// Read in a full state's worth of bits to initialize it -static inline void HUF_init_state(HUF_dtable *dtable, u16 *state, const u8 *src, - i64 *offset); - -/// Initialize a Huffman decoding table using the table of bit counts provided -static void HUF_init_dtable(HUF_dtable *table, u8 *bits, int num_symbs); -/// Initialize a Huffman decoding table using the table of weights provided -/// Weights follow the definition provided in the Zstandard specification -static void HUF_init_dtable_usingweights(HUF_dtable *table, u8 *weights, - int num_symbs); +static inline void HUF_init_state(const HUF_dtable *const dtable, + u16 *const state, const u8 *const src, + i64 *const offset); /// Decompresses a single Huffman stream, returns the number of bytes decoded. /// `src_len` must be the exact length of the Huffman-coded block. -static size_t HUF_decompress_1stream(HUF_dtable *table, u8 *dst, size_t dst_len, - const u8 *src, size_t src_len); +static size_t HUF_decompress_1stream(const HUF_dtable *const dtable, u8 *dst, + const size_t dst_len, const u8 *src, + size_t src_len); /// Same as previous but decodes 4 streams, formatted as in the Zstandard /// specification. /// `src_len` must be the exact length of the Huffman-coded block. -static size_t HUF_decompress_4stream(HUF_dtable *dtable, u8 *dst, - size_t dst_len, const u8 *src, - size_t src_len); +static size_t HUF_decompress_4stream(const HUF_dtable *const dtable, u8 *dst, + const size_t dst_len, const u8 *const src, + const size_t src_len); + +/// Initialize a Huffman decoding table using the table of bit counts provided +static void HUF_init_dtable(HUF_dtable *const table, const u8 *const bits, + const int num_symbs); +/// Initialize a Huffman decoding table using the table of weights provided +/// Weights follow the definition provided in the Zstandard specification +static void HUF_init_dtable_usingweights(HUF_dtable *const table, + const u8 *const weights, + const int num_symbs); /// Free the malloc'ed parts of a decoding table -static void HUF_free_dtable(HUF_dtable *dtable); +static void HUF_free_dtable(HUF_dtable *const dtable); /// Deep copy a decoding table, so that it can be used and free'd without /// impacting the source table. -static void HUF_copy_dtable(HUF_dtable *dst, const HUF_dtable *src); +static void HUF_copy_dtable(HUF_dtable *const dst, const HUF_dtable *const src); /*** END HUFFMAN PRIMITIVES ***********/ /*** FSE PRIMITIVES *******************/ @@ -151,46 +154,53 @@ typedef struct { } FSE_dtable; /// Return the symbol for the current state -static inline u8 FSE_peek_symbol(FSE_dtable *dtable, u16 state); +static inline u8 FSE_peek_symbol(const FSE_dtable *const dtable, + const u16 state); /// Read the number of bits necessary to update state, update, and shift offset /// back to reflect the bits read -static inline void FSE_update_state(FSE_dtable *dtable, u16 *state, - const u8 *src, i64 *offset); +static inline void FSE_update_state(const FSE_dtable *const dtable, + u16 *const state, const u8 *const src, + i64 *const offset); /// Combine peek and update: decode a symbol and update the state -static inline u8 FSE_decode_symbol(FSE_dtable *dtable, u16 *state, - const u8 *src, i64 *offset); +static inline u8 FSE_decode_symbol(const FSE_dtable *const dtable, + u16 *const state, const u8 *const src, + i64 *const offset); /// Read bits from the stream to initialize the state and shift offset back -static inline void FSE_init_state(FSE_dtable *dtable, u16 *state, const u8 *src, - i64 *offset); +static inline void FSE_init_state(const FSE_dtable *const dtable, + u16 *const state, const u8 *const src, + i64 *const offset); /// Decompress two interleaved bitstreams (e.g. compressed Huffman weights) /// using an FSE decoding table. `src_len` must be the exact length of the /// block. -static size_t FSE_decompress_interleaved2(FSE_dtable *dtable, u8 *dst, - size_t dst_len, const u8 *src, - size_t src_len); +static size_t FSE_decompress_interleaved2(const FSE_dtable *const dtable, + u8 *dst, const size_t dst_len, + const u8 *const src, + const size_t src_len); /// Initialize a decoding table using normalized frequencies. -static void FSE_init_dtable(FSE_dtable *dtable, const i16 *norm_freqs, - int num_symbs, int accuracy_log); +static void FSE_init_dtable(FSE_dtable *const dtable, + const i16 *const norm_freqs, const int num_symbs, + const int accuracy_log); /// Decode an FSE header as defined in the Zstandard format specification and /// use the decoded frequencies to initialize a decoding table. -static size_t FSE_decode_header(FSE_dtable *dtable, const u8 *src, - size_t src_len, int max_accuracy_log); +static size_t FSE_decode_header(FSE_dtable *const dtable, const u8 *const src, + const size_t src_len, + const int max_accuracy_log); /// Initialize an FSE table that will always return the same symbol and consume /// 0 bits per symbol, to be used for RLE mode in sequence commands -static void FSE_init_dtable_rle(FSE_dtable *dtable, u8 symb); +static void FSE_init_dtable_rle(FSE_dtable *const dtable, const u8 symb); /// Free the malloc'ed parts of a decoding table -static void FSE_free_dtable(FSE_dtable *dtable); +static void FSE_free_dtable(FSE_dtable *const dtable); /// Deep copy a decoding table, so that it can be used and free'd without /// impacting the source table. -static void FSE_copy_dtable(FSE_dtable *dst, const FSE_dtable *src); +static void FSE_copy_dtable(FSE_dtable *const dst, const FSE_dtable *const src); /*** END FSE PRIMITIVES ***************/ /******* END IMPLEMENTATION PRIMITIVE PROTOTYPES ******************************/ @@ -291,47 +301,46 @@ typedef struct { /// Accepts a dict argument, which may be NULL indicating no dictionary. /// See /// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#frame-concatenation -static void decode_frame(io_streams_t *streams, dictionary_t *dict); +static void decode_frame(io_streams_t *const streams, + const dictionary_t *const dict); // Decode data in a compressed block -static void decompress_block(io_streams_t *streams, frame_context_t *ctx, - size_t block_len); +static void decompress_block(io_streams_t *const streams, + frame_context_t *const ctx, + const size_t block_len); // Decode the literals section of a block -static size_t decode_literals(io_streams_t *streams, frame_context_t *ctx, - u8 **literals); +static size_t decode_literals(io_streams_t *const streams, + frame_context_t *const ctx, u8 **const literals); // Decode the sequences part of a block -static size_t decode_sequences(frame_context_t *ctx, const u8 *src, - size_t src_len, sequence_command_t **sequences); +static size_t decode_sequences(frame_context_t *const ctx, const u8 *const src, + const size_t src_len, + sequence_command_t **const sequences); // Execute the decoded sequences on the literals block -static size_t execute_sequences(io_streams_t *streams, frame_context_t *ctx, - sequence_command_t *sequences, - size_t num_sequences, const u8 *literals, - size_t literals_len); +static void execute_sequences(io_streams_t *const streams, + frame_context_t *const ctx, + const sequence_command_t *const sequences, + const size_t num_sequences, + const u8 *literals, + size_t literals_len); // Parse a provided dictionary blob for use in decompression -static void parse_dictionary(dictionary_t *dict, const u8 *src, size_t src_len); -static void free_dictionary(dictionary_t *dict); +static void parse_dictionary(dictionary_t *const dict, const u8 *const src, + const size_t src_len); +static void free_dictionary(dictionary_t *const dict); /******* END ZSTD HELPER STRUCTS AND PROTOTYPES *******************************/ -size_t ZSTD_decompress(void *dst, size_t dst_len, const void *src, - size_t src_len) { +size_t ZSTD_decompress(void *const dst, const size_t dst_len, + const void *const src, const size_t src_len) { return ZSTD_decompress_with_dict(dst, dst_len, src, src_len, NULL, 0); } -size_t ZSTD_decompress_usingDict(void *_ctx, void *dst, size_t dst_len, - const void *src, size_t src_len, - const void *dict, size_t dict_len) { - // _ctx needed to match ZSTD lib signature - return ZSTD_decompress_with_dict(dst, dst_len, src, src_len, dict, - dict_len); -} - -size_t ZSTD_decompress_with_dict(void *dst, size_t dst_len, const void *src, - size_t src_len, const void *dict, - size_t dict_len) { +size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len, + const void *const src, const size_t src_len, + const void *const dict, + const size_t dict_len) { dictionary_t parsed_dict; memset(&parsed_dict, 0, sizeof(dictionary_t)); // dict_len < 8 is not a valid dictionary @@ -351,21 +360,26 @@ size_t ZSTD_decompress_with_dict(void *dst, size_t dst_len, const void *src, /******* FRAME DECODING ******************************************************/ -static void decode_data_frame(io_streams_t *streams, dictionary_t *dict); -static void init_frame_context(io_streams_t *streams, frame_context_t *context, - dictionary_t *dict); -static void free_frame_context(frame_context_t *context); -static void parse_frame_header(frame_header_t *header, const u8 *src, - size_t src_len); -static void frame_context_apply_dict(frame_context_t *ctx, dictionary_t *dict); +static void decode_data_frame(io_streams_t *const streams, + const dictionary_t *const dict); +static void init_frame_context(io_streams_t *const streams, + frame_context_t *const context, + const dictionary_t *const dict); +static void free_frame_context(frame_context_t *const context); +static void parse_frame_header(frame_header_t *const header, + const u8 *const src, const size_t src_len); +static void frame_context_apply_dict(frame_context_t *const ctx, + const dictionary_t *const dict); -static void decompress_data(io_streams_t *streams, frame_context_t *ctx); +static void decompress_data(io_streams_t *const streams, + frame_context_t *const ctx); -static void decode_frame(io_streams_t *streams, dictionary_t *dict) { +static void decode_frame(io_streams_t *const streams, + const dictionary_t *const dict) { if (streams->src_len < 4) { INP_SIZE(); } - u32 magic_number = read_bits_LE(streams->src, 32, 0); + const u32 magic_number = read_bits_LE(streams->src, 32, 0); streams->src += 4; streams->src_len -= 4; @@ -374,7 +388,7 @@ static void decode_frame(io_streams_t *streams, dictionary_t *dict) { if (streams->src_len < 4) { INP_SIZE(); } - size_t frame_size = read_bits_LE(streams->src, 32, 32); + const size_t frame_size = read_bits_LE(streams->src, 32, 32); if (streams->src_len < 4 + frame_size) { INP_SIZE(); @@ -396,7 +410,8 @@ static void decode_frame(io_streams_t *streams, dictionary_t *dict) { /// are skippable frames. /// See /// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#general-structure-of-zstandard-frame-format -static void decode_data_frame(io_streams_t *streams, dictionary_t *dict) { +static void decode_data_frame(io_streams_t *const streams, + const dictionary_t *const dict) { frame_context_t ctx; // Initialize the context that needs to be carried from block to block @@ -414,8 +429,10 @@ static void decode_data_frame(io_streams_t *streams, dictionary_t *dict) { /// Takes the information provided in the header and dictionary, and initializes /// the context for this frame -static void init_frame_context(io_streams_t *streams, frame_context_t *context, - dictionary_t *dict) { +static void init_frame_context(io_streams_t *const streams, + frame_context_t *const context, + const dictionary_t *const dict) { + // Most fields in context are correct when initialized to 0 memset(context, 0x00, sizeof(frame_context_t)); // Parse data from the frame header @@ -432,7 +449,7 @@ static void init_frame_context(io_streams_t *streams, frame_context_t *context, frame_context_apply_dict(context, dict); } -static void free_frame_context(frame_context_t *context) { +static void free_frame_context(frame_context_t *const context) { HUF_free_dtable(&context->literals_dtable); FSE_free_dtable(&context->ll_dtable); @@ -442,20 +459,20 @@ static void free_frame_context(frame_context_t *context) { memset(context, 0, sizeof(frame_context_t)); } -static void parse_frame_header(frame_header_t *header, const u8 *src, - size_t src_len) { +static void parse_frame_header(frame_header_t *const header, + const u8 *const src, const size_t src_len) { if (src_len < 1) { INP_SIZE(); } - u8 descriptor = read_bits_LE(src, 8, 0); + const u8 descriptor = read_bits_LE(src, 8, 0); // decode frame header descriptor into flags - u8 frame_content_size_flag = descriptor >> 6; - u8 single_segment_flag = (descriptor >> 5) & 1; - u8 reserved_bit = (descriptor >> 3) & 1; - u8 content_checksum_flag = (descriptor >> 2) & 1; - u8 dictionary_id_flag = descriptor & 3; + const u8 frame_content_size_flag = descriptor >> 6; + const u8 single_segment_flag = (descriptor >> 5) & 1; + const u8 reserved_bit = (descriptor >> 3) & 1; + const u8 content_checksum_flag = (descriptor >> 2) & 1; + const u8 dictionary_id_flag = descriptor & 3; if (reserved_bit != 0) { CORRUPTION(); @@ -536,7 +553,8 @@ static void parse_frame_header(frame_header_t *header, const u8 *src, /// A dictionary acts as initializing values for the frame context before /// decompression, so we implement it by applying it's predetermined /// tables and content to the context before beginning decompression -static void frame_context_apply_dict(frame_context_t *ctx, dictionary_t *dict) { +static void frame_context_apply_dict(frame_context_t *const ctx, + const dictionary_t *const dict) { // If the content pointer is NULL then it must be an empty dict if (!dict || !dict->content) return; @@ -574,8 +592,8 @@ static void frame_context_apply_dict(frame_context_t *ctx, dictionary_t *dict) { } /// Decompress the data from a frame block by block -static void decompress_data(io_streams_t *streams, frame_context_t *ctx) { - +static void decompress_data(io_streams_t *const streams, + frame_context_t *const ctx) { int last_block = 0; do { if (streams->src_len < 3) { @@ -583,8 +601,8 @@ static void decompress_data(io_streams_t *streams, frame_context_t *ctx) { } // Parse the block header last_block = streams->src[0] & 1; - int block_type = (streams->src[0] >> 1) & 3; - size_t block_len = read_bits_LE(streams->src, 21, 3); + const int block_type = (streams->src[0] >> 1) & 3; + const size_t block_len = read_bits_LE(streams->src, 21, 3); streams->src += 3; streams->src_len -= 3; @@ -656,8 +674,8 @@ static void decompress_data(io_streams_t *streams, frame_context_t *ctx) { /******* END FRAME DECODING ***************************************************/ /******* BLOCK DECOMPRESSION **************************************************/ -static void decompress_block(io_streams_t *streams, frame_context_t *ctx, - size_t block_len) { +static void decompress_block(io_streams_t *const streams, frame_context_t *const ctx, + const size_t block_len) { if (streams->src_len < block_len) { INP_SIZE(); } @@ -666,15 +684,15 @@ static void decompress_block(io_streams_t *streams, frame_context_t *ctx, // Part 1: decode the literals block u8 *literals = NULL; - size_t literals_size = decode_literals(streams, ctx, &literals); + const size_t literals_size = decode_literals(streams, ctx, &literals); // Part 2: decode the sequences block if (streams->src > end_of_block) { INP_SIZE(); } - size_t sequences_size = end_of_block - streams->src; + const size_t sequences_size = end_of_block - streams->src; sequence_command_t *sequences = NULL; - size_t num_sequences = + const size_t num_sequences = decode_sequences(ctx, streams->src, sequences_size, &sequences); streams->src += sequences_size; @@ -689,18 +707,22 @@ static void decompress_block(io_streams_t *streams, frame_context_t *ctx, /******* END BLOCK DECOMPRESSION **********************************************/ /******* LITERALS DECODING ****************************************************/ -static size_t decode_literals_simple(io_streams_t *streams, u8 **literals, - int block_type, int size_format); -static size_t decode_literals_compressed(io_streams_t *streams, - frame_context_t *ctx, u8 **literals, - int block_type, int size_format); +static size_t decode_literals_simple(io_streams_t *const streams, + u8 **const literals, const int block_type, + const int size_format); +static size_t decode_literals_compressed(io_streams_t *const streams, + frame_context_t *const ctx, + u8 **const literals, + const int block_type, + const int size_format); static size_t decode_huf_table(const u8 *src, size_t src_len, - HUF_dtable *dtable); -static size_t fse_decode_hufweights(const u8 *src, size_t src_len, u8 *weights, - int *num_symbs, size_t compressed_size); + HUF_dtable *const dtable); +static size_t fse_decode_hufweights(const u8 *const src, const size_t src_len, + u8 *const weights, int *const num_symbs, + const size_t compressed_size); -static size_t decode_literals(io_streams_t *streams, frame_context_t *ctx, - u8 **literals) { +static size_t decode_literals(io_streams_t *const streams, + frame_context_t *const ctx, u8 **const literals) { if (streams->src_len < 1) { INP_SIZE(); } @@ -720,8 +742,9 @@ static size_t decode_literals(io_streams_t *streams, frame_context_t *ctx, } /// Decodes literals blocks in raw or RLE form -static size_t decode_literals_simple(io_streams_t *streams, u8 **literals, - int block_type, int size_format) { +static size_t decode_literals_simple(io_streams_t *const streams, + u8 **const literals, const int block_type, + const int size_format) { size_t size; switch (size_format) { // These cases are in the form X0 @@ -787,9 +810,11 @@ static size_t decode_literals_simple(io_streams_t *streams, u8 **literals, } /// Decodes Huffman compressed literals -static size_t decode_literals_compressed(io_streams_t *streams, - frame_context_t *ctx, u8 **literals, - int block_type, int size_format) { +static size_t decode_literals_compressed(io_streams_t *const streams, + frame_context_t *const ctx, + u8 **const literals, + const int block_type, + const int size_format) { size_t regenerated_size, compressed_size; // Only size_format=0 has 1 stream, so default to 4 int num_streams = 4; @@ -846,8 +871,8 @@ static size_t decode_literals_compressed(io_streams_t *streams, // Decode provided Huffman table HUF_free_dtable(&ctx->literals_dtable); - size_t size = decode_huf_table(streams->src, compressed_size, - &ctx->literals_dtable); + const size_t size = decode_huf_table(streams->src, compressed_size, + &ctx->literals_dtable); streams->src += size; streams->src_len -= size; compressed_size -= size; @@ -873,14 +898,14 @@ static size_t decode_literals_compressed(io_streams_t *streams, // Decode the Huffman table description static size_t decode_huf_table(const u8 *src, size_t src_len, - HUF_dtable *dtable) { + HUF_dtable *const dtable) { if (src_len < 1) { INP_SIZE(); } const u8 *const osrc = src; - u8 header = src[0]; + const u8 header = src[0]; u8 weights[HUF_MAX_SYMBS]; memset(weights, 0, sizeof(weights)); @@ -892,13 +917,16 @@ static size_t decode_huf_table(const u8 *src, size_t src_len, if (header >= 128) { // Direct representation, read the weights out num_symbs = header - 127; - size_t bytes = (num_symbs + 1) / 2; + const size_t bytes = (num_symbs + 1) / 2; if (bytes > src_len) { INP_SIZE(); } for (int i = 0; i < num_symbs; i++) { + // read_bits_LE isn't applicable here because the weights are order + // reversed within each byte + // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#huffman-tree-header if (i % 2 == 0) { weights[i] = src[i / 2] >> 4; } else { @@ -911,7 +939,7 @@ static size_t decode_huf_table(const u8 *src, size_t src_len, } else { // The weights are FSE encoded, decode them before we can construct the // table - size_t size = + const size_t size = fse_decode_hufweights(src, src_len, weights, &num_symbs, header); src += size; src_len -= size; @@ -922,14 +950,16 @@ static size_t decode_huf_table(const u8 *src, size_t src_len, return src - osrc; } -static size_t fse_decode_hufweights(const u8 *src, size_t src_len, u8 *weights, - int *num_symbs, size_t compressed_size) { +static size_t fse_decode_hufweights(const u8 *const src, const size_t src_len, + u8 *const weights, int *const num_symbs, + const size_t compressed_size) { const int MAX_ACCURACY_LOG = 7; FSE_dtable dtable; // Construct the FSE table - size_t read = FSE_decode_header(&dtable, src, src_len, MAX_ACCURACY_LOG); + const size_t read = + FSE_decode_header(&dtable, src, src_len, MAX_ACCURACY_LOG); if (src_len < compressed_size) { INP_SIZE(); @@ -1001,16 +1031,20 @@ static const u8 SEQ_MATCH_LENGTH_EXTRA_BITS[53] = { /// Offset decoding is simpler so we just need a maximum code value static const u8 SEQ_MAX_CODES[3] = {35, -1, 52}; -static void decompress_sequences(frame_context_t *ctx, const u8 *src, - size_t src_len, sequence_command_t *sequences, - size_t num_sequences); -static sequence_command_t decode_sequence(sequence_state_t *state, - const u8 *src, i64 *offset); -static size_t decode_seq_table(const u8 *src, size_t src_len, FSE_dtable *table, - seq_part_t type, seq_mode_t mode); +static void decompress_sequences(frame_context_t *const ctx, const u8 *src, + size_t src_len, + sequence_command_t *const sequences, + const size_t num_sequences); +static sequence_command_t decode_sequence(sequence_state_t *const state, + const u8 *const src, + i64 *const offset); +static size_t decode_seq_table(const u8 *src, size_t src_len, + FSE_dtable *const table, const seq_part_t type, + const seq_mode_t mode); -static size_t decode_sequences(frame_context_t *ctx, const u8 *src, - size_t src_len, sequence_command_t **sequences) { +static size_t decode_sequences(frame_context_t *const ctx, const u8 *src, + size_t src_len, + sequence_command_t **const sequences) { size_t num_sequences; // Decode the sequence header and allocate space for the output @@ -1050,9 +1084,10 @@ static size_t decode_sequences(frame_context_t *ctx, const u8 *src, } /// Decompress the FSE encoded sequence commands -static void decompress_sequences(frame_context_t *ctx, const u8 *src, - size_t src_len, sequence_command_t *sequences, - size_t num_sequences) { +static void decompress_sequences(frame_context_t *const ctx, const u8 *src, + size_t src_len, + sequence_command_t *const sequences, + const size_t num_sequences) { if (src_len < 1) { INP_SIZE(); } @@ -1064,21 +1099,31 @@ static void decompress_sequences(frame_context_t *ctx, const u8 *src, CORRUPTION(); } - sequence_state_t state; - size_t read; - // Update the tables we have stored in the context - read = decode_seq_table(src, src_len, &ctx->ll_dtable, seq_literal_length, - (compression_modes >> 6) & 3); - src += read; - src_len -= read; - read = decode_seq_table(src, src_len, &ctx->of_dtable, seq_offset, - (compression_modes >> 4) & 3); - src += read; - src_len -= read; - read = decode_seq_table(src, src_len, &ctx->ml_dtable, seq_match_length, - (compression_modes >> 2) & 3); - src += read; - src_len -= read; + { + size_t read; + // Update the tables we have stored in the context + read = decode_seq_table(src, src_len, &ctx->ll_dtable, + seq_literal_length, + (compression_modes >> 6) & 3); + src += read; + src_len -= read; + } + + { + const size_t read = + decode_seq_table(src, src_len, &ctx->of_dtable, seq_offset, + (compression_modes >> 4) & 3); + src += read; + src_len -= read; + } + + { + const size_t read = decode_seq_table(src, src_len, &ctx->ml_dtable, + seq_match_length, + (compression_modes >> 2) & 3); + src += read; + src_len -= read; + } // Check to make sure none of the tables are uninitialized if (!ctx->ll_dtable.symbols || !ctx->of_dtable.symbols || @@ -1086,12 +1131,13 @@ static void decompress_sequences(frame_context_t *ctx, const u8 *src, CORRUPTION(); } - // Now use the context's tables + sequence_state_t state; + // Copy the context's tables into the local state memcpy(&state.ll_table, &ctx->ll_dtable, sizeof(FSE_dtable)); memcpy(&state.of_table, &ctx->of_dtable, sizeof(FSE_dtable)); memcpy(&state.ml_table, &ctx->ml_dtable, sizeof(FSE_dtable)); - int padding = 8 - log2inf(src[src_len - 1]); + const int padding = 8 - log2inf(src[src_len - 1]); i64 offset = src_len * 8 - padding; FSE_init_state(&state.ll_table, &state.ll_state, src, &offset); @@ -1111,12 +1157,13 @@ static void decompress_sequences(frame_context_t *ctx, const u8 *src, } // Decode a single sequence and update the state -static sequence_command_t decode_sequence(sequence_state_t *state, - const u8 *src, i64 *offset) { +static sequence_command_t decode_sequence(sequence_state_t *const state, + const u8 *const src, + i64 *const offset) { // Decode symbols, but don't update states - u8 of_code = FSE_peek_symbol(&state->of_table, state->of_state); - u8 ll_code = FSE_peek_symbol(&state->ll_table, state->ll_state); - u8 ml_code = FSE_peek_symbol(&state->ml_table, state->ml_state); + const u8 of_code = FSE_peek_symbol(&state->of_table, state->of_state); + const u8 ll_code = FSE_peek_symbol(&state->ll_table, state->ll_state); + const u8 ml_code = FSE_peek_symbol(&state->ml_table, state->ml_state); // Offset doesn't need a max value as it's not decoded using a table if (ll_code > SEQ_MAX_CODES[seq_literal_length] || @@ -1147,9 +1194,9 @@ static sequence_command_t decode_sequence(sequence_state_t *state, } /// Given a sequence part and table mode, decode the FSE distribution -static size_t decode_seq_table(const u8 *src, size_t src_len, FSE_dtable *table, - seq_part_t type, seq_mode_t mode) { - +static size_t decode_seq_table(const u8 *src, size_t src_len, + FSE_dtable *const table, const seq_part_t type, + const seq_mode_t mode) { // Constant arrays indexed by seq_part_t const i16 *const default_distributions[] = {SEQ_LITERAL_LENGTH_DEFAULT_DIST, SEQ_OFFSET_DEFAULT_DIST, @@ -1178,7 +1225,7 @@ static size_t decode_seq_table(const u8 *src, size_t src_len, FSE_dtable *table, if (src_len < 1) { INP_SIZE(); } - u8 symb = src[0]; + const u8 symb = src[0]; src++; src_len--; FSE_init_dtable_rle(table, symb); @@ -1204,15 +1251,17 @@ static size_t decode_seq_table(const u8 *src, size_t src_len, FSE_dtable *table, /******* END SEQUENCE DECODING ************************************************/ /******* SEQUENCE EXECUTION ***************************************************/ -static size_t execute_sequences(io_streams_t *streams, frame_context_t *ctx, - sequence_command_t *sequences, - size_t num_sequences, const u8 *literals, - size_t literals_len) { - u64 *offset_hist = ctx->previous_offsets; +static void execute_sequences(io_streams_t *const streams, + frame_context_t *const ctx, + const sequence_command_t *const sequences, + const size_t num_sequences, + const u8 *literals, + size_t literals_len) { + u64 *const offset_hist = ctx->previous_offsets; size_t total_output = ctx->current_total_output; for (size_t i = 0; i < num_sequences; i++) { - sequence_command_t seq = sequences[i]; + const sequence_command_t seq = sequences[i]; if (seq.literal_length > literals_len) { CORRUPTION(); @@ -1312,46 +1361,48 @@ static size_t execute_sequences(io_streams_t *streams, frame_context_t *ctx, /******* END SEQUENCE EXECUTION ***********************************************/ /******* OUTPUT SIZE COUNTING *************************************************/ -size_t traverse_frame(frame_header_t *header, const u8 *src, size_t src_len); +size_t traverse_frame(const frame_header_t *const header, const u8 *src, + size_t src_len); /// Get the decompressed size of an input stream so memory can be allocated in /// advance. /// This is more complex than the implementation in the reference /// implementation, as this API allows for the decompression of multiple /// concatenated frames. -size_t ZSTD_get_decompressed_size(const void *src, size_t src_len) { +size_t ZSTD_get_decompressed_size(const void *src, const size_t src_len) { const u8 *ip = (const u8 *) src; + size_t ip_len = src_len; size_t dst_size = 0; // Each frame header only gives us the size of its frame, so iterate over all // frames - while (src_len > 0) { - if (src_len < 4) { + while (ip_len > 0) { + if (ip_len < 4) { INP_SIZE(); } - u32 magic_number = read_bits_LE(ip, 32, 0); + const u32 magic_number = read_bits_LE(ip, 32, 0); ip += 4; - src_len -= 4; + ip_len -= 4; if (magic_number >= 0x184D2A50U && magic_number <= 0x184D2A5F) { // skippable frame, this has no impact on output size - if (src_len < 4) { + if (ip_len < 4) { INP_SIZE(); } - size_t frame_size = read_bits_LE(ip, 32, 32); + const size_t frame_size = read_bits_LE(ip, 32, 32); - if (src_len < 4 + frame_size) { + if (ip_len < 4 + frame_size) { INP_SIZE(); } // skip over frame ip += 4 + frame_size; - src_len -= 4 + frame_size; + ip_len -= 4 + frame_size; } else if (magic_number == 0xFD2FB528U) { // ZSTD frame frame_header_t header; - parse_frame_header(&header, ip, src_len); + parse_frame_header(&header, ip, ip_len); if (header.frame_content_size == 0 && !header.single_segment_flag) { // Content size not provided, we can't tell @@ -1361,9 +1412,9 @@ size_t ZSTD_get_decompressed_size(const void *src, size_t src_len) { dst_size += header.frame_content_size; // we need to traverse the frame to find when the next one starts - size_t traversed = traverse_frame(&header, ip, src_len); + const size_t traversed = traverse_frame(&header, ip, ip_len); ip += traversed; - src_len -= traversed; + ip_len -= traversed; } else { // not a real frame ERROR("Invalid magic number"); @@ -1375,7 +1426,8 @@ size_t ZSTD_get_decompressed_size(const void *src, size_t src_len) { /// Iterate over each block in a frame to find the end of it, to get to the /// start of the next frame -size_t traverse_frame(frame_header_t *header, const u8 *src, size_t src_len) { +size_t traverse_frame(const frame_header_t *const header, const u8 *src, + size_t src_len) { const u8 *const src_beg = src; const u8 *const src_end = src + src_len; src += header->header_size; @@ -1389,8 +1441,8 @@ size_t traverse_frame(frame_header_t *header, const u8 *src, size_t src_len) { } // Parse the block header last_block = src[0] & 1; - int block_type = (src[0] >> 1) & 3; - size_t block_len = read_bits_LE(src, 21, 3); + const int block_type = (src[0] >> 1) & 3; + const size_t block_len = read_bits_LE(src, 21, 3); src += 3; switch (block_type) { @@ -1432,16 +1484,16 @@ size_t traverse_frame(frame_header_t *header, const u8 *src, size_t src_len) { /******* END OUTPUT SIZE COUNTING *********************************************/ /******* DICTIONARY PARSING ***************************************************/ -static void init_raw_content_dict(dictionary_t *dict, const u8 *src, - size_t src_len); +static void init_raw_content_dict(dictionary_t *const dict, const u8 *const src, + const size_t src_len); -static void parse_dictionary(dictionary_t *dict, const u8 *src, +static void parse_dictionary(dictionary_t *const dict, const u8 *src, size_t src_len) { memset(dict, 0, sizeof(dictionary_t)); if (src_len < 8) { INP_SIZE(); } - u32 magic_number = read_bits_LE(src, 32, 0); + const u32 magic_number = read_bits_LE(src, 32, 0); if (magic_number != 0xEC30A437) { // raw content dict init_raw_content_dict(dict, src, src_len); @@ -1454,25 +1506,26 @@ static void parse_dictionary(dictionary_t *dict, const u8 *src, // Parse the provided entropy tables in order { - size_t read = decode_huf_table(src, src_len, &dict->literals_dtable); + const size_t read = + decode_huf_table(src, src_len, &dict->literals_dtable); src += read; src_len -= read; } { - size_t read = decode_seq_table(src, src_len, &dict->of_dtable, - seq_offset, seq_fse); + const size_t read = decode_seq_table(src, src_len, &dict->of_dtable, + seq_offset, seq_fse); src += read; src_len -= read; } { - size_t read = decode_seq_table(src, src_len, &dict->ml_dtable, - seq_match_length, seq_fse); + const size_t read = decode_seq_table(src, src_len, &dict->ml_dtable, + seq_match_length, seq_fse); src += read; src_len -= read; } { - size_t read = decode_seq_table(src, src_len, &dict->ll_dtable, - seq_literal_length, seq_fse); + const size_t read = decode_seq_table(src, src_len, &dict->ll_dtable, + seq_literal_length, seq_fse); src += read; src_len -= read; } @@ -1505,8 +1558,8 @@ static void parse_dictionary(dictionary_t *dict, const u8 *src, } /// If parse_dictionary is given a raw content dictionary, it delegates here -static void init_raw_content_dict(dictionary_t *dict, const u8 *src, - size_t src_len) { +static void init_raw_content_dict(dictionary_t *const dict, const u8 *const src, + const size_t src_len) { dict->dictionary_id = 0; // Copy in the content dict->content = malloc(src_len); @@ -1519,7 +1572,7 @@ static void init_raw_content_dict(dictionary_t *dict, const u8 *src, } /// Free an allocated dictionary -static void free_dictionary(dictionary_t *dict) { +static void free_dictionary(dictionary_t *const dict) { HUF_free_dtable(&dict->literals_dtable); FSE_free_dtable(&dict->ll_dtable); FSE_free_dtable(&dict->of_dtable); @@ -1531,179 +1584,50 @@ static void free_dictionary(dictionary_t *dict) { } /******* END DICTIONARY PARSING ***********************************************/ -/******* CIRCULAR BUFFER ******************************************************/ -static void cbuf_init(cbuf_t *buf, size_t size) { - buf->ptr = malloc(size); - - if (!buf->ptr) { - BAD_ALLOC(); - } - - memset(buf->ptr, 0x3f, size); - - buf->size = size; - buf->idx = 0; - buf->last_flush = 0; -} - -static size_t cbuf_write_data(cbuf_t *buf, const u8 *src, size_t src_len) { - if (buf->size == 0 && src_len > 0) { - CORRUPTION(); - } - size_t max_len = buf->size - buf->idx; - size_t len = MIN(src_len, max_len); - - memcpy(buf->ptr + buf->idx, src, len); - - buf->idx += len; - - return len; -} - -static size_t cbuf_write_data_full(cbuf_t *buf, const u8 *src, size_t src_len, - u8 *out, size_t out_len) { - size_t written = 0; - size_t flushed = 0; - while (1) { - written += cbuf_write_data(buf, src + written, src_len - written); - if (written == src_len) { - break; - } else { - flushed += cbuf_flush(buf, out + flushed, out_len - flushed); - } - } - - return flushed; -} - -static size_t cbuf_copy_offset(cbuf_t *buf, size_t offset, size_t len) { - if (buf->size == 0 && len > 0) { - CORRUPTION(); - } - if (offset > buf->size) { - CORRUPTION(); - } - size_t max_len = buf->size - buf->idx; - len = MIN(len, max_len); - - size_t read_off = (buf->idx + buf->size - offset) % buf->size; - - for (size_t i = 0; i < len; i++) { - buf->ptr[buf->idx++] = buf->ptr[read_off++]; - if (read_off == buf->size) { - read_off = 0; - } - } - - return len; -} - -static size_t cbuf_copy_offset_full(cbuf_t *buf, size_t offset, size_t len, - u8 *out, size_t out_len) { - size_t written = 0; - size_t flushed = 0; - while (1) { - written += cbuf_copy_offset(buf, offset, len - written); - if (written == len) { - break; - } else { - flushed += cbuf_flush(buf, out + flushed, out_len - flushed); - } - } - - return flushed; -} - -static size_t cbuf_repeat_byte(cbuf_t *buf, u8 byte, size_t len) { - if (buf->size == 0 && len > 0) { - CORRUPTION(); - } - size_t max_len = buf->size - buf->idx; - len = MIN(len, max_len); - - memset(buf->ptr + buf->idx, byte, len); - - return len; -} - -static size_t cbuf_repeat_byte_full(cbuf_t *buf, u8 byte, size_t len, u8 *out, - size_t out_len) { - size_t written = 0; - size_t flushed = 0; - while (1) { - written += cbuf_repeat_byte(buf, byte, len - written); - if (written == len) { - break; - } else { - flushed += cbuf_flush(buf, out + flushed, out_len - flushed); - } - } - - return flushed; -} - -static size_t cbuf_flush(cbuf_t *buf, u8 *dst, size_t dst_len) { - if (buf->idx < buf->last_flush) { - CORRUPTION(); - } - - size_t len = buf->idx - buf->last_flush; - - if (dst && len > dst_len) { - OUT_SIZE(); - } - - // allow for NULL buffers to indicate flushing to nowhere - if (dst) { - memcpy(dst, buf->ptr + buf->last_flush, len); - } - - // we could have a 0 size buffer - if (buf->size) { - buf->idx = buf->idx % buf->size; - } - buf->last_flush = buf->idx; - - return len; -} - -static void cbuf_free(cbuf_t *buf) { - free(buf->ptr); - memset(buf, 0, sizeof(cbuf_t)); -} -/******* END CIRCULAR BUFFER **************************************************/ - /******* BITSTREAM OPERATIONS *************************************************/ -static inline u64 read_bits_LE(const u8 *src, int num, size_t offset) { +/// Read `num` bits (up to 64) from `src + offset`, where `offset` is in bits +static inline u64 read_bits_LE(const u8 *src, const int num, + const size_t offset) { if (num > 64) { return -1; } + // Skip over bytes that aren't in range src += offset / 8; - offset %= 8; + size_t bit_offset = offset % 8; u64 res = 0; int shift = 0; int left = num; while (left > 0) { u64 mask = left >= 8 ? 0xff : (((u64)1 << left) - 1); - res += (((u64)*src++ >> offset) & mask) << shift; - shift += 8 - offset; - left -= 8 - offset; - offset = 0; + // Dead the next byte, shift it to account for the offset, and then mask + // out the top part if we don't need all the bits + res += (((u64)*src++ >> bit_offset) & mask) << shift; + shift += 8 - bit_offset; + left -= 8 - bit_offset; + bit_offset = 0; } return res; } -static inline u64 STREAM_read_bits(const u8 *src, int bits, i64 *offset) { +/// Read bits from the end of a HUF or FSE bitstream. `offset` is in bits, so +/// it updates `offset` to `offset - bits`, and then reads `bits` bits from +/// `src + offset`. If the offset becomes negative, the extra bits at the +/// bottom are filled in with `0` bits instead of reading from before `src`. +static inline u64 STREAM_read_bits(const u8 *const src, const int bits, + i64 *const offset) { *offset = *offset - bits; size_t actual_off = *offset; + size_t actual_bits = bits; + // Don't actually read bits from before the start of src, so if `*offset < + // 0` fix actual_off and actual_bits to reflect the quantity to read if (*offset < 0) { - bits += *offset; + actual_bits += *offset; actual_off = 0; } - u64 res = read_bits_LE(src, bits, actual_off); + u64 res = read_bits_LE(src, actual_bits, actual_off); if (*offset < 0) { // Fill in the bottom "overflowed" bits with 0's @@ -1714,16 +1638,9 @@ static inline u64 STREAM_read_bits(const u8 *src, int bits, i64 *offset) { /******* END BITSTREAM OPERATIONS *********************************************/ /******* BIT COUNTING OPERATIONS **********************************************/ -static inline int log2sup(u64 num) { - for (int i = 0; i < 64; i++) { - if (((u64)1 << i) >= num) { - return i; - } - } - return -1; -} - -static inline int log2inf(u64 num) { +/// Returns `x`, where `2^x` is the largest power of 2 less than or equal to +/// `num`, or `-1` if `num == 0`. +static inline int log2inf(const u64 num) { for (int i = 63; i >= 0; i--) { if (((u64)1 << i) <= num) { return i; @@ -1734,33 +1651,38 @@ static inline int log2inf(u64 num) { /******* END BIT COUNTING OPERATIONS ******************************************/ /******* HUFFMAN PRIMITIVES ***************************************************/ -static inline u8 HUF_decode_symbol(HUF_dtable *dtable, u16 *state, - const u8 *src, i64 *offset) { +static inline u8 HUF_decode_symbol(const HUF_dtable *const dtable, + u16 *const state, const u8 *const src, + i64 *const offset) { // Look up the symbol and number of bits to read const u8 symb = dtable->symbols[*state]; const u8 bits = dtable->num_bits[*state]; const u16 rest = STREAM_read_bits(src, bits, offset); + // Shift `bits` bits out of the state, keeping the low order bits that + // weren't necessary to determine this symbol. Then add in the new bits + // read from the stream. *state = ((*state << bits) + rest) & (((u16)1 << dtable->max_bits) - 1); return symb; } -static inline void HUF_init_state(HUF_dtable *dtable, u16 *state, const u8 *src, - i64 *offset) { - // Read in a full dtable->max_bits to initialize the state +static inline void HUF_init_state(const HUF_dtable *const dtable, + u16 *const state, const u8 *const src, + i64 *const offset) { + // Read in a full `dtable->max_bits` bits to initialize the state const u8 bits = dtable->max_bits; *state = STREAM_read_bits(src, bits, offset); } -static size_t HUF_decompress_1stream(HUF_dtable *dtable, u8 *dst, - size_t dst_len, const u8 *src, +static size_t HUF_decompress_1stream(const HUF_dtable *const dtable, u8 *dst, + const size_t dst_len, const u8 *src, size_t src_len) { - u8 *const dst_max = dst + dst_len; - u8 *const odst = dst; + const u8 *const dst_max = dst + dst_len; + const u8 *const odst = dst; // To maintain similarity with FSE, start from the end // Find the last 1 bit - int padding = 8 - log2inf(src[src_len - 1]); + const int padding = 8 - log2inf(src[src_len - 1]); i64 offset = src_len * 8 - padding; u16 state; @@ -1768,6 +1690,7 @@ static size_t HUF_decompress_1stream(HUF_dtable *dtable, u8 *dst, HUF_init_state(dtable, &state, src, &offset); while (dst < dst_max && offset > -dtable->max_bits) { + // Iterate over the stream, decoding one symbol at a time *dst++ = HUF_decode_symbol(dtable, &state, src, &offset); } // If we stopped before consuming all the input, we didn't have enough space @@ -1775,8 +1698,11 @@ static size_t HUF_decompress_1stream(HUF_dtable *dtable, u8 *dst, OUT_SIZE(); } - // The current state should be the `max_bits` preceding the start as - // everything from `src` onward should be consumed + // When all symbols have been decoded, the final state value shouldn't have + // any data from the stream, so it should have "read" dtable->max_bits from + // before the start of `src` + // Therefore `offset`, the edge to start reading new bits at, should be + // dtable->max_bits before the start of the stream if (offset != -dtable->max_bits) { CORRUPTION(); } @@ -1784,28 +1710,18 @@ static size_t HUF_decompress_1stream(HUF_dtable *dtable, u8 *dst, return dst - odst; } -static size_t HUF_decompress_4stream(HUF_dtable *dtable, u8 *dst, - size_t dst_len, const u8 *src, - size_t src_len) { - // Decode each stream independently for simplicity - // If we wanted to we could decode all 4 at the same time for speed, - // utilizing - // more execution units - - const u8 *src1, *src2, *src3, *src4, *src_end; - u8 *dst1, *dst2, *dst3, *dst4, *dst_end; - - size_t total_out = 0; - +static size_t HUF_decompress_4stream(const HUF_dtable *const dtable, u8 *dst, + const size_t dst_len, const u8 *const src, + const size_t src_len) { if (src_len < 6) { INP_SIZE(); } - src1 = src + 6; - src2 = src1 + read_bits_LE(src, 16, 0); - src3 = src2 + read_bits_LE(src, 16, 16); - src4 = src3 + read_bits_LE(src, 16, 32); - src_end = src + src_len; + const u8 *const src1 = src + 6; + const u8 *const src2 = src1 + read_bits_LE(src, 16, 0); + const u8 *const src3 = src2 + read_bits_LE(src, 16, 16); + const u8 *const src4 = src3 + read_bits_LE(src, 16, 32); + const u8 *const src_end = src + src_len; // We can't test with all 4 sizes because the 4th size is a function of the // other 3 and the provided length @@ -1813,26 +1729,32 @@ static size_t HUF_decompress_4stream(HUF_dtable *dtable, u8 *dst, INP_SIZE(); } - size_t segment_size = (dst_len + 3) / 4; - dst1 = dst; - dst2 = dst1 + segment_size; - dst3 = dst2 + segment_size; - dst4 = dst3 + segment_size; - dst_end = dst + dst_len; + const size_t segment_size = (dst_len + 3) / 4; + u8 *const dst1 = dst; + u8 *const dst2 = dst1 + segment_size; + u8 *const dst3 = dst2 + segment_size; + u8 *const dst4 = dst3 + segment_size; + u8 *const dst_end = dst + dst_len; - total_out += - HUF_decompress_1stream(dtable, dst1, segment_size, src1, src2 - src1); - total_out += - HUF_decompress_1stream(dtable, dst2, segment_size, src2, src3 - src2); - total_out += - HUF_decompress_1stream(dtable, dst3, segment_size, src3, src4 - src3); + size_t total_out = 0; + + // Decode each stream independently for simplicity + // If we wanted to we could decode all 4 at the same time for speed, + // utilizing more execution units + total_out += HUF_decompress_1stream(dtable, dst1, segment_size, src1, + src2 - src1); + total_out += HUF_decompress_1stream(dtable, dst2, segment_size, src2, + src3 - src2); + total_out += HUF_decompress_1stream(dtable, dst3, segment_size, src3, + src4 - src3); total_out += HUF_decompress_1stream(dtable, dst4, dst_end - dst4, src4, src_end - src4); return total_out; } -static void HUF_init_dtable(HUF_dtable *table, u8 *bits, int num_symbs) { +static void HUF_init_dtable(HUF_dtable *const table, const u8 *const bits, + const int num_symbs) { memset(table, 0, sizeof(HUF_dtable)); if (num_symbs > HUF_MAX_SYMBS) { ERROR("Too many symbols for Huffman"); @@ -1852,7 +1774,7 @@ static void HUF_init_dtable(HUF_dtable *table, u8 *bits, int num_symbs) { rank_count[bits[i]]++; } - size_t table_size = 1 << max_bits; + const size_t table_size = 1 << max_bits; table->max_bits = max_bits; table->symbols = malloc(table_size); table->num_bits = malloc(table_size); @@ -1881,6 +1803,9 @@ static void HUF_init_dtable(HUF_dtable *table, u8 *bits, int num_symbs) { if (bits[i] != 0) { // Allocate a code for this symbol and set its range in the table const u16 code = rank_idx[bits[i]]; + // Since the code doesn't care about the bottom `max_bits - bits[i]` + // bits of state, it gets a range that spans all possible values of + // the lower bits const u16 len = 1 << (max_bits - bits[i]); memset(&table->symbols[code], i, len); rank_idx[bits[i]] += len; @@ -1888,8 +1813,9 @@ static void HUF_init_dtable(HUF_dtable *table, u8 *bits, int num_symbs) { } } -static void HUF_init_dtable_usingweights(HUF_dtable *table, u8 *weights, - int num_symbs) { +static void HUF_init_dtable_usingweights(HUF_dtable *const table, + const u8 *const weights, + const int num_symbs) { // +1 because the last weight is not transmitted in the header if (num_symbs + 1 > HUF_MAX_SYMBS) { ERROR("Too many symbols for Huffman"); @@ -1903,37 +1829,40 @@ static void HUF_init_dtable_usingweights(HUF_dtable *table, u8 *weights, } // Find the first power of 2 larger than the sum - int max_bits = log2inf(weight_sum) + 1; - u64 left_over = ((u64)1 << max_bits) - weight_sum; + const int max_bits = log2inf(weight_sum) + 1; + const u64 left_over = ((u64)1 << max_bits) - weight_sum; // If the left over isn't a power of 2, the weights are invalid if (left_over & (left_over - 1)) { CORRUPTION(); } - int last_weight = log2inf(left_over) + 1; + // left_over is used to find the last weight as it's not transmitted + // by inverting 2^(weight - 1) we can determine the value of last_weight + const int last_weight = log2inf(left_over) + 1; for (int i = 0; i < num_symbs; i++) { bits[i] = weights[i] > 0 ? (max_bits + 1 - weights[i]) : 0; } bits[num_symbs] = - max_bits + 1 - last_weight; // last weight is always non-zero + max_bits + 1 - last_weight; // Last weight is always non-zero HUF_init_dtable(table, bits, num_symbs + 1); } -static void HUF_free_dtable(HUF_dtable *dtable) { +static void HUF_free_dtable(HUF_dtable *const dtable) { free(dtable->symbols); free(dtable->num_bits); memset(dtable, 0, sizeof(HUF_dtable)); } -static void HUF_copy_dtable(HUF_dtable *dst, const HUF_dtable *src) { +static void HUF_copy_dtable(HUF_dtable *const dst, + const HUF_dtable *const src) { if (src->max_bits == 0) { memset(dst, 0, sizeof(HUF_dtable)); return; } - size_t size = (size_t)1 << src->max_bits; + const size_t size = (size_t)1 << src->max_bits; dst->max_bits = src->max_bits; dst->symbols = malloc(size); @@ -1948,46 +1877,56 @@ static void HUF_copy_dtable(HUF_dtable *dst, const HUF_dtable *src) { /******* END HUFFMAN PRIMITIVES ***********************************************/ /******* FSE PRIMITIVES *******************************************************/ -static inline u8 FSE_peek_symbol(FSE_dtable *dtable, u16 state) { +/// Allow a symbol to be decoded without updating state +static inline u8 FSE_peek_symbol(const FSE_dtable *const dtable, + const u16 state) { return dtable->symbols[state]; } -static inline void FSE_update_state(FSE_dtable *dtable, u16 *state, - const u8 *src, i64 *offset) { +/// Consumes bits from the input and uses the current state to determine the +/// next state +static inline void FSE_update_state(const FSE_dtable *const dtable, + u16 *const state, const u8 *const src, + i64 *const offset) { const u8 bits = dtable->num_bits[*state]; const u16 rest = STREAM_read_bits(src, bits, offset); *state = dtable->new_state_base[*state] + rest; } -// Decodes a single FSE symbol and updates the offset -static inline u8 FSE_decode_symbol(FSE_dtable *dtable, u16 *state, - const u8 *src, i64 *offset) { +/// Decodes a single FSE symbol and updates the offset +static inline u8 FSE_decode_symbol(const FSE_dtable *const dtable, + u16 *const state, const u8 *const src, + i64 *const offset) { const u8 symb = FSE_peek_symbol(dtable, *state); FSE_update_state(dtable, state, src, offset); return symb; } -static inline void FSE_init_state(FSE_dtable *dtable, u16 *state, const u8 *src, - i64 *offset) { +static inline void FSE_init_state(const FSE_dtable *const dtable, + u16 *const state, const u8 *const src, + i64 *const offset) { + // Read in a full `accuracy_log` bits to initialize the state const u8 bits = dtable->accuracy_log; *state = STREAM_read_bits(src, bits, offset); } -static size_t FSE_decompress_interleaved2(FSE_dtable *dtable, u8 *dst, - size_t dst_len, const u8 *src, - size_t src_len) { +static size_t FSE_decompress_interleaved2(const FSE_dtable *const dtable, + u8 *dst, const size_t dst_len, + const u8 *const src, + const size_t src_len) { if (src_len == 0) { INP_SIZE(); } - u8 *dst_max = dst + dst_len; - u8 *const odst = dst; + const u8 *const dst_max = dst + dst_len; + const u8 *const odst = dst; // Find the last 1 bit - int padding = 8 - log2inf(src[src_len - 1]); + const int padding = 8 - log2inf(src[src_len - 1]); i64 offset = src_len * 8 - padding; + // The end of the stream contains the 2 states, in this order u16 state1, state2; FSE_init_state(dtable, &state1, src, &offset); FSE_init_state(dtable, &state2, src, &offset); @@ -2002,7 +1941,7 @@ static size_t FSE_decompress_interleaved2(FSE_dtable *dtable, u8 *dst, *dst++ = FSE_decode_symbol(dtable, &state1, src, &offset); if (offset < 0) { // There's still a symbol to decode in state2 - *dst++ = FSE_decode_symbol(dtable, &state2, src, &offset); + *dst++ = FSE_peek_symbol(dtable, state2); break; } @@ -2012,17 +1951,18 @@ static size_t FSE_decompress_interleaved2(FSE_dtable *dtable, u8 *dst, *dst++ = FSE_decode_symbol(dtable, &state2, src, &offset); if (offset < 0) { // There's still a symbol to decode in state1 - *dst++ = FSE_decode_symbol(dtable, &state1, src, &offset); + *dst++ = FSE_peek_symbol(dtable, state1); break; } } - // number of symbols read + // Number of symbols read return dst - odst; } -static void FSE_init_dtable(FSE_dtable *dtable, const i16 *norm_freqs, - int num_symbs, int accuracy_log) { +static void FSE_init_dtable(FSE_dtable *const dtable, + const i16 *const norm_freqs, const int num_symbs, + const int accuracy_log) { if (accuracy_log > FSE_MAX_ACCURACY_LOG) { ERROR("FSE accuracy too large"); } @@ -2032,7 +1972,7 @@ static void FSE_init_dtable(FSE_dtable *dtable, const i16 *norm_freqs, dtable->accuracy_log = accuracy_log; - size_t size = (size_t)1 << accuracy_log; + const size_t size = (size_t)1 << accuracy_log; dtable->symbols = malloc(size * sizeof(u8)); dtable->num_bits = malloc(size * sizeof(u8)); dtable->new_state_base = malloc(size * sizeof(u16)); @@ -2057,8 +1997,8 @@ static void FSE_init_dtable(FSE_dtable *dtable, const i16 *norm_freqs, } // Place the rest in the table - u16 step = (size >> 1) + (size >> 3) + 3; - u16 mask = size - 1; + const u16 step = (size >> 1) + (size >> 3) + 3; + const u16 mask = size - 1; u16 pos = 0; for (int s = 0; s < num_symbs; s++) { if (norm_freqs[s] <= 0) { @@ -2068,6 +2008,7 @@ static void FSE_init_dtable(FSE_dtable *dtable, const i16 *norm_freqs, state_desc[s] = norm_freqs[s]; for (int i = 0; i < norm_freqs[s]; i++) { + // Give `norm_freqs[s]` states to symbol s dtable->symbols[pos] = s; do { pos = (pos + step) & mask; @@ -2087,18 +2028,21 @@ static void FSE_init_dtable(FSE_dtable *dtable, const i16 *norm_freqs, for (int i = 0; i < size; i++) { u8 symbol = dtable->symbols[i]; u16 next_state_desc = state_desc[symbol]++; - // Fills in the table appropriately next_state_desc increases by symbol + // Fills in the table appropriately, next_state_desc increases by symbol // over time, decreasing number of bits dtable->num_bits[i] = (u8)(accuracy_log - log2inf(next_state_desc)); - // baseline increases until the bit threshold is passed, at which point + // Baseline increases until the bit threshold is passed, at which point // it resets to 0 dtable->new_state_base[i] = ((u16)next_state_desc << dtable->num_bits[i]) - size; } } -static size_t FSE_decode_header(FSE_dtable *dtable, const u8 *src, - size_t src_len, int max_accuracy_log) { +/// Decode an FSE header as defined in the Zstandard format specification and +/// use the decoded frequencies to initialize a decoding table. +static size_t FSE_decode_header(FSE_dtable *const dtable, const u8 *const src, + const size_t src_len, + const int max_accuracy_log) { if (max_accuracy_log > FSE_MAX_ACCURACY_LOG) { ERROR("FSE accuracy too large"); } @@ -2106,7 +2050,7 @@ static size_t FSE_decode_header(FSE_dtable *dtable, const u8 *src, INP_SIZE(); } - int accuracy_log = 5 + read_bits_LE(src, 4, 0); + const int accuracy_log = 5 + read_bits_LE(src, 4, 0); if (accuracy_log > max_accuracy_log) { ERROR("FSE accuracy too large"); } @@ -2116,17 +2060,19 @@ static size_t FSE_decode_header(FSE_dtable *dtable, const u8 *src, i16 frequencies[FSE_MAX_SYMBS]; int symb = 0; + // Offset of 4 because 4 bits were already read in for accuracy size_t offset = 4; while (remaining > 1 && symb < FSE_MAX_SYMBS) { - int bits = log2sup(remaining + - 1); // the number of possible values we could read + // Log of the number of possible values we could read + int bits = log2inf(remaining) + 1; + u16 val = read_bits_LE(src, bits, offset); offset += bits; - // try to mask out the lower bits to see if it qualifies for the "small + // Try to mask out the lower bits to see if it qualifies for the "small // value" threshold - u16 lower_mask = ((u16)1 << (bits - 1)) - 1; - u16 threshold = ((u16)1 << bits) - 1 - remaining; + const u16 lower_mask = ((u16)1 << (bits - 1)) - 1; + const u16 threshold = ((u16)1 << bits) - 1 - remaining; if ((val & lower_mask) < threshold) { offset--; @@ -2135,8 +2081,8 @@ static size_t FSE_decode_header(FSE_dtable *dtable, const u8 *src, val = val - threshold; } - i16 proba = (i16)val - 1; - // a value of -1 is possible, and has special meaning + const i16 proba = (i16)val - 1; + // A value of -1 is possible, and has special meaning remaining -= proba < 0 ? -proba : proba; frequencies[symb] = proba; @@ -2144,7 +2090,7 @@ static size_t FSE_decode_header(FSE_dtable *dtable, const u8 *src, // Handle the special probability = 0 case if (proba == 0) { - // read the next two bits to see how many more 0s + // Read the next two bits to see how many more 0s int repeat = read_bits_LE(src, 2, offset); offset += 2; @@ -2172,7 +2118,7 @@ static size_t FSE_decode_header(FSE_dtable *dtable, const u8 *src, return (offset + 7) / 8; } -static void FSE_init_dtable_rle(FSE_dtable *dtable, u8 symb) { +static void FSE_init_dtable_rle(FSE_dtable *const dtable, const u8 symb) { dtable->symbols = malloc(sizeof(u8)); dtable->num_bits = malloc(sizeof(u8)); dtable->new_state_base = malloc(sizeof(u16)); @@ -2189,14 +2135,14 @@ static void FSE_init_dtable_rle(FSE_dtable *dtable, u8 symb) { dtable->accuracy_log = 0; } -static void FSE_free_dtable(FSE_dtable *dtable) { +static void FSE_free_dtable(FSE_dtable *const dtable) { free(dtable->symbols); free(dtable->num_bits); free(dtable->new_state_base); memset(dtable, 0, sizeof(FSE_dtable)); } -static void FSE_copy_dtable(FSE_dtable *dst, const FSE_dtable *src) { +static void FSE_copy_dtable(FSE_dtable *const dst, const FSE_dtable *const src) { if (src->accuracy_log == 0) { memset(dst, 0, sizeof(FSE_dtable)); return; diff --git a/contrib/educational_decoder/zstd_decompress.h b/contrib/educational_decoder/zstd_decompress.h index 6e1736720..16f4da3eb 100644 --- a/contrib/educational_decoder/zstd_decompress.h +++ b/contrib/educational_decoder/zstd_decompress.h @@ -7,10 +7,10 @@ * of patent rights can be found in the PATENTS file in the same directory. */ -size_t ZSTD_decompress(void *dst, size_t dst_len, const void *src, - size_t src_len); -size_t ZSTD_decompress_with_dict(void *dst, size_t dst_len, const void *src, - size_t src_len, const void *dict, - size_t dict_len); -size_t ZSTD_get_decompressed_size(const void *src, size_t src_len); +size_t ZSTD_decompress(void *const dst, const size_t dst_len, + const void *const src, const size_t src_len); +size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len, + const void *const src, const size_t src_len, + const void *const dict, const size_t dict_len); +size_t ZSTD_get_decompressed_size(const void *const src, const size_t src_len); From 823d8c233bd7a12aefee349d7556855ea3894487 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Wed, 1 Feb 2017 10:41:04 -0800 Subject: [PATCH 006/223] Minor security fixes --- contrib/educational_decoder/harness.c | 2 +- contrib/educational_decoder/zstd_decompress.c | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/educational_decoder/harness.c b/contrib/educational_decoder/harness.c index 107a16a22..cff8239d6 100644 --- a/contrib/educational_decoder/harness.c +++ b/contrib/educational_decoder/harness.c @@ -98,7 +98,7 @@ int main(int argc, char **argv) { } size_t decompressed = - ZSTD_decompress_with_dict(output, input_size * MAX_COMPRESSION_RATIO, + ZSTD_decompress_with_dict(output, decompressed_size, input, input_size, dict, dict_size); write_file(argv[2], output, decompressed); diff --git a/contrib/educational_decoder/zstd_decompress.c b/contrib/educational_decoder/zstd_decompress.c index 3c1c56730..e2fbcf2cf 100644 --- a/contrib/educational_decoder/zstd_decompress.c +++ b/contrib/educational_decoder/zstd_decompress.c @@ -1331,6 +1331,8 @@ static void execute_sequences(io_streams_t *const streams, } match_length -= dict_copy; } + } else if (offset > ctx->header.window_size) { + CORRUPTION(); } // We must copy byte by byte because the match length might be larger From 18ce8b54ddeb7cd80de8978d7fb0b66b966089d7 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Wed, 1 Feb 2017 17:05:45 -0800 Subject: [PATCH 007/223] Switch IO to go through streams --- contrib/educational_decoder/harness.c | 15 +- contrib/educational_decoder/zstd_decompress.c | 1263 ++++++++--------- 2 files changed, 604 insertions(+), 674 deletions(-) diff --git a/contrib/educational_decoder/harness.c b/contrib/educational_decoder/harness.c index cff8239d6..683278dfc 100644 --- a/contrib/educational_decoder/harness.c +++ b/contrib/educational_decoder/harness.c @@ -18,6 +18,9 @@ typedef unsigned char u8; // compression ratio is at most 16 #define MAX_COMPRESSION_RATIO (16) +// Protect against allocating too much memory for output +#define MAX_OUTPUT_SIZE ((size_t)1024 * 1024 * 1024) + u8 *input; u8 *output; u8 *dict; @@ -86,11 +89,17 @@ int main(int argc, char **argv) { size_t decompressed_size = ZSTD_get_decompressed_size(input, input_size); if (decompressed_size == -1) { decompressed_size = MAX_COMPRESSION_RATIO * input_size; - fprintf(stderr, "WARNING: Compressed data does contain decompressed " - "size, going to assume the compression ratio is at " - "most %d (decompressed size of at most %zu)\n", + fprintf(stderr, "WARNING: Compressed data does not contain " + "decompressed size, going to assume the compression " + "ratio is at most %d (decompressed size of at most " + "%zu)\n", MAX_COMPRESSION_RATIO, decompressed_size); } + if (decompressed_size > MAX_OUTPUT_SIZE) { + fprintf(stderr, + "Required output size too large for this implementation\n"); + return 1; + } output = malloc(decompressed_size); if (!output) { fprintf(stderr, "failed to allocate memory\n"); diff --git a/contrib/educational_decoder/zstd_decompress.c b/contrib/educational_decoder/zstd_decompress.c index e2fbcf2cf..8f28313e4 100644 --- a/contrib/educational_decoder/zstd_decompress.c +++ b/contrib/educational_decoder/zstd_decompress.c @@ -48,6 +48,7 @@ size_t ZSTD_get_decompressed_size(const void *const src, const size_t src_len); #define OUT_SIZE() ERROR("Output buffer too small for output") #define CORRUPTION() ERROR("Corruption detected while decompressing") #define BAD_ALLOC() ERROR("Memory allocation error") +#define IMPOSSIBLE() ERROR("An impossibility has occurred") typedef uint8_t u8; typedef uint16_t u16; @@ -65,6 +66,62 @@ typedef int64_t i64; /// file. They implement low-level functionality needed for the higher level /// decompression functions. +/*** IO STREAM OPERATIONS *************/ +/// These structs are the interface for IO, and do bounds checking on all +/// operations. They should be used opaquely to ensure safety. + +/// Output is always done byte-by-byte +typedef struct { + u8 *ptr; + size_t len; +} ostream_t; + +/// Input often reads a few bits at a time, so maintain an internal offset +typedef struct { + const u8 *ptr; + int bit_offset; + size_t len; +} istream_t; + +/// The following two functions are the only ones that allow the istream to be +/// non-byte aligned + +/// Reads `num` bits from a bitstream, and updates the internal offset +static inline u64 IO_read_bits(istream_t *const in, const int num); +/// Rewinds the stream by `num` bits +static inline void IO_rewind_bits(istream_t *const in, const int num); +/// If the remaining bits in a byte will be unused, advance to the end of the +/// byte +static inline void IO_align_stream(istream_t *const in); + +/// Write the given byte into the output stream +static inline void IO_write_byte(ostream_t *const out, u8 symb); + +/// Returns the number of bytes left to be read in this stream. The stream must +/// be byte aligned. +static inline size_t IO_istream_len(const istream_t *const in); + +/// Returns a pointer where `len` bytes can be read, and advances the internal +/// state. The stream must be byte aligned. +static inline const u8 *IO_read_bytes(istream_t *const in, size_t len); +/// Returns a pointer where `len` bytes can be written, and advances the internal +/// state. The stream must be byte aligned. +static inline u8 *IO_write_bytes(ostream_t *const out, size_t len); + +/// Advance the inner state by `len` bytes. The stream must be byte aligned. +static inline void IO_advance_input(istream_t *const in, size_t len); + +/// Returns an `ostream_t` constructed from the given pointer and length +static inline ostream_t IO_make_ostream(u8 *out, size_t len); +/// Returns an `istream_t` constructed from the given pointer and length +static inline istream_t IO_make_istream(const u8 *in, size_t len); + +/// Returns an `istream_t` with the same base as `in`, and length `len` +/// Then, advance `in` to account for the consumed bytes +/// `in` must be byte aligned +static inline istream_t IO_make_sub_istream(istream_t *const in, size_t len); +/*** END IO STREAM OPERATIONS *********/ + /*** BITSTREAM OPERATIONS *************/ /// Read `num` bits (up to 64) from `src + offset`, where `offset` is in bits static inline u64 read_bits_LE(const u8 *src, const int num, @@ -109,15 +166,13 @@ static inline void HUF_init_state(const HUF_dtable *const dtable, /// Decompresses a single Huffman stream, returns the number of bytes decoded. /// `src_len` must be the exact length of the Huffman-coded block. -static size_t HUF_decompress_1stream(const HUF_dtable *const dtable, u8 *dst, - const size_t dst_len, const u8 *src, - size_t src_len); +static size_t HUF_decompress_1stream(const HUF_dtable *const dtable, + ostream_t *const out, istream_t *const in); /// Same as previous but decodes 4 streams, formatted as in the Zstandard /// specification. /// `src_len` must be the exact length of the Huffman-coded block. -static size_t HUF_decompress_4stream(const HUF_dtable *const dtable, u8 *dst, - const size_t dst_len, const u8 *const src, - const size_t src_len); +static size_t HUF_decompress_4stream(const HUF_dtable *const dtable, + ostream_t *const out, istream_t *const in); /// Initialize a Huffman decoding table using the table of bit counts provided static void HUF_init_dtable(HUF_dtable *const table, const u8 *const bits, @@ -176,9 +231,8 @@ static inline void FSE_init_state(const FSE_dtable *const dtable, /// using an FSE decoding table. `src_len` must be the exact length of the /// block. static size_t FSE_decompress_interleaved2(const FSE_dtable *const dtable, - u8 *dst, const size_t dst_len, - const u8 *const src, - const size_t src_len); + ostream_t *const out, + istream_t *const in); /// Initialize a decoding table using normalized frequencies. static void FSE_init_dtable(FSE_dtable *const dtable, @@ -187,8 +241,7 @@ static void FSE_init_dtable(FSE_dtable *const dtable, /// Decode an FSE header as defined in the Zstandard format specification and /// use the decoded frequencies to initialize a decoding table. -static size_t FSE_decode_header(FSE_dtable *const dtable, const u8 *const src, - const size_t src_len, +static void FSE_decode_header(FSE_dtable *const dtable, istream_t *const in, const int max_accuracy_log); /// Initialize an FSE table that will always return the same symbol and consume @@ -207,16 +260,6 @@ static void FSE_copy_dtable(FSE_dtable *const dst, const FSE_dtable *const src); /******* ZSTD HELPER STRUCTS AND PROTOTYPES ***********************************/ -/// Input and output pointers to allow them to be advanced by -/// functions that consume input/produce output -typedef struct { - u8 *dst; - size_t dst_len; - - const u8 *src; - size_t src_len; -} io_streams_t; - /// A small structure that can be reused in various places that need to access /// frame header information typedef struct { @@ -233,9 +276,6 @@ typedef struct { int content_checksum_flag; // Whether or not the output for this frame is in a single segment int single_segment_flag; - - // The size in bytes of this header - int header_size; } frame_header_t; /// The context needed to decode blocks in a frame @@ -256,9 +296,8 @@ typedef struct { FSE_dtable ml_dtable; FSE_dtable of_dtable; - // The last 3 offsets for the special "repeat offsets". Array size is 4 so - // that previous_offsets[1] corresponds to the most recent offset - u64 previous_offsets[4]; + // The last 3 offsets for the special "repeat offsets". + u64 previous_offsets[3]; } frame_context_t; /// The decoded contents of a dictionary so that it doesn't have to be repeated @@ -275,7 +314,7 @@ typedef struct { size_t content_size; // Offset history to prepopulate the frame's history - u64 previous_offsets[4]; + u64 previous_offsets[3]; u32 dictionary_id; } dictionary_t; @@ -301,34 +340,31 @@ typedef struct { /// Accepts a dict argument, which may be NULL indicating no dictionary. /// See /// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#frame-concatenation -static void decode_frame(io_streams_t *const streams, +static void decode_frame(ostream_t *const out, istream_t *const in, const dictionary_t *const dict); // Decode data in a compressed block -static void decompress_block(io_streams_t *const streams, - frame_context_t *const ctx, - const size_t block_len); +static void decompress_block(frame_context_t *const ctx, ostream_t *const out, + istream_t *const in); // Decode the literals section of a block -static size_t decode_literals(io_streams_t *const streams, - frame_context_t *const ctx, u8 **const literals); +static size_t decode_literals(frame_context_t *const ctx, istream_t *const in, + u8 **const literals); // Decode the sequences part of a block -static size_t decode_sequences(frame_context_t *const ctx, const u8 *const src, - const size_t src_len, +static size_t decode_sequences(frame_context_t *const ctx, istream_t *const in, sequence_command_t **const sequences); // Execute the decoded sequences on the literals block -static void execute_sequences(io_streams_t *const streams, - frame_context_t *const ctx, +static void execute_sequences(frame_context_t *const ctx, ostream_t *const out, + const u8 *const literals, + const size_t literals_len, const sequence_command_t *const sequences, - const size_t num_sequences, - const u8 *literals, - size_t literals_len); + const size_t num_sequences); // Parse a provided dictionary blob for use in decompression -static void parse_dictionary(dictionary_t *const dict, const u8 *const src, - const size_t src_len); +static void parse_dictionary(dictionary_t *const dict, const u8 *src, + size_t src_len); static void free_dictionary(dictionary_t *const dict); /******* END ZSTD HELPER STRUCTS AND PROTOTYPES *******************************/ @@ -348,58 +384,46 @@ size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len, parse_dictionary(&parsed_dict, (const u8 *)dict, dict_len); } - io_streams_t streams = {(u8 *)dst, dst_len, (const u8 *)src, src_len}; - while (streams.src_len > 0) { - decode_frame(&streams, &parsed_dict); + istream_t in = {(const u8 *)src, 0, src_len}; + ostream_t out = {(u8 *)dst, dst_len}; + while (IO_istream_len(&in) > 0) { + decode_frame(&out, &in, &parsed_dict); } free_dictionary(&parsed_dict); - return streams.dst - (u8 *)dst; + return out.ptr - (u8 *)dst; } /******* FRAME DECODING ******************************************************/ -static void decode_data_frame(io_streams_t *const streams, +static void decode_data_frame(ostream_t *const out, istream_t *const in, const dictionary_t *const dict); -static void init_frame_context(io_streams_t *const streams, - frame_context_t *const context, +static void init_frame_context(frame_context_t *const context, + istream_t *const in, const dictionary_t *const dict); static void free_frame_context(frame_context_t *const context); static void parse_frame_header(frame_header_t *const header, - const u8 *const src, const size_t src_len); + istream_t *const in); static void frame_context_apply_dict(frame_context_t *const ctx, const dictionary_t *const dict); -static void decompress_data(io_streams_t *const streams, - frame_context_t *const ctx); +static void decompress_data(frame_context_t *const ctx, ostream_t *const out, + istream_t *const in); -static void decode_frame(io_streams_t *const streams, +static void decode_frame(ostream_t *const out, istream_t *const in, const dictionary_t *const dict) { - if (streams->src_len < 4) { - INP_SIZE(); - } - const u32 magic_number = read_bits_LE(streams->src, 32, 0); + const u32 magic_number = IO_read_bits(in, 32); - streams->src += 4; - streams->src_len -= 4; - if (magic_number >= 0x184D2A50U && magic_number <= 0x184D2A5F) { - // skippable frame - if (streams->src_len < 4) { - INP_SIZE(); - } - const size_t frame_size = read_bits_LE(streams->src, 32, 32); - - if (streams->src_len < 4 + frame_size) { - INP_SIZE(); - } + if ((magic_number & ~0xFU) == 0x184D2A50U) { + // Skippable frame + const size_t frame_size = IO_read_bits(in, 32); // skip over frame - streams->src += 4 + frame_size; - streams->src_len -= 4 + frame_size; + IO_advance_input(in, frame_size); } else if (magic_number == 0xFD2FB528U) { // ZSTD frame - decode_data_frame(streams, dict); + decode_data_frame(out, in, dict); } else { // not a real frame ERROR("Invalid magic number"); @@ -410,40 +434,38 @@ static void decode_frame(io_streams_t *const streams, /// are skippable frames. /// See /// https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#general-structure-of-zstandard-frame-format -static void decode_data_frame(io_streams_t *const streams, +static void decode_data_frame(ostream_t *const out, istream_t *const in, const dictionary_t *const dict) { frame_context_t ctx; // Initialize the context that needs to be carried from block to block - init_frame_context(streams, &ctx, dict); + init_frame_context(&ctx, in, dict); if (ctx.header.frame_content_size != 0 && - ctx.header.frame_content_size > streams->dst_len) { + ctx.header.frame_content_size > out->len) { OUT_SIZE(); } - decompress_data(streams, &ctx); + decompress_data(&ctx, out, in); free_frame_context(&ctx); } /// Takes the information provided in the header and dictionary, and initializes /// the context for this frame -static void init_frame_context(io_streams_t *const streams, - frame_context_t *const context, +static void init_frame_context(frame_context_t *const context, + istream_t *const in, const dictionary_t *const dict) { // Most fields in context are correct when initialized to 0 - memset(context, 0x00, sizeof(frame_context_t)); + memset(context, 0, sizeof(frame_context_t)); // Parse data from the frame header - parse_frame_header(&context->header, streams->src, streams->src_len); - streams->src += context->header.header_size; - streams->src_len -= context->header.header_size; + parse_frame_header(&context->header, in); // Set up the offset history for the repeat offset commands - context->previous_offsets[1] = 1; - context->previous_offsets[2] = 4; - context->previous_offsets[3] = 8; + context->previous_offsets[0] = 1; + context->previous_offsets[1] = 4; + context->previous_offsets[2] = 8; // Apply details from the dict if it exists frame_context_apply_dict(context, dict); @@ -460,12 +482,8 @@ static void free_frame_context(frame_context_t *const context) { } static void parse_frame_header(frame_header_t *const header, - const u8 *const src, const size_t src_len) { - if (src_len < 1) { - INP_SIZE(); - } - - const u8 descriptor = read_bits_LE(src, 8, 0); + istream_t *const in) { + const u8 descriptor = IO_read_bits(in, 8); // decode frame header descriptor into flags const u8 frame_content_size_flag = descriptor >> 6; @@ -478,28 +496,20 @@ static void parse_frame_header(frame_header_t *const header, CORRUPTION(); } - int header_size = 1; - header->single_segment_flag = single_segment_flag; header->content_checksum_flag = content_checksum_flag; // decode window size if (!single_segment_flag) { - if (src_len < header_size + 1) { - INP_SIZE(); - } - // Use the algorithm from the specification to compute window size // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#window_descriptor - u8 window_descriptor = src[header_size]; + u8 window_descriptor = IO_read_bits(in, 8); u8 exponent = window_descriptor >> 3; u8 mantissa = window_descriptor & 7; size_t window_base = (size_t)1 << (10 + exponent); size_t window_add = (window_base / 8) * mantissa; header->window_size = window_base + window_add; - - header_size++; } // decode dictionary id if it exists @@ -507,13 +517,7 @@ static void parse_frame_header(frame_header_t *const header, const int bytes_array[] = {0, 1, 2, 4}; const int bytes = bytes_array[dictionary_id_flag]; - if (src_len < header_size + bytes) { - INP_SIZE(); - } - - header->dictionary_id = read_bits_LE(src + header_size, bytes * 8, 0); - - header_size += bytes; + header->dictionary_id = IO_read_bits(in, bytes * 8); } else { header->dictionary_id = 0; } @@ -525,17 +529,10 @@ static void parse_frame_header(frame_header_t *const header, const int bytes_array[] = {1, 2, 4, 8}; const int bytes = bytes_array[frame_content_size_flag]; - if (src_len < header_size + bytes) { - INP_SIZE(); - } - - header->frame_content_size = - read_bits_LE(src + header_size, bytes * 8, 0); + header->frame_content_size = IO_read_bits(in, bytes * 8); if (bytes == 2) { header->frame_content_size += 256; } - - header_size += bytes; } else { header->frame_content_size = 0; } @@ -546,8 +543,6 @@ static void parse_frame_header(frame_header_t *const header, // back to the dictionary or not on large offsets header->window_size = header->frame_content_size; } - - header->header_size = header_size; } /// A dictionary acts as initializing values for the frame context before @@ -559,20 +554,15 @@ static void frame_context_apply_dict(frame_context_t *const ctx, if (!dict || !dict->content) return; - if (ctx->header.dictionary_id == 0 && dict->dictionary_id != 0) { - // The dictionary is unneeded, and shouldn't be used as it may interfere - // with the default offset history - return; - } - - // If the dictionary id is 0, it doesn't matter if we provide the wrong raw - // content dict, it won't change anything + // If the requested dictionary_id is non-zero, the correct dictionary must + // be present if (ctx->header.dictionary_id != 0 && ctx->header.dictionary_id != dict->dictionary_id) { - ERROR("Wrong/no dictionary provided"); + ERROR("Wrong dictionary provided"); } - // Copy the pointer in so we can reference it in sequence execution + // Copy the dict content to the context for references during sequence + // execution ctx->dict_content = dict->content; ctx->dict_content_len = dict->content_size; @@ -592,188 +582,137 @@ static void frame_context_apply_dict(frame_context_t *const ctx, } /// Decompress the data from a frame block by block -static void decompress_data(io_streams_t *const streams, - frame_context_t *const ctx) { +static void decompress_data(frame_context_t *const ctx, ostream_t *const out, + istream_t *const in) { int last_block = 0; do { - if (streams->src_len < 3) { - INP_SIZE(); - } // Parse the block header - last_block = streams->src[0] & 1; - const int block_type = (streams->src[0] >> 1) & 3; - const size_t block_len = read_bits_LE(streams->src, 21, 3); - - streams->src += 3; - streams->src_len -= 3; + last_block = IO_read_bits(in, 1); + const int block_type = IO_read_bits(in, 2); + const size_t block_len = IO_read_bits(in, 21); switch (block_type) { case 0: { // Raw, uncompressed block - if (streams->src_len < block_len) { - INP_SIZE(); - } - if (streams->dst_len < block_len) { - OUT_SIZE(); - } - + const u8 *const read_ptr = IO_read_bytes(in, block_len); + u8 *const write_ptr = IO_write_bytes(out, block_len); + // // Copy the raw data into the output - memcpy(streams->dst, streams->src, block_len); - - streams->src += block_len; - streams->src_len -= block_len; - - streams->dst += block_len; - streams->dst_len -= block_len; + memcpy(write_ptr, read_ptr, block_len); ctx->current_total_output += block_len; break; } case 1: { // RLE block, repeat the first byte N times - if (streams->src_len < 1) { - INP_SIZE(); - } - if (streams->dst_len < block_len) { - OUT_SIZE(); - } + const u8 *const read_ptr = IO_read_bytes(in, 1); + u8 *const write_ptr = IO_write_bytes(out, block_len); // Copy `block_len` copies of `streams->src[0]` to the output - memset(streams->dst, streams->src[0], block_len); - - streams->dst += block_len; - streams->dst_len -= block_len; - - streams->src += 1; - streams->src_len -= 1; + memset(write_ptr, read_ptr[0], block_len); ctx->current_total_output += block_len; break; } - case 2: - // Compressed block, this is mode complex - decompress_block(streams, ctx, block_len); + case 2: { + // Compressed block + // Create a sub-stream for the block + istream_t block_stream = IO_make_sub_istream(in, block_len); + decompress_block(ctx, out, &block_stream); break; + } case 3: // Reserved block type CORRUPTION(); break; + default: + IMPOSSIBLE(); } } while (!last_block); if (ctx->header.content_checksum_flag) { // This program does not support checking the checksum, so skip over it // if it's present - if (streams->src_len < 4) { - INP_SIZE(); - } - streams->src += 4; - streams->src_len -= 4; + IO_advance_input(in, 4); } } /******* END FRAME DECODING ***************************************************/ /******* BLOCK DECOMPRESSION **************************************************/ -static void decompress_block(io_streams_t *const streams, frame_context_t *const ctx, - const size_t block_len) { - if (streams->src_len < block_len) { - INP_SIZE(); - } - // We need this to determine how long the compressed literals block was - const u8 *const end_of_block = streams->src + block_len; - +static void decompress_block(frame_context_t *const ctx, ostream_t *const out, + istream_t *const in) { // Part 1: decode the literals block u8 *literals = NULL; - const size_t literals_size = decode_literals(streams, ctx, &literals); + const size_t literals_size = decode_literals(ctx, in, &literals); // Part 2: decode the sequences block - if (streams->src > end_of_block) { - INP_SIZE(); - } - const size_t sequences_size = end_of_block - streams->src; sequence_command_t *sequences = NULL; const size_t num_sequences = - decode_sequences(ctx, streams->src, sequences_size, &sequences); - - streams->src += sequences_size; - streams->src_len -= sequences_size; + decode_sequences(ctx, in, &sequences); // Part 3: combine literals and sequence commands to generate output - execute_sequences(streams, ctx, sequences, num_sequences, literals, - literals_size); + execute_sequences(ctx, out, literals, literals_size, sequences, + num_sequences); free(literals); free(sequences); } /******* END BLOCK DECOMPRESSION **********************************************/ /******* LITERALS DECODING ****************************************************/ -static size_t decode_literals_simple(io_streams_t *const streams, - u8 **const literals, const int block_type, +static size_t decode_literals_simple(istream_t *const in, u8 **const literals, + const int block_type, const int size_format); -static size_t decode_literals_compressed(io_streams_t *const streams, - frame_context_t *const ctx, +static size_t decode_literals_compressed(frame_context_t *const ctx, + istream_t *const in, u8 **const literals, const int block_type, const int size_format); -static size_t decode_huf_table(const u8 *src, size_t src_len, - HUF_dtable *const dtable); -static size_t fse_decode_hufweights(const u8 *const src, const size_t src_len, - u8 *const weights, int *const num_symbs, - const size_t compressed_size); +static void decode_huf_table(istream_t *const in, HUF_dtable *const dtable); +static void fse_decode_hufweights(ostream_t *weights, istream_t *const in, + int *const num_symbs); -static size_t decode_literals(io_streams_t *const streams, - frame_context_t *const ctx, u8 **const literals) { - if (streams->src_len < 1) { - INP_SIZE(); - } +static size_t decode_literals(frame_context_t *const ctx, istream_t *const in, + u8 **const literals) { // Decode literals header - int block_type = streams->src[0] & 3; - int size_format = (streams->src[0] >> 2) & 3; + int block_type = IO_read_bits(in, 2); + int size_format = IO_read_bits(in, 2); if (block_type <= 1) { // Raw or RLE literals block - return decode_literals_simple(streams, literals, block_type, + return decode_literals_simple(in, literals, block_type, size_format); } else { // Huffman compressed literals - return decode_literals_compressed(streams, ctx, literals, block_type, + return decode_literals_compressed(ctx, in, literals, block_type, size_format); } } /// Decodes literals blocks in raw or RLE form -static size_t decode_literals_simple(io_streams_t *const streams, - u8 **const literals, const int block_type, +static size_t decode_literals_simple(istream_t *const in, u8 **const literals, + const int block_type, const int size_format) { size_t size; switch (size_format) { - // These cases are in the form X0 - // In this case, the X bit is actually part of the size field + // These cases are in the form ?0 + // In this case, the ? bit is actually part of the size field case 0: case 2: - size = read_bits_LE(streams->src, 5, 3); - streams->src += 1; - streams->src_len -= 1; + // "Size_Format uses 1 bit. Regenerated_Size uses 5 bits (0-31)." + IO_rewind_bits(in, 1); + size = IO_read_bits(in, 2); break; case 1: - if (streams->src_len < 2) { - INP_SIZE(); - } - size = read_bits_LE(streams->src, 12, 4); - streams->src += 2; - streams->src_len -= 2; + // "Size_Format uses 2 bits. Regenerated_Size uses 12 bits (0-4095)." + size = IO_read_bits(in, 12); break; case 3: - if (streams->src_len < 2) { - INP_SIZE(); - } - size = read_bits_LE(streams->src, 20, 4); - streams->src += 3; - streams->src_len -= 3; + // "Size_Format uses 2 bits. Regenerated_Size uses 20 bits (0-1048575)." + size = IO_read_bits(in, 20); break; default: - // Impossible - size = -1; + // Size format is in range 0-3 + IMPOSSIBLE(); } if (size > MAX_LITERALS_SIZE) { @@ -786,32 +725,28 @@ static size_t decode_literals_simple(io_streams_t *const streams, } switch (block_type) { - case 0: + case 0: { // Raw data - if (size > streams->src_len) { - INP_SIZE(); - } - memcpy(*literals, streams->src, size); - streams->src += size; - streams->src_len -= size; + const u8 *const read_ptr = IO_read_bytes(in, size); + memcpy(*literals, read_ptr, size); break; - case 1: + } + case 1: { // Single repeated byte - if (1 > streams->src_len) { - INP_SIZE(); - } - memset(*literals, streams->src[0], size); - streams->src += 1; - streams->src_len -= 1; + const u8 *const read_ptr = IO_read_bytes(in, 1); + memset(*literals, read_ptr[0], size); break; } + default: + IMPOSSIBLE(); + } return size; } /// Decodes Huffman compressed literals -static size_t decode_literals_compressed(io_streams_t *const streams, - frame_context_t *const ctx, +static size_t decode_literals_compressed(frame_context_t *const ctx, + istream_t *const in, u8 **const literals, const int block_type, const int size_format) { @@ -820,98 +755,78 @@ static size_t decode_literals_compressed(io_streams_t *const streams, int num_streams = 4; switch (size_format) { case 0: + // "A single stream. Both Compressed_Size and Regenerated_Size use 10 + // bits (0-1023)." num_streams = 1; // Fall through as it has the same size format case 1: - if (streams->src_len < 3) { - INP_SIZE(); - } - regenerated_size = read_bits_LE(streams->src, 10, 4); - compressed_size = read_bits_LE(streams->src, 10, 14); - streams->src += 3; - streams->src_len -= 3; + // "4 streams. Both Compressed_Size and Regenerated_Size use 10 bits + // (0-1023)." + regenerated_size = IO_read_bits(in, 10); + compressed_size = IO_read_bits(in, 10); break; case 2: - if (streams->src_len < 4) { - INP_SIZE(); - } - regenerated_size = read_bits_LE(streams->src, 14, 4); - compressed_size = read_bits_LE(streams->src, 14, 18); - streams->src += 4; - streams->src_len -= 4; + // "4 streams. Both Compressed_Size and Regenerated_Size use 14 bits + // (0-16383)." + regenerated_size = IO_read_bits(in, 14); + compressed_size = IO_read_bits(in, 14); break; case 3: - if (streams->src_len < 5) { - INP_SIZE(); - } - regenerated_size = read_bits_LE(streams->src, 18, 4); - compressed_size = read_bits_LE(streams->src, 18, 22); - streams->src += 5; - streams->src_len -= 5; + // "4 streams. Both Compressed_Size and Regenerated_Size use 18 bits + // (0-262143)." + regenerated_size = IO_read_bits(in, 18); + compressed_size = IO_read_bits(in, 18); break; default: // Impossible - compressed_size = regenerated_size = -1; + IMPOSSIBLE(); } if (regenerated_size > MAX_LITERALS_SIZE || compressed_size > regenerated_size) { CORRUPTION(); } - if (compressed_size > streams->src_len) { - INP_SIZE(); - } - *literals = malloc(regenerated_size); if (!*literals) { BAD_ALLOC(); } + ostream_t lit_stream = IO_make_ostream(*literals, regenerated_size); + istream_t huf_stream = IO_make_sub_istream(in, compressed_size); + if (block_type == 2) { // Decode provided Huffman table HUF_free_dtable(&ctx->literals_dtable); - const size_t size = decode_huf_table(streams->src, compressed_size, - &ctx->literals_dtable); - streams->src += size; - streams->src_len -= size; - compressed_size -= size; + decode_huf_table(&huf_stream, &ctx->literals_dtable); } else { - // If we're to repeat the previous Huffman table, make sure it exists + // If the previous Huffman table is being repeated, ensure it exists if (!ctx->literals_dtable.symbols) { CORRUPTION(); } } + size_t symbols_decoded; if (num_streams == 1) { - HUF_decompress_1stream(&ctx->literals_dtable, *literals, - regenerated_size, streams->src, compressed_size); + symbols_decoded = HUF_decompress_1stream(&ctx->literals_dtable, &lit_stream, &huf_stream); } else { - HUF_decompress_4stream(&ctx->literals_dtable, *literals, - regenerated_size, streams->src, compressed_size); + symbols_decoded = HUF_decompress_4stream(&ctx->literals_dtable, &lit_stream, &huf_stream); + } + + if (symbols_decoded != regenerated_size) { + CORRUPTION(); } - streams->src += compressed_size; - streams->src_len -= compressed_size; return regenerated_size; } // Decode the Huffman table description -static size_t decode_huf_table(const u8 *src, size_t src_len, - HUF_dtable *const dtable) { - if (src_len < 1) { - INP_SIZE(); - } +static void decode_huf_table(istream_t *const in, HUF_dtable *const dtable) { + const u8 header = IO_read_bits(in, 8); - const u8 *const osrc = src; - - const u8 header = src[0]; u8 weights[HUF_MAX_SYMBS]; memset(weights, 0, sizeof(weights)); - src++; - src_len--; - int num_symbs; if (header >= 128) { @@ -919,67 +834,56 @@ static size_t decode_huf_table(const u8 *src, size_t src_len, num_symbs = header - 127; const size_t bytes = (num_symbs + 1) / 2; - if (bytes > src_len) { - INP_SIZE(); - } + const u8 *const weight_src = IO_read_bytes(in, bytes); for (int i = 0; i < num_symbs; i++) { // read_bits_LE isn't applicable here because the weights are order // reversed within each byte // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#huffman-tree-header if (i % 2 == 0) { - weights[i] = src[i / 2] >> 4; + weights[i] = weight_src[i / 2] >> 4; } else { - weights[i] = src[i / 2] & 0xf; + weights[i] = weight_src[i / 2] & 0xf; } } - - src += bytes; - src_len -= bytes; } else { // The weights are FSE encoded, decode them before we can construct the // table - const size_t size = - fse_decode_hufweights(src, src_len, weights, &num_symbs, header); - src += size; - src_len -= size; + istream_t fse_stream = IO_make_sub_istream(in, header); + ostream_t weight_stream = IO_make_ostream(weights, HUF_MAX_SYMBS); + fse_decode_hufweights(&weight_stream, &fse_stream, &num_symbs); } // Construct the table using the decoded weights HUF_init_dtable_usingweights(dtable, weights, num_symbs); - return src - osrc; } -static size_t fse_decode_hufweights(const u8 *const src, const size_t src_len, - u8 *const weights, int *const num_symbs, - const size_t compressed_size) { +static void fse_decode_hufweights(ostream_t *weights, istream_t *const in, + int *const num_symbs) { const int MAX_ACCURACY_LOG = 7; FSE_dtable dtable; // Construct the FSE table - const size_t read = - FSE_decode_header(&dtable, src, src_len, MAX_ACCURACY_LOG); - - if (src_len < compressed_size) { - INP_SIZE(); - } + FSE_decode_header(&dtable, in, MAX_ACCURACY_LOG); // Decode the weights - *num_symbs = FSE_decompress_interleaved2( - &dtable, weights, HUF_MAX_SYMBS, src + read, compressed_size - read); + *num_symbs = FSE_decompress_interleaved2(&dtable, weights, in); FSE_free_dtable(&dtable); - - return compressed_size; } /******* END LITERALS DECODING ************************************************/ /******* SEQUENCE DECODING ****************************************************/ /// The combination of FSE states needed to decode sequences typedef struct { - u16 ll_state, of_state, ml_state; - FSE_dtable ll_table, of_table, ml_table; + FSE_dtable ll_table; + FSE_dtable of_table; + FSE_dtable ml_table; + + u16 ll_state; + u16 of_state; + u16 ml_state; } sequence_state_t; /// Different modes to signal to decode_seq_tables what to do @@ -1031,47 +935,36 @@ static const u8 SEQ_MATCH_LENGTH_EXTRA_BITS[53] = { /// Offset decoding is simpler so we just need a maximum code value static const u8 SEQ_MAX_CODES[3] = {35, -1, 52}; -static void decompress_sequences(frame_context_t *const ctx, const u8 *src, - size_t src_len, +static void decompress_sequences(frame_context_t *const ctx, + istream_t *const in, sequence_command_t *const sequences, const size_t num_sequences); static sequence_command_t decode_sequence(sequence_state_t *const state, const u8 *const src, i64 *const offset); -static size_t decode_seq_table(const u8 *src, size_t src_len, - FSE_dtable *const table, const seq_part_t type, - const seq_mode_t mode); +static void decode_seq_table(istream_t *const in, FSE_dtable *const table, + const seq_part_t type, const seq_mode_t mode); -static size_t decode_sequences(frame_context_t *const ctx, const u8 *src, - size_t src_len, +static size_t decode_sequences(frame_context_t *const ctx, istream_t *in, sequence_command_t **const sequences) { size_t num_sequences; // Decode the sequence header and allocate space for the output - if (src_len < 1) { - INP_SIZE(); - } - if (src[0] == 0) { + u8 header = IO_read_bits(in, 8); + if (header == 0) { + // "There are no sequences. The sequence section stops there. + // Regenerated content is defined entirely by literals section." *sequences = NULL; return 0; - } else if (src[0] < 128) { - num_sequences = src[0]; - src++; - src_len--; - } else if (src[0] < 255) { - if (src_len < 2) { - INP_SIZE(); - } - num_sequences = ((src[0] - 128) << 8) + src[1]; - src += 2; - src_len -= 2; + } else if (header < 128) { + // "Number_of_Sequences = byte0 . Uses 1 byte." + num_sequences = header; + } else if (header < 255) { + // "Number_of_Sequences = ((byte0-128) << 8) + byte1 . Uses 2 bytes." + num_sequences = ((header - 128) << 8) + IO_read_bits(in, 8); } else { - if (src_len < 3) { - INP_SIZE(); - } - num_sequences = src[1] + ((u64)src[2] << 8) + 0x7F00; - src += 3; - src_len -= 3; + // "Number_of_Sequences = byte1 + (byte2<<8) + 0x7F00 . Uses 3 bytes." + num_sequences = IO_read_bits(in, 16) + 0x7F00; } *sequences = malloc(num_sequences * sizeof(sequence_command_t)); @@ -1079,51 +972,29 @@ static size_t decode_sequences(frame_context_t *const ctx, const u8 *src, BAD_ALLOC(); } - decompress_sequences(ctx, src, src_len, *sequences, num_sequences); + decompress_sequences(ctx, in, *sequences, num_sequences); return num_sequences; } /// Decompress the FSE encoded sequence commands -static void decompress_sequences(frame_context_t *const ctx, const u8 *src, - size_t src_len, +static void decompress_sequences(frame_context_t *const ctx, istream_t *in, sequence_command_t *const sequences, const size_t num_sequences) { - if (src_len < 1) { - INP_SIZE(); - } - u8 compression_modes = src[0]; - src++; - src_len--; + u8 compression_modes = IO_read_bits(in, 8); if ((compression_modes & 3) != 0) { CORRUPTION(); } - { - size_t read; - // Update the tables we have stored in the context - read = decode_seq_table(src, src_len, &ctx->ll_dtable, - seq_literal_length, - (compression_modes >> 6) & 3); - src += read; - src_len -= read; - } + // Update the tables we have stored in the context + decode_seq_table(in, &ctx->ll_dtable, seq_literal_length, + (compression_modes >> 6) & 3); - { - const size_t read = - decode_seq_table(src, src_len, &ctx->of_dtable, seq_offset, - (compression_modes >> 4) & 3); - src += read; - src_len -= read; - } + decode_seq_table(in, &ctx->of_dtable, seq_offset, + (compression_modes >> 4) & 3); - { - const size_t read = decode_seq_table(src, src_len, &ctx->ml_dtable, - seq_match_length, - (compression_modes >> 2) & 3); - src += read; - src_len -= read; - } + decode_seq_table(in, &ctx->ml_dtable, seq_match_length, + (compression_modes >> 2) & 3); // Check to make sure none of the tables are uninitialized if (!ctx->ll_dtable.symbols || !ctx->of_dtable.symbols || @@ -1137,8 +1008,13 @@ static void decompress_sequences(frame_context_t *const ctx, const u8 *src, memcpy(&state.of_table, &ctx->of_dtable, sizeof(FSE_dtable)); memcpy(&state.ml_table, &ctx->ml_dtable, sizeof(FSE_dtable)); - const int padding = 8 - log2inf(src[src_len - 1]); - i64 offset = src_len * 8 - padding; + size_t len = IO_istream_len(in); + const u8 *const src = IO_read_bytes(in, len); + + // "After writing the last bit containing information, the compressor writes + // a single 1-bit and then fills the byte with 0-7 0 bits of padding." + const int padding = 8 - log2inf(src[len - 1]); + i64 offset = len * 8 - padding; FSE_init_state(&state.ll_table, &state.ll_state, src, &offset); FSE_init_state(&state.of_table, &state.of_state, src, &offset); @@ -1153,7 +1029,7 @@ static void decompress_sequences(frame_context_t *const ctx, const u8 *src, CORRUPTION(); } - // Don't free our tables so they can be used in the next block + // Don't free tables so they can be used in the next block } // Decode a single sequence and update the state @@ -1194,9 +1070,8 @@ static sequence_command_t decode_sequence(sequence_state_t *const state, } /// Given a sequence part and table mode, decode the FSE distribution -static size_t decode_seq_table(const u8 *src, size_t src_len, - FSE_dtable *const table, const seq_part_t type, - const seq_mode_t mode) { +static void decode_seq_table(istream_t *const in, FSE_dtable *const table, + const seq_part_t type, const seq_mode_t mode) { // Constant arrays indexed by seq_part_t const i16 *const default_distributions[] = {SEQ_LITERAL_LENGTH_DEFAULT_DIST, SEQ_OFFSET_DEFAULT_DIST, @@ -1207,7 +1082,7 @@ static size_t decode_seq_table(const u8 *src, size_t src_len, const size_t max_accuracies[] = {9, 8, 9}; if (mode != seq_repeat) { - // ree old one before overwriting + // Free old one before overwriting FSE_free_dtable(table); } @@ -1218,102 +1093,102 @@ static size_t decode_seq_table(const u8 *src, size_t src_len, const size_t accuracy_log = default_distribution_accuracies[type]; FSE_init_dtable(table, distribution, symbs, accuracy_log); - - return 0; + break; } case seq_rle: { - if (src_len < 1) { - INP_SIZE(); - } - const u8 symb = src[0]; - src++; - src_len--; + const u8 symb = IO_read_bits(in, 8); FSE_init_dtable_rle(table, symb); - - return 1; + break; } case seq_fse: { - size_t read = - FSE_decode_header(table, src, src_len, max_accuracies[type]); - src += read; - src_len -= read; - - return read; + FSE_decode_header(table, in, max_accuracies[type]); + break; } case seq_repeat: - // Don't have to do anything here as we're not changing the table - return 0; + // Nothing to do here, table will be unchanged + break; default: // Impossible, as mode is from 0-3 - return -1; + IMPOSSIBLE(); + break; } } /******* END SEQUENCE DECODING ************************************************/ /******* SEQUENCE EXECUTION ***************************************************/ -static void execute_sequences(io_streams_t *const streams, - frame_context_t *const ctx, +static void execute_sequences(frame_context_t *const ctx, ostream_t *const out, + const u8 *const literals, + const size_t literals_len, const sequence_command_t *const sequences, - const size_t num_sequences, - const u8 *literals, - size_t literals_len) { + const size_t num_sequences) { + istream_t litstream = IO_make_istream(literals, literals_len); + u64 *const offset_hist = ctx->previous_offsets; size_t total_output = ctx->current_total_output; for (size_t i = 0; i < num_sequences; i++) { const sequence_command_t seq = sequences[i]; - if (seq.literal_length > literals_len) { - CORRUPTION(); + { + if (seq.literal_length > IO_istream_len(&litstream)) { + CORRUPTION(); + } + + u8 *const write_ptr = IO_write_bytes(out, seq.literal_length); + const u8 *const read_ptr = + IO_read_bytes(&litstream, seq.literal_length); + // Copy literals to output + memcpy(write_ptr, read_ptr, seq.literal_length); + + total_output += seq.literal_length; } - if (streams->dst_len < seq.literal_length + seq.match_length) { - OUT_SIZE(); - } - // Copy literals to output - memcpy(streams->dst, literals, seq.literal_length); - - literals += seq.literal_length; - literals_len -= seq.literal_length; - - streams->dst += seq.literal_length; - streams->dst_len -= seq.literal_length; - - total_output += seq.literal_length; - size_t offset; // Offsets are special, we need to handle the repeat offsets if (seq.offset <= 3) { - u32 idx = seq.offset; + // "The first 3 values define a repeated offset and we will call + // them Repeated_Offset1, Repeated_Offset2, and Repeated_Offset3. + // They are sorted in recency order, with Repeated_Offset1 meaning + // 'most recent one'". + + // Use 0 indexing for the array + u32 idx = seq.offset - 1; if (seq.literal_length == 0) { - // Special case when literal length is 0 + // "There is an exception though, when current sequence's + // literals length is 0. In this case, repeated offsets are + // shifted by one, so Repeated_Offset1 becomes Repeated_Offset2, + // Repeated_Offset2 becomes Repeated_Offset3, and + // Repeated_Offset3 becomes Repeated_Offset1 - 1_byte." idx++; } - if (idx == 1) { - offset = offset_hist[1]; + if (idx == 0) { + offset = offset_hist[0]; } else { - // If idx == 4 then literal length was 0 and the offset was 3 - offset = idx < 4 ? offset_hist[idx] : offset_hist[1] - 1; + // If idx == 3 then literal length was 0 and the offset was 3, + // as per the exception listed above + offset = idx < 3 ? offset_hist[idx] : offset_hist[0] - 1; - // If idx == 2 we don't need to modify offset_hist[3] - if (idx > 2) { - offset_hist[3] = offset_hist[2]; + // If idx == 1 we don't need to modify offset_hist[2] + if (idx > 1) { + offset_hist[2] = offset_hist[1]; } - offset_hist[2] = offset_hist[1]; - offset_hist[1] = offset; + offset_hist[1] = offset_hist[0]; + offset_hist[0] = offset; } } else { offset = seq.offset - 3; // Shift back history - offset_hist[3] = offset_hist[2]; offset_hist[2] = offset_hist[1]; - offset_hist[1] = offset; + offset_hist[1] = offset_hist[0]; + offset_hist[0] = offset; } size_t match_length = seq.match_length; + + u8 *write_ptr = IO_write_bytes(out, match_length); if (total_output <= ctx->header.window_size) { // In this case offset might go back into the dictionary if (offset > total_output + ctx->dict_content_len) { @@ -1322,13 +1197,16 @@ static void execute_sequences(io_streams_t *const streams, } if (offset > total_output) { + // "The rest of the dictionary is its content. The content act + // as a "past" in front of data to compress or decompress, so it + // can be referenced in sequence commands." const size_t dict_copy = MIN(offset - total_output, match_length); const size_t dict_offset = ctx->dict_content_len - (offset - total_output); - for (size_t i = 0; i < dict_copy; i++) { - *streams->dst++ = ctx->dict_content[dict_offset + i]; - } + + memcpy(write_ptr, ctx->dict_content + dict_offset, dict_copy); + write_ptr += dict_copy; match_length -= dict_copy; } } else if (offset > ctx->header.window_size) { @@ -1340,31 +1218,29 @@ static void execute_sequences(io_streams_t *const streams, // ex: if the output so far was "abc", a command with offset=3 and // match_length=6 would produce "abcabcabc" as the new output for (size_t i = 0; i < match_length; i++) { - *streams->dst = *(streams->dst - offset); - streams->dst++; + *write_ptr = *(write_ptr - offset); + write_ptr++; } - streams->dst_len -= seq.match_length; total_output += seq.match_length; } - if (streams->dst_len < literals_len) { - OUT_SIZE(); - } - // Copy any leftover literals - memcpy(streams->dst, literals, literals_len); - streams->dst += literals_len; - streams->dst_len -= literals_len; + { + size_t len = IO_istream_len(&litstream); + u8 *const write_ptr = IO_write_bytes(out, len); + const u8 *const read_ptr = IO_read_bytes(&litstream, len); + // Copy any leftover literals + memcpy(write_ptr, read_ptr, len); - total_output += literals_len; + total_output += len; + } ctx->current_total_output = total_output; } /******* END SEQUENCE EXECUTION ***********************************************/ /******* OUTPUT SIZE COUNTING *************************************************/ -size_t traverse_frame(const frame_header_t *const header, const u8 *src, - size_t src_len); +static void traverse_frame(const frame_header_t *const header, istream_t *const in); /// Get the decompressed size of an input stream so memory can be allocated in /// advance. @@ -1372,115 +1248,75 @@ size_t traverse_frame(const frame_header_t *const header, const u8 *src, /// implementation, as this API allows for the decompression of multiple /// concatenated frames. size_t ZSTD_get_decompressed_size(const void *src, const size_t src_len) { - const u8 *ip = (const u8 *) src; - size_t ip_len = src_len; - size_t dst_size = 0; + istream_t in = IO_make_istream(src, src_len); + size_t dst_size = 0; - // Each frame header only gives us the size of its frame, so iterate over all - // frames - while (ip_len > 0) { - if (ip_len < 4) { - INP_SIZE(); + // Each frame header only gives us the size of its frame, so iterate over + // all + // frames + while (IO_istream_len(&in) > 0) { + const u32 magic_number = IO_read_bits(&in, 32); + + if ((magic_number & ~0xFU) == 0x184D2A50U) { + // skippable frame, this has no impact on output size + const size_t frame_size = IO_read_bits(&in, 32); + IO_advance_input(&in, frame_size); + } else if (magic_number == 0xFD2FB528U) { + // ZSTD frame + frame_header_t header; + parse_frame_header(&header, &in); + + if (header.frame_content_size == 0 && !header.single_segment_flag) { + // Content size not provided, we can't tell + return -1; + } + + dst_size += header.frame_content_size; + + // Consume the input from the frame to reach the start of the next + traverse_frame(&header, &in); + } else { + // not a real frame + ERROR("Invalid magic number"); + } } - const u32 magic_number = read_bits_LE(ip, 32, 0); - - ip += 4; - ip_len -= 4; - if (magic_number >= 0x184D2A50U && magic_number <= 0x184D2A5F) { - // skippable frame, this has no impact on output size - if (ip_len < 4) { - INP_SIZE(); - } - const size_t frame_size = read_bits_LE(ip, 32, 32); - - if (ip_len < 4 + frame_size) { - INP_SIZE(); - } - - // skip over frame - ip += 4 + frame_size; - ip_len -= 4 + frame_size; - } else if (magic_number == 0xFD2FB528U) { - // ZSTD frame - frame_header_t header; - parse_frame_header(&header, ip, ip_len); - - if (header.frame_content_size == 0 && !header.single_segment_flag) { - // Content size not provided, we can't tell - return -1; - } - - dst_size += header.frame_content_size; - - // we need to traverse the frame to find when the next one starts - const size_t traversed = traverse_frame(&header, ip, ip_len); - ip += traversed; - ip_len -= traversed; - } else { - // not a real frame - ERROR("Invalid magic number"); - } - } - - return dst_size; + return dst_size; } /// Iterate over each block in a frame to find the end of it, to get to the /// start of the next frame -size_t traverse_frame(const frame_header_t *const header, const u8 *src, - size_t src_len) { - const u8 *const src_beg = src; - const u8 *const src_end = src + src_len; - src += header->header_size; - src_len += header->header_size; - +static void traverse_frame(const frame_header_t *const header, istream_t *const in) { int last_block = 0; do { - if (src + 3 > src_end) { - INP_SIZE(); - } // Parse the block header - last_block = src[0] & 1; - const int block_type = (src[0] >> 1) & 3; - const size_t block_len = read_bits_LE(src, 21, 3); + last_block = IO_read_bits(in, 1); + const int block_type = IO_read_bits(in, 2); + const size_t block_len = IO_read_bits(in, 21); - src += 3; switch (block_type) { case 0: // Raw block, block_len bytes - if (src + block_len > src_end) { - INP_SIZE(); - } - src += block_len; + IO_advance_input(in, block_len); break; case 1: // RLE block, 1 byte - if (src + 1 > src_end) { - INP_SIZE(); - } - src++; + IO_advance_input(in, 1); break; case 2: // Compressed block, compressed size is block_len - if (src + block_len > src_end) { - INP_SIZE(); - } - src += block_len; + IO_advance_input(in, block_len); break; case 3: // Reserved block type CORRUPTION(); break; + default: + IMPOSSIBLE(); } } while (!last_block); if (header->content_checksum_flag) { - if (src + 4 > src_end) { - INP_SIZE(); - } - src += 4; + IO_advance_input(in, 4); } - - return src - src_beg; } /******* END OUTPUT SIZE COUNTING *********************************************/ @@ -1495,68 +1331,46 @@ static void parse_dictionary(dictionary_t *const dict, const u8 *src, if (src_len < 8) { INP_SIZE(); } - const u32 magic_number = read_bits_LE(src, 32, 0); + + istream_t in = IO_make_istream(src, src_len); + + const u32 magic_number = IO_read_bits(&in, 32); if (magic_number != 0xEC30A437) { // raw content dict init_raw_content_dict(dict, src, src_len); return; } - dict->dictionary_id = read_bits_LE(src, 32, 32); - src += 8; - src_len -= 8; + dict->dictionary_id = IO_read_bits(&in, 32); // Parse the provided entropy tables in order - { - const size_t read = - decode_huf_table(src, src_len, &dict->literals_dtable); - src += read; - src_len -= read; - } - { - const size_t read = decode_seq_table(src, src_len, &dict->of_dtable, - seq_offset, seq_fse); - src += read; - src_len -= read; - } - { - const size_t read = decode_seq_table(src, src_len, &dict->ml_dtable, - seq_match_length, seq_fse); - src += read; - src_len -= read; - } - { - const size_t read = decode_seq_table(src, src_len, &dict->ll_dtable, - seq_literal_length, seq_fse); - src += read; - src_len -= read; - } + decode_huf_table(&in, &dict->literals_dtable); + decode_seq_table(&in, &dict->of_dtable, seq_offset, seq_fse); + decode_seq_table(&in, &dict->ml_dtable, seq_match_length, seq_fse); + decode_seq_table(&in, &dict->ll_dtable, seq_literal_length, seq_fse); - if (src_len < 12) { - INP_SIZE(); - } // Read in the previous offset history - dict->previous_offsets[1] = read_bits_LE(src, 32, 0); - dict->previous_offsets[2] = read_bits_LE(src, 32, 32); - dict->previous_offsets[3] = read_bits_LE(src, 32, 64); - - src += 12; - src_len -= 12; + dict->previous_offsets[0] = IO_read_bits(&in, 32); + dict->previous_offsets[1] = IO_read_bits(&in, 32); + dict->previous_offsets[2] = IO_read_bits(&in, 32); // Ensure the provided offsets aren't too large - for (int i = 1; i <= 3; i++) { + for (int i = 0; i < 3; i++) { if (dict->previous_offsets[i] > src_len) { ERROR("Dictionary corrupted"); } } + // The rest is the content - dict->content = malloc(src_len); + dict->content_size = IO_istream_len(&in); + dict->content = malloc(dict->content_size); if (!dict->content) { BAD_ALLOC(); } - dict->content_size = src_len; - memcpy(dict->content, src, src_len); + const u8 *const content = IO_read_bytes(&in, dict->content_size); + + memcpy(dict->content, content, dict->content_size); } /// If parse_dictionary is given a raw content dictionary, it delegates here @@ -1586,6 +1400,143 @@ static void free_dictionary(dictionary_t *const dict) { } /******* END DICTIONARY PARSING ***********************************************/ +/******* IO STREAM OPERATIONS *************************************************/ +#define UNALIGNED() ERROR("Attempting to operate on a non-byte aligned stream") +/// Reads `num` bits from a bitstream, and updates the internal offset +static inline u64 IO_read_bits(istream_t *const in, const int num) { + if (num > 64) { + return -1; + } + + const size_t bytes = (num + in->bit_offset + 7) / 8; + const size_t full_bytes = (num + in->bit_offset) / 8; + if (bytes > in->len) { + INP_SIZE(); + } + + const u64 result = read_bits_LE(in->ptr, num, in->bit_offset); + + in->bit_offset = (num + in->bit_offset) % 8; + in->ptr += full_bytes; + in->len -= full_bytes; + + return result; +} + +/// If a non-zero number of bits have been read from the current byte, advance +/// the offset to the next byte +static inline void IO_rewind_bits(istream_t *const in, int num) { + if (num < 0) { + ERROR("Attempting to rewind stream by a negative number of bits"); + } + + const int new_offset = in->bit_offset - num; + const i64 bytes = (new_offset - 7) / 8; + + in->ptr += bytes; + in->len -= bytes; + in->bit_offset = ((new_offset % 8) + 8) % 8; +} + +/// If the remaining bits in a byte will be unused, advance to the end of the +/// byte +static inline void IO_align_stream(istream_t *const in) { + if (in->bit_offset != 0) { + if (in->len == 0) { + INP_SIZE(); + } + in->ptr++; + in->len--; + in->bit_offset = 0; + } +} + +/// Write the given byte into the output stream +static inline void IO_write_byte(ostream_t *const out, u8 symb) { + if (out->len == 0) { + OUT_SIZE(); + } + + out->ptr[0] = symb; + out->ptr++; + out->len--; +} + +/// Returns the number of bytes left to be read in this stream. The stream must +/// be byte aligned. +static inline size_t IO_istream_len(const istream_t *const in) { + return in->len; +} + +/// Returns a pointer where `len` bytes can be read, and advances the internal +/// state. The stream must be byte aligned. +static inline const u8 *IO_read_bytes(istream_t *const in, size_t len) { + if (len > in->len) { + INP_SIZE(); + } + if (in->bit_offset != 0) { + UNALIGNED(); + } + const u8 *const ptr = in->ptr; + in->ptr += len; + in->len -= len; + + return ptr; +} +/// Returns a pointer to write `len` bytes to, and advances the internal state +static inline u8 *IO_write_bytes(ostream_t *const out, size_t len) { + if (len > out->len) { + INP_SIZE(); + } + u8 *const ptr = out->ptr; + out->ptr += len; + out->len -= len; + + return ptr; +} + +/// Advance the inner state by `len` bytes +static inline void IO_advance_input(istream_t *const in, size_t len) { + if (len > in->len) { + INP_SIZE(); + } + if (in->bit_offset != 0) { + UNALIGNED(); + } + + in->ptr += len; + in->len -= len; +} + +/// Returns an `ostream_t` constructed from the given pointer and length +static inline ostream_t IO_make_ostream(u8 *out, size_t len) { + return (ostream_t) { out, len }; +} + +/// Returns an `istream_t` constructed from the given pointer and length +static inline istream_t IO_make_istream(const u8 *in, size_t len) { + return (istream_t) { in, 0, len }; +} + +/// Returns an `istream_t` with the same base as `in`, and length `len` +/// Then, advance `in` to account for the consumed bytes +/// `in` must be byte aligned +static inline istream_t IO_make_sub_istream(istream_t *const in, size_t len) { + if (len > in->len) { + INP_SIZE(); + } + if (in->bit_offset != 0) { + UNALIGNED(); + } + const istream_t sub = { in->ptr, in->bit_offset, len }; + + in->ptr += len; + in->len -= len; + + return sub; +} +/******* END IO STREAM OPERATIONS *********************************************/ + /******* BITSTREAM OPERATIONS *************************************************/ /// Read `num` bits (up to 64) from `src + offset`, where `offset` is in bits static inline u64 read_bits_LE(const u8 *src, const int num, @@ -1676,28 +1627,29 @@ static inline void HUF_init_state(const HUF_dtable *const dtable, *state = STREAM_read_bits(src, bits, offset); } -static size_t HUF_decompress_1stream(const HUF_dtable *const dtable, u8 *dst, - const size_t dst_len, const u8 *src, - size_t src_len) { - const u8 *const dst_max = dst + dst_len; - const u8 *const odst = dst; +static size_t HUF_decompress_1stream(const HUF_dtable *const dtable, + ostream_t *const out, + istream_t *const in) { + const size_t len = IO_istream_len(in); + if (len == 0) { + INP_SIZE(); + } + const u8 *const src = IO_read_bytes(in, len); // To maintain similarity with FSE, start from the end // Find the last 1 bit - const int padding = 8 - log2inf(src[src_len - 1]); + const int padding = 8 - log2inf(src[len - 1]); - i64 offset = src_len * 8 - padding; + i64 offset = len * 8 - padding; u16 state; HUF_init_state(dtable, &state, src, &offset); - while (dst < dst_max && offset > -dtable->max_bits) { + size_t symbols_written = 0; + while (offset > -dtable->max_bits) { // Iterate over the stream, decoding one symbol at a time - *dst++ = HUF_decode_symbol(dtable, &state, src, &offset); - } - // If we stopped before consuming all the input, we didn't have enough space - if (dst == dst_max && offset > -dtable->max_bits) { - OUT_SIZE(); + IO_write_byte(out, HUF_decode_symbol(dtable, &state, src, &offset)); + symbols_written++; } // When all symbols have been decoded, the final state value shouldn't have @@ -1709,50 +1661,30 @@ static size_t HUF_decompress_1stream(const HUF_dtable *const dtable, u8 *dst, CORRUPTION(); } - return dst - odst; + return symbols_written; } -static size_t HUF_decompress_4stream(const HUF_dtable *const dtable, u8 *dst, - const size_t dst_len, const u8 *const src, - const size_t src_len) { - if (src_len < 6) { - INP_SIZE(); - } +static size_t HUF_decompress_4stream(const HUF_dtable *const dtable, + ostream_t *const out, istream_t *const in) { + const size_t csize1 = IO_read_bits(in, 16); + const size_t csize2 = IO_read_bits(in, 16); + const size_t csize3 = IO_read_bits(in, 16); - const u8 *const src1 = src + 6; - const u8 *const src2 = src1 + read_bits_LE(src, 16, 0); - const u8 *const src3 = src2 + read_bits_LE(src, 16, 16); - const u8 *const src4 = src3 + read_bits_LE(src, 16, 32); - const u8 *const src_end = src + src_len; - - // We can't test with all 4 sizes because the 4th size is a function of the - // other 3 and the provided length - if (src4 - src >= src_len) { - INP_SIZE(); - } - - const size_t segment_size = (dst_len + 3) / 4; - u8 *const dst1 = dst; - u8 *const dst2 = dst1 + segment_size; - u8 *const dst3 = dst2 + segment_size; - u8 *const dst4 = dst3 + segment_size; - u8 *const dst_end = dst + dst_len; - - size_t total_out = 0; + istream_t in1 = IO_make_sub_istream(in, csize1); + istream_t in2 = IO_make_sub_istream(in, csize2); + istream_t in3 = IO_make_sub_istream(in, csize3); + istream_t in4 = IO_make_sub_istream(in, IO_istream_len(in)); + size_t total_output = 0; // Decode each stream independently for simplicity // If we wanted to we could decode all 4 at the same time for speed, // utilizing more execution units - total_out += HUF_decompress_1stream(dtable, dst1, segment_size, src1, - src2 - src1); - total_out += HUF_decompress_1stream(dtable, dst2, segment_size, src2, - src3 - src2); - total_out += HUF_decompress_1stream(dtable, dst3, segment_size, src3, - src4 - src3); - total_out += HUF_decompress_1stream(dtable, dst4, dst_end - dst4, src4, - src_end - src4); + total_output += HUF_decompress_1stream(dtable, out, &in1); + total_output += HUF_decompress_1stream(dtable, out, &in2); + total_output += HUF_decompress_1stream(dtable, out, &in3); + total_output += HUF_decompress_1stream(dtable, out, &in4); - return total_out; + return total_output; } static void HUF_init_dtable(HUF_dtable *const table, const u8 *const bits, @@ -1827,6 +1759,10 @@ static void HUF_init_dtable_usingweights(HUF_dtable *const table, u64 weight_sum = 0; for (int i = 0; i < num_symbs; i++) { + // Weights are in the same range as bit count + if (weights[i] > HUF_MAX_BITS) { + CORRUPTION(); + } weight_sum += weights[i] > 0 ? (u64)1 << (weights[i] - 1) : 0; } @@ -1913,20 +1849,17 @@ static inline void FSE_init_state(const FSE_dtable *const dtable, } static size_t FSE_decompress_interleaved2(const FSE_dtable *const dtable, - u8 *dst, const size_t dst_len, - const u8 *const src, - const size_t src_len) { - if (src_len == 0) { + ostream_t *const out, + istream_t *const in) { + const size_t len = IO_istream_len(in); + if (len == 0) { INP_SIZE(); } - - const u8 *const dst_max = dst + dst_len; - const u8 *const odst = dst; + const u8 *const src = IO_read_bytes(in, len); // Find the last 1 bit - const int padding = 8 - log2inf(src[src_len - 1]); - - i64 offset = src_len * 8 - padding; + const int padding = 8 - log2inf(src[len - 1]); + i64 offset = len * 8 - padding; // The end of the stream contains the 2 states, in this order u16 state1, state2; @@ -1936,30 +1869,28 @@ static size_t FSE_decompress_interleaved2(const FSE_dtable *const dtable, // Decode until we overflow the stream // Since we decode in reverse order, overflowing the stream is offset going // negative + size_t symbols_written = 0; while (1) { - if (dst > dst_max - 2) { - OUT_SIZE(); - } - *dst++ = FSE_decode_symbol(dtable, &state1, src, &offset); + IO_write_byte(out, FSE_decode_symbol(dtable, &state1, src, &offset)); + symbols_written++; if (offset < 0) { // There's still a symbol to decode in state2 - *dst++ = FSE_peek_symbol(dtable, state2); + IO_write_byte(out, FSE_peek_symbol(dtable, state2)); + symbols_written++; break; } - if (dst > dst_max - 2) { - OUT_SIZE(); - } - *dst++ = FSE_decode_symbol(dtable, &state2, src, &offset); + IO_write_byte(out, FSE_decode_symbol(dtable, &state2, src, &offset)); + symbols_written++; if (offset < 0) { // There's still a symbol to decode in state1 - *dst++ = FSE_peek_symbol(dtable, state1); + IO_write_byte(out, FSE_peek_symbol(dtable, state1)); + symbols_written++; break; } } - // Number of symbols read - return dst - odst; + return symbols_written; } static void FSE_init_dtable(FSE_dtable *const dtable, @@ -2042,17 +1973,13 @@ static void FSE_init_dtable(FSE_dtable *const dtable, /// Decode an FSE header as defined in the Zstandard format specification and /// use the decoded frequencies to initialize a decoding table. -static size_t FSE_decode_header(FSE_dtable *const dtable, const u8 *const src, - const size_t src_len, +static void FSE_decode_header(FSE_dtable *const dtable, istream_t *const in, const int max_accuracy_log) { if (max_accuracy_log > FSE_MAX_ACCURACY_LOG) { ERROR("FSE accuracy too large"); } - if (src_len < 1) { - INP_SIZE(); - } - const int accuracy_log = 5 + read_bits_LE(src, 4, 0); + const int accuracy_log = 5 + IO_read_bits(in, 4); if (accuracy_log > max_accuracy_log) { ERROR("FSE accuracy too large"); } @@ -2062,14 +1989,11 @@ static size_t FSE_decode_header(FSE_dtable *const dtable, const u8 *const src, i16 frequencies[FSE_MAX_SYMBS]; int symb = 0; - // Offset of 4 because 4 bits were already read in for accuracy - size_t offset = 4; while (remaining > 1 && symb < FSE_MAX_SYMBS) { // Log of the number of possible values we could read int bits = log2inf(remaining) + 1; - u16 val = read_bits_LE(src, bits, offset); - offset += bits; + u16 val = IO_read_bits(in, bits); // Try to mask out the lower bits to see if it qualifies for the "small // value" threshold @@ -2077,7 +2001,7 @@ static size_t FSE_decode_header(FSE_dtable *const dtable, const u8 *const src, const u16 threshold = ((u16)1 << bits) - 1 - remaining; if ((val & lower_mask) < threshold) { - offset--; + IO_rewind_bits(in, 1); val = val & lower_mask; } else if (val > lower_mask) { val = val - threshold; @@ -2093,22 +2017,21 @@ static size_t FSE_decode_header(FSE_dtable *const dtable, const u8 *const src, // Handle the special probability = 0 case if (proba == 0) { // Read the next two bits to see how many more 0s - int repeat = read_bits_LE(src, 2, offset); - offset += 2; + int repeat = IO_read_bits(in, 2); while (1) { for (int i = 0; i < repeat && symb < FSE_MAX_SYMBS; i++) { frequencies[symb++] = 0; } if (repeat == 3) { - repeat = read_bits_LE(src, 2, offset); - offset += 2; + repeat = IO_read_bits(in, 2); } else { break; } } } } + IO_align_stream(in); if (remaining != 1 || symb >= FSE_MAX_SYMBS) { CORRUPTION(); @@ -2116,8 +2039,6 @@ static size_t FSE_decode_header(FSE_dtable *const dtable, const u8 *const src, // Initialize the decoding table using the determined weights FSE_init_dtable(dtable, frequencies, symb, accuracy_log); - - return (offset + 7) / 8; } static void FSE_init_dtable_rle(FSE_dtable *const dtable, const u8 symb) { From f191be2fe6cc68254c5488d958df96b3d5eb85a8 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Fri, 3 Feb 2017 18:04:00 -0800 Subject: [PATCH 008/223] Inlined portions of specification for clarity --- contrib/educational_decoder/zstd_decompress.c | 370 +++++++++++++++--- 1 file changed, 309 insertions(+), 61 deletions(-) diff --git a/contrib/educational_decoder/zstd_decompress.c b/contrib/educational_decoder/zstd_decompress.c index 8f28313e4..b46bb487d 100644 --- a/contrib/educational_decoder/zstd_decompress.c +++ b/contrib/educational_decoder/zstd_decompress.c @@ -386,6 +386,11 @@ size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len, istream_t in = {(const u8 *)src, 0, src_len}; ostream_t out = {(u8 *)dst, dst_len}; + + // "A content compressed by Zstandard is transformed into a Zstandard frame. + // Multiple frames can be appended into a single file or stream. A frame is + // totally independent, has a defined beginning and end, and a set of + // parameters which tells the decoder how to decompress it." while (IO_istream_len(&in) > 0) { decode_frame(&out, &in, &parsed_dict); } @@ -415,19 +420,43 @@ static void decode_frame(ostream_t *const out, istream_t *const in, const dictionary_t *const dict) { const u32 magic_number = IO_read_bits(in, 32); + // Skippable frame + // + // "Magic_Number + // + // 4 Bytes, little-endian format. Value : 0x184D2A5?, which means any value + // from 0x184D2A50 to 0x184D2A5F. All 16 values are valid to identify a + // skippable frame." if ((magic_number & ~0xFU) == 0x184D2A50U) { - // Skippable frame + // "Skippable frames allow the insertion of user-defined data into a + // flow of concatenated frames. Its design is pretty straightforward, + // with the sole objective to allow the decoder to quickly skip over + // user-defined data and continue decoding. + // + // Skippable frames defined in this specification are compatible with + // LZ4 ones." const size_t frame_size = IO_read_bits(in, 32); // skip over frame IO_advance_input(in, frame_size); - } else if (magic_number == 0xFD2FB528U) { + + return; + } + + // Zstandard frame + // + // "Magic_Number + // + // 4 Bytes, little-endian format. Value : 0xFD2FB528" + if (magic_number == 0xFD2FB528U) { // ZSTD frame decode_data_frame(out, in, dict); - } else { - // not a real frame - ERROR("Invalid magic number"); + + return; } + + // not a real frame + ERROR("Invalid magic number"); } /// Decode a frame that contains compressed data. Not all frames do as there @@ -483,6 +512,17 @@ static void free_frame_context(frame_context_t *const context) { static void parse_frame_header(frame_header_t *const header, istream_t *const in) { + // "The first header's byte is called the Frame_Header_Descriptor. It tells + // which other fields are present. Decoding this byte is enough to tell the + // size of Frame_Header. + // + // Bit number Field name + // 7-6 Frame_Content_Size_flag + // 5 Single_Segment_flag + // 4 Unused_bit + // 3 Reserved_bit + // 2 Content_Checksum_flag + // 1-0 Dictionary_ID_flag" const u8 descriptor = IO_read_bits(in, 8); // decode frame header descriptor into flags @@ -501,12 +541,18 @@ static void parse_frame_header(frame_header_t *const header, // decode window size if (!single_segment_flag) { - // Use the algorithm from the specification to compute window size - // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#window_descriptor + // "Provides guarantees on maximum back-reference distance that will be + // used within compressed data. This information is important for + // decoders to allocate enough memory. + // + // Bit numbers 7-3 2-0 + // Field name Exponent Mantissa" u8 window_descriptor = IO_read_bits(in, 8); u8 exponent = window_descriptor >> 3; u8 mantissa = window_descriptor & 7; + // Use the algorithm from the specification to compute window size + // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#window_descriptor size_t window_base = (size_t)1 << (10 + exponent); size_t window_add = (window_base / 8) * mantissa; header->window_size = window_base + window_add; @@ -514,6 +560,10 @@ static void parse_frame_header(frame_header_t *const header, // decode dictionary id if it exists if (dictionary_id_flag) { + // "This is a variable size field, which contains the ID of the + // dictionary required to properly decode the frame. Note that this + // field is optional. When it's not present, it's up to the caller to + // make sure it uses the correct dictionary. Format is little-endian." const int bytes_array[] = {0, 1, 2, 4}; const int bytes = bytes_array[dictionary_id_flag]; @@ -524,6 +574,11 @@ static void parse_frame_header(frame_header_t *const header, // decode frame content size if it exists if (single_segment_flag || frame_content_size_flag) { + // "This is the original (uncompressed) size. This information is + // optional. The Field_Size is provided according to value of + // Frame_Content_Size_flag. The Field_Size can be equal to 0 (not + // present), 1, 2, 4 or 8 bytes. Format is little-endian." + // // if frame_content_size_flag == 0 but single_segment_flag is set, we // still have a 1 byte field const int bytes_array[] = {1, 2, 4, 8}; @@ -531,6 +586,7 @@ static void parse_frame_header(frame_header_t *const header, header->frame_content_size = IO_read_bits(in, bytes * 8); if (bytes == 2) { + // "When Field_Size is 2, the offset of 256 is added." header->frame_content_size += 256; } } else { @@ -538,9 +594,10 @@ static void parse_frame_header(frame_header_t *const header, } if (single_segment_flag) { - // in this case the effective window size is frame_content_size this - // impacts sequence decoding as we need to determine whether to fall - // back to the dictionary or not on large offsets + // "The Window_Descriptor byte is optional. It is absent when + // Single_Segment_flag is set. In this case, the maximum back-reference + // distance is the content size itself, which can be any value from 1 to + // 2^64-1 bytes (16 EB)." header->window_size = header->frame_content_size; } } @@ -584,16 +641,31 @@ static void frame_context_apply_dict(frame_context_t *const ctx, /// Decompress the data from a frame block by block static void decompress_data(frame_context_t *const ctx, ostream_t *const out, istream_t *const in) { + // "A frame encapsulates one or multiple blocks. Each block can be + // compressed or not, and has a guaranteed maximum content size, which + // depends on frame parameters. Unlike frames, each block depends on + // previous blocks for proper decoding. However, each block can be + // decompressed without waiting for its successor, allowing streaming + // operations." int last_block = 0; do { - // Parse the block header + // "Last_Block + // + // The lowest bit signals if this block is the last one. Frame ends + // right after this block. + // + // Block_Type and Block_Size + // + // The next 2 bits represent the Block_Type, while the remaining 21 bits + // represent the Block_Size. Format is little-endian." last_block = IO_read_bits(in, 1); const int block_type = IO_read_bits(in, 2); const size_t block_len = IO_read_bits(in, 21); switch (block_type) { case 0: { - // Raw, uncompressed block + // "Raw_Block - this is an uncompressed block. Block_Size is the + // number of bytes to read and copy." const u8 *const read_ptr = IO_read_bytes(in, block_len); u8 *const write_ptr = IO_write_bytes(out, block_len); // @@ -604,7 +676,9 @@ static void decompress_data(frame_context_t *const ctx, ostream_t *const out, break; } case 1: { - // RLE block, repeat the first byte N times + // "RLE_Block - this is a single byte, repeated N times. In which + // case, Block_Size is the size to regenerate, while the + // "compressed" block is just 1 byte (the byte to repeat)." const u8 *const read_ptr = IO_read_bytes(in, 1); u8 *const write_ptr = IO_write_bytes(out, block_len); @@ -615,14 +689,18 @@ static void decompress_data(frame_context_t *const ctx, ostream_t *const out, break; } case 2: { - // Compressed block + // "Compressed_Block - this is a Zstandard compressed block, + // detailed in another section of this specification. Block_Size is + // the compressed size. + // Create a sub-stream for the block istream_t block_stream = IO_make_sub_istream(in, block_len); decompress_block(ctx, out, &block_stream); break; } case 3: - // Reserved block type + // "Reserved - this is not a block. This value cannot be used with + // current version of this specification." CORRUPTION(); break; default: @@ -641,6 +719,12 @@ static void decompress_data(frame_context_t *const ctx, ostream_t *const out, /******* BLOCK DECOMPRESSION **************************************************/ static void decompress_block(frame_context_t *const ctx, ostream_t *const out, istream_t *const in) { + // "A compressed block consists of 2 sections : + // + // Literals_Section + // Sequences_Section" + + // Part 1: decode the literals block u8 *literals = NULL; const size_t literals_size = decode_literals(ctx, in, &literals); @@ -673,7 +757,22 @@ static void fse_decode_hufweights(ostream_t *weights, istream_t *const in, static size_t decode_literals(frame_context_t *const ctx, istream_t *const in, u8 **const literals) { - // Decode literals header + // "Literals can be stored uncompressed or compressed using Huffman prefix + // codes. When compressed, an optional tree description can be present, + // followed by 1 or 4 streams." + // + // "Literals_Section_Header + // + // Header is in charge of describing how literals are packed. It's a + // byte-aligned variable-size bitfield, ranging from 1 to 5 bytes, using + // little-endian convention." + // + // "Literals_Block_Type + // + // This field uses 2 lowest bits of first byte, describing 4 different block + // types" + // + // size_format takes between 1 and 2 bits int block_type = IO_read_bits(in, 2); int size_format = IO_read_bits(in, 2); @@ -726,13 +825,13 @@ static size_t decode_literals_simple(istream_t *const in, u8 **const literals, switch (block_type) { case 0: { - // Raw data + // "Raw_Literals_Block - Literals are stored uncompressed." const u8 *const read_ptr = IO_read_bytes(in, size); memcpy(*literals, read_ptr, size); break; } case 1: { - // Single repeated byte + // "RLE_Literals_Block - Literals consist of a single byte value repeated N times." const u8 *const read_ptr = IO_read_bytes(in, 1); memset(*literals, read_ptr[0], size); break; @@ -796,6 +895,8 @@ static size_t decode_literals_compressed(frame_context_t *const ctx, if (block_type == 2) { // Decode provided Huffman table + // "This section is only present when Literals_Block_Type type is + // Compressed_Literals_Block (2)." HUF_free_dtable(&ctx->literals_dtable); decode_huf_table(&huf_stream, &ctx->literals_dtable); @@ -824,22 +925,32 @@ static size_t decode_literals_compressed(frame_context_t *const ctx, static void decode_huf_table(istream_t *const in, HUF_dtable *const dtable) { const u8 header = IO_read_bits(in, 8); + // "All literal values from zero (included) to last present one (excluded) + // are represented by Weight with values from 0 to Max_Number_of_Bits." + + // "This is a single byte value (0-255), which describes how to decode the list of weights." u8 weights[HUF_MAX_SYMBS]; memset(weights, 0, sizeof(weights)); int num_symbs; if (header >= 128) { - // Direct representation, read the weights out + // "This is a direct representation, where each Weight is written + // directly as a 4 bits field (0-15). The full representation occupies + // ((Number_of_Symbols+1)/2) bytes, meaning it uses a last full byte + // even if Number_of_Symbols is odd. Number_of_Symbols = headerByte - + // 127" num_symbs = header - 127; const size_t bytes = (num_symbs + 1) / 2; const u8 *const weight_src = IO_read_bytes(in, bytes); for (int i = 0; i < num_symbs; i++) { - // read_bits_LE isn't applicable here because the weights are order - // reversed within each byte - // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#huffman-tree-header + // "They are encoded forward, 2 + // weights to a byte with the first weight taking the top four bits + // and the second taking the bottom four (e.g. the following + // operations could be used to read the weights: Weight[0] = + // (Byte[0] >> 4), Weight[1] = (Byte[0] & 0xf), etc.)." if (i % 2 == 0) { weights[i] = weight_src[i / 2] >> 4; } else { @@ -864,7 +975,9 @@ static void fse_decode_hufweights(ostream_t *weights, istream_t *const in, FSE_dtable dtable; - // Construct the FSE table + // "An FSE bitstream starts by a header, describing probabilities + // distribution. It will create a Decoding Table. For a list of Huffman + // weights, maximum accuracy is 7 bits." FSE_decode_header(&dtable, in, MAX_ACCURACY_LOG); // Decode the weights @@ -947,9 +1060,19 @@ static void decode_seq_table(istream_t *const in, FSE_dtable *const table, static size_t decode_sequences(frame_context_t *const ctx, istream_t *in, sequence_command_t **const sequences) { + // "A compressed block is a succession of sequences . A sequence is a + // literal copy command, followed by a match copy command. A literal copy + // command specifies a length. It is the number of bytes to be copied (or + // extracted) from the literal section. A match copy command specifies an + // offset and a length. The offset gives the position to copy from, which + // can be within a previous block." + size_t num_sequences; - // Decode the sequence header and allocate space for the output + // "Number_of_Sequences + // + // This is a variable size field using between 1 and 3 bytes. Let's call its + // first byte byte0." u8 header = IO_read_bits(in, 8); if (header == 0) { // "There are no sequences. The sequence section stops there. @@ -980,12 +1103,33 @@ static size_t decode_sequences(frame_context_t *const ctx, istream_t *in, static void decompress_sequences(frame_context_t *const ctx, istream_t *in, sequence_command_t *const sequences, const size_t num_sequences) { + // "The Sequences_Section regroup all symbols required to decode commands. + // There are 3 symbol types : literals lengths, offsets and match lengths. + // They are encoded together, interleaved, in a single bitstream." + + // "Symbol compression modes + // + // This is a single byte, defining the compression mode of each symbol + // type." + // + // Bit number : Field name + // 7-6 : Literals_Lengths_Mode + // 5-4 : Offsets_Mode + // 3-2 : Match_Lengths_Mode + // 1-0 : Reserved u8 compression_modes = IO_read_bits(in, 8); if ((compression_modes & 3) != 0) { + // Reserved bits set CORRUPTION(); } + // "Following the header, up to 3 distribution tables can be described. When + // present, they are in this order : + // + // Literals lengths + // Offsets + // Match Lengths" // Update the tables we have stored in the context decode_seq_table(in, &ctx->ll_dtable, seq_literal_length, (compression_modes >> 6) & 3); @@ -1016,6 +1160,12 @@ static void decompress_sequences(frame_context_t *const ctx, istream_t *in, const int padding = 8 - log2inf(src[len - 1]); i64 offset = len * 8 - padding; + // "The bitstream starts with initial state values, each using the required + // number of bits in their respective accuracy, decoded previously from + // their normalized distribution. + // + // It starts by Literals_Length_State, followed by Offset_State, and finally + // Match_Length_State." FSE_init_state(&state.ll_table, &state.ll_state, src, &offset); FSE_init_state(&state.of_table, &state.of_state, src, &offset); FSE_init_state(&state.ml_table, &state.ml_state, src, &offset); @@ -1036,6 +1186,10 @@ static void decompress_sequences(frame_context_t *const ctx, istream_t *in, static sequence_command_t decode_sequence(sequence_state_t *const state, const u8 *const src, i64 *const offset) { + // "Each symbol is a code in its own context, which specifies Baseline and + // Number_of_Bits to add. Codes are FSE compressed, and interleaved with raw + // additional bits in the same bitstream." + // Decode symbols, but don't update states const u8 of_code = FSE_peek_symbol(&state->of_table, state->of_state); const u8 ll_code = FSE_peek_symbol(&state->ll_table, state->ll_state); @@ -1049,18 +1203,24 @@ static sequence_command_t decode_sequence(sequence_state_t *const state, // Read the interleaved bits sequence_command_t seq; - // Offset computation works differently + // "Decoding starts by reading the Number_of_Bits required to decode Offset. + // It then does the same for Match_Length, and then for Literals_Length." seq.offset = ((u32)1 << of_code) + STREAM_read_bits(src, of_code, offset); + seq.match_length = SEQ_MATCH_LENGTH_BASELINES[ml_code] + STREAM_read_bits(src, SEQ_MATCH_LENGTH_EXTRA_BITS[ml_code], offset); + seq.literal_length = SEQ_LITERAL_LENGTH_BASELINES[ll_code] + STREAM_read_bits(src, SEQ_LITERAL_LENGTH_EXTRA_BITS[ll_code], offset); + // "If it is not the last sequence in the block, the next operation is to + // update states. Using the rules pre-calculated in the decoding tables, + // Literals_Length_State is updated, followed by Match_Length_State, and + // then Offset_State." // If the stream is complete don't read bits to update state if (*offset != 0) { - // Update state in the order specified in the specification FSE_update_state(&state->ll_table, &state->ll_state, src, offset); FSE_update_state(&state->ml_table, &state->ml_state, src, offset); FSE_update_state(&state->of_table, &state->of_state, src, offset); @@ -1088,6 +1248,7 @@ static void decode_seq_table(istream_t *const in, FSE_dtable *const table, switch (mode) { case seq_predefined: { + // "Predefined_Mode : uses a predefined distribution table." const i16 *distribution = default_distributions[type]; const size_t symbs = default_distribution_lengths[type]; const size_t accuracy_log = default_distribution_accuracies[type]; @@ -1096,15 +1257,20 @@ static void decode_seq_table(istream_t *const in, FSE_dtable *const table, break; } case seq_rle: { + // "RLE_Mode : it's a single code, repeated Number_of_Sequences times." const u8 symb = IO_read_bits(in, 8); FSE_init_dtable_rle(table, symb); break; } case seq_fse: { + // "FSE_Compressed_Mode : standard FSE compression. A distribution table + // will be present " FSE_decode_header(table, in, max_accuracies[type]); break; } case seq_repeat: + // "Repeat_Mode : re-use distribution table from previous compressed + // block." // Nothing to do here, table will be unchanged break; default: @@ -1322,8 +1488,8 @@ static void traverse_frame(const frame_header_t *const header, istream_t *const /******* END OUTPUT SIZE COUNTING *********************************************/ /******* DICTIONARY PARSING ***************************************************/ -static void init_raw_content_dict(dictionary_t *const dict, const u8 *const src, - const size_t src_len); +static void init_dictionary_content(dictionary_t *const dict, + istream_t *const in); static void parse_dictionary(dictionary_t *const dict, const u8 *src, size_t src_len) { @@ -1337,13 +1503,20 @@ static void parse_dictionary(dictionary_t *const dict, const u8 *src, const u32 magic_number = IO_read_bits(&in, 32); if (magic_number != 0xEC30A437) { // raw content dict - init_raw_content_dict(dict, src, src_len); + IO_rewind_bits(&in, 32); + init_dictionary_content(dict, &in); return; } dict->dictionary_id = IO_read_bits(&in, 32); - // Parse the provided entropy tables in order + // "Entropy_Tables : following the same format as the tables in compressed + // blocks. They are stored in following order : Huffman tables for literals, + // FSE table for offsets, FSE table for match lengths, and FSE table for + // literals lengths. It's finally followed by 3 offset values, populating + // recent offsets (instead of using {1,4,8}), stored in order, 4-bytes + // little-endian each, for a total of 12 bytes. Each recent offset must have + // a value < dictionary size." decode_huf_table(&in, &dict->literals_dtable); decode_seq_table(&in, &dict->of_dtable, seq_offset, seq_fse); decode_seq_table(&in, &dict->ml_dtable, seq_match_length, seq_fse); @@ -1355,38 +1528,33 @@ static void parse_dictionary(dictionary_t *const dict, const u8 *src, dict->previous_offsets[2] = IO_read_bits(&in, 32); // Ensure the provided offsets aren't too large + // "Each recent offset must have a value < dictionary size." for (int i = 0; i < 3; i++) { if (dict->previous_offsets[i] > src_len) { ERROR("Dictionary corrupted"); } } - // The rest is the content - dict->content_size = IO_istream_len(&in); + // "Content : The rest of the dictionary is its content. The content act as + // a "past" in front of data to compress or decompress, so it can be + // referenced in sequence commands." + init_dictionary_content(dict, &in); +} + +static void init_dictionary_content(dictionary_t *const dict, + istream_t *const in) { + // Copy in the content + dict->content_size = IO_istream_len(in); dict->content = malloc(dict->content_size); if (!dict->content) { BAD_ALLOC(); } - const u8 *const content = IO_read_bytes(&in, dict->content_size); + const u8 *const content = IO_read_bytes(in, dict->content_size); memcpy(dict->content, content, dict->content_size); } -/// If parse_dictionary is given a raw content dictionary, it delegates here -static void init_raw_content_dict(dictionary_t *const dict, const u8 *const src, - const size_t src_len) { - dict->dictionary_id = 0; - // Copy in the content - dict->content = malloc(src_len); - if (!dict->content) { - BAD_ALLOC(); - } - - dict->content_size = src_len; - memcpy(dict->content, src, src_len); -} - /// Free an allocated dictionary static void free_dictionary(dictionary_t *const dict) { HUF_free_dtable(&dict->literals_dtable); @@ -1636,8 +1804,15 @@ static size_t HUF_decompress_1stream(const HUF_dtable *const dtable, } const u8 *const src = IO_read_bytes(in, len); - // To maintain similarity with FSE, start from the end - // Find the last 1 bit + // "Each bitstream must be read backward, that is starting from the end down + // to the beginning. Therefore it's necessary to know the size of each + // bitstream. + // + // It's also necessary to know exactly which bit is the latest. This is + // detected by a final bit flag : the highest bit of latest byte is a + // final-bit-flag. Consequently, a last byte of 0 is not possible. And the + // final-bit-flag itself is not part of the useful bitstream. Hence, the + // last byte contains between 0 and 7 useful bits." const int padding = 8 - log2inf(src[len - 1]); i64 offset = len * 8 - padding; @@ -1651,6 +1826,10 @@ static size_t HUF_decompress_1stream(const HUF_dtable *const dtable, IO_write_byte(out, HUF_decode_symbol(dtable, &state, src, &offset)); symbols_written++; } + // "The process continues up to reading the required number of symbols per + // stream. If a bitstream is not entirely and exactly consumed, hence + // reaching exactly its beginning position with all bits consumed, the + // decoding process is considered faulty." // When all symbols have been decoded, the final state value shouldn't have // any data from the stream, so it should have "read" dtable->max_bits from @@ -1666,6 +1845,11 @@ static size_t HUF_decompress_1stream(const HUF_dtable *const dtable, static size_t HUF_decompress_4stream(const HUF_dtable *const dtable, ostream_t *const out, istream_t *const in) { + // "Compressed size is provided explicitly : in the 4-streams variant, + // bitstreams are preceded by 3 unsigned little-endian 16-bits values. Each + // value represents the compressed size of one stream, in order. The last + // stream size is deducted from total compressed size and from previously + // decoded stream sizes" const size_t csize1 = IO_read_bits(in, 16); const size_t csize2 = IO_read_bits(in, 16); const size_t csize3 = IO_read_bits(in, 16); @@ -1719,6 +1903,10 @@ static void HUF_init_dtable(HUF_dtable *const table, const u8 *const bits, BAD_ALLOC(); } + // "Symbols are sorted by Weight. Within same Weight, symbols keep natural + // order. Symbols with a Weight of zero are removed. Then, starting from + // lowest weight, prefix codes are distributed in order." + u32 rank_idx[HUF_MAX_BITS + 1]; // Initialize the starting codes for each rank (number of bits) rank_idx[max_bits] = 0; @@ -1779,6 +1967,7 @@ static void HUF_init_dtable_usingweights(HUF_dtable *const table, const int last_weight = log2inf(left_over) + 1; for (int i = 0; i < num_symbs; i++) { + // "Number_of_Bits = Number_of_Bits ? Max_Number_of_Bits + 1 - Weight : 0" bits[i] = weights[i] > 0 ? (max_bits + 1 - weights[i]) : 0; } bits[num_symbs] = @@ -1857,12 +2046,23 @@ static size_t FSE_decompress_interleaved2(const FSE_dtable *const dtable, } const u8 *const src = IO_read_bytes(in, len); - // Find the last 1 bit + // "Each bitstream must be read backward, that is starting from the end down + // to the beginning. Therefore it's necessary to know the size of each + // bitstream. + // + // It's also necessary to know exactly which bit is the latest. This is + // detected by a final bit flag : the highest bit of latest byte is a + // final-bit-flag. Consequently, a last byte of 0 is not possible. And the + // final-bit-flag itself is not part of the useful bitstream. Hence, the + // last byte contains between 0 and 7 useful bits." const int padding = 8 - log2inf(src[len - 1]); i64 offset = len * 8 - padding; - // The end of the stream contains the 2 states, in this order u16 state1, state2; + // "The first state (State1) encodes the even indexed symbols, and the + // second (State2) encodes the odd indexes. State1 is initialized first, and + // then State2, and they take turns decoding a single symbol and updating + // their state." FSE_init_state(dtable, &state1, src, &offset); FSE_init_state(dtable, &state2, src, &offset); @@ -1871,6 +2071,11 @@ static size_t FSE_decompress_interleaved2(const FSE_dtable *const dtable, // negative size_t symbols_written = 0; while (1) { + // "The number of symbols to decode is determined by tracking bitStream + // overflow condition: If updating state after decoding a symbol would + // require more bits than remain in the stream, it is assumed the extra + // bits are 0. Then, the symbols for each of the final states are + // decoded and the process is complete." IO_write_byte(out, FSE_decode_symbol(dtable, &state1, src, &offset)); symbols_written++; if (offset < 0) { @@ -1920,6 +2125,10 @@ static void FSE_init_dtable(FSE_dtable *const dtable, // which can be larger than a byte can store u16 state_desc[FSE_MAX_SYMBS]; + // "Symbols are scanned in their natural order for "less than 1" + // probabilities. Symbols with this probability are being attributed a + // single cell, starting from the end of the table. These symbols define a + // full state reset, reading Accuracy_Log bits." int high_threshold = size; for (int s = 0; s < num_symbs; s++) { // Scan for low probability symbols to put at the top @@ -1929,6 +2138,9 @@ static void FSE_init_dtable(FSE_dtable *const dtable, } } + // "All remaining symbols are sorted in their natural order. Starting from + // symbol 0 and table position 0, each symbol gets attributed as many cells + // as its probability. Cell allocation is spreaded, not linear." // Place the rest in the table const u16 step = (size >> 1) + (size >> 3) + 3; const u16 mask = size - 1; @@ -1943,11 +2155,12 @@ static void FSE_init_dtable(FSE_dtable *const dtable, for (int i = 0; i < norm_freqs[s]; i++) { // Give `norm_freqs[s]` states to symbol s dtable->symbols[pos] = s; + // "A position is skipped if already occupied, typically by a "less + // than 1" probability symbol." do { pos = (pos + step) & mask; } while (pos >= - high_threshold); // Make sure we don't occupy a spot taken - // by the low prob symbols + high_threshold); // Note: no other collision checking is necessary as `step` is // coprime to `size`, so the cycle will visit each position exactly // once @@ -1975,30 +2188,53 @@ static void FSE_init_dtable(FSE_dtable *const dtable, /// use the decoded frequencies to initialize a decoding table. static void FSE_decode_header(FSE_dtable *const dtable, istream_t *const in, const int max_accuracy_log) { + // "An FSE distribution table describes the probabilities of all symbols + // from 0 to the last present one (included) on a normalized scale of 1 << + // Accuracy_Log . + // + // It's a bitstream which is read forward, in little-endian fashion. It's + // not necessary to know its exact size, since it will be discovered and + // reported by the decoding process. if (max_accuracy_log > FSE_MAX_ACCURACY_LOG) { ERROR("FSE accuracy too large"); } + // The bitstream starts by reporting on which scale it operates. + // Accuracy_Log = low4bits + 5. Note that maximum Accuracy_Log for literal + // and match lengths is 9, and for offsets is 8. Higher values are + // considered errors." const int accuracy_log = 5 + IO_read_bits(in, 4); if (accuracy_log > max_accuracy_log) { ERROR("FSE accuracy too large"); } - // The +1 facilitates the `-1` probabilities - i32 remaining = (1 << accuracy_log) + 1; + // "Then follows each symbol value, from 0 to last present one. The number + // of bits used by each field is variable. It depends on : + // + // Remaining probabilities + 1 : example : Presuming an Accuracy_Log of 8, + // and presuming 100 probabilities points have already been distributed, the + // decoder may read any value from 0 to 255 - 100 + 1 == 156 (inclusive). + // Therefore, it must read log2sup(156) == 8 bits. + // + // Value decoded : small values use 1 less bit : example : Presuming values + // from 0 to 156 (inclusive) are possible, 255-156 = 99 values are remaining + // in an 8-bits field. They are used this way : first 99 values (hence from + // 0 to 98) use only 7 bits, values from 99 to 156 use 8 bits. " + + i32 remaining = 1 << accuracy_log; i16 frequencies[FSE_MAX_SYMBS]; int symb = 0; - while (remaining > 1 && symb < FSE_MAX_SYMBS) { + while (remaining > 0 && symb < FSE_MAX_SYMBS) { // Log of the number of possible values we could read - int bits = log2inf(remaining) + 1; + int bits = log2inf(remaining + 1) + 1; u16 val = IO_read_bits(in, bits); // Try to mask out the lower bits to see if it qualifies for the "small // value" threshold const u16 lower_mask = ((u16)1 << (bits - 1)) - 1; - const u16 threshold = ((u16)1 << bits) - 1 - remaining; + const u16 threshold = ((u16)1 << bits) - 1 - (remaining + 1); if ((val & lower_mask) < threshold) { IO_rewind_bits(in, 1); @@ -2007,14 +2243,23 @@ static void FSE_decode_header(FSE_dtable *const dtable, istream_t *const in, val = val - threshold; } + // "Probability is obtained from Value decoded by following formula : + // Proba = value - 1" const i16 proba = (i16)val - 1; - // A value of -1 is possible, and has special meaning + + // "It means value 0 becomes negative probability -1. -1 is a special + // probability, which means "less than 1". Its effect on distribution + // table is described in next paragraph. For the purpose of calculating + // cumulated distribution, it counts as one." remaining -= proba < 0 ? -proba : proba; frequencies[symb] = proba; symb++; - // Handle the special probability = 0 case + // "When a symbol has a probability of zero, it is followed by a 2-bits + // repeat flag. This repeat flag tells how many probabilities of zeroes + // follow the current one. It provides a number ranging from 0 to 3. If + // it is a 3, another 2-bits repeat flag follows, and so on." if (proba == 0) { // Read the next two bits to see how many more 0s int repeat = IO_read_bits(in, 2); @@ -2033,7 +2278,10 @@ static void FSE_decode_header(FSE_dtable *const dtable, istream_t *const in, } IO_align_stream(in); - if (remaining != 1 || symb >= FSE_MAX_SYMBS) { + // "When last symbol reaches cumulated total of 1 << Accuracy_Log, decoding + // is complete. If the last symbol makes cumulated total go above 1 << + // Accuracy_Log, distribution is considered corrupted." + if (remaining != 0 || symb >= FSE_MAX_SYMBS) { CORRUPTION(); } From d44d363ec176734191027779a435d387ba9f1d37 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 6 Feb 2017 10:01:06 -0800 Subject: [PATCH 009/223] changed download URL for github_users sample set --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b5e16dff7..6336d9006 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ To solve this situation, Zstd offers a __training mode__, which can be used to t Training Zstandard is achieved by provide it with a few samples (one file per sample). The result of this training is stored in a file called "dictionary", which must be loaded before compression and decompression. Using this dictionary, the compression ratio achievable on small data improves dramatically. -The following example uses the `github-users` [sample set](https://www.dropbox.com/s/mnktkomhkjbf1i2/github_users.tar.zst?dl=0), created from [github public API](https://developer.github.com/v3/users/#get-all-users). +The following example uses the `github-users` [sample set](https://github.com/facebook/zstd/releases/tag/v1.1.3), created from [github public API](https://developer.github.com/v3/users/#get-all-users). It consists of roughly 10K records weighting about 1KB each. Compression Ratio | Compression Speed | Decompression Speed From 7060aee8c2de9080135133d07221237e7db6adb7 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 6 Feb 2017 19:43:13 +0100 Subject: [PATCH 010/223] platform.h added to build_package.bat --- lib/dll/example/build_package.bat | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/dll/example/build_package.bat b/lib/dll/example/build_package.bat index ce738a56c..b225af8d8 100644 --- a/lib/dll/example/build_package.bat +++ b/lib/dll/example/build_package.bat @@ -4,6 +4,7 @@ COPY tests\fullbench.c bin\example\ COPY programs\datagen.c bin\example\ COPY programs\datagen.h bin\example\ COPY programs\util.h bin\example\ +COPY programs\platform.h bin\example\ COPY lib\common\mem.h bin\example\ COPY lib\common\zstd_errors.h bin\example\ COPY lib\common\zstd_internal.h bin\example\ From 2cb8ee878437fb627bbee838ad8d125dafd84285 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 6 Feb 2017 11:32:13 -0800 Subject: [PATCH 011/223] Change zlib include to be a system include --- programs/fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index a9e1574a6..087bf9504 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -39,7 +39,7 @@ # include "zstdmt_compress.h" #endif #ifdef ZSTD_GZDECOMPRESS -# include "zlib.h" +# include # if !defined(z_const) # define z_const # endif From 7e3fc73795064bdbb49f59d7af1657b994c8b057 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 6 Feb 2017 11:54:31 -0800 Subject: [PATCH 012/223] Ensure can be included in HAVE_ZLIB test --- programs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/Makefile b/programs/Makefile index 599bef694..ae798c2a6 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -67,7 +67,7 @@ endif # zlib detection VOID = /dev/null -HAVE_ZLIB := $(shell echo "int main(){}" | $(CC) -o $(VOID) -x c - -lz 2> $(VOID) && echo 1 || echo 0) +HAVE_ZLIB := $(shell echo "\#include \nint main(){}" | $(CC) -o $(VOID) -x c - -lz 2> $(VOID) && echo 1 || echo 0) ifeq ($(HAVE_ZLIB), 1) ZLIBCPP = -DZSTD_GZDECOMPRESS ZLIBLD = -lz From 816edeb9c244dcd8007433bb2bb1fb69e3ca0dc1 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 6 Feb 2017 17:39:54 -0800 Subject: [PATCH 013/223] corrected contributor's name --- NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 4f7463056..a8a2ade2d 100644 --- a/NEWS +++ b/NEWS @@ -12,7 +12,7 @@ API : new : ZDICT_finalizeDictionary() API : fix : ZSTD_initCStream_usingCDict() properly writes dictID into frame header, by Gregory Szorc (#511) API : fix : all symbols properly exposed in libzstd, by Nick Terrell build : support for Solaris target, by Przemyslaw Skibinski -doc : clarified specification, by Andrew Purcell +doc : clarified specification, by Sean Purcell v1.1.2 API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init From 94abd6a26cad4b2993fc0b5fce19a8b01d4a35ad Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 7 Feb 2017 16:36:19 +0100 Subject: [PATCH 014/223] SET_REALTIME_PRIORITY --- programs/bench.c | 2 +- programs/util.h | 8 ++++---- zlibWrapper/examples/zwrapbench.c | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index dcb23b1f2..c8f1dcf0a 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -457,7 +457,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, if (!pch) pch = strrchr(displayName, '/'); /* Linux */ if (pch) displayName = pch+1; - SET_HIGH_PRIORITY; + SET_REALTIME_PRIORITY; if (g_displayLevel == 1 && !g_additionalParam) DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10)); diff --git a/programs/util.h b/programs/util.h index 651027bae..656b3a96b 100644 --- a/programs/util.h +++ b/programs/util.h @@ -44,7 +44,7 @@ extern "C" { ******************************************/ #if defined(_WIN32) # include -# define SET_HIGH_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) +# define SET_REALTIME_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) # define UTIL_sleep(s) Sleep(1000*s) # define UTIL_sleepMilli(milli) Sleep(milli) #elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */ @@ -52,9 +52,9 @@ extern "C" { # include /* setpriority */ # include /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */ # if defined(PRIO_PROCESS) -# define SET_HIGH_PRIORITY setpriority(PRIO_PROCESS, 0, -20) +# define SET_REALTIME_PRIORITY setpriority(PRIO_PROCESS, 0, -20) # else -# define SET_HIGH_PRIORITY /* disabled */ +# define SET_REALTIME_PRIORITY /* disabled */ # endif # define UTIL_sleep(s) sleep(s) # if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L) /* nanosleep requires POSIX.1-2001 */ @@ -63,7 +63,7 @@ extern "C" { # define UTIL_sleepMilli(milli) /* disabled */ # endif #else -# define SET_HIGH_PRIORITY /* disabled */ +# define SET_REALTIME_PRIORITY /* disabled */ # define UTIL_sleep(s) /* disabled */ # define UTIL_sleepMilli(milli) /* disabled */ #endif diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index e5c54438b..328c1096b 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -591,7 +591,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, if (!pch) pch = strrchr(displayName, '/'); /* Linux */ if (pch) displayName = pch+1; - SET_HIGH_PRIORITY; + SET_REALTIME_PRIORITY; if (g_displayLevel == 1 && !g_additionalParam) DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbIterations, (U32)(g_blockSize>>10)); From d05014c7390ec99a6b13ba32c46529acfb5759d1 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 7 Feb 2017 16:48:01 +0100 Subject: [PATCH 015/223] added the "--rt-prio" option --- programs/bench.c | 23 +++++++++++++---------- programs/bench.h | 2 +- programs/zstdcli.c | 6 ++++-- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index c8f1dcf0a..5870eaf72 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -449,7 +449,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, const char* displayName, int cLevel, int cLevelLast, const size_t* fileSizes, unsigned nbFiles, const void* dictBuffer, size_t dictBufferSize, - ZSTD_compressionParameters *compressionParams) + ZSTD_compressionParameters *compressionParams, int setRealTimePrio) { int l; @@ -457,7 +457,10 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, if (!pch) pch = strrchr(displayName, '/'); /* Linux */ if (pch) displayName = pch+1; - SET_REALTIME_PRIORITY; + if (setRealTimePrio) { + DISPLAYLEVEL(2, "Note : switching to a real-time priority \n"); + SET_REALTIME_PRIORITY; + } if (g_displayLevel == 1 && !g_additionalParam) DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, (U32)benchedSize, g_nbSeconds, (U32)(g_blockSize>>10)); @@ -505,8 +508,8 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize, if (totalSize == 0) EXM_THROW(12, "no data to bench"); } -static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, - int cLevel, int cLevelLast, ZSTD_compressionParameters *compressionParams) +static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel, + int cLevelLast, ZSTD_compressionParameters *compressionParams, int setRealTimePrio) { void* srcBuffer; size_t benchedSize; @@ -545,7 +548,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co BMK_benchCLevel(srcBuffer, benchedSize, displayName, cLevel, cLevelLast, fileSizes, nbFiles, - dictBuffer, dictBufferSize, compressionParams); + dictBuffer, dictBufferSize, compressionParams, setRealTimePrio); } /* clean up */ @@ -555,7 +558,7 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co } -static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams) +static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility, ZSTD_compressionParameters* compressionParams, int setRealTimePrio) { char name[20] = {0}; size_t benchedSize = 10000000; @@ -569,7 +572,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility /* Bench */ snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); - BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams); + BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0, compressionParams, setRealTimePrio); /* clean up */ free(srcBuffer); @@ -577,7 +580,7 @@ static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, - int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams) + int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio) { double const compressibility = (double)g_compressibilityDefault / 100; @@ -587,8 +590,8 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* di if (cLevelLast > cLevel) DISPLAYLEVEL(2, "Benchmarking levels from %d to %d\n", cLevel, cLevelLast); if (nbFiles == 0) - BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams); + BMK_syntheticTest(cLevel, cLevelLast, compressibility, compressionParams, setRealTimePrio); else - BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams); + BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast, compressionParams, setRealTimePrio); return 0; } diff --git a/programs/bench.h b/programs/bench.h index 2918c02bf..77a527f8f 100644 --- a/programs/bench.h +++ b/programs/bench.h @@ -16,7 +16,7 @@ #include "zstd.h" /* ZSTD_compressionParameters */ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles,const char* dictFileName, - int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams); + int cLevel, int cLevelLast, ZSTD_compressionParameters* compressionParams, int setRealTimePrio); /* Set Parameters */ void BMK_setNbSeconds(unsigned nbLoops); diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 6ca294fc2..30391eb0e 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -267,7 +267,8 @@ int main(int argCount, const char* argv[]) nextArgumentsAreFiles=0, ultra=0, lastCommand = 0, - nbThreads = 1; + nbThreads = 1, + setRealTimePrio = 0; unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ size_t blockSize = 0; zstd_operation_mode operation = zom_compress; @@ -356,6 +357,7 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; } if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } + if (!strcmp(argument, "--rt-prio")) { setRealTimePrio = 1; continue; } /* long commands with arguments */ #ifndef ZSTD_NODICT @@ -574,7 +576,7 @@ int main(int argCount, const char* argv[]) BMK_setBlockSize(blockSize); BMK_setNbThreads(nbThreads); BMK_setNbSeconds(bench_nbSeconds); - BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams); + BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams, setRealTimePrio); #endif (void)bench_nbSeconds; goto _end; From 0665a359aa95ccc5386a1bb7cad42909fc2cca86 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 7 Feb 2017 20:12:59 +0100 Subject: [PATCH 016/223] "--rt-prio" renamed to "--priority=rt" --- programs/zstdcli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 30391eb0e..604e4c5f2 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -357,7 +357,7 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; } if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } - if (!strcmp(argument, "--rt-prio")) { setRealTimePrio = 1; continue; } + if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } /* long commands with arguments */ #ifndef ZSTD_NODICT From 71c5263c00e7b18e16ae394b23cef18487cf83fd Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Tue, 7 Feb 2017 11:35:07 -0800 Subject: [PATCH 017/223] Attribute cover dictionary code --- lib/dictBuilder/cover.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c index c5b606db6..1ced645b5 100644 --- a/lib/dictBuilder/cover.c +++ b/lib/dictBuilder/cover.c @@ -7,6 +7,16 @@ * of patent rights can be found in the PATENTS file in the same directory. */ +/* ***************************************************************************** + * Constructs a dictionary using a heuristic based on the following paper: + * + * Liao, Petri, Moffat, Wirth + * Effective Construction of Relative Lempel-Ziv Dictionaries + * Published in WWW 2016. + * + * Adapted from code originally written by @ot (Giuseppe Ottaviano). + ******************************************************************************/ + /*-************************************* * Dependencies ***************************************/ @@ -621,13 +631,6 @@ static ZDICT_params_t COVER_translateParams(COVER_params_t parameters) { return zdictParams; } -/** - * Constructs a dictionary using a heuristic based on the following paper: - * - * Liao, Petri, Moffat, Wirth - * Effective Construction of Relative Lempel-Ziv Dictionaries - * Published in WWW 2016. - */ ZDICTLIB_API size_t COVER_trainFromBuffer( void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, COVER_params_t parameters) { From 00ea51f8066d5b624d2c221ee141b9ea593b9f1e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 7 Feb 2017 12:05:28 -0800 Subject: [PATCH 018/223] completed NEWS for v1.1.3 --- NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS b/NEWS index a8a2ade2d..24860c957 100644 --- a/NEWS +++ b/NEWS @@ -1,7 +1,7 @@ v1.1.3 cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`) cli : new : experimental target `make zstdmt`, with multi-threading support -cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell +cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell, based on prior work by Giuseppe Ottaviano. cli : new : advanced commands for detailed parameters, by Przemyslaw Skibinski cli : fix zstdless on Mac OS-X, by Andrew Janke cli : fix #232 "compress non-files" From eb52dbd4fe052b4786250635877fadeebfd127f5 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Tue, 7 Feb 2017 14:44:11 -0800 Subject: [PATCH 019/223] Minor changes to educational decoder --- contrib/educational_decoder/zstd_decompress.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/contrib/educational_decoder/zstd_decompress.c b/contrib/educational_decoder/zstd_decompress.c index b46bb487d..856255987 100644 --- a/contrib/educational_decoder/zstd_decompress.c +++ b/contrib/educational_decoder/zstd_decompress.c @@ -1258,7 +1258,7 @@ static void decode_seq_table(istream_t *const in, FSE_dtable *const table, } case seq_rle: { // "RLE_Mode : it's a single code, repeated Number_of_Sequences times." - const u8 symb = IO_read_bits(in, 8); + const u8 symb = IO_read_bytes(in, 1)[0]; FSE_init_dtable_rle(table, symb); break; } @@ -1572,8 +1572,8 @@ static void free_dictionary(dictionary_t *const dict) { #define UNALIGNED() ERROR("Attempting to operate on a non-byte aligned stream") /// Reads `num` bits from a bitstream, and updates the internal offset static inline u64 IO_read_bits(istream_t *const in, const int num) { - if (num > 64) { - return -1; + if (num > 64 || num <= 0) { + ERROR("Attempt to read an invalid number of bits"); } const size_t bytes = (num + in->bit_offset + 7) / 8; @@ -1710,7 +1710,7 @@ static inline istream_t IO_make_sub_istream(istream_t *const in, size_t len) { static inline u64 read_bits_LE(const u8 *src, const int num, const size_t offset) { if (num > 64) { - return -1; + ERROR("Attempt to read an invalid number of bits"); } // Skip over bytes that aren't in range @@ -1871,6 +1871,11 @@ static size_t HUF_decompress_4stream(const HUF_dtable *const dtable, return total_output; } +/// Initializes a Huffman table using canonical Huffman codes +/// For more explanation on canonical Huffman codes see +/// http://www.cs.uofs.edu/~mccloske/courses/cmps340/huff_canonical_dec2015.html +/// Codes within a level are allocated in symbol order (i.e. smaller symbols get +/// earlier codes) static void HUF_init_dtable(HUF_dtable *const table, const u8 *const bits, const int num_symbs) { memset(table, 0, sizeof(HUF_dtable)); @@ -2004,6 +2009,9 @@ static void HUF_copy_dtable(HUF_dtable *const dst, /******* END HUFFMAN PRIMITIVES ***********************************************/ /******* FSE PRIMITIVES *******************************************************/ +/// For more description of FSE see +/// https://github.com/Cyan4973/FiniteStateEntropy/ + /// Allow a symbol to be decoded without updating state static inline u8 FSE_peek_symbol(const FSE_dtable *const dtable, const u16 state) { From 40580ff669a4e5bb4d681973fb61281b2ac2d140 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 8 Feb 2017 13:49:06 +0100 Subject: [PATCH 020/223] added description of "--priority=rt" --- programs/zstd.1 | 3 ++- programs/zstdcli.c | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/programs/zstd.1 b/programs/zstd.1 index 384f69e3a..684fb868a 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -289,7 +289,8 @@ and weight typically 100x the target dictionary size (for example, 10 MB for a 1 .TP .B \-B# cut file into independent blocks of size # (default: no block) - +.B \--priority=rt + set process priority to real-time .SH ADVANCED COMPRESSION OPTIONS .TP diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 604e4c5f2..a7dbda313 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -148,6 +148,7 @@ static int usage_advanced(const char* programName) DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n"); DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n"); DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n"); + DISPLAY( "--priority=rt : set process priority to real-time\n"); #endif return 0; } From 4b4f8c2d71ddd22df117b1330495acaa1100d95d Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 8 Feb 2017 13:58:04 +0100 Subject: [PATCH 021/223] turn off test-pool for qemu-ppc64-static --- tests/Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index f49d23018..07069d3ac 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -225,7 +225,10 @@ zstd-playTests: datagen file $(ZSTD) ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST) -test: test-zstd test-fullbench test-fuzzer test-zstream test-longmatch test-invalidDictionaries test-pool +test: test-zstd test-fullbench test-fuzzer test-zstream test-longmatch test-invalidDictionaries +ifneq ($(QEMU_SYS),qemu-ppc64-static) +test: test-pool +endif test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32 From ca20edd96024e71011f83e526d831028c877e30f Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 8 Feb 2017 14:32:49 +0100 Subject: [PATCH 022/223] fixed zlib detection with MinGW --- programs/Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index ae798c2a6..b189224f3 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -67,7 +67,10 @@ endif # zlib detection VOID = /dev/null -HAVE_ZLIB := $(shell echo "\#include \nint main(){}" | $(CC) -o $(VOID) -x c - -lz 2> $(VOID) && echo 1 || echo 0) +HAVE_ZLIB := $(shell echo -e "\#include \nint main(){}" | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && echo 1 || echo 0) +ifeq ($(HAVE_ZLIB), 1) +TEMP := $(shell rm have_zlib$(EXT)) +endif ifeq ($(HAVE_ZLIB), 1) ZLIBCPP = -DZSTD_GZDECOMPRESS ZLIBLD = -lz @@ -152,7 +155,7 @@ clean: @$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda @$(RM) core *.o tmp* result* *.gcda dictionary *.zst \ zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \ - *.gcda default.profraw have_zlib + *.gcda default.profraw have_zlib$(EXT) @echo Cleaning completed clean_decomp_o: From cfd4dc299a9ed70c0e2b25b22ac5e973ccf5bdb1 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 8 Feb 2017 15:17:55 +0100 Subject: [PATCH 023/223] add "--format=gzip" option --- programs/Makefile | 2 +- programs/fileio.h | 7 +++++++ programs/zstdcli.c | 4 ++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/programs/Makefile b/programs/Makefile index b189224f3..c96029631 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -72,7 +72,7 @@ ifeq ($(HAVE_ZLIB), 1) TEMP := $(shell rm have_zlib$(EXT)) endif ifeq ($(HAVE_ZLIB), 1) -ZLIBCPP = -DZSTD_GZDECOMPRESS +ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS ZLIBLD = -lz endif diff --git a/programs/fileio.h b/programs/fileio.h index daff0312e..2b6275573 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -31,9 +31,16 @@ extern "C" { #endif +/*-************************************* +* Types +***************************************/ +typedef enum { FIO_zstdCompression, FIO_gzipCompression } FIO_compresionType_t; + + /*-************************************* * Parameters ***************************************/ +void FIO_setCompresionType(FIO_compresionType_t compresionType); void FIO_overwriteMode(void); void FIO_setNotificationLevel(unsigned level); void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index a7dbda313..651255b02 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -123,6 +123,9 @@ static int usage_advanced(const char* programName) DISPLAY( " -T# : use # threads for compression (default:1) \n"); DISPLAY( " -B# : select size of independent sections (default:0==automatic) \n"); #endif +#ifdef ZSTD_GZCOMPRESS + DISPLAY( "--format=gzip : output .gz files \n"); +#endif #endif #ifndef ZSTD_NODECOMPRESS DISPLAY( "--test : test compressed file integrity \n"); @@ -359,6 +362,7 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } + if (!strcmp(argument, "--format=gzip")) { FIO_setCompresionType(FIO_gzipCompression); continue; } /* long commands with arguments */ #ifndef ZSTD_NODICT From 02018c83cfaff4289907e30ae217cf41e1d47df1 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 8 Feb 2017 16:54:23 +0100 Subject: [PATCH 024/223] added FIO_compressGzFrame --- programs/fileio.c | 89 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 80 insertions(+), 9 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 087bf9504..0f6e00670 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -38,7 +38,7 @@ #ifdef ZSTD_MULTITHREAD # include "zstdmt_compress.h" #endif -#ifdef ZSTD_GZDECOMPRESS +#if defined(ZSTD_GZCOMPRESS) || defined(ZSTD_GZDECOMPRESS) # include # if !defined(z_const) # define z_const @@ -95,6 +95,8 @@ static clock_t g_time = 0; /*-************************************* * Local Parameters - Not thread safe ***************************************/ +static FIO_compresionType_t g_compresionType = FIO_zstdCompression; +void FIO_setCompresionType(FIO_compresionType_t compresionType) { g_compresionType = compresionType; } static U32 g_overwrite = 0; void FIO_overwriteMode(void) { g_overwrite=1; } static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ @@ -335,13 +337,75 @@ static void FIO_freeCResources(cRess_t ress) } +#ifdef ZSTD_GZCOMPRESS +static unsigned long long FIO_compressGzFrame(cRess_t* ress, const char* srcFileName, U64 const srcFileSize, int compressionLevel, U64* readsize) +{ + unsigned long long inFileSize = 0, outFileSize = 0; + z_stream strm; + + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.next_in = 0; + strm.avail_in = Z_NULL; + if (deflateInit2(&strm, compressionLevel, Z_DEFLATED, 15 /* maxWindowLogSize */ + 16 /* gzip only */, 8, Z_DEFAULT_STRATEGY) != Z_OK) + EXM_THROW(70, "deflateInit2 error"); /* see http://www.zlib.net/manual.html */ + + strm.next_out = (Bytef*)ress->dstBuffer; + strm.avail_out = (uInt)ress->dstBufferSize; + + while (1) { + int ret; + if (strm.avail_in == 0) { + size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); + if (inSize == 0) break; + inFileSize += inSize; + strm.next_in = (z_const unsigned char*)ress->srcBuffer; + strm.avail_in = (uInt)inSize; + } + ret = deflate(&strm, Z_NO_FLUSH); + if (ret != Z_OK) EXM_THROW(71, "zstd: %s: deflate error %d \n", srcFileName, ret); + { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; + if (decompBytes) { + if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(73, "Write error : cannot write to output file"); + outFileSize += decompBytes; + strm.next_out = (Bytef*)ress->dstBuffer; + strm.avail_out = (uInt)ress->dstBufferSize; + } + } + if (!srcFileSize) DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", (U32)(inFileSize>>20), (double)outFileSize/inFileSize*100) + else DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", (U32)(inFileSize>>20), (U32)(srcFileSize>>20), (double)outFileSize/inFileSize*100); + } + + while (1) { + int ret = deflate(&strm, Z_FINISH); + if (ret != Z_OK && ret != Z_STREAM_END) EXM_THROW(75, "zstd: %s: deflate error %d \n", srcFileName, ret); + { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; + if (decompBytes) { + if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(77, "Write error : cannot write to output file"); + outFileSize += decompBytes; + strm.next_out = (Bytef*)ress->dstBuffer; + strm.avail_out = (uInt)ress->dstBufferSize; + } + } + if (ret == Z_STREAM_END) break; + } + + deflateEnd(&strm); + *readsize = inFileSize; + + return outFileSize; +} +#endif + + /*! FIO_compressFilename_internal() : * same as FIO_compressFilename_extRess(), with `ress.desFile` already opened. * @return : 0 : compression completed correctly, * 1 : missing or pb opening srcFileName */ static int FIO_compressFilename_internal(cRess_t ress, - const char* dstFileName, const char* srcFileName) + const char* dstFileName, const char* srcFileName, int compressionLevel) { FILE* const srcFile = ress.srcFile; FILE* const dstFile = ress.dstFile; @@ -349,6 +413,12 @@ static int FIO_compressFilename_internal(cRess_t ress, U64 compressedfilesize = 0; U64 const fileSize = UTIL_getFileSize(srcFileName); + if (g_compresionType) { + compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); + // printf("g_compresionType=%d compressionLevel=%d compressedfilesize=%d\n", g_compresionType, compressionLevel, (int)compressedfilesize); + goto finish; + } + /* init */ #ifdef ZSTD_MULTITHREAD { size_t const resetError = ZSTDMT_resetCStream(ress.cctx, fileSize); @@ -406,6 +476,7 @@ static int FIO_compressFilename_internal(cRess_t ress, } } +finish: /* Status */ DISPLAYLEVEL(2, "\r%79s\r", ""); DISPLAYLEVEL(2,"%-20s :%6.2f%% (%6llu => %6llu bytes, %s) \n", srcFileName, @@ -423,7 +494,7 @@ static int FIO_compressFilename_internal(cRess_t ress, * 1 : missing or pb opening srcFileName */ static int FIO_compressFilename_srcFile(cRess_t ress, - const char* dstFileName, const char* srcFileName) + const char* dstFileName, const char* srcFileName, int compressionLevel) { int result; @@ -436,7 +507,7 @@ static int FIO_compressFilename_srcFile(cRess_t ress, ress.srcFile = FIO_openSrcFile(srcFileName); if (!ress.srcFile) return 1; /* srcFile could not be opened */ - result = FIO_compressFilename_internal(ress, dstFileName, srcFileName); + result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, compressionLevel); fclose(ress.srcFile); if (g_removeSrcFile && !result) { if (remove(srcFileName)) EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno)); } /* remove source file : --rm */ @@ -449,7 +520,7 @@ static int FIO_compressFilename_srcFile(cRess_t ress, * 1 : pb */ static int FIO_compressFilename_dstFile(cRess_t ress, - const char* dstFileName, const char* srcFileName) + const char* dstFileName, const char* srcFileName, int compressionLevel) { int result; stat_t statbuf; @@ -459,7 +530,7 @@ static int FIO_compressFilename_dstFile(cRess_t ress, if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ if (strcmp (srcFileName, stdinmark) && UTIL_getFileStat(srcFileName, &statbuf)) stat_result = 1; - result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName); + result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, compressionLevel); if (fclose(ress.dstFile)) { DISPLAYLEVEL(1, "zstd: %s: %s \n", dstFileName, strerror(errno)); result=1; } /* error closing dstFile */ if (result!=0) { if (remove(dstFileName)) EXM_THROW(1, "zstd: %s: %s", dstFileName, strerror(errno)); } /* remove operation artefact */ @@ -475,7 +546,7 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, U64 const srcSize = UTIL_getFileSize(srcFileName); cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams); - int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName); + int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel); double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC; DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds); @@ -507,7 +578,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile ress.dstFile = stdout; SET_BINARY_MODE(stdout); for (u=0; u Date: Wed, 8 Feb 2017 17:37:14 +0100 Subject: [PATCH 025/223] .gz suffix for gzip compressed files --- programs/fileio.c | 24 +++++++++++++++--------- programs/zstdcli.c | 6 ++++-- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 0f6e00670..6b5f80ff4 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -342,20 +342,21 @@ static unsigned long long FIO_compressGzFrame(cRess_t* ress, const char* srcFile { unsigned long long inFileSize = 0, outFileSize = 0; z_stream strm; + int ret; strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; + + if (deflateInit2(&strm, compressionLevel, Z_DEFLATED, 15 /* maxWindowLogSize */ + 16 /* gzip only */, 8, Z_DEFAULT_STRATEGY) != Z_OK) + EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); /* see http://www.zlib.net/manual.html */ + strm.next_in = 0; strm.avail_in = Z_NULL; - if (deflateInit2(&strm, compressionLevel, Z_DEFLATED, 15 /* maxWindowLogSize */ + 16 /* gzip only */, 8, Z_DEFAULT_STRATEGY) != Z_OK) - EXM_THROW(70, "deflateInit2 error"); /* see http://www.zlib.net/manual.html */ - strm.next_out = (Bytef*)ress->dstBuffer; strm.avail_out = (uInt)ress->dstBufferSize; while (1) { - int ret; if (strm.avail_in == 0) { size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile); if (inSize == 0) break; @@ -364,7 +365,7 @@ static unsigned long long FIO_compressGzFrame(cRess_t* ress, const char* srcFile strm.avail_in = (uInt)inSize; } ret = deflate(&strm, Z_NO_FLUSH); - if (ret != Z_OK) EXM_THROW(71, "zstd: %s: deflate error %d \n", srcFileName, ret); + if (ret != Z_OK) EXM_THROW(72, "zstd: %s: deflate error %d \n", srcFileName, ret); { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; if (decompBytes) { if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(73, "Write error : cannot write to output file"); @@ -378,20 +379,21 @@ static unsigned long long FIO_compressGzFrame(cRess_t* ress, const char* srcFile } while (1) { - int ret = deflate(&strm, Z_FINISH); - if (ret != Z_OK && ret != Z_STREAM_END) EXM_THROW(75, "zstd: %s: deflate error %d \n", srcFileName, ret); + ret = deflate(&strm, Z_FINISH); { size_t const decompBytes = ress->dstBufferSize - strm.avail_out; if (decompBytes) { - if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(77, "Write error : cannot write to output file"); + if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(75, "Write error : cannot write to output file"); outFileSize += decompBytes; strm.next_out = (Bytef*)ress->dstBuffer; strm.avail_out = (uInt)ress->dstBufferSize; } } if (ret == Z_STREAM_END) break; + if (ret != Z_BUF_ERROR) EXM_THROW(77, "zstd: %s: deflate error %d \n", srcFileName, ret); } - deflateEnd(&strm); + ret = deflateEnd(&strm); + if (ret != Z_OK) EXM_THROW(79, "zstd: %s: deflateEnd error %d \n", srcFileName, ret); *readsize = inFileSize; return outFileSize; @@ -414,9 +416,13 @@ static int FIO_compressFilename_internal(cRess_t ress, U64 const fileSize = UTIL_getFileSize(srcFileName); if (g_compresionType) { +#ifdef ZSTD_GZCOMPRESS compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); // printf("g_compresionType=%d compressionLevel=%d compressedfilesize=%d\n", g_compresionType, compressionLevel, (int)compressedfilesize); goto finish; +#else + EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", srcFileName); +#endif } /* init */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 651255b02..08e169ed8 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -49,6 +49,7 @@ #define AUTHOR "Yann Collet" #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR +#define GZ_EXTENSION ".gz" #define ZSTD_EXTENSION ".zst" #define ZSTD_CAT "zstdcat" #define ZSTD_UNZSTD "unzstd" @@ -286,6 +287,7 @@ int main(int argCount, const char* argv[]) const char* programName = argv[0]; const char* outFileName = NULL; const char* dictFileName = NULL; + const char* suffix = ZSTD_EXTENSION; unsigned maxDictSize = g_defaultMaxDictSize; unsigned dictID = 0; int dictCLevel = g_defaultDictCLevel; @@ -362,7 +364,7 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } - if (!strcmp(argument, "--format=gzip")) { FIO_setCompresionType(FIO_gzipCompression); continue; } + if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompresionType(FIO_gzipCompression); continue; } /* long commands with arguments */ #ifndef ZSTD_NODICT @@ -647,7 +649,7 @@ int main(int argCount, const char* argv[]) if ((filenameIdx==1) && outFileName) operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, &compressionParams); else - operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : ZSTD_EXTENSION, dictFileName, cLevel, &compressionParams); + operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName ? outFileName : suffix, dictFileName, cLevel, &compressionParams); #else DISPLAY("Compression not supported\n"); #endif From 4f9eaa7bb35e57e0e294c8a397410d86e656f432 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 8 Feb 2017 18:08:09 +0100 Subject: [PATCH 026/223] fixed gcc warnings --- programs/fileio.c | 3 ++- programs/zstdcli.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 6b5f80ff4..430127771 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -419,10 +419,11 @@ static int FIO_compressFilename_internal(cRess_t ress, #ifdef ZSTD_GZCOMPRESS compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); // printf("g_compresionType=%d compressionLevel=%d compressedfilesize=%d\n", g_compresionType, compressionLevel, (int)compressedfilesize); - goto finish; #else + (void)compressionLevel; EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", srcFileName); #endif + goto finish; } /* init */ diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 08e169ed8..30aa0b949 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -125,7 +125,7 @@ static int usage_advanced(const char* programName) DISPLAY( " -B# : select size of independent sections (default:0==automatic) \n"); #endif #ifdef ZSTD_GZCOMPRESS - DISPLAY( "--format=gzip : output .gz files \n"); + DISPLAY( "--format=gzip : compress files to the .gz format \n"); #endif #endif #ifndef ZSTD_NODECOMPRESS From b5e46b1255177728246ffaf759316c242c2a6a6a Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Wed, 8 Feb 2017 12:00:21 -0800 Subject: [PATCH 027/223] Remove test-longmatch from test target and only run it once --- .travis.yml | 2 +- tests/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 90306dab1..885e4517a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,7 @@ matrix: # Standard Ubuntu 12.04 LTS Server Edition 64 bit - - env: Ubu=12.04 Cmd="make -C programs zstd-small zstd-decompress zstd-compress && make -C programs clean && make -C tests versionsTest" + - env: Ubu=12.04 Cmd="make -C programs zstd-small zstd-decompress zstd-compress && make -C programs clean && make -C tests versionsTest test-longmatch" os: linux sudo: required diff --git a/tests/Makefile b/tests/Makefile index f49d23018..452f080ea 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -225,7 +225,7 @@ zstd-playTests: datagen file $(ZSTD) ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST) -test: test-zstd test-fullbench test-fuzzer test-zstream test-longmatch test-invalidDictionaries test-pool +test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-pool test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32 From 93901fe85ca3eb342dfbbbc6c4778c5f653a438c Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 8 Feb 2017 21:11:18 +0100 Subject: [PATCH 028/223] remove redundant "ifeq ($(HAVE_ZLIB), 1)" --- programs/Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index b189224f3..8257e394e 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -70,8 +70,6 @@ VOID = /dev/null HAVE_ZLIB := $(shell echo -e "\#include \nint main(){}" | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && echo 1 || echo 0) ifeq ($(HAVE_ZLIB), 1) TEMP := $(shell rm have_zlib$(EXT)) -endif -ifeq ($(HAVE_ZLIB), 1) ZLIBCPP = -DZSTD_GZDECOMPRESS ZLIBLD = -lz endif From 4e709712e1a01981a39da3e8458635f688927953 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Tue, 7 Feb 2017 13:50:09 -0800 Subject: [PATCH 029/223] Decompressed size functions now handle multiframes and distinguish cases - Add ZSTD_findDecompressedSize - Traverses multiple frames to find total output size - Add ZSTD_getFrameContentSize - Gets the decompressed size of a single frame by reading header - Deprecate ZSTD_getDecompressedSize --- examples/dictionary_decompression.c | 2 +- examples/simple_decompression.c | 2 +- lib/decompress/zstd_decompress.c | 146 ++++++++++++++++++++++++++-- lib/dll/libzstd.def | 2 + lib/legacy/zstd_legacy.h | 24 +++++ lib/legacy/zstd_v01.c | 31 ++++++ lib/legacy/zstd_v01.h | 8 ++ lib/legacy/zstd_v02.c | 37 +++++++ lib/legacy/zstd_v02.h | 8 ++ lib/legacy/zstd_v03.c | 37 +++++++ lib/legacy/zstd_v03.h | 8 ++ lib/legacy/zstd_v04.c | 33 +++++++ lib/legacy/zstd_v04.h | 8 ++ lib/legacy/zstd_v05.c | 29 ++++++ lib/legacy/zstd_v05.h | 7 ++ lib/legacy/zstd_v06.c | 31 ++++++ lib/legacy/zstd_v06.h | 7 ++ lib/legacy/zstd_v07.c | 32 ++++++ lib/legacy/zstd_v07.h | 8 ++ lib/zstd.h | 40 +++++++- programs/bench.c | 4 +- tests/fuzzer.c | 4 +- tests/symbols.c | 2 + tests/zstreamtest.c | 2 +- 24 files changed, 496 insertions(+), 16 deletions(-) diff --git a/examples/dictionary_decompression.c b/examples/dictionary_decompression.c index db9417b30..2aa71b268 100644 --- a/examples/dictionary_decompression.c +++ b/examples/dictionary_decompression.c @@ -76,7 +76,7 @@ static void decompress(const char* fname, const ZSTD_DDict* ddict) { size_t cSize; void* const cBuff = loadFile_orDie(fname, &cSize); - unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize); + unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize); if (rSize==0) { fprintf(stderr, "%s : original size unknown \n", fname); exit(6); diff --git a/examples/simple_decompression.c b/examples/simple_decompression.c index 62a881fcb..3a75e164c 100644 --- a/examples/simple_decompression.c +++ b/examples/simple_decompression.c @@ -63,7 +63,7 @@ static void decompress(const char* fname) { size_t cSize; void* const cBuff = loadFile_X(fname, &cSize); - unsigned long long const rSize = ZSTD_getDecompressedSize(cBuff, cSize); + unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize); if (rSize==0) { printf("%s : original size unknown. Use streaming decompression instead. \n", fname); exit(5); diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 9c04503d2..00c0d937e 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -306,6 +306,100 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t return 0; } +static size_t ZSTD_frameSrcSize(const void* src, size_t srcSize); + +/** ZSTD_getFrameContentSize() : +* compatible with legacy mode +* @return : decompressed size of the single frame pointed to be `src` if known, otherwise +* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined +* - ZSTD_CONTENTSIZE_ERROR if an error occured (e.g. invalid magic number, srcSize too small) */ +unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) +{ +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) + if (ZSTD_isLegacy(src, srcSize)) { + unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize); + return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret; + } +#endif + { + ZSTD_frameParams fParams; + if (ZSTD_getFrameParams(&fParams, src, srcSize) != 0) return ZSTD_CONTENTSIZE_ERROR; + if (fParams.windowSize == 0) { + /* Either skippable or empty frame, size == 0 either way */ + return 0; + } else if (fParams.frameContentSize != 0) { + return fParams.frameContentSize; + } else { + return ZSTD_CONTENTSIZE_UNKNOWN; + } + } +} + +/** ZSTD_findDecompressedSize() : + * compatible with legacy mode + * `srcSize` must be the exact length of some number of ZSTD compressed and/or + * skippable frames + * @return : decompressed size of the frames contained */ +unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) +{ + { + unsigned long long totalDstSize = 0; + while (srcSize >= ZSTD_frameHeaderSize_prefix) { + const U32 magicNumber = MEM_readLE32(src); + + if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { + size_t skippableSize; + if (srcSize < ZSTD_skippableHeaderSize) + return ERROR(srcSize_wrong); + skippableSize = MEM_readLE32((const BYTE *)src + 4) + + ZSTD_skippableHeaderSize; + if (srcSize < skippableSize) { + /* srcSize_wrong */ + return 0; + } + + src = (const BYTE *)src + skippableSize; + srcSize -= skippableSize; + continue; + } + + { + unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); + if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret; + + /* check for overflow */ + if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR; + totalDstSize += ret; + } + { + size_t frameSrcSize; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) + if (ZSTD_isLegacy(src, srcSize)) + { + frameSrcSize = ZSTD_frameSrcSizeLegacy(src, srcSize); + } + else +#endif + { + frameSrcSize = ZSTD_frameSrcSize(src, srcSize); + } + if (ZSTD_isError(frameSrcSize)) { + return 0; + } + + src = (const BYTE *)src + frameSrcSize; + srcSize -= frameSrcSize; + } + } + + if (srcSize) { + /* srcSize_wrong */ + return 0; + } + + return totalDstSize; + } +} /** ZSTD_getDecompressedSize() : * compatible with legacy mode @@ -316,14 +410,8 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t - frame header not complete (`srcSize` too small) */ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) { -#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) - if (ZSTD_isLegacy(src, srcSize)) return ZSTD_getDecompressedSize_legacy(src, srcSize); -#endif - { ZSTD_frameParams fparams; - size_t const frResult = ZSTD_getFrameParams(&fparams, src, srcSize); - if (frResult!=0) return 0; - return fparams.frameContentSize; - } + unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); + return ret >= ZSTD_CONTENTSIZE_ERROR ? 0 : ret; } @@ -1363,6 +1451,48 @@ size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t len return length; } +static size_t ZSTD_frameSrcSize(const void *src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + const BYTE* const ipstart = ip; + size_t remainingSize = srcSize; + ZSTD_frameParams fParams; + + size_t const headerSize = ZSTD_frameHeaderSize(ip, remainingSize); + if (ZSTD_isError(headerSize)) return headerSize; + + /* Frame Header */ + { + size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize); + if (ZSTD_isError(ret)) return ret; + if (ret > 0) return ERROR(srcSize_wrong); + } + + ip += headerSize; + remainingSize -= headerSize; + + /* Loop on each block */ + while (1) { + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + + if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) return ERROR(srcSize_wrong); + + ip += ZSTD_blockHeaderSize + cBlockSize; + remainingSize -= ZSTD_blockHeaderSize + cBlockSize; + + if (blockProperties.lastBlock) break; + } + + if (fParams.checksumFlag) { /* Frame content checksum verification */ + if (remainingSize < 4) return ERROR(srcSize_wrong); + ip += 4; + remainingSize -= 4; + } + + return ip - ipstart; +} /*! ZSTD_decompressFrame() : * `dctx` must be properly initialized */ diff --git a/lib/dll/libzstd.def b/lib/dll/libzstd.def index 0a3259e9e..51d0c1925 100644 --- a/lib/dll/libzstd.def +++ b/lib/dll/libzstd.def @@ -58,6 +58,8 @@ EXPORTS ZSTD_getBlockSizeMax ZSTD_getCParams ZSTD_getDecompressedSize + ZSTD_findDecompressedSize + ZSTD_getFrameContentSize ZSTD_getErrorName ZSTD_getFrameParams ZSTD_getParams diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h index 2a9f36aa2..c0369ab4f 100644 --- a/lib/legacy/zstd_legacy.h +++ b/lib/legacy/zstd_legacy.h @@ -123,6 +123,30 @@ MEM_STATIC size_t ZSTD_decompressLegacy( } } +MEM_STATIC size_t ZSTD_frameSrcSizeLegacy(const void *src, + size_t compressedSize) +{ + U32 const version = ZSTD_isLegacy(src, compressedSize); + switch(version) + { + case 1 : + return ZSTDv01_frameSrcSize(src, compressedSize); + case 2 : + return ZSTDv02_frameSrcSize(src, compressedSize); + case 3 : + return ZSTDv03_frameSrcSize(src, compressedSize); + case 4 : + return ZSTDv04_frameSrcSize(src, compressedSize); + case 5 : + return ZSTDv05_frameSrcSize(src, compressedSize); + case 6 : + return ZSTDv06_frameSrcSize(src, compressedSize); + case 7 : + return ZSTDv07_frameSrcSize(src, compressedSize); + default : + return ERROR(prefix_unknown); + } +} MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version) { diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c index 6fd30c0ac..c9b676ad8 100644 --- a/lib/legacy/zstd_v01.c +++ b/lib/legacy/zstd_v01.c @@ -1992,6 +1992,37 @@ size_t ZSTDv01_decompress(void* dst, size_t maxDstSize, const void* src, size_t return ZSTDv01_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize); } +size_t ZSTDv01_frameSrcSize(const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + size_t remainingSize = srcSize; + U32 magicNumber; + blockProperties_t blockProperties; + + /* Frame Header */ + if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); + magicNumber = ZSTD_readBE32(src); + if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown); + ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize; + + /* Loop on each block */ + while (1) + { + size_t blockSize = ZSTDv01_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTDv01_isError(blockSize)) return blockSize; + + ip += ZSTD_blockHeaderSize; + remainingSize -= ZSTD_blockHeaderSize; + if (blockSize > remainingSize) return ERROR(srcSize_wrong); + + if (blockSize == 0) break; /* bt_end */ + + ip += blockSize; + remainingSize -= blockSize; + } + + return ip - (const BYTE*)src; +} /******************************* * Streaming Decompression API diff --git a/lib/legacy/zstd_v01.h b/lib/legacy/zstd_v01.h index 0f2323db9..9e5d553bf 100644 --- a/lib/legacy/zstd_v01.h +++ b/lib/legacy/zstd_v01.h @@ -34,6 +34,14 @@ ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize, const void* src, size_t compressedSize); +/** +ZSTDv01_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.1.x format + compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' + return : the number of bytes that would be read to decompress this frame + or an errorCode if it fails (which can be tested using ZSTDv01_isError()) +*/ +size_t ZSTDv01_frameSrcSize(const void* src, size_t compressedSize); + /** ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error */ diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c index b8a12ab5e..f3b107af3 100644 --- a/lib/legacy/zstd_v02.c +++ b/lib/legacy/zstd_v02.c @@ -3378,6 +3378,38 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize); } +static size_t ZSTD_frameSrcSize(const void *src, size_t srcSize) +{ + + const BYTE* ip = (const BYTE*)src; + size_t remainingSize = srcSize; + U32 magicNumber; + blockProperties_t blockProperties; + + /* Frame Header */ + if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); + magicNumber = MEM_readLE32(src); + if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown); + ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize; + + /* Loop on each block */ + while (1) + { + size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + + ip += ZSTD_blockHeaderSize; + remainingSize -= ZSTD_blockHeaderSize; + if (cBlockSize > remainingSize) return ERROR(srcSize_wrong); + + if (cBlockSize == 0) break; /* bt_end */ + + ip += cBlockSize; + remainingSize -= cBlockSize; + } + + return ip - (const BYTE*)src; +} /******************************* * Streaming Decompression API @@ -3492,6 +3524,11 @@ size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize, return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize); } +size_t ZSTDv02_frameSrcSize(const void *src, size_t compressedSize) +{ + return ZSTD_frameSrcSize(src, compressedSize); +} + ZSTDv02_Dctx* ZSTDv02_createDCtx(void) { return (ZSTDv02_Dctx*)ZSTD_createDCtx(); diff --git a/lib/legacy/zstd_v02.h b/lib/legacy/zstd_v02.h index a371bd181..45dc5f6ca 100644 --- a/lib/legacy/zstd_v02.h +++ b/lib/legacy/zstd_v02.h @@ -34,6 +34,14 @@ ZSTDv02_decompress() : decompress ZSTD frames compliant with v0.2.x format size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize, const void* src, size_t compressedSize); +/** +ZSTDv02_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.2.x format + compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' + return : the number of bytes that would be read to decompress this frame + or an errorCode if it fails (which can be tested using ZSTDv02_isError()) +*/ +size_t ZSTDv02_frameSrcSize(const void* src, size_t compressedSize); + /** ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error */ diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c index 6459da3d4..2eba77ccc 100644 --- a/lib/legacy/zstd_v03.c +++ b/lib/legacy/zstd_v03.c @@ -3019,6 +3019,38 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize); } +static size_t ZSTD_frameSrcSize(const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + size_t remainingSize = srcSize; + U32 magicNumber; + blockProperties_t blockProperties; + + /* Frame Header */ + if (srcSize < ZSTD_frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); + magicNumber = MEM_readLE32(src); + if (magicNumber != ZSTD_magicNumber) return ERROR(prefix_unknown); + ip += ZSTD_frameHeaderSize; remainingSize -= ZSTD_frameHeaderSize; + + /* Loop on each block */ + while (1) + { + size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + + ip += ZSTD_blockHeaderSize; + remainingSize -= ZSTD_blockHeaderSize; + if (cBlockSize > remainingSize) return ERROR(srcSize_wrong); + + if (cBlockSize == 0) break; /* bt_end */ + + ip += cBlockSize; + remainingSize -= cBlockSize; + } + + return ip - (const BYTE*)src; +} + /******************************* * Streaming Decompression API @@ -3133,6 +3165,11 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize, return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize); } +size_t ZSTDv03_frameSrcSize(const void* src, size_t srcSize) +{ + return ZSTD_frameSrcSize(src, srcSize); +} + ZSTDv03_Dctx* ZSTDv03_createDCtx(void) { return (ZSTDv03_Dctx*)ZSTD_createDCtx(); diff --git a/lib/legacy/zstd_v03.h b/lib/legacy/zstd_v03.h index 8b89737b1..24dba1f59 100644 --- a/lib/legacy/zstd_v03.h +++ b/lib/legacy/zstd_v03.h @@ -35,6 +35,14 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize, const void* src, size_t compressedSize); /** +ZSTDv03_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.3.x format + compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' + return : the number of bytes that would be read to decompress this frame + or an errorCode if it fails (which can be tested using ZSTDv03_isError()) +*/ +size_t ZSTDv03_frameSrcSize(const void* src, size_t compressedSize); + + /** ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error */ unsigned ZSTDv03_isError(size_t code); diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index 723242c6c..7c900e1e0 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -3326,6 +3326,35 @@ static size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx, return op-ostart; } +static size_t ZSTD_frameSrcSize(const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + size_t remainingSize = srcSize; + blockProperties_t blockProperties; + + /* Frame Header */ + if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); + if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) return ERROR(prefix_unknown); + ip += ZSTD_frameHeaderSize_min; remainingSize -= ZSTD_frameHeaderSize_min; + + /* Loop on each block */ + while (1) + { + size_t cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + + ip += ZSTD_blockHeaderSize; + remainingSize -= ZSTD_blockHeaderSize; + if (cBlockSize > remainingSize) return ERROR(srcSize_wrong); + + if (cBlockSize == 0) break; /* bt_end */ + + ip += cBlockSize; + remainingSize -= cBlockSize; + } + + return ip - (const BYTE*)src; +} /* ****************************** * Streaming Decompression API @@ -3753,6 +3782,10 @@ size_t ZSTDv04_decompress(void* dst, size_t maxDstSize, const void* src, size_t #endif } +size_t ZSTDv04_frameSrcSize(const void* src, size_t srcSize) +{ + return ZSTD_frameSrcSize(src, srcSize); +} size_t ZSTDv04_resetDCtx(ZSTDv04_Dctx* dctx) { return ZSTD_resetDCtx(dctx); } diff --git a/lib/legacy/zstd_v04.h b/lib/legacy/zstd_v04.h index 370553b18..671ba6dc0 100644 --- a/lib/legacy/zstd_v04.h +++ b/lib/legacy/zstd_v04.h @@ -34,6 +34,14 @@ ZSTDv04_decompress() : decompress ZSTD frames compliant with v0.4.x format size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize, const void* src, size_t compressedSize); +/** +ZSTDv04_getFrameSrcSize() : get the source length of a ZSTD frame compliant with v0.4.x format + compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' + return : the number of bytes that would be read to decompress this frame + or an errorCode if it fails (which can be tested using ZSTDv04_isError()) +*/ +size_t ZSTDv04_frameSrcSize(const void* src, size_t compressedSize); + /** ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error */ diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index f13592420..85c0f0e73 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -3583,6 +3583,35 @@ size_t ZSTDv05_decompress(void* dst, size_t maxDstSize, const void* src, size_t #endif } +size_t ZSTDv05_frameSrcSize(const void *src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + size_t remainingSize = srcSize; + blockProperties_t blockProperties; + + /* Frame Header */ + if (srcSize < ZSTDv05_frameHeaderSize_min) return ERROR(srcSize_wrong); + if (MEM_readLE32(src) != ZSTDv05_MAGICNUMBER) return ERROR(prefix_unknown); + ip += ZSTDv05_frameHeaderSize_min; remainingSize -= ZSTDv05_frameHeaderSize_min; + + /* Loop on each block */ + while (1) + { + size_t cBlockSize = ZSTDv05_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTDv05_isError(cBlockSize)) return cBlockSize; + + ip += ZSTDv05_blockHeaderSize; + remainingSize -= ZSTDv05_blockHeaderSize; + if (cBlockSize > remainingSize) return ERROR(srcSize_wrong); + + if (cBlockSize == 0) break; /* bt_end */ + + ip += cBlockSize; + remainingSize -= cBlockSize; + } + + return ip - (const BYTE*)src; +} /* ****************************** * Streaming Decompression API diff --git a/lib/legacy/zstd_v05.h b/lib/legacy/zstd_v05.h index da26d96cf..ef2d18d10 100644 --- a/lib/legacy/zstd_v05.h +++ b/lib/legacy/zstd_v05.h @@ -32,6 +32,13 @@ extern "C" { size_t ZSTDv05_decompress( void* dst, size_t dstCapacity, const void* src, size_t compressedSize); +/** +ZSTDv05_getFrameSrcSize() : get the source length of a ZSTD frame + compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' + return : the number of bytes that would be read to decompress this frame + or an errorCode if it fails (which can be tested using ZSTDv05_isError()) +*/ +size_t ZSTDv05_frameSrcSize(const void* src, size_t compressedSize); /* ************************************* * Helper functions diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index 8be4bc377..92c1a45c6 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -3729,6 +3729,37 @@ size_t ZSTDv06_decompress(void* dst, size_t dstCapacity, const void* src, size_t #endif } +size_t ZSTDv06_frameSrcSize(const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + size_t remainingSize = srcSize; + blockProperties_t blockProperties = { bt_compressed, 0 }; + + /* Frame Header */ + { size_t const frameHeaderSize = ZSTDv06_frameHeaderSize(src, ZSTDv06_frameHeaderSize_min); + if (ZSTDv06_isError(frameHeaderSize)) return frameHeaderSize; + if (MEM_readLE32(src) != ZSTDv06_MAGICNUMBER) return ERROR(prefix_unknown); + if (srcSize < frameHeaderSize+ZSTDv06_blockHeaderSize) return ERROR(srcSize_wrong); + ip += frameHeaderSize; remainingSize -= frameHeaderSize; + } + + /* Loop on each block */ + while (1) { + size_t const cBlockSize = ZSTDv06_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTDv06_isError(cBlockSize)) return cBlockSize; + + ip += ZSTDv06_blockHeaderSize; + remainingSize -= ZSTDv06_blockHeaderSize; + if (cBlockSize > remainingSize) return ERROR(srcSize_wrong); + + if (cBlockSize == 0) break; /* bt_end */ + + ip += cBlockSize; + remainingSize -= cBlockSize; + } + + return ip - (const BYTE*)src; +} /*_****************************** * Streaming Decompression API diff --git a/lib/legacy/zstd_v06.h b/lib/legacy/zstd_v06.h index 14040abdd..1fad7311e 100644 --- a/lib/legacy/zstd_v06.h +++ b/lib/legacy/zstd_v06.h @@ -41,6 +41,13 @@ extern "C" { ZSTDLIBv06_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity, const void* src, size_t compressedSize); +/** +ZSTDv06_getFrameSrcSize() : get the source length of a ZSTD frame + compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' + return : the number of bytes that would be read to decompress this frame + or an errorCode if it fails (which can be tested using ZSTDv06_isError()) +*/ +size_t ZSTDv06_frameSrcSize(const void* src, size_t compressedSize); /* ************************************* * Helper functions diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index b1fcdc766..4ee055ddd 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -3968,6 +3968,38 @@ size_t ZSTDv07_decompress(void* dst, size_t dstCapacity, const void* src, size_t #endif } +size_t ZSTDv07_frameSrcSize(const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + size_t remainingSize = srcSize; + + /* check */ + if (srcSize < ZSTDv07_frameHeaderSize_min+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong); + + /* Frame Header */ + { size_t const frameHeaderSize = ZSTDv07_frameHeaderSize(src, ZSTDv07_frameHeaderSize_min); + if (ZSTDv07_isError(frameHeaderSize)) return frameHeaderSize; + if (MEM_readLE32(src) != ZSTDv07_MAGICNUMBER) return ERROR(prefix_unknown); + if (srcSize < frameHeaderSize+ZSTDv07_blockHeaderSize) return ERROR(srcSize_wrong); + ip += frameHeaderSize; remainingSize -= frameHeaderSize; + } + + /* Loop on each block */ + while (1) { + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTDv07_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTDv07_isError(cBlockSize)) return cBlockSize; + + ip += ZSTDv07_blockHeaderSize; + remainingSize -= ZSTDv07_blockHeaderSize; + if (cBlockSize > remainingSize) return ERROR(srcSize_wrong); + + ip += cBlockSize; + remainingSize -= cBlockSize; + } + + return ip - (const BYTE*)src; +} /*_****************************** * Streaming Decompression API diff --git a/lib/legacy/zstd_v07.h b/lib/legacy/zstd_v07.h index 30725dcf7..b02b3dce4 100644 --- a/lib/legacy/zstd_v07.h +++ b/lib/legacy/zstd_v07.h @@ -48,6 +48,14 @@ unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize); ZSTDLIBv07_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity, const void* src, size_t compressedSize); +/** +ZSTDv07_getFrameSrcSize() : get the source length of a ZSTD frame + compressedSize : The size of the 'src' buffer, at least as large as the frame pointed to by 'src' + return : the number of bytes that would be read to decompress this frame + or an errorCode if it fails (which can be tested using ZSTDv07_isError()) +*/ +size_t ZSTDv07_frameSrcSize(const void* src, size_t compressedSize); + /*====== Helper functions ======*/ ZSTDLIBv07_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ ZSTDLIBv07_API const char* ZSTDv07_getErrorName(size_t code); /*!< provides readable string from an error code */ diff --git a/lib/zstd.h b/lib/zstd.h index f5cbf4b48..90f0bca59 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -80,7 +80,7 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, int compressionLevel); /*! ZSTD_decompress() : - `compressedSize` : must be the _exact_ size of a single compressed frame. + `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. `dstCapacity` is an upper bound of originalSize. If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), @@ -88,7 +88,45 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, const void* src, size_t compressedSize); +#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) +#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) + +/*! ZSTD_getFrameContentSize() : +* `src` should point to the start of a ZSTD encoded frame +* @return : decompressed size of the frame pointed to be `src` if known, otherwise +* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined +* - ZSTD_CONTENTSIZE_ERROR if an error occured (e.g. invalid magic number, srcSize too small) */ +ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); + +/*! ZSTD_findDecompressedSize() : +* `src` should point the start of a series of ZSTD encoded and/or skippable frames +* `srcSize` must be the _exact_ size of this series +* (i.e. there should be a frame boundary exactly `srcSize` bytes after `src`) +* @return : the decompressed size of all data in the contained frames, as a 64-bit value _if known_ +* - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN +* - if an error occurred: ZSTD_CONTENTSIZE_ERROR +* +* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. +* When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. +* In which case, it's necessary to use streaming mode to decompress data. +* Optionally, application can still use ZSTD_decompress() while relying on implied limits. +* (For example, data may be necessarily cut into blocks <= 16 KB). +* note 2 : decompressed size is always present when compression is done with ZSTD_compress() +* note 3 : decompressed size can be very large (64-bits value), +* potentially larger than what local system can handle as a single memory segment. +* In which case, it's necessary to use streaming mode to decompress data. +* note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. +* Always ensure result fits within application's authorized limits. +* Each application can set its own limits. +* note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to +* read each contained frame header. This is efficient as most of the data is skipped, +* however it does mean that all frame data must be present and valid. */ +ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); + /*! ZSTD_getDecompressedSize() : +* WARNING: This function is now obsolete. ZSTD_findDecompressedSize should be used, +* or if only exactly one ZSTD frame is needed, ZSTD_getFrameContentSize can be used. +* * 'src' is the start of a zstd compressed frame. * @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise. * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. diff --git a/programs/bench.c b/programs/bench.c index dcb23b1f2..73489643f 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -184,7 +184,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, U64 dSize64 = 0; U32 fileNb; for (fileNb=0; fileNb Date: Tue, 7 Feb 2017 16:16:55 -0800 Subject: [PATCH 030/223] ZSTD_decompress now handles multiple frames --- lib/decompress/zstd_decompress.c | 119 ++++++++++++++++++++++++------- lib/legacy/zstd_v07.c | 3 + tests/fuzzer.c | 40 +++++++++++ 3 files changed, 137 insertions(+), 25 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 00c0d937e..18ee070cf 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -354,8 +354,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) skippableSize = MEM_readLE32((const BYTE *)src + 4) + ZSTD_skippableHeaderSize; if (srcSize < skippableSize) { - /* srcSize_wrong */ - return 0; + return ZSTD_CONTENTSIZE_ERROR; } src = (const BYTE *)src + skippableSize; @@ -384,7 +383,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) frameSrcSize = ZSTD_frameSrcSize(src, srcSize); } if (ZSTD_isError(frameSrcSize)) { - return 0; + return ZSTD_CONTENTSIZE_ERROR; } src = (const BYTE *)src + frameSrcSize; @@ -393,8 +392,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) } if (srcSize) { - /* srcSize_wrong */ - return 0; + return ZSTD_CONTENTSIZE_ERROR; } return totalDstSize; @@ -1498,22 +1496,22 @@ static size_t ZSTD_frameSrcSize(const void *src, size_t srcSize) * `dctx` must be properly initialized */ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, - const void* src, size_t srcSize) + const void** srcPtr, size_t *srcSizePtr) { - const BYTE* ip = (const BYTE*)src; + const BYTE* ip = (const BYTE*)(*srcPtr); BYTE* const ostart = (BYTE* const)dst; BYTE* const oend = ostart + dstCapacity; BYTE* op = ostart; - size_t remainingSize = srcSize; + size_t remainingSize = *srcSizePtr; /* check */ - if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); + if (remainingSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); /* Frame Header */ - { size_t const frameHeaderSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix); + { size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_frameHeaderSize_prefix); if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; - if (srcSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); - CHECK_F(ZSTD_decodeFrameHeader(dctx, src, frameHeaderSize)); + if (remainingSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); + CHECK_F(ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize)); ip += frameHeaderSize; remainingSize -= frameHeaderSize; } @@ -1558,25 +1556,98 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, if (remainingSize<4) return ERROR(checksum_wrong); checkRead = MEM_readLE32(ip); if (checkRead != checkCalc) return ERROR(checksum_wrong); + ip += 4; remainingSize -= 4; } - if (remainingSize) return ERROR(srcSize_wrong); + // Allow caller to get size read + *srcPtr = ip; + *srcSizePtr = remainingSize; return op-ostart; } +static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void *dict, size_t dictSize, + const ZSTD_DCtx* refContext) +{ + void* const dststart = dst; + while (srcSize >= ZSTD_frameHeaderSize_prefix) { + U32 magicNumber; + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(src, srcSize)) { + size_t const frameSize = ZSTD_frameSrcSizeLegacy(src, srcSize); + size_t decodedSize; + if (ZSTD_isError(frameSize)) return frameSize; + + decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize); + + dst = (BYTE*)dst + decodedSize; + dstCapacity -= decodedSize; + + src = (const BYTE*)src + frameSize; + srcSize -= frameSize; + + continue; + } +#endif + + magicNumber = MEM_readLE32(src); + if (magicNumber != ZSTD_MAGICNUMBER) { + if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { + size_t skippableSize; + if (srcSize < ZSTD_skippableHeaderSize) + return ERROR(srcSize_wrong); + skippableSize = MEM_readLE32((const BYTE *)src + 4) + + ZSTD_skippableHeaderSize; + if (srcSize < skippableSize) { + return ERROR(srcSize_wrong); + } + + src = (const BYTE *)src + skippableSize; + srcSize -= skippableSize; + continue; + } else { + return ERROR(prefix_unknown); + } + } + + if (refContext) { + /* we were called from ZSTD_decompress_usingDDict */ + ZSTD_refDCtx(dctx, refContext); + } else { + /* this will initialize correctly with no dict if dict == NULL, so + * use this in all cases but ddict */ + CHECK_F(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize)); + } + ZSTD_checkContinuity(dctx, dst); + + { + const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, + &src, &srcSize); + if (ZSTD_isError(res)) return res; + /* don't need to bounds check this, ZSTD_decompressFrame will have + * already */ + dst = (BYTE*)dst + res; + dstCapacity -= res; + } + } + + if (srcSize) { + return ERROR(srcSize_wrong); + } + + return (BYTE*)dst - (BYTE*)dststart; +} size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize) { -#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) - if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize); -#endif - CHECK_F(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize)); - ZSTD_checkContinuity(dctx, dst); - return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize); + return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL); } @@ -1973,12 +2044,10 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, const void* src, size_t srcSize, const ZSTD_DDict* ddict) { -#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) - if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dictContent, ddict->dictSize); -#endif - ZSTD_refDCtx(dctx, ddict->refContext); - ZSTD_checkContinuity(dctx, dst); - return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize); + /* pass content and size in case legacy frames are encountered */ + return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, + ddict->dictContent, ddict->dictSize, + ddict->refContext); } diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index 4ee055ddd..6228ee870 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -3992,6 +3992,9 @@ size_t ZSTDv07_frameSrcSize(const void* src, size_t srcSize) ip += ZSTDv07_blockHeaderSize; remainingSize -= ZSTDv07_blockHeaderSize; + + if (blockProperties.blockType == bt_end) break; + if (cBlockSize > remainingSize) return ERROR(srcSize_wrong); ip += cBlockSize; diff --git a/tests/fuzzer.c b/tests/fuzzer.c index e7f25aae5..4239af455 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -167,6 +167,46 @@ static int basicUnitTests(U32 seed, double compressibility) if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; } DISPLAYLEVEL(4, "OK \n"); + /* Simple API multiframe test */ + DISPLAYLEVEL(4, "test%3i : compress multiple frames : ", testNb++); + { size_t off = 0; + int i; + int const segs = 4; + /* only use the first half so we don't push against size limit of compressedBuffer */ + size_t const segSize = (CNBuffSize / 2) / segs; + for (i = 0; i < segs; i++) { + CHECK_V(r, + ZSTD_compress( + (BYTE *)compressedBuffer + off, CNBuffSize - off, + (BYTE *)CNBuffer + segSize * i, + segSize, 5)); + off += r; + if (i == segs/2) { + /* insert skippable frame */ + const U32 skipLen = 128 KB; + MEM_writeLE32((BYTE*)compressedBuffer + off, ZSTD_MAGIC_SKIPPABLE_START); + MEM_writeLE32((BYTE*)compressedBuffer + off + 4, skipLen); + off += skipLen + ZSTD_skippableHeaderSize; + } + } + cSize = off; + } + DISPLAYLEVEL(4, "OK \n"); + + DISPLAYLEVEL(4, "test%3i : get decompressed size of multiple frames : ", testNb++); + { unsigned long long const r = ZSTD_findDecompressedSize(compressedBuffer, cSize); + if (r != CNBuffSize / 2) goto _output_error; } + DISPLAYLEVEL(4, "OK \n"); + + DISPLAYLEVEL(4, "test%3i : decompress multiple frames : ", testNb++); + { CHECK_V(r, ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize)); + if (r != CNBuffSize / 2) goto _output_error; } + DISPLAYLEVEL(4, "OK \n"); + + DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); + if (memcmp(decodedBuffer, CNBuffer, CNBuffSize / 2) != 0) goto _output_error; + DISPLAYLEVEL(4, "OK \n"); + /* Dictionary and CCtx Duplication tests */ { ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx(); ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx(); From 0f5c95af441718f2e8b38ab1b02d8462084efd35 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Tue, 7 Feb 2017 16:33:48 -0800 Subject: [PATCH 031/223] Disambiguate pledgedSrcSize == 0 - Modify ZSTD CLI to only set contentSizeFlag if it _knows_ the size - Change pzstd to stop setting contentSizeFlag without accurate pledgedSrcSize --- contrib/pzstd/Options.h | 2 +- lib/compress/zstd_compress.c | 2 +- lib/zstd.h | 4 ++-- programs/fileio.c | 12 +++++++----- programs/util.h | 5 +++++ 5 files changed, 16 insertions(+), 9 deletions(-) diff --git a/contrib/pzstd/Options.h b/contrib/pzstd/Options.h index 97c3885ec..d58de017e 100644 --- a/contrib/pzstd/Options.h +++ b/contrib/pzstd/Options.h @@ -54,7 +54,7 @@ struct Options { ZSTD_parameters determineParameters() const { ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, 0); - params.fParams.contentSizeFlag = 1; + params.fParams.contentSizeFlag = 0; params.fParams.checksumFlag = checksum; if (maxWindowLog != 0 && params.cParams.windowLog > maxWindowLog) { params.cParams.windowLog = maxWindowLog; diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index b6cf37646..508ce1fa5 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2362,7 +2362,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ U32 const checksumFlag = params.fParams.checksumFlag>0; U32 const windowSize = 1U << params.cParams.windowLog; - U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize > (pledgedSrcSize-1)); + U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize > pledgedSrcSize); BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); U32 const fcsCode = params.fParams.contentSizeFlag ? (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : /* 0-3 */ diff --git a/lib/zstd.h b/lib/zstd.h index 90f0bca59..5de11b012 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -558,7 +558,7 @@ ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct */ ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */ ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, - ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */ + ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */ ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */ ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); @@ -616,7 +616,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); /*===== Buffer-less streaming compression functions =====*/ ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); +ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */ ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize); ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); diff --git a/programs/fileio.c b/programs/fileio.c index 087bf9504..0054de42e 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -271,8 +271,8 @@ typedef struct { } cRess_t; static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, - U64 srcSize, ZSTD_compressionParameters* comprParams) -{ + U64 srcSize, int srcRegFile, + ZSTD_compressionParameters* comprParams) { cRess_t ress; memset(&ress, 0, sizeof(ress)); @@ -298,7 +298,7 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, size_t const dictBuffSize = FIO_loadFile(&dictBuffer, dictFileName); if (dictFileName && (dictBuffer==NULL)) EXM_THROW(32, "zstd: allocation error : can't create dictBuffer"); { ZSTD_parameters params = ZSTD_getParams(cLevel, srcSize, dictBuffSize); - params.fParams.contentSizeFlag = 1; + params.fParams.contentSizeFlag = srcRegFile; params.fParams.checksumFlag = g_checksumFlag; params.fParams.noDictIDFlag = !g_dictIDFlag; if (comprParams->windowLog) params.cParams.windowLog = comprParams->windowLog; @@ -473,8 +473,9 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName, { clock_t const start = clock(); U64 const srcSize = UTIL_getFileSize(srcFileName); + int const regFile = UTIL_isRegFile(srcFileName); - cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams); + cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, regFile, comprParams); int const result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName); double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC; @@ -495,7 +496,8 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile char* dstFileName = (char*)malloc(FNSPACE); size_t const suffixSize = suffix ? strlen(suffix) : 0; U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ; - cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams); + int const regFile = (nbFiles != 1) ? 0 : UTIL_isRegFile(inFileNamesTable[0]); + cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, regFile, comprParams); /* init */ if (dstFileName==NULL) EXM_THROW(27, "FIO_compressMultipleFilenames : allocation error for dstFileName"); diff --git a/programs/util.h b/programs/util.h index 651027bae..54dab4e49 100644 --- a/programs/util.h +++ b/programs/util.h @@ -182,6 +182,11 @@ UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) return 1; } +UTIL_STATIC int UTIL_isRegFile(const char* infilename) +{ + stat_t statbuf; + return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */ +} UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) { From f07ddf88e8f98457b4b7fe94bf615f302d4100c8 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Wed, 8 Feb 2017 13:56:32 -0800 Subject: [PATCH 032/223] Test multiframe legacy decoding with simple and streaming APIs --- tests/.gitignore | 1 + tests/Makefile | 12 ++- tests/legacy.c | 226 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 237 insertions(+), 2 deletions(-) create mode 100644 tests/legacy.c diff --git a/tests/.gitignore b/tests/.gitignore index 53520238f..b7ba51b67 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -16,6 +16,7 @@ paramgrill32 roundTripCrash longmatch symbols +legacy pool invalidDictionaries diff --git a/tests/Makefile b/tests/Makefile index f49d23018..83e10ad7c 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -27,7 +27,7 @@ TESTARTEFACT := versionsTest namespaceTest CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) -CFLAGS ?= -O3 +CFLAGS ?= -g CFLAGS += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \ -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef CFLAGS += $(MOREFLAGS) @@ -149,6 +149,11 @@ longmatch : $(ZSTD_FILES) longmatch.c invalidDictionaries : $(ZSTD_FILES) invalidDictionaries.c $(CC) $(FLAGS) $^ -o $@$(EXT) +legacy : CFLAGS+= -DZSTD_LEGACY_SUPPORT=1 +legacy : CPPFLAGS+= -I$(ZSTDDIR)/legacy +legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + symbols : symbols.c $(MAKE) -C $(ZSTDDIR) libzstd ifneq (,$(filter Windows%,$(OS))) @@ -225,7 +230,7 @@ zstd-playTests: datagen file $(ZSTD) ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST) -test: test-zstd test-fullbench test-fuzzer test-zstream test-longmatch test-invalidDictionaries test-pool +test: test-zstd test-fullbench test-fuzzer test-zstream test-longmatch test-invalidDictionaries test-pool test-legacy test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32 @@ -291,6 +296,9 @@ test-invalidDictionaries: invalidDictionaries test-symbols: symbols $(QEMU_SYS) ./symbols +test-legacy: legacy + $(QEMU_SYS) ./legacy + test-pool: pool $(QEMU_SYS) ./pool diff --git a/tests/legacy.c b/tests/legacy.c new file mode 100644 index 000000000..5d93c68fa --- /dev/null +++ b/tests/legacy.c @@ -0,0 +1,226 @@ +/** + * Copyright (c) 2017-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +/* + This program uses hard-coded data compressed with Zstd legacy versions + and tests that the API decompresses them correctly +*/ + +/*=========================================== +* Dependencies +*==========================================*/ +#include /* size_t */ +#include /* malloc, free */ +#include /* fprintf */ +#include /* strlen */ +#include "zstd.h" +#include "zstd_errors.h" + +/*=========================================== +* Macros +*==========================================*/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) + +/*=========================================== +* Precompressed frames +*==========================================*/ +const char* const COMPRESSED; /* content is at end of file */ +size_t const COMPRESSED_SIZE = 917; +const char* const EXPECTED; /* content is at end of file */ + + +int testSimpleAPI(void) +{ + size_t const size = strlen(EXPECTED); + char* const output = malloc(size); + + if (!output) { + DISPLAY("ERROR: Not enough memory!\n"); + return 1; + } + + { + size_t const ret = ZSTD_decompress(output, size, COMPRESSED, COMPRESSED_SIZE); + if (ZSTD_isError(ret)) { + if (ret == ZSTD_error_prefix_unknown) { + DISPLAY("ERROR: Invalid frame magic number, was this compiled " + "without legacy support?\n"); + } else { + DISPLAY("ERROR: %s\n", ZSTD_getErrorName(ret)); + } + return 1; + } + if (ret != size) { + DISPLAY("ERROR: Wrong decoded size\n"); + } + } + if (memcmp(EXPECTED, output, size) != 0) { + DISPLAY("ERROR: Wrong decoded output produced\n"); + return 1; + } + + DISPLAY("Simple API OK\n"); + return 0; +} + +int testStreamingAPI(void) +{ + size_t const outBuffSize = ZSTD_DStreamOutSize(); + char* const outBuff = malloc(outBuffSize); + ZSTD_DStream* const stream = ZSTD_createDStream(); + ZSTD_inBuffer input = { COMPRESSED, COMPRESSED_SIZE, 0 }; + size_t outputPos = 0; + int needsInit = 1; + + if (outBuff == NULL) { + DISPLAY("ERROR: Could not allocate memory\n"); + return 1; + } + if (stream == NULL) { + DISPLAY("ERROR: Could not create dstream\n"); + return 1; + } + + while (1) { + ZSTD_outBuffer output = {outBuff, outBuffSize, 0}; + if (needsInit) { + size_t const ret = ZSTD_initDStream(stream); + if (ZSTD_isError(ret)) { + DISPLAY("ERROR: %s\n", ZSTD_getErrorName(ret)); + return 1; + } + } + { + size_t const ret = ZSTD_decompressStream(stream, &output, &input); + if (ZSTD_isError(ret)) { + DISPLAY("ERROR: %s\n", ZSTD_getErrorName(ret)); + return 1; + } + + if (ret == 0) { + needsInit = 1; + } + } + + if (memcmp(outBuff, EXPECTED + outputPos, output.pos) != 0) { + DISPLAY("ERROR: Wrong decoded output produced\n"); + return 1; + } + outputPos += output.pos; + if (input.pos == input.size && output.pos < output.size) { + break; + } + } + + DISPLAY("Streaming API OK\n"); + return 0; +} + +int main(void) +{ + int ret; + + ret = testSimpleAPI(); + if (ret) return ret; + ret = testStreamingAPI(); + if (ret) return ret; + + DISPLAY("OK\n"); + + return 0; +} + +/* Consists of the "EXPECTED" string compressed with default settings on + - v0.4.3 + - v0.5.0 + - v0.6.0 + - v0.7.0 + - v0.8.0 +*/ +const char* const COMPRESSED = + "\x24\xB5\x2F\xFD\x00\x00\x00\xBB\xB0\x02\xC0\x10\x00\x1E\xB0\x01" + "\x02\x00\x00\x80\x00\xE8\x92\x34\x12\x97\xC8\xDF\xE9\xF3\xEF\x53" + "\xEA\x1D\x27\x4F\x0C\x44\x90\x0C\x8D\xF1\xB4\x89\x17\x00\x18\x00" + "\x18\x00\x3F\xE6\xE2\xE3\x74\xD6\xEC\xC9\x4A\xE0\x71\x71\x42\x3E" + "\x64\x4F\x6A\x45\x4E\x78\xEC\x49\x03\x3F\xC6\x80\xAB\x8F\x75\x5E" + "\x6F\x2E\x3E\x7E\xC6\xDC\x45\x69\x6C\xC5\xFD\xC7\x40\xB8\x84\x8A" + "\x01\xEB\xA8\xD1\x40\x39\x90\x4C\x64\xF8\xEB\x53\xE6\x18\x0B\x67" + "\x12\xAD\xB8\x99\xB3\x5A\x6F\x8A\x19\x03\x01\x50\x67\x56\xF5\x9F" + "\x35\x84\x60\xA0\x60\x91\xC9\x0A\xDC\xAB\xAB\xE0\xE2\x81\xFA\xCF" + "\xC6\xBA\x01\x0E\x00\x54\x00\x00\x19\x00\x00\x54\x14\x00\x24\x24" + "\x04\xFE\x04\x84\x4E\x41\x00\x27\xE2\x02\xC4\xB1\x00\xD2\x51\x00" + "\x79\x58\x41\x28\x00\xE0\x0C\x01\x68\x65\x00\x04\x13\x0C\xDA\x0C" + "\x80\x22\x06\xC0\x00\x00\x25\xB5\x2F\xFD\x00\x00\x00\xAD\x12\xB0" + "\x7D\x1E\xB0\x01\x02\x00\x00\x80\x00\xE8\x92\x34\x12\x97\xC8\xDF" + "\xE9\xF3\xEF\x53\xEA\x1D\x27\x4F\x0C\x44\x90\x0C\x8D\xF1\xB4\x89" + "\x03\x01\x50\x67\x56\xF5\x9F\x35\x84\x60\xA0\x60\x91\xC9\x0A\xDC" + "\xAB\xAB\xE0\xE2\x81\xFA\xCF\xC6\xBA\xEB\xA8\xD1\x40\x39\x90\x4C" + "\x64\xF8\xEB\x53\xE6\x18\x0B\x67\x12\xAD\xB8\x99\xB3\x5A\x6F\x8A" + "\xF9\x63\x0C\xB8\xFA\x58\xE7\xF5\xE6\xE2\xE3\x67\xCC\x5D\x94\xC6" + "\x56\xDC\x7F\x0C\x84\x4B\xA8\xF8\x63\x2E\x3E\x4E\x67\xCD\x9E\xAC" + "\x04\x1E\x17\x27\xE4\x43\xF6\xA4\x56\xE4\x84\xC7\x9E\x34\x0E\x00" + "\x00\x32\x40\x80\xA8\x00\x01\x49\x81\xE0\x3C\x01\x29\x1D\x00\x87" + "\xCE\x80\x75\x08\x80\x72\x24\x00\x7B\x52\x00\x94\x00\x20\xCC\x01" + "\x86\xD2\x00\x81\x09\x83\xC1\x34\xA0\x88\x01\xC0\x00\x00\x26\xB5" + "\x2F\xFD\x42\xEF\x00\x00\xA6\x12\xB0\x7D\x1E\xB0\x01\x02\x00\x00" + "\x54\xA0\xBA\x24\x8D\xC4\x25\xF2\x77\xFA\xFC\xFB\x94\x7A\xC7\xC9" + "\x13\x03\x11\x24\x43\x63\x3C\x6D\x22\x03\x01\x50\x67\x56\xF5\x9F" + "\x35\x84\x60\xA0\x60\x91\xC9\x0A\xDC\xAB\xAB\xE0\xE2\x81\xFA\xCF" + "\xC6\xBA\xEB\xA8\xD1\x40\x39\x90\x4C\x64\xF8\xEB\x53\xE6\x18\x0B" + "\x67\x12\xAD\xB8\x99\xB3\x5A\x6F\x8A\xF9\x63\x0C\xB8\xFA\x58\xE7" + "\xF5\xE6\xE2\xE3\x67\xCC\x5D\x94\xC6\x56\xDC\x7F\x0C\x84\x4B\xA8" + "\xF8\x63\x2E\x3E\x4E\x67\xCD\x9E\xAC\x04\x1E\x17\x27\xE4\x43\xF6" + "\xA4\x56\xE4\x84\xC7\x9E\x34\x0E\x00\x35\x0B\x71\xB5\xC0\x2A\x5C" + "\x26\x94\x22\x20\x8B\x4C\x8D\x13\x47\x58\x67\x15\x6C\xF1\x1C\x4B" + "\x54\x10\x9D\x31\x50\x85\x4B\x54\x0E\x01\x4B\x3D\x01\xC0\x00\x00" + "\x27\xB5\x2F\xFD\x20\xEF\x00\x00\xA6\x12\xE4\x84\x1F\xB0\x01\x10" + "\x00\x00\x00\x35\x59\xA6\xE7\xA1\xEF\x7C\xFC\xBD\x3F\xFF\x9F\xEF" + "\xEE\xEF\x61\xC3\xAA\x31\x1D\x34\x38\x22\x22\x04\x44\x21\x80\x32" + "\xAD\x28\xF3\xD6\x28\x0C\x0A\x0E\xD6\x5C\xAC\x19\x8D\x20\x5F\x45" + "\x02\x2E\x17\x50\x66\x6D\xAC\x8B\x9C\x6E\x07\x73\x46\xBB\x44\x14" + "\xE7\x98\xC3\xB9\x17\x32\x6E\x33\x7C\x0E\x21\xB1\xDB\xCB\x89\x51" + "\x23\x34\xAB\x9D\xBC\x6D\x20\xF5\x03\xA9\x91\x4C\x2E\x1F\x59\xDB" + "\xD9\x35\x67\x4B\x0C\x95\x79\x10\x00\x85\xA6\x96\x95\x2E\xDF\x78" + "\x7B\x4A\x5C\x09\x76\x97\xD1\x5C\x96\x12\x75\x35\xA3\x55\x4A\xD4" + "\x0B\x00\x35\x0B\x71\xB5\xC0\x2A\x5C\xE6\x08\x45\xF1\x39\x43\xF1" + "\x1C\x4B\x54\x10\x9D\x31\x50\x85\x4B\x54\x0E\x01\x4B\x3D\x01\xC0" + "\x00\x00\x28\xB5\x2F\xFD\x24\xEF\x35\x05\x00\x92\x0B\x21\x1F\xB0" + "\x01\x10\x00\x00\x00\x35\x59\xA6\xE7\xA1\xEF\x7C\xFC\xBD\x3F\xFF" + "\x9F\xEF\xEE\xEF\x61\xC3\xAA\x31\x1D\x34\x38\x22\x22\x04\x44\x21" + "\x80\x32\xAD\x28\xF3\xD6\x28\x0C\x0A\x0E\xD6\x5C\xAC\x19\x8D\x20" + "\x5F\x45\x02\x2E\x17\x50\x66\x6D\xAC\x8B\x9C\x6E\x07\x73\x46\xBB" + "\x44\x14\xE7\x98\xC3\xB9\x17\x32\x6E\x33\x7C\x0E\x21\xB1\xDB\xCB" + "\x89\x51\x23\x34\xAB\x9D\xBC\x6D\x20\xF5\x03\xA9\x91\x4C\x2E\x1F" + "\x59\xDB\xD9\x35\x67\x4B\x0C\x95\x79\x10\x00\x85\xA6\x96\x95\x2E" + "\xDF\x78\x7B\x4A\x5C\x09\x76\x97\xD1\x5C\x96\x12\x75\x35\xA3\x55" + "\x4A\xD4\x0B\x00\x35\x0B\x71\xB5\xC0\x2A\x5C\xE6\x08\x45\xF1\x39" + "\x43\xF1\x1C\x4B\x54\x10\x9D\x31\x50\x85\x4B\x54\x0E\x01\x4B\x3D" + "\x01\xD2\x2F\x21\x80"; + +const char* const EXPECTED = + "snowden is snowed in / he's now then in his snow den / when does the snow end?\n" + "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n" + "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n" + + "snowden is snowed in / he's now then in his snow den / when does the snow end?\n" + "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n" + "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n" + + "snowden is snowed in / he's now then in his snow den / when does the snow end?\n" + "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n" + "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n" + + "snowden is snowed in / he's now then in his snow den / when does the snow end?\n" + "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n" + "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n" + + "snowden is snowed in / he's now then in his snow den / when does the snow end?\n" + "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n" + "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n"; + From a0f9006e5a8c844a9d36d85307d3ad5d5b24ed78 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Wed, 8 Feb 2017 17:25:01 -0800 Subject: [PATCH 033/223] #undef _POSIX_C_SOURCE if already defined --- programs/platform.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/programs/platform.h b/programs/platform.h index f30528aa9..b54b94d75 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -1,6 +1,6 @@ /** * platform.h - compiler and OS detection - * + * * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. * All rights reserved. * @@ -23,7 +23,7 @@ extern "C" { ****************************************/ #if defined(_MSC_VER) # define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */ -# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before and */ +# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before and */ # if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */ # define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ # endif @@ -52,7 +52,7 @@ extern "C" { * Turn on Large Files support (>4GB) for 32-bit Linux/Unix ***********************************************************/ #if !defined(__64BIT__) /* No point defining Large file for 64 bit */ -# if !defined(_FILE_OFFSET_BITS) +# if !defined(_FILE_OFFSET_BITS) # define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ # endif # if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */ @@ -77,6 +77,9 @@ extern "C" { # define PLATFORM_POSIX_VERSION 200112L # else # if defined(__linux__) || defined(__linux) +# ifdef _POSIX_C_SOURCE +# undef _POSIX_C_SOURCE +# endif # define _POSIX_C_SOURCE 200112L /* use feature test macro */ # endif # include /* declares _POSIX_VERSION */ From e0b3265e87e30b57d5bf8c6c5acbab20c44d9f67 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Wed, 8 Feb 2017 15:31:47 -0800 Subject: [PATCH 034/223] Fix ZSTD_getErrorString and add tests --- lib/common/zstd_common.c | 2 +- tests/fuzzer.c | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c index 749b2870c..8408a589a 100644 --- a/lib/common/zstd_common.c +++ b/lib/common/zstd_common.c @@ -41,7 +41,7 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } /*! ZSTD_getErrorString() : * provides error code string from enum */ -const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); } +const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); } /*=************************************************************** diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 60546c07a..84d2c0237 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -505,6 +505,16 @@ static int basicUnitTests(U32 seed, double compressibility) if (r != _3BYTESTESTLENGTH) goto _output_error; } DISPLAYLEVEL(4, "OK \n"); + /* error string tests */ + DISPLAYLEVEL(4, "test%3i : testing ZSTD error code strings : ", testNb++); + if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error; + if (strcmp("No error detected", ZSTD_getErrorString(ZSTD_error_no_error)) != 0) goto _output_error; + if (strcmp("Unspecified error code", ZSTD_getErrorString((ZSTD_ErrorCode)(0-ZSTD_error_GENERIC))) != 0) goto _output_error; + if (strcmp("Error (generic)", ZSTD_getErrorName((size_t)0-ZSTD_error_GENERIC)) != 0) goto _output_error; + if (strcmp("Error (generic)", ZSTD_getErrorString(ZSTD_error_GENERIC)) != 0) goto _output_error; + if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error; + DISPLAYLEVEL(4, "OK \n"); + _end: free(CNBuffer); free(compressedBuffer); From 545987996a198c090fd7d1e591157505a5b02b41 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Wed, 8 Feb 2017 17:38:17 -0800 Subject: [PATCH 035/223] Fix deprecation warnings for clang with C++14 --- lib/deprecated/zbuff.h | 4 +++- lib/dictBuilder/zdict.h | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/deprecated/zbuff.h b/lib/deprecated/zbuff.h index 85f973557..f62091976 100644 --- a/lib/deprecated/zbuff.h +++ b/lib/deprecated/zbuff.h @@ -42,7 +42,9 @@ extern "C" { #ifdef ZBUFF_DISABLE_DEPRECATE_WARNINGS # define ZBUFF_DEPRECATED(message) ZSTDLIB_API /* disable deprecation warnings */ #else -# if (defined(__GNUC__) && (__GNUC__ >= 5)) || defined(__clang__) +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZBUFF_DEPRECATED(message) [[deprecated(message)]] ZSTDLIB_API +# elif (defined(__GNUC__) && (__GNUC__ >= 5)) || defined(__clang__) # define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated(message))) # elif defined(__GNUC__) && (__GNUC__ >= 3) # define ZBUFF_DEPRECATED(message) ZSTDLIB_API __attribute__((deprecated)) diff --git a/lib/dictBuilder/zdict.h b/lib/dictBuilder/zdict.h index 4d0a62a2c..4ead4474f 100644 --- a/lib/dictBuilder/zdict.h +++ b/lib/dictBuilder/zdict.h @@ -174,7 +174,7 @@ ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBuffer #else # define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) # if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ -# define ZDICT_DEPRECATED(message) ZDICTLIB_API [[deprecated(message)]] +# define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API # elif (ZDICT_GCC_VERSION >= 405) || defined(__clang__) # define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message))) # elif (ZDICT_GCC_VERSION >= 301) From 13127fd05b7ce947b57a0175c54a1b1c7353916f Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 9 Feb 2017 11:32:14 +0100 Subject: [PATCH 036/223] don't use "echo -e" --- programs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/Makefile b/programs/Makefile index 8257e394e..a7de2e1af 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -67,7 +67,7 @@ endif # zlib detection VOID = /dev/null -HAVE_ZLIB := $(shell echo -e "\#include \nint main(){}" | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && echo 1 || echo 0) +HAVE_ZLIB := $(shell echo $$'\#include \nint main(){}' | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && echo 1 || echo 0) ifeq ($(HAVE_ZLIB), 1) TEMP := $(shell rm have_zlib$(EXT)) ZLIBCPP = -DZSTD_GZDECOMPRESS From 896638a8a296674cac145aff51b6dc7f724c4b94 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 9 Feb 2017 17:01:17 +0100 Subject: [PATCH 037/223] echo replaced with printf --- programs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/Makefile b/programs/Makefile index a7de2e1af..862ba35dd 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -67,7 +67,7 @@ endif # zlib detection VOID = /dev/null -HAVE_ZLIB := $(shell echo $$'\#include \nint main(){}' | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && echo 1 || echo 0) +HAVE_ZLIB := $(shell printf '\#include \nint main(){}' | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && echo 1 || echo 0) ifeq ($(HAVE_ZLIB), 1) TEMP := $(shell rm have_zlib$(EXT)) ZLIBCPP = -DZSTD_GZDECOMPRESS From 2db72492657aa824fe934ffe9eb493099612effa Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Thu, 9 Feb 2017 10:50:43 -0800 Subject: [PATCH 038/223] Make pledgedSrcSize meaning clear for other functions - Added tests - Moved new size functions to static link only --- lib/compress/zstd_compress.c | 26 +++++++++-- lib/zstd.h | 88 +++++++++++++++++++----------------- tests/Makefile | 2 +- tests/zstreamtest.c | 28 ++++++++++++ 4 files changed, 97 insertions(+), 47 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 508ce1fa5..765c8e34d 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -344,8 +344,12 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long { if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong); + memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); - ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, pledgedSrcSize, ZSTDcrp_noMemset); + { ZSTD_parameters params = srcCCtx->params; + params.fParams.contentSizeFlag = (pledgedSrcSize > 0); + ZSTD_resetCCtx_advanced(dstCCtx, params, pledgedSrcSize, ZSTDcrp_noMemset); + } /* copy tables */ { size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog); @@ -2362,7 +2366,7 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ U32 const checksumFlag = params.fParams.checksumFlag>0; U32 const windowSize = 1U << params.cParams.windowLog; - U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize > pledgedSrcSize); + U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); U32 const fcsCode = params.fParams.contentSizeFlag ? (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : /* 0-3 */ @@ -2845,7 +2849,11 @@ static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) { size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize) { if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize)) - else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, pledgedSrcSize)); + else { + ZSTD_parameters params = cdict->refContext->params; + params.fParams.contentSizeFlag = (pledgedSrcSize > 0); + CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, params, pledgedSrcSize)); + } return 0; } @@ -2939,7 +2947,7 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs) size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; } -size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) +static size_t ZSTD_resetCStream_internal(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) { if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once => can't reset */ @@ -2957,6 +2965,14 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) return 0; /* ready to go */ } +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) +{ + + zcs->params.fParams.contentSizeFlag = (pledgedSrcSize > 0); + + return ZSTD_resetCStream_internal(zcs, pledgedSrcSize); +} + size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize) @@ -2988,7 +3004,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, zcs->checksum = params.fParams.checksumFlag > 0; zcs->params = params; - return ZSTD_resetCStream(zcs, pledgedSrcSize); + return ZSTD_resetCStream_internal(zcs, pledgedSrcSize); } /* note : cdict must outlive compression session */ diff --git a/lib/zstd.h b/lib/zstd.h index 5de11b012..900a4c0c5 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -88,44 +88,12 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, const void* src, size_t compressedSize); -#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) -#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) - -/*! ZSTD_getFrameContentSize() : -* `src` should point to the start of a ZSTD encoded frame -* @return : decompressed size of the frame pointed to be `src` if known, otherwise -* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined -* - ZSTD_CONTENTSIZE_ERROR if an error occured (e.g. invalid magic number, srcSize too small) */ -ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); - -/*! ZSTD_findDecompressedSize() : -* `src` should point the start of a series of ZSTD encoded and/or skippable frames -* `srcSize` must be the _exact_ size of this series -* (i.e. there should be a frame boundary exactly `srcSize` bytes after `src`) -* @return : the decompressed size of all data in the contained frames, as a 64-bit value _if known_ -* - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN -* - if an error occurred: ZSTD_CONTENTSIZE_ERROR -* -* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. -* When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. -* In which case, it's necessary to use streaming mode to decompress data. -* Optionally, application can still use ZSTD_decompress() while relying on implied limits. -* (For example, data may be necessarily cut into blocks <= 16 KB). -* note 2 : decompressed size is always present when compression is done with ZSTD_compress() -* note 3 : decompressed size can be very large (64-bits value), -* potentially larger than what local system can handle as a single memory segment. -* In which case, it's necessary to use streaming mode to decompress data. -* note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. -* Always ensure result fits within application's authorized limits. -* Each application can set its own limits. -* note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to -* read each contained frame header. This is efficient as most of the data is skipped, -* however it does mean that all frame data must be present and valid. */ -ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); - /*! ZSTD_getDecompressedSize() : -* WARNING: This function is now obsolete. ZSTD_findDecompressedSize should be used, -* or if only exactly one ZSTD frame is needed, ZSTD_getFrameContentSize can be used. +* NOTE: This function is planned to be obsolete, in favour of ZSTD_findDecompressedSize, +* or for single frames, ZSTD_getFrameContentSize. Both of the new functions return more +* descriptive return values, disambiguating empty frames from size unknown or errors. +* Additionally, ZSTD_findDecompressedSize handles multi-frame inputs, returning the total +* size of all frames put together. * * 'src' is the start of a zstd compressed frame. * @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise. @@ -369,6 +337,9 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output #define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */ #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U +#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) +#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) + #define ZSTD_WINDOWLOG_MAX_32 25 #define ZSTD_WINDOWLOG_MAX_64 27 #define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) @@ -422,6 +393,41 @@ typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); typedef void (*ZSTD_freeFunction) (void* opaque, void* address); typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; +/*************************************** +* Decompressed size functions +***************************************/ +/*! ZSTD_getFrameContentSize() : +* `src` should point to the start of a ZSTD encoded frame +* @return : decompressed size of the frame pointed to be `src` if known, otherwise +* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined +* - ZSTD_CONTENTSIZE_ERROR if an error occured (e.g. invalid magic number, srcSize too small) */ +ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); + +/*! ZSTD_findDecompressedSize() : +* `src` should point the start of a series of ZSTD encoded and/or skippable frames +* `srcSize` must be the _exact_ size of this series +* (i.e. there should be a frame boundary exactly `srcSize` bytes after `src`) +* @return : the decompressed size of all data in the contained frames, as a 64-bit value _if known_ +* - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN +* - if an error occurred: ZSTD_CONTENTSIZE_ERROR +* +* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. +* When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. +* In which case, it's necessary to use streaming mode to decompress data. +* Optionally, application can still use ZSTD_decompress() while relying on implied limits. +* (For example, data may be necessarily cut into blocks <= 16 KB). +* note 2 : decompressed size is always present when compression is done with ZSTD_compress() +* note 3 : decompressed size can be very large (64-bits value), +* potentially larger than what local system can handle as a single memory segment. +* In which case, it's necessary to use streaming mode to decompress data. +* note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. +* Always ensure result fits within application's authorized limits. +* Each application can set its own limits. +* note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to +* read each contained frame header. This is efficient as most of the data is skipped, +* however it does mean that all frame data must be present and valid. */ +ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); + /*************************************** * Advanced compression functions @@ -555,12 +561,12 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); /*===== Advanced Streaming compression functions =====*/ ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); -ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct */ +ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */ ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */ ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */ -ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */ +ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before. note: pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */ ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); @@ -617,8 +623,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */ -ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); -ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize); +ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */ +ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */ ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); diff --git a/tests/Makefile b/tests/Makefile index 83e10ad7c..5c966534c 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -27,7 +27,7 @@ TESTARTEFACT := versionsTest namespaceTest CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) -CFLAGS ?= -g +CFLAGS ?= -O3 CFLAGS += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \ -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef CFLAGS += $(MOREFLAGS) diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 747e9a445..5680d27c1 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -438,6 +438,34 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */ DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); } + /* Unknown srcSize */ + DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly : ", testNb++); + { ZSTD_parameters params = ZSTD_getParams(5, 0, 0); + params.fParams.contentSizeFlag = 1; + ZSTD_initCStream_advanced(zc, NULL, 0, params, 0); } /* cstream advanced should write the 0 size field */ + inBuff.src = CNBuffer; + inBuff.size = 0; + inBuff.pos = 0; + outBuff.dst = compressedBuffer; + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + if (ZSTD_isError(ZSTD_compressStream(zc, &outBuff, &inBuff))) goto _output_error; + if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; + cSize = outBuff.pos; + if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error; + + ZSTD_resetCStream(zc, 0); /* resetCStream should treat 0 as unknown */ + inBuff.src = CNBuffer; + inBuff.size = 0; + inBuff.pos = 0; + outBuff.dst = compressedBuffer; + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + if (ZSTD_isError(ZSTD_compressStream(zc, &outBuff, &inBuff))) goto _output_error; + if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; + cSize = outBuff.pos; + if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error; + DISPLAYLEVEL(3, "OK \n"); _end: FUZ_freeDictionary(dictionary); From 84b37cc1f1f0d13a3e137a6b871921ff9f0f4d80 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Thu, 9 Feb 2017 12:27:32 -0800 Subject: [PATCH 039/223] Fix failing unit test --- tests/fuzzer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fuzzer.c b/tests/fuzzer.c index 4239af455..603c74c2a 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -220,7 +220,7 @@ static int basicUnitTests(U32 seed, double compressibility) DISPLAYLEVEL(4, "test%3i : load dictionary into context : ", testNb++); CHECK( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) ); - CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, CNBuffSize - dictSize) ); + CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */ DISPLAYLEVEL(4, "OK \n"); DISPLAYLEVEL(4, "test%3i : compress with flat dictionary : ", testNb++); From 269b2cd3d8461da4e615e2cb89cbe61ca8b3d810 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Thu, 9 Feb 2017 13:25:30 -0800 Subject: [PATCH 040/223] Documentation updates --- lib/zstd.h | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/zstd.h b/lib/zstd.h index 900a4c0c5..bbfefb8db 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -89,11 +89,14 @@ ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, const void* src, size_t compressedSize); /*! ZSTD_getDecompressedSize() : -* NOTE: This function is planned to be obsolete, in favour of ZSTD_findDecompressedSize, -* or for single frames, ZSTD_getFrameContentSize. Both of the new functions return more -* descriptive return values, disambiguating empty frames from size unknown or errors. -* Additionally, ZSTD_findDecompressedSize handles multi-frame inputs, returning the total -* size of all frames put together. +* NOTE: This function is planned to be obsolete, in favour of ZSTD_getFrameContentSize. +* ZSTD_getFrameContentSize functions the same way, returning the decompressed size of a single +* frame, but distinguishes empty frames from frames with an unknown size, or errors. +* +* Additionally, ZSTD_findDecompressedSize can be used instead. It can handle multiple +* concatenated frames in one buffer, and so is more general. +* As a result however, it requires more computation and entire frames to be passed to it, +* as opposed to ZSTD_getFrameContentSize which requires only a single frame's header. * * 'src' is the start of a zstd compressed frame. * @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise. @@ -398,6 +401,8 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v ***************************************/ /*! ZSTD_getFrameContentSize() : * `src` should point to the start of a ZSTD encoded frame +* `srcSize` must be at least as large as the frame header. A value greater than or equal +* to `ZSTD_frameHeaderSize_max` is guaranteed to be large enough in all cases. * @return : decompressed size of the frame pointed to be `src` if known, otherwise * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined * - ZSTD_CONTENTSIZE_ERROR if an error occured (e.g. invalid magic number, srcSize too small) */ From 9cde3f8b2e7a9382b69f4a8362812405273e7cec Mon Sep 17 00:00:00 2001 From: ds77 Date: Thu, 9 Feb 2017 18:12:00 +0100 Subject: [PATCH 041/223] use _stat64() on MinGW On MinGW, use _stat64() and struct _stat64 instead of stat() and struct stat_t. This fixes reporting incorrect sizes for large files. --- programs/util.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/programs/util.h b/programs/util.h index 651027bae..f5bd42306 100644 --- a/programs/util.h +++ b/programs/util.h @@ -141,7 +141,7 @@ UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond) /*-**************************************** * File functions ******************************************/ -#if defined(_MSC_VER) +#if defined(_MSC_VER) || defined(__MINGW32__) #define chmod _chmod typedef struct _stat64 stat_t; #else @@ -172,7 +172,7 @@ UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf) UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) { int r; -#if defined(_MSC_VER) +#if defined(_MSC_VER) || defined(__MINGW32__) r = _stat64(infilename, statbuf); if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */ #else @@ -186,7 +186,7 @@ UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) { int r; -#if defined(_MSC_VER) +#if defined(_MSC_VER) || defined(__MINGW32__) struct _stat64 statbuf; r = _stat64(infilename, &statbuf); if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ @@ -212,7 +212,7 @@ UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFi UTIL_STATIC int UTIL_doesFileExists(const char* infilename) { int r; -#if defined(_MSC_VER) +#if defined(_MSC_VER) || defined(__MINGW32__) struct _stat64 statbuf; r = _stat64(infilename, &statbuf); if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ @@ -228,7 +228,7 @@ UTIL_STATIC int UTIL_doesFileExists(const char* infilename) UTIL_STATIC U32 UTIL_isDirectory(const char* infilename) { int r; -#if defined(_MSC_VER) +#if defined(_MSC_VER) || defined(__MINGW32__) struct _stat64 statbuf; r = _stat64(infilename, &statbuf); if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; From d08019813be545a66de0afa8718261df44efd7dc Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Thu, 9 Feb 2017 14:20:52 -0800 Subject: [PATCH 042/223] Improvement from @inikep --- programs/platform.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/programs/platform.h b/programs/platform.h index b54b94d75..def8945bd 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -77,10 +77,9 @@ extern "C" { # define PLATFORM_POSIX_VERSION 200112L # else # if defined(__linux__) || defined(__linux) -# ifdef _POSIX_C_SOURCE -# undef _POSIX_C_SOURCE +# ifndef _POSIX_C_SOURCE +# define _POSIX_C_SOURCE 200112L /* use feature test macro */ # endif -# define _POSIX_C_SOURCE 200112L /* use feature test macro */ # endif # include /* declares _POSIX_VERSION */ # if defined(_POSIX_VERSION) /* POSIX compliant */ From 429e13099a81db842cd27c523f0e086a49827958 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 10 Feb 2017 10:36:44 +0100 Subject: [PATCH 043/223] fix 64-bit file support for MinGW --- programs/platform.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/platform.h b/programs/platform.h index f30528aa9..5801f6ec3 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -51,7 +51,7 @@ extern "C" { /* ********************************************************* * Turn on Large Files support (>4GB) for 32-bit Linux/Unix ***********************************************************/ -#if !defined(__64BIT__) /* No point defining Large file for 64 bit */ +#if !defined(__64BIT__) || defined(__MINGW32__) /* No point defining Large file for 64 bit but MinGW requires it */ # if !defined(_FILE_OFFSET_BITS) # define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ # endif From 19f61b534e67e911ce888048aff3516dbdc3f7bd Mon Sep 17 00:00:00 2001 From: - <-> Date: Fri, 10 Feb 2017 10:56:45 +0100 Subject: [PATCH 044/223] use _stat64 only when targetting Win2k or later --- programs/util.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/programs/util.h b/programs/util.h index f5bd42306..b0f12363a 100644 --- a/programs/util.h +++ b/programs/util.h @@ -141,7 +141,7 @@ UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond) /*-**************************************** * File functions ******************************************/ -#if defined(_MSC_VER) || defined(__MINGW32__) +#if defined(_MSC_VER) || defined(__MINGW32__) && _WIN32_WINNT >= _WIN32_WINNT_WIN2K #define chmod _chmod typedef struct _stat64 stat_t; #else @@ -172,7 +172,7 @@ UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf) UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) { int r; -#if defined(_MSC_VER) || defined(__MINGW32__) +#if defined(_MSC_VER) || defined(__MINGW32__) && _WIN32_WINNT >= _WIN32_WINNT_WIN2K r = _stat64(infilename, statbuf); if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */ #else @@ -186,7 +186,7 @@ UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) { int r; -#if defined(_MSC_VER) || defined(__MINGW32__) +#if defined(_MSC_VER) || defined(__MINGW32__) && _WIN32_WINNT >= _WIN32_WINNT_WIN2K struct _stat64 statbuf; r = _stat64(infilename, &statbuf); if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ @@ -212,7 +212,7 @@ UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFi UTIL_STATIC int UTIL_doesFileExists(const char* infilename) { int r; -#if defined(_MSC_VER) || defined(__MINGW32__) +#if defined(_MSC_VER) || defined(__MINGW32__) && _WIN32_WINNT >= _WIN32_WINNT_WIN2K struct _stat64 statbuf; r = _stat64(infilename, &statbuf); if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ @@ -228,7 +228,7 @@ UTIL_STATIC int UTIL_doesFileExists(const char* infilename) UTIL_STATIC U32 UTIL_isDirectory(const char* infilename) { int r; -#if defined(_MSC_VER) || defined(__MINGW32__) +#if defined(_MSC_VER) || defined(__MINGW32__) && _WIN32_WINNT >= _WIN32_WINNT_WIN2K struct _stat64 statbuf; r = _stat64(infilename, &statbuf); if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; From bdadb82d5d0f189992171b400f612ae484cee1ea Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 10 Feb 2017 11:01:52 +0100 Subject: [PATCH 045/223] fixed "mingw32" AppVeyor target --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index bfdbfe6c0..950a2983b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -5,7 +5,7 @@ environment: MAKE_PARAMS: '"make test && make lib && make -C tests test-symbols fullbench-dll fullbench-lib"' PLATFORM: "mingw64" - COMPILER: "gcc" - MAKE_PARAMS: "make test" + MAKE_PARAMS: '"make test"' PLATFORM: "mingw32" - COMPILER: "visual" CONFIGURATION: "Debug" From cb8d2d9d963d054895c7411f573b6b94727685a9 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 10 Feb 2017 12:01:14 +0100 Subject: [PATCH 046/223] appveyor.yml: add clang target --- appveyor.yml | 51 +++++++++++++++++++++++---------------------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 950a2983b..225515c00 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,6 +1,9 @@ version: 1.0.{build} environment: matrix: + - COMPILER: "gcc" + MAKE_PARAMS: '"make -C tests zstd fullbench fuzzer paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"' + PLATFORM: "clang" - COMPILER: "gcc" MAKE_PARAMS: '"make test && make lib && make -C tests test-symbols fullbench-dll fullbench-lib"' PLATFORM: "mingw64" @@ -25,7 +28,6 @@ install: - MKDIR bin - if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH% - if [%COMPILER%]==[gcc] ( - SET "CLANG_PARAMS=-C tests zstd fullbench fuzzer paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"" && SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" && SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && COPY C:\msys64\usr\bin\make.exe C:\MinGW\bin\make.exe && @@ -38,31 +40,7 @@ build_script: - ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION% - if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL% - if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% - - if [%PLATFORM%]==[mingw64] ( - make clean && - ECHO *** && - ECHO *** Building clang && - ECHO *** && - ECHO make %CLANG_PARAMS% && - make %CLANG_PARAMS% && - COPY tests\fuzzer.exe tests\fuzzer_clang.exe && - ECHO *** && - ECHO *** Building cmake for %PLATFORM% && - ECHO *** && - mkdir build\cmake\build && - cd build\cmake\build && - cmake -G "Visual Studio 14 2015 Win64" .. && - cd ..\..\.. && - make clean && - ECHO *** && - ECHO *** Building pzstd for %PLATFORM% && - ECHO *** && - make -C contrib\pzstd googletest-mingw64 && - make -C contrib\pzstd pzstd.exe && - make -C contrib\pzstd tests && - make -C contrib\pzstd check && - make -C contrib\pzstd clean - ) + - if [%PLATFORM%]==[clang] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% - if [%COMPILER%]==[gcc] ( ECHO *** && ECHO *** Building %PLATFORM% && @@ -72,6 +50,7 @@ build_script: ECHO %MAKE_PARAMS% && sh -c %MAKE_PARAMS% ) + - if [%PLATFORM%]==[clang] COPY tests\fuzzer.exe tests\fuzzer_clang.exe - if [%COMPILER%]==[gcc] if [%PLATFORM%]==[mingw64] ( COPY programs\zstd.exe bin\zstd.exe && appveyor PushArtifact bin\zstd.exe @@ -135,8 +114,24 @@ build_script: test_script: - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION% - SET FUZZERTEST=-T1mn - - if [%COMPILER%]==[gcc] ( - if [%PLATFORM%]==[mingw64] tests\fuzzer_clang.exe %FUZZERTEST% + - if [%COMPILER%]==[gcc] if [%PLATFORM%]==[clang] ( + tests\fuzzer_clang.exe %FUZZERTEST% && + ECHO *** && + ECHO *** Building cmake for %PLATFORM% && + ECHO *** && + mkdir build\cmake\build && + cd build\cmake\build && + cmake -G "Visual Studio 14 2015 Win64" .. && + cd ..\..\.. && + make clean && + ECHO *** && + ECHO *** Building pzstd for %PLATFORM% && + ECHO *** && + make -C contrib\pzstd googletest-mingw64 && + make -C contrib\pzstd pzstd.exe && + make -C contrib\pzstd tests && + make -C contrib\pzstd check && + make -C contrib\pzstd clean ) - if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] ( CD tests && From bc2bfa4c9ae2b78dbdeac6232cef4e1213a568b5 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 10 Feb 2017 12:32:30 +0100 Subject: [PATCH 047/223] fix missing " --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 225515c00..fecc5fedc 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,7 +2,7 @@ version: 1.0.{build} environment: matrix: - COMPILER: "gcc" - MAKE_PARAMS: '"make -C tests zstd fullbench fuzzer paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion"' + MAKE_PARAMS: '"make -C tests zstd fullbench fuzzer paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion""' PLATFORM: "clang" - COMPILER: "gcc" MAKE_PARAMS: '"make test && make lib && make -C tests test-symbols fullbench-dll fullbench-lib"' From 70597841925a90f09827760ac8c5310cd0d43b8b Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 10 Feb 2017 12:41:31 +0100 Subject: [PATCH 048/223] appveyor.yml: reordering of tests --- appveyor.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index fecc5fedc..d603e9216 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,14 +2,14 @@ version: 1.0.{build} environment: matrix: - COMPILER: "gcc" - MAKE_PARAMS: '"make -C tests zstd fullbench fuzzer paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion""' PLATFORM: "clang" + MAKE_PARAMS: '"make -C tests zstd fullbench fuzzer paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion""' - COMPILER: "gcc" - MAKE_PARAMS: '"make test && make lib && make -C tests test-symbols fullbench-dll fullbench-lib"' PLATFORM: "mingw64" + MAKE_PARAMS: '"make test && make lib && make -C tests test-symbols fullbench-dll fullbench-lib"' - COMPILER: "gcc" - MAKE_PARAMS: '"make test"' PLATFORM: "mingw32" + MAKE_PARAMS: '"make -C test-zstd test-fullbench test-fuzzer test-invalidDictionaries"' - COMPILER: "visual" CONFIGURATION: "Debug" PLATFORM: "x64" From 192e20338f0da5e8f4224c78191256a37e9aa3f1 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 10 Feb 2017 13:10:25 +0100 Subject: [PATCH 049/223] appveyor.yml: fixed clang test --- appveyor.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index d603e9216..51ff488a4 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,15 +1,15 @@ version: 1.0.{build} environment: matrix: - - COMPILER: "gcc" - PLATFORM: "clang" - MAKE_PARAMS: '"make -C tests zstd fullbench fuzzer paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion""' - COMPILER: "gcc" PLATFORM: "mingw64" MAKE_PARAMS: '"make test && make lib && make -C tests test-symbols fullbench-dll fullbench-lib"' - COMPILER: "gcc" PLATFORM: "mingw32" - MAKE_PARAMS: '"make -C test-zstd test-fullbench test-fuzzer test-invalidDictionaries"' + MAKE_PARAMS: '"make -C tests test-zstd test-fullbench test-fuzzer test-invalidDictionaries"' + - COMPILER: "gcc" + PLATFORM: "clang" + MAKE_PARAMS: '"make -C tests zstd fullbench fuzzer paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion""' - COMPILER: "visual" CONFIGURATION: "Debug" PLATFORM: "x64" From 7ec315df0dd4bef8921bceb089bdbea2b88a9382 Mon Sep 17 00:00:00 2001 From: - <-> Date: Fri, 10 Feb 2017 13:27:43 +0100 Subject: [PATCH 050/223] fix previous commit * struct _stat64 is not defined by (non-w64) MinGW releases, __stat64 should be everywhere * proper detection of _stat64() availability (as in MinGW sys/stat.h) --- programs/util.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/programs/util.h b/programs/util.h index b0f12363a..fe3d84489 100644 --- a/programs/util.h +++ b/programs/util.h @@ -141,9 +141,9 @@ UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond) /*-**************************************** * File functions ******************************************/ -#if defined(_MSC_VER) || defined(__MINGW32__) && _WIN32_WINNT >= _WIN32_WINNT_WIN2K +#if defined(_MSC_VER) || defined(__MINGW32__) && defined(__MSVCRT__) && _WIN32_WINNT >= _WIN32_WINNT_WIN2K #define chmod _chmod - typedef struct _stat64 stat_t; + typedef struct __stat64 stat_t; #else typedef struct stat stat_t; #endif @@ -172,7 +172,7 @@ UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf) UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) { int r; -#if defined(_MSC_VER) || defined(__MINGW32__) && _WIN32_WINNT >= _WIN32_WINNT_WIN2K +#if defined(_MSC_VER) || defined(__MINGW32__) && defined(__MSVCRT__) && _WIN32_WINNT >= _WIN32_WINNT_WIN2K r = _stat64(infilename, statbuf); if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */ #else @@ -186,8 +186,8 @@ UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) { int r; -#if defined(_MSC_VER) || defined(__MINGW32__) && _WIN32_WINNT >= _WIN32_WINNT_WIN2K - struct _stat64 statbuf; +#if defined(_MSC_VER) || defined(__MINGW32__) && defined(__MSVCRT__) && _WIN32_WINNT >= _WIN32_WINNT_WIN2K + struct __stat64 statbuf; r = _stat64(infilename, &statbuf); if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ #else @@ -212,8 +212,8 @@ UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFi UTIL_STATIC int UTIL_doesFileExists(const char* infilename) { int r; -#if defined(_MSC_VER) || defined(__MINGW32__) && _WIN32_WINNT >= _WIN32_WINNT_WIN2K - struct _stat64 statbuf; +#if defined(_MSC_VER) || defined(__MINGW32__) && defined(__MSVCRT__) && _WIN32_WINNT >= _WIN32_WINNT_WIN2K + struct __stat64 statbuf; r = _stat64(infilename, &statbuf); if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ #else @@ -228,8 +228,8 @@ UTIL_STATIC int UTIL_doesFileExists(const char* infilename) UTIL_STATIC U32 UTIL_isDirectory(const char* infilename) { int r; -#if defined(_MSC_VER) || defined(__MINGW32__) && _WIN32_WINNT >= _WIN32_WINNT_WIN2K - struct _stat64 statbuf; +#if defined(_MSC_VER) || defined(__MINGW32__) && defined(__MSVCRT__) && _WIN32_WINNT >= _WIN32_WINNT_WIN2K + struct __stat64 statbuf; r = _stat64(infilename, &statbuf); if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; #else From 45f0c207ab8e3955a6068cc845eceb175623fd63 Mon Sep 17 00:00:00 2001 From: ds77 Date: Fri, 10 Feb 2017 18:37:57 +0100 Subject: [PATCH 051/223] use _stati64() in UTIL_getFileSize() when compiling with mingw, get rid of introduces previously preprocessor checks. --- programs/util.h | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/programs/util.h b/programs/util.h index fe3d84489..b12179106 100644 --- a/programs/util.h +++ b/programs/util.h @@ -141,7 +141,7 @@ UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond) /*-**************************************** * File functions ******************************************/ -#if defined(_MSC_VER) || defined(__MINGW32__) && defined(__MSVCRT__) && _WIN32_WINNT >= _WIN32_WINNT_WIN2K +#if defined(_MSC_VER) #define chmod _chmod typedef struct __stat64 stat_t; #else @@ -172,7 +172,7 @@ UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf) UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) { int r; -#if defined(_MSC_VER) || defined(__MINGW32__) && defined(__MSVCRT__) && _WIN32_WINNT >= _WIN32_WINNT_WIN2K +#if defined(_MSC_VER) r = _stat64(infilename, statbuf); if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */ #else @@ -186,10 +186,14 @@ UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) { int r; -#if defined(_MSC_VER) || defined(__MINGW32__) && defined(__MSVCRT__) && _WIN32_WINNT >= _WIN32_WINNT_WIN2K - struct __stat64 statbuf; +#if defined(_MSC_VER) + struct _stat64 statbuf; r = _stat64(infilename, &statbuf); if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ +#elif defined(__MINGW32__) && defined (__MSVCRT__) + struct _stati64 statbuf; + r = _stati64(infilename, &statbuf); + if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ #else struct stat statbuf; r = stat(infilename, &statbuf); @@ -212,7 +216,7 @@ UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFi UTIL_STATIC int UTIL_doesFileExists(const char* infilename) { int r; -#if defined(_MSC_VER) || defined(__MINGW32__) && defined(__MSVCRT__) && _WIN32_WINNT >= _WIN32_WINNT_WIN2K +#if defined(_MSC_VER) struct __stat64 statbuf; r = _stat64(infilename, &statbuf); if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ @@ -228,7 +232,7 @@ UTIL_STATIC int UTIL_doesFileExists(const char* infilename) UTIL_STATIC U32 UTIL_isDirectory(const char* infilename) { int r; -#if defined(_MSC_VER) || defined(__MINGW32__) && defined(__MSVCRT__) && _WIN32_WINNT >= _WIN32_WINNT_WIN2K +#if defined(_MSC_VER) struct __stat64 statbuf; r = _stat64(infilename, &statbuf); if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; From 645f5b98563d584e7a12aa7179c05252688acdd4 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 10 Feb 2017 20:09:28 +0100 Subject: [PATCH 052/223] fix for original MinGW --- programs/platform.h | 2 +- programs/util.h | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/programs/platform.h b/programs/platform.h index 5aeee3325..1b53e1f85 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -51,7 +51,7 @@ extern "C" { /* ********************************************************* * Turn on Large Files support (>4GB) for 32-bit Linux/Unix ***********************************************************/ -#if !defined(__64BIT__) || defined(__MINGW32__) /* No point defining Large file for 64 bit but MinGW requires it */ +#if !defined(__64BIT__) /* No point defining Large file for 64 bit */ # if !defined(_FILE_OFFSET_BITS) # define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ # endif diff --git a/programs/util.h b/programs/util.h index 656b3a96b..58077ccc4 100644 --- a/programs/util.h +++ b/programs/util.h @@ -141,9 +141,9 @@ UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond) /*-**************************************** * File functions ******************************************/ -#if defined(_MSC_VER) +#if defined(_MSC_VER) || defined(__MINGW32__) #define chmod _chmod - typedef struct _stat64 stat_t; + typedef struct __stat64 stat_t; #else typedef struct stat stat_t; #endif @@ -172,7 +172,7 @@ UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf) UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) { int r; -#if defined(_MSC_VER) +#if defined(_MSC_VER) || defined(__MINGW32__) r = _stat64(infilename, statbuf); if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */ #else @@ -186,8 +186,8 @@ UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) { int r; -#if defined(_MSC_VER) - struct _stat64 statbuf; +#if defined(_MSC_VER) || defined(__MINGW32__) + struct __stat64 statbuf; r = _stat64(infilename, &statbuf); if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ #else From d7bfcac18a098769917d57c5f52fe8798d2ce35f Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Fri, 10 Feb 2017 11:38:57 -0800 Subject: [PATCH 053/223] Expose frameSrcSize to experimental API --- lib/decompress/zstd_decompress.c | 102 +++++++++++++++---------------- lib/legacy/zstd_legacy.h | 16 ++--- lib/legacy/zstd_v01.c | 2 +- lib/legacy/zstd_v01.h | 2 +- lib/legacy/zstd_v02.c | 6 +- lib/legacy/zstd_v02.h | 2 +- lib/legacy/zstd_v03.c | 6 +- lib/legacy/zstd_v03.h | 2 +- lib/legacy/zstd_v04.c | 6 +- lib/legacy/zstd_v04.h | 2 +- lib/legacy/zstd_v05.c | 2 +- lib/legacy/zstd_v05.h | 2 +- lib/legacy/zstd_v06.c | 2 +- lib/legacy/zstd_v06.h | 2 +- lib/legacy/zstd_v07.c | 2 +- lib/legacy/zstd_v07.h | 2 +- lib/zstd.h | 11 ++++ 17 files changed, 89 insertions(+), 80 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 18ee070cf..b8670315c 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -306,8 +306,6 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t return 0; } -static size_t ZSTD_frameSrcSize(const void* src, size_t srcSize); - /** ZSTD_getFrameContentSize() : * compatible with legacy mode * @return : decompressed size of the single frame pointed to be `src` if known, otherwise @@ -371,17 +369,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) totalDstSize += ret; } { - size_t frameSrcSize; -#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) - if (ZSTD_isLegacy(src, srcSize)) - { - frameSrcSize = ZSTD_frameSrcSizeLegacy(src, srcSize); - } - else -#endif - { - frameSrcSize = ZSTD_frameSrcSize(src, srcSize); - } + size_t const frameSrcSize = ZSTD_getFrameCompressedSize(src, srcSize); if (ZSTD_isError(frameSrcSize)) { return ZSTD_CONTENTSIZE_ERROR; } @@ -1449,47 +1437,57 @@ size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t len return length; } -static size_t ZSTD_frameSrcSize(const void *src, size_t srcSize) +/** ZSTD_getFrameCompressedSize() : + * compatible with legacy mode + * `src` must point to the start of a ZSTD or ZSTD legacy frame + * `srcSize` must be at least as large as the frame contained + * @return : the compressed size of the frame starting at `src` */ +size_t ZSTD_getFrameCompressedSize(const void *src, size_t srcSize) { - const BYTE* ip = (const BYTE*)src; - const BYTE* const ipstart = ip; - size_t remainingSize = srcSize; - ZSTD_frameParams fParams; - - size_t const headerSize = ZSTD_frameHeaderSize(ip, remainingSize); - if (ZSTD_isError(headerSize)) return headerSize; - - /* Frame Header */ +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) + if (ZSTD_isLegacy(src, srcSize)) return ZSTD_getFrameCompressedSizeLegacy(src, srcSize); +#endif { - size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize); - if (ZSTD_isError(ret)) return ret; - if (ret > 0) return ERROR(srcSize_wrong); + const BYTE* ip = (const BYTE*)src; + const BYTE* const ipstart = ip; + size_t remainingSize = srcSize; + ZSTD_frameParams fParams; + + size_t const headerSize = ZSTD_frameHeaderSize(ip, remainingSize); + if (ZSTD_isError(headerSize)) return headerSize; + + /* Frame Header */ + { + size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize); + if (ZSTD_isError(ret)) return ret; + if (ret > 0) return ERROR(srcSize_wrong); + } + + ip += headerSize; + remainingSize -= headerSize; + + /* Loop on each block */ + while (1) { + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + + if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) return ERROR(srcSize_wrong); + + ip += ZSTD_blockHeaderSize + cBlockSize; + remainingSize -= ZSTD_blockHeaderSize + cBlockSize; + + if (blockProperties.lastBlock) break; + } + + if (fParams.checksumFlag) { /* Frame content checksum */ + if (remainingSize < 4) return ERROR(srcSize_wrong); + ip += 4; + remainingSize -= 4; + } + + return ip - ipstart; } - - ip += headerSize; - remainingSize -= headerSize; - - /* Loop on each block */ - while (1) { - blockProperties_t blockProperties; - size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); - if (ZSTD_isError(cBlockSize)) return cBlockSize; - - if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) return ERROR(srcSize_wrong); - - ip += ZSTD_blockHeaderSize + cBlockSize; - remainingSize -= ZSTD_blockHeaderSize + cBlockSize; - - if (blockProperties.lastBlock) break; - } - - if (fParams.checksumFlag) { /* Frame content checksum verification */ - if (remainingSize < 4) return ERROR(srcSize_wrong); - ip += 4; - remainingSize -= 4; - } - - return ip - ipstart; } /*! ZSTD_decompressFrame() : @@ -1578,7 +1576,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) if (ZSTD_isLegacy(src, srcSize)) { - size_t const frameSize = ZSTD_frameSrcSizeLegacy(src, srcSize); + size_t const frameSize = ZSTD_getFrameCompressedSizeLegacy(src, srcSize); size_t decodedSize; if (ZSTD_isError(frameSize)) return frameSize; diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h index c0369ab4f..b0a7b71d6 100644 --- a/lib/legacy/zstd_legacy.h +++ b/lib/legacy/zstd_legacy.h @@ -123,26 +123,26 @@ MEM_STATIC size_t ZSTD_decompressLegacy( } } -MEM_STATIC size_t ZSTD_frameSrcSizeLegacy(const void *src, +MEM_STATIC size_t ZSTD_getFrameCompressedSizeLegacy(const void *src, size_t compressedSize) { U32 const version = ZSTD_isLegacy(src, compressedSize); switch(version) { case 1 : - return ZSTDv01_frameSrcSize(src, compressedSize); + return ZSTDv01_getFrameCompressedSize(src, compressedSize); case 2 : - return ZSTDv02_frameSrcSize(src, compressedSize); + return ZSTDv02_getFrameCompressedSize(src, compressedSize); case 3 : - return ZSTDv03_frameSrcSize(src, compressedSize); + return ZSTDv03_getFrameCompressedSize(src, compressedSize); case 4 : - return ZSTDv04_frameSrcSize(src, compressedSize); + return ZSTDv04_getFrameCompressedSize(src, compressedSize); case 5 : - return ZSTDv05_frameSrcSize(src, compressedSize); + return ZSTDv05_getFrameCompressedSize(src, compressedSize); case 6 : - return ZSTDv06_frameSrcSize(src, compressedSize); + return ZSTDv06_getFrameCompressedSize(src, compressedSize); case 7 : - return ZSTDv07_frameSrcSize(src, compressedSize); + return ZSTDv07_getFrameCompressedSize(src, compressedSize); default : return ERROR(prefix_unknown); } diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c index c9b676ad8..a0c78a4b8 100644 --- a/lib/legacy/zstd_v01.c +++ b/lib/legacy/zstd_v01.c @@ -1992,7 +1992,7 @@ size_t ZSTDv01_decompress(void* dst, size_t maxDstSize, const void* src, size_t return ZSTDv01_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize); } -size_t ZSTDv01_frameSrcSize(const void* src, size_t srcSize) +size_t ZSTDv01_getFrameCompressedSize(const void* src, size_t srcSize) { const BYTE* ip = (const BYTE*)src; size_t remainingSize = srcSize; diff --git a/lib/legacy/zstd_v01.h b/lib/legacy/zstd_v01.h index 9e5d553bf..21959fcd6 100644 --- a/lib/legacy/zstd_v01.h +++ b/lib/legacy/zstd_v01.h @@ -40,7 +40,7 @@ ZSTDv01_getFrameSrcSize() : get the source length of a ZSTD frame compliant with return : the number of bytes that would be read to decompress this frame or an errorCode if it fails (which can be tested using ZSTDv01_isError()) */ -size_t ZSTDv01_frameSrcSize(const void* src, size_t compressedSize); +size_t ZSTDv01_getFrameCompressedSize(const void* src, size_t compressedSize); /** ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c index f3b107af3..6cbf80234 100644 --- a/lib/legacy/zstd_v02.c +++ b/lib/legacy/zstd_v02.c @@ -3378,7 +3378,7 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize); } -static size_t ZSTD_frameSrcSize(const void *src, size_t srcSize) +static size_t ZSTD_getFrameCompressedSize(const void *src, size_t srcSize) { const BYTE* ip = (const BYTE*)src; @@ -3524,9 +3524,9 @@ size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize, return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize); } -size_t ZSTDv02_frameSrcSize(const void *src, size_t compressedSize) +size_t ZSTDv02_getFrameCompressedSize(const void *src, size_t compressedSize) { - return ZSTD_frameSrcSize(src, compressedSize); + return ZSTD_getFrameCompressedSize(src, compressedSize); } ZSTDv02_Dctx* ZSTDv02_createDCtx(void) diff --git a/lib/legacy/zstd_v02.h b/lib/legacy/zstd_v02.h index 45dc5f6ca..9542fc0ee 100644 --- a/lib/legacy/zstd_v02.h +++ b/lib/legacy/zstd_v02.h @@ -40,7 +40,7 @@ ZSTDv02_getFrameSrcSize() : get the source length of a ZSTD frame compliant with return : the number of bytes that would be read to decompress this frame or an errorCode if it fails (which can be tested using ZSTDv02_isError()) */ -size_t ZSTDv02_frameSrcSize(const void* src, size_t compressedSize); +size_t ZSTDv02_getFrameCompressedSize(const void* src, size_t compressedSize); /** ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c index 2eba77ccc..98b93c49b 100644 --- a/lib/legacy/zstd_v03.c +++ b/lib/legacy/zstd_v03.c @@ -3019,7 +3019,7 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize); } -static size_t ZSTD_frameSrcSize(const void* src, size_t srcSize) +static size_t ZSTD_getFrameCompressedSize(const void* src, size_t srcSize) { const BYTE* ip = (const BYTE*)src; size_t remainingSize = srcSize; @@ -3165,9 +3165,9 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize, return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize); } -size_t ZSTDv03_frameSrcSize(const void* src, size_t srcSize) +size_t ZSTDv03_getFrameCompressedSize(const void* src, size_t srcSize) { - return ZSTD_frameSrcSize(src, srcSize); + return ZSTD_getFrameCompressedSize(src, srcSize); } ZSTDv03_Dctx* ZSTDv03_createDCtx(void) diff --git a/lib/legacy/zstd_v03.h b/lib/legacy/zstd_v03.h index 24dba1f59..46969410a 100644 --- a/lib/legacy/zstd_v03.h +++ b/lib/legacy/zstd_v03.h @@ -40,7 +40,7 @@ ZSTDv03_getFrameSrcSize() : get the source length of a ZSTD frame compliant with return : the number of bytes that would be read to decompress this frame or an errorCode if it fails (which can be tested using ZSTDv03_isError()) */ -size_t ZSTDv03_frameSrcSize(const void* src, size_t compressedSize); +size_t ZSTDv03_getFrameCompressedSize(const void* src, size_t compressedSize); /** ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c index 7c900e1e0..8c929b053 100644 --- a/lib/legacy/zstd_v04.c +++ b/lib/legacy/zstd_v04.c @@ -3326,7 +3326,7 @@ static size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx, return op-ostart; } -static size_t ZSTD_frameSrcSize(const void* src, size_t srcSize) +static size_t ZSTD_getFrameCompressedSize(const void* src, size_t srcSize) { const BYTE* ip = (const BYTE*)src; size_t remainingSize = srcSize; @@ -3782,9 +3782,9 @@ size_t ZSTDv04_decompress(void* dst, size_t maxDstSize, const void* src, size_t #endif } -size_t ZSTDv04_frameSrcSize(const void* src, size_t srcSize) +size_t ZSTDv04_getFrameCompressedSize(const void* src, size_t srcSize) { - return ZSTD_frameSrcSize(src, srcSize); + return ZSTD_getFrameCompressedSize(src, srcSize); } size_t ZSTDv04_resetDCtx(ZSTDv04_Dctx* dctx) { return ZSTD_resetDCtx(dctx); } diff --git a/lib/legacy/zstd_v04.h b/lib/legacy/zstd_v04.h index 671ba6dc0..bcef1fe96 100644 --- a/lib/legacy/zstd_v04.h +++ b/lib/legacy/zstd_v04.h @@ -40,7 +40,7 @@ ZSTDv04_getFrameSrcSize() : get the source length of a ZSTD frame compliant with return : the number of bytes that would be read to decompress this frame or an errorCode if it fails (which can be tested using ZSTDv04_isError()) */ -size_t ZSTDv04_frameSrcSize(const void* src, size_t compressedSize); +size_t ZSTDv04_getFrameCompressedSize(const void* src, size_t compressedSize); /** ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c index 85c0f0e73..9689b170c 100644 --- a/lib/legacy/zstd_v05.c +++ b/lib/legacy/zstd_v05.c @@ -3583,7 +3583,7 @@ size_t ZSTDv05_decompress(void* dst, size_t maxDstSize, const void* src, size_t #endif } -size_t ZSTDv05_frameSrcSize(const void *src, size_t srcSize) +size_t ZSTDv05_getFrameCompressedSize(const void *src, size_t srcSize) { const BYTE* ip = (const BYTE*)src; size_t remainingSize = srcSize; diff --git a/lib/legacy/zstd_v05.h b/lib/legacy/zstd_v05.h index ef2d18d10..157dbc57b 100644 --- a/lib/legacy/zstd_v05.h +++ b/lib/legacy/zstd_v05.h @@ -38,7 +38,7 @@ ZSTDv05_getFrameSrcSize() : get the source length of a ZSTD frame return : the number of bytes that would be read to decompress this frame or an errorCode if it fails (which can be tested using ZSTDv05_isError()) */ -size_t ZSTDv05_frameSrcSize(const void* src, size_t compressedSize); +size_t ZSTDv05_getFrameCompressedSize(const void* src, size_t compressedSize); /* ************************************* * Helper functions diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index 92c1a45c6..4c8f06823 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -3729,7 +3729,7 @@ size_t ZSTDv06_decompress(void* dst, size_t dstCapacity, const void* src, size_t #endif } -size_t ZSTDv06_frameSrcSize(const void* src, size_t srcSize) +size_t ZSTDv06_getFrameCompressedSize(const void* src, size_t srcSize) { const BYTE* ip = (const BYTE*)src; size_t remainingSize = srcSize; diff --git a/lib/legacy/zstd_v06.h b/lib/legacy/zstd_v06.h index 1fad7311e..ef1feb2f2 100644 --- a/lib/legacy/zstd_v06.h +++ b/lib/legacy/zstd_v06.h @@ -47,7 +47,7 @@ ZSTDv06_getFrameSrcSize() : get the source length of a ZSTD frame return : the number of bytes that would be read to decompress this frame or an errorCode if it fails (which can be tested using ZSTDv06_isError()) */ -size_t ZSTDv06_frameSrcSize(const void* src, size_t compressedSize); +size_t ZSTDv06_getFrameCompressedSize(const void* src, size_t compressedSize); /* ************************************* * Helper functions diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index 6228ee870..441e4bc39 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -3968,7 +3968,7 @@ size_t ZSTDv07_decompress(void* dst, size_t dstCapacity, const void* src, size_t #endif } -size_t ZSTDv07_frameSrcSize(const void* src, size_t srcSize) +size_t ZSTDv07_getFrameCompressedSize(const void* src, size_t srcSize) { const BYTE* ip = (const BYTE*)src; size_t remainingSize = srcSize; diff --git a/lib/legacy/zstd_v07.h b/lib/legacy/zstd_v07.h index b02b3dce4..a79cbb883 100644 --- a/lib/legacy/zstd_v07.h +++ b/lib/legacy/zstd_v07.h @@ -54,7 +54,7 @@ ZSTDv07_getFrameSrcSize() : get the source length of a ZSTD frame return : the number of bytes that would be read to decompress this frame or an errorCode if it fails (which can be tested using ZSTDv07_isError()) */ -size_t ZSTDv07_frameSrcSize(const void* src, size_t compressedSize); +size_t ZSTDv07_getFrameCompressedSize(const void* src, size_t compressedSize); /*====== Helper functions ======*/ ZSTDLIBv07_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ diff --git a/lib/zstd.h b/lib/zstd.h index bbfefb8db..e2f752ddc 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -396,6 +396,17 @@ typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); typedef void (*ZSTD_freeFunction) (void* opaque, void* address); typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; +/*************************************** +* Compressed size functions +***************************************/ + +/*! ZSTD_getFrameCompressedSize() : + * `src` should point to the start of a ZSTD encoded frame + * `srcSize` must be at least as large as the frame + * @return : the compressed size of the frame pointed to by `src`, suitable to pass to + * `ZSTD_decompress` or similar, or an error code if given invalid input. */ +ZSTDLIB_API size_t ZSTD_getFrameCompressedSize(const void* src, size_t srcSize); + /*************************************** * Decompressed size functions ***************************************/ From eb132530cd13031a9d48adfada2b166b1aa488dc Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 10 Feb 2017 21:15:49 +0100 Subject: [PATCH 054/223] revert last commit --- programs/util.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/programs/util.h b/programs/util.h index 58077ccc4..656b3a96b 100644 --- a/programs/util.h +++ b/programs/util.h @@ -141,9 +141,9 @@ UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond) /*-**************************************** * File functions ******************************************/ -#if defined(_MSC_VER) || defined(__MINGW32__) +#if defined(_MSC_VER) #define chmod _chmod - typedef struct __stat64 stat_t; + typedef struct _stat64 stat_t; #else typedef struct stat stat_t; #endif @@ -172,7 +172,7 @@ UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf) UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) { int r; -#if defined(_MSC_VER) || defined(__MINGW32__) +#if defined(_MSC_VER) r = _stat64(infilename, statbuf); if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */ #else @@ -186,8 +186,8 @@ UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) { int r; -#if defined(_MSC_VER) || defined(__MINGW32__) - struct __stat64 statbuf; +#if defined(_MSC_VER) + struct _stat64 statbuf; r = _stat64(infilename, &statbuf); if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ #else From 168d9b8006882ed72721272ad6c1c69c1712d87a Mon Sep 17 00:00:00 2001 From: ds77 Date: Sun, 12 Feb 2017 10:27:18 +0100 Subject: [PATCH 055/223] fix seeking 2GB+ files under Windows Replace fseek() in FIO_fwriteSparse() and FIO_fwriteSparseEnd() with macro expanding to 64-bit fseek version provided by the platform (includes fallback workaround using Win32 API). --- programs/fileio.c | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 087bf9504..51b1a94ec 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -91,6 +91,32 @@ static clock_t g_time = 0; #define MIN(a,b) ((a) < (b) ? (a) : (b)) +#if defined(_MSC_VER) && _MSC_VER >= 1400 +# define LONG_SEEK _fseeki64 +#elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__) +# define LONG_SEEK fseeko64 +#elif defined(_WIN32) && !defined(__DJGPP__) +# include + static int LONG_SEEK(FILE* file, __int64 offset, int origin) { + LARGE_INTEGER off; + DWORD method; + off.QuadPart = offset; + if (origin == SEEK_END) + method = FILE_END; + else if (origin == SEEK_CUR) + method = FILE_CURRENT; + else + method = FILE_BEGIN; + + if (SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, NULL, method)) + return 0; + else + return -1; + } +#else +# define LONG_SEEK fseek +#endif + /*-************************************* * Local Parameters - Not thread safe @@ -598,7 +624,7 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi /* avoid int overflow */ if (storedSkips > 1 GB) { - int const seekResult = fseek(file, 1 GB, SEEK_CUR); + int const seekResult = LONG_SEEK(file, 1 GB, SEEK_CUR); if (seekResult != 0) EXM_THROW(71, "1 GB skip error (sparse file support)"); storedSkips -= 1 GB; } @@ -614,7 +640,7 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi storedSkips += (unsigned)(nb0T * sizeof(size_t)); if (nb0T != seg0SizeT) { /* not all 0s */ - int const seekResult = fseek(file, storedSkips, SEEK_CUR); + int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR); if (seekResult) EXM_THROW(72, "Sparse skip error ; try --no-sparse"); storedSkips = 0; seg0SizeT -= nb0T; @@ -634,7 +660,7 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi for ( ; (restPtr < restEnd) && (*restPtr == 0); restPtr++) ; storedSkips += (unsigned) (restPtr - restStart); if (restPtr != restEnd) { - int seekResult = fseek(file, storedSkips, SEEK_CUR); + int seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR); if (seekResult) EXM_THROW(74, "Sparse skip error ; try --no-sparse"); storedSkips = 0; { size_t const sizeCheck = fwrite(restPtr, 1, restEnd - restPtr, file); @@ -647,7 +673,7 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips) { if (storedSkips-->0) { /* implies g_sparseFileSupport>0 */ - int const seekResult = fseek(file, storedSkips, SEEK_CUR); + int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR); if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n"); { const char lastZeroByte[1] = { 0 }; size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file); From 6220bfc9247d50568b77c3b79b4f41b401761c85 Mon Sep 17 00:00:00 2001 From: ds77 Date: Mon, 13 Feb 2017 12:00:59 +0100 Subject: [PATCH 056/223] fix indentation in previous commit --- programs/fileio.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index 51b1a94ec..be1ada30b 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -109,9 +109,9 @@ static clock_t g_time = 0; method = FILE_BEGIN; if (SetFilePointerEx((HANDLE) _get_osfhandle(_fileno(file)), off, NULL, method)) - return 0; + return 0; else - return -1; + return -1; } #else # define LONG_SEEK fseek From 09c8e5390dbaaaca84b27e99d55e1052f5e8dcef Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 13 Feb 2017 12:45:53 +0100 Subject: [PATCH 057/223] __builtin_bswap requires gcc 4.3+ --- lib/common/mem.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/common/mem.h b/lib/common/mem.h index aff044de1..1c223fe5e 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -182,7 +182,7 @@ MEM_STATIC U32 MEM_swap32(U32 in) { #if defined(_MSC_VER) /* Visual Studio */ return _byteswap_ulong(in); -#elif defined (__GNUC__) +#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403) return __builtin_bswap32(in); #else return ((in << 24) & 0xff000000 ) | @@ -196,7 +196,7 @@ MEM_STATIC U64 MEM_swap64(U64 in) { #if defined(_MSC_VER) /* Visual Studio */ return _byteswap_uint64(in); -#elif defined (__GNUC__) +#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403) return __builtin_bswap64(in); #else return ((in << 56) & 0xff00000000000000ULL) | From 35bf23c08675edfab1e95aef83bb06e92110f1c9 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 13 Feb 2017 13:57:29 +0100 Subject: [PATCH 058/223] MinGW-w64 requires _FILE_OFFSET_BITS 64 --- programs/platform.h | 4 ++-- programs/util.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/programs/platform.h b/programs/platform.h index 1b53e1f85..89a9f6cd4 100644 --- a/programs/platform.h +++ b/programs/platform.h @@ -51,8 +51,8 @@ extern "C" { /* ********************************************************* * Turn on Large Files support (>4GB) for 32-bit Linux/Unix ***********************************************************/ -#if !defined(__64BIT__) /* No point defining Large file for 64 bit */ -# if !defined(_FILE_OFFSET_BITS) +#if !defined(__64BIT__) || defined(__MINGW32__) /* No point defining Large file for 64 bit but MinGW-w64 requires it */ +# if !defined(_FILE_OFFSET_BITS) # define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ # endif # if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */ diff --git a/programs/util.h b/programs/util.h index 16bd3bdfc..fe132d38d 100644 --- a/programs/util.h +++ b/programs/util.h @@ -187,7 +187,7 @@ UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) { int r; #if defined(_MSC_VER) - struct _stat64 statbuf; + struct __stat64 statbuf; r = _stat64(infilename, &statbuf); if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ #elif defined(__MINGW32__) && defined (__MSVCRT__) From 64f7221958084b10f9f9c8b86639a96a73b106a2 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 13 Feb 2017 21:00:41 +0100 Subject: [PATCH 059/223] limit zlib compression level to Z_BEST_COMPRESSION --- programs/fileio.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/programs/fileio.c b/programs/fileio.c index 430127771..9f1560b43 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -344,6 +344,8 @@ static unsigned long long FIO_compressGzFrame(cRess_t* ress, const char* srcFile z_stream strm; int ret; + if (compressionLevel > Z_BEST_COMPRESSION) compressionLevel = Z_BEST_COMPRESSION; + strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; From 1a195b3b7a4cb9efaa26e7d9c5c64173e65ba8e8 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 13 Feb 2017 22:56:31 +0100 Subject: [PATCH 060/223] fixed unitialized variable warning --- programs/fileio.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index ca83db6f0..b8066a7a3 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -376,8 +376,8 @@ static unsigned long long FIO_compressGzFrame(cRess_t* ress, const char* srcFile strm.zfree = Z_NULL; strm.opaque = Z_NULL; - if (deflateInit2(&strm, compressionLevel, Z_DEFLATED, 15 /* maxWindowLogSize */ + 16 /* gzip only */, 8, Z_DEFAULT_STRATEGY) != Z_OK) - EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); /* see http://www.zlib.net/manual.html */ + ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED, 15 /* maxWindowLogSize */ + 16 /* gzip only */, 8, Z_DEFAULT_STRATEGY); + if (ret != Z_OK) EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); /* see http://www.zlib.net/manual.html */ strm.next_in = 0; strm.avail_in = Z_NULL; From 08e6a88a97f8a472a8e89aa1ea49b6b8669ff595 Mon Sep 17 00:00:00 2001 From: ds77 Date: Tue, 14 Feb 2017 00:14:24 +0100 Subject: [PATCH 061/223] avoid empty translation unit warning without #pragma --- lib/common/threading.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/common/threading.c b/lib/common/threading.c index b56e594b2..32d58796a 100644 --- a/lib/common/threading.c +++ b/lib/common/threading.c @@ -15,10 +15,11 @@ * This file will hold wrapper for systems, which do not support pthreads */ -/* ====== Compiler specifics ====== */ -#if defined(_MSC_VER) -# pragma warning(disable : 4206) /* disable: C4206: translation unit is empty (when ZSTD_MULTITHREAD is not defined) */ -#endif +/* When ZSTD_MULTITHREAD is not defined, this file would become an empty translation unit. +* Include some ISO C header code to prevent this and portably avoid related warnings. +* (Visual C++: C4206 / GCC: -Wpedantic / Clang: -Wempty-translation-unit) +*/ +#include #if defined(ZSTD_MULTITHREAD) && defined(_WIN32) From 58af614ef27c84efb5b6e1685bebfe38f8e9493e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 13 Feb 2017 18:32:12 -0800 Subject: [PATCH 062/223] push version and NEWS to v1.1.4 --- NEWS | 7 +++++++ lib/zstd.h | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 24860c957..09f1d010d 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,10 @@ +v1.1.4 +cli : new : advanced comnmand --priority=rt, by Przemyslaw Skibinski +cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77 +API : new : ZSTD_getFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize(), by Sean Purcell +API : change : ZSTD_compress*() with srcSize==0 create an empty-frame of known size, by Sean Purcell +doc : new : educational decoder, by Sean Purcell + v1.1.3 cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`) cli : new : experimental target `make zstdmt`, with multi-threading support diff --git a/lib/zstd.h b/lib/zstd.h index e2f752ddc..a71867837 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -56,7 +56,7 @@ extern "C" { /*------ Version ------*/ #define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MINOR 1 -#define ZSTD_VERSION_RELEASE 3 +#define ZSTD_VERSION_RELEASE 4 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_QUOTE(str) #str From ecf90ca24b7031444133bc236a51683bd059dfc7 Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Mon, 13 Feb 2017 18:27:34 -0800 Subject: [PATCH 063/223] [zstdmt] Fix MSAN failure with ZSTD_p_forceWindow Reproduction steps: ``` make zstreamtest CC=clang CFLAGS="-O3 -g -fsanitize=memory -fsanitize-memory-track-origins" ./zstreamtest -vv -t4178 -i4178 -s4531 ``` How to get to the error in gdb (may be a more efficient way): * 2 breaks at zstd_compress.c:2418 -- in ZSTD_compressContinue_internal() * 2 breaks at zstd_compress.c:2276 -- in ZSTD_compressBlock_internal() * 1 break at zstd_compress.c:1547 Why the error occurred: When `zc->forceWindow == 1`, after calling `ZSTD_loadDictionaryContent()` we have `zc->loadedDictEnd == zc->nextToUpdate == 0`. But, we've really loaded up to `iend` into the dictionary. Then in `ZSTD_compressBlock_internal()` we see that `current > zc->nextToUpdate + 384`, so we load the last 192 bytes a second time. In this case the bytes we are loading are a block of all 0s, starting in the previous block. So when we are loading the last 192 bytes, we find a `match` in the future, 183 bytes beyond `ip`. Since the block is all 0s, the match extends to the end of the block. But in `ZSTD_count()` we only check that `pIn < pInLoopLimit`, but since `pMatch > pIn`, `pMatch` eventually points past the end of the buffer, causing the MSAN failure. The fix: The line changed sets sets `zc->nextToUpdate` to the end of the dictionary. This is the behavior that existed before `ZSTD_p_forceWindow` was introduced. This fixes the exposing test case. Since the code doesn't fail without `zc->forceWindow`, it makes sense that this works. I've run the command `./zstreamtest -T2mn` 64 times without failures. CI should also verify nothing obvious broke. --- lib/compress/zstd_compress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 765c8e34d..91e81d9c2 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2512,7 +2512,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t return ERROR(GENERIC); /* strategy doesn't exist; impossible */ } - zc->nextToUpdate = zc->loadedDictEnd; + zc->nextToUpdate = (U32)(iend - zc->base); return 0; } From 2bb6fc2a944d30d0ec3ec18d3db0fc462cf06ccf Mon Sep 17 00:00:00 2001 From: zefanxu2 Date: Mon, 13 Feb 2017 21:12:59 -0600 Subject: [PATCH 064/223] fix memory leak --- examples/simple_compression.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/simple_compression.c b/examples/simple_compression.c index deb0bbfc4..2aab48f47 100644 --- a/examples/simple_compression.c +++ b/examples/simple_compression.c @@ -127,6 +127,6 @@ int main(int argc, const char** argv) const char* const outFilename = createOutFilename_orDie(inFilename); compress_orDie(inFilename, outFilename); - + free(outFilename); return 0; } From 98509a70acc58c71464181c3564f9bfd9cc319ed Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 14 Feb 2017 09:23:32 +0100 Subject: [PATCH 065/223] fixed function name --- programs/fileio.c | 11 +++++------ programs/fileio.h | 4 ++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index b8066a7a3..e905948bf 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -121,8 +121,8 @@ static clock_t g_time = 0; /*-************************************* * Local Parameters - Not thread safe ***************************************/ -static FIO_compresionType_t g_compresionType = FIO_zstdCompression; -void FIO_setCompresionType(FIO_compresionType_t compresionType) { g_compresionType = compresionType; } +static FIO_compressionType_t g_compressionType = FIO_zstdCompression; +void FIO_setCompressionType(FIO_compressionType_t compressionType) { g_compressionType = compressionType; } static U32 g_overwrite = 0; void FIO_overwriteMode(void) { g_overwrite=1; } static U32 g_sparseFileSupport = 1; /* 0 : no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ @@ -376,8 +376,8 @@ static unsigned long long FIO_compressGzFrame(cRess_t* ress, const char* srcFile strm.zfree = Z_NULL; strm.opaque = Z_NULL; - ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED, 15 /* maxWindowLogSize */ + 16 /* gzip only */, 8, Z_DEFAULT_STRATEGY); - if (ret != Z_OK) EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); /* see http://www.zlib.net/manual.html */ + ret = deflateInit2(&strm, compressionLevel, Z_DEFLATED, 15 /* maxWindowLogSize */ + 16 /* gzip only */, 8, Z_DEFAULT_STRATEGY); /* see http://www.zlib.net/manual.html */ + if (ret != Z_OK) EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret); strm.next_in = 0; strm.avail_in = Z_NULL; @@ -443,10 +443,9 @@ static int FIO_compressFilename_internal(cRess_t ress, U64 compressedfilesize = 0; U64 const fileSize = UTIL_getFileSize(srcFileName); - if (g_compresionType) { + if (g_compressionType) { #ifdef ZSTD_GZCOMPRESS compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); - // printf("g_compresionType=%d compressionLevel=%d compressedfilesize=%d\n", g_compresionType, compressionLevel, (int)compressedfilesize); #else (void)compressionLevel; EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", srcFileName); diff --git a/programs/fileio.h b/programs/fileio.h index 2b6275573..e828b385b 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -34,13 +34,13 @@ extern "C" { /*-************************************* * Types ***************************************/ -typedef enum { FIO_zstdCompression, FIO_gzipCompression } FIO_compresionType_t; +typedef enum { FIO_zstdCompression, FIO_gzipCompression } FIO_compressionType_t; /*-************************************* * Parameters ***************************************/ -void FIO_setCompresionType(FIO_compresionType_t compresionType); +void FIO_setCompressionType(FIO_compressionType_t compressionType); void FIO_overwriteMode(void); void FIO_setNotificationLevel(unsigned level); void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */ From 442c75f13214bc476ee96b865023f3d2b9845e1e Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 14 Feb 2017 09:38:51 +0100 Subject: [PATCH 066/223] removed UTIL_doesFileExists (replaced with UTIL_isRegFile) --- programs/fileio.c | 2 +- programs/util.h | 48 +++++++++++++++++------------------------------ 2 files changed, 18 insertions(+), 32 deletions(-) diff --git a/programs/fileio.c b/programs/fileio.c index b2998ea12..b8d4ef02d 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -191,7 +191,7 @@ static FILE* FIO_openSrcFile(const char* srcFileName) f = stdin; SET_BINARY_MODE(stdin); } else { - if (!UTIL_doesFileExists(srcFileName)) { + if (!UTIL_isRegFile(srcFileName)) { DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", srcFileName); return NULL; } diff --git a/programs/util.h b/programs/util.h index 7f710c686..364aa650f 100644 --- a/programs/util.h +++ b/programs/util.h @@ -182,12 +182,29 @@ UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf) return 1; } + UTIL_STATIC int UTIL_isRegFile(const char* infilename) { stat_t statbuf; return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */ } + +UTIL_STATIC U32 UTIL_isDirectory(const char* infilename) +{ + int r; + stat_t statbuf; +#if defined(_MSC_VER) + r = _stat64(infilename, &statbuf); + if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; +#else + r = stat(infilename, &statbuf); + if (!r && S_ISDIR(statbuf.st_mode)) return 1; +#endif + return 0; +} + + UTIL_STATIC U64 UTIL_getFileSize(const char* infilename) { int r; @@ -218,37 +235,6 @@ UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFi } -UTIL_STATIC int UTIL_doesFileExists(const char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct __stat64 statbuf; - r = _stat64(infilename, &statbuf); - if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ -#else - struct stat statbuf; - r = stat(infilename, &statbuf); - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ -#endif - return 1; -} - - -UTIL_STATIC U32 UTIL_isDirectory(const char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct __stat64 statbuf; - r = _stat64(infilename, &statbuf); - if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; -#else - struct stat statbuf; - r = stat(infilename, &statbuf); - if (!r && S_ISDIR(statbuf.st_mode)) return 1; -#endif - return 0; -} - /* * A modified version of realloc(). * If UTIL_realloc() fails the original block is freed. From abd6302423d262ef04c299a9585813ed0bc4b01a Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 14 Feb 2017 09:39:09 +0100 Subject: [PATCH 067/223] Windows resources updated to v1.1.4 --- programs/windres/zstd32.res | Bin 1044 -> 1044 bytes programs/windres/zstd64.res | Bin 1044 -> 1044 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/programs/windres/zstd32.res b/programs/windres/zstd32.res index 748192a988648a0be3b17ce47353f28015003815..b5dd78db717ce9c7ddb2f69a9bc81804e51aedf6 100644 GIT binary patch delta 33 pcmbQjF@`OkT;H4FH692nqlI delta 33 pcmbQjF@`OkT;H4FH692nqlI delta 33 pcmbQjF@ Date: Tue, 14 Feb 2017 09:45:33 +0100 Subject: [PATCH 068/223] Avoid fseek()'s 2GiB barrier with MacOS and *BSD --- programs/fileio.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/programs/fileio.c b/programs/fileio.c index b8d4ef02d..c152fdcbe 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -91,8 +91,13 @@ static clock_t g_time = 0; #define MIN(a,b) ((a) < (b) ? (a) : (b)) +/* ************************************************************ +* Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW +***************************************************************/ #if defined(_MSC_VER) && _MSC_VER >= 1400 # define LONG_SEEK _fseeki64 +#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ +# define fseek fseeko #elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__) # define LONG_SEEK fseeko64 #elif defined(_WIN32) && !defined(__DJGPP__) From 970419535ff739200acd0957d9abb48acf000c80 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 14 Feb 2017 09:47:29 +0100 Subject: [PATCH 069/223] fixed function name (2) --- programs/zstdcli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 30aa0b949..33c0cd8d5 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -364,7 +364,7 @@ int main(int argCount, const char* argv[]) if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } - if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompresionType(FIO_gzipCompression); continue; } + if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; } /* long commands with arguments */ #ifndef ZSTD_NODICT From ce13d087d9af867a1540b6351fd98b212d91dbe4 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 14 Feb 2017 09:52:52 +0100 Subject: [PATCH 070/223] fix LONG_SEEK --- programs/fileio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/fileio.c b/programs/fileio.c index c152fdcbe..b384f3d1f 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -97,7 +97,7 @@ static clock_t g_time = 0; #if defined(_MSC_VER) && _MSC_VER >= 1400 # define LONG_SEEK _fseeki64 #elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ -# define fseek fseeko +# define LONG_SEEK fseeko #elif defined(__MINGW32__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) && defined(__MSVCRT__) # define LONG_SEEK fseeko64 #elif defined(_WIN32) && !defined(__DJGPP__) From 74b81ada256f45b0ea69f50c3b9b3918faacc91a Mon Sep 17 00:00:00 2001 From: Nick Terrell Date: Tue, 14 Feb 2017 10:08:14 -0800 Subject: [PATCH 071/223] Don't run test-pool with QEMU > make test -n ... ./pool > make test -n QEMU_SYS=valgrind ... ./legacy # ./pool not run --- tests/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index f64be1695..2b58c949c 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -231,7 +231,7 @@ zstd-playTests: datagen ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST) test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy -ifneq ($(QEMU_SYS),qemu-ppc64-static) +ifeq ($(QEMU_SYS),) test: test-pool endif From c09d16ba8c9328c8c6d0955b291470caabff0d83 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Tue, 14 Feb 2017 10:45:19 -0800 Subject: [PATCH 072/223] preset behavior for gzip, gunzip and gzcat when zstd is called through a link named gzip, gunzip or gzcat, provides the same behavior as the related program. gzip compresses using --format=gz both gzip and gunzip enable --rm by default --- NEWS | 5 +++-- programs/zstdcli.c | 8 +++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 09f1d010d..a710da8b9 100644 --- a/NEWS +++ b/NEWS @@ -1,8 +1,9 @@ v1.1.4 -cli : new : advanced comnmand --priority=rt, by Przemyslaw Skibinski +cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski +cli : new : advanced benchmark command --priority=rt cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77 API : new : ZSTD_getFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize(), by Sean Purcell -API : change : ZSTD_compress*() with srcSize==0 create an empty-frame of known size, by Sean Purcell +API : change : ZSTD_compress*() with srcSize==0 create an empty-frame of known size doc : new : educational decoder, by Sean Purcell v1.1.3 diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 33c0cd8d5..588111913 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -51,8 +51,11 @@ #define GZ_EXTENSION ".gz" #define ZSTD_EXTENSION ".zst" -#define ZSTD_CAT "zstdcat" #define ZSTD_UNZSTD "unzstd" +#define ZSTD_CAT "zstdcat" +#define ZSTD_GZ "gzip" +#define ZSTD_GUNZIP "gunzip" +#define ZSTD_GZCAT "gzcat" #define KB *(1 <<10) #define MB *(1 <<20) @@ -319,6 +322,9 @@ int main(int argCount, const char* argv[]) /* preset behaviors */ if (!strcmp(programName, ZSTD_UNZSTD)) operation=zom_decompress; if (!strcmp(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; displayLevel=1; } + if (!strcmp(programName, ZSTD_GZ)) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); FIO_setRemoveSrcFile(1); } /* behave like gzip */ + if (!strcmp(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(1); } /* behave like gunzip */ + if (!strcmp(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; displayLevel=1; } /* behave like gzcat */ memset(&compressionParams, 0, sizeof(compressionParams)); /* command switches */ From 9b5a1e9d973dca9d4cc312c8dc6e441d4ab94dfd Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 14 Feb 2017 20:06:41 +0100 Subject: [PATCH 073/223] added circle.yml --- circle.yml | 42 ++++++++++++++++++++++++++++++++++++++++++ tests/Makefile | 2 +- 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 circle.yml diff --git a/circle.yml b/circle.yml new file mode 100644 index 000000000..c189d3b65 --- /dev/null +++ b/circle.yml @@ -0,0 +1,42 @@ +dependencies: + override: + - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update + - sudo apt-get -y install qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu + - sudo apt-get -y install qemu-system-arm gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross + - sudo apt-get -y install libc6-dev-i386 clang gcc-5 gcc-6 valgrind + +test: + override: + # Tests compilers and C standards + - clang -v; make clangtest && make clean + - g++ -v; make gpptest && make clean + - gcc -v; make gnu90test && make clean + - gcc -v; make c99test && make clean + - gcc -v; make gnu99test && make clean + - gcc-5 -v; make gcc5test && make clean + - gcc-6 -v; make gcc6test && make clean + # Shorter tests + - make cmaketest && make clean + - make zlibwrapper && make clean + - make -C lib all && make clean + - make -C tests dll && make clean + - make -C tests test-symbols && make clean + - make -C tests test-zstd-nolegacy && make clean + - make -C tests test-longmatch && make clean + - pyenv global 3.4.4; make -C tests versionsTest && make clean + - make -C programs zstd-small zstd-decompress zstd-compress && make -C programs clean + - make travis-install && make clean + # Longer tests + - make test && make clean + - gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean + - make usan && make clean + - make asan && make clean + - make asan32 && make clean + # Valgrind tests + - CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean + - make -C tests valgrindTest && make clean + # ARM, AArch64, PowerPC, PowerPC64 tests + - make ppctest && make clean + - make ppc64test && make clean + - make armtest && make clean + - make aarch64test && make clean diff --git a/tests/Makefile b/tests/Makefile index f64be1695..17b0146e2 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -184,7 +184,7 @@ clean: fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT)\ zstreamtest$(EXT) zstreamtest32$(EXT) \ datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \ - symbols$(EXT) invalidDictionaries$(EXT) pool$(EXT) + symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) pool$(EXT) @echo Cleaning completed From 90e5412a4e2ea9c059a37b5d71ec5bb4105361ad Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 14 Feb 2017 23:30:23 +0100 Subject: [PATCH 074/223] added -I/usr/include/x86_64-linux-gnu for asan32 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d86db7cb3..3ce12e67d 100644 --- a/Makefile +++ b/Makefile @@ -138,7 +138,7 @@ msan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer" # datagen.c fails this test for no obvious reason asan32: clean - $(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address" + $(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu" uasan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined" From 3aaa1dae4e49d3fcf8c85568de79c5f2eafaeacd Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 15 Feb 2017 09:17:39 +0100 Subject: [PATCH 075/223] simplified zlib detection --- programs/Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/programs/Makefile b/programs/Makefile index efe684432..0a9ab5a79 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -67,9 +67,8 @@ endif # zlib detection VOID = /dev/null -HAVE_ZLIB := $(shell printf '\#include \nint main(){}' | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && echo 1 || echo 0) +HAVE_ZLIB := $(shell printf '\#include \nint main(){}' | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0) ifeq ($(HAVE_ZLIB), 1) -TEMP := $(shell rm have_zlib$(EXT)) ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS ZLIBLD = -lz endif From 6e59b3ce0180b02e81bfa8fc1829567cb0c2e1a9 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 15 Feb 2017 17:03:16 +0100 Subject: [PATCH 076/223] added UTIL_fseek --- programs/util.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/programs/util.h b/programs/util.h index 364aa650f..c03bd1d19 100644 --- a/programs/util.h +++ b/programs/util.h @@ -39,6 +39,20 @@ extern "C" { #include "mem.h" /* U32, U64 */ +/* ************************************************************ +* Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW +***************************************************************/ +#if defined(_MSC_VER) && (_MSC_VER >= 1400) +# define UTIL_fseek _fseeki64 +#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ +# define UTIL_fseek fseeko +#elif defined(__MINGW32__) && defined(__MSVCRT__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) +# define UTIL_fseek fseeko64 +#else +# define UTIL_fseek fseek +#endif + + /*-**************************************** * Sleep functions: Windows - Posix - others ******************************************/ From acb6e57ad211a0e897310acf5bfec2b72da23002 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 15 Feb 2017 17:13:35 +0100 Subject: [PATCH 077/223] use FindFirstFileA instead of FindFirstFile --- programs/util.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/programs/util.h b/programs/util.h index c03bd1d19..0f588f110 100644 --- a/programs/util.h +++ b/programs/util.h @@ -269,7 +269,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ { char* path; int dirLength, fnameLength, pathLength, nbFiles = 0; - WIN32_FIND_DATA cFile; + WIN32_FIND_DATAA cFile; HANDLE hFile; dirLength = (int)strlen(dirName); @@ -281,7 +281,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ path[dirLength+1] = '*'; path[dirLength+2] = 0; - hFile=FindFirstFile(path, &cFile); + hFile=FindFirstFileA(path, &cFile); if (hFile == INVALID_HANDLE_VALUE) { fprintf(stderr, "Cannot open directory '%s'\n", dirName); return 0; @@ -318,7 +318,7 @@ UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_ } } free(path); - } while (FindNextFile(hFile, &cFile)); + } while (FindNextFileA(hFile, &cFile)); FindClose(hFile); return nbFiles; From 4596037042851fbcf3800fe33fa9424ae52e99d0 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 15 Feb 2017 12:00:03 -0800 Subject: [PATCH 078/223] updated fse version feature minor refactoring (removing FSE_abs()) also : fix a few minor issues recently introduced in examples --- examples/dictionary_decompression.c | 1 + examples/simple_compression.c | 6 +-- examples/simple_decompression.c | 21 +++++---- lib/common/entropy_common.c | 24 +++++----- lib/common/fse.h | 70 ++++++++++++++++++++--------- lib/compress/fse_compress.c | 10 ++--- 6 files changed, 80 insertions(+), 52 deletions(-) diff --git a/examples/dictionary_decompression.c b/examples/dictionary_decompression.c index 2aa71b268..75183505d 100644 --- a/examples/dictionary_decompression.c +++ b/examples/dictionary_decompression.c @@ -13,6 +13,7 @@ #include // strerror #include // errno #include // stat +#define ZSTD_STATIC_LINKING_ONLY // ZSTD_findDecompressedSize #include // presumes zstd library is installed diff --git a/examples/simple_compression.c b/examples/simple_compression.c index 2aab48f47..9d448712e 100644 --- a/examples/simple_compression.c +++ b/examples/simple_compression.c @@ -102,7 +102,7 @@ static void compress_orDie(const char* fname, const char* oname) } -static const char* createOutFilename_orDie(const char* filename) +static char* createOutFilename_orDie(const char* filename) { size_t const inL = strlen(filename); size_t const outL = inL + 5; @@ -110,7 +110,7 @@ static const char* createOutFilename_orDie(const char* filename) memset(outSpace, 0, outL); strcat(outSpace, filename); strcat(outSpace, ".zst"); - return (const char*)outSpace; + return (char*)outSpace; } int main(int argc, const char** argv) @@ -125,7 +125,7 @@ int main(int argc, const char** argv) return 1; } - const char* const outFilename = createOutFilename_orDie(inFilename); + char* const outFilename = createOutFilename_orDie(inFilename); compress_orDie(inFilename, outFilename); free(outFilename); return 0; diff --git a/examples/simple_decompression.c b/examples/simple_decompression.c index 3a75e164c..09b27baa6 100644 --- a/examples/simple_decompression.c +++ b/examples/simple_decompression.c @@ -6,17 +6,16 @@ * LICENSE-examples file in the root directory of this source tree. */ - - #include // malloc, exit #include // printf #include // strerror #include // errno #include // stat +#define ZSTD_STATIC_LINKING_ONLY // ZSTD_findDecompressedSize #include // presumes zstd library is installed -static off_t fsize_X(const char *filename) +static off_t fsize_orDie(const char *filename) { struct stat st; if (stat(filename, &st) == 0) return st.st_size; @@ -25,7 +24,7 @@ static off_t fsize_X(const char *filename) exit(1); } -static FILE* fopen_X(const char *filename, const char *instruction) +static FILE* fopen_orDie(const char *filename, const char *instruction) { FILE* const inFile = fopen(filename, instruction); if (inFile) return inFile; @@ -34,7 +33,7 @@ static FILE* fopen_X(const char *filename, const char *instruction) exit(2); } -static void* malloc_X(size_t size) +static void* malloc_orDie(size_t size) { void* const buff = malloc(size); if (buff) return buff; @@ -43,11 +42,11 @@ static void* malloc_X(size_t size) exit(3); } -static void* loadFile_X(const char* fileName, size_t* size) +static void* loadFile_orDie(const char* fileName, size_t* size) { - off_t const buffSize = fsize_X(fileName); - FILE* const inFile = fopen_X(fileName, "rb"); - void* const buffer = malloc_X(buffSize); + off_t const buffSize = fsize_orDie(fileName); + FILE* const inFile = fopen_orDie(fileName, "rb"); + void* const buffer = malloc_orDie(buffSize); size_t const readSize = fread(buffer, 1, buffSize, inFile); if (readSize != (size_t)buffSize) { printf("fread: %s : %s \n", fileName, strerror(errno)); @@ -62,13 +61,13 @@ static void* loadFile_X(const char* fileName, size_t* size) static void decompress(const char* fname) { size_t cSize; - void* const cBuff = loadFile_X(fname, &cSize); + void* const cBuff = loadFile_orDie(fname, &cSize); unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize); if (rSize==0) { printf("%s : original size unknown. Use streaming decompression instead. \n", fname); exit(5); } - void* const rBuff = malloc_X((size_t)rSize); + void* const rBuff = malloc_orDie((size_t)rSize); size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize); diff --git a/lib/common/entropy_common.c b/lib/common/entropy_common.c index 83fd97154..72bc398da 100644 --- a/lib/common/entropy_common.c +++ b/lib/common/entropy_common.c @@ -43,6 +43,12 @@ #include "huf.h" +/*-**************************************** +* Version +******************************************/ +unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; } + + /*-**************************************** * FSE Error Management ******************************************/ @@ -62,8 +68,6 @@ const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } /*-************************************************************** * FSE NCount encoding-decoding ****************************************************************/ -static short FSE_abs(short a) { return (short)(a<0 ? -a : a); } - size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, const void* headerBuffer, size_t hbSize) { @@ -117,21 +121,21 @@ size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* t } else { bitStream >>= 2; } } - { short const max = (short)((2*threshold-1)-remaining); - short count; + { int const max = (2*threshold-1) - remaining; + int count; if ((bitStream & (threshold-1)) < (U32)max) { - count = (short)(bitStream & (threshold-1)); - bitCount += nbBits-1; + count = bitStream & (threshold-1); + bitCount += nbBits-1; } else { - count = (short)(bitStream & (2*threshold-1)); + count = bitStream & (2*threshold-1); if (count >= threshold) count -= max; - bitCount += nbBits; + bitCount += nbBits; } count--; /* extra accuracy */ - remaining -= FSE_abs(count); - normalizedCounter[charnum++] = count; + remaining -= count < 0 ? -count : count; /* -1 means +1 */ + normalizedCounter[charnum++] = (short)count; previous0 = !count; while (remaining < threshold) { nbBits--; diff --git a/lib/common/fse.h b/lib/common/fse.h index 8b07d184d..baac39032 100644 --- a/lib/common/fse.h +++ b/lib/common/fse.h @@ -45,6 +45,32 @@ extern "C" { #include /* size_t, ptrdiff_t */ +/*-***************************************** +* FSE_PUBLIC_API : control library symbols visibility +******************************************/ +#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) +# define FSE_PUBLIC_API __attribute__ ((visibility ("default"))) +#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ +# define FSE_PUBLIC_API __declspec(dllexport) +#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) +# define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define FSE_PUBLIC_API +#endif + +/*------ Version ------*/ +#define FSE_VERSION_MAJOR 0 +#define FSE_VERSION_MINOR 9 +#define FSE_VERSION_RELEASE 0 + +#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE +#define FSE_QUOTE(str) #str +#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str) +#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION) + +#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE) +FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ + /*-**************************************** * FSE simple functions ******************************************/ @@ -56,8 +82,8 @@ extern "C" { if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead. if FSE_isError(return), compression failed (more details using FSE_getErrorName()) */ -size_t FSE_compress(void* dst, size_t dstCapacity, - const void* src, size_t srcSize); +FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity, + const void* src, size_t srcSize); /*! FSE_decompress(): Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', @@ -69,18 +95,18 @@ size_t FSE_compress(void* dst, size_t dstCapacity, Why ? : making this distinction requires a header. Header management is intentionally delegated to the user layer, which can better manage special cases. */ -size_t FSE_decompress(void* dst, size_t dstCapacity, - const void* cSrc, size_t cSrcSize); +FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity, + const void* cSrc, size_t cSrcSize); /*-***************************************** * Tool functions ******************************************/ -size_t FSE_compressBound(size_t size); /* maximum compressed size */ +FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */ /* Error Management */ -unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ -const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ +FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ +FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ /*-***************************************** @@ -94,7 +120,7 @@ const char* FSE_getErrorName(size_t code); /* provides error code string (usef if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression. if FSE_isError(return), it's an error code. */ -size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); +FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); /*-***************************************** @@ -127,50 +153,50 @@ or to save and provide normalized distribution using external method. @return : the count of the most frequent symbol (which is not identified). if return == srcSize, there is only one symbol. Can also return an error code, which can be tested with FSE_isError(). */ -size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); +FSE_PUBLIC_API size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); /*! FSE_optimalTableLog(): dynamically downsize 'tableLog' when conditions are met. It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. @return : recommended tableLog (necessarily <= 'maxTableLog') */ -unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); +FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); /*! FSE_normalizeCount(): normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). @return : tableLog, or an errorCode, which can be tested using FSE_isError() */ -size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue); +FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue); /*! FSE_NCountWriteBound(): Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. Typically useful for allocation purpose. */ -size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); +FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); /*! FSE_writeNCount(): Compactly save 'normalizedCounter' into 'buffer'. @return : size of the compressed table, or an errorCode, which can be tested using FSE_isError(). */ -size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); +FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); /*! Constructor and Destructor of FSE_CTable. Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ -FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue); -void FSE_freeCTable (FSE_CTable* ct); +FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue); +FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct); /*! FSE_buildCTable(): Builds `ct`, which must be already allocated, using FSE_createCTable(). @return : 0, or an errorCode, which can be tested using FSE_isError() */ -size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); +FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); /*! FSE_compress_usingCTable(): Compress `src` using `ct` into `dst` which must be already allocated. @return : size of compressed data (<= `dstCapacity`), or 0 if compressed data could not fit into `dst`, or an errorCode, which can be tested using FSE_isError() */ -size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct); +FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct); /*! Tutorial : @@ -223,25 +249,25 @@ If there is an error, the function will return an ErrorCode (which can be tested @return : size read from 'rBuffer', or an errorCode, which can be tested using FSE_isError(). maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ -size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize); +FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize); /*! Constructor and Destructor of FSE_DTable. Note that its size depends on 'tableLog' */ typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ -FSE_DTable* FSE_createDTable(unsigned tableLog); -void FSE_freeDTable(FSE_DTable* dt); +FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog); +FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt); /*! FSE_buildDTable(): Builds 'dt', which must be already allocated, using FSE_createDTable(). return : 0, or an errorCode, which can be tested using FSE_isError() */ -size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); +FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); /*! FSE_decompress_usingDTable(): Decompress compressed source `cSrc` of size `cSrcSize` using `dt` into `dst` which must be already allocated. @return : size of regenerated data (necessarily <= `dstCapacity`), or an errorCode, which can be tested using FSE_isError() */ -size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt); +FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt); /*! Tutorial : diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c index 6627facfe..337b7a6ff 100644 --- a/lib/compress/fse_compress.c +++ b/lib/compress/fse_compress.c @@ -201,8 +201,6 @@ size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog) return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ } -static short FSE_abs(short a) { return (short)(a<0 ? -a : a); } - static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, unsigned writeIsSafe) @@ -258,16 +256,16 @@ static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, bitStream >>= 16; bitCount -= 16; } } - { short count = normalizedCounter[charnum++]; - const short max = (short)((2*threshold-1)-remaining); - remaining -= FSE_abs(count); - if (remaining<1) return ERROR(GENERIC); + { int count = normalizedCounter[charnum++]; + int const max = (2*threshold-1)-remaining; + remaining -= count < 0 ? -count : count; count++; /* +1 for extra accuracy */ if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ bitStream += count << bitCount; bitCount += nbBits; bitCount -= (count>=1; } if (bitCount>16) { From 887eaa9e21d5a35614ed35e1cae0025e5f071fa8 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Wed, 15 Feb 2017 16:43:45 -0800 Subject: [PATCH 079/223] Fix wildcopy overwriting data still in window --- lib/decompress/zstd_decompress.c | 2 +- lib/legacy/zstd_v06.c | 2 +- lib/legacy/zstd_v07.c | 2 +- tests/zstreamtest.c | 24 ++++++++++++++++++++++++ 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index b8670315c..52949be99 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -2260,7 +2260,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB /* Adapt buffer sizes to frame header instructions */ { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); - size_t const neededOutSize = zds->fParams.windowSize + blockSize; + size_t const neededOutSize = zds->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH; zds->blockSize = blockSize; if (zds->inBuffSize < blockSize) { ZSTD_free(zds->inBuff, zds->customMem); diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index 4c8f06823..1d65e8f7d 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -4108,7 +4108,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd, zbd->inBuff = (char*)malloc(blockSize); if (zbd->inBuff == NULL) return ERROR(memory_allocation); } - { size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize; + { size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize + WILDCOPY_OVERLENGTH; if (zbd->outBuffSize < neededOutSize) { free(zbd->outBuff); zbd->outBuffSize = neededOutSize; diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index 441e4bc39..c93a217f4 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -4483,7 +4483,7 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd, zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize); if (zbd->inBuff == NULL) return ERROR(memory_allocation); } - { size_t const neededOutSize = zbd->fParams.windowSize + blockSize; + { size_t const neededOutSize = zbd->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH; if (zbd->outBuffSize < neededOutSize) { zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); zbd->outBuffSize = neededOutSize; diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 5680d27c1..323a087ce 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -467,6 +467,30 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error; DISPLAYLEVEL(3, "OK \n"); + /* Overlen overwriting window data bug */ + DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++); + { const char* testCase = + "\x28\xB5\x2F\xFD\x04\x00\x4C\x00\x00\x10\x61\x61\x01\x00\xFC\x2A" + "\xC0\x02\x44\x00\x00\x08\x62\x01\x00\xFC\x2A\x10\x02\x00\x00\x00" + "\x4D\x00\x00\x00\x02\x40\x00\x01\x64\xE0\xE6\x19\xC1\xFB\x54\x9E"; + ZSTD_DStream* zds = ZSTD_createDStream(); + + ZSTD_initDStream(zds); + inBuff.src = testCase; + inBuff.size = 48; + inBuff.pos = 0; + outBuff.dst = decodedBuffer; + outBuff.size = CNBufferSize; + outBuff.pos = 0; + + while (inBuff.pos < inBuff.size) { + size_t const r = ZSTD_decompressStream(zds, &outBuff, &inBuff); + /* Bug will cause checksum to fail */ + if (ZSTD_isError(r)) goto _output_error; + } + } + DISPLAYLEVEL(3, "OK \n"); + _end: FUZ_freeDictionary(dictionary); ZSTD_freeCStream(zc); From e0d2a146d1ef7d5d357a3fabe2db5e6c06012784 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 16 Feb 2017 12:29:08 +0100 Subject: [PATCH 080/223] .travis.yml: detect "$TRAVIS_EVENT_TYPE" = "cron" --- .travis.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 885e4517a..b87d7d476 100644 --- a/.travis.yml +++ b/.travis.yml @@ -169,7 +169,13 @@ matrix: script: - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') - # dev => normal tests; other feature branches => short tests (number > 11) - - if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 11 ] || [ "$TRAVIS_BRANCH" = "dev" ] && [ "$TRAVIS_BRANCH" != "master" ]; then sh -c "$Cmd"; fi - # master => long tests, as this is the final step towards a Release - - if [ "$TRAVIS_BRANCH" = "master" ]; then FUZZERTEST=-T10mn sh -c "$Cmd"; fi + + # cron & master => long tests, as this is the final step towards a Release + - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "master" ]; then + FUZZERTEST=-T10mn sh -c "$Cmd" || travis_terminate 1; + else + # dev => normal tests; other feature branches => short tests (number > 11) + if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 11 ] || [ "$TRAVIS_BRANCH" = "dev" ]; then + sh -c "$Cmd" || travis_terminate 1; + fi + fi From b0511aeca541d4e274b33d58c379b1d793eef345 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 16 Feb 2017 12:33:25 +0100 Subject: [PATCH 081/223] fix travis.yml --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b87d7d476..82b2b03bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -170,11 +170,11 @@ matrix: script: - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') - # cron & master => long tests, as this is the final step towards a Release + # cron & master => long tests, as this is the final step towards a Release + # dev => normal tests; other feature branches => short tests (number > 11) - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "master" ]; then FUZZERTEST=-T10mn sh -c "$Cmd" || travis_terminate 1; else - # dev => normal tests; other feature branches => short tests (number > 11) if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 11 ] || [ "$TRAVIS_BRANCH" = "dev" ]; then sh -c "$Cmd" || travis_terminate 1; fi From f8a5749c224e678286de8da97e367617dfa661d3 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 16 Feb 2017 13:08:30 +0100 Subject: [PATCH 082/223] circle.yml: run only short tests --- circle.yml | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/circle.yml b/circle.yml index c189d3b65..2f531a91f 100644 --- a/circle.yml +++ b/circle.yml @@ -16,27 +16,31 @@ test: - gcc-5 -v; make gcc5test && make clean - gcc-6 -v; make gcc6test && make clean # Shorter tests - - make cmaketest && make clean - - make zlibwrapper && make clean - - make -C lib all && make clean - - make -C tests dll && make clean - - make -C tests test-symbols && make clean - - make -C tests test-zstd-nolegacy && make clean + - make cmaketest && make clean + - make -C lib all && make clean + - make -C tests dll && make clean + - make -C tests test-zstd && make clean + - make -C tests test-fullbench && make clean + - make -C tests test-fuzzer && make clean + - make -C tests test-zstream && make clean + - make -C tests test-invalidDictionaries && make clean + - make -C tests test-legacy && make clean + - make -C tests test-symbols && make clean - make -C tests test-longmatch && make clean - - pyenv global 3.4.4; make -C tests versionsTest && make clean - make -C programs zstd-small zstd-decompress zstd-compress && make -C programs clean - make travis-install && make clean # Longer tests - - make test && make clean - - gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean - - make usan && make clean - - make asan && make clean - - make asan32 && make clean + #- make -C tests test-zstd-nolegacy && make clean + #- pyenv global 3.4.4; make -C tests versionsTest && make clean + #- make zlibwrapper && make clean + #- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean + #- make uasan && make clean + #- make asan32 && make clean # Valgrind tests - - CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean - - make -C tests valgrindTest && make clean + #- CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean + #- make -C tests valgrindTest && make clean # ARM, AArch64, PowerPC, PowerPC64 tests - - make ppctest && make clean - - make ppc64test && make clean - - make armtest && make clean - - make aarch64test && make clean + #- make ppctest && make clean + #- make ppc64test && make clean + #- make armtest && make clean + #- make aarch64test && make clean From 9e97a8a45a395481e30c9f7110245aa7a08dc0e1 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 16 Feb 2017 13:36:12 +0100 Subject: [PATCH 083/223] check $CIRCLE_NODE_INDEX --- circle.yml | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/circle.yml b/circle.yml index 2f531a91f..c04dd46b3 100644 --- a/circle.yml +++ b/circle.yml @@ -8,27 +8,29 @@ dependencies: test: override: # Tests compilers and C standards - - clang -v; make clangtest && make clean - - g++ -v; make gpptest && make clean - - gcc -v; make gnu90test && make clean - - gcc -v; make c99test && make clean - - gcc -v; make gnu99test && make clean - - gcc-5 -v; make gcc5test && make clean - - gcc-6 -v; make gcc6test && make clean + - [[ "$CIRCLE_NODE_INDEX" == "0" ]] && clang -v && make clangtest && make clean + parallel: true + #- g++ -v; make gpptest && make clean + #- gcc -v; make gnu90test && make clean + #- gcc -v; make c99test && make clean + #- gcc -v; make gnu99test && make clean + #- gcc-5 -v; make gcc5test && make clean + #- gcc-6 -v; make gcc6test && make clean # Shorter tests - - make cmaketest && make clean - - make -C lib all && make clean - - make -C tests dll && make clean - - make -C tests test-zstd && make clean - - make -C tests test-fullbench && make clean - - make -C tests test-fuzzer && make clean - - make -C tests test-zstream && make clean - - make -C tests test-invalidDictionaries && make clean - - make -C tests test-legacy && make clean - - make -C tests test-symbols && make clean - - make -C tests test-longmatch && make clean - - make -C programs zstd-small zstd-decompress zstd-compress && make -C programs clean - - make travis-install && make clean + - if [[ "$CIRCLE_NODE_TOTAL" < "2" ] || [ "$CIRCLE_NODE_INDEX" == "1" ]] && make cmaketest && make clean + parallel: true + #- make -C lib all && make clean + #- make -C tests dll && make clean + #- make -C tests test-zstd && make clean + #- make -C tests test-fullbench && make clean + #- make -C tests test-fuzzer && make clean + #- make -C tests test-zstream && make clean + #- make -C tests test-invalidDictionaries && make clean + #- make -C tests test-legacy && make clean + #- make -C tests test-symbols && make clean + #- make -C tests test-longmatch && make clean + #- make -C programs zstd-small zstd-decompress zstd-compress && make -C programs clean + #- make travis-install && make clean # Longer tests #- make -C tests test-zstd-nolegacy && make clean #- pyenv global 3.4.4; make -C tests versionsTest && make clean From d3ff834562a87475618ea1069bf394c414807375 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 16 Feb 2017 13:45:40 +0100 Subject: [PATCH 084/223] check CIRCLE_NODE_TOTAL --- circle.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/circle.yml b/circle.yml index c04dd46b3..716507e6b 100644 --- a/circle.yml +++ b/circle.yml @@ -8,18 +8,22 @@ dependencies: test: override: # Tests compilers and C standards - - [[ "$CIRCLE_NODE_INDEX" == "0" ]] && clang -v && make clangtest && make clean + - | + [[ "$CIRCLE_NODE_INDEX" == "0" ]] && clang -v && make clangtest && make clean + parallel: true + - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then g++ -v; make gpptest && make clean; fi parallel: true - #- g++ -v; make gpptest && make clean #- gcc -v; make gnu90test && make clean #- gcc -v; make c99test && make clean #- gcc -v; make gnu99test && make clean #- gcc-5 -v; make gcc5test && make clean #- gcc-6 -v; make gcc6test && make clean # Shorter tests - - if [[ "$CIRCLE_NODE_TOTAL" < "2" ] || [ "$CIRCLE_NODE_INDEX" == "1" ]] && make cmaketest && make clean + - | + [[ "$CIRCLE_NODE_TOTAL" < "2" ] || [ "$CIRCLE_NODE_INDEX" == "1" ]] && make cmaketest && make clean + parallel: true + - if [[ "$CIRCLE_NODE_TOTAL" < "2" ] || [ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C lib all && make clean; fi parallel: true - #- make -C lib all && make clean #- make -C tests dll && make clean #- make -C tests test-zstd && make clean #- make -C tests test-fullbench && make clean From cb7694486164b2af8eac47c87f4e8742545ff035 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 16 Feb 2017 13:51:21 +0100 Subject: [PATCH 085/223] final colon --- circle.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/circle.yml b/circle.yml index 716507e6b..c9245d3da 100644 --- a/circle.yml +++ b/circle.yml @@ -9,9 +9,9 @@ test: override: # Tests compilers and C standards - | - [[ "$CIRCLE_NODE_INDEX" == "0" ]] && clang -v && make clangtest && make clean + [[ "$CIRCLE_NODE_INDEX" == "0" ]] && clang -v && make clangtest && make clean: parallel: true - - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then g++ -v; make gpptest && make clean; fi + - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then g++ -v; make gpptest && make clean; fi: parallel: true #- gcc -v; make gnu90test && make clean #- gcc -v; make c99test && make clean @@ -20,9 +20,9 @@ test: #- gcc-6 -v; make gcc6test && make clean # Shorter tests - | - [[ "$CIRCLE_NODE_TOTAL" < "2" ] || [ "$CIRCLE_NODE_INDEX" == "1" ]] && make cmaketest && make clean + [[ "$CIRCLE_NODE_TOTAL" < "2" ] || [ "$CIRCLE_NODE_INDEX" == "1" ]] && make cmaketest && make clean: parallel: true - - if [[ "$CIRCLE_NODE_TOTAL" < "2" ] || [ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C lib all && make clean; fi + - if [[ "$CIRCLE_NODE_TOTAL" < "2" ] || [ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C lib all && make clean; fi: parallel: true #- make -C tests dll && make clean #- make -C tests test-zstd && make clean From fa492a3eca53ece95dab01cecdfa6d598be81837 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 16 Feb 2017 14:39:21 +0100 Subject: [PATCH 086/223] Tests for thread 1 (when CIRCLE_NODE_TOTAL=1) or thread 2 --- circle.yml | 54 ++++++++++++++++++++++++++---------------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/circle.yml b/circle.yml index c9245d3da..aaf25617d 100644 --- a/circle.yml +++ b/circle.yml @@ -5,36 +5,34 @@ dependencies: - sudo apt-get -y install qemu-system-arm gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross - sudo apt-get -y install libc6-dev-i386 clang gcc-5 gcc-6 valgrind -test: - override: - # Tests compilers and C standards + # use default "parallel: true" for commands in the machine, checkout, dependencies and database build phase + post: + # Tests for thread 1 - | - [[ "$CIRCLE_NODE_INDEX" == "0" ]] && clang -v && make clangtest && make clean: - parallel: true - - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then g++ -v; make gpptest && make clean; fi: - parallel: true - #- gcc -v; make gnu90test && make clean - #- gcc -v; make c99test && make clean - #- gcc -v; make gnu99test && make clean - #- gcc-5 -v; make gcc5test && make clean - #- gcc-6 -v; make gcc6test && make clean - # Shorter tests + [[ "$CIRCLE_NODE_INDEX" == "0" ]] && clang -v && make clangtest && make clean + - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then g++ -v; make gpptest && make clean; fi + - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make gnu90test && make clean; fi + - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make c99test && make clean; fi + - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make gnu99test && make clean; fi + - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc-5 -v; make gcc5test && make clean; fi + - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc-6 -v; make gcc6test && make clean; fi + - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make cmaketest && make clean; fi + - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make travis-install && make clean; fi + - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make lib && make clean; fi + - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make -C programs zstd-small zstd-decompress zstd-compress && make clean; fi + + # Tests for thread 1 (when CIRCLE_NODE_TOTAL=1) or thread 2 - | - [[ "$CIRCLE_NODE_TOTAL" < "2" ] || [ "$CIRCLE_NODE_INDEX" == "1" ]] && make cmaketest && make clean: - parallel: true - - if [[ "$CIRCLE_NODE_TOTAL" < "2" ] || [ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C lib all && make clean; fi: - parallel: true - #- make -C tests dll && make clean - #- make -C tests test-zstd && make clean - #- make -C tests test-fullbench && make clean - #- make -C tests test-fuzzer && make clean - #- make -C tests test-zstream && make clean - #- make -C tests test-invalidDictionaries && make clean - #- make -C tests test-legacy && make clean - #- make -C tests test-symbols && make clean - #- make -C tests test-longmatch && make clean - #- make -C programs zstd-small zstd-decompress zstd-compress && make -C programs clean - #- make travis-install && make clean + [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]] && make -C tests test-zstd && make clean + - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-fullbench && make clean; fi + - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-fuzzer && make clean; fi + - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-zstream && make clean; fi + - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy && make clean; fi + - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-symbols && make clean; fi + - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-longmatch && make clean; fi + - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-invalidDictionaries && make clean; fi + - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests dll && make clean; fi + # Longer tests #- make -C tests test-zstd-nolegacy && make clean #- pyenv global 3.4.4; make -C tests versionsTest && make clean From 9a0161d376585c5b237d2f651fb8618a6f71e364 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 16 Feb 2017 15:13:33 +0100 Subject: [PATCH 087/223] imporved test-zstd --- circle.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/circle.yml b/circle.yml index aaf25617d..1723628a1 100644 --- a/circle.yml +++ b/circle.yml @@ -22,8 +22,7 @@ dependencies: - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make -C programs zstd-small zstd-decompress zstd-compress && make clean; fi # Tests for thread 1 (when CIRCLE_NODE_TOTAL=1) or thread 2 - - | - [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]] && make -C tests test-zstd && make clean + - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-zstd && make clean; fi - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-fullbench && make clean; fi - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-fuzzer && make clean; fi - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-zstream && make clean; fi From 6b64abb2870aa19691b7fe97ffdafcd9c4aeb244 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 16 Feb 2017 15:28:08 +0100 Subject: [PATCH 088/223] improved clangtest --- circle.yml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/circle.yml b/circle.yml index 1723628a1..feb0fed0a 100644 --- a/circle.yml +++ b/circle.yml @@ -8,17 +8,16 @@ dependencies: # use default "parallel: true" for commands in the machine, checkout, dependencies and database build phase post: # Tests for thread 1 - - | - [[ "$CIRCLE_NODE_INDEX" == "0" ]] && clang -v && make clangtest && make clean - - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then g++ -v; make gpptest && make clean; fi - - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make gnu90test && make clean; fi - - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make c99test && make clean; fi - - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make gnu99test && make clean; fi - - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc-5 -v; make gcc5test && make clean; fi - - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc-6 -v; make gcc6test && make clean; fi - - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make cmaketest && make clean; fi - - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make travis-install && make clean; fi - - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make lib && make clean; fi + - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then clang -v; make clangtest && make clean; fi + - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then g++ -v; make gpptest && make clean; fi + - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make gnu90test && make clean; fi + - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make c99test && make clean; fi + - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make gnu99test && make clean; fi + - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc-5 -v; make gcc5test && make clean; fi + - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc-6 -v; make gcc6test && make clean; fi + - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make cmaketest && make clean; fi + - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make travis-install && make clean; fi + - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make lib && make clean; fi - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make -C programs zstd-small zstd-decompress zstd-compress && make clean; fi # Tests for thread 1 (when CIRCLE_NODE_TOTAL=1) or thread 2 From 21d9022b8896991f2f9140fce8d6b40ee26da018 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 16 Feb 2017 15:59:00 +0100 Subject: [PATCH 089/223] two groups of tests --- circle.yml | 48 +++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/circle.yml b/circle.yml index feb0fed0a..bb2a35a4b 100644 --- a/circle.yml +++ b/circle.yml @@ -7,29 +7,31 @@ dependencies: # use default "parallel: true" for commands in the machine, checkout, dependencies and database build phase post: - # Tests for thread 1 - - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then clang -v; make clangtest && make clean; fi - - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then g++ -v; make gpptest && make clean; fi - - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make gnu90test && make clean; fi - - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make c99test && make clean; fi - - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make gnu99test && make clean; fi - - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc-5 -v; make gcc5test && make clean; fi - - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc-6 -v; make gcc6test && make clean; fi - - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make cmaketest && make clean; fi - - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make travis-install && make clean; fi - - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make lib && make clean; fi - - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make -C programs zstd-small zstd-decompress zstd-compress && make clean; fi - - # Tests for thread 1 (when CIRCLE_NODE_TOTAL=1) or thread 2 - - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-zstd && make clean; fi - - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-fullbench && make clean; fi - - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-fuzzer && make clean; fi - - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-zstream && make clean; fi - - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy && make clean; fi - - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-symbols && make clean; fi - - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-longmatch && make clean; fi - - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-invalidDictionaries && make clean; fi - - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests dll && make clean; fi + - | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then + clang -v; make clangtest + g++ -v; make gpptest + gcc -v; make gnu90test + gcc -v; make c99test + gcc -v; make gnu99test + make gcc5test + gcc-6 -v; make gcc6test + make cmaketest + make travis-install + make lib + make -C programs zstd-small zstd-decompress zstd-compress; + fi + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then + make -C tests test-zstd + make -C tests test-fullbench + make -C tests test-fuzzer + make -C tests test-zstream + make -C tests test-legacy + make -C tests test-symbols + make -C tests test-longmatch + make -C tests test-invalidDictionaries + make -C tests dll + fi # Longer tests #- make -C tests test-zstd-nolegacy && make clean From 84452ca29faf49c211c0137ae30cf1af278d170c Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 16 Feb 2017 16:27:40 +0100 Subject: [PATCH 090/223] more balanced tests --- circle.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/circle.yml b/circle.yml index bb2a35a4b..dc9ae7f1f 100644 --- a/circle.yml +++ b/circle.yml @@ -14,17 +14,17 @@ dependencies: gcc -v; make gnu90test gcc -v; make c99test gcc -v; make gnu99test - make gcc5test + gcc-5 -v; make gcc5test gcc-6 -v; make gcc6test make cmaketest make travis-install make lib - make -C programs zstd-small zstd-decompress zstd-compress; + make -C programs zstd-small zstd-decompress zstd-compress + make -C tests test-fuzzer fi if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-zstd make -C tests test-fullbench - make -C tests test-fuzzer make -C tests test-zstream make -C tests test-legacy make -C tests test-symbols @@ -33,6 +33,10 @@ dependencies: make -C tests dll fi +test: + override: + - echo Circle CI tests finished + # Longer tests #- make -C tests test-zstd-nolegacy && make clean #- pyenv global 3.4.4; make -C tests versionsTest && make clean From 6babbff58dbcd61e2b7268c32bacbd16681b77e9 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 16 Feb 2017 17:52:49 +0100 Subject: [PATCH 091/223] move MOREFLAGS to circle.yml --- Makefile | 2 +- circle.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3ce12e67d..d86db7cb3 100644 --- a/Makefile +++ b/Makefile @@ -138,7 +138,7 @@ msan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer" # datagen.c fails this test for no obvious reason asan32: clean - $(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu" + $(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address" uasan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined" diff --git a/circle.yml b/circle.yml index dc9ae7f1f..02a5dd6cc 100644 --- a/circle.yml +++ b/circle.yml @@ -44,6 +44,7 @@ test: #- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean #- make uasan && make clean #- make asan32 && make clean + #- make -C tests test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu" # Valgrind tests #- CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean #- make -C tests valgrindTest && make clean From 40dadd65cc118781c70595059a531a5689780051 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 16 Feb 2017 18:19:36 +0100 Subject: [PATCH 092/223] join tests into pairs --- circle.yml | 50 ++++++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/circle.yml b/circle.yml index 02a5dd6cc..5af1bd24e 100644 --- a/circle.yml +++ b/circle.yml @@ -8,30 +8,32 @@ dependencies: # use default "parallel: true" for commands in the machine, checkout, dependencies and database build phase post: - | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then - clang -v; make clangtest - g++ -v; make gpptest - gcc -v; make gnu90test - gcc -v; make c99test - gcc -v; make gnu99test - gcc-5 -v; make gcc5test - gcc-6 -v; make gcc6test - make cmaketest - make travis-install - make lib - make -C programs zstd-small zstd-decompress zstd-compress - make -C tests test-fuzzer - fi - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then - make -C tests test-zstd - make -C tests test-fullbench - make -C tests test-zstream - make -C tests test-legacy - make -C tests test-symbols - make -C tests test-longmatch - make -C tests test-invalidDictionaries - make -C tests dll - fi + if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make cmaketest && make clean; fi + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-invalidDictionaries && make clean; fi + - | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then g++ -v; make gpptest && make clean; fi + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy && make clean; fi + - | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make gnu90test && make clean; fi + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-symbols && make clean; fi + - | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make c99test && make clean; fi + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-longmatch && make clean; fi + - | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make gnu99test && make clean; fi + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests dll && make clean; fi + - | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then clang -v; make clangtest && make clean; fi + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C programs zstd-small zstd-decompress zstd-compress && make clean lib && make clean; fi + - | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then travis-install && make clean; fi + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-fullbench && make clean; fi + - | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc-5 -v; make gcc5test && gcc-6 -v && make gcc6test && make clean; fi + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-zstream && make clean; fi + - | + if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make -C tests test-zstd && make clean; fi + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-fuzzer && make clean; fi test: override: From c1f25eaec0f8fde1782f55995f7d8095fdc931e2 Mon Sep 17 00:00:00 2001 From: Dima Krasner Date: Thu, 16 Feb 2017 19:35:56 +0200 Subject: [PATCH 093/223] Added a Meson project --- build/README.md | 1 + build/meson/meson.build | 70 +++++++++++++++++++++++++++++++++++ build/meson/meson_options.txt | 1 + 3 files changed, 72 insertions(+) create mode 100644 build/meson/meson.build create mode 100644 build/meson/meson_options.txt diff --git a/build/README.md b/build/README.md index c4abe9efd..ca88e5cfd 100644 --- a/build/README.md +++ b/build/README.md @@ -5,6 +5,7 @@ Projects for various integrated development environments (IDE) The following projects are included with the zstd distribution: - `cmake` - CMake project contributed by Artyom Dymchenko +- `meson` - Meson project contributed by Dima Krasner - `VS2005` - Visual Studio 2005 project - `VS2008` - Visual Studio 2008 project - `VS2010` - Visual Studio 2010 project (which also works well with Visual Studio 2012, 2013, 2015) diff --git a/build/meson/meson.build b/build/meson/meson.build new file mode 100644 index 000000000..b17128bdd --- /dev/null +++ b/build/meson/meson.build @@ -0,0 +1,70 @@ +project('zstd', 'c', license: 'BSD') + +libm = meson.get_compiler('c').find_library('m', required: true) + +lib_dir = join_paths(meson.source_root(), '..', '..', 'lib') +common_dir = join_paths(lib_dir, 'common') +compress_dir = join_paths(lib_dir, 'compress') +decompress_dir = join_paths(lib_dir, 'decompress') +dictbuilder_dir = join_paths(lib_dir, 'dictBuilder') +deprecated_dir = join_paths(lib_dir, 'deprecated') + +lib_srcs = [join_paths(common_dir, 'entropy_common.c'), join_paths(common_dir, 'fse_decompress.c'), join_paths(common_dir, 'threading.c'), join_paths(common_dir, 'pool.c'), join_paths(common_dir, 'zstd_common.c'), join_paths(common_dir, 'error_private.c'), join_paths(common_dir, 'xxhash.c'), join_paths(compress_dir, 'fse_compress.c'), join_paths(compress_dir, 'huf_compress.c'), join_paths(compress_dir, 'zstd_compress.c'), join_paths(compress_dir, 'zstdmt_compress.c'), join_paths(decompress_dir, 'huf_decompress.c'), join_paths(decompress_dir, 'zstd_decompress.c'), join_paths(dictbuilder_dir, 'cover.c'), join_paths(dictbuilder_dir, 'divsufsort.c'), join_paths(dictbuilder_dir, 'zdict.c'), join_paths(deprecated_dir, 'zbuff_common.c'), join_paths(deprecated_dir, 'zbuff_compress.c'), join_paths(deprecated_dir, 'zbuff_decompress.c')] + +libzstd_includes = [include_directories(common_dir, dictbuilder_dir, compress_dir, lib_dir)] + +if get_option('legacy_support') + message('Enabling legacy support') + libzstd_cflags = ['-DZSTD_LEGACY_SUPPORT=1'] + + legacy_dir = join_paths(lib_dir, 'legacy') + libzstd_includes += [include_directories(legacy_dir)] + lib_srcs += [join_paths(legacy_dir, 'zstd_v01.c'), join_paths(legacy_dir, 'zstd_v02.c'), join_paths(legacy_dir, 'zstd_v03.c'), join_paths(legacy_dir, 'zstd_v04.c'), join_paths(legacy_dir, 'zstd_v05.c'), join_paths(legacy_dir, 'zstd_v06.c'), join_paths(legacy_dir, 'zstd_v07.c')] +else + libzstd_cflags = [] +endif + +libzstd = library('zstd', + lib_srcs, + include_directories: libzstd_includes, + c_args: libzstd_cflags, + install: true) + +programs_dir = join_paths(meson.source_root(), '..', '..', 'programs') + +zstd = executable('zstd', + join_paths(programs_dir, 'bench.c'), join_paths(programs_dir, 'datagen.c'), join_paths(programs_dir, 'dibio.c'), join_paths(programs_dir, 'fileio.c'), join_paths(programs_dir, 'zstdcli.c'), + include_directories: libzstd_includes, + c_args: ['-DZSTD_NODICT', '-DZSTD_NOBENCH'], + link_with: libzstd, + install: true) + +tests_dir = join_paths(meson.source_root(), '..', '..', 'tests') +datagen_c = join_paths(programs_dir, 'datagen.c') +test_includes = libzstd_includes + [include_directories(programs_dir)] + +fullbench = executable('fullbench', + datagen_c, join_paths(tests_dir, 'fullbench.c'), + include_directories: test_includes, + link_with: libzstd) +test('fullbench', fullbench) + +fuzzer = executable('fuzzer', + datagen_c, join_paths(tests_dir, 'fuzzer.c'), + include_directories: test_includes, + link_with: libzstd) +test('fuzzer', fuzzer) + +if target_machine.system() != 'windows' + paramgrill = executable('paramgrill', + datagen_c, join_paths(tests_dir, 'paramgrill.c'), + include_directories: test_includes, + link_with: libzstd, + dependencies: libm) + test('paramgrill', paramgrill) + + datagen = executable('datagen', + datagen_c, join_paths(tests_dir, 'datagencli.c'), + include_directories: test_includes, + link_with: libzstd) +endif diff --git a/build/meson/meson_options.txt b/build/meson/meson_options.txt new file mode 100644 index 000000000..0b3d62a39 --- /dev/null +++ b/build/meson/meson_options.txt @@ -0,0 +1 @@ +option('legacy_support', type: 'boolean', value: false) From 6f508421ebd867f81ee21bd0a2b65df11619d6b2 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 16 Feb 2017 18:45:17 +0100 Subject: [PATCH 094/223] faster start of containers --- circle.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/circle.yml b/circle.yml index 5af1bd24e..de983f24f 100644 --- a/circle.yml +++ b/circle.yml @@ -1,9 +1,9 @@ dependencies: override: - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update - - sudo apt-get -y install qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu - - sudo apt-get -y install qemu-system-arm gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross - - sudo apt-get -y install libc6-dev-i386 clang gcc-5 gcc-6 valgrind + #- sudo apt-get -y install qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu valgrind + #- sudo apt-get -y install qemu-system-arm gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross + - sudo apt-get -y install libc6-dev-i386 clang gcc-5 gcc-6 # use default "parallel: true" for commands in the machine, checkout, dependencies and database build phase post: @@ -24,7 +24,7 @@ dependencies: if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests dll && make clean; fi - | if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then clang -v; make clangtest && make clean; fi - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C programs zstd-small zstd-decompress zstd-compress && make clean lib && make clean; fi + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C programs zstd-small zstd-decompress zstd-compress zstd32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean lib && make clean; fi - | if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then travis-install && make clean; fi if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-fullbench && make clean; fi From 7a8811f3f56c84bb274615e7fc5568f571c47e23 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Thu, 16 Feb 2017 19:04:22 +0100 Subject: [PATCH 095/223] circle.yml: make travis-install --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index de983f24f..d8b033597 100644 --- a/circle.yml +++ b/circle.yml @@ -26,7 +26,7 @@ dependencies: if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then clang -v; make clangtest && make clean; fi if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C programs zstd-small zstd-decompress zstd-compress zstd32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean lib && make clean; fi - | - if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then travis-install && make clean; fi + if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make travis-install && make clean; fi if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-fullbench && make clean; fi - | if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc-5 -v; make gcc5test && gcc-6 -v && make gcc6test && make clean; fi From 6b010dec80124576e50635de96cc8266d7bd1cde Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Thu, 16 Feb 2017 12:05:40 -0800 Subject: [PATCH 096/223] execSequence copies up to 2*WILDCOPY_OVERLENGTH extra --- lib/decompress/zstd_decompress.c | 2 +- lib/legacy/zstd_v06.c | 2 +- lib/legacy/zstd_v07.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 52949be99..404d0b83d 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -2260,7 +2260,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB /* Adapt buffer sizes to frame header instructions */ { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); - size_t const neededOutSize = zds->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH; + size_t const neededOutSize = zds->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2; zds->blockSize = blockSize; if (zds->inBuffSize < blockSize) { ZSTD_free(zds->inBuff, zds->customMem); diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c index 1d65e8f7d..f586db226 100644 --- a/lib/legacy/zstd_v06.c +++ b/lib/legacy/zstd_v06.c @@ -4108,7 +4108,7 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd, zbd->inBuff = (char*)malloc(blockSize); if (zbd->inBuff == NULL) return ERROR(memory_allocation); } - { size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize + WILDCOPY_OVERLENGTH; + { size_t const neededOutSize = ((size_t)1 << zbd->fParams.windowLog) + blockSize + WILDCOPY_OVERLENGTH * 2; if (zbd->outBuffSize < neededOutSize) { free(zbd->outBuff); zbd->outBuffSize = neededOutSize; diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c index c93a217f4..07099d5ab 100644 --- a/lib/legacy/zstd_v07.c +++ b/lib/legacy/zstd_v07.c @@ -4483,7 +4483,7 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd, zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize); if (zbd->inBuff == NULL) return ERROR(memory_allocation); } - { size_t const neededOutSize = zbd->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH; + { size_t const neededOutSize = zbd->fParams.windowSize + blockSize + WILDCOPY_OVERLENGTH * 2; if (zbd->outBuffSize < neededOutSize) { zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); zbd->outBuffSize = neededOutSize; From 0ed3901b05ff099e485107caed272576732ce424 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Thu, 16 Feb 2017 13:29:47 -0800 Subject: [PATCH 097/223] Update overlength match test case --- tests/zstreamtest.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 323a087ce..9a9fed98d 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -469,15 +469,23 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* Overlen overwriting window data bug */ DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++); - { const char* testCase = - "\x28\xB5\x2F\xFD\x04\x00\x4C\x00\x00\x10\x61\x61\x01\x00\xFC\x2A" - "\xC0\x02\x44\x00\x00\x08\x62\x01\x00\xFC\x2A\x10\x02\x00\x00\x00" - "\x4D\x00\x00\x00\x02\x40\x00\x01\x64\xE0\xE6\x19\xC1\xFB\x54\x9E"; + { /* This test has a window size of 1024 bytes and consists of 3 blocks: + 1. 'a' repeated 517 times + 2. 'b' repeated 516 times + 3. a compressed block with no literals and 3 sequence commands: + litlength = 0, offset = 24, match length = 24 + litlength = 0, offset = 24, match length = 3 (this one creates an overlength write of length 2*WILDCOPY_OVERLENGTH - 3) + litlength = 0, offset = 1021, match length = 3 (this one will try to read from overwritten data if the buffer is too small) */ + + const char* testCase = + "\x28\xB5\x2F\xFD\x04\x00\x4C\x00\x00\x10\x61\x61\x01\x00\x00\x2A" + "\x80\x05\x44\x00\x00\x08\x62\x01\x00\x00\x2A\x20\x04\x5D\x00\x00" + "\x00\x03\x40\x00\x00\x64\x60\x27\xB0\xE0\x0C\x67\x62\xCE\xE0"; ZSTD_DStream* zds = ZSTD_createDStream(); ZSTD_initDStream(zds); inBuff.src = testCase; - inBuff.size = 48; + inBuff.size = 47; inBuff.pos = 0; outBuff.dst = decodedBuffer; outBuff.size = CNBufferSize; From 65add2934a0c9f0e83b82b060121542c48286b3f Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Fri, 17 Feb 2017 07:54:37 +0100 Subject: [PATCH 098/223] circle.yml: set FUZZERTEST=-T4mn --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index d8b033597..69c988545 100644 --- a/circle.yml +++ b/circle.yml @@ -33,7 +33,7 @@ dependencies: if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-zstream && make clean; fi - | if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make -C tests test-zstd && make clean; fi - if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-fuzzer && make clean; fi + if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-fuzzer FUZZERTEST=-T4mn && make clean; fi test: override: From faaded197611de64be10fd6e53db192e5d0e90f8 Mon Sep 17 00:00:00 2001 From: Dima Krasner Date: Fri, 17 Feb 2017 12:27:13 +0200 Subject: [PATCH 099/223] Added multi-threaded library support --- build/meson/meson.build | 14 +++++++++++--- build/meson/meson_options.txt | 1 + 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/build/meson/meson.build b/build/meson/meson.build index b17128bdd..0f9d31176 100644 --- a/build/meson/meson.build +++ b/build/meson/meson.build @@ -9,7 +9,7 @@ decompress_dir = join_paths(lib_dir, 'decompress') dictbuilder_dir = join_paths(lib_dir, 'dictBuilder') deprecated_dir = join_paths(lib_dir, 'deprecated') -lib_srcs = [join_paths(common_dir, 'entropy_common.c'), join_paths(common_dir, 'fse_decompress.c'), join_paths(common_dir, 'threading.c'), join_paths(common_dir, 'pool.c'), join_paths(common_dir, 'zstd_common.c'), join_paths(common_dir, 'error_private.c'), join_paths(common_dir, 'xxhash.c'), join_paths(compress_dir, 'fse_compress.c'), join_paths(compress_dir, 'huf_compress.c'), join_paths(compress_dir, 'zstd_compress.c'), join_paths(compress_dir, 'zstdmt_compress.c'), join_paths(decompress_dir, 'huf_decompress.c'), join_paths(decompress_dir, 'zstd_decompress.c'), join_paths(dictbuilder_dir, 'cover.c'), join_paths(dictbuilder_dir, 'divsufsort.c'), join_paths(dictbuilder_dir, 'zdict.c'), join_paths(deprecated_dir, 'zbuff_common.c'), join_paths(deprecated_dir, 'zbuff_compress.c'), join_paths(deprecated_dir, 'zbuff_decompress.c')] +libzstd_srcs = [join_paths(common_dir, 'entropy_common.c'), join_paths(common_dir, 'fse_decompress.c'), join_paths(common_dir, 'threading.c'), join_paths(common_dir, 'pool.c'), join_paths(common_dir, 'zstd_common.c'), join_paths(common_dir, 'error_private.c'), join_paths(common_dir, 'xxhash.c'), join_paths(compress_dir, 'fse_compress.c'), join_paths(compress_dir, 'huf_compress.c'), join_paths(compress_dir, 'zstd_compress.c'), join_paths(compress_dir, 'zstdmt_compress.c'), join_paths(decompress_dir, 'huf_decompress.c'), join_paths(decompress_dir, 'zstd_decompress.c'), join_paths(dictbuilder_dir, 'cover.c'), join_paths(dictbuilder_dir, 'divsufsort.c'), join_paths(dictbuilder_dir, 'zdict.c'), join_paths(deprecated_dir, 'zbuff_common.c'), join_paths(deprecated_dir, 'zbuff_compress.c'), join_paths(deprecated_dir, 'zbuff_decompress.c')] libzstd_includes = [include_directories(common_dir, dictbuilder_dir, compress_dir, lib_dir)] @@ -19,15 +19,23 @@ if get_option('legacy_support') legacy_dir = join_paths(lib_dir, 'legacy') libzstd_includes += [include_directories(legacy_dir)] - lib_srcs += [join_paths(legacy_dir, 'zstd_v01.c'), join_paths(legacy_dir, 'zstd_v02.c'), join_paths(legacy_dir, 'zstd_v03.c'), join_paths(legacy_dir, 'zstd_v04.c'), join_paths(legacy_dir, 'zstd_v05.c'), join_paths(legacy_dir, 'zstd_v06.c'), join_paths(legacy_dir, 'zstd_v07.c')] + libzstd_srcs += [join_paths(legacy_dir, 'zstd_v01.c'), join_paths(legacy_dir, 'zstd_v02.c'), join_paths(legacy_dir, 'zstd_v03.c'), join_paths(legacy_dir, 'zstd_v04.c'), join_paths(legacy_dir, 'zstd_v05.c'), join_paths(legacy_dir, 'zstd_v06.c'), join_paths(legacy_dir, 'zstd_v07.c')] else libzstd_cflags = [] endif +if get_option('multithread') + add_global_arguments('-DZSTD_MULTITHREAD', language: 'c') + libzstd_deps = [dependency('threads')] +else + libzstd_deps = [] +endif + libzstd = library('zstd', - lib_srcs, + libzstd_srcs, include_directories: libzstd_includes, c_args: libzstd_cflags, + dependencies: libzstd_deps, install: true) programs_dir = join_paths(meson.source_root(), '..', '..', 'programs') diff --git a/build/meson/meson_options.txt b/build/meson/meson_options.txt index 0b3d62a39..0a12f43e1 100644 --- a/build/meson/meson_options.txt +++ b/build/meson/meson_options.txt @@ -1 +1,2 @@ +option('multithread', type: 'boolean', value: false) option('legacy_support', type: 'boolean', value: false) From da145123c5f90586cd42973e92efbe9fcb0debf6 Mon Sep 17 00:00:00 2001 From: Dima Krasner Date: Fri, 17 Feb 2017 12:29:23 +0200 Subject: [PATCH 100/223] Updated the README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 6336d9006..7782972db 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,10 @@ A `cmake` project generator is provided within `build/cmake`. It can generate Makefiles or other build scripts to create `zstd` binary, and `libzstd` dynamic and static libraries. +#### Meson + +A Meson project is provided within `build/meson`. + #### Visual (Windows) Going into `build` directory, you will find additional possibilities : From 4c05b09f27041d177623b640950123936c4e1345 Mon Sep 17 00:00:00 2001 From: Dima Krasner Date: Fri, 17 Feb 2017 12:32:16 +0200 Subject: [PATCH 101/223] Added a message when multhread=true --- build/meson/meson.build | 1 + 1 file changed, 1 insertion(+) diff --git a/build/meson/meson.build b/build/meson/meson.build index 0f9d31176..369461335 100644 --- a/build/meson/meson.build +++ b/build/meson/meson.build @@ -25,6 +25,7 @@ else endif if get_option('multithread') + message('Enabling multi-threading support') add_global_arguments('-DZSTD_MULTITHREAD', language: 'c') libzstd_deps = [dependency('threads')] else From 042419ec2acb8d6bcf42cf337ae54c966a8e576d Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Fri, 17 Feb 2017 16:24:26 -0800 Subject: [PATCH 102/223] Restructure Format Specification --- doc/zstd_compression_format.md | 1289 +++++++++++++++++--------------- 1 file changed, 677 insertions(+), 612 deletions(-) diff --git a/doc/zstd_compression_format.md b/doc/zstd_compression_format.md index df983284f..f08dc9537 100644 --- a/doc/zstd_compression_format.md +++ b/doc/zstd_compression_format.md @@ -3,7 +3,7 @@ Zstandard Compression Format ### Notices -Copyright (c) 2016 Yann Collet +Copyright (c) 2016-present Yann Collet, Facebook, Inc. Permission is granted to copy and distribute this document for any purpose and without charge, @@ -16,8 +16,7 @@ Distribution of this document is unlimited. ### Version -0.2.3 (27/01/17) - +0.2.4 (17/02/17) Introduction ------------ @@ -57,17 +56,15 @@ Whenever it does not support a parameter defined in the compressed stream, it must produce a non-ambiguous error code and associated error message explaining which parameter is unsupported. -Overall conventions ------------ +### Overall conventions In this document: - square brackets i.e. `[` and `]` are used to indicate optional fields or parameters. -- a naming convention for identifiers is `Mixed_Case_With_Underscores` +- the naming convention for identifiers is `Mixed_Case_With_Underscores` -Definitions ------------ -A content compressed by Zstandard is transformed into a Zstandard __frame__. +### Definitions +Content compressed by Zstandard is transformed into a Zstandard __frame__. Multiple frames can be appended into a single file or stream. -A frame is totally independent, has a defined beginning and end, +A frame is completely independent, has a defined beginning and end, and a set of parameters which tells the decoder how to decompress it. A frame encapsulates one or multiple __blocks__. @@ -77,63 +74,33 @@ Unlike frames, each block depends on previous blocks for proper decoding. However, each block can be decompressed without waiting for its successor, allowing streaming operations. +Overview +--------- +- [Frames](#frames) + - [Zstandard frames](#zstandard-frames) + - [Blocks](#blocks) + - [Literals Section](#literals-section) + - [Sequences Section](#sequences-section) + - [Sequence Execution](#sequence-execution) + - [Skippable frames](#skippable-frames) +- [Entropy Encoding](#entropy-encoding) + - [FSE](#fse) + - [Huffman Coding](#huffman-coding) +- [Dictionary Format](#dictionary-format) -Frame Concatenation -------------------- +Frames +------ +Zstandard compressed data is made of up one or more __frames__. +Each frame is independent and can be decompressed indepedently of other frames. +The decompressed content of multiple concatenated frames is the concatenation of +each frames decompressed content. -In some circumstances, it may be required to append multiple frames, -for example in order to add new data to an existing compressed file -without re-framing it. +There are two frame formats defined by Zstandard: + Zstandard frames and Skippable frames. +Zstandard frames contain compressed data, while +skippable frames contain no data and can be used for metadata. -In such case, each frame brings its own set of descriptor flags. -Each frame is considered independent. -The only relation between frames is their sequential order. - -The ability to decode multiple concatenated frames -within a single stream or file is left outside of this specification. -As an example, the reference `zstd` command line utility is able -to decode all concatenated frames in their sequential order, -delivering the final decompressed result as if it was a single content. - - -Skippable Frames ----------------- - -| `Magic_Number` | `Frame_Size` | `User_Data` | -|:--------------:|:------------:|:-----------:| -| 4 bytes | 4 bytes | n bytes | - -Skippable frames allow the insertion of user-defined data -into a flow of concatenated frames. -Its design is pretty straightforward, -with the sole objective to allow the decoder to quickly skip -over user-defined data and continue decoding. - -Skippable frames defined in this specification are compatible with [LZ4] ones. - -[LZ4]:http://www.lz4.org - -__`Magic_Number`__ - -4 Bytes, little-endian format. -Value : 0x184D2A5?, which means any value from 0x184D2A50 to 0x184D2A5F. -All 16 values are valid to identify a skippable frame. - -__`Frame_Size`__ - -This is the size, in bytes, of the following `User_Data` -(without including the magic number nor the size field itself). -This field is represented using 4 Bytes, little-endian format, unsigned 32-bits. -This means `User_Data` can’t be bigger than (2^32-1) bytes. - -__`User_Data`__ - -The `User_Data` can be anything. Data will just be skipped by the decoder. - - - -General Structure of Zstandard Frame format -------------------------------------------- +## Zstandard frames The structure of a single Zstandard frame is following: | `Magic_Number` | `Frame_Header` |`Data_Block`| [More data blocks] | [`Content_Checksum`] | @@ -147,11 +114,11 @@ Value : 0xFD2FB528 __`Frame_Header`__ -2 to 14 Bytes, detailed in [next part](#the-structure-of-frame_header). +2 to 14 Bytes, detailed in [`Frame_Header`](#frame_header). __`Data_Block`__ -Detailed in [next chapter](#the-structure-of-data_block). +Detailed in [`Blocks`](#blocks). That’s where compressed data is stored. __`Content_Checksum`__ @@ -162,10 +129,9 @@ of [xxh64() hash function](http://www.xxhash.org) digesting the original (decoded) data as input, and a seed of zero. The low 4 bytes of the checksum are stored in little endian format. +### `Frame_Header` -The structure of `Frame_Header` -------------------------------- -The `Frame_Header` has a variable size, which uses a minimum of 2 bytes, +The `Frame_Header` has a variable size, with a minimum of 2 bytes, and up to 14 bytes depending on optional parameters. The structure of `Frame_Header` is following: @@ -173,10 +139,10 @@ The structure of `Frame_Header` is following: | ------------------------- | --------------------- | ----------------- | ---------------------- | | 1 byte | 0-1 byte | 0-4 bytes | 0-8 bytes | -### `Frame_Header_Descriptor` +#### `Frame_Header_Descriptor` The first header's byte is called the `Frame_Header_Descriptor`. -It tells which other fields are present. +It describes which other fields are present. Decoding this byte is enough to tell the size of `Frame_Header`. | Bit number | Field name | @@ -188,7 +154,7 @@ Decoding this byte is enough to tell the size of `Frame_Header`. | 2 | `Content_Checksum_flag` | | 1-0 | `Dictionary_ID_flag` | -In this table, bit 7 is highest bit, while bit 0 is lowest. +In this table, bit 7 the is highest bit, while bit 0 the is lowest. __`Frame_Content_Size_flag`__ @@ -216,7 +182,7 @@ but `Window_Descriptor` byte is skipped. As a consequence, the decoder must allocate a memory segment of size equal or bigger than `Frame_Content_Size`. -In order to preserve the decoder from unreasonable memory requirement, +In order to preserve the decoder from unreasonable memory requirements, a decoder can reject a compressed frame which requests a memory size beyond decoder's authorized range. @@ -256,7 +222,7 @@ It also specifies the size of this field as `Field_Size`. | ---------- | --- | --- | --- | --- | |`Field_Size`| 0 | 1 | 2 | 4 | -### `Window_Descriptor` +#### `Window_Descriptor` Provides guarantees on maximum back-reference distance that will be used within compressed data. @@ -294,12 +260,12 @@ It's merely a recommendation though, decoders are free to support larger or lower limits, depending on local limitations. -### `Dictionary_ID` +#### `Dictionary_ID` This is a variable size field, which contains the ID of the dictionary required to properly decode the frame. Note that this field is optional. When it's not present, -it's up to the caller to make sure it uses the correct dictionary. +it's up to the decoder to make sure it uses the correct dictionary. Format is little-endian. Field size depends on `Dictionary_ID_flag`. @@ -319,7 +285,7 @@ the following ranges are reserved for future use and should not be used : - high range : >= (2^31) -### `Frame_Content_Size` +#### `Frame_Content_Size` This is the original (uncompressed) size. This information is optional. The `Field_Size` is provided according to value of `Frame_Content_Size_flag`. @@ -337,10 +303,14 @@ When `Field_Size` is 1, 4 or 8 bytes, the value is read directly. When `Field_Size` is 2, _the offset of 256 is added_. It's allowed to represent a small size (for example `18`) using any compatible variant. +Blocks +------- +After the magic number and header of each block, +there are some number of blocks. +Each frame must have at least one block but there is no upper limit +on the number of blocks per frame. -The structure of `Data_Block` ------------------------------ -The structure of `Data_Block` is following: +The structure of a block is as follows: | `Last_Block` | `Block_Type` | `Block_Size` | `Block_Content` | |:------------:|:------------:|:------------:|:---------------:| @@ -351,8 +321,9 @@ The block header (`Last_Block`, `Block_Type`, and `Block_Size`) uses 3-bytes. __`Last_Block`__ The lowest bit signals if this block is the last one. -Frame ends right after this block. -It may be followed by an optional `Content_Checksum` . +The frame will end after this one. +It may be followed by an optional `Content_Checksum` +(see [Zstandard Frames](#zstandard-frames)). __`Block_Type` and `Block_Size`__ @@ -367,15 +338,19 @@ There are 4 block types : | `Block_Type` | `Raw_Block` | `RLE_Block` | `Compressed_Block` | `Reserved`| - `Raw_Block` - this is an uncompressed block. - `Block_Size` is the number of bytes to read and copy. + `Block_Content` contains `Block_Size` bytes to read and copy + as decoded data. + - `RLE_Block` - this is a single byte, repeated N times. - In which case, `Block_Size` is the size to regenerate, - while the "compressed" block is just 1 byte (the byte to repeat). -- `Compressed_Block` - this is a [Zstandard compressed block](#the-format-of-compressed_block), - detailed in another section of this specification. - `Block_Size` is the compressed size. - Decompressed size is unknown, + `Block_Content` consists of a single byte, + and `Block_Size` is the number of times this byte should be repeated. + +- `Compressed_Block` - this is a [Zstandard compressed block](#compressed-blocks), + explained later on. + `Block_Size` is the length of `Block_Content`, the compressed data. + The decompressed size is unknown, but its maximum possible value is guaranteed (see below) + - `Reserved` - this is not a block. This value cannot be used with current version of this specification. @@ -384,42 +359,36 @@ Block sizes must respect a few rules : - Block decompressed size is always <= maximum back-reference distance. - Block decompressed size is always <= 128 KB. - -__`Block_Content`__ - -The `Block_Content` is where the actual data to decode stands. -It might be compressed or not, depending on previous field indications. A data block is not necessarily "full" : since an arbitrary “flush” may happen anytime, -block decompressed content can be any size, +block decompressed content can be any size (even empty), up to `Block_Maximum_Decompressed_Size`, which is the smallest of : - Maximum back-reference distance - 128 KB - - -The format of `Compressed_Block` --------------------------------- -The size of `Compressed_Block` must be provided using `Block_Size` field from `Data_Block`. -The `Compressed_Block` has a guaranteed maximum regenerated size, -in order to properly allocate destination buffer. -See [`Data_Block`](#the-structure-of-data_block) for more details. +Compressed Blocks +----------------- +To decompress a compressed block, the compressed size must be provided from +`Block_Size` field in the block header. A compressed block consists of 2 sections : -- [`Literals_Section`](#literals_section) -- [`Sequences_Section`](#sequences_section) +- [Literals Section](#literals-section) +- [Sequences Section](#sequences-section) -### Prerequisites +The results of the two sections are then combined to produce the decompressed +data in [Sequence Execution](#sequence-execution) + +#### Prerequisites To decode a compressed block, the following elements are necessary : -- Previous decoded blocks, up to a distance of `Window_Size`, - or all previous blocks when `Single_Segment_flag` is set. -- List of "recent offsets" from previous compressed block. -- Decoding tables of previous compressed block for each symbol type +- Previous decoded data, up to a distance of `Window_Size`, + or all previous data when `Single_Segment_flag` is set. +- List of "recent offsets" from the previous compressed block. +- Decoding tables of the previous compressed block for each symbol type (literals, literals lengths, match lengths, offsets). - -### `Literals_Section` - +Literals Section +---------------- +During sequence execution, symbols from the literals section During sequence phase, literals will be entangled with match copy operations. All literals are regrouped in the first part of the block. They can be decoded first, and then copied during sequence operations, @@ -443,7 +412,7 @@ using little-endian convention. | --------------------- | ------------- | ------------------ | ----------------- | | 2 bits | 1 - 2 bits | 5 - 20 bits | 0 - 18 bits | -In this representation, bits on the left are smallest bits. +In this representation, bits on the left are the lowest bits. __`Literals_Block_Type`__ @@ -464,14 +433,16 @@ This field uses 2 lowest bits of first byte, describing 4 different block types - `Repeat_Stats_Literals_Block` - This is a Huffman-compressed block, using Huffman tree _from previous Huffman-compressed literals block_. Huffman tree description will be skipped. + Note: If this mode is used without any previous Huffman-table in the frame + (or [dictionary](#dictionary-format)), this should be treated as corruption. __`Size_Format`__ `Size_Format` is divided into 2 families : -- For `Compressed_Block`, it requires to decode both `Compressed_Size` - and `Regenerated_Size` (the decompressed size). It will also decode the number of streams. - For `Raw_Literals_Block` and `RLE_Literals_Block` it's enough to decode `Regenerated_Size`. +- For `Compressed_Block`, its required to decode both `Compressed_Size` + and `Regenerated_Size` (the decompressed size). It will also decode the number of streams. For values spanning several bytes, convention is little-endian. @@ -490,32 +461,595 @@ __`Size_Format` for `Raw_Literals_Block` and `RLE_Literals_Block`__ : `Literals_Section_Header` has 3 bytes. `Regenerated_Size = (Header[0]>>4) + (Header[1]<<4) + (Header[2]<<12)` +Only Stream1 is present for these cases. Note : it's allowed to represent a short value (for example `13`) using a long format, accepting the increased compressed data size. __`Size_Format` for `Compressed_Literals_Block` and `Repeat_Stats_Literals_Block`__ : - Value 00 : _A single stream_. - Both `Compressed_Size` and `Regenerated_Size` use 10 bits (0-1023). + Both `Regenerated_Size` and `Compressed_Size` use 10 bits (0-1023). `Literals_Section_Header` has 3 bytes. - Value 01 : 4 streams. - Both `Compressed_Size` and `Regenerated_Size` use 10 bits (0-1023). + Both `Regenerated_Size` and `Compressed_Size` use 10 bits (0-1023). `Literals_Section_Header` has 3 bytes. - Value 10 : 4 streams. - Both `Compressed_Size` and `Regenerated_Size` use 14 bits (0-16383). + Both `Regenerated_Size` and `Compressed_Size` use 14 bits (0-16383). `Literals_Section_Header` has 4 bytes. - Value 11 : 4 streams. - Both `Compressed_Size` and `Regenerated_Size` use 18 bits (0-262143). + Both `Regenerated_Size` and `Compressed_Size` use 18 bits (0-262143). `Literals_Section_Header` has 5 bytes. Both `Compressed_Size` and `Regenerated_Size` fields follow little-endian convention. Note: `Compressed_Size` __includes__ the size of the Huffman Tree description if it is present. +### Raw Literals Block +The data in Stream1 is `Regenerated_Size` bytes long, and contains the raw literals data +to be used in sequence execution. + +### RLE Literals Block +Stream1 consists of a single byte which should be repeated `Regenerated_Size` times +to generate the decoded literals. + +### Compressed Literals Block and Repeat Stats Literals Block +Both of these modes contain Huffman encoded data + #### `Huffman_Tree_Description` - This section is only present when `Literals_Block_Type` type is `Compressed_Literals_Block` (`2`). +The format of the Huffman tree description can be found at [Huffman Tree description](#huffman-tree-description). +The size Huffman Tree description will be determined during the decoding process, +and must be used to determine where the compressed Huffman streams begin. +If repeat stats mode is used, the Huffman table used in the previous compressed block will +be used to decompress this block as well. + +Huffman compressed data consists either 1 or 4 Huffman-coded streams. + +If only one stream is present, it is a single bitstream occupying the entire +remaining portion of the literals block, encoded as described at +[Huffman-Coded Streams](#huffman-coded-streams). + +If there are four streams, the literals section header only provides enough +information to know the regenerated and compressed sizes of all four streams combined. +The regenerated size of each stream is equal to `(totalSize+3)/4`, except for the last stream, +which may be up to 3 bytes smaller, to reach a total decompressed size match that described +in the literals header. + +The compressed size of each stream is provided explicitly: the first 6 bytes of the compressed +data consist of three 2-byte little endian fields, describing the compressed sizes +of the first three streams. +The last streams size is computed from the total compressed size and the size of the other +three streams. + +`stream4CSize = totalCSize - 6 - stream1CSize - stream2CSize - stream3CSize`. + +Note: remember that totalCSize may be smaller than the `Compressed_Size` found in the literals +block header as `Compressed_Size` also contains the size of the Huffman Tree description if it +is present. + +Each of these 4 bitstreams is then decoded independently as a Huffman-Coded stream, +as described at [Huffman-Coded Streams](#huffman-coded-streams) + +Sequences Section +----------------- +A compressed block is a succession of _sequences_ . +A sequence is a literal copy command, followed by a match copy command. +A literal copy command specifies a length. +It is the number of bytes to be copied (or extracted) from the literal section. +A match copy command specifies an offset and a length. + +When all _sequences_ are decoded, +if there is are any literals left in the _literal section_, +these bytes are added at the end of the block. + +This is described in more detail in [Sequence Execution](#sequence-execution) + +The `Sequences_Section` regroup all symbols required to decode commands. +There are 3 symbol types : literals lengths, offsets and match lengths. +They are encoded together, interleaved, in a single _bitstream_. + +The `Sequences_Section` starts by a header, +followed by optional probability tables for each symbol type, +followed by the bitstream. + +| `Sequences_Section_Header` | [`Literals_Length_Table`] | [`Offset_Table`] | [`Match_Length_Table`] | bitStream | +| -------------------------- | ------------------------- | ---------------- | ---------------------- | --------- | + +To decode the `Sequences_Section`, it's required to know its size. +This size is deduced from `blockSize - literalSectionSize`. + + +#### `Sequences_Section_Header` + +Consists of 2 items: +- `Number_of_Sequences` +- Symbol compression modes + +__`Number_of_Sequences`__ + +This is a variable size field using between 1 and 3 bytes. +Let's call its first byte `byte0`. +- `if (byte0 == 0)` : there are no sequences. + The sequence section stops there. + Regenerated content is defined entirely by literals section. +- `if (byte0 < 128)` : `Number_of_Sequences = byte0` . Uses 1 byte. +- `if (byte0 < 255)` : `Number_of_Sequences = ((byte0-128) << 8) + byte1` . Uses 2 bytes. +- `if (byte0 == 255)`: `Number_of_Sequences = byte1 + (byte2<<8) + 0x7F00` . Uses 3 bytes. + +__Symbol compression modes__ + +This is a single byte, defining the compression mode of each symbol type. + +|Bit number| 7-6 | 5-4 | 3-2 | 1-0 | +| -------- | ----------------------- | -------------- | -------------------- | ---------- | +|Field name| `Literals_Lengths_Mode` | `Offsets_Mode` | `Match_Lengths_Mode` | `Reserved` | + +The last field, `Reserved`, must be all-zeroes. + +`Literals_Lengths_Mode`, `Offsets_Mode` and `Match_Lengths_Mode` define the `Compression_Mode` of +literals lengths, offsets, and match lengths respectively. + +They follow the same enumeration : + +| Value | 0 | 1 | 2 | 3 | +| ------------------ | ----------------- | ---------- | --------------------- | ------------- | +| `Compression_Mode` | `Predefined_Mode` | `RLE_Mode` | `FSE_Compressed_Mode` | `Repeat_Mode` | + +- `Predefined_Mode` : A predefined FSE distribution table is used, defined in + [default distributions](#default-distributions). + The table takes no space in the compressed data. +- `RLE_Mode` : The table description consists of a single byte. + This code will be repeated for every sequence. +- `Repeat_Mode` : The table used in the previous compressed block will be used again. + No distribution table will be present. + Note: this includes RLE mode, so if repeat_mode follows rle_mode the same symbol will be repeated. + If this mode is used without any previous sequence table in the frame + (or [dictionary](#dictionary-format)) to repeat, this should be treated as corruption. +- `FSE_Compressed_Mode` : standard FSE compression. + A distribution table will be present. + The format of this distribution table is described in (FSE Table Description)[#fse-table-description]. + Note that the maximum allowed accuracy log for literals length and match length tables is 9, + and the maximum accuracy log for the offsets table is 8. + +#### The codes for literals lengths, match lengths, and offsets. + +Each symbol is a _code_ in its own context, +which specifies `Baseline` and `Number_of_Bits` to add. +_Codes_ are FSE compressed, +and interleaved with raw additional bits in the same bitstream. + +##### Literals length codes + +Literals length codes are values ranging from `0` to `35` included. +They define lengths from 0 to 131071 bytes. +The literals length is equal to the decoded `Baseline` plus +the result of reading `Number_of_Bits` bits from the bitstream, +as a little-endian value. + +| `Literals_Length_Code` | 0-15 | +| ---------------------- | ---------------------- | +| length | `Literals_Length_Code` | +| `Number_of_Bits` | 0 | + +| `Literals_Length_Code` | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | +| ---------------------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | +| `Baseline` | 16 | 18 | 20 | 22 | 24 | 28 | 32 | 40 | +| `Number_of_Bits` | 1 | 1 | 1 | 1 | 2 | 2 | 3 | 3 | + +| `Literals_Length_Code` | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | +| ---------------------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | +| `Baseline` | 48 | 64 | 128 | 256 | 512 | 1024 | 2048 | 4096 | +| `Number_of_Bits` | 4 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | + +| `Literals_Length_Code` | 32 | 33 | 34 | 35 | +| ---------------------- | ---- | ---- | ---- | ---- | +| `Baseline` | 8192 |16384 |32768 |65536 | +| `Number_of_Bits` | 13 | 14 | 15 | 16 | + + +##### Match length codes + +Match length codes are values ranging from `0` to `52` included. +They define lengths from 3 to 131074 bytes. +The match length is equal to the decoded `Baseline` plus +the result of reading `Number_of_Bits` bits from the bitstream, +as a little-endian value. + +| `Match_Length_Code` | 0-31 | +| ------------------- | ----------------------- | +| value | `Match_Length_Code` + 3 | +| `Number_of_Bits` | 0 | + +| `Match_Length_Code` | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | +| ------------------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | +| `Baseline` | 35 | 37 | 39 | 41 | 43 | 47 | 51 | 59 | +| `Number_of_Bits` | 1 | 1 | 1 | 1 | 2 | 2 | 3 | 3 | + +| `Match_Length_Code` | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | +| ------------------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | +| `Baseline` | 67 | 83 | 99 | 131 | 259 | 515 | 1027 | 2051 | +| `Number_of_Bits` | 4 | 4 | 5 | 7 | 8 | 9 | 10 | 11 | + +| `Match_Length_Code` | 48 | 49 | 50 | 51 | 52 | +| ------------------- | ---- | ---- | ---- | ---- | ---- | +| `Baseline` | 4099 | 8195 |16387 |32771 |65539 | +| `Number_of_Bits` | 12 | 13 | 14 | 15 | 16 | + +##### Offset codes + +Offset codes are values ranging from `0` to `N`. + +A decoder is free to limit its maximum `N` supported. +Recommendation is to support at least up to `22`. +For information, at the time of this writing. +the reference decoder supports a maximum `N` value of `28` in 64-bits mode. + +An offset code is also the number of additional bits to read in little-endian fashion, +and can be translated into an `Offset_Value` using the following formulas : + +``` +Offset_Value = (1 << offsetCode) + readNBits(offsetCode); +if (Offset_Value > 3) offset = Offset_Value - 3; +``` +It means that maximum `Offset_Value` is `(2^(N+1))-1` and it supports back-reference distance up to `(2^(N+1))-4` +but is limited by [maximum back-reference distance](#window_descriptor). + +`Offset_Value` from 1 to 3 are special : they define "repeat codes". +This is described in more detail in [Repeat Offsets](#repeat-offsets). + +#### Decoding Sequences +FSE bitstreams are read in reverse direction than written. In zstd, +the compressor writes bits forward into a block and the decompressor +must read the bitstream _backwards_. + +To find the start of the bitstream it is therefore necessary to +know the offset of the last byte of the block which can be found +by counting `Block_Size` bytes after the block header. + +After writing the last bit containing information, the compressor +writes a single `1`-bit and then fills the byte with 0-7 `0` bits of +padding. The last byte of the compressed bitstream cannot be `0` for +that reason. + +When decompressing, the last byte containing the padding is the first +byte to read. The decompressor needs to skip 0-7 initial `0`-bits and +the first `1`-bit it occurs. Afterwards, the useful part of the bitstream +begins. + +FSE decoding requires a 'state' to be carried from symbol to symbol. +For more explanation on FSE decoding, see the [FSE section](#fse). + +For sequence decoding, a separate state must be kept track of for each of +literal lengths, offsets, and match lengths. +Some FSE primitives are also used. +For more details on the operation of these primitives, see the [FSE section](#fse). + +##### Starting states +The bitstream starts with initial FSE state values, +each using the required number of bits in their respective _accuracy_, +decoded previously from their normalized distribution. + +It starts by `Literals_Length_State`, +followed by `Offset_State`, +and finally `Match_Length_State`. + +Reminder : always keep in mind that all values are read _backward_, +so the 'start' of the bitstream is at the highest position in memory, +immediately before the last `1`-bit for padding. + +After decoding the starting states, a single sequence is decoded +`Number_Of_Sequences` times. +These sequences are decoded in order from first to last. +Since the compressor writes the bitstream in the forward direction, +this means the compressor must encode the sequences starting with the last +one and ending with the first. + +##### Decoding a sequence +For each of the symbol types, the FSE state can be used to determine the appropriate code. +The code then defines the baseline and number of bits to read for each type. +See the [description of the codes] for how to determine these values. + +[description of the codes]: #the-codes-for-literals-lengths-match-lengths-and-offsets + +Decoding starts by reading the `Number_of_Bits` required to decode `Offset`. +It then does the same for `Match_Length`, +and then for `Literals_Length`. +This sequence is then used for [sequence execution](#sequence-execution). + +If it is not the last sequence in the block, +the next operation is to update states. +Using the rules pre-calculated in the decoding tables, +`Literals_Length_State` is updated, +followed by `Match_Length_State`, +and then `Offset_State`. +See the [FSE section](#fse) for details on how to update states from the bitstream. + +This operation will be repeated `Number_of_Sequences` times. +At the end, the bitstream shall be entirely consumed, +otherwise the bitstream is considered corrupted. + +#### Default Distributions +If `Predefined_Mode` is selected for a symbol type, +its FSE decoding table is generated from a predefined distribution table defined here. +For details on how to convert this distribution into a decoding table, see the [FSE section]. + +[FSE section]: #from-normalized-distribution-to-decoding-tables + +Sequence Execution +------------------ +Once literals and sequences have been decoded, +they are combined to produce the decoded content of a block. + +Each sequence consists of a tuple of (`literals_length`, `offset_value`, `match_length`), +decoded as described in the [Sequences Section)[#sequences-section]. +To execute a sequence, first copy `literals_length` bytes from the literals section +to the output. + +Then `match_length` bytes are copied from previous decoded data. +The offset to copy from is determined by `offset_value`: +if `offset_value > 3`, then the offset is `offset_value - 3`. +If `offset_value` is from 1-3, the offset is a special repeat offset value. +See the [repeat offset](#repeat-offsets) section for how the offset is determined +in this case. + +The offset is defined as from the current position, so an offset of 6 +and a match length of 3 means that 3 bytes should be copied from 6 bytes back. +Note that all offsets must be at most equal to the window size defined by the frame header. + +#### Repeat offsets +As seen in [Sequence Execution](#sequence-execution), +the first 3 values define a repeated offset and we will call them +`Repeated_Offset1`, `Repeated_Offset2`, and `Repeated_Offset3`. +They are sorted in recency order, with `Repeated_Offset1` meaning "most recent one". + +If `offset_value == 1`, then the offset used is `Repeated_Offset1`, etc. + +There is an exception though, when current sequence's `literals_length = 0`. +In this case, repeated offsets are shifted by one, +so an `offset_value` of 1 means `Repeated_Offset2`, +an `offset_value` of 2 means `Repeated_Offset3`, +and an `offset_value` of 3 means `Repeated_Offset1 - 1_byte`. + +In the first block, the offset history is populated with the following values : 1, 4 and 8 (in order). + +Then each block gets its starting offset history from the ending values of the most recent compressed block. +Note that non-compressed blocks are skipped, +they do not contribute to offset history. + +[Offset Codes]: #offset-codes + +###### Offset updates rules + +The newest offset takes the lead in offset history, +shifting others back (up to its previous place if it was already present). + +This means that when `Repeated_Offset1` (most recent) is used, history is unmodified. +When `Repeated_Offset2` is used, it's swapped with `Repeated_Offset1`. +If any other offset is used, it becomes `Repeated_Offset1` and the rest are shift back by one. + +Skippable Frames +---------------- + +| `Magic_Number` | `Frame_Size` | `User_Data` | +|:--------------:|:------------:|:-----------:| +| 4 bytes | 4 bytes | n bytes | + +Skippable frames allow the insertion of user-defined data +into a flow of concatenated frames. +Its design is pretty straightforward, +with the sole objective to allow the decoder to quickly skip +over user-defined data and continue decoding. + +Skippable frames defined in this specification are compatible with [LZ4] ones. + +[LZ4]:http://www.lz4.org + +__`Magic_Number`__ + +4 Bytes, little-endian format. +Value : 0x184D2A5?, which means any value from 0x184D2A50 to 0x184D2A5F. +All 16 values are valid to identify a skippable frame. + +__`Frame_Size`__ + +This is the size, in bytes, of the following `User_Data` +(without including the magic number nor the size field itself). +This field is represented using 4 Bytes, little-endian format, unsigned 32-bits. +This means `User_Data` can’t be bigger than (2^32-1) bytes. + +__`User_Data`__ + +The `User_Data` can be anything. Data will just be skipped by the decoder. + +Entropy Encoding +---------------- +Two types of entropy encoding are used by the Zstandard format: +FSE, and Huffman coding. + +FSE +--- +FSE, or FiniteStateEntropy is an entropy coding based on [ANS]. +FSE encoding/decoding involves a state that is carried over between symbols, +so decoding must be done in the opposite direction as encoding. +Therefore, all FSE bitstreams are read from end to beginning. + +For additional details on FSE, see [Finite State Entropy]. + +[Finite State Entropy]:https://github.com/Cyan4973/FiniteStateEntropy/ + +FSE decoding involves a decoding table which has a power of 2 size and three elements: +`Symbol`, `Num_Bits`, and `Baseline`. +The `log2` of the table size is its `Accuracy_Log`. +The FSE state represents an index in this table. +The next symbol in the stream is the symbol indicated by the table value for that state. +To obtain the next state value, +the decoder should consume `Num_Bits` bits from the stream as a little endian value and add it to baseline. + +To obtain the initial state value, consume `Accuracy_Log` bits from the stream as a little endian value. + +[ANS]: https://en.wikipedia.org/wiki/Asymmetric_Numeral_Systems + +### FSE Table Description +To decode FSE streams, it is necessary to construct the decoding table. +The Zstandard format encodes FSE table descriptions as follows: + +An FSE distribution table describes the probabilities of all symbols +from `0` to the last present one (included) +on a normalized scale of `1 << Accuracy_Log` . + +It's a bitstream which is read forward, in little-endian fashion. +It's not necessary to know its exact size, +since it will be discovered and reported by the decoding process. + +The bitstream starts by reporting on which scale it operates. +`Accuracy_Log = low4bits + 5`. + +Then follows each symbol value, from `0` to last present one. +The number of bits used by each field is variable. +It depends on : + +- Remaining probabilities + 1 : + __example__ : + Presuming an `Accuracy_Log` of 8, + and presuming 100 probabilities points have already been distributed, + the decoder may read any value from `0` to `255 - 100 + 1 == 156` (inclusive). + Therefore, it must read `log2sup(156) == 8` bits. + +- Value decoded : small values use 1 less bit : + __example__ : + Presuming values from 0 to 156 (inclusive) are possible, + 255-156 = 99 values are remaining in an 8-bits field. + They are used this way : + first 99 values (hence from 0 to 98) use only 7 bits, + values from 99 to 156 use 8 bits. + This is achieved through this scheme : + + | Value read | Value decoded | Number of bits used | + | ---------- | ------------- | ------------------- | + | 0 - 98 | 0 - 98 | 7 | + | 99 - 127 | 99 - 127 | 8 | + | 128 - 226 | 0 - 98 | 7 | + | 227 - 255 | 128 - 156 | 8 | + +Symbols probabilities are read one by one, in order. + +Probability is obtained from Value decoded by following formula : +`Proba = value - 1` + +It means value `0` becomes negative probability `-1`. +`-1` is a special probability, which means "less than 1". +Its effect on distribution table is described in the [next section]. +For the purpose of calculating total allocated probability points, it counts as one. + +[next section]:#from-normalized-distribution-to-decoding-tables + +When a symbol has a __probability__ of `zero`, +it is followed by a 2-bits repeat flag. +This repeat flag tells how many probabilities of zeroes follow the current one. +It provides a number ranging from 0 to 3. +If it is a 3, another 2-bits repeat flag follows, and so on. + +When last symbol reaches cumulated total of `1 << Accuracy_Log`, +decoding is complete. +If the last symbol makes cumulated total go above `1 << Accuracy_Log`, +distribution is considered corrupted. + +Then the decoder can tell how many bytes were used in this process, +and how many symbols are present. +The bitstream consumes a round number of bytes. +Any remaining bit within the last byte is just unused. + +##### From normalized distribution to decoding tables + +The distribution of normalized probabilities is enough +to create a unique decoding table. + +It follows the following build rule : + +The table has a size of `Table_Size = 1 << Accuracy_Log`. +Each cell describes the symbol decoded, +and instructions to get the next state. + +Symbols are scanned in their natural order for "less than 1" probabilities. +Symbols with this probability are being attributed a single cell, +starting from the end of the table. +These symbols define a full state reset, reading `Accuracy_Log` bits. + +All remaining symbols are sorted in their natural order. +Starting from symbol `0` and table position `0`, +each symbol gets attributed as many cells as its probability. +Cell allocation is spreaded, not linear : +each successor position follow this rule : + +``` +position += (tableSize>>1) + (tableSize>>3) + 3; +position &= tableSize-1; +``` + +A position is skipped if already occupied by a "less than 1" probability symbol. +`position` does not reset between symbols, it simply iterates through +each position in the table, switching to the next symbol when enough +states have been allocated to the current one. + +The result is a list of state values. +Each state will decode the current symbol. + +To get the `Number_of_Bits` and `Baseline` required for next state, +it's first necessary to sort all states in their natural order. +The lower states will need 1 more bit than higher ones. + +__Example__ : +Presuming a symbol has a probability of 5. +It receives 5 state values. States are sorted in natural order. + +Next power of 2 is 8. +Space of probabilities is divided into 8 equal parts. +Presuming the `Accuracy_Log` is 7, it defines 128 states. +Divided by 8, each share is 16 large. + +In order to reach 8, 8-5=3 lowest states will count "double", +taking shares twice larger, +requiring one more bit in the process. + +Numbering starts from higher states using less bits. + +| state order | 0 | 1 | 2 | 3 | 4 | +| ---------------- | ----- | ----- | ------ | ---- | ----- | +| width | 32 | 32 | 32 | 16 | 16 | +| `Number_of_Bits` | 5 | 5 | 5 | 4 | 4 | +| range number | 2 | 4 | 6 | 0 | 1 | +| `Baseline` | 32 | 64 | 96 | 0 | 16 | +| range | 32-63 | 64-95 | 96-127 | 0-15 | 16-31 | + +The next state is determined from current state +by reading the required `Number_of_Bits`, and adding the specified `Baseline`. + +See [Appendix A] for the results of this process applied to the default distributions. + +[Appendix A]: #appendix-a---decoding-tables-for-predefined-codes + +Huffman Coding +-------------- +Zstandard Huffman-coded streams are read backwards, +similar to the FSE bitstreams. +Therefore, to find the start of the bitstream it is therefore necessary to +know the offset of the last byte of the Huffman-coded stream. + +After writing the last bit containing information, the compressor +writes a single `1`-bit and then fills the byte with 0-7 `0` bits of +padding. The last byte of the compressed bitstream cannot be `0` for +that reason. + +When decompressing, the last byte containing the padding is the first +byte to read. The decompressor needs to skip 0-7 initial `0`-bits and +the first `1`-bit it occurs. Afterwards, the useful part of the bitstream +begins. + +The bitstream contains Huffman-coded symbols in little-endian order, +with the codes defined by the method below. + +### Huffman Tree Description Prefix coding represents symbols from an a priori known alphabet by bit sequences (codewords), one codeword for each symbol, in a manner such that different symbols may be represented @@ -598,19 +1132,7 @@ which describes how to decode the list of weights. ##### Finite State Entropy (FSE) compression of Huffman weights -FSE decoding uses three operations: `Init_State`, `Decode_Symbol`, and `Update_State`. -`Init_State` reads in the initial state value from a bitstream, -`Decode_Symbol` outputs a symbol based on the current state, -and `Update_State` goes to a new state based on the current state and some number of consumed bits. - -FSE streams must be read in reverse from the order they're encoded in, -so bitstreams start at a certain offset and works backwards towards their base. - -For more on how FSE bitstreams work, see [Finite State Entropy]. - -[Finite State Entropy]:https://github.com/Cyan4973/FiniteStateEntropy/ - -The series of Huffman weights is compressed using FSE compression. +In this case, the series of Huffman weights is compressed using FSE compression. It's a single bitstream with 2 interleaved states, sharing a single distribution table. @@ -622,17 +1144,16 @@ and last symbol's weight is not represented. An FSE bitstream starts by a header, describing probabilities distribution. It will create a Decoding Table. -The table must be pre-allocated, so a maximum accuracy must be fixed. -For a list of Huffman weights, maximum accuracy is 7 bits. +For a list of Huffman weights, the maximum accuracy log is 7 bits. +For more description see the [FSE header description](#fse-table-description) -The FSE header format is [described in a relevant chapter](#fse-distribution-table--condensed-format), -as well as the [FSE bitstream](#bitstream). -The main difference is that Huffman header compression uses 2 states, +The Huffman header compression uses 2 states, which share the same FSE distribution table. The first state (`State1`) encodes the even indexed symbols, and the second (`State2`) encodes the odd indexes. State1 is initialized first, and then State2, and they take turns decoding a single symbol and updating their state. +For more details on these FSE operations, see the [FSE section](#fse). The number of symbols to decode is determined by tracking bitStream overflow condition: @@ -667,39 +1188,9 @@ it gives the following distribution : | `Number_of_Bits` | 0 | 4 | 4 | 3 | 2 | 1 | | prefix codes | N/A | 0000| 0001| 001 | 01 | 1 | - -#### The content of Huffman-compressed literal stream - -##### Bitstreams sizes - -As seen in a previous paragraph, -there are 2 types of Huffman-compressed literals : -a single stream and 4 streams. - -Encoding using 4 streams is useful for CPU with multiple execution units and out-of-order operations. -Since each stream can be decoded independently, -it's possible to decode them up to 4x faster than a single stream, -presuming the CPU has enough parallelism available. - -For single stream, header provides both the compressed and regenerated size. -For 4 streams though, -header only provides compressed and regenerated size of all 4 streams combined. -In order to properly decode the 4 streams, -it's necessary to know the compressed and regenerated size of each stream. - -Regenerated size of each stream can be calculated by `(totalSize+3)/4`, -except for last one, which can be up to 3 bytes smaller, to reach `totalSize`. - -Compressed size is provided explicitly : in the 4-streams variant, -bitstreams are preceded by 3 unsigned little-endian 16-bits values. -Each value represents the compressed size of one stream, in order. -The last stream size is deducted from total compressed size -and from previously decoded stream sizes : - -`stream4CSize = totalCSize - 6 - stream1CSize - stream2CSize - stream3CSize`. - - -##### Bitstreams read and decode +### Huffman-coded Streams +Given a Huffman decoding table, +it's possible to decode a Huffman-coded stream. Each bitstream must be read _backward_, that is starting from the end down to the beginning. @@ -736,446 +1227,10 @@ If a bitstream is not entirely and exactly consumed, hence reaching exactly its beginning position with _all_ bits consumed, the decoding process is considered faulty. -### `Sequences_Section` - -A compressed block is a succession of _sequences_ . -A sequence is a literal copy command, followed by a match copy command. -A literal copy command specifies a length. -It is the number of bytes to be copied (or extracted) from the literal section. -A match copy command specifies an offset and a length. -The offset gives the position to copy from, -which can be within a previous block. - -When all _sequences_ are decoded, -if there is are any literals left in the _literal section_, -these bytes are added at the end of the block. - -The `Sequences_Section` regroup all symbols required to decode commands. -There are 3 symbol types : literals lengths, offsets and match lengths. -They are encoded together, interleaved, in a single _bitstream_. - -The `Sequences_Section` starts by a header, -followed by optional probability tables for each symbol type, -followed by the bitstream. - -| `Sequences_Section_Header` | [`Literals_Length_Table`] | [`Offset_Table`] | [`Match_Length_Table`] | bitStream | -| -------------------------- | ------------------------- | ---------------- | ---------------------- | --------- | - -To decode the `Sequences_Section`, it's required to know its size. -This size is deducted from `blockSize - literalSectionSize`. - - -#### `Sequences_Section_Header` - -Consists of 2 items: -- `Number_of_Sequences` -- Symbol compression modes - -__`Number_of_Sequences`__ - -This is a variable size field using between 1 and 3 bytes. -Let's call its first byte `byte0`. -- `if (byte0 == 0)` : there are no sequences. - The sequence section stops there. - Regenerated content is defined entirely by literals section. -- `if (byte0 < 128)` : `Number_of_Sequences = byte0` . Uses 1 byte. -- `if (byte0 < 255)` : `Number_of_Sequences = ((byte0-128) << 8) + byte1` . Uses 2 bytes. -- `if (byte0 == 255)`: `Number_of_Sequences = byte1 + (byte2<<8) + 0x7F00` . Uses 3 bytes. - -__Symbol compression modes__ - -This is a single byte, defining the compression mode of each symbol type. - -|Bit number| 7-6 | 5-4 | 3-2 | 1-0 | -| -------- | ----------------------- | -------------- | -------------------- | ---------- | -|Field name| `Literals_Lengths_Mode` | `Offsets_Mode` | `Match_Lengths_Mode` | `Reserved` | - -The last field, `Reserved`, must be all-zeroes. - -`Literals_Lengths_Mode`, `Offsets_Mode` and `Match_Lengths_Mode` define the `Compression_Mode` of -literals lengths, offsets, and match lengths respectively. - -They follow the same enumeration : - -| Value | 0 | 1 | 2 | 3 | -| ------------------ | ----------------- | ---------- | --------------------- | ------------- | -| `Compression_Mode` | `Predefined_Mode` | `RLE_Mode` | `FSE_Compressed_Mode` | `Repeat_Mode` | - -- `Predefined_Mode` : uses a predefined distribution table. -- `RLE_Mode` : it's a single code, repeated `Number_of_Sequences` times. -- `Repeat_Mode` : re-use distribution table from previous compressed block. -- `FSE_Compressed_Mode` : standard FSE compression. - A distribution table will be present. - It will be described in [next part](#distribution-tables). - -#### The codes for literals lengths, match lengths, and offsets. - -Each symbol is a _code_ in its own context, -which specifies `Baseline` and `Number_of_Bits` to add. -_Codes_ are FSE compressed, -and interleaved with raw additional bits in the same bitstream. - -##### Literals length codes - -Literals length codes are values ranging from `0` to `35` included. -They define lengths from 0 to 131071 bytes. - -| `Literals_Length_Code` | 0-15 | -| ---------------------- | ---------------------- | -| length | `Literals_Length_Code` | -| `Number_of_Bits` | 0 | - -| `Literals_Length_Code` | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -| ---------------------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | -| `Baseline` | 16 | 18 | 20 | 22 | 24 | 28 | 32 | 40 | -| `Number_of_Bits` | 1 | 1 | 1 | 1 | 2 | 2 | 3 | 3 | - -| `Literals_Length_Code` | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -| ---------------------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | -| `Baseline` | 48 | 64 | 128 | 256 | 512 | 1024 | 2048 | 4096 | -| `Number_of_Bits` | 4 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | - -| `Literals_Length_Code` | 32 | 33 | 34 | 35 | -| ---------------------- | ---- | ---- | ---- | ---- | -| `Baseline` | 8192 |16384 |32768 |65536 | -| `Number_of_Bits` | 13 | 14 | 15 | 16 | - -##### Default distribution for literals length codes - -When `Compression_Mode` is `Predefined_Mode`, -a predefined distribution is used for FSE compression. - -Its definition is below. It uses an accuracy of 6 bits (64 states). -``` -short literalsLength_defaultDistribution[36] = - { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, - -1,-1,-1,-1 }; -``` - -##### Match length codes - -Match length codes are values ranging from `0` to `52` included. -They define lengths from 3 to 131074 bytes. - -| `Match_Length_Code` | 0-31 | -| ------------------- | ----------------------- | -| value | `Match_Length_Code` + 3 | -| `Number_of_Bits` | 0 | - -| `Match_Length_Code` | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | -| ------------------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | -| `Baseline` | 35 | 37 | 39 | 41 | 43 | 47 | 51 | 59 | -| `Number_of_Bits` | 1 | 1 | 1 | 1 | 2 | 2 | 3 | 3 | - -| `Match_Length_Code` | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | -| ------------------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- | -| `Baseline` | 67 | 83 | 99 | 131 | 259 | 515 | 1027 | 2051 | -| `Number_of_Bits` | 4 | 4 | 5 | 7 | 8 | 9 | 10 | 11 | - -| `Match_Length_Code` | 48 | 49 | 50 | 51 | 52 | -| ------------------- | ---- | ---- | ---- | ---- | ---- | -| `Baseline` | 4099 | 8195 |16387 |32771 |65539 | -| `Number_of_Bits` | 12 | 13 | 14 | 15 | 16 | - -##### Default distribution for match length codes - -When `Compression_Mode` is defined as `Predefined_Mode`, -a predefined distribution is used for FSE compression. - -Its definition is below. It uses an accuracy of 6 bits (64 states). -``` -short matchLengths_defaultDistribution[53] = - { 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1, - -1,-1,-1,-1,-1 }; -``` - -##### Offset codes - -Offset codes are values ranging from `0` to `N`. - -A decoder is free to limit its maximum `N` supported. -Recommendation is to support at least up to `22`. -For information, at the time of this writing. -the reference decoder supports a maximum `N` value of `28` in 64-bits mode. - -An offset code is also the number of additional bits to read, -and can be translated into an `Offset_Value` using the following formulas : - -``` -Offset_Value = (1 << offsetCode) + readNBits(offsetCode); -if (Offset_Value > 3) offset = Offset_Value - 3; -``` -It means that maximum `Offset_Value` is `(2^(N+1))-1` and it supports back-reference distance up to `(2^(N+1))-4` -but is limited by [maximum back-reference distance](#window_descriptor). - -`Offset_Value` from 1 to 3 are special : they define "repeat codes", -which means one of the previous offsets will be repeated. -They are sorted in recency order, with 1 meaning the most recent one. -See [Repeat offsets](#repeat-offsets) paragraph. - - -##### Default distribution for offset codes - -When `Compression_Mode` is defined as `Predefined_Mode`, -a predefined distribution is used for FSE compression. - -Below is its definition. It uses an accuracy of 5 bits (32 states), -and supports a maximum `N` of 28, allowing offset values up to 536,870,908 . - -If any sequence in the compressed block requires an offset larger than this, -it's not possible to use the default distribution to represent it. - -``` -short offsetCodes_defaultDistribution[29] = - { 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 }; -``` - -#### Distribution tables - -Following the header, up to 3 distribution tables can be described. -When present, they are in this order : -- Literals lengths -- Offsets -- Match Lengths - -The content to decode depends on their respective encoding mode : -- `Predefined_Mode` : no content. Use the predefined distribution table. -- `RLE_Mode` : 1 byte. This is the only code to use across the whole compressed block. -- `FSE_Compressed_Mode` : A distribution table is present. -- `Repeat_Mode` : no content. Re-use distribution from previous compressed block. - -##### FSE distribution table : condensed format - -An FSE distribution table describes the probabilities of all symbols -from `0` to the last present one (included) -on a normalized scale of `1 << Accuracy_Log` . - -It's a bitstream which is read forward, in little-endian fashion. -It's not necessary to know its exact size, -since it will be discovered and reported by the decoding process. - -The bitstream starts by reporting on which scale it operates. -`Accuracy_Log = low4bits + 5`. -Note that maximum `Accuracy_Log` for literal and match lengths is `9`, -and for offsets is `8`. Higher values are considered errors. - -Then follows each symbol value, from `0` to last present one. -The number of bits used by each field is variable. -It depends on : - -- Remaining probabilities + 1 : - __example__ : - Presuming an `Accuracy_Log` of 8, - and presuming 100 probabilities points have already been distributed, - the decoder may read any value from `0` to `255 - 100 + 1 == 156` (inclusive). - Therefore, it must read `log2sup(156) == 8` bits. - -- Value decoded : small values use 1 less bit : - __example__ : - Presuming values from 0 to 156 (inclusive) are possible, - 255-156 = 99 values are remaining in an 8-bits field. - They are used this way : - first 99 values (hence from 0 to 98) use only 7 bits, - values from 99 to 156 use 8 bits. - This is achieved through this scheme : - - | Value read | Value decoded | Number of bits used | - | ---------- | ------------- | ------------------- | - | 0 - 98 | 0 - 98 | 7 | - | 99 - 127 | 99 - 127 | 8 | - | 128 - 226 | 0 - 98 | 7 | - | 227 - 255 | 128 - 156 | 8 | - -Symbols probabilities are read one by one, in order. - -Probability is obtained from Value decoded by following formula : -`Proba = value - 1` - -It means value `0` becomes negative probability `-1`. -`-1` is a special probability, which means "less than 1". -Its effect on distribution table is described in [next paragraph]. -For the purpose of calculating cumulated distribution, it counts as one. - -[next paragraph]:#fse-decoding--from-normalized-distribution-to-decoding-tables - -When a symbol has a __probability__ of `zero`, -it is followed by a 2-bits repeat flag. -This repeat flag tells how many probabilities of zeroes follow the current one. -It provides a number ranging from 0 to 3. -If it is a 3, another 2-bits repeat flag follows, and so on. - -When last symbol reaches cumulated total of `1 << Accuracy_Log`, -decoding is complete. -If the last symbol makes cumulated total go above `1 << Accuracy_Log`, -distribution is considered corrupted. - -Then the decoder can tell how many bytes were used in this process, -and how many symbols are present. -The bitstream consumes a round number of bytes. -Any remaining bit within the last byte is just unused. - -##### FSE decoding : from normalized distribution to decoding tables - -The distribution of normalized probabilities is enough -to create a unique decoding table. - -It follows the following build rule : - -The table has a size of `tableSize = 1 << Accuracy_Log`. -Each cell describes the symbol decoded, -and instructions to get the next state. - -Symbols are scanned in their natural order for "less than 1" probabilities. -Symbols with this probability are being attributed a single cell, -starting from the end of the table. -These symbols define a full state reset, reading `Accuracy_Log` bits. - -All remaining symbols are sorted in their natural order. -Starting from symbol `0` and table position `0`, -each symbol gets attributed as many cells as its probability. -Cell allocation is spreaded, not linear : -each successor position follow this rule : - -``` -position += (tableSize>>1) + (tableSize>>3) + 3; -position &= tableSize-1; -``` - -A position is skipped if already occupied, -typically by a "less than 1" probability symbol. -`position` does not reset between symbols, it simply iterates through -each position in the table, switching to the next symbol when enough -states have been allocated to the current one. - -The result is a list of state values. -Each state will decode the current symbol. - -To get the `Number_of_Bits` and `Baseline` required for next state, -it's first necessary to sort all states in their natural order. -The lower states will need 1 more bit than higher ones. - -__Example__ : -Presuming a symbol has a probability of 5. -It receives 5 state values. States are sorted in natural order. - -Next power of 2 is 8. -Space of probabilities is divided into 8 equal parts. -Presuming the `Accuracy_Log` is 7, it defines 128 states. -Divided by 8, each share is 16 large. - -In order to reach 8, 8-5=3 lowest states will count "double", -taking shares twice larger, -requiring one more bit in the process. - -Numbering starts from higher states using less bits. - -| state order | 0 | 1 | 2 | 3 | 4 | -| ---------------- | ----- | ----- | ------ | ---- | ----- | -| width | 32 | 32 | 32 | 16 | 16 | -| `Number_of_Bits` | 5 | 5 | 5 | 4 | 4 | -| range number | 2 | 4 | 6 | 0 | 1 | -| `Baseline` | 32 | 64 | 96 | 0 | 16 | -| range | 32-63 | 64-95 | 96-127 | 0-15 | 16-31 | - -The next state is determined from current state -by reading the required `Number_of_Bits`, and adding the specified `Baseline`. - - -#### Bitstream - -FSE bitstreams are read in reverse direction than written. In zstd, -the compressor writes bits forward into a block and the decompressor -must read the bitstream _backwards_. - -To find the start of the bitstream it is therefore necessary to -know the offset of the last byte of the block which can be found -by counting `Block_Size` bytes after the block header. - -After writing the last bit containing information, the compressor -writes a single `1`-bit and then fills the byte with 0-7 `0` bits of -padding. The last byte of the compressed bitstream cannot be `0` for -that reason. - -When decompressing, the last byte containing the padding is the first -byte to read. The decompressor needs to skip 0-7 initial `0`-bits and -the first `1`-bit it occurs. Afterwards, the useful part of the bitstream -begins. - -##### Starting states - -The bitstream starts with initial state values, -each using the required number of bits in their respective _accuracy_, -decoded previously from their normalized distribution. - -It starts by `Literals_Length_State`, -followed by `Offset_State`, -and finally `Match_Length_State`. - -Reminder : always keep in mind that all values are read _backward_. - -##### Decoding a sequence - -A state gives a code. -A code provides `Baseline` and `Number_of_Bits` to add. -See [Symbol Decoding] section for details on each symbol. - -Decoding starts by reading the `Number_of_Bits` required to decode `Offset`. -It then does the same for `Match_Length`, -and then for `Literals_Length`. - -`Offset`, `Match_Length`, and `Literals_Length` define a sequence. -It starts by inserting the number of literals defined by `Literals_Length`, -then continue by copying `Match_Length` bytes from `currentPos - Offset`. - -If it is not the last sequence in the block, -the next operation is to update states. -Using the rules pre-calculated in the decoding tables, -`Literals_Length_State` is updated, -followed by `Match_Length_State`, -and then `Offset_State`. - -This operation will be repeated `Number_of_Sequences` times. -At the end, the bitstream shall be entirely consumed, -otherwise the bitstream is considered corrupted. - -[Symbol Decoding]:#the-codes-for-literals-lengths-match-lengths-and-offsets - -##### Repeat offsets - -As seen in [Offset Codes], the first 3 values define a repeated offset and we will call them `Repeated_Offset1`, `Repeated_Offset2`, and `Repeated_Offset3`. -They are sorted in recency order, with `Repeated_Offset1` meaning "most recent one". - -There is an exception though, when current sequence's literals length is `0`. -In this case, repeated offsets are shifted by one, -so `Repeated_Offset1` becomes `Repeated_Offset2`, `Repeated_Offset2` becomes `Repeated_Offset3`, -and `Repeated_Offset3` becomes `Repeated_Offset1 - 1_byte`. - -In the first block, the offset history is populated with the following values : 1, 4 and 8 (in order). - -Then each block gets its starting offset history from the ending values of the most recent compressed block. -Note that non-compressed blocks are skipped, -they do not contribute to offset history. - -[Offset Codes]: #offset-codes - -###### Offset updates rules - -The newest offset takes the lead in offset history, -shifting others back (up to its previous place if it was already present). - -This means that when `Repeated_Offset1` (most recent) is used, history is unmodified. -When `Repeated_Offset2` is used, it's swapped with `Repeated_Offset1`. -If any other offset is used, it becomes `Repeated_Offset1` and the rest are shift back by one. - - -Dictionary format +Dictionary Format ----------------- -`zstd` is compatible with "raw content" dictionaries, free of any format restriction, +Zstandard is compatible with "raw content" dictionaries, free of any format restriction, except that they must be at least 8 bytes. These dictionaries function as if they were just the `Content` block of a formatted dictionary. @@ -1203,10 +1258,15 @@ _Reserved ranges :_ - low range : 1 - 32767 - high range : >= (2^31) -__`Entropy_Tables`__ : following the same format as the tables in [compressed blocks]. +__`Entropy_Tables`__ : following the same format as the tables in compressed blocks. + See the relevant [FSE](#fse-table-description) + and [Huffman](#huffman-tree-description) sections for how to decode these tables. They are stored in following order : Huffman tables for literals, FSE table for offsets, FSE table for match lengths, and FSE table for literals lengths. + These tables populate the Repeat Stats literals mode and + Repeat distribution mode for sequence decoding. + It's finally followed by 3 offset values, populating recent offsets (instead of using `{1,4,8}`), stored in order, 4-bytes little-endian each, for a total of 12 bytes. Each recent offset must have a value < dictionary size. @@ -1214,9 +1274,13 @@ __`Entropy_Tables`__ : following the same format as the tables in [compressed bl __`Content`__ : The rest of the dictionary is its content. The content act as a "past" in front of data to compress or decompress, so it can be referenced in sequence commands. + As long as the amount of data decoded from this frame is less than or + equal to the window-size, sequence commands may specify offsets longer + than the lenght of total decoded output so far to reference back to the + dictionary. After the total output has surpassed the window size however, + this is no longer allowed and the dictionary is no longer accessible. [compressed blocks]: #the-format-of-compressed_block - Appendix A - Decoding tables for predefined codes ------------------------------------------------- @@ -1402,6 +1466,7 @@ to crosscheck that an implementation implements the decoding table generation al Version changes --------------- +- 0.2.4 : section restructuring, by Sean Purcell - 0.2.3 : clarified several details, by Sean Purcell - 0.2.2 : added predefined codes, by Johannes Rudolph - 0.2.1 : clarify field names, by Przemyslaw Skibinski From 107c9a4e421f28fd80d4b6b3e3751c651dc2ef7c Mon Sep 17 00:00:00 2001 From: Dima Krasner Date: Sat, 18 Feb 2017 23:30:57 +0200 Subject: [PATCH 103/223] Moved to contrib --- README.md | 2 +- build/README.md | 1 - contrib/meson/README | 3 +++ {build => contrib}/meson/meson.build | 0 {build => contrib}/meson/meson_options.txt | 0 5 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 contrib/meson/README rename {build => contrib}/meson/meson.build (100%) rename {build => contrib}/meson/meson_options.txt (100%) diff --git a/README.md b/README.md index 7782972db..229571b3e 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ to create `zstd` binary, and `libzstd` dynamic and static libraries. #### Meson -A Meson project is provided within `build/meson`. +A Meson project is provided within `contrib/meson`. #### Visual (Windows) diff --git a/build/README.md b/build/README.md index ca88e5cfd..c4abe9efd 100644 --- a/build/README.md +++ b/build/README.md @@ -5,7 +5,6 @@ Projects for various integrated development environments (IDE) The following projects are included with the zstd distribution: - `cmake` - CMake project contributed by Artyom Dymchenko -- `meson` - Meson project contributed by Dima Krasner - `VS2005` - Visual Studio 2005 project - `VS2008` - Visual Studio 2008 project - `VS2010` - Visual Studio 2010 project (which also works well with Visual Studio 2012, 2013, 2015) diff --git a/contrib/meson/README b/contrib/meson/README new file mode 100644 index 000000000..0b5331e6d --- /dev/null +++ b/contrib/meson/README @@ -0,0 +1,3 @@ +This Meson project is provided with no guarantee and maintained by Dima Krasner . + +It outputs one libzstd, either shared or static, depending on default_library. diff --git a/build/meson/meson.build b/contrib/meson/meson.build similarity index 100% rename from build/meson/meson.build rename to contrib/meson/meson.build diff --git a/build/meson/meson_options.txt b/contrib/meson/meson_options.txt similarity index 100% rename from build/meson/meson_options.txt rename to contrib/meson/meson_options.txt From 3b89eb0c209e5be4eb3fd3bffa236e5f14f6e678 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Mon, 20 Feb 2017 01:26:05 -0800 Subject: [PATCH 104/223] updated NEWS with meson build by Dima Krasner --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index a710da8b9..be3349754 100644 --- a/NEWS +++ b/NEWS @@ -4,6 +4,7 @@ cli : new : advanced benchmark command --priority=rt cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77 API : new : ZSTD_getFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize(), by Sean Purcell API : change : ZSTD_compress*() with srcSize==0 create an empty-frame of known size +build:new : meson build system in contrib/meson, by Dima Krasner doc : new : educational decoder, by Sean Purcell v1.1.3 From 83775d9e05a4014d44c97fe913547169aba174c3 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 20 Feb 2017 11:11:50 +0100 Subject: [PATCH 105/223] replace times() with clock_gettime(CLOCK_MONOTONIC, x) --- programs/util.h | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/programs/util.h b/programs/util.h index 0f588f110..54719cc87 100644 --- a/programs/util.h +++ b/programs/util.h @@ -109,20 +109,18 @@ extern "C" { /*-**************************************** * Time functions ******************************************/ -#if (PLATFORM_POSIX_VERSION >= 1) -#include -#include /* times */ - typedef U64 UTIL_time_t; - UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=sysconf(_SC_CLK_TCK); } - UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { struct tms junk; clock_t newTicks = (clock_t) times(&junk); (void)junk; *x = (UTIL_time_t)newTicks; } - UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / ticksPerSecond; } - UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / ticksPerSecond; } -#elif defined(_WIN32) /* Windows */ +#if defined(_WIN32) /* Windows */ typedef LARGE_INTEGER UTIL_time_t; UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { if (!QueryPerformanceFrequency(ticksPerSecond)) fprintf(stderr, "ERROR: QueryPerformance not present\n"); } UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { QueryPerformanceCounter(x); } UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; } UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; } +#elif (_POSIX_TIMERS > 0) && defined(_POSIX_MONOTONIC_CLOCK) /* defined in */ + typedef struct timespec UTIL_time_t; + UTIL_STATIC void UTIL_initTimer(UTIL_time_t* clockResolution) { clock_getres(CLOCK_MONOTONIC, clockResolution); } + UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { if (clock_gettime(CLOCK_MONOTONIC, x) == -1) { fprintf(stderr, "clock_gettime error"); }; } + UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return (1000000000ULL * (clockEnd.tv_sec - clockStart.tv_sec) + (clockEnd.tv_nsec - clockStart.tv_nsec))/1000ULL; } + UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd.tv_sec - clockStart.tv_sec) + (clockEnd.tv_nsec - clockStart.tv_nsec); } #else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */ typedef clock_t UTIL_time_t; UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=0; } From e052c60540728543d02fd111cfbac1597652384a Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 20 Feb 2017 11:27:11 +0100 Subject: [PATCH 106/223] introduce UTIL_freq_t --- programs/bench.c | 2 +- programs/util.h | 25 ++++++++++++++----------- zlibWrapper/examples/zwrapbench.c | 2 +- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index 40c700d11..40373fbd4 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -169,7 +169,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, size_t cSize = 0; double ratio = 0.; U32 nbBlocks; - UTIL_time_t ticksPerSecond; + UTIL_freq_t ticksPerSecond; /* checks */ if (!compressedBuffer || !resultBuffer || !blockTable || !ctx || !dctx) diff --git a/programs/util.h b/programs/util.h index 54719cc87..8dd0df8b9 100644 --- a/programs/util.h +++ b/programs/util.h @@ -110,28 +110,31 @@ extern "C" { * Time functions ******************************************/ #if defined(_WIN32) /* Windows */ + typedef LARGE_INTEGER UTIL_freq_t; typedef LARGE_INTEGER UTIL_time_t; - UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { if (!QueryPerformanceFrequency(ticksPerSecond)) fprintf(stderr, "ERROR: QueryPerformance not present\n"); } + UTIL_STATIC void UTIL_initTimer(UTIL_freq_t* ticksPerSecond) { if (!QueryPerformanceFrequency(ticksPerSecond)) fprintf(stderr, "ERROR: QueryPerformance not present\n"); } UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { QueryPerformanceCounter(x); } - UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; } - UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; } + UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; } + UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; } #elif (_POSIX_TIMERS > 0) && defined(_POSIX_MONOTONIC_CLOCK) /* defined in */ + typedef struct timespec UTIL_freq_t; typedef struct timespec UTIL_time_t; - UTIL_STATIC void UTIL_initTimer(UTIL_time_t* clockResolution) { clock_getres(CLOCK_MONOTONIC, clockResolution); } + UTIL_STATIC void UTIL_initTimer(UTIL_freq_t* clockResolution) { clock_getres(CLOCK_MONOTONIC, clockResolution); } UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { if (clock_gettime(CLOCK_MONOTONIC, x) == -1) { fprintf(stderr, "clock_gettime error"); }; } - UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return (1000000000ULL * (clockEnd.tv_sec - clockStart.tv_sec) + (clockEnd.tv_nsec - clockStart.tv_nsec))/1000ULL; } - UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd.tv_sec - clockStart.tv_sec) + (clockEnd.tv_nsec - clockStart.tv_nsec); } + UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return (1000000000ULL * (clockEnd.tv_sec - clockStart.tv_sec) + (clockEnd.tv_nsec - clockStart.tv_nsec))/1000ULL; } + UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd.tv_sec - clockStart.tv_sec) + (clockEnd.tv_nsec - clockStart.tv_nsec); } #else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */ + typedef clock_t UTIL_freq_t; typedef clock_t UTIL_time_t; - UTIL_STATIC void UTIL_initTimer(UTIL_time_t* ticksPerSecond) { *ticksPerSecond=0; } + UTIL_STATIC void UTIL_initTimer(UTIL_freq_t* ticksPerSecond) { *ticksPerSecond=0; } UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { *x = clock(); } - UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } - UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } + UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } + UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { (void)ticksPerSecond; return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; } #endif /* returns time span in microseconds */ -UTIL_STATIC U64 UTIL_clockSpanMicro( UTIL_time_t clockStart, UTIL_time_t ticksPerSecond ) +UTIL_STATIC U64 UTIL_clockSpanMicro( UTIL_time_t clockStart, UTIL_freq_t ticksPerSecond ) { UTIL_time_t clockEnd; UTIL_getTime(&clockEnd); @@ -139,7 +142,7 @@ UTIL_STATIC U64 UTIL_clockSpanMicro( UTIL_time_t clockStart, UTIL_time_t ticksPe } -UTIL_STATIC void UTIL_waitForNextTick(UTIL_time_t ticksPerSecond) +UTIL_STATIC void UTIL_waitForNextTick(UTIL_freq_t ticksPerSecond) { UTIL_time_t clockStart, clockEnd; UTIL_getTime(&clockStart); diff --git a/zlibWrapper/examples/zwrapbench.c b/zlibWrapper/examples/zwrapbench.c index 328c1096b..23c3ca4da 100644 --- a/zlibWrapper/examples/zwrapbench.c +++ b/zlibWrapper/examples/zwrapbench.c @@ -160,7 +160,7 @@ static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize, ZSTD_CCtx* const ctx = ZSTD_createCCtx(); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); U32 nbBlocks; - UTIL_time_t ticksPerSecond; + UTIL_freq_t ticksPerSecond; /* checks */ if (!compressedBuffer || !resultBuffer || !blockTable || !ctx || !dctx) From da4a0f30af8f4d65a8e3883f73980c36008d48bc Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Mon, 20 Feb 2017 12:18:15 +0100 Subject: [PATCH 107/223] util.h: use mach_absolute_time for macOS --- programs/util.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/programs/util.h b/programs/util.h index 8dd0df8b9..ef526b62e 100644 --- a/programs/util.h +++ b/programs/util.h @@ -116,6 +116,14 @@ extern "C" { UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { QueryPerformanceCounter(x); } UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; } UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart; } +#elif defined(__APPLE__) && defined(__MACH__) + #include + typedef mach_timebase_info_data_t UTIL_freq_t; + typedef U64 UTIL_time_t; + UTIL_STATIC void UTIL_initTimer(UTIL_freq_t* rate) { mach_timebase_info(&rate); } + UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { *x = mach_absolute_time(); } + UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_freq_t rate, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return (((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom))/1000ULL; } + UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_freq_t rate, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom); } #elif (_POSIX_TIMERS > 0) && defined(_POSIX_MONOTONIC_CLOCK) /* defined in */ typedef struct timespec UTIL_freq_t; typedef struct timespec UTIL_time_t; From 517577bf531b09204ad95ffeef4a45abb7c2df9c Mon Sep 17 00:00:00 2001 From: Anders Oleson Date: Mon, 20 Feb 2017 12:08:59 -0800 Subject: [PATCH 108/223] spelling fixes in comments i.e. occurred labeled Huffman --- lib/common/mem.h | 4 ++-- lib/common/threading.h | 2 +- lib/compress/huf_compress.c | 2 +- lib/compress/zstd_compress.c | 2 +- lib/compress/zstd_opt.h | 2 +- lib/decompress/huf_decompress.c | 2 +- lib/decompress/zstd_decompress.c | 6 +++--- lib/zstd.h | 4 ++-- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/common/mem.h b/lib/common/mem.h index 1c223fe5e..7a3f72141 100644 --- a/lib/common/mem.h +++ b/lib/common/mem.h @@ -76,11 +76,11 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. * The below switch allow to select different access method for improved performance. * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable). * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. * Method 2 : direct access. This method is portable but violate C standard. * It can generate buggy code on targets depending on alignment. - * In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6) * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. * Prefer these methods in priority order (0 > 1 > 2) */ diff --git a/lib/common/threading.h b/lib/common/threading.h index 74b2ec042..c0086139e 100644 --- a/lib/common/threading.h +++ b/lib/common/threading.h @@ -73,7 +73,7 @@ int _pthread_join(pthread_t* thread, void** value_ptr); */ -#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection mathod */ +#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */ /* === POSIX Systems === */ # include diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c index bf464daaf..7869ccf64 100644 --- a/lib/compress/huf_compress.c +++ b/lib/compress/huf_compress.c @@ -127,7 +127,7 @@ struct HUF_CElt_s { }; /* typedef'd to HUF_CElt within "huf.h" */ /*! HUF_writeCTable() : - `CTable` : huffman tree to save, using huf representation. + `CTable` : Huffman tree to save, using huf representation. @return : size of saved CTable */ size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 91e81d9c2..924189b0c 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -1752,7 +1752,7 @@ static size_t ZSTD_BtFindBestMatch_selectMLS_extDict ( #define NEXT_IN_CHAIN(d, mask) chainTable[(d) & mask] /* Update chains up to ip (excluded) - Assumption : always within prefix (ie. not within extDict) */ + Assumption : always within prefix (i.e. not within extDict) */ FORCE_INLINE U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls) { diff --git a/lib/compress/zstd_opt.h b/lib/compress/zstd_opt.h index 8862bbd6b..ac418b61c 100644 --- a/lib/compress/zstd_opt.h +++ b/lib/compress/zstd_opt.h @@ -203,7 +203,7 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B /* Update hashTable3 up to ip (excluded) - Assumption : always within prefix (ie. not within extDict) */ + Assumption : always within prefix (i.e. not within extDict) */ FORCE_INLINE U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* zc, const BYTE* ip) { diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c index a342dfb1e..889a22a8c 100644 --- a/lib/decompress/huf_decompress.c +++ b/lib/decompress/huf_decompress.c @@ -102,7 +102,7 @@ size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize) /* Table header */ { DTableDesc dtd = HUF_getDTableDesc(DTable); - if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, huffman tree cannot fit in */ + if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ dtd.tableType = 0; dtd.tableLog = (BYTE)tableLog; memcpy(DTable, &dtd, sizeof(dtd)); diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index 404d0b83d..eda8b9dd5 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -310,7 +310,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t * compatible with legacy mode * @return : decompressed size of the single frame pointed to be `src` if known, otherwise * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined -* - ZSTD_CONTENTSIZE_ERROR if an error occured (e.g. invalid magic number, srcSize too small) */ +* - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) @@ -1049,7 +1049,7 @@ size_t ZSTD_execSequence(BYTE* op, if (sequence.offset < 8) { /* close range match, overlap */ static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ - static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */ + static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ int const sub2 = dec64table[sequence.offset]; op[0] = match[0]; op[1] = match[1]; @@ -1270,7 +1270,7 @@ size_t ZSTD_execSequenceLong(BYTE* op, if (sequence.offset < 8) { /* close range match, overlap */ static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ - static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */ + static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ int const sub2 = dec64table[sequence.offset]; op[0] = match[0]; op[1] = match[1]; diff --git a/lib/zstd.h b/lib/zstd.h index a71867837..c0a1c7d1c 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -39,7 +39,7 @@ extern "C" { zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22. - Levels >= 20, labelled `--ultra`, should be used with caution, as they require more memory. + Levels >= 20, labeled `--ultra`, should be used with caution, as they require more memory. Compression can be done in: - a single step (described as Simple API) - a single step, reusing a context (described as Explicit memory management) @@ -416,7 +416,7 @@ ZSTDLIB_API size_t ZSTD_getFrameCompressedSize(const void* src, size_t srcSize); * to `ZSTD_frameHeaderSize_max` is guaranteed to be large enough in all cases. * @return : decompressed size of the frame pointed to be `src` if known, otherwise * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined -* - ZSTD_CONTENTSIZE_ERROR if an error occured (e.g. invalid magic number, srcSize too small) */ +* - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */ ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); /*! ZSTD_findDecompressedSize() : From 6e18d33122a17648497fdddb3ce7806e9dfb593d Mon Sep 17 00:00:00 2001 From: Soojin Nam Date: Tue, 21 Feb 2017 09:51:40 +0900 Subject: [PATCH 109/223] original size unknown --- examples/dictionary_decompression.c | 2 +- examples/simple_decompression.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/dictionary_decompression.c b/examples/dictionary_decompression.c index 75183505d..deaf3888e 100644 --- a/examples/dictionary_decompression.c +++ b/examples/dictionary_decompression.c @@ -78,7 +78,7 @@ static void decompress(const char* fname, const ZSTD_DDict* ddict) size_t cSize; void* const cBuff = loadFile_orDie(fname, &cSize); unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize); - if (rSize==0) { + if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) { fprintf(stderr, "%s : original size unknown \n", fname); exit(6); } diff --git a/examples/simple_decompression.c b/examples/simple_decompression.c index 09b27baa6..e23f14887 100644 --- a/examples/simple_decompression.c +++ b/examples/simple_decompression.c @@ -63,7 +63,7 @@ static void decompress(const char* fname) size_t cSize; void* const cBuff = loadFile_orDie(fname, &cSize); unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize); - if (rSize==0) { + if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) { printf("%s : original size unknown. Use streaming decompression instead. \n", fname); exit(5); } From 1b59333c828b15d207bf4e861b09ab094f247dc9 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 07:33:45 +0100 Subject: [PATCH 110/223] util.h: restore times() --- programs/util.h | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/programs/util.h b/programs/util.h index ef526b62e..b89ae3bf5 100644 --- a/programs/util.h +++ b/programs/util.h @@ -124,13 +124,14 @@ extern "C" { UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { *x = mach_absolute_time(); } UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_freq_t rate, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return (((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom))/1000ULL; } UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_freq_t rate, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom); } -#elif (_POSIX_TIMERS > 0) && defined(_POSIX_MONOTONIC_CLOCK) /* defined in */ - typedef struct timespec UTIL_freq_t; - typedef struct timespec UTIL_time_t; - UTIL_STATIC void UTIL_initTimer(UTIL_freq_t* clockResolution) { clock_getres(CLOCK_MONOTONIC, clockResolution); } - UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { if (clock_gettime(CLOCK_MONOTONIC, x) == -1) { fprintf(stderr, "clock_gettime error"); }; } - UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return (1000000000ULL * (clockEnd.tv_sec - clockStart.tv_sec) + (clockEnd.tv_nsec - clockStart.tv_nsec))/1000ULL; } - UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd.tv_sec - clockStart.tv_sec) + (clockEnd.tv_nsec - clockStart.tv_nsec); } +#elif (PLATFORM_POSIX_VERSION >= 200112L) + #include /* times */ + typedef U64 UTIL_freq_t; + typedef U64 UTIL_time_t; + UTIL_STATIC void UTIL_initTimer(UTIL_freq_t* ticksPerSecond) { *ticksPerSecond=sysconf(_SC_CLK_TCK); } + UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { struct tms junk; clock_t newTicks = (clock_t) times(&junk); (void)junk; *x = (UTIL_time_t)newTicks; } + UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / ticksPerSecond; } + UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_freq_t ticksPerSecond, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / ticksPerSecond; } #else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */ typedef clock_t UTIL_freq_t; typedef clock_t UTIL_time_t; From 54a7f8591f130983d8c2821e39aef853d00c48df Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 08:01:55 +0100 Subject: [PATCH 111/223] travis.yml: remove tests that overlap with Circle CI --- .travis.yml | 66 +++++++++++++++++------------------------------------ 1 file changed, 21 insertions(+), 45 deletions(-) diff --git a/.travis.yml b/.travis.yml index 82b2b03bb..f4e38f2ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ matrix: # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - - env: Ubu=12.04cont Cmd="make zlibwrapper && make clean && make -C tests test-symbols && make clean && make -C tests test-zstd-nolegacy && make clean && make cmaketest && make clean && make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean" + - env: Ubu=12.04cont Cmd="make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean" os: linux sudo: false language: cpp @@ -33,10 +33,6 @@ matrix: # Standard Ubuntu 12.04 LTS Server Edition 64 bit - - env: Ubu=12.04 Cmd="make -C programs zstd-small zstd-decompress zstd-compress && make -C programs clean && make -C tests versionsTest test-longmatch" - os: linux - sudo: required - - env: Ubu=12.04 Cmd="make asan32" os: linux sudo: required @@ -65,6 +61,23 @@ matrix: # Ubuntu 14.04 LTS Server Edition 64 bit + - env: Ubu=14.04 Cmd="make -C contrib/pzstd googletest32 + && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean" + os: linux + dist: trusty + sudo: required + install: + - export CXX="g++-4.8" CC="gcc-4.8" + addons: + apt: + packages: + - libc6-dev-i386 + - g++-multilib + - gcc-4.8 + - gcc-4.8-multilib + - g++-4.8 + - g++-4.8-multilib + - env: Ubu=14.04 Cmd="make armtest" dist: trusty sudo: required @@ -107,6 +120,8 @@ matrix: - qemu-user-static - gcc-powerpc-linux-gnu + + # other feature branches => short tests - env: Ubu=14.04 Cmd='make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest' os: linux dist: trusty @@ -116,14 +131,7 @@ matrix: packages: - valgrind - - - # other feature branches => short tests - - env: Ubu=12.04cont Cmd="make test && make clean && make travis-install" - os: linux - sudo: false - - - env: Ubu=14.04 Cmd="make -C tests test32" + - env: Ubu=14.04 Cmd="make zlibwrapper && make clean && make -C tests test-zstd-nolegacy && make clean && make -C tests test32 versionsTest" os: linux dist: trusty sudo: required @@ -133,39 +141,7 @@ matrix: - libc6-dev-i386 - gcc-multilib - - env: Ubu=14.04 Cmd="make gpptest && make clean && make gnu90test && make clean - && make c99test && make clean && make gnu99test && make clean - && make clangtest && make clean && make -C contrib/pzstd googletest32 - && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean" - os: linux - dist: trusty - sudo: required - install: - - export CXX="g++-4.8" CC="gcc-4.8" - addons: - apt: - packages: - - libc6-dev-i386 - - g++-multilib - - gcc-4.8 - - gcc-4.8-multilib - - g++-4.8 - - g++-4.8-multilib - - env: Ubu=14.04 Cmd="make gcc5test && make clean && make gcc6test && make clean && make -C tests dll" - os: linux - dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-multilib - - gcc-5 - - gcc-5-multilib - - gcc-6 - - gcc-6-multilib script: - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') From 3a4da1fd86315d2d478c6d9b12423fd1cdddcc1f Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 08:39:02 +0100 Subject: [PATCH 112/223] travis.yml: join pzstd tests --- .travis.yml | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index f4e38f2ac..21905393d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,21 +8,6 @@ matrix: # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - - env: Ubu=12.04cont Cmd="make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean" - os: linux - sudo: false - language: cpp - install: - - export CXX="g++-4.8" CC="gcc-4.8" - - export TESTFLAGS='--gtest_filter=-*ExtremelyLarge*' - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-4.8 - - g++-4.8 - - env: Ubu=12.04cont Cmd="make usan" os: linux sudo: false @@ -61,8 +46,8 @@ matrix: # Ubuntu 14.04 LTS Server Edition 64 bit - - env: Ubu=14.04 Cmd="make -C contrib/pzstd googletest32 - && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean" + - env: Ubu=14.04 Cmd="make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean + && make -C contrib/pzstd googletest32 && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean" os: linux dist: trusty sudo: required @@ -77,7 +62,7 @@ matrix: - gcc-4.8-multilib - g++-4.8 - g++-4.8-multilib - + - env: Ubu=14.04 Cmd="make armtest" dist: trusty sudo: required @@ -122,7 +107,7 @@ matrix: # other feature branches => short tests - - env: Ubu=14.04 Cmd='make -C lib all && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest' + - env: Ubu=14.04 Cmd='make lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest' os: linux dist: trusty sudo: required From d2e5a56a23164a22abc51a92e22e492672005b26 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 09:59:19 +0100 Subject: [PATCH 113/223] travis.yml: switch asan32 to Ubuntu 14.04 --- .travis.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 21905393d..b95ebbee2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,9 +17,9 @@ matrix: sudo: false - # Standard Ubuntu 12.04 LTS Server Edition 64 bit - - env: Ubu=12.04 Cmd="make asan32" + - env: Ubu=14.04 Cmd="make asan32" os: linux + dist: trusty sudo: required addons: apt: @@ -29,6 +29,8 @@ matrix: - libc6-dev-i386 - gcc-multilib + + # Standard Ubuntu 12.04 LTS Server Edition 64 bit - env: Ubu=12.04 Cmd='cd contrib/pzstd && make googletest && make tsan && make check && make clean && make asan && make check && make clean && cd ../..' os: linux sudo: required @@ -131,12 +133,13 @@ matrix: script: - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') - # cron & master => long tests, as this is the final step towards a Release - # dev => normal tests; other feature branches => short tests (number > 11) + # cron & master => long tests, as this is the final step towards a Release + # dev => normal tests + # other feature branches => short tests (number > 10) - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "master" ]; then FUZZERTEST=-T10mn sh -c "$Cmd" || travis_terminate 1; else - if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 11 ] || [ "$TRAVIS_BRANCH" = "dev" ]; then + if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 10 ] || [ "$TRAVIS_BRANCH" = "dev" ]; then sh -c "$Cmd" || travis_terminate 1; fi fi From 74dcd8d15fdf1732300b793186ea9491fa1bebed Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 12:22:05 +0100 Subject: [PATCH 114/223] bench.c: use a single ticksPerSecond --- .travis.yml | 2 ++ programs/bench.c | 19 ++++++++----------- programs/util.h | 2 +- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index b95ebbee2..d685132f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,8 @@ matrix: packages: - libc6-dev-i386 - gcc-multilib + - libclang-3.5-dev + - libclang-common-3.5-dev # Standard Ubuntu 12.04 LTS Server Edition 64 bit diff --git a/programs/bench.c b/programs/bench.c index 40373fbd4..663b30743 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -230,7 +230,7 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, /* Bench */ { U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL); U64 const crcOrig = g_decodeOnly ? 0 : XXH64(srcBuffer, srcSize, 0); - UTIL_time_t coolTime, coolTick; + UTIL_time_t coolTime; U64 const maxTime = (g_nbSeconds * TIMELOOP_MICROSEC) + 1; U64 totalCTime=0, totalDTime=0; U32 cCompleted=g_decodeOnly, dCompleted=0; @@ -238,27 +238,25 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, const char* const marks[NB_MARKS] = { " |", " /", " =", "\\" }; U32 markNb = 0; - UTIL_initTimer(&coolTick); UTIL_getTime(&coolTime); DISPLAYLEVEL(2, "\r%79s\r", ""); while (!cCompleted || !dCompleted) { /* overheat protection */ - if (UTIL_clockSpanMicro(coolTime, coolTick) > ACTIVEPERIOD_MICROSEC) { + if (UTIL_clockSpanMicro(coolTime, ticksPerSecond) > ACTIVEPERIOD_MICROSEC) { DISPLAYLEVEL(2, "\rcooling down ... \r"); UTIL_sleep(COOLPERIOD_SEC); UTIL_getTime(&coolTime); } if (!g_decodeOnly) { - UTIL_time_t clockTick, clockStart; + UTIL_time_t clockStart; /* Compression */ DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (U32)srcSize); if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ UTIL_sleepMilli(1); /* give processor time to other processes */ UTIL_waitForNextTick(ticksPerSecond); - UTIL_initTimer(&clockTick); UTIL_getTime(&clockStart); if (!cCompleted) { /* still some time to do compression tests */ @@ -300,9 +298,9 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, blockTable[blockNb].cSize = rSize; } nbLoops++; - } while (UTIL_clockSpanMicro(clockStart, clockTick) < clockLoop); + } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop); ZSTD_freeCDict(cdict); - { U64 const clockSpanMicro = UTIL_clockSpanMicro(clockStart, clockTick); + { U64 const clockSpanMicro = UTIL_clockSpanMicro(clockStart, ticksPerSecond); if (clockSpanMicro < fastestC*nbLoops) fastestC = clockSpanMicro / nbLoops; totalCTime += clockSpanMicro; cCompleted = (totalCTime >= maxTime); @@ -332,10 +330,9 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, if (!dCompleted) { U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1; U32 nbLoops = 0; - UTIL_time_t clockStart, clockTick; + UTIL_time_t clockStart; ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictBufferSize); if (!ddict) EXM_THROW(2, "ZSTD_createDDict() allocation failure"); - UTIL_initTimer(&clockTick); UTIL_getTime(&clockStart); do { U32 blockNb; @@ -353,9 +350,9 @@ static int BMK_benchMem(const void* srcBuffer, size_t srcSize, blockTable[blockNb].resSize = regenSize; } nbLoops++; - } while (UTIL_clockSpanMicro(clockStart, clockTick) < clockLoop); + } while (UTIL_clockSpanMicro(clockStart, ticksPerSecond) < clockLoop); ZSTD_freeDDict(ddict); - { U64 const clockSpanMicro = UTIL_clockSpanMicro(clockStart, clockTick); + { U64 const clockSpanMicro = UTIL_clockSpanMicro(clockStart, ticksPerSecond); if (clockSpanMicro < fastestD*nbLoops) fastestD = clockSpanMicro / nbLoops; totalDTime += clockSpanMicro; dCompleted = (totalDTime >= maxTime); diff --git a/programs/util.h b/programs/util.h index b89ae3bf5..59e19d027 100644 --- a/programs/util.h +++ b/programs/util.h @@ -120,7 +120,7 @@ extern "C" { #include typedef mach_timebase_info_data_t UTIL_freq_t; typedef U64 UTIL_time_t; - UTIL_STATIC void UTIL_initTimer(UTIL_freq_t* rate) { mach_timebase_info(&rate); } + UTIL_STATIC void UTIL_initTimer(UTIL_freq_t* rate) { mach_timebase_info(rate); } UTIL_STATIC void UTIL_getTime(UTIL_time_t* x) { *x = mach_absolute_time(); } UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_freq_t rate, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return (((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom))/1000ULL; } UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_freq_t rate, UTIL_time_t clockStart, UTIL_time_t clockEnd) { return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom); } From 4ec26e53b788d3bec99eadd77eb1c9726b7f3aca Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 13:40:28 +0100 Subject: [PATCH 115/223] travis.yml: use clang-4.0 for asan32 test --- .travis.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index d685132f5..0ac8efb80 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,19 +17,18 @@ matrix: sudo: false - - env: Ubu=14.04 Cmd="make asan32" + - env: Ubu=14.04 Cmd='make -C tests test32 CC=clang-4.0 MOREFLAGS="-g -fsanitize=address"' os: linux dist: trusty sudo: required addons: apt: sources: - - ubuntu-toolchain-r-test + - llvm-toolchain-trusty-4.0 packages: - libc6-dev-i386 - gcc-multilib - - libclang-3.5-dev - - libclang-common-3.5-dev + - clang-4.0 # Standard Ubuntu 12.04 LTS Server Edition 64 bit From 3a751edeaedf79f320aae63de4134e0fcf54786e Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 15:57:03 +0100 Subject: [PATCH 116/223] uasan --- .travis.yml | 71 ++++++++++++++++++++--------------------------------- 1 file changed, 27 insertions(+), 44 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0ac8efb80..7379fc504 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,20 @@ matrix: os: linux sudo: false + - env: Ubu=12.04cont Cmd="make uasan" + os: linux + sudo: false + + - env: Ubu=14.04 Cmd='make test CC=clang-4.0 MOREFLAGS="-g -fsanitize=address -fsanitize=undefined"' + os: linux + dist: trusty + sudo: required + addons: + apt: + sources: + - llvm-toolchain-trusty-4.0 + packages: + - clang-4.0 - env: Ubu=14.04 Cmd='make -C tests test32 CC=clang-4.0 MOREFLAGS="-g -fsanitize=address"' os: linux @@ -31,9 +45,13 @@ matrix: - clang-4.0 - # Standard Ubuntu 12.04 LTS Server Edition 64 bit - - env: Ubu=12.04 Cmd='cd contrib/pzstd && make googletest && make tsan && make check && make clean && make asan && make check && make clean && cd ../..' + # Ubuntu 14.04 LTS Server Edition 64 bit + - env: Ubu=14.04 Cmd='cd contrib/pzstd && make googletest pzstd tests check && make clean + && make googletest32 all32 check && make clean + && make googletest tsan check && make clean + && make asan check && make clean' os: linux + dist: trusty sudo: required install: - export CXX="g++-6" CC="gcc-6" @@ -43,30 +61,15 @@ matrix: apt: sources: - ubuntu-toolchain-r-test - packages: - - gcc-6 - - g++-6 - - - # Ubuntu 14.04 LTS Server Edition 64 bit - - env: Ubu=14.04 Cmd="make -C contrib/pzstd googletest pzstd tests check && make -C contrib/pzstd clean - && make -C contrib/pzstd googletest32 && make -C contrib/pzstd all32 && make -C contrib/pzstd check && make -C contrib/pzstd clean" - os: linux - dist: trusty - sudo: required - install: - - export CXX="g++-4.8" CC="gcc-4.8" - addons: - apt: packages: - libc6-dev-i386 - g++-multilib - - gcc-4.8 - - gcc-4.8-multilib - - g++-4.8 - - g++-4.8-multilib + - gcc-6 + - gcc-6-multilib + - g++-6 + - g++-6-multilib - - env: Ubu=14.04 Cmd="make armtest" + - env: Ubu=14.04 Cmd="make armtest && make clean && make aarch64test" dist: trusty sudo: required addons: @@ -76,19 +79,10 @@ matrix: - qemu-user-static - gcc-arm-linux-gnueabi - libc6-dev-armel-cross - - - env: Ubu=14.04 Cmd="make aarch64test" - dist: trusty - sudo: required - addons: - apt: - packages: - - qemu-system-arm - - qemu-user-static - gcc-aarch64-linux-gnu - libc6-dev-arm64-cross - - env: Ubu=14.04 Cmd='make ppctest' + - env: Ubu=14.04 Cmd='make ppctest && make clean && make ppc64test' dist: trusty sudo: required addons: @@ -98,17 +92,6 @@ matrix: - qemu-user-static - gcc-powerpc-linux-gnu - - env: Ubu=14.04 Cmd='make ppc64test' - dist: trusty - sudo: required - addons: - apt: - packages: - - qemu-system-ppc - - qemu-user-static - - gcc-powerpc-linux-gnu - - # other feature branches => short tests - env: Ubu=14.04 Cmd='make lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest' os: linux @@ -137,7 +120,7 @@ script: # cron & master => long tests, as this is the final step towards a Release # dev => normal tests # other feature branches => short tests (number > 10) - - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "master" ]; then + - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "asan" ]; then FUZZERTEST=-T10mn sh -c "$Cmd" || travis_terminate 1; else if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 10 ] || [ "$TRAVIS_BRANCH" = "dev" ]; then From 684858e7b7924d7789395ec5950d4b29315a4516 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 18:17:24 +0100 Subject: [PATCH 117/223] fix memory leaks --- .travis.yml | 15 +-------------- contrib/pzstd/Makefile | 17 +++++++++++++++++ programs/zstdcli.c | 4 ++-- tests/zstreamtest.c | 2 ++ 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7379fc504..38ed23431 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,14 +8,6 @@ matrix: # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - - env: Ubu=12.04cont Cmd="make usan" - os: linux - sudo: false - - - env: Ubu=12.04cont Cmd="make asan" - os: linux - sudo: false - - env: Ubu=12.04cont Cmd="make uasan" os: linux sudo: false @@ -46,17 +38,12 @@ matrix: # Ubuntu 14.04 LTS Server Edition 64 bit - - env: Ubu=14.04 Cmd='cd contrib/pzstd && make googletest pzstd tests check && make clean - && make googletest32 all32 check && make clean - && make googletest tsan check && make clean - && make asan check && make clean' + - env: Ubu=14.04 Cmd='cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && test-pzstd-asan' os: linux dist: trusty sudo: required install: - export CXX="g++-6" CC="gcc-6" - - export LDFLAGS="-fuse-ld=gold" - - export TESTFLAGS='--gtest_filter=-*ExtremelyLarge*' addons: apt: sources: diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile index f148bfd8e..10a133dd7 100644 --- a/contrib/pzstd/Makefile +++ b/contrib/pzstd/Makefile @@ -85,6 +85,23 @@ endif .PHONY: default default: all +.PHONY: test-pzstd +test-pzstd: TESTFLAGS='--gtest_filter=-*ExtremelyLarge*' +test-pzstd: clean googletest pzstd tests check + +.PHONY: test-pzstd32 +test-pzstd32: clean googletest32 all32 check + +.PHONY: test-pzstd-tsan +test-pzstd-tsan: LDFLAGS="-fuse-ld=gold" +test-pzstd-tsan: TESTFLAGS='--gtest_filter=-*ExtremelyLarge*' +test-pzstd-tsan: clean googletest tsan check + +.PHONY: test-pzstd-asan +test-pzstd-asan: LDFLAGS="-fuse-ld=gold" +test-pzstd-asan: TESTFLAGS='--gtest_filter=-*ExtremelyLarge*' +test-pzstd-asan: clean asan check + .PHONY: check check: $(TESTPROG) ./utils/test/BufferTest$(EXT) $(TESTFLAGS) diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 588111913..a7b4fddc8 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -399,7 +399,7 @@ int main(int argCount, const char* argv[]) while (argument[0]!=0) { if (lastCommand) { DISPLAY("error : command must be followed by argument \n"); - return 1; + CLEAN_RETURN(1); } #ifndef ZSTD_NOCOMPRESS /* compression Level */ @@ -555,7 +555,7 @@ int main(int argCount, const char* argv[]) filenameTable[filenameIdx++] = argument; } - if (lastCommand) { DISPLAY("error : command must be followed by argument \n"); return 1; } /* forgotten argument */ + if (lastCommand) { DISPLAY("error : command must be followed by argument \n"); CLEAN_RETURN(1); } /* forgotten argument */ /* Welcome message (if verbose) */ DISPLAYLEVEL(3, WELCOME_MESSAGE); diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 9a9fed98d..c22a284c7 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -496,6 +496,8 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo /* Bug will cause checksum to fail */ if (ZSTD_isError(r)) goto _output_error; } + + ZSTD_freeDStream(zds); } DISPLAYLEVEL(3, "OK \n"); From d8114e5802edfb236758d7a4d7ce24795e74afa2 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 18:59:56 +0100 Subject: [PATCH 118/223] zstd_compress.c: fix memory leaks --- contrib/pzstd/Makefile | 10 +++++----- lib/compress/zstd_compress.c | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile index 10a133dd7..21ef935c6 100644 --- a/contrib/pzstd/Makefile +++ b/contrib/pzstd/Makefile @@ -86,20 +86,20 @@ endif default: all .PHONY: test-pzstd -test-pzstd: TESTFLAGS='--gtest_filter=-*ExtremelyLarge*' +test-pzstd: TESTFLAGS=--gtest_filter=-*ExtremelyLarge* test-pzstd: clean googletest pzstd tests check .PHONY: test-pzstd32 test-pzstd32: clean googletest32 all32 check .PHONY: test-pzstd-tsan -test-pzstd-tsan: LDFLAGS="-fuse-ld=gold" -test-pzstd-tsan: TESTFLAGS='--gtest_filter=-*ExtremelyLarge*' +test-pzstd-tsan: LDFLAGS=-fuse-ld=gold +test-pzstd-tsan: TESTFLAGS=--gtest_filter=-*ExtremelyLarge* test-pzstd-tsan: clean googletest tsan check .PHONY: test-pzstd-asan -test-pzstd-asan: LDFLAGS="-fuse-ld=gold" -test-pzstd-asan: TESTFLAGS='--gtest_filter=-*ExtremelyLarge*' +test-pzstd-asan: LDFLAGS=-fuse-ld=gold +test-pzstd-asan: TESTFLAGS=--gtest_filter=-*ExtremelyLarge* test-pzstd-asan: clean asan check .PHONY: check diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 924189b0c..0e0f9d373 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -2786,7 +2786,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, u if (!cdict || !cctx) { ZSTD_free(cdict, customMem); - ZSTD_free(cctx, customMem); + ZSTD_freeCCtx(cctx); return NULL; } @@ -2804,8 +2804,8 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, u { size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0); if (ZSTD_isError(errorCode)) { ZSTD_free(cdict->dictBuffer, customMem); - ZSTD_free(cctx, customMem); ZSTD_free(cdict, customMem); + ZSTD_freeCCtx(cctx); return NULL; } } From 3bee41a70eaf343fbcae3637b3f6edbe52f35ed8 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Tue, 21 Feb 2017 10:20:36 -0800 Subject: [PATCH 119/223] Add default distributions and fix typos --- doc/zstd_compression_format.md | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/doc/zstd_compression_format.md b/doc/zstd_compression_format.md index f08dc9537..d4b46548a 100644 --- a/doc/zstd_compression_format.md +++ b/doc/zstd_compression_format.md @@ -776,13 +776,44 @@ For details on how to convert this distribution into a decoding table, see the [ [FSE section]: #from-normalized-distribution-to-decoding-tables +##### Literals Length +The decoding table uses an accuracy log of 6 bits (64 states). +``` +short literalsLength_defaultDistribution[36] = + { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, + -1,-1,-1,-1 }; +``` + +##### Match Length +The decoding table uses an accuracy log of 6 bits (64 states). +``` +short matchLengths_defaultDistribution[53] = + { 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1, + -1,-1,-1,-1,-1 }; +``` + +##### Offset Codes +The decoding table uses an accuracy log of 5 bits (32 states), +and supports a maximum `N` value of 28, allowing offset values up to 536,870,908 . + +If any sequence in the compressed block requires a larger offset than this, +it's not possible to use the default distribution to represent it. +``` +short offsetCodes_defaultDistribution[29] = + { 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 }; +``` + Sequence Execution ------------------ Once literals and sequences have been decoded, they are combined to produce the decoded content of a block. Each sequence consists of a tuple of (`literals_length`, `offset_value`, `match_length`), -decoded as described in the [Sequences Section)[#sequences-section]. +decoded as described in the [Sequences Section](#sequences-section). To execute a sequence, first copy `literals_length` bytes from the literals section to the output. @@ -1266,7 +1297,6 @@ __`Entropy_Tables`__ : following the same format as the tables in compressed blo FSE table for match lengths, and FSE table for literals lengths. These tables populate the Repeat Stats literals mode and Repeat distribution mode for sequence decoding. - It's finally followed by 3 offset values, populating recent offsets (instead of using `{1,4,8}`), stored in order, 4-bytes little-endian each, for a total of 12 bytes. Each recent offset must have a value < dictionary size. From 346ce32adeb57c468d40f7c4e8ed75c0c84a4f4e Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 20:10:21 +0100 Subject: [PATCH 120/223] legacy.c: fix memory leaks --- contrib/pzstd/Makefile | 2 +- tests/legacy.c | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/contrib/pzstd/Makefile b/contrib/pzstd/Makefile index 21ef935c6..cec6959e6 100644 --- a/contrib/pzstd/Makefile +++ b/contrib/pzstd/Makefile @@ -134,7 +134,7 @@ debug: pzstd$(EXT) tests roundtrip .PHONY: tsan tsan: PZSTD_CCXXFLAGS += -fsanitize=thread -fPIC -tsan: PZSTD_LDFLAGS += -fsanitize=thread -pie +tsan: PZSTD_LDFLAGS += -fsanitize=thread tsan: debug .PHONY: asan diff --git a/tests/legacy.c b/tests/legacy.c index 5d93c68fa..e84e31273 100644 --- a/tests/legacy.c +++ b/tests/legacy.c @@ -65,6 +65,7 @@ int testSimpleAPI(void) return 1; } + free(output); DISPLAY("Simple API OK\n"); return 0; } @@ -118,6 +119,8 @@ int testStreamingAPI(void) } } + free(outBuff); + ZSTD_freeDStream(stream); DISPLAY("Streaming API OK\n"); return 0; } From 97cfec5e12b64bb0494f658143080b1735b8df0e Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 20:44:35 +0100 Subject: [PATCH 121/223] travis.yml: reduce number of jobs --- .travis.yml | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 38ed23431..1020df8e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,38 +7,34 @@ matrix: os: osx - # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes) - - env: Ubu=12.04cont Cmd="make uasan" - os: linux - sudo: false - - - env: Ubu=14.04 Cmd='make test CC=clang-4.0 MOREFLAGS="-g -fsanitize=address -fsanitize=undefined"' + # Ubuntu 14.04 LTS Server Edition 64 bit + - env: Ubu=14.04 Cmd='make test CC=gcc-6 MOREFLAGS="-g -fsanitize=address -fsanitize=undefined"' os: linux dist: trusty sudo: required addons: apt: sources: - - llvm-toolchain-trusty-4.0 + - ubuntu-toolchain-r-test packages: - - clang-4.0 + - gcc-6 + - gcc-6-multilib - - env: Ubu=14.04 Cmd='make -C tests test32 CC=clang-4.0 MOREFLAGS="-g -fsanitize=address"' + - env: Ubu=14.04 Cmd='make -C tests test32 CC=gcc-6 MOREFLAGS="-g -fsanitize=address"' os: linux dist: trusty sudo: required addons: apt: sources: - - llvm-toolchain-trusty-4.0 + - ubuntu-toolchain-r-test packages: - libc6-dev-i386 - gcc-multilib - - clang-4.0 + - gcc-6 + - gcc-6-multilib - - # Ubuntu 14.04 LTS Server Edition 64 bit - - env: Ubu=14.04 Cmd='cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && test-pzstd-asan' + - env: Ubu=14.04 Cmd='cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' os: linux dist: trusty sudo: required From 4d7a24328b3312c4531b68cb75d3d7e8e411cb4c Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 21:12:09 +0100 Subject: [PATCH 122/223] travis.yml: added LDFLAGS=-fuse-ld=gold --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1020df8e8..6d5f22bb1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ matrix: # Ubuntu 14.04 LTS Server Edition 64 bit - - env: Ubu=14.04 Cmd='make test CC=gcc-6 MOREFLAGS="-g -fsanitize=address -fsanitize=undefined"' + - env: Ubu=14.04 Cmd='make test CC=gcc-6 MOREFLAGS="-g -fsanitize=address -fsanitize=undefined" LDFLAGS=-fuse-ld=gold' os: linux dist: trusty sudo: required @@ -18,9 +18,8 @@ matrix: - ubuntu-toolchain-r-test packages: - gcc-6 - - gcc-6-multilib - - env: Ubu=14.04 Cmd='make -C tests test32 CC=gcc-6 MOREFLAGS="-g -fsanitize=address"' + - env: Ubu=14.04 Cmd='make -C tests test32 CC=gcc-6 MOREFLAGS="-g -fsanitize=address" LDFLAGS=-fuse-ld=gold' os: linux dist: trusty sudo: required From 7704c3ca1ae07b76c4593418d22ebf89b64b9710 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 21:48:14 +0100 Subject: [PATCH 123/223] travis.yml: use CFLAGS=-Og with -fsanitize --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6d5f22bb1..cf3bbb5bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ matrix: # Ubuntu 14.04 LTS Server Edition 64 bit - - env: Ubu=14.04 Cmd='make test CC=gcc-6 MOREFLAGS="-g -fsanitize=address -fsanitize=undefined" LDFLAGS=-fuse-ld=gold' + - env: Ubu=14.04 Cmd='LDFLAGS=-fuse-ld=gold CFLAGS=-Og make test CC=gcc-6 MOREFLAGS="-fsanitize=address -fsanitize=undefined"' os: linux dist: trusty sudo: required @@ -19,7 +19,7 @@ matrix: packages: - gcc-6 - - env: Ubu=14.04 Cmd='make -C tests test32 CC=gcc-6 MOREFLAGS="-g -fsanitize=address" LDFLAGS=-fuse-ld=gold' + - env: Ubu=14.04 Cmd='LDFLAGS=-fuse-ld=gold CFLAGS=-Og make -C tests test32 CC=gcc-6 MOREFLAGS="-fsanitize=address -fsanitize=undefined"' os: linux dist: trusty sudo: required From 8a51c692184e524d1f3bb167750c0e378f7c8f35 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 22:48:04 +0100 Subject: [PATCH 124/223] travis.yml: added uasan-test and uasan-test32 --- .travis.yml | 46 ++++++++++++++-------------------------------- Makefile | 3 +++ 2 files changed, 17 insertions(+), 32 deletions(-) diff --git a/.travis.yml b/.travis.yml index cf3bbb5bf..dba15d7ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,33 +8,7 @@ matrix: # Ubuntu 14.04 LTS Server Edition 64 bit - - env: Ubu=14.04 Cmd='LDFLAGS=-fuse-ld=gold CFLAGS=-Og make test CC=gcc-6 MOREFLAGS="-fsanitize=address -fsanitize=undefined"' - os: linux - dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - gcc-6 - - - env: Ubu=14.04 Cmd='LDFLAGS=-fuse-ld=gold CFLAGS=-Og make -C tests test32 CC=gcc-6 MOREFLAGS="-fsanitize=address -fsanitize=undefined"' - os: linux - dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - libc6-dev-i386 - - gcc-multilib - - gcc-6 - - gcc-6-multilib - - - env: Ubu=14.04 Cmd='cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' - os: linux + - env: Ubu=14.04 Cmd='make uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' dist: trusty sudo: required install: @@ -47,9 +21,19 @@ matrix: - libc6-dev-i386 - g++-multilib - gcc-6 - - gcc-6-multilib - g++-6 - - g++-6-multilib + + - env: Ubu=14.04 Cmd='CC=gcc-6 make uasan-test32 && make clean zlibwrapper && make -C tests clean test-zstd-nolegacy versionsTest' + dist: trusty + sudo: required + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - libc6-dev-i386 + - gcc-multilib + - gcc-6 - env: Ubu=14.04 Cmd="make armtest && make clean && make aarch64test" dist: trusty @@ -76,7 +60,6 @@ matrix: # other feature branches => short tests - env: Ubu=14.04 Cmd='make lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest' - os: linux dist: trusty sudo: required addons: @@ -84,8 +67,7 @@ matrix: packages: - valgrind - - env: Ubu=14.04 Cmd="make zlibwrapper && make clean && make -C tests test-zstd-nolegacy && make clean && make -C tests test32 versionsTest" - os: linux + - env: Ubu=14.04 Cmd="make -C tests test32" dist: trusty sudo: required addons: diff --git a/Makefile b/Makefile index d86db7cb3..128c72bb0 100644 --- a/Makefile +++ b/Makefile @@ -143,6 +143,9 @@ asan32: clean uasan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined" +uasan-%: clean + LDFLAGS=-fuse-ld=gold CFLAGS="-Og -fsanitize=address -fsanitize=undefined" $(MAKE) -C $(TESTDIR) $* + endif From f58ac79f513cda1acd468de1853dde6ecfe793aa Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Tue, 21 Feb 2017 23:40:21 +0100 Subject: [PATCH 125/223] fix uasan-test32 --- .travis.yml | 7 ++++--- Makefile | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index dba15d7ea..8688035e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ matrix: # Ubuntu 14.04 LTS Server Edition 64 bit - - env: Ubu=14.04 Cmd='make uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' + - env: Ubu=14.04 Cmd='LDFLAGS=-fuse-ld=gold make uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' dist: trusty sudo: required install: @@ -22,6 +22,7 @@ matrix: - g++-multilib - gcc-6 - g++-6 + - g++-6-multilib - env: Ubu=14.04 Cmd='CC=gcc-6 make uasan-test32 && make clean zlibwrapper && make -C tests clean test-zstd-nolegacy versionsTest' dist: trusty @@ -83,11 +84,11 @@ script: # cron & master => long tests, as this is the final step towards a Release # dev => normal tests - # other feature branches => short tests (number > 10) + # other feature branches => short tests (number > 5) - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "asan" ]; then FUZZERTEST=-T10mn sh -c "$Cmd" || travis_terminate 1; else - if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 10 ] || [ "$TRAVIS_BRANCH" = "dev" ]; then + if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 5 ] || [ "$TRAVIS_BRANCH" = "dev" ]; then sh -c "$Cmd" || travis_terminate 1; fi fi diff --git a/Makefile b/Makefile index 128c72bb0..ff624e907 100644 --- a/Makefile +++ b/Makefile @@ -144,7 +144,7 @@ uasan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined" uasan-%: clean - LDFLAGS=-fuse-ld=gold CFLAGS="-Og -fsanitize=address -fsanitize=undefined" $(MAKE) -C $(TESTDIR) $* + CFLAGS="-Og -fsanitize=address -fsanitize=undefined" $(MAKE) -C $(TESTDIR) $* endif From 971c1613189f8305c558287656f88d559430a2f8 Mon Sep 17 00:00:00 2001 From: Soojin Nam Date: Wed, 22 Feb 2017 16:04:48 +0900 Subject: [PATCH 126/223] test for fail to decompress --- examples/dictionary_decompression.c | 6 +++++- examples/simple_decompression.c | 19 ++++++++++++------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/examples/dictionary_decompression.c b/examples/dictionary_decompression.c index deaf3888e..ef739c189 100644 --- a/examples/dictionary_decompression.c +++ b/examples/dictionary_decompression.c @@ -78,10 +78,14 @@ static void decompress(const char* fname, const ZSTD_DDict* ddict) size_t cSize; void* const cBuff = loadFile_orDie(fname, &cSize); unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize); - if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) { + if (rSize==ZSTD_CONTENTSIZE_ERROR) { + fprintf(stderr, "%s : it was not compressed by zstd.\n", fname); + exit(5); + } else if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) { fprintf(stderr, "%s : original size unknown \n", fname); exit(6); } + void* const rBuff = malloc_orDie((size_t)rSize); ZSTD_DCtx* const dctx = ZSTD_createDCtx(); diff --git a/examples/simple_decompression.c b/examples/simple_decompression.c index e23f14887..fa4e3e680 100644 --- a/examples/simple_decompression.c +++ b/examples/simple_decompression.c @@ -20,7 +20,7 @@ static off_t fsize_orDie(const char *filename) struct stat st; if (stat(filename, &st) == 0) return st.st_size; /* error */ - printf("stat: %s : %s \n", filename, strerror(errno)); + fprintf(stderr, "stat: %s : %s \n", filename, strerror(errno)); exit(1); } @@ -29,7 +29,7 @@ static FILE* fopen_orDie(const char *filename, const char *instruction) FILE* const inFile = fopen(filename, instruction); if (inFile) return inFile; /* error */ - printf("fopen: %s : %s \n", filename, strerror(errno)); + fprintf(stderr, "fopen: %s : %s \n", filename, strerror(errno)); exit(2); } @@ -38,7 +38,7 @@ static void* malloc_orDie(size_t size) void* const buff = malloc(size); if (buff) return buff; /* error */ - printf("malloc: %s \n", strerror(errno)); + fprintf(stderr, "malloc: %s \n", strerror(errno)); exit(3); } @@ -49,7 +49,7 @@ static void* loadFile_orDie(const char* fileName, size_t* size) void* const buffer = malloc_orDie(buffSize); size_t const readSize = fread(buffer, 1, buffSize, inFile); if (readSize != (size_t)buffSize) { - printf("fread: %s : %s \n", fileName, strerror(errno)); + fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno)); exit(4); } fclose(inFile); /* can't fail (read only) */ @@ -63,16 +63,21 @@ static void decompress(const char* fname) size_t cSize; void* const cBuff = loadFile_orDie(fname, &cSize); unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize); - if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) { - printf("%s : original size unknown. Use streaming decompression instead. \n", fname); + if (rSize==ZSTD_CONTENTSIZE_ERROR) { + fprintf(stderr, "%s : it was not compressed by zstd.\n", fname); exit(5); + } else if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) { + fprintf(stderr, + "%s : original size unknown. Use streaming decompression instead.\n", fname); + exit(6); } + void* const rBuff = malloc_orDie((size_t)rSize); size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize); if (dSize != rSize) { - printf("error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize)); + fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize)); exit(7); } From 5dd18b314b482020be0014f6a0257d20a183b630 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 22 Feb 2017 08:15:17 +0100 Subject: [PATCH 127/223] travis.yml: reduce number of jobs to 7 --- .travis.yml | 3 ++- Makefile | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8688035e2..41d90f380 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ matrix: # Ubuntu 14.04 LTS Server Edition 64 bit - - env: Ubu=14.04 Cmd='LDFLAGS=-fuse-ld=gold make uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' + - env: Ubu=14.04 Cmd='make uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' dist: trusty sudo: required install: @@ -35,6 +35,7 @@ matrix: - libc6-dev-i386 - gcc-multilib - gcc-6 + - gcc-6-multilib - env: Ubu=14.04 Cmd="make armtest && make clean && make aarch64test" dist: trusty diff --git a/Makefile b/Makefile index ff624e907..128c72bb0 100644 --- a/Makefile +++ b/Makefile @@ -144,7 +144,7 @@ uasan: clean $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined" uasan-%: clean - CFLAGS="-Og -fsanitize=address -fsanitize=undefined" $(MAKE) -C $(TESTDIR) $* + LDFLAGS=-fuse-ld=gold CFLAGS="-Og -fsanitize=address -fsanitize=undefined" $(MAKE) -C $(TESTDIR) $* endif From 21911ad6cbc76a4aeab23fa0ecc64fec6c4da2c9 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 22 Feb 2017 08:54:56 +0100 Subject: [PATCH 128/223] move Ubuntu packages install to Makefile --- .travis.yml | 64 ++++++----------------------------------------------- Makefile | 25 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 57 deletions(-) diff --git a/.travis.yml b/.travis.yml index 41d90f380..6ea9d31dc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,77 +6,27 @@ matrix: - env: Ubu=OS_X_Mavericks Cmd="make gnu90test && make clean && make test && make clean && make travis-install" os: osx - # Ubuntu 14.04 LTS Server Edition 64 bit - - env: Ubu=14.04 Cmd='make uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' + - env: Ubu=14.04 Cmd='make gpp6install uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' dist: trusty - sudo: required install: - export CXX="g++-6" CC="gcc-6" - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - libc6-dev-i386 - - g++-multilib - - gcc-6 - - g++-6 - - g++-6-multilib - - env: Ubu=14.04 Cmd='CC=gcc-6 make uasan-test32 && make clean zlibwrapper && make -C tests clean test-zstd-nolegacy versionsTest' + - env: Ubu=14.04 Cmd='CC=gcc-6 make gcc6install uasan-test32 && make clean zlibwrapper && make -C tests clean test-zstd-nolegacy versionsTest' dist: trusty - sudo: required - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - libc6-dev-i386 - - gcc-multilib - - gcc-6 - - gcc-6-multilib - - env: Ubu=14.04 Cmd="make armtest && make clean && make aarch64test" + - env: Ubu=14.04 Cmd="make arminstall armtest && make clean && make aarch64test" dist: trusty - sudo: required - addons: - apt: - packages: - - qemu-system-arm - - qemu-user-static - - gcc-arm-linux-gnueabi - - libc6-dev-armel-cross - - gcc-aarch64-linux-gnu - - libc6-dev-arm64-cross - - env: Ubu=14.04 Cmd='make ppctest && make clean && make ppc64test' + - env: Ubu=14.04 Cmd='make ppcinstall ppctest && make clean && make ppc64test' dist: trusty - sudo: required - addons: - apt: - packages: - - qemu-system-ppc - - qemu-user-static - - gcc-powerpc-linux-gnu # other feature branches => short tests - - env: Ubu=14.04 Cmd='make lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest' + - env: Ubu=14.04 Cmd='make valgrindinstall lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest' dist: trusty - sudo: required - addons: - apt: - packages: - - valgrind - - env: Ubu=14.04 Cmd="make -C tests test32" + - env: Ubu=14.04 Cmd="make libc6install && make -C tests test32" dist: trusty - sudo: required - addons: - apt: - packages: - - libc6-dev-i386 - - gcc-multilib @@ -87,7 +37,7 @@ script: # dev => normal tests # other feature branches => short tests (number > 5) - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "asan" ]; then - FUZZERTEST=-T10mn sh -c "$Cmd" || travis_terminate 1; + FUZZERTEST=-T5mn sh -c "$Cmd" || travis_terminate 1; else if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 5 ] || [ "$TRAVIS_BRANCH" = "dev" ]; then sh -c "$Cmd" || travis_terminate 1; diff --git a/Makefile b/Makefile index 128c72bb0..709d2f0ea 100644 --- a/Makefile +++ b/Makefile @@ -146,6 +146,31 @@ uasan: clean uasan-%: clean LDFLAGS=-fuse-ld=gold CFLAGS="-Og -fsanitize=address -fsanitize=undefined" $(MAKE) -C $(TESTDIR) $* +apt-install: + sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES) + +apt-add-repo: + sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + sudo apt-get update -y -qq + +ppcinstall: + APT_PACKAGES="qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu" $(MAKE) apt-install + +arminstall: + APT_PACKAGES="qemu-system-arm qemu-user-static gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross" $(MAKE) apt-install + +valgrindinstall: + APT_PACKAGES="valgrind" $(MAKE) apt-install + +libc6install: + APT_PACKAGES="libc6-dev-i386 gcc-multilib" $(MAKE) apt-install + +gcc6install: apt-add-repo + APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-6 gcc-6-multilib" $(MAKE) apt-install + +gpp6install: apt-add-repo + APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install + endif From 2e8ae51f8cf32ce22ec3caebd6c153c6248dfab9 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 22 Feb 2017 09:21:04 +0100 Subject: [PATCH 129/223] travis.yml: set "dist: trusty" as default --- .travis.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6ea9d31dc..e6b011003 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,6 @@ language: c +sudo: required +dist: trusty matrix: fast_finish: true include: @@ -8,27 +10,15 @@ matrix: # Ubuntu 14.04 LTS Server Edition 64 bit - env: Ubu=14.04 Cmd='make gpp6install uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan' - dist: trusty install: - export CXX="g++-6" CC="gcc-6" - - env: Ubu=14.04 Cmd='CC=gcc-6 make gcc6install uasan-test32 && make clean zlibwrapper && make -C tests clean test-zstd-nolegacy versionsTest' - dist: trusty - - env: Ubu=14.04 Cmd="make arminstall armtest && make clean && make aarch64test" - dist: trusty - - env: Ubu=14.04 Cmd='make ppcinstall ppctest && make clean && make ppc64test' - dist: trusty # other feature branches => short tests - env: Ubu=14.04 Cmd='make valgrindinstall lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest' - dist: trusty - - env: Ubu=14.04 Cmd="make libc6install && make -C tests test32" - dist: trusty - - script: - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') From 3d836bfd18e67f406024e874e97aedddb3ea0355 Mon Sep 17 00:00:00 2001 From: Przemyslaw Skibinski Date: Wed, 22 Feb 2017 09:36:42 +0100 Subject: [PATCH 130/223] travis.yml: fix versionsTest target --- .travis.yml | 7 +++---- tests/Makefile | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index e6b011003..958633de7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,12 +22,11 @@ matrix: script: - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:') - # cron & master => long tests, as this is the final step towards a Release - # dev => normal tests + # dev && pull requests => normal tests # other feature branches => short tests (number > 5) - - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "asan" ]; then - FUZZERTEST=-T5mn sh -c "$Cmd" || travis_terminate 1; + - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "master" ]; then + FUZZERTEST=-T7mn sh -c "$Cmd" || travis_terminate 1; else if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 5 ] || [ "$TRAVIS_BRANCH" = "dev" ]; then sh -c "$Cmd" || travis_terminate 1; diff --git a/tests/Makefile b/tests/Makefile index c5b8bdfa7..937ec96d3 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -170,7 +170,7 @@ namespaceTest: if $(CC) namespaceTest.c ../lib/common/xxhash.c -o $@ ; then echo compilation should fail; exit 1 ; fi $(RM) $@ -versionsTest: +versionsTest: clean $(PYTHON) test-zstd-versions.py clean: From 508404514cbde1a3bbdfc50cd5c23fefcdac0dff Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 22 Feb 2017 00:57:50 -0800 Subject: [PATCH 131/223] added `manual` target to contrib/gen_html/Makefile --- contrib/gen_html/Makefile | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/contrib/gen_html/Makefile b/contrib/gen_html/Makefile index c68e560a1..ea68b11fc 100644 --- a/contrib/gen_html/Makefile +++ b/contrib/gen_html/Makefile @@ -7,12 +7,18 @@ # of patent rights can be found in the PATENTS file in the same directory. # ########################################################################## - CFLAGS ?= -O3 CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wno-comment CFLAGS += $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) +FLAGS = $(CPPFLAGS) $(CFLAGS) $(CXXFLAGS) $(LDFLAGS) +ZSTDAPI = ../../lib/zstd.h +ZSTDMANUAL = ../../doc/zstd_manual.html +LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(ZSTDAPI)` +LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(ZSTDAPI)` +LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(ZSTDAPI)` +LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) +LIBVER := $(shell echo $(LIBVER_SCRIPT)) # Define *.exe as extension for Windows systems @@ -23,14 +29,23 @@ EXT = endif -.PHONY: default gen_html - +.PHONY: default default: gen_html +.PHONY: all +all: manual + gen_html: gen_html.cpp - $(CXX) $(FLAGS) $^ -o $@$(EXT) + $(CXX) $(FLAGS) $^ -o $@$(EXT) +$(ZSTDMANUAL): gen_html $(ZSTDAPI) + echo "Update zstd manual in /doc" + ./gen_html $(LIBVER) $(ZSTDAPI) $(ZSTDMANUAL) +.PHONY: manual +manual: gen_html $(ZSTDMANUAL) + +.PHONY: clean clean: @$(RM) gen_html$(EXT) @echo Cleaning completed From 7757577341bf9bbefeb3686a9127c32bb1fc3d3c Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 22 Feb 2017 01:10:43 -0800 Subject: [PATCH 132/223] added `manual` target in root Makefile `manual` target is added to `all` target --- Makefile | 11 ++- doc/zstd_manual.html | 192 +++++++++++++++++++++++++++++++------------ 2 files changed, 148 insertions(+), 55 deletions(-) diff --git a/Makefile b/Makefile index d86db7cb3..ffa7a62a2 100644 --- a/Makefile +++ b/Makefile @@ -26,8 +26,7 @@ endif default: lib zstd-release .PHONY: all -all: allmost - CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ $@ +all: | allmost examples manual .PHONY: allmost allmost: @@ -68,6 +67,14 @@ zlibwrapper: test: $(MAKE) -C $(TESTDIR) $@ +.PHONY: examples +examples: + CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ all + +.PHONY: manual +manual: + $(MAKE) -C contrib/gen_html $@ + .PHONY: clean clean: @$(MAKE) -C $(ZSTDDIR) $@ > $(VOID) diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html index 1badcbd79..23224d77a 100644 --- a/doc/zstd_manual.html +++ b/doc/zstd_manual.html @@ -1,10 +1,10 @@ -zstd 1.1.2 Manual +zstd 1.1.4 Manual -

zstd 1.1.2 Manual

+

zstd 1.1.4 Manual

Contents

    @@ -19,13 +19,15 @@
  1. Streaming decompression - HowTo
  2. START OF ADVANCED AND EXPERIMENTAL FUNCTIONS
  3. Advanced types
    4. -
  4. Advanced compression functions
    5. -
  5. Advanced decompression functions
    6. -
  6. Advanced streaming functions
    7. -
  7. Buffer-less and synchronous inner streaming functions
    8. -
  8. Buffer-less streaming compression (synchronous mode)
    9. -
  9. Buffer-less streaming decompression (synchronous mode)
    10. -
  10. Block functions
    11. +
  11. Compressed size functions
    12. +
  12. Decompressed size functions
    13. +
  13. Advanced compression functions
    14. +
  14. Advanced decompression functions
    15. +
  15. Advanced streaming functions
    16. +
  16. Buffer-less and synchronous inner streaming functions
    17. +
  17. Buffer-less streaming compression (synchronous mode)
    18. +
  18. Buffer-less streaming decompression (synchronous mode)
    19. +
  19. Block functions

Introduction

@@ -63,7 +65,7 @@
 
 
size_t ZSTD_decompress( void* dst, size_t dstCapacity,
                               const void* src, size_t compressedSize);
-
    `compressedSize` : must be the _exact_ size of a single compressed frame.
+
    `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
     `dstCapacity` is an upper bound of originalSize.
     If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
     @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
@@ -71,7 +73,16 @@
 


 
 
unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
-
   'src' is the start of a zstd compressed frame.
+
   NOTE: This function is planned to be obsolete, in favour of ZSTD_getFrameContentSize.
+   ZSTD_getFrameContentSize functions the same way, returning the decompressed size of a single
+   frame, but distinguishes empty frames from frames with an unknown size, or errors.
+
+   Additionally, ZSTD_findDecompressedSize can be used instead.  It can handle multiple
+   concatenated frames in one buffer, and so is more general.
+   As a result however, it requires more computation and entire frames to be passed to it,
+   as opposed to ZSTD_getFrameContentSize which requires only a single frame's header.
+
+   'src' is the start of a zstd compressed frame.
    @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
     note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
              When `return==0`, data to decompress could be any size.
@@ -88,21 +99,29 @@
     note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. 
 


 
-
Helper functions
int         ZSTD_maxCLevel(void);               /*!< maximum compression level available */
+
Helper functions
int         ZSTD_maxCLevel(void);               /*!< maximum compression level available */
 size_t      ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case scenario */
 unsigned    ZSTD_isError(size_t code);          /*!< tells if a `size_t` function result is an error code */
 const char* ZSTD_getErrorName(size_t code);     /*!< provides readable string from an error code */
-


+


 
Explicit memory management

 
+
Compression context
   When compressing many times,
+   it is recommended to allocate a context just once, and re-use it for each successive compression operation.
+   This will make workload friendlier for system's memory.
+   Use one context per thread for parallel execution in multi-threaded environments. 
+
typedef struct ZSTD_CCtx_s ZSTD_CCtx;
+ZSTD_CCtx* ZSTD_createCCtx(void);
+size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
+


 
size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
 
    Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). 
 


 
-
Decompression context
typedef struct ZSTD_DCtx_s ZSTD_DCtx;
+
Decompression context
typedef struct ZSTD_DCtx_s ZSTD_DCtx;
 ZSTD_DCtx* ZSTD_createDCtx(void);
 size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
-


+


 
size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 
   Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()). 
 


@@ -131,11 +150,11 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 
 
Fast dictionary API

 
-
ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
+
ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel);
 
   When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
    ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
    ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
-   `dict` can be released after ZSTD_CDict creation. 
+   `dictBuffer` can be released after ZSTD_CDict creation, as its content is copied within CDict 
 


 
 
size_t      ZSTD_freeCDict(ZSTD_CDict* CDict);
@@ -151,9 +170,9 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
    Note that compression level is decided during dictionary creation. 
 


 
-
ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize);
+
ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
 
   Create a digested dictionary, ready to start decompression operation without startup delay.
-   `dict` can be released after creation. 
+   dictBuffer can be released after DDict creation, as its content is copied inside DDict 
 


 
 
size_t      ZSTD_freeDDict(ZSTD_DDict* ddict);
@@ -271,9 +290,9 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 } ZSTD_compressionParameters;
 


 
typedef struct {
-    unsigned contentSizeFlag; /**< 1: content size will be in frame header (if known). */
-    unsigned checksumFlag;    /**< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */
-    unsigned noDictIDFlag;    /**< 1: no dict ID will be saved into frame header (if dictionary compression) */
+    unsigned contentSizeFlag; /**< 1: content size will be in frame header (when known) */
+    unsigned checksumFlag;    /**< 1: generate a 32-bits checksum at end of frame, for error detection */
+    unsigned noDictIDFlag;    /**< 1: no dictID will be saved into frame header (if dictionary compression) */
 } ZSTD_frameParameters;
 


 
typedef struct {
@@ -281,11 +300,56 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
     ZSTD_frameParameters fParams;
 } ZSTD_parameters;
 


-
Custom memory allocation functions
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
+
Custom memory allocation functions
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
 typedef void  (*ZSTD_freeFunction) (void* opaque, void* address);
 typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
-


-
Advanced compression functions

+


+
Compressed size functions

+
+
size_t ZSTD_getFrameCompressedSize(const void* src, size_t srcSize);
+
  `src` should point to the start of a ZSTD encoded frame
+  `srcSize` must be at least as large as the frame
+  @return : the compressed size of the frame pointed to by `src`, suitable to pass to
+      `ZSTD_decompress` or similar, or an error code if given invalid input. 
+


+
+
Decompressed size functions

+
+
unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
+
   `src` should point to the start of a ZSTD encoded frame
+   `srcSize` must be at least as large as the frame header.  A value greater than or equal
+       to `ZSTD_frameHeaderSize_max` is guaranteed to be large enough in all cases.
+   @return : decompressed size of the frame pointed to be `src` if known, otherwise
+             - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
+             - ZSTD_CONTENTSIZE_ERROR if an error occured (e.g. invalid magic number, srcSize too small) 
+


+
+
unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
+
   `src` should point the start of a series of ZSTD encoded and/or skippable frames
+   `srcSize` must be the _exact_ size of this series
+       (i.e. there should be a frame boundary exactly `srcSize` bytes after `src`)
+   @return : the decompressed size of all data in the contained frames, as a 64-bit value _if known_
+             - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
+             - if an error occurred: ZSTD_CONTENTSIZE_ERROR
+
+    note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
+             When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
+             In which case, it's necessary to use streaming mode to decompress data.
+             Optionally, application can still use ZSTD_decompress() while relying on implied limits.
+             (For example, data may be necessarily cut into blocks <= 16 KB).
+    note 2 : decompressed size is always present when compression is done with ZSTD_compress()
+    note 3 : decompressed size can be very large (64-bits value),
+             potentially larger than what local system can handle as a single memory segment.
+             In which case, it's necessary to use streaming mode to decompress data.
+    note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
+             Always ensure result fits within application's authorized limits.
+             Each application can set its own limits.
+    note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
+             read each contained frame header.  This is efficient as most of the data is skipped,
+             however it does mean that all frame data must be present and valid. 
+


+
+
Advanced compression functions

 
 
size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
 
  Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters.
@@ -300,7 +364,22 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 
  Gives the amount of memory used by a given ZSTD_CCtx 
 


 
-
ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
+
typedef enum {
+    ZSTD_p_forceWindow   /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0)*/
+} ZSTD_CCtxParameter;
+


+
size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value);
+
  Set advanced parameters, selected through enum ZSTD_CCtxParameter
+  @result : 0, or an error code (which can be tested with ZSTD_isError()) 
+


+
+
ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
+
  Create a digested dictionary for compression
+  Dictionary content is simply referenced, and therefore stays in dictBuffer.
+  It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict 
+


+
+
ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
                                                   ZSTD_parameters params, ZSTD_customMem customMem);
 
  Create a ZSTD_CDict using external alloc and free, and customized compression parameters 
 


@@ -336,7 +415,7 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 
   Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter 
 


 
-
Advanced decompression functions

+
Advanced decompression functions

 
 
unsigned ZSTD_isFrame(const void* buffer, size_t size);
 
  Tells if the content of `buffer` starts with a valid Frame Identifier.
@@ -357,6 +436,12 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 
  Gives the amount of memory used by a given ZSTD_DCtx 
 


 
+
ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
+
  Create a digested dictionary, ready to start decompression operation without startup delay.
+  Dictionary content is simply referenced, and therefore stays in dictBuffer.
+  It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict 
+


+
 
size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 
  Gives the amount of memory used by a given ZSTD_DDict 
 


@@ -385,33 +470,33 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
   When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. 
 


 
-
Advanced streaming functions

+
Advanced streaming functions

 
-
Advanced Streaming compression functions
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
-size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   /**< pledgedSrcSize must be correct */
-size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
+
Advanced Streaming compression functions
ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
+size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize);   /**< pledgedSrcSize must be correct, a size of 0 means unknown.  for a frame size of 0 use initCStream_advanced */
+size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
 size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
-                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be zero == unknown */
+                                             ZSTD_parameters params, unsigned long long pledgedSrcSize);  /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
 size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict);  /**< note : cdict will just be referenced, and must outlive compression session */
-size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);  /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */
+size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize);  /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before. note: pledgedSrcSize must be correct, a size of 0 means unknown.  for a frame size of 0 use initCStream_advanced */
 size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
-


-
Advanced Streaming decompression functions
typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
+


+
Advanced Streaming decompression functions
typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
 ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
-size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
+size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
 size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
 size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict);  /**< note : ddict will just be referenced, and must outlive decompression session */
 size_t ZSTD_resetDStream(ZSTD_DStream* zds);  /**< re-use decompression parameters from previous init; saves dictionary loading */
 size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
-


-
Buffer-less and synchronous inner streaming functions
+


+
Buffer-less and synchronous inner streaming functions
   This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
   But it's also a complex one, with many restrictions (documented below).
   Prefer using normal streaming API for an easier experience
  
 

 
-
Buffer-less streaming compression (synchronous mode)
+
Buffer-less streaming compression (synchronous mode)
   A ZSTD_CCtx object is required to track streaming operations.
   Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
   ZSTD_CCtx object can be re-used multiple times within successive compression operations.
@@ -434,20 +519,21 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
     In which case, it will "discard" the relevant memory section from its history.
 
   Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
-  It's possible to use a NULL,0 src content, in which case, it will write a final empty block to end the frame,
-  Without last block mark, frames will be considered unfinished (broken) by decoders.
+  It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
+  Without last block mark, frames will be considered unfinished (corrupted) by decoders.
 
-  You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame.
+  `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress some new frame.
 

 
-
Buffer-less streaming compression functions
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
+
Buffer-less streaming compression functions
size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
 size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
-size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
-size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
+size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
+size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**<  note: if pledgedSrcSize can be 0, indicating unknown size.  if it is non-zero, it must be accurate.  for 0 size frames, use compressBegin_advanced */
+size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize can be 0, indicating unknown size.  if it is non-zero, it must be accurate.  for 0 size frames, use compressBegin_advanced */
 size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
-


-
Buffer-less streaming decompression (synchronous mode)
+


+
Buffer-less streaming decompression (synchronous mode)
   A ZSTD_DCtx object is required to track streaming operations.
   Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it.
   A ZSTD_DCtx object can be re-used multiple times.
@@ -490,7 +576,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const vo
   Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType().
   This information is not required to properly decode a frame.
 
-  == Special case : skippable frames ==
+  == Special case : skippable frames 
 
   Skippable frames allow integration of user-defined data into a flow of concatenated frames.
   Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows :
@@ -509,7 +595,7 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const vo
     unsigned checksumFlag;
 } ZSTD_frameParams;
 


-
Buffer-less streaming decompression functions
size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize);   /**< doesn't consume input, see details below */
+
Buffer-less streaming decompression functions
size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize);   /**< doesn't consume input, see details below */
 size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
 size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
 void   ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
@@ -517,8 +603,8 @@ size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
 size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
 ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
-


-
Block functions
+


+
Block functions
     Block functions produce and decode raw zstd blocks, without frame metadata.
     Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
     User will have to take in charge required information to regenerate data, such as compressed and content sizes.
@@ -542,10 +628,10 @@ ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
         Use ZSTD_insertBlock() in such a case.
 

 
-
Raw zstd block functions
size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
+
Raw zstd block functions
size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
 size_t ZSTD_compressBlock  (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
 size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize);  /**< insert block into `dctx` history. Useful for uncompressed blocks */
-


+


 
 

From 337ec875b61f56cc98bf297887da80029741b8d4 Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Wed, 22 Feb 2017 10:31:30 +0100
Subject: [PATCH 133/223] minor tweaks

---
 .travis.yml       | 2 +-
 programs/Makefile | 2 +-
 tests/Makefile    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 958633de7..c1985d785 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,7 +12,7 @@ matrix:
     - env: Ubu=14.04 Cmd='make gpp6install uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan'
       install:
         - export CXX="g++-6" CC="gcc-6"
-    - env: Ubu=14.04 Cmd='CC=gcc-6 make gcc6install uasan-test32 && make clean zlibwrapper && make -C tests clean test-zstd-nolegacy versionsTest'
+    - env: Ubu=14.04 Cmd='CC=gcc-6 make gcc6install uasan-test32 && make clean zlibwrapper && make -C tests clean test-zstd-nolegacy && make -C tests versionsTest'
     - env: Ubu=14.04 Cmd="make arminstall armtest && make clean && make aarch64test"
     - env: Ubu=14.04 Cmd='make ppcinstall ppctest && make clean && make ppc64test'
 
diff --git a/programs/Makefile b/programs/Makefile
index 0a9ab5a79..db718d14c 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -148,7 +148,7 @@ generate_res:
 	windres/generate_res.bat
 
 clean:
-	$(MAKE) -C ../lib clean
+	$(MAKE) -C $(ZSTDDIR) clean
 	@$(RM) $(ZSTDDIR)/decompress/*.o $(ZSTDDIR)/decompress/zstd_decompress.gcda
 	@$(RM) core *.o tmp* result* *.gcda dictionary *.zst \
         zstd$(EXT) zstd32$(EXT) zstd-compress$(EXT) zstd-decompress$(EXT) \
diff --git a/tests/Makefile b/tests/Makefile
index 937ec96d3..17286a022 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -174,7 +174,7 @@ versionsTest: clean
 	$(PYTHON) test-zstd-versions.py
 
 clean:
-	$(MAKE) -C ../lib clean
+	$(MAKE) -C $(ZSTDDIR) clean
 	@$(RM) -fR $(TESTARTEFACT)
 	@$(RM) -f core *.o tmp* result* *.gcda dictionary *.zst \
         $(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \

From d41c048394fb4123d567560013e72424bea5a920 Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Wed, 22 Feb 2017 11:07:28 +0100
Subject: [PATCH 134/223] added arm-ppc-compilation Makefile target

---
 .travis.yml | 2 +-
 Makefile    | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index c1985d785..a4d4a50ab 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,7 +17,7 @@ matrix:
     - env: Ubu=14.04 Cmd='make ppcinstall ppctest && make clean && make ppc64test'
 
     # other feature branches => short tests
-    - env: Ubu=14.04 Cmd='make valgrindinstall lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest'
+    - env: Ubu=14.04 Cmd='make arminstall ppcinstall arm-ppc-compilation && make valgrindinstall lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest'
     - env: Ubu=14.04 Cmd="make libc6install && make -C tests test32"
 
 script:
diff --git a/Makefile b/Makefile
index 709d2f0ea..ed8f16107 100644
--- a/Makefile
+++ b/Makefile
@@ -128,6 +128,12 @@ ppc64test: clean
 	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
 	$(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static"
 
+arm-ppc-compilation:
+	$(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
+	$(MAKE) -C $(PRGDIR) clean zstd CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static"
+	$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static"
+	$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static"
+
 usan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"
 

From bbbd43509950701d076cfdc3f0ef325a80f5d55a Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Wed, 22 Feb 2017 11:21:34 +0100
Subject: [PATCH 135/223] travis.yml: test arm-ppc-compilation target

---
 .travis.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index a4d4a50ab..b20c43329 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,20 +5,20 @@ matrix:
   fast_finish: true
   include:
     # OS X Mavericks
-    - env: Ubu=OS_X_Mavericks Cmd="make gnu90test && make clean && make test && make clean && make travis-install"
+    - env: Cmd="make gnu90test && make clean && make test && make clean && make travis-install"
       os: osx
 
     # Ubuntu 14.04 LTS Server Edition 64 bit
-    - env: Ubu=14.04 Cmd='make gpp6install uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan'
+    - env: Cmd='make gpp6install uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan'
       install:
         - export CXX="g++-6" CC="gcc-6"
-    - env: Ubu=14.04 Cmd='CC=gcc-6 make gcc6install uasan-test32 && make clean zlibwrapper && make -C tests clean test-zstd-nolegacy && make -C tests versionsTest'
-    - env: Ubu=14.04 Cmd="make arminstall armtest && make clean && make aarch64test"
-    - env: Ubu=14.04 Cmd='make ppcinstall ppctest && make clean && make ppc64test'
+    - env: Cmd='CC=gcc-6 make gcc6install uasan-test32 && make clean zlibwrapper && make -C tests clean test-zstd-nolegacy && make -C tests versionsTest'
+    - env: Cmd="make arminstall armtest && make clean && make aarch64test"
+    - env: Cmd='make ppcinstall ppctest && make clean && make ppc64test'
 
     # other feature branches => short tests
-    - env: Ubu=14.04 Cmd='make arminstall ppcinstall arm-ppc-compilation && make valgrindinstall lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest'
-    - env: Ubu=14.04 Cmd="make libc6install && make -C tests test32"
+    - env: Cmd='make valgrindinstall arminstall ppcinstall arm-ppc-compilation && make clean lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest'
+    - env: Cmd="make libc6install && make -C tests test32"
 
 script:
   - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:')

From 88ba64702eaffd88089bdf518c6fea9e7b80be6c Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 22 Feb 2017 10:52:36 -0800
Subject: [PATCH 136/223] fixed c90/gnu90/gnu99 tests

---
 Makefile | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index ffa7a62a2..1524dd42a 100644
--- a/Makefile
+++ b/Makefile
@@ -82,6 +82,7 @@ clean:
 	@$(MAKE) -C $(TESTDIR) $@ > $(VOID)
 	@$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID)
 	@$(MAKE) -C examples/ $@ > $(VOID)
+	@$(MAKE) -C contrib/gen_html $@ > $(VOID)
 	@$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
 	@echo Cleaning completed
 
@@ -170,16 +171,16 @@ cmaketest:
 	cd $(BUILDIR)/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
 
 c90test: clean
-	CFLAGS="-std=c90" $(MAKE) all  # will fail, due to // and long long
+	CFLAGS="-std=c90" $(MAKE) allmost  # will fail, due to missing support for `long long`
 
 gnu90test: clean
-	CFLAGS="-std=gnu90" $(MAKE) all
+	CFLAGS="-std=gnu90" $(MAKE) allmost
 
 c99test: clean
 	CFLAGS="-std=c99" $(MAKE) allmost
 
 gnu99test: clean
-	CFLAGS="-std=gnu99" $(MAKE) all
+	CFLAGS="-std=gnu99" $(MAKE) allmost
 
 c11test: clean
 	CFLAGS="-std=c11" $(MAKE) allmost

From 1f3d54ddb44c2834fbda6ef2b13eb354b7e82db4 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 22 Feb 2017 11:08:00 -0800
Subject: [PATCH 137/223] fixed malloc(0) potential issue

Added test cases to cover #556 patch
---
 examples/Makefile               | 17 ++++++++++++-----
 examples/simple_decompression.c |  2 +-
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/examples/Makefile b/examples/Makefile
index 741022869..b84983f08 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -9,7 +9,7 @@
 
 # This Makefile presumes libzstd is installed, using `sudo make install`
 
-LDFLAGS+= -lzstd
+LDFLAGS += -lzstd
 
 .PHONY: default all clean test
 
@@ -52,16 +52,23 @@ clean:
 test: all
 	cp README.md tmp
 	cp Makefile tmp2
-	@echo starting simple compression
+	@echo -- Simple compression tests
 	./simple_compression tmp
 	./simple_decompression tmp.zst
 	./streaming_decompression tmp.zst > /dev/null
-	@echo starting streaming compression
+	@echo -- Streaming compression tests
 	./streaming_compression tmp
 	./streaming_decompression tmp.zst > /dev/null
-	@echo starting multiple streaming compression
+	@echo -- Edge cases detection
+	! ./streaming_decompression tmp    # invalid input, must fail
+	! ./simple_decompression tmp       # invalid input, must fail
+	! ./simple_decompression tmp.zst   # unknown input size, must fail
+	touch tmpNull                      # create 0-size file
+	./simple_compression tmpNull
+	./simple_decompression tmpNull.zst # 0-size frame : must work
+	@echo -- Multiple streaming tests
 	./multiple_streaming_compression *.c
-	@echo starting dictionary compression
+	@echo -- Dictionary compression tests
 	./dictionary_compression tmp2 tmp README.md
 	./dictionary_decompression tmp2.zst tmp.zst README.md
 	$(RM) tmp* *.zst
diff --git a/examples/simple_decompression.c b/examples/simple_decompression.c
index fa4e3e680..4b7ea59e5 100644
--- a/examples/simple_decompression.c
+++ b/examples/simple_decompression.c
@@ -35,7 +35,7 @@ static FILE* fopen_orDie(const char *filename, const char *instruction)
 
 static void* malloc_orDie(size_t size)
 {
-    void* const buff = malloc(size);
+    void* const buff = malloc(size + !size);   /* avoid allocating size of 0 : may return NULL (implementation dependent) */
     if (buff) return buff;
     /* error */
     fprintf(stderr, "malloc: %s \n", strerror(errno));

From 9050e1925ece4cddd6aa82b1a4526750081c3243 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Wed, 22 Feb 2017 12:12:32 -0800
Subject: [PATCH 138/223] Change name to to findFrameCompressedSize and add
 skippable support

---
 NEWS                             |  2 +-
 lib/decompress/zstd_decompress.c | 15 +++++++++------
 lib/legacy/zstd_legacy.h         | 16 ++++++++--------
 lib/legacy/zstd_v01.c            |  2 +-
 lib/legacy/zstd_v01.h            |  2 +-
 lib/legacy/zstd_v02.c            |  6 +++---
 lib/legacy/zstd_v02.h            |  2 +-
 lib/legacy/zstd_v03.c            |  6 +++---
 lib/legacy/zstd_v03.h            |  2 +-
 lib/legacy/zstd_v04.c            |  6 +++---
 lib/legacy/zstd_v04.h            |  2 +-
 lib/legacy/zstd_v05.c            |  2 +-
 lib/legacy/zstd_v05.h            |  2 +-
 lib/legacy/zstd_v06.c            |  2 +-
 lib/legacy/zstd_v06.h            |  2 +-
 lib/legacy/zstd_v07.c            |  2 +-
 lib/legacy/zstd_v07.h            |  2 +-
 lib/zstd.h                       |  4 ++--
 tests/fuzzer.c                   |  7 +++++++
 tests/symbols.c                  |  1 +
 20 files changed, 48 insertions(+), 37 deletions(-)

diff --git a/NEWS b/NEWS
index be3349754..96ff25fd7 100644
--- a/NEWS
+++ b/NEWS
@@ -2,7 +2,7 @@ v1.1.4
 cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski
 cli : new : advanced benchmark command --priority=rt
 cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77
-API : new : ZSTD_getFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize(), by Sean Purcell
+API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize(), by Sean Purcell
 API : change : ZSTD_compress*() with srcSize==0 create an empty-frame of known size
 build:new : meson build system in contrib/meson, by Dima Krasner
 doc : new : educational decoder, by Sean Purcell
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index eda8b9dd5..362da1a76 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -369,7 +369,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
                 totalDstSize += ret;
             }
             {
-                size_t const frameSrcSize = ZSTD_getFrameCompressedSize(src, srcSize);
+                size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize);
                 if (ZSTD_isError(frameSrcSize)) {
                     return ZSTD_CONTENTSIZE_ERROR;
                 }
@@ -1437,17 +1437,20 @@ size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t len
     return length;
 }
 
-/** ZSTD_getFrameCompressedSize() :
+/** ZSTD_findFrameCompressedSize() :
  *  compatible with legacy mode
  *  `src` must point to the start of a ZSTD or ZSTD legacy frame
  *  `srcSize` must be at least as large as the frame contained
  *  @return : the compressed size of the frame starting at `src` */
-size_t ZSTD_getFrameCompressedSize(const void *src, size_t srcSize)
+size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
 {
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
-    if (ZSTD_isLegacy(src, srcSize)) return ZSTD_getFrameCompressedSizeLegacy(src, srcSize);
+    if (ZSTD_isLegacy(src, srcSize)) return ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
 #endif
-    {
+    if (srcSize >= ZSTD_skippableHeaderSize &&
+            (MEM_readLE32(src) & 0xFFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
+        return ZSTD_skippableHeaderSize + MEM_readLE32((const BYTE*)src + 4);
+    } else {
         const BYTE* ip = (const BYTE*)src;
         const BYTE* const ipstart = ip;
         size_t remainingSize = srcSize;
@@ -1576,7 +1579,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
 
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
         if (ZSTD_isLegacy(src, srcSize)) {
-            size_t const frameSize = ZSTD_getFrameCompressedSizeLegacy(src, srcSize);
+            size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
             size_t decodedSize;
             if (ZSTD_isError(frameSize)) return frameSize;
 
diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h
index b0a7b71d6..707e76f0a 100644
--- a/lib/legacy/zstd_legacy.h
+++ b/lib/legacy/zstd_legacy.h
@@ -123,26 +123,26 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
     }
 }
 
-MEM_STATIC size_t ZSTD_getFrameCompressedSizeLegacy(const void *src,
+MEM_STATIC size_t ZSTD_findFrameCompressedSizeLegacy(const void *src,
                                              size_t compressedSize)
 {
     U32 const version = ZSTD_isLegacy(src, compressedSize);
     switch(version)
     {
         case 1 :
-            return ZSTDv01_getFrameCompressedSize(src, compressedSize);
+            return ZSTDv01_findFrameCompressedSize(src, compressedSize);
         case 2 :
-            return ZSTDv02_getFrameCompressedSize(src, compressedSize);
+            return ZSTDv02_findFrameCompressedSize(src, compressedSize);
         case 3 :
-            return ZSTDv03_getFrameCompressedSize(src, compressedSize);
+            return ZSTDv03_findFrameCompressedSize(src, compressedSize);
         case 4 :
-            return ZSTDv04_getFrameCompressedSize(src, compressedSize);
+            return ZSTDv04_findFrameCompressedSize(src, compressedSize);
         case 5 :
-            return ZSTDv05_getFrameCompressedSize(src, compressedSize);
+            return ZSTDv05_findFrameCompressedSize(src, compressedSize);
         case 6 :
-            return ZSTDv06_getFrameCompressedSize(src, compressedSize);
+            return ZSTDv06_findFrameCompressedSize(src, compressedSize);
         case 7 :
-            return ZSTDv07_getFrameCompressedSize(src, compressedSize);
+            return ZSTDv07_findFrameCompressedSize(src, compressedSize);
         default :
             return ERROR(prefix_unknown);
     }
diff --git a/lib/legacy/zstd_v01.c b/lib/legacy/zstd_v01.c
index a0c78a4b8..bcacb8d5d 100644
--- a/lib/legacy/zstd_v01.c
+++ b/lib/legacy/zstd_v01.c
@@ -1992,7 +1992,7 @@ size_t ZSTDv01_decompress(void* dst, size_t maxDstSize, const void* src, size_t
     return ZSTDv01_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
 }
 
-size_t ZSTDv01_getFrameCompressedSize(const void* src, size_t srcSize)
+size_t ZSTDv01_findFrameCompressedSize(const void* src, size_t srcSize)
 {
     const BYTE* ip = (const BYTE*)src;
     size_t remainingSize = srcSize;
diff --git a/lib/legacy/zstd_v01.h b/lib/legacy/zstd_v01.h
index 21959fcd6..13cb3acfd 100644
--- a/lib/legacy/zstd_v01.h
+++ b/lib/legacy/zstd_v01.h
@@ -40,7 +40,7 @@ ZSTDv01_getFrameSrcSize() : get the source length of a ZSTD frame compliant with
     return : the number of bytes that would be read to decompress this frame
              or an errorCode if it fails (which can be tested using ZSTDv01_isError())
 */
-size_t ZSTDv01_getFrameCompressedSize(const void* src, size_t compressedSize);
+size_t ZSTDv01_findFrameCompressedSize(const void* src, size_t compressedSize);
 
 /**
 ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error
diff --git a/lib/legacy/zstd_v02.c b/lib/legacy/zstd_v02.c
index 6cbf80234..2297b28c8 100644
--- a/lib/legacy/zstd_v02.c
+++ b/lib/legacy/zstd_v02.c
@@ -3378,7 +3378,7 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
     return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
 }
 
-static size_t ZSTD_getFrameCompressedSize(const void *src, size_t srcSize)
+static size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
 {
 
     const BYTE* ip = (const BYTE*)src;
@@ -3524,9 +3524,9 @@ size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize,
 	return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
 }
 
-size_t ZSTDv02_getFrameCompressedSize(const void *src, size_t compressedSize)
+size_t ZSTDv02_findFrameCompressedSize(const void *src, size_t compressedSize)
 {
-    return ZSTD_getFrameCompressedSize(src, compressedSize);
+    return ZSTD_findFrameCompressedSize(src, compressedSize);
 }
 
 ZSTDv02_Dctx* ZSTDv02_createDCtx(void)
diff --git a/lib/legacy/zstd_v02.h b/lib/legacy/zstd_v02.h
index 9542fc0ee..d14f0293c 100644
--- a/lib/legacy/zstd_v02.h
+++ b/lib/legacy/zstd_v02.h
@@ -40,7 +40,7 @@ ZSTDv02_getFrameSrcSize() : get the source length of a ZSTD frame compliant with
     return : the number of bytes that would be read to decompress this frame
              or an errorCode if it fails (which can be tested using ZSTDv02_isError())
 */
-size_t ZSTDv02_getFrameCompressedSize(const void* src, size_t compressedSize);
+size_t ZSTDv02_findFrameCompressedSize(const void* src, size_t compressedSize);
 
 /**
 ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error
diff --git a/lib/legacy/zstd_v03.c b/lib/legacy/zstd_v03.c
index 98b93c49b..ef654931f 100644
--- a/lib/legacy/zstd_v03.c
+++ b/lib/legacy/zstd_v03.c
@@ -3019,7 +3019,7 @@ static size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, siz
     return ZSTD_decompressDCtx(&ctx, dst, maxDstSize, src, srcSize);
 }
 
-static size_t ZSTD_getFrameCompressedSize(const void* src, size_t srcSize)
+static size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
 {
     const BYTE* ip = (const BYTE*)src;
     size_t remainingSize = srcSize;
@@ -3165,9 +3165,9 @@ size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize,
 	return ZSTD_decompress(dst, maxOriginalSize, src, compressedSize);
 }
 
-size_t ZSTDv03_getFrameCompressedSize(const void* src, size_t srcSize)
+size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t srcSize)
 {
-    return ZSTD_getFrameCompressedSize(src, srcSize);
+    return ZSTD_findFrameCompressedSize(src, srcSize);
 }
 
 ZSTDv03_Dctx* ZSTDv03_createDCtx(void)
diff --git a/lib/legacy/zstd_v03.h b/lib/legacy/zstd_v03.h
index 46969410a..07f7597bb 100644
--- a/lib/legacy/zstd_v03.h
+++ b/lib/legacy/zstd_v03.h
@@ -40,7 +40,7 @@ ZSTDv03_getFrameSrcSize() : get the source length of a ZSTD frame compliant with
     return : the number of bytes that would be read to decompress this frame
              or an errorCode if it fails (which can be tested using ZSTDv03_isError())
 */
-size_t ZSTDv03_getFrameCompressedSize(const void* src, size_t compressedSize);
+size_t ZSTDv03_findFrameCompressedSize(const void* src, size_t compressedSize);
 
     /**
 ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error
diff --git a/lib/legacy/zstd_v04.c b/lib/legacy/zstd_v04.c
index 8c929b053..09040e68e 100644
--- a/lib/legacy/zstd_v04.c
+++ b/lib/legacy/zstd_v04.c
@@ -3326,7 +3326,7 @@ static size_t ZSTD_decompress_usingDict(ZSTD_DCtx* ctx,
     return op-ostart;
 }
 
-static size_t ZSTD_getFrameCompressedSize(const void* src, size_t srcSize)
+static size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize)
 {
     const BYTE* ip = (const BYTE*)src;
     size_t remainingSize = srcSize;
@@ -3782,9 +3782,9 @@ size_t ZSTDv04_decompress(void* dst, size_t maxDstSize, const void* src, size_t
 #endif
 }
 
-size_t ZSTDv04_getFrameCompressedSize(const void* src, size_t srcSize)
+size_t ZSTDv04_findFrameCompressedSize(const void* src, size_t srcSize)
 {
-    return ZSTD_getFrameCompressedSize(src, srcSize);
+    return ZSTD_findFrameCompressedSize(src, srcSize);
 }
 
 size_t ZSTDv04_resetDCtx(ZSTDv04_Dctx* dctx) { return ZSTD_resetDCtx(dctx); }
diff --git a/lib/legacy/zstd_v04.h b/lib/legacy/zstd_v04.h
index bcef1fe96..1b5439d39 100644
--- a/lib/legacy/zstd_v04.h
+++ b/lib/legacy/zstd_v04.h
@@ -40,7 +40,7 @@ ZSTDv04_getFrameSrcSize() : get the source length of a ZSTD frame compliant with
     return : the number of bytes that would be read to decompress this frame
              or an errorCode if it fails (which can be tested using ZSTDv04_isError())
 */
-size_t ZSTDv04_getFrameCompressedSize(const void* src, size_t compressedSize);
+size_t ZSTDv04_findFrameCompressedSize(const void* src, size_t compressedSize);
 
 /**
 ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error
diff --git a/lib/legacy/zstd_v05.c b/lib/legacy/zstd_v05.c
index 9689b170c..a6f5f5dbb 100644
--- a/lib/legacy/zstd_v05.c
+++ b/lib/legacy/zstd_v05.c
@@ -3583,7 +3583,7 @@ size_t ZSTDv05_decompress(void* dst, size_t maxDstSize, const void* src, size_t
 #endif
 }
 
-size_t ZSTDv05_getFrameCompressedSize(const void *src, size_t srcSize)
+size_t ZSTDv05_findFrameCompressedSize(const void *src, size_t srcSize)
 {
     const BYTE* ip = (const BYTE*)src;
     size_t remainingSize = srcSize;
diff --git a/lib/legacy/zstd_v05.h b/lib/legacy/zstd_v05.h
index 157dbc57b..8ce662fd9 100644
--- a/lib/legacy/zstd_v05.h
+++ b/lib/legacy/zstd_v05.h
@@ -38,7 +38,7 @@ ZSTDv05_getFrameSrcSize() : get the source length of a ZSTD frame
     return : the number of bytes that would be read to decompress this frame
              or an errorCode if it fails (which can be tested using ZSTDv05_isError())
 */
-size_t ZSTDv05_getFrameCompressedSize(const void* src, size_t compressedSize);
+size_t ZSTDv05_findFrameCompressedSize(const void* src, size_t compressedSize);
 
 /* *************************************
 *  Helper functions
diff --git a/lib/legacy/zstd_v06.c b/lib/legacy/zstd_v06.c
index f586db226..a4258b67a 100644
--- a/lib/legacy/zstd_v06.c
+++ b/lib/legacy/zstd_v06.c
@@ -3729,7 +3729,7 @@ size_t ZSTDv06_decompress(void* dst, size_t dstCapacity, const void* src, size_t
 #endif
 }
 
-size_t ZSTDv06_getFrameCompressedSize(const void* src, size_t srcSize)
+size_t ZSTDv06_findFrameCompressedSize(const void* src, size_t srcSize)
 {
     const BYTE* ip = (const BYTE*)src;
     size_t remainingSize = srcSize;
diff --git a/lib/legacy/zstd_v06.h b/lib/legacy/zstd_v06.h
index ef1feb2f2..10c9c7725 100644
--- a/lib/legacy/zstd_v06.h
+++ b/lib/legacy/zstd_v06.h
@@ -47,7 +47,7 @@ ZSTDv06_getFrameSrcSize() : get the source length of a ZSTD frame
     return : the number of bytes that would be read to decompress this frame
              or an errorCode if it fails (which can be tested using ZSTDv06_isError())
 */
-size_t ZSTDv06_getFrameCompressedSize(const void* src, size_t compressedSize);
+size_t ZSTDv06_findFrameCompressedSize(const void* src, size_t compressedSize);
 
 /* *************************************
 *  Helper functions
diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c
index 07099d5ab..e67916b3c 100644
--- a/lib/legacy/zstd_v07.c
+++ b/lib/legacy/zstd_v07.c
@@ -3968,7 +3968,7 @@ size_t ZSTDv07_decompress(void* dst, size_t dstCapacity, const void* src, size_t
 #endif
 }
 
-size_t ZSTDv07_getFrameCompressedSize(const void* src, size_t srcSize)
+size_t ZSTDv07_findFrameCompressedSize(const void* src, size_t srcSize)
 {
     const BYTE* ip = (const BYTE*)src;
     size_t remainingSize = srcSize;
diff --git a/lib/legacy/zstd_v07.h b/lib/legacy/zstd_v07.h
index a79cbb883..cc95c661b 100644
--- a/lib/legacy/zstd_v07.h
+++ b/lib/legacy/zstd_v07.h
@@ -54,7 +54,7 @@ ZSTDv07_getFrameSrcSize() : get the source length of a ZSTD frame
     return : the number of bytes that would be read to decompress this frame
              or an errorCode if it fails (which can be tested using ZSTDv07_isError())
 */
-size_t ZSTDv07_getFrameCompressedSize(const void* src, size_t compressedSize);
+size_t ZSTDv07_findFrameCompressedSize(const void* src, size_t compressedSize);
 
 /*======  Helper functions  ======*/
 ZSTDLIBv07_API unsigned    ZSTDv07_isError(size_t code);          /*!< tells if a `size_t` function result is an error code */
diff --git a/lib/zstd.h b/lib/zstd.h
index c0a1c7d1c..9820b5379 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -400,12 +400,12 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 *  Compressed size functions
 ***************************************/
 
-/*! ZSTD_getFrameCompressedSize() :
+/*! ZSTD_findFrameCompressedSize() :
  *  `src` should point to the start of a ZSTD encoded frame
  *  `srcSize` must be at least as large as the frame
  *  @return : the compressed size of the frame pointed to by `src`, suitable to pass to
  *      `ZSTD_decompress` or similar, or an error code if given invalid input. */
-ZSTDLIB_API size_t ZSTD_getFrameCompressedSize(const void* src, size_t srcSize);
+ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
 
 /***************************************
 *  Decompressed size functions
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 590bcb393..033998525 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -545,6 +545,13 @@ static int basicUnitTests(U32 seed, double compressibility)
       if (r != _3BYTESTESTLENGTH) goto _output_error; }
     DISPLAYLEVEL(4, "OK \n");
 
+    /* findFrameCompressedSize on skippable frames */
+    DISPLAYLEVEL(4, "test%3i : frame compressed size of skippable frame : ", testNb++);
+    {   const char* frame = "\x50\x2a\x4d\x18\x05\x0\x0\0abcde";
+        size_t const frameSrcSize = 13;
+        if (ZSTD_findFrameCompressedSize(frame, frameSrcSize) != frameSrcSize) goto _output_error; }
+    DISPLAYLEVEL(4, "OK \n");
+
     /* error string tests */
     DISPLAYLEVEL(4, "test%3i : testing ZSTD error code strings : ", testNb++);
     if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error;
diff --git a/tests/symbols.c b/tests/symbols.c
index afdb00064..7dacfc058 100644
--- a/tests/symbols.c
+++ b/tests/symbols.c
@@ -16,6 +16,7 @@ static const void *symbols[] = {
   &ZSTD_decompress,
   &ZSTD_getDecompressedSize,
   &ZSTD_findDecompressedSize,
+  &ZSTD_findFrameCompressedSize,
   &ZSTD_getFrameContentSize,
   &ZSTD_maxCLevel,
   &ZSTD_compressBound,

From 9757cc811bc575e9cbd9d2d469cb054d0a92e2ed Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Wed, 22 Feb 2017 12:27:15 -0800
Subject: [PATCH 139/223] Update comment

---
 lib/decompress/zstd_decompress.c | 2 +-
 lib/zstd.h                       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 362da1a76..e38ef79ba 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1439,7 +1439,7 @@ size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t len
 
 /** ZSTD_findFrameCompressedSize() :
  *  compatible with legacy mode
- *  `src` must point to the start of a ZSTD or ZSTD legacy frame
+ *  `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame
  *  `srcSize` must be at least as large as the frame contained
  *  @return : the compressed size of the frame starting at `src` */
 size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
diff --git a/lib/zstd.h b/lib/zstd.h
index 9820b5379..49050607e 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -401,7 +401,7 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 ***************************************/
 
 /*! ZSTD_findFrameCompressedSize() :
- *  `src` should point to the start of a ZSTD encoded frame
+ *  `src` should point to the start of a ZSTD encoded frame or skippable frame
  *  `srcSize` must be at least as large as the frame
  *  @return : the compressed size of the frame pointed to by `src`, suitable to pass to
  *      `ZSTD_decompress` or similar, or an error code if given invalid input. */

From 64417cd2ff07dcebb0fc55067608c52a4606de1f Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Wed, 22 Feb 2017 13:29:01 -0800
Subject: [PATCH 140/223] Describe ambiguity around skippable frames

---
 lib/zstd.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/zstd.h b/lib/zstd.h
index 49050607e..e597c5db5 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -700,6 +700,9 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
   c) Frame Content - any content (User Data) of length equal to Frame Size
   For skippable frames ZSTD_decompressContinue() always returns 0.
   For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
+    Note : If fparamsPtr->frameContentSize==0, it is ambiguous: the frame might actually be a Zstd encoded frame with no content.
+           For purposes of decompression, it is valid in both cases to skip the frame using
+           ZSTD_findFrameCompressedSize to find its size in bytes.
   It also returns Frame Size as fparamsPtr->frameContentSize.
 */
 

From 83038d236acd5a77bc4655c1e6efc723f716d4a9 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Wed, 22 Feb 2017 13:52:48 -0800
Subject: [PATCH 141/223] Fix bug in FSE distribution normalization

---
 lib/compress/fse_compress.c | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c
index 337b7a6ff..6708fb9d7 100644
--- a/lib/compress/fse_compress.c
+++ b/lib/compress/fse_compress.c
@@ -506,6 +506,7 @@ unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxS
 
 static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue)
 {
+    short const NOT_YET_ASSIGNED = -2;
     U32 s;
     U32 distributed = 0;
     U32 ToDistribute;
@@ -531,7 +532,8 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
             total -= count[s];
             continue;
         }
-        norm[s]=-2;
+
+        norm[s]=NOT_YET_ASSIGNED;
     }
     ToDistribute = (1 << tableLog) - distributed;
 
@@ -539,7 +541,7 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
         /* risk of rounding to zero */
         lowOne = (U32)((total * 3) / (ToDistribute * 2));
         for (s=0; s<=maxSymbolValue; s++) {
-            if ((norm[s] == -2) && (count[s] <= lowOne)) {
+            if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) {
                 norm[s] = 1;
                 distributed++;
                 total -= count[s];
@@ -559,12 +561,19 @@ static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count,
         return 0;
     }
 
+    if (total == 0) {
+        /* all of the symbols were low enough for the lowOne or lowThreshold */
+        for (s=0; ToDistribute > 0; s = (s+1)%(maxSymbolValue+1))
+            if (norm[s] > 0) ToDistribute--, norm[s]++;
+        return 0;
+    }
+
     {   U64 const vStepLog = 62 - tableLog;
         U64 const mid = (1ULL << (vStepLog-1)) - 1;
         U64 const rStep = ((((U64)1<> vStepLog);
                 U32 const sEnd = (U32)(end >> vStepLog);

From f119b6205577ba74793a0d99bfa5915517b41560 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Wed, 22 Feb 2017 15:59:15 -0800
Subject: [PATCH 142/223] Create a tool that generates random, valid, Zstd
 frames for decoder testing

Note: Does not handle dictionaries currently
---
 circle.yml           |    2 +-
 tests/.gitignore     |    1 +
 tests/Makefile       |   12 +-
 tests/decodecorpus.c | 1424 ++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 1436 insertions(+), 3 deletions(-)
 create mode 100644 tests/decodecorpus.c

diff --git a/circle.yml b/circle.yml
index 69c988545..3102633e6 100644
--- a/circle.yml
+++ b/circle.yml
@@ -12,7 +12,7 @@ dependencies:
       if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-invalidDictionaries && make clean; fi
     - |
       if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then g++ -v; make gpptest     && make clean; fi
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy              && make clean; fi
+      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-decodecorpus && make clean; fi
     - |
       if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make gnu90test   && make clean; fi
       if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-symbols             && make clean; fi
diff --git a/tests/.gitignore b/tests/.gitignore
index b7ba51b67..dc468dee4 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -17,6 +17,7 @@ roundTripCrash
 longmatch
 symbols
 legacy
+decodecorpus
 pool
 invalidDictionaries
 
diff --git a/tests/Makefile b/tests/Makefile
index 17286a022..5b0e29c67 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -56,6 +56,7 @@ VOID = /dev/null
 ZSTREAM_TESTTIME = -T2mn
 FUZZERTEST ?= -T5mn
 ZSTDRTTEST = --test-large-data
+DECODECORPUS_TESTTIME = -T30
 
 .PHONY: default all all32 dll clean test test32 test-all namespaceTest versionsTest
 
@@ -154,6 +155,9 @@ legacy : CPPFLAGS+= -I$(ZSTDDIR)/legacy
 legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
+decodecorpus	: $(filter-out $(ZSTDDIR)/compress/zstd_compress.c, $(wildcard $(ZSTD_FILES))) decodecorpus.c
+	$(CC)      $(FLAGS) $^ -o $@$(EXT) -lm
+
 symbols  : symbols.c
 	$(MAKE) -C $(ZSTDDIR) libzstd
 ifneq (,$(filter Windows%,$(OS)))
@@ -184,7 +188,8 @@ clean:
         fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT)\
         zstreamtest$(EXT) zstreamtest32$(EXT) \
         datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \
-        symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) pool$(EXT)
+        symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) pool$(EXT) \
+	decodecorpus$(EXT)
 	@echo Cleaning completed
 
 
@@ -230,7 +235,7 @@ zstd-playTests: datagen
 	file $(ZSTD)
 	ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
 
-test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy
+test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus
 ifeq ($(QEMU_SYS),)
 test: test-pool
 endif
@@ -302,6 +307,9 @@ test-symbols: symbols
 test-legacy: legacy
 	$(QEMU_SYS) ./legacy
 
+test-decodecorpus: decodecorpus
+	$(QEMU_SYS) ./decodecorpus -t $(DECODECORPUS_TESTTIME)
+
 test-pool: pool
 	$(QEMU_SYS) ./pool
 
diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
new file mode 100644
index 000000000..b817d7f0d
--- /dev/null
+++ b/tests/decodecorpus.c
@@ -0,0 +1,1424 @@
+/**
+ * Copyright (c) 2017-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include "zstd.h"
+#include "zstd_internal.h"
+#include "mem.h"
+
+// Direct access to internal compression functions is required
+#include "zstd_compress.c"
+
+#define XXH_STATIC_LINKING_ONLY
+#include "xxhash.h"     /* XXH64 */
+
+#ifndef MIN
+    #define MIN(a, b) ((a) < (b) ? (a) : (b))
+#endif
+
+#ifndef MAX_PATH
+    #ifdef PATH_MAX
+        #define MAX_PATH PATH_MAX
+    #else
+        #define MAX_PATH 256
+    #endif
+#endif
+
+/*-************************************
+*  DISPLAY Macros
+**************************************/
+#define DISPLAY(...)          fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...)  if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+static U32 g_displayLevel = 0;
+
+#define DISPLAYUPDATE(...)                                                     \
+    do {                                                                       \
+        if ((clockSpan(g_displayClock) > g_refreshRate) ||                     \
+            (g_displayLevel >= 4)) {                                           \
+            g_displayClock = clock();                                          \
+            DISPLAY(__VA_ARGS__);                                              \
+            if (g_displayLevel >= 4) fflush(stderr);                           \
+        }                                                                      \
+    } while (0)
+static const clock_t g_refreshRate = CLOCKS_PER_SEC / 6;
+static clock_t g_displayClock = 0;
+
+static clock_t clockSpan(clock_t cStart)
+{
+    return clock() - cStart;   /* works even when overflow; max span ~ 30mn */
+}
+
+#define CHECKERR(code)                                                         \
+    do {                                                                       \
+        if (ZSTD_isError(code)) {                                              \
+            DISPLAY("Error occurred while generating data: %s\n",              \
+                    ZSTD_getErrorName(code));                                  \
+            exit(1);                                                           \
+        }                                                                      \
+    } while (0)
+
+/*-*******************************************************
+*  Random function
+*********************************************************/
+#define CLAMP(x, a, b) ((x) < (a) ? (a) : ((x) > (b) ? (b) : (x)))
+
+static unsigned RAND(unsigned* src)
+{
+#define RAND_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+    static const U32 prime1 = 2654435761U;
+    static const U32 prime2 = 2246822519U;
+    U32 rand32 = *src;
+    rand32 *= prime1;
+    rand32 += prime2;
+    rand32  = RAND_rotl32(rand32, 13);
+    *src = rand32;
+    return RAND_rotl32(rand32, 27);
+#undef RAND_rotl32
+}
+
+#define DISTSIZE (8192)
+
+/* Write `size` bytes into `ptr`, all of which are less than or equal to `maxSymb` */
+static void RAND_bufferMaxSymb(U32* seed, void* ptr, size_t size, int maxSymb)
+{
+    size_t i;
+    BYTE* op = ptr;
+
+    for (i = 0; i < size; i++) {
+        op[i] = RAND(seed) % (maxSymb + 1);
+    }
+}
+
+/* Write `size` random bytes into `ptr` */
+static void RAND_buffer(U32* seed, void* ptr, size_t size)
+{
+    size_t i;
+    BYTE* op = ptr;
+
+    for (i = 0; i + 4 <= size; i += 4) {
+        MEM_writeLE32(op + i, RAND(seed));
+    }
+    for (; i < size; i++) {
+        op[i] = RAND(seed) & 0xff;
+    }
+}
+
+/* Write `size` bytes into `ptr` following the distribution `dist` */
+static void RAND_bufferDist(U32* seed, BYTE* dist, void* ptr, size_t size)
+{
+    size_t i;
+    BYTE* op = ptr;
+
+    for (i = 0; i < size; i++) {
+        op[i] = dist[RAND(seed) % DISTSIZE];
+    }
+}
+
+/* Generate a random distribution where the frequency of each symbol follows a
+ * geometric distribution defined by `weight`
+ * `dist` should have size at least `DISTSIZE` */
+static void RAND_genDist(U32* seed, BYTE* dist, double weight)
+{
+    size_t i = 0;
+    size_t statesLeft = DISTSIZE;
+    BYTE symb = RAND(seed) % 256;
+    BYTE step = (RAND(seed) % 256) | 1; /* force it to be odd so it's relatively prime to 256 */
+
+    while (i < DISTSIZE) {
+        size_t states = ((size_t)(weight * statesLeft)) + 1;
+        size_t j;
+        for (j = 0; j < states && i < DISTSIZE; j++, i++) {
+            dist[i] = symb;
+        }
+
+        symb += step;
+        statesLeft -= states;
+    }
+}
+
+/* Generates a random number in the range [min, max) */
+static inline U32 RAND_range(U32* seed, U32 min, U32 max)
+{
+    return (RAND(seed) % (max-min)) + min;
+}
+
+#define ROUND(x) ((U32)(x + 0.5))
+
+/* Generates a random number in an exponential distribution with mean `mean` */
+static double RAND_exp(U32* seed, double mean)
+{
+    double const u = RAND(seed) / (double) UINT_MAX;
+    return log(1-u) * (-mean);
+}
+
+/*-*******************************************************
+*  Constants and Structs
+*********************************************************/
+const char *BLOCK_TYPES[] = {"raw", "rle", "compressed"};
+
+#define MAX_DECOMPRESSED_SIZE_LOG 20
+#define MAX_DECOMPRESSED_SIZE (1ULL << MAX_DECOMPRESSED_SIZE_LOG)
+
+#define MAX_WINDOW_LOG 22 /* Recommended support is 8MB, so limit to 4MB + mantissa */
+#define MAX_BLOCK_SIZE (128ULL * 1024)
+
+#define MIN_SEQ_LEN (3)
+#define MAX_NB_SEQ ((MAX_BLOCK_SIZE + MIN_SEQ_LEN - 1) / MIN_SEQ_LEN)
+
+BYTE CONTENT_BUFFER[MAX_DECOMPRESSED_SIZE];
+BYTE FRAME_BUFFER[MAX_DECOMPRESSED_SIZE * 2];
+BYTE LITERAL_BUFFER[MAX_BLOCK_SIZE];
+
+seqDef SEQUENCE_BUFFER[MAX_NB_SEQ];
+BYTE SEQUENCE_LITERAL_BUFFER[MAX_BLOCK_SIZE]; /* storeSeq expects a place to copy literals to */
+BYTE SEQUENCE_LLCODE[MAX_BLOCK_SIZE];
+BYTE SEQUENCE_MLCODE[MAX_BLOCK_SIZE];
+BYTE SEQUENCE_OFCODE[MAX_BLOCK_SIZE];
+
+unsigned WKSP[1024];
+
+typedef struct {
+    size_t contentSize; /* 0 means unknown (unless contentSize == windowSize == 0) */
+    unsigned windowSize; /* contentSize >= windowSize means single segment */
+} frameHeader_t;
+
+/* For repeat modes */
+typedef struct {
+    U32 rep[ZSTD_REP_NUM];
+
+    int hufInit;
+    /* the distribution used in the previous block for repeat mode */
+    BYTE hufDist[DISTSIZE];
+    U32 hufTable [256]; /* HUF_CElt is an incomplete type */
+
+    int fseInit;
+    FSE_CTable offcodeCTable  [FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
+    FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
+    FSE_CTable litlengthCTable  [FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
+
+    /* Symbols that were present in the previous distribution, for use with
+     * set_repeat */
+    BYTE litlengthSymbolSet[36];
+    BYTE offsetSymbolSet[29];
+    BYTE matchlengthSymbolSet[53];
+} cblockStats_t;
+
+typedef struct {
+    void* data;
+    void* dataStart;
+    void* dataEnd;
+
+    void* src;
+    void* srcStart;
+    void* srcEnd;
+
+    frameHeader_t header;
+
+    cblockStats_t stats;
+    cblockStats_t oldStats; /* so they can be rolled back if uncompressible */
+} frame_t;
+
+/*-*******************************************************
+*  Generator Functions
+*********************************************************/
+
+/* Generate and write a random frame header */
+static void writeFrameHeader(U32* seed, frame_t* frame)
+{
+    BYTE* const op = frame->data;
+    size_t pos = 0;
+    frameHeader_t fl;
+
+    BYTE windowByte = 0;
+
+    int singleSegment = 0;
+    int contentSizeFlag = 0;
+    int fcsCode = 0;
+
+    memset(&fl, 0, sizeof(fl));
+
+    /* generate window size */
+    {
+        /* Follow window algorithm from specification */
+        int const exponent = RAND(seed) % (MAX_WINDOW_LOG - 10);
+        int const mantissa = RAND(seed) % 8;
+        windowByte = (exponent << 3) | mantissa;
+        fl.windowSize = (1U << (exponent + 10));
+        fl.windowSize += fl.windowSize / 8 * mantissa;
+    }
+
+    {
+        /* Generate random content size */
+        size_t highBit;
+        if (RAND(seed) & 7) {
+            /* do content of at least 128 bytes */
+            highBit = 1ULL << RAND_range(seed, 7, MAX_DECOMPRESSED_SIZE_LOG);
+        } else if (RAND(seed) & 3) {
+            /* do small content */
+            highBit = 1ULL << RAND_range(seed, 0, 7);
+        } else {
+            /* 0 size frame */
+            highBit = 0;
+        }
+        fl.contentSize = highBit ? highBit + (RAND(seed) % highBit) : 0;
+
+        /* provide size sometimes */
+        contentSizeFlag = RAND(seed) & 1;
+
+        if (contentSizeFlag && !(RAND(seed) & 7)) {
+            /* do single segment sometimes */
+            fl.windowSize = fl.contentSize;
+            singleSegment = 1;
+        }
+    }
+
+    if (contentSizeFlag) {
+        /* Determine how large fcs field has to be */
+        int minFcsCode = (fl.contentSize >= 256) +
+                               (fl.contentSize >= 65536 + 256) +
+                               (fl.contentSize > 0xFFFFFFFFU);
+        if (!singleSegment && !minFcsCode) {
+            minFcsCode = 1;
+        }
+        fcsCode = minFcsCode + (RAND(seed) % (4 - minFcsCode));
+        if (fcsCode == 1 && fl.contentSize < 256) fcsCode++;
+    }
+
+    /* write out the header */
+    MEM_writeLE32(op + pos, ZSTD_MAGICNUMBER);
+    pos += 4;
+
+    {
+        BYTE const frameHeaderDescriptor =
+                (fcsCode << 6) | (singleSegment << 5) | (1 << 2);
+        op[pos++] = frameHeaderDescriptor;
+    }
+
+    if (!singleSegment) {
+        op[pos++] = windowByte;
+    }
+
+    if (contentSizeFlag) {
+        switch (fcsCode) {
+        default: /* Impossible */
+        case 0: op[pos++] = fl.contentSize; break;
+        case 1: MEM_writeLE16(op + pos, fl.contentSize - 256); pos += 2; break;
+        case 2: MEM_writeLE32(op + pos, fl.contentSize); pos += 4; break;
+        case 3: MEM_writeLE64(op + pos, fl.contentSize); pos += 8; break;
+        }
+    }
+
+    DISPLAYLEVEL(2, " frame content size:\t%zu\n", fl.contentSize);
+    DISPLAYLEVEL(2, " frame window size:\t%u\n", fl.windowSize);
+    DISPLAYLEVEL(2, " content size flag:\t%d\n", contentSizeFlag);
+    DISPLAYLEVEL(2, " single segment flag:\t%d\n", singleSegment);
+
+    frame->data = op + pos;
+    frame->header = fl;
+}
+
+/* Write a literal block in either raw or RLE form, return the literals size */
+static size_t writeLiteralsBlockSimple(U32* seed, frame_t* frame, size_t contentSize)
+{
+    BYTE* op = (BYTE*)frame->data;
+    int const type = RAND(seed) % 2;
+    int const sizeFormatDesc = RAND(seed) % 8;
+    size_t litSize;
+    size_t maxLitSize = MIN(contentSize, MAX_BLOCK_SIZE);
+
+    if (sizeFormatDesc == 0) {
+        /* Size_FormatDesc = ?0 */
+        maxLitSize = MIN(maxLitSize, 31);
+    } else if (sizeFormatDesc <= 4) {
+        /* Size_FormatDesc = 01 */
+        maxLitSize = MIN(maxLitSize, 4095);
+    } else {
+        /* Size_Format = 11 */
+        maxLitSize = MIN(maxLitSize, 1048575);
+    }
+
+    litSize = RAND(seed) % (maxLitSize + 1);
+    if (frame->src == frame->srcStart && litSize == 0) {
+        litSize = 1; /* no empty literals if there's nothing preceding this block */
+    }
+    if (litSize + 3 > contentSize) {
+        litSize = contentSize; /* no matches shorter than 3 are allowed */
+    }
+    /* use smallest size format that fits */
+    if (litSize < 32) {
+        op[0] = (type | (0 << 2) | (litSize << 3)) & 0xff;
+        op += 1;
+    } else if (litSize < 4096) {
+        op[0] = (type | (1 << 2) | (litSize << 4)) & 0xff;
+        op[1] = (litSize >> 4) & 0xff;
+        op += 2;
+    } else {
+        op[0] = (type | (3 << 2) | (litSize << 4)) & 0xff;
+        op[1] = (litSize >> 4) & 0xff;
+        op[2] = (litSize >> 12) & 0xff;
+        op += 3;
+    }
+
+    if (type == 0) {
+        /* Raw literals */
+        DISPLAYLEVEL(4, "   raw literals\n");
+
+        RAND_buffer(seed, LITERAL_BUFFER, litSize);
+        memcpy(op, LITERAL_BUFFER, litSize);
+        op += litSize;
+    } else {
+        /* RLE literals */
+        BYTE const symb = RAND(seed) % 256;
+
+        DISPLAYLEVEL(4, "   rle literals: 0x%02x\n", (U32)symb);
+
+        memset(LITERAL_BUFFER, symb, litSize);
+        op[0] = symb;
+        op++;
+    }
+
+    frame->data = op;
+
+    return litSize;
+}
+
+/* Generate a Huffman header for the given source */
+static size_t writeHufHeader(U32* seed, HUF_CElt* hufTable, void* dst, size_t dstSize,
+                                 const void* src, size_t srcSize)
+{
+    BYTE* const ostart = (BYTE*)dst;
+    BYTE* op = ostart;
+
+    unsigned huffLog = 11;
+    U32 maxSymbolValue = 255;
+
+    U32 count[HUF_SYMBOLVALUE_MAX+1];
+
+    /* Scan input and build symbol stats */
+    {   size_t const largest = FSE_count_wksp (count, &maxSymbolValue, (const BYTE*)src, srcSize, WKSP);
+        if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 0; }   /* single symbol, rle */
+        if (largest <= (srcSize >> 7)+1) return 0;   /* Fast heuristic : not compressible enough */
+    }
+
+    /* Build Huffman Tree */
+    /* Max Huffman log is 11, min is highbit(maxSymbolValue)+1 */
+    huffLog = RAND_range(seed, ZSTD_highbit32(maxSymbolValue)+1, huffLog+1);
+    DISPLAYLEVEL(6, "     huffman log: %u\n", huffLog);
+    {   size_t const maxBits = HUF_buildCTable_wksp (hufTable, count, maxSymbolValue, huffLog, WKSP, sizeof(WKSP));
+        CHECKERR(maxBits);
+        huffLog = (U32)maxBits;
+    }
+
+    /* Write table description header */
+    {   size_t const hSize = HUF_writeCTable (op, dstSize, hufTable, maxSymbolValue, huffLog);
+        if (hSize + 12 >= srcSize) return 0;   /* not useful to try compression */
+        op += hSize;
+    }
+
+    return op - ostart;
+}
+
+/* Write a Huffman coded literals block and return the litearls size */
+static size_t writeLiteralsBlockCompressed(U32* seed, frame_t* frame, size_t contentSize)
+{
+    BYTE* origop = (BYTE*)frame->data;
+    BYTE* opend = (BYTE*)frame->dataEnd;
+    BYTE* op;
+    BYTE* const ostart = origop;
+    int const sizeFormat = RAND(seed) % 4;
+    size_t litSize;
+    size_t hufHeaderSize = 0;
+    size_t compressedSize = 0;
+    size_t maxLitSize = MIN(contentSize-3, MAX_BLOCK_SIZE);
+
+    symbolEncodingType_e hType;
+
+    if (contentSize < 64) {
+        /* make sure we get reasonably-sized literals for compression */
+        return ERROR(GENERIC);
+    }
+
+    DISPLAYLEVEL(4, "   compressed literals\n");
+
+    switch (sizeFormat) {
+    case 0: /* fall through, size is the same as case 1 */
+    case 1:
+        maxLitSize = MIN(maxLitSize, 1023);
+        origop += 3;
+        break;
+    case 2:
+        maxLitSize = MIN(maxLitSize, 16383);
+        origop += 4;
+        break;
+    case 3:
+        maxLitSize = MIN(maxLitSize, 262143);
+        origop += 5;
+        break;
+    default:; /* impossible */
+    }
+
+    do {
+        op = origop;
+        do {
+            litSize = RAND(seed) % (maxLitSize + 1);
+        } while (litSize < 32); /* avoid small literal sizes */
+        if (litSize + 3 > contentSize) {
+            litSize = contentSize; /* no matches shorter than 3 are allowed */
+        }
+
+        /* most of the time generate a new distribution */
+        if ((RAND(seed) & 3) || !frame->stats.hufInit) {
+            do {
+                if (RAND(seed) & 3) {
+                    /* add 10 to ensure some compressability */
+                    double const weight = ((RAND(seed) % 90) + 10) / 100.0;
+
+                    DISPLAYLEVEL(5, "    distribution weight: %d%%\n",
+                                 (int)(weight * 100));
+
+                    RAND_genDist(seed, frame->stats.hufDist, weight);
+                } else {
+                    /* sometimes do restricted range literals to force
+                     * non-huffman headers */
+                    DISPLAYLEVEL(5, "    small range literals\n");
+                    RAND_bufferMaxSymb(seed, frame->stats.hufDist, DISTSIZE,
+                                       15);
+                }
+                RAND_bufferDist(seed, frame->stats.hufDist, LITERAL_BUFFER,
+                                litSize);
+
+                /* generate the header from the distribution instead of the
+                 * actual data to avoid bugs with symbols that were in the
+                 * distribution but never showed up in the output */
+                hufHeaderSize = writeHufHeader(
+                        seed, (HUF_CElt*)frame->stats.hufTable, op, opend - op,
+                        frame->stats.hufDist, DISTSIZE);
+                CHECKERR(hufHeaderSize);
+                /* repeat until a valid header is written */
+            } while (hufHeaderSize == 0);
+            op += hufHeaderSize;
+            hType = set_compressed;
+
+            frame->stats.hufInit = 1;
+        } else {
+            /* repeat the distribution/table from last time */
+            DISPLAYLEVEL(5, "    huffman repeat stats\n");
+            RAND_bufferDist(seed, frame->stats.hufDist, LITERAL_BUFFER,
+                            litSize);
+            hufHeaderSize = 0;
+            hType = set_repeat;
+        }
+
+        do {
+            compressedSize =
+                    sizeFormat == 0
+                            ? HUF_compress1X_usingCTable(
+                                      op, opend - op, LITERAL_BUFFER, litSize,
+                                      (HUF_CElt*)frame->stats.hufTable)
+                            : HUF_compress4X_usingCTable(
+                                      op, opend - op, LITERAL_BUFFER, litSize,
+                                      (HUF_CElt*)frame->stats.hufTable);
+            CHECKERR(compressedSize);
+            /* this only occurs when it could not compress or similar */
+        } while (compressedSize <= 0);
+
+        op += compressedSize;
+
+        compressedSize += hufHeaderSize;
+        DISPLAYLEVEL(5, "    regenerated size: %zu\n", litSize);
+        DISPLAYLEVEL(5, "    compressed size: %zu\n", compressedSize);
+        if (compressedSize >= litSize) {
+            DISPLAYLEVEL(5, "     trying again\n");
+            /* if we have to try again, reset the stats so we don't accidentally
+             * try to repeat a distribution we just made */
+            frame->stats = frame->oldStats;
+        } else {
+            break;
+        }
+    } while (1);
+
+    /* write header */
+    switch (sizeFormat) {
+    case 0: /* fall through, size is the same as case 1 */
+    case 1: {
+        U32 const header = hType | (sizeFormat << 2) | ((U32)litSize << 4) |
+                           ((U32)compressedSize << 14);
+        MEM_writeLE24(ostart, header);
+        break;
+    }
+    case 2: {
+        U32 const header = hType | (sizeFormat << 2) | ((U32)litSize << 4) |
+                           ((U32)compressedSize << 18);
+        MEM_writeLE32(ostart, header);
+        break;
+    }
+    case 3: {
+        U32 const header = hType | (sizeFormat << 2) | ((U32)litSize << 4) |
+                           ((U32)compressedSize << 22);
+        MEM_writeLE32(ostart, header);
+        ostart[4] = (BYTE)(compressedSize >> 10);
+        break;
+    }
+    default:; /* impossible */
+    }
+
+    frame->data = op;
+    return litSize;
+}
+
+static size_t writeLiteralsBlock(U32* seed, frame_t* frame, size_t contentSize)
+{
+    /* only do compressed for larger segments to avoid compressibility issues */
+    if (RAND(seed) & 7 && contentSize >= 64) {
+        return writeLiteralsBlockCompressed(seed, frame, contentSize);
+    } else {
+        return writeLiteralsBlockSimple(seed, frame, contentSize);
+    }
+}
+
+static inline void initSeqStore(seqStore_t *seqStore) {
+    seqStore->sequencesStart = SEQUENCE_BUFFER;
+    seqStore->litStart = SEQUENCE_LITERAL_BUFFER;
+    seqStore->llCode = SEQUENCE_LLCODE;
+    seqStore->mlCode = SEQUENCE_MLCODE;
+    seqStore->ofCode = SEQUENCE_OFCODE;
+
+    ZSTD_resetSeqStore(seqStore);
+}
+
+/* Randomly generate sequence commands */
+static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
+                                size_t contentSize, size_t literalsSize)
+{
+    /* The total length of all the matches */
+    size_t const remainingMatch = contentSize - literalsSize;
+    size_t excessMatch;
+    U32 i;
+
+    U32 numSequences;
+
+    const BYTE* literals = LITERAL_BUFFER;
+    BYTE* srcPtr = frame->src;
+
+    if (literalsSize == contentSize) {
+        numSequences = 0;
+    } else {
+        /* each match must be at least MIN_SEQ_LEN, so this is the maximum
+         * number of sequences we can have */
+        U32 const maxSequences = (U32)remainingMatch / MIN_SEQ_LEN;
+        numSequences = (RAND(seed) % maxSequences) + 1;
+
+        /* the extra match lengths we have to allocate to each sequence */
+        excessMatch = remainingMatch - numSequences * MIN_SEQ_LEN;
+    }
+
+    DISPLAYLEVEL(5, "    total match lengths: %zu\n", remainingMatch);
+
+    for (i = 0; i < numSequences; i++) {
+        /* Generate match and literal lengths by exponential distribution to
+         * ensure nice numbers */
+        U32 matchLen =
+                MIN_SEQ_LEN +
+                ROUND(RAND_exp(seed, excessMatch / (double)(numSequences - i)));
+        U32 literalLen =
+                (RAND(seed) & 7)
+                        ? ROUND(RAND_exp(seed,
+                                         literalsSize /
+                                                 (double)(numSequences - i)))
+                        : 0;
+        /* actual offset, code to send, and point to copy up to when shifting
+         * codes in the repeat offsets history */
+        U32 offset, offsetCode, repIndex;
+
+        /* bounds checks */
+        matchLen = MIN(matchLen, excessMatch + MIN_SEQ_LEN);
+        literalLen = MIN(literalLen, literalsSize);
+        if (i == 0 && srcPtr == frame->srcStart && literalLen == 0) literalLen = 1;
+        if (i + 1 == numSequences) matchLen = MIN_SEQ_LEN + excessMatch;
+
+        memcpy(srcPtr, literals, literalLen);
+        srcPtr += literalLen;
+
+        do {
+            if (RAND(seed) & 7) {
+                /* do a normal offset */
+                offset = (RAND(seed) %
+                          MIN(frame->header.windowSize,
+                              (BYTE*)srcPtr - (BYTE*)frame->srcStart)) +
+                         1;
+                offsetCode = offset + ZSTD_REP_MOVE;
+                repIndex = 2;
+            } else {
+                /* do a repeat offset */
+                offsetCode = RAND(seed) % 3;
+                if (literalLen > 0) {
+                    offset = frame->stats.rep[offsetCode];
+                    repIndex = offsetCode;
+                } else {
+                    /* special case */
+                    offset = offsetCode == 2 ? frame->stats.rep[0] - 1
+                                           : frame->stats.rep[offsetCode + 1];
+                    repIndex = MIN(2, offsetCode + 1);
+                }
+            }
+        } while (offset > (BYTE*)srcPtr - (BYTE*)frame->srcStart || offset == 0);
+
+        {   size_t j;
+            for (j = 0; j < matchLen; j++) {
+                *srcPtr = *(srcPtr-offset);
+                srcPtr++;
+            }
+        }
+
+        {   int r;
+            for (r = repIndex; r > 0; r--) {
+                frame->stats.rep[r] = frame->stats.rep[r - 1];
+            }
+            frame->stats.rep[0] = offset;
+        }
+
+        DISPLAYLEVEL(6, "      LL: %5u OF: %5u ML: %5u", literalLen, offset, matchLen);
+        DISPLAYLEVEL(7, " srcPos: %8zu seqNb: %3u",
+                     (BYTE*)srcPtr - (BYTE*)frame->srcStart, i);
+        DISPLAYLEVEL(6, "\n");
+        if (offsetCode < 3) {
+            DISPLAYLEVEL(7, "        repeat offset: %d\n", repIndex);
+        }
+        /* use libzstd sequence handling */
+        ZSTD_storeSeq(seqStore, literalLen, literals, offsetCode,
+                      matchLen - MINMATCH);
+
+        literalsSize -= literalLen;
+        excessMatch -= (matchLen - MIN_SEQ_LEN);
+        literals += literalLen;
+    }
+
+    memcpy(srcPtr, literals, literalsSize);
+    srcPtr += literalsSize;
+    DISPLAYLEVEL(6, "      excess literals: %5zu", literalsSize);
+    DISPLAYLEVEL(7, " srcPos: %8zu", (BYTE*)srcPtr - (BYTE*)frame->srcStart);
+    DISPLAYLEVEL(6, "\n");
+
+    return numSequences;
+}
+
+static void initSymbolSet(const BYTE* symbols, size_t len, BYTE* set, BYTE maxSymbolValue)
+{
+    size_t i;
+
+    memset(set, 0, (size_t)maxSymbolValue+1);
+
+    for (i = 0; i < len; i++) {
+        set[symbols[i]] = 1;
+    }
+}
+
+static int isSymbolSubset(const BYTE* symbols, size_t len, const BYTE* set, BYTE maxSymbolValue)
+{
+    size_t i;
+
+    for (i = 0; i < len; i++) {
+        if (symbols[i] > maxSymbolValue || !set[symbols[i]]) {
+            return 0;
+        }
+    }
+    return 1;
+}
+
+static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr,
+                             size_t nbSeq)
+{
+    /* This code is mostly copied from ZSTD_compressSequences in zstd_compress.c */
+    U32 count[MaxSeq+1];
+    S16 norm[MaxSeq+1];
+    FSE_CTable* CTable_LitLength = frame->stats.litlengthCTable;
+    FSE_CTable* CTable_OffsetBits = frame->stats.offcodeCTable;
+    FSE_CTable* CTable_MatchLength = frame->stats.matchlengthCTable;
+    U32 LLtype, Offtype, MLtype;   /* compressed, raw or rle */
+    const seqDef* const sequences = seqStorePtr->sequencesStart;
+    const BYTE* const ofCodeTable = seqStorePtr->ofCode;
+    const BYTE* const llCodeTable = seqStorePtr->llCode;
+    const BYTE* const mlCodeTable = seqStorePtr->mlCode;
+    BYTE* const oend = (BYTE*)frame->dataEnd;
+    BYTE* op = (BYTE*)frame->data;
+    BYTE* seqHead;
+    BYTE scratchBuffer[1<>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2;
+    else op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3;
+
+    /* seqHead : flags for FSE encoding type */
+    seqHead = op++;
+
+    if (nbSeq==0) {
+        frame->data = op;
+
+        return 0;
+    }
+
+    /* convert length/distances into codes */
+    ZSTD_seqToCodes(seqStorePtr);
+
+    /* CTable for Literal Lengths */
+    {   U32 max = MaxLL;
+        size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, WKSP);
+        if (mostFrequent == nbSeq) {
+            /* do RLE if we have the chance */
+            *op++ = llCodeTable[0];
+            FSE_buildCTable_rle(CTable_LitLength, (BYTE)max);
+            LLtype = set_rle;
+        } else if (frame->stats.fseInit && !(RAND(seed) & 3) &&
+                   isSymbolSubset(llCodeTable, nbSeq,
+                                  frame->stats.litlengthSymbolSet, 35)) {
+            /* maybe do repeat mode if we're allowed to */
+            LLtype = set_repeat;
+        } else if (!(RAND(seed) & 3)) {
+            /* maybe use the default distribution */
+            FSE_buildCTable_wksp(CTable_LitLength, LL_defaultNorm, MaxLL, LL_defaultNormLog, scratchBuffer, sizeof(scratchBuffer));
+            LLtype = set_basic;
+        } else {
+            /* fall back on a full table */
+            size_t nbSeq_1 = nbSeq;
+            const U32 tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max);
+            if (count[llCodeTable[nbSeq-1]]>1) { count[llCodeTable[nbSeq-1]]--; nbSeq_1--; }
+            FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
+            { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog);   /* overflow protected */
+              if (FSE_isError(NCountSize)) return ERROR(GENERIC);
+              op += NCountSize; }
+            FSE_buildCTable_wksp(CTable_LitLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer));
+            LLtype = set_compressed;
+    }   }
+
+    /* CTable for Offsets */
+    /* see Literal Lengths for descriptions of mode choices */
+    {   U32 max = MaxOff;
+        size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, WKSP);
+        if (mostFrequent == nbSeq) {
+            *op++ = ofCodeTable[0];
+            FSE_buildCTable_rle(CTable_OffsetBits, (BYTE)max);
+            Offtype = set_rle;
+        } else if (frame->stats.fseInit && !(RAND(seed) & 3) &&
+                   isSymbolSubset(ofCodeTable, nbSeq,
+                                  frame->stats.offsetSymbolSet, 28)) {
+            Offtype = set_repeat;
+        } else if (!(RAND(seed) & 3)) {
+            FSE_buildCTable_wksp(CTable_OffsetBits, OF_defaultNorm, MaxOff, OF_defaultNormLog, scratchBuffer, sizeof(scratchBuffer));
+            Offtype = set_basic;
+        } else {
+            size_t nbSeq_1 = nbSeq;
+            const U32 tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max);
+            if (count[ofCodeTable[nbSeq-1]]>1) { count[ofCodeTable[nbSeq-1]]--; nbSeq_1--; }
+            FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
+            { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog);   /* overflow protected */
+              if (FSE_isError(NCountSize)) return ERROR(GENERIC);
+              op += NCountSize; }
+            FSE_buildCTable_wksp(CTable_OffsetBits, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer));
+            Offtype = set_compressed;
+    }   }
+
+    /* CTable for MatchLengths */
+    /* see Literal Lengths for descriptions of mode choices */
+    {   U32 max = MaxML;
+        size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, WKSP);
+        if (mostFrequent == nbSeq) {
+            *op++ = *mlCodeTable;
+            FSE_buildCTable_rle(CTable_MatchLength, (BYTE)max);
+            MLtype = set_rle;
+        } else if (frame->stats.fseInit && !(RAND(seed) & 3) &&
+                   isSymbolSubset(mlCodeTable, nbSeq,
+                                  frame->stats.matchlengthSymbolSet, 52)) {
+            MLtype = set_repeat;
+        } else if (!(RAND(seed) & 3)) {
+            /* sometimes do default distribution */
+            FSE_buildCTable_wksp(CTable_MatchLength, ML_defaultNorm, MaxML, ML_defaultNormLog, scratchBuffer, sizeof(scratchBuffer));
+            MLtype = set_basic;
+        } else {
+            /* fall back on table */
+            size_t nbSeq_1 = nbSeq;
+            const U32 tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max);
+            if (count[mlCodeTable[nbSeq-1]]>1) { count[mlCodeTable[nbSeq-1]]--; nbSeq_1--; }
+            FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max);
+            { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog);   /* overflow protected */
+              if (FSE_isError(NCountSize)) return ERROR(GENERIC);
+              op += NCountSize; }
+            FSE_buildCTable_wksp(CTable_MatchLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer));
+            MLtype = set_compressed;
+    }   }
+    frame->stats.fseInit = 1;
+    initSymbolSet(llCodeTable, nbSeq, frame->stats.litlengthSymbolSet, 35);
+    initSymbolSet(ofCodeTable, nbSeq, frame->stats.offsetSymbolSet, 28);
+    initSymbolSet(mlCodeTable, nbSeq, frame->stats.matchlengthSymbolSet, 52);
+
+    DISPLAYLEVEL(5, "    LL type: %d OF type: %d ML type: %d\n", LLtype, Offtype, MLtype);
+
+    *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
+
+    /* Encoding Sequences */
+    {   BIT_CStream_t blockStream;
+        FSE_CState_t  stateMatchLength;
+        FSE_CState_t  stateOffsetBits;
+        FSE_CState_t  stateLitLength;
+
+        CHECK_E(BIT_initCStream(&blockStream, op, oend-op), dstSize_tooSmall); /* not enough space remaining */
+
+        /* first symbols */
+        FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]);
+        FSE_initCState2(&stateOffsetBits,  CTable_OffsetBits,  ofCodeTable[nbSeq-1]);
+        FSE_initCState2(&stateLitLength,   CTable_LitLength,   llCodeTable[nbSeq-1]);
+        BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]);
+        if (MEM_32bits()) BIT_flushBits(&blockStream);
+        BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]);
+        if (MEM_32bits()) BIT_flushBits(&blockStream);
+        BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]);
+        BIT_flushBits(&blockStream);
+
+        {   size_t n;
+            for (n=nbSeq-2 ; n= 64-7-(LLFSELog+MLFSELog+OffFSELog)))
+                    BIT_flushBits(&blockStream);                                /* (7)*/
+                BIT_addBits(&blockStream, sequences[n].litLength, llBits);
+                if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream);
+                BIT_addBits(&blockStream, sequences[n].matchLength, mlBits);
+                if (MEM_32bits()) BIT_flushBits(&blockStream);                  /* (7)*/
+                BIT_addBits(&blockStream, sequences[n].offset, ofBits);         /* 31 */
+                BIT_flushBits(&blockStream);                                    /* (7)*/
+        }   }
+
+        FSE_flushCState(&blockStream, &stateMatchLength);
+        FSE_flushCState(&blockStream, &stateOffsetBits);
+        FSE_flushCState(&blockStream, &stateLitLength);
+
+        {   size_t const streamSize = BIT_closeCStream(&blockStream);
+            if (streamSize==0) return ERROR(dstSize_tooSmall);   /* not enough space */
+            op += streamSize;
+    }   }
+
+    frame->data = op;
+
+    return 0;
+}
+
+static size_t writeSequencesBlock(U32* seed, frame_t* frame, size_t contentSize,
+                                  size_t literalsSize)
+{
+    seqStore_t seqStore;
+    size_t numSequences;
+
+
+    initSeqStore(&seqStore);
+
+    /* randomly generate sequences */
+    numSequences = generateSequences(seed, frame, &seqStore, contentSize, literalsSize);
+    /* write them out to the frame data */
+    CHECKERR(writeSequences(seed, frame, &seqStore, numSequences));
+
+    return numSequences;
+}
+
+static size_t writeCompressedBlock(U32* seed, frame_t* frame, size_t contentSize)
+{
+    BYTE* const blockStart = (BYTE*)frame->data;
+    size_t literalsSize;
+    size_t nbSeq;
+
+    DISPLAYLEVEL(4, "  compressed block:\n");
+
+    literalsSize = writeLiteralsBlock(seed, frame, contentSize);
+
+    DISPLAYLEVEL(4, "   literals size: %zu\n", literalsSize);
+
+    nbSeq = writeSequencesBlock(seed, frame, contentSize, literalsSize);
+
+    DISPLAYLEVEL(4, "   number of sequences: %zu\n", nbSeq);
+
+    return (BYTE*)frame->data - blockStart;
+}
+
+static void writeBlock(U32* seed, frame_t* frame, size_t contentSize,
+                       int lastBlock)
+{
+    int const blockTypeDesc = RAND(seed) % 8;
+    size_t blockSize;
+    int blockType;
+
+    BYTE *const header = (BYTE*)frame->data;
+    BYTE *op = header + 3;
+
+    DISPLAYLEVEL(3, " block:\n");
+    DISPLAYLEVEL(3, "  block content size: %zu\n", contentSize);
+    DISPLAYLEVEL(3, "  last block: %s\n", lastBlock ? "yes" : "no");
+
+    if (blockTypeDesc == 0) {
+        /* Raw data frame */
+
+        RAND_buffer(seed, frame->src, contentSize);
+        memcpy(op, frame->src, contentSize);
+
+        op += contentSize;
+        blockType = 0;
+        blockSize = contentSize;
+    } else if (blockTypeDesc == 1) {
+        /* RLE */
+        BYTE const symbol = RAND(seed) & 0xff;
+
+        op[0] = symbol;
+        memset(frame->src, symbol, contentSize);
+
+        op++;
+        blockType = 1;
+        blockSize = contentSize;
+    } else {
+        /* compressed, most common */
+        size_t compressedSize;
+        blockType = 2;
+
+        frame->oldStats = frame->stats;
+
+        frame->data = op;
+        compressedSize = writeCompressedBlock(seed, frame, contentSize);
+        if (compressedSize > contentSize) {
+            blockType = 0;
+            memcpy(op, frame->src, contentSize);
+
+            op += contentSize;
+            blockSize = contentSize; /* fall back on raw block if data doesn't
+                                        compress */
+
+            frame->stats = frame->oldStats; /* don't update the stats */
+        } else {
+            op += compressedSize;
+            blockSize = compressedSize;
+        }
+    }
+    frame->src = (BYTE*)frame->src + contentSize;
+
+    DISPLAYLEVEL(3, "  block type: %s\n", BLOCK_TYPES[blockType]);
+    DISPLAYLEVEL(3, "  block size field: %zu\n", blockSize);
+
+    header[0] = (lastBlock | (blockType << 1) | (blockSize << 3)) & 0xff;
+    MEM_writeLE16(header + 1, blockSize >> 5);
+
+    frame->data = op;
+}
+
+static void writeBlocks(U32* seed, frame_t* frame)
+{
+    size_t contentLeft = frame->header.contentSize;
+    size_t const maxBlockSize = MIN(MAX_BLOCK_SIZE, frame->header.windowSize);
+    while (1) {
+        /* 1 in 4 chance of ending frame */
+        int const lastBlock = contentLeft > maxBlockSize ? 0 : !(RAND(seed) & 3);
+        size_t blockContentSize;
+        if (lastBlock) {
+            blockContentSize = contentLeft;
+        } else {
+            if (contentLeft > 0 && (RAND(seed) & 7)) {
+                /* some variable size blocks */
+                blockContentSize = RAND(seed) % (MIN(maxBlockSize, contentLeft)+1);
+            } else if (contentLeft > maxBlockSize && (RAND(seed) & 1)) {
+                /* some full size blocks */
+                blockContentSize = maxBlockSize;
+            } else {
+                /* some empty blocks */
+                blockContentSize = 0;
+            }
+        }
+
+        writeBlock(seed, frame, blockContentSize, lastBlock);
+
+        contentLeft -= blockContentSize;
+        if (lastBlock) break;
+    }
+}
+
+static void writeChecksum(frame_t* frame)
+{
+    /* write checksum so implementations can verify their output */
+    U64 digest = XXH64(frame->srcStart, (BYTE*)frame->src-(BYTE*)frame->srcStart, 0);
+    DISPLAYLEVEL(2, "  checksum: %08x\n", (U32)digest);
+    MEM_writeLE32(frame->data, (U32)digest);
+    frame->data = (BYTE*)frame->data + 4;
+}
+
+static void outputBuffer(const void* buf, size_t size, const char* const path)
+{
+    /* write data out to file */
+    const BYTE* ip = (const BYTE*)buf;
+    FILE* out;
+    if (path) {
+        out = fopen(path, "wb");
+    } else {
+        out = stdout;
+    }
+    if (!out) {
+        fprintf(stderr, "Failed to open file at %s: ", path);
+        perror(NULL);
+        exit(1);
+    }
+
+    {
+        size_t fsize = size;
+        size_t written = 0;
+        while (written < fsize) {
+            written += fwrite(ip + written, 1, fsize - written, out);
+            if (ferror(out)) {
+                fprintf(stderr, "Failed to write to file at %s: ", path);
+                perror(NULL);
+                exit(1);
+            }
+        }
+    }
+
+    if (path) {
+        fclose(out);
+    }
+}
+
+static void initFrame(frame_t* fr)
+{
+    memset(fr, 0, sizeof(*fr));
+    fr->data = fr->dataStart = FRAME_BUFFER;
+    fr->dataEnd = FRAME_BUFFER + sizeof(FRAME_BUFFER);
+    fr->src = fr->srcStart = CONTENT_BUFFER;
+    fr->srcEnd = CONTENT_BUFFER + sizeof(CONTENT_BUFFER);
+
+    /* init repeat codes */
+    fr->stats.rep[0] = 1;
+    fr->stats.rep[1] = 4;
+    fr->stats.rep[2] = 8;
+}
+
+/* Return the final seed */
+static U32 generateFrame(U32 seed, frame_t* fr)
+{
+    /* generate a complete frame */
+    DISPLAYLEVEL(1, "frame seed: %u\n", seed);
+
+    initFrame(fr);
+
+    writeFrameHeader(&seed, fr);
+    writeBlocks(&seed, fr);
+    writeChecksum(fr);
+
+    return seed;
+}
+
+/*-*******************************************************
+*  Test Mode
+*********************************************************/
+
+BYTE DECOMPRESSED_BUFFER[MAX_DECOMPRESSED_SIZE];
+
+static size_t testDecodeSimple(frame_t* fr)
+{
+    /* test decoding the generated data with the simple API */
+    size_t const ret = ZSTD_decompress(DECOMPRESSED_BUFFER, MAX_DECOMPRESSED_SIZE,
+                           fr->dataStart, (BYTE*)fr->data - (BYTE*)fr->dataStart);
+
+    if (ZSTD_isError(ret)) return ret;
+
+    if (memcmp(DECOMPRESSED_BUFFER, fr->srcStart,
+               (BYTE*)fr->src - (BYTE*)fr->srcStart) != 0) {
+        return ERROR(corruption_detected);
+    }
+
+    return ret;
+}
+
+static size_t testDecodeStreaming(frame_t* fr)
+{
+    /* test decoding the generated data with the streaming API */
+    ZSTD_DStream* zd = ZSTD_createDStream();
+    ZSTD_inBuffer in;
+    ZSTD_outBuffer out;
+    size_t ret;
+
+    if (!zd) return ERROR(memory_allocation);
+
+    in.src = fr->dataStart;
+    in.pos = 0;
+    in.size = (BYTE*)fr->data - (BYTE*)fr->dataStart;
+
+    out.dst = DECOMPRESSED_BUFFER;
+    out.pos = 0;
+    out.size = ZSTD_DStreamOutSize();
+
+    ZSTD_initDStream(zd);
+    while (1) {
+        ret = ZSTD_decompressStream(zd, &out, &in);
+        if (ZSTD_isError(ret)) goto cleanup; /* error */
+        if (ret == 0) break; /* frame is done */
+
+        /* force decoding to be done in chunks */
+        out.size += MIN(ZSTD_DStreamOutSize(), MAX_DECOMPRESSED_SIZE - out.size);
+    }
+
+    ret = out.pos;
+
+    if (memcmp(out.dst, fr->srcStart, out.pos) != 0) {
+        return ERROR(corruption_detected);
+    }
+
+cleanup:
+    ZSTD_freeDStream(zd);
+    return ret;
+}
+
+static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS)
+{
+    unsigned fnum;
+
+    clock_t const startClock = clock();
+    clock_t const maxClockSpan = testDurationS * CLOCKS_PER_SEC;
+
+    if (numFiles == 0 && !testDurationS) numFiles = 1;
+
+    DISPLAY("seed: %u\n", seed);
+
+    for (fnum = 0; fnum < numFiles || clockSpan(startClock) < maxClockSpan; fnum++) {
+        frame_t fr;
+
+        if (fnum < numFiles)
+            DISPLAYUPDATE("\r%u/%u        ", fnum, numFiles);
+        else
+            DISPLAYUPDATE("\r%u           ", fnum);
+
+        seed = generateFrame(seed, &fr);
+
+        {   size_t const r = testDecodeSimple(&fr);
+            if (ZSTD_isError(r)) {
+                DISPLAY("Error in simple mode on test seed %u: %s\n", seed + fnum,
+                        ZSTD_getErrorName(r));
+                return 1;
+            }
+        }
+        {   size_t const r = testDecodeStreaming(&fr);
+            if (ZSTD_isError(r)) {
+                DISPLAY("Error in streaming mode on test seed %u: %s\n", seed + fnum,
+                        ZSTD_getErrorName(r));
+                return 1;
+            }
+        }
+    }
+
+    DISPLAY("\r%u tests completed: ", fnum);
+    DISPLAY("OK\n");
+
+    return 0;
+}
+
+/*-*******************************************************
+*  File I/O
+*********************************************************/
+
+static int generateFile(U32 seed, const char* const path,
+                         const char* const origPath)
+{
+    frame_t fr;
+
+    generateFrame(seed, &fr);
+
+    outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, path);
+    if (origPath) {
+        outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, origPath);
+    }
+    return 0;
+}
+
+static int generateCorpus(U32 seed, unsigned numFiles, const char* const path,
+                         const char* const origPath)
+{
+    char outPath[MAX_PATH];
+    unsigned fnum;
+
+    if (!path) {
+        DISPLAY("Error: valid path is required in multiple files mode\n");
+        return 1;
+    }
+
+    DISPLAY("seed: %u\n", seed);
+
+    for (fnum = 0; fnum < numFiles; fnum++) {
+        frame_t fr;
+
+        DISPLAYUPDATE("\r%u/%u        ", fnum, numFiles);
+
+        seed = generateFrame(seed, &fr);
+
+        if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) {
+            DISPLAY("Error: path too long\n");
+            return 1;
+        }
+        outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, outPath);
+
+        if (origPath) {
+            if (snprintf(outPath, MAX_PATH, "%s/z%06u", origPath, fnum) + 1 > MAX_PATH) {
+                DISPLAY("Error: path too long\n");
+                return 1;
+            }
+            outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, outPath);
+        }
+    }
+
+    DISPLAY("\r%u/%u      \n", fnum, numFiles);
+
+    return 0;
+}
+
+
+/*_*******************************************************
+*  Command line
+*********************************************************/
+static U32 makeSeed(void)
+{
+    U32 t = time(NULL);
+    return XXH32(&t, sizeof(t), 0) % 65536;
+}
+
+static unsigned readInt(const char** argument)
+{
+    unsigned val = 0;
+    while ((**argument>='0') && (**argument<='9')) {
+        val *= 10;
+        val += **argument - '0';
+        (*argument)++;
+    }
+    return val;
+}
+
+static void usage(const char* programName)
+{
+    DISPLAY( "Usage :\n");
+    DISPLAY( "      %s [args]\n", programName);
+    DISPLAY( "\n");
+    DISPLAY( "Arguments :\n");
+    DISPLAY( " -p : select output path (default:stdout)\n");
+    DISPLAY( "                in multiple files mode this should be a directory\n");
+    DISPLAY( " -o : select path to output original file (default:no output)\n");
+    DISPLAY( "                in multiple files mode this should be a directory\n");
+    DISPLAY( " -s#      : select seed (default:random based on time)\n");
+    DISPLAY( " -n#      : number of files to generate (default:1)\n");
+    DISPLAY( " -t       : activate test mode (test files against libzstd instead of outputting them)\n");
+    DISPLAY( " -T#      : length of time to run tests for\n");
+    DISPLAY( " -v       : increase verbosity level (default:0, max:7)\n");
+    DISPLAY( " -h       : display help and exit\n");
+}
+
+int main(int argc, char** argv)
+{
+    U32 seed = 0;
+    int seedset = 0;
+    unsigned numFiles = 0;
+    unsigned testDuration = 0;
+    int testMode = 0;
+    const char* path = NULL;
+    const char* origPath = NULL;
+
+    int argNb;
+
+    /* Check command line */
+    for (argNb=1; argNb
Date: Thu, 23 Feb 2017 18:28:48 +0100
Subject: [PATCH 143/223] zlibWrapper: better description of
 ZWRAP_useZSTDcompression

---
 zlibWrapper/zstd_zlibwrapper.h | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/zlibWrapper/zstd_zlibwrapper.h b/zlibWrapper/zstd_zlibwrapper.h
index 382716921..0ebd87612 100644
--- a/zlibWrapper/zstd_zlibwrapper.h
+++ b/zlibWrapper/zstd_zlibwrapper.h
@@ -30,7 +30,13 @@ const char * zstdVersion(void);
 
 
 /*** COMPRESSION ***/
-/* enables/disables zstd compression during runtime */
+/* ZWRAP_useZSTDcompression() enables/disables zstd compression during runtime.
+   By default zstd compression is disabled. To enable zstd compression please use one of the methods:
+   - compilation with the additional option -DZWRAP_USE_ZSTD=1 
+   - using '#define ZWRAP_USE_ZSTD 1' in source code before '#include "zstd_zlibwrapper.h"'
+   - calling ZWRAP_useZSTDcompression(1)
+   All above-mentioned methods will enable zstd compression for all threads.
+   Be aware that ZWRAP_useZSTDcompression() is not thread-safe and may lead to a race condition. */
 void ZWRAP_useZSTDcompression(int turn_on);
 
 /* checks if zstd compression is turned on */
@@ -54,7 +60,11 @@ int ZWRAP_deflateReset_keepDict(z_streamp strm);
 /*** DECOMPRESSION ***/
 typedef enum { ZWRAP_FORCE_ZLIB, ZWRAP_AUTO } ZWRAP_decompress_type;
 
-/* enables/disables automatic recognition of zstd/zlib compressed data during runtime */
+/* ZWRAP_setDecompressionType() enables/disables automatic recognition of zstd/zlib compressed data during runtime.
+   By default auto-detection of zstd and zlib streams in enabled (ZWRAP_AUTO).
+   Forcing zlib decompression with ZWRAP_setDecompressionType(ZWRAP_FORCE_ZLIB) slightly improves
+   decompression speed of zlib-encoded streams.
+   Be aware that ZWRAP_setDecompressionType() is not thread-safe and may lead to a race condition. */
 void ZWRAP_setDecompressionType(ZWRAP_decompress_type type);
 
 /* checks zstd decompression type */

From 485ca8c35279a6b43db9af6e7e897c0440998f6e Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Thu, 23 Feb 2017 10:27:00 -0800
Subject: [PATCH 144/223] Update tests/README.md

---
 tests/README.md | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/tests/README.md b/tests/README.md
index 79c067abb..24a28ab7b 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -10,12 +10,14 @@ This directory contains the following programs and scripts:
 - `test-zstd-versions.py` : compatibility test between zstd versions stored on Github (v0.1+)
 - `zbufftest`  : Test tool to check ZBUFF (a buffered streaming API) integrity
 - `zstreamtest` : Fuzzer test tool for zstd streaming API
+- `legacy` : Test tool to test decoding of legacy zstd frames
+- `decodecorpus` : Tool to generate valid Zstandard frames, for verifying decoder implementations
 
 
 #### `test-zstd-versions.py` - script for testing zstd interoperability between versions
 
 This script creates `versionsTest` directory to which zstd repository is cloned.
-Then all taged (released) versions of zstd are compiled.
+Then all tagged (released) versions of zstd are compiled.
 In the following step interoperability between zstd versions is checked.
 
 
@@ -64,3 +66,25 @@ optional arguments:
   --sleepTime SLEEPTIME
                         frequency of repository checking in seconds
 ```
+
+#### `decodecorpus` - tool to generate Zstandard frames for decoder testing
+Command line tool to generate test .zst files.
+
+This tool will generate .zst files with checksums,
+as well as optionally output the corresponding correct uncompressed data for
+extra verfication.
+
+Example:
+```
+./decodecorpus -ptestfiles -otestfiles -n10000 -s5
+```
+will generate 10,000 sample .zst files using a seed of 5 in the `testfiles` directory,
+with the zstd checksum field set,
+as well as the 10,000 original files for more detailed comparison of decompression results.
+
+```
+./decodecorpus -t -T1mn
+```
+will choose a random seed, and for 1 minute,
+generate random test frames and ensure that the
+zstd library correctly decompresses them in both simple and streaming modes.

From 3cd8d50c3476b4997380663b9b8be950faf248fd Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Thu, 23 Feb 2017 13:06:50 -0800
Subject: [PATCH 145/223] Update CLI and link from educational decoder

---
 contrib/educational_decoder/README.md | 10 ++++
 tests/decodecorpus.c                  | 80 ++++++++++++++++++---------
 2 files changed, 63 insertions(+), 27 deletions(-)

diff --git a/contrib/educational_decoder/README.md b/contrib/educational_decoder/README.md
index 2e2186e02..e3b9bf58e 100644
--- a/contrib/educational_decoder/README.md
+++ b/contrib/educational_decoder/README.md
@@ -17,3 +17,13 @@ It also contains implementations of Huffman and FSE table decoding.
 
     harness   [dictionary]
 
+As an additional resource to be used with this decoder,
+see the `decodecorpus` tool in the [tests] directory.
+It generates valid Zstandard frames that can be used to verify
+a Zstandard decoder implementation.
+Note that to use the tool to verify this decoder implementation,
+the --content-size flag should be set,
+as this decoder does not handle streaming decoding,
+and so it must know the decompressed size in advance.
+
+[tests]: https://github.com/facebook/zstd/blob/dev/tests/
diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index b817d7f0d..df12dd56d 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -235,12 +235,16 @@ typedef struct {
 *  Generator Functions
 *********************************************************/
 
+struct {
+    int contentSize; /* force the content size to be present */
+} opts; /* advanced options on generation */
+
 /* Generate and write a random frame header */
 static void writeFrameHeader(U32* seed, frame_t* frame)
 {
     BYTE* const op = frame->data;
     size_t pos = 0;
-    frameHeader_t fl;
+    frameHeader_t fh;
 
     BYTE windowByte = 0;
 
@@ -248,7 +252,7 @@ static void writeFrameHeader(U32* seed, frame_t* frame)
     int contentSizeFlag = 0;
     int fcsCode = 0;
 
-    memset(&fl, 0, sizeof(fl));
+    memset(&fh, 0, sizeof(fh));
 
     /* generate window size */
     {
@@ -256,8 +260,8 @@ static void writeFrameHeader(U32* seed, frame_t* frame)
         int const exponent = RAND(seed) % (MAX_WINDOW_LOG - 10);
         int const mantissa = RAND(seed) % 8;
         windowByte = (exponent << 3) | mantissa;
-        fl.windowSize = (1U << (exponent + 10));
-        fl.windowSize += fl.windowSize / 8 * mantissa;
+        fh.windowSize = (1U << (exponent + 10));
+        fh.windowSize += fh.windowSize / 8 * mantissa;
     }
 
     {
@@ -273,28 +277,28 @@ static void writeFrameHeader(U32* seed, frame_t* frame)
             /* 0 size frame */
             highBit = 0;
         }
-        fl.contentSize = highBit ? highBit + (RAND(seed) % highBit) : 0;
+        fh.contentSize = highBit ? highBit + (RAND(seed) % highBit) : 0;
 
         /* provide size sometimes */
-        contentSizeFlag = RAND(seed) & 1;
+        contentSizeFlag = opts.contentSize | (RAND(seed) & 1);
 
-        if (contentSizeFlag && !(RAND(seed) & 7)) {
+        if (contentSizeFlag && (fh.contentSize == 0 || !(RAND(seed) & 7))) {
             /* do single segment sometimes */
-            fl.windowSize = fl.contentSize;
+            fh.windowSize = fh.contentSize;
             singleSegment = 1;
         }
     }
 
     if (contentSizeFlag) {
         /* Determine how large fcs field has to be */
-        int minFcsCode = (fl.contentSize >= 256) +
-                               (fl.contentSize >= 65536 + 256) +
-                               (fl.contentSize > 0xFFFFFFFFU);
+        int minFcsCode = (fh.contentSize >= 256) +
+                               (fh.contentSize >= 65536 + 256) +
+                               (fh.contentSize > 0xFFFFFFFFU);
         if (!singleSegment && !minFcsCode) {
             minFcsCode = 1;
         }
         fcsCode = minFcsCode + (RAND(seed) % (4 - minFcsCode));
-        if (fcsCode == 1 && fl.contentSize < 256) fcsCode++;
+        if (fcsCode == 1 && fh.contentSize < 256) fcsCode++;
     }
 
     /* write out the header */
@@ -314,20 +318,20 @@ static void writeFrameHeader(U32* seed, frame_t* frame)
     if (contentSizeFlag) {
         switch (fcsCode) {
         default: /* Impossible */
-        case 0: op[pos++] = fl.contentSize; break;
-        case 1: MEM_writeLE16(op + pos, fl.contentSize - 256); pos += 2; break;
-        case 2: MEM_writeLE32(op + pos, fl.contentSize); pos += 4; break;
-        case 3: MEM_writeLE64(op + pos, fl.contentSize); pos += 8; break;
+        case 0: op[pos++] = fh.contentSize; break;
+        case 1: MEM_writeLE16(op + pos, fh.contentSize - 256); pos += 2; break;
+        case 2: MEM_writeLE32(op + pos, fh.contentSize); pos += 4; break;
+        case 3: MEM_writeLE64(op + pos, fh.contentSize); pos += 8; break;
         }
     }
 
-    DISPLAYLEVEL(2, " frame content size:\t%zu\n", fl.contentSize);
-    DISPLAYLEVEL(2, " frame window size:\t%u\n", fl.windowSize);
+    DISPLAYLEVEL(2, " frame content size:\t%zu\n", fh.contentSize);
+    DISPLAYLEVEL(2, " frame window size:\t%u\n", fh.windowSize);
     DISPLAYLEVEL(2, " content size flag:\t%d\n", contentSizeFlag);
     DISPLAYLEVEL(2, " single segment flag:\t%d\n", singleSegment);
 
     frame->data = op + pos;
-    frame->header = fl;
+    frame->header = fh;
 }
 
 /* Write a literal block in either raw or RLE form, return the literals size */
@@ -1245,6 +1249,8 @@ static int generateFile(U32 seed, const char* const path,
 {
     frame_t fr;
 
+    DISPLAY("seed: %u\n", seed);
+
     generateFrame(seed, &fr);
 
     outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, path);
@@ -1260,11 +1266,6 @@ static int generateCorpus(U32 seed, unsigned numFiles, const char* const path,
     char outPath[MAX_PATH];
     unsigned fnum;
 
-    if (!path) {
-        DISPLAY("Error: valid path is required in multiple files mode\n");
-        return 1;
-    }
-
     DISPLAY("seed: %u\n", seed);
 
     for (fnum = 0; fnum < numFiles; fnum++) {
@@ -1330,7 +1331,15 @@ static void usage(const char* programName)
     DISPLAY( " -t       : activate test mode (test files against libzstd instead of outputting them)\n");
     DISPLAY( " -T#      : length of time to run tests for\n");
     DISPLAY( " -v       : increase verbosity level (default:0, max:7)\n");
-    DISPLAY( " -h       : display help and exit\n");
+    DISPLAY( " -h/H     : display help/long help and exit\n");
+}
+
+static void advancedUsage(const char* programName)
+{
+    usage(programName);
+    DISPLAY( "\n");
+    DISPLAY( "Advanced arguments :\n");
+    DISPLAY( " --content-size    : always include the content size in the frame header\n");
 }
 
 int main(int argc, char** argv)
@@ -1359,6 +1368,9 @@ int main(int argc, char** argv)
                 case 'h':
                     usage(argv[0]);
                     return 0;
+                case 'H':
+                    advancedUsage(argv[0]);
+                    return 0;
                 case 'v':
                     argument++;
                     g_displayLevel++;
@@ -1395,6 +1407,16 @@ int main(int argc, char** argv)
                     argument++;
                     testMode = 1;
                     break;
+                case '-':
+                    argument++;
+                    if (strcmp(argument, "content-size") == 0) {
+                        opts.contentSize = 1;
+                    } else {
+                        advancedUsage(argv[0]);
+                        return 1;
+                    }
+                    argument += strlen(argument);
+                    break;
                 default:
                     usage(argv[0]);
                     return 1;
@@ -1414,11 +1436,15 @@ int main(int argc, char** argv)
         }
     }
 
+    if (!path) {
+        DISPLAY("Error: path is required in file generation mode\n");
+        usage(argv[0]);
+        return 1;
+    }
+
     if (numFiles == 0) {
         return generateFile(seed, path, origPath);
     } else {
         return generateCorpus(seed, numFiles, path, origPath);
     }
-
-    return 0;
 }

From 1d1932480e11190e7ec652236690c460bfee642c Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Thu, 23 Feb 2017 14:34:52 -0800
Subject: [PATCH 146/223] Move educational_decoder to doc/ and add doc README

- Also make some minor bugfixes to educational decoder
---
 doc/README.md                                 | 20 +++++++++++++++++++
 .../educational_decoder/README.md             |  0
 .../educational_decoder/harness.c             |  0
 .../educational_decoder/zstd_decompress.c     |  6 +++---
 .../educational_decoder/zstd_decompress.h     |  0
 5 files changed, 23 insertions(+), 3 deletions(-)
 create mode 100644 doc/README.md
 rename {contrib => doc}/educational_decoder/README.md (100%)
 rename {contrib => doc}/educational_decoder/harness.c (100%)
 rename {contrib => doc}/educational_decoder/zstd_decompress.c (99%)
 rename {contrib => doc}/educational_decoder/zstd_decompress.h (100%)

diff --git a/doc/README.md b/doc/README.md
new file mode 100644
index 000000000..47cfe3617
--- /dev/null
+++ b/doc/README.md
@@ -0,0 +1,20 @@
+Zstandard Documentation
+=======================
+
+This directory contains material defining the Zstandard format,
+as well as for help using the `zstd` library.
+
+__`zstd_compression_format.md`__ : This document defines the Zstandard compression format.
+Compliant decoders must adhere to this document,
+and compliant encoders must generate data that follows it.
+
+__`educational_decoder`__ : This directory contains an implementation of a Zstandard decoder,
+compliant with the Zstandard compression format.
+It can be used, for example, to better understand the format,
+or as the basis for a separate implementation a Zstandard decoder/encoder.
+
+__`zstd_manual.html`__ : Documentation on the functions found in `zstd.h`.
+See [http://zstd.net/zstd_manual.html](http://zstd.net/zstd_manual.html) for
+the manual released with the latest official `zstd` release.
+
+
diff --git a/contrib/educational_decoder/README.md b/doc/educational_decoder/README.md
similarity index 100%
rename from contrib/educational_decoder/README.md
rename to doc/educational_decoder/README.md
diff --git a/contrib/educational_decoder/harness.c b/doc/educational_decoder/harness.c
similarity index 100%
rename from contrib/educational_decoder/harness.c
rename to doc/educational_decoder/harness.c
diff --git a/contrib/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c
similarity index 99%
rename from contrib/educational_decoder/zstd_decompress.c
rename to doc/educational_decoder/zstd_decompress.c
index 856255987..ae4eaa81c 100644
--- a/contrib/educational_decoder/zstd_decompress.c
+++ b/doc/educational_decoder/zstd_decompress.c
@@ -799,7 +799,7 @@ static size_t decode_literals_simple(istream_t *const in, u8 **const literals,
     case 2:
         // "Size_Format uses 1 bit. Regenerated_Size uses 5 bits (0-31)."
         IO_rewind_bits(in, 1);
-        size = IO_read_bits(in, 2);
+        size = IO_read_bits(in, 5);
         break;
     case 1:
         // "Size_Format uses 2 bits. Regenerated_Size uses 12 bits (0-4095)."
@@ -881,7 +881,7 @@ static size_t decode_literals_compressed(frame_context_t *const ctx,
         IMPOSSIBLE();
     }
     if (regenerated_size > MAX_LITERALS_SIZE ||
-        compressed_size > regenerated_size) {
+        compressed_size >= regenerated_size) {
         CORRUPTION();
     }
 
@@ -1654,7 +1654,7 @@ static inline const u8 *IO_read_bytes(istream_t *const in, size_t len) {
 /// Returns a pointer to write `len` bytes to, and advances the internal state
 static inline u8 *IO_write_bytes(ostream_t *const out, size_t len) {
     if (len > out->len) {
-        INP_SIZE();
+        OUT_SIZE();
     }
     u8 *const ptr = out->ptr;
     out->ptr += len;
diff --git a/contrib/educational_decoder/zstd_decompress.h b/doc/educational_decoder/zstd_decompress.h
similarity index 100%
rename from contrib/educational_decoder/zstd_decompress.h
rename to doc/educational_decoder/zstd_decompress.h

From d590291d869e8c51b3879fd3415b97e91aa37f11 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Thu, 23 Feb 2017 15:53:44 -0800
Subject: [PATCH 147/223] Fix -Wsign-compare issues in decodecorpus.c

https://travis-ci.org/facebook/zstd/jobs/204423280
---
 tests/decodecorpus.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index df12dd56d..d75025a83 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -661,7 +661,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
                 /* do a normal offset */
                 offset = (RAND(seed) %
                           MIN(frame->header.windowSize,
-                              (BYTE*)srcPtr - (BYTE*)frame->srcStart)) +
+                              (size_t)((BYTE*)srcPtr - (BYTE*)frame->srcStart))) +
                          1;
                 offsetCode = offset + ZSTD_REP_MOVE;
                 repIndex = 2;
@@ -678,7 +678,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
                     repIndex = MIN(2, offsetCode + 1);
                 }
             }
-        } while (offset > (BYTE*)srcPtr - (BYTE*)frame->srcStart || offset == 0);
+        } while (offset > (size_t)((BYTE*)srcPtr - (BYTE*)frame->srcStart) || offset == 0);
 
         {   size_t j;
             for (j = 0; j < matchLen; j++) {
@@ -695,7 +695,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
         }
 
         DISPLAYLEVEL(6, "      LL: %5u OF: %5u ML: %5u", literalLen, offset, matchLen);
-        DISPLAYLEVEL(7, " srcPos: %8zu seqNb: %3u",
+        DISPLAYLEVEL(7, " srcPos: %8tu seqNb: %3u",
                      (BYTE*)srcPtr - (BYTE*)frame->srcStart, i);
         DISPLAYLEVEL(6, "\n");
         if (offsetCode < 3) {
@@ -713,7 +713,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
     memcpy(srcPtr, literals, literalsSize);
     srcPtr += literalsSize;
     DISPLAYLEVEL(6, "      excess literals: %5zu", literalsSize);
-    DISPLAYLEVEL(7, " srcPos: %8zu", (BYTE*)srcPtr - (BYTE*)frame->srcStart);
+    DISPLAYLEVEL(7, " srcPos: %8tu", (BYTE*)srcPtr - (BYTE*)frame->srcStart);
     DISPLAYLEVEL(6, "\n");
 
     return numSequences;

From 831b4890ce7db8c3771e2ef9bfc2b5c929b5ac2b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 23 Feb 2017 23:09:10 -0800
Subject: [PATCH 148/223] minor tests/Makefile refactoring

and update of zstd_manual,html
---
 doc/zstd_manual.html           | 13 ++++++++-----
 lib/compress/zstdmt_compress.c | 10 +++++-----
 lib/zstd.h                     |  2 +-
 tests/.gitignore               |  2 ++
 tests/Makefile                 | 34 ++++++++++++++++++++++++++--------
 5 files changed, 42 insertions(+), 19 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 23224d77a..02656c230 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -34,7 +34,7 @@
   zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
   at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and
   decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22.
-  Levels >= 20, labelled `--ultra`, should be used with caution, as they require more memory.
+  Levels >= 20, labeled `--ultra`, should be used with caution, as they require more memory.
   Compression can be done in:
     - a single step (described as Simple API)
     - a single step, reusing a context (described as Explicit memory management)
@@ -306,8 +306,8 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 


 
Compressed size functions

 
-
size_t ZSTD_getFrameCompressedSize(const void* src, size_t srcSize);
-
  `src` should point to the start of a ZSTD encoded frame
+
size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
+
  `src` should point to the start of a ZSTD encoded frame or skippable frame
   `srcSize` must be at least as large as the frame
   @return : the compressed size of the frame pointed to by `src`, suitable to pass to
       `ZSTD_decompress` or similar, or an error code if given invalid input. 
@@ -321,7 +321,7 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
        to `ZSTD_frameHeaderSize_max` is guaranteed to be large enough in all cases.
    @return : decompressed size of the frame pointed to be `src` if known, otherwise
              - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
-             - ZSTD_CONTENTSIZE_ERROR if an error occured (e.g. invalid magic number, srcSize too small) 
+             - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) 
 


 
 
unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
@@ -365,7 +365,7 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 


 
 
typedef enum {
-    ZSTD_p_forceWindow   /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0)*/
+    ZSTD_p_forceWindow   /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */
 } ZSTD_CCtxParameter;
 


 
size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value);
@@ -585,6 +585,9 @@ size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const vo
   c) Frame Content - any content (User Data) of length equal to Frame Size
   For skippable frames ZSTD_decompressContinue() always returns 0.
   For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
+    Note : If fparamsPtr->frameContentSize==0, it is ambiguous: the frame might actually be a Zstd encoded frame with no content.
+           For purposes of decompression, it is valid in both cases to skip the frame using
+           ZSTD_findFrameCompressedSize to find its size in bytes.
   It also returns Frame Size as fparamsPtr->frameContentSize.
 

 
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 97d2b38ee..f32c334a9 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -231,16 +231,16 @@ void ZSTDMT_compressChunk(void* jobDescription)
     const void* const src = (const char*)job->srcStart + job->dictSize;
     buffer_t const dstBuff = job->dstBuff;
     DEBUGLOG(3, "job (first:%u) (last:%u) : dictSize %u, srcSize %u", job->firstChunk, job->lastChunk, (U32)job->dictSize, (U32)job->srcSize);
-    if (job->cdict) {
+    if (job->cdict) {  /* should only happen for first segment */
         size_t const initError = ZSTD_compressBegin_usingCDict(job->cctx, job->cdict, job->fullFrameSize);
         if (job->cdict) DEBUGLOG(3, "using CDict ");
         if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
-    } else {
-        size_t const initError = ZSTD_compressBegin_advanced(job->cctx, job->srcStart, job->dictSize, job->params, job->fullFrameSize);
+    } else {  /* srcStart points at reloaded section */
+        size_t const initError = ZSTD_compressBegin_advanced(job->cctx, job->srcStart, job->dictSize, job->params, 0);
         if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
         ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceWindow, 1);
     }
-    if (!job->firstChunk) {  /* flush frame header */
+    if (!job->firstChunk) {  /* flush and overwrite frame header when it's not first segment */
         size_t const hSize = ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, src, 0);
         if (ZSTD_isError(hSize)) { job->cSize = hSize; goto _endJob; }
         ZSTD_invalidateRepCodes(job->cctx);
@@ -248,7 +248,7 @@ void ZSTDMT_compressChunk(void* jobDescription)
 
     DEBUGLOG(4, "Compressing : ");
     DEBUG_PRINTHEX(4, job->srcStart, 12);
-    job->cSize = (job->lastChunk) ?   /* last chunk signal */
+    job->cSize = (job->lastChunk) ?
                  ZSTD_compressEnd     (job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize) :
                  ZSTD_compressContinue(job->cctx, dstBuff.start, dstBuff.size, src, job->srcSize);
     DEBUGLOG(3, "compressed %u bytes into %u bytes   (first:%u) (last:%u)", (unsigned)job->srcSize, (unsigned)job->cSize, job->firstChunk, job->lastChunk);
diff --git a/lib/zstd.h b/lib/zstd.h
index e597c5db5..f35805447 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -462,7 +462,7 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
 ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
 
 typedef enum {
-    ZSTD_p_forceWindow   /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0)*/
+    ZSTD_p_forceWindow   /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */
 } ZSTD_CCtxParameter;
 /*! ZSTD_setCCtxParameter() :
  *  Set advanced parameters, selected through enum ZSTD_CCtxParameter
diff --git a/tests/.gitignore b/tests/.gitignore
index dc468dee4..f408a7491 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -9,6 +9,8 @@ zbufftest32
 zbufftest-dll
 zstreamtest
 zstreamtest32
+zstreamtest_asan
+zstreamtest_tsan
 zstreamtest-dll
 datagen
 paramgrill
diff --git a/tests/Makefile b/tests/Makefile
index 5b0e29c67..4fae769d3 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -35,22 +35,28 @@ FLAGS    = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
 
 
 ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
-ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c
+ZSTDCOMP_FILES   := $(ZSTDDIR)/compress/*.c
 ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c
-ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
+ZSTD_FILES  := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
 ZBUFF_FILES := $(ZSTDDIR)/deprecated/*.c
 ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
 
+ZSTD_OBJ  := $(patsubst %.c,%.o, $(wildcard $(ZSTD_FILES)) )
+ZBUFF_OBJ := $(patsubst %.c,%.o, $(wildcard $(ZBUFF_FILES)) )
+ZDICT_OBJ := $(patsubst %.c,%.o, $(wildcard $(ZDICT_FILES)) )
 
 
 # Define *.exe as extension for Windows systems
 ifneq (,$(filter Windows%,$(OS)))
 EXT =.exe
-MULTITHREAD = -DZSTD_MULTITHREAD
+MULTITHREAD_CPP = -DZSTD_MULTITHREAD
+MULTITHREAD_LD  =
 else
 EXT =
-MULTITHREAD = -pthread -DZSTD_MULTITHREAD
+MULTITHREAD_CPP = -DZSTD_MULTITHREAD
+MULTITHREAD_LD  = -lpthread
 endif
+MULTITHREAD = $(MULTITHREAD_CPP) $(MULTITHREAD_LD)
 
 VOID = /dev/null
 ZSTREAM_TESTTIME = -T2mn
@@ -124,11 +130,23 @@ zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zbufftest.c
 	$(MAKE) -C $(ZSTDDIR) libzstd
 	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@$(EXT)
 
-zstreamtest  : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c zstreamtest.c
-	$(CC)      $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
+ZSTREAMFILES := $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c zstreamtest.c
+zstreamtest : CPPFLAGS += $(MULTITHREAD_CPP)
+zstreamtest : LDFLAGS += $(MULTITHREAD_LD)
+zstreamtest : $(ZSTREAMFILES)
+	$(CC) $(FLAGS) $^ -o $@$(EXT)
 
-zstreamtest32 : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c zstreamtest.c
-	$(CC) -m32  $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
+zstreamtest32 : CFLAGS += -m32
+zstreamtest32 : $(ZSTREAMFILES)
+	$(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
+
+zstreamtest_asan : CFLAGS += -fsanitize=address
+zstreamtest_asan : $(ZSTREAMFILES)
+	$(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
+
+zstreamtest_tsan : CFLAGS += -fsanitize=thread
+zstreamtest_tsan : $(ZSTREAMFILES)
+	$(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT)
 
 zstreamtest-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd
 zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/datagen.c zstreamtest.c

From b68ea5d87b39269cca3c5941831d680849bd5907 Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Fri, 24 Feb 2017 08:18:44 +0100
Subject: [PATCH 149/223] rearrange Travis tests

---
 .travis.yml | 36 ++++++++++++++++++++----------------
 1 file changed, 20 insertions(+), 16 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index b20c43329..3c77a17e1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,31 +4,35 @@ dist: trusty
 matrix:
   fast_finish: true
   include:
+    # other feature branches => short tests
+    - env: Cmd="make libc6install && make -C tests test32"
+    - env: Cmd='make valgrindinstall arminstall ppcinstall arm-ppc-compilation && make clean lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest'
+
+    - env: Cmd='CC=gcc-6 make gcc6install uasan-test'
+    - env: Cmd='CC=gcc-6 make gcc6install uasan-test32'
+    - env: Cmd="make arminstall armtest && make clean && make aarch64test"
+    - env: Cmd='make ppcinstall ppctest && make clean && make ppc64test'
+    - env: Cmd='make gpp6install zlibwrapper && make -C tests clean test-zstd-nolegacy && make -C tests versionsTest && make clean && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan'
+      install:
+        - export CXX="g++-6" CC="gcc-6"
+
     # OS X Mavericks
     - env: Cmd="make gnu90test && make clean && make test && make clean && make travis-install"
       os: osx
 
-    # Ubuntu 14.04 LTS Server Edition 64 bit
-    - env: Cmd='make gpp6install uasan-test && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan'
-      install:
-        - export CXX="g++-6" CC="gcc-6"
-    - env: Cmd='CC=gcc-6 make gcc6install uasan-test32 && make clean zlibwrapper && make -C tests clean test-zstd-nolegacy && make -C tests versionsTest'
-    - env: Cmd="make arminstall armtest && make clean && make aarch64test"
-    - env: Cmd='make ppcinstall ppctest && make clean && make ppc64test'
-
-    # other feature branches => short tests
-    - env: Cmd='make valgrindinstall arminstall ppcinstall arm-ppc-compilation && make clean lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest'
-    - env: Cmd="make libc6install && make -C tests test32"
-
 script:
   - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:')
-  #  cron & master          => long tests, as this is the final step towards a Release
-  #  dev && pull requests   => normal tests
-  #  other feature branches => short tests (number > 5)
+  #  cron & master          => full tests, as this is the final step towards a Release
+  #  pull requests          => normal tests (job numbers 1-3)
+  #  other feature branches => short tests (job numbers 1-2)
   - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "master" ]; then
         FUZZERTEST=-T7mn sh -c "$Cmd" || travis_terminate 1;
     else
-        if [ "$TRAVIS_PULL_REQUEST" = "true" ] || [ $JOB_NUMBER -gt 5 ] || [ "$TRAVIS_BRANCH" = "dev" ]; then
+        if [ "$TRAVIS_PULL_REQUEST" == "true" ] && [ $JOB_NUMBER -lt 4 ]; then
             sh -c "$Cmd" || travis_terminate 1;
+        else
+            if [ $JOB_NUMBER -lt 3 ]; then
+                sh -c "$Cmd" || travis_terminate 1;
+            fi
         fi
     fi

From 14312d833e1249de7bb0caa916d83993d2fa33be Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 23 Feb 2017 23:42:12 -0800
Subject: [PATCH 150/223] zstdmt : fix : loading prefix from previous segments

There used to be a (very small) chance that
loading prefix from previous segment
would be confused with a real zstd dictionary.
For that to happen, the prefix needs to start
with the same value as dictionary magic.
That's 1 chance in 4 billions if all values have equal probability.
But in fact, since some values are more common (0x00000000 for example)
others are less common, and dictionary magic was selected to be one of them,
so probabilities are likely even lower.

Anyway, this risk is no down to zero
by adding a new CCtx parameter : ZSTD_p_forceRawDict

Current parameter policy : the parameter "stick" to its CCtx,
so any dictionary loading after ZSTD_p_forceRawDict is set
will be loaded in "raw" ("content only") mode,
even if CCtx is re-used multiple times with multiple different dictionary.
It's up to the user to reset this value differently if it needs so.
---
 doc/zstd_manual.html           | 3 ++-
 lib/compress/zstd_compress.c   | 7 +++++--
 lib/compress/zstdmt_compress.c | 3 ++-
 lib/zstd.h                     | 3 ++-
 4 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 02656c230..77e8974de 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -365,7 +365,8 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 


 
 
typedef enum {
-    ZSTD_p_forceWindow   /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */
+    ZSTD_p_forceWindow,   /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */
+    ZSTD_p_forceRawDict   /* Force loading dictionary in "content-only" mode (no header analysis) */
 } ZSTD_CCtxParameter;
 


 
size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value);
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 0e0f9d373..d684e6a0d 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -62,6 +62,7 @@ struct ZSTD_CCtx_s {
     U32   hashLog3;         /* dispatch table : larger == faster, more memory */
     U32   loadedDictEnd;    /* index of end of dictionary */
     U32   forceWindow;      /* force back-references to respect limit of 1<forceWindow = value>0; cctx->loadedDictEnd = 0; return 0;
+    case ZSTD_p_forceRawDict : cctx->forceRawDict = value>0; return 0;
     default: return ERROR(parameter_unknown);
     }
 }
@@ -2613,8 +2615,9 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, si
 {
     if ((dict==NULL) || (dictSize<=8)) return 0;
 
-    /* default : dict is pure content */
-    if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return ZSTD_loadDictionaryContent(zc, dict, dictSize);
+    /* dict as pure content */
+    if ((MEM_readLE32(dict) != ZSTD_DICT_MAGIC) || (zc->forceRawDict))
+        return ZSTD_loadDictionaryContent(zc, dict, dictSize);
     zc->dictID = zc->params.fParams.noDictIDFlag ? 0 :  MEM_readLE32((const char*)dict+4);
 
     /* known magic number : dict is parsed for entropy stats and content */
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index f32c334a9..483ea157e 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -236,8 +236,9 @@ void ZSTDMT_compressChunk(void* jobDescription)
         if (job->cdict) DEBUGLOG(3, "using CDict ");
         if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
     } else {  /* srcStart points at reloaded section */
+        size_t const dictModeError = ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceRawDict, 1);  /* Force loading dictionary in "content-only" mode (no header analysis) */
         size_t const initError = ZSTD_compressBegin_advanced(job->cctx, job->srcStart, job->dictSize, job->params, 0);
-        if (ZSTD_isError(initError)) { job->cSize = initError; goto _endJob; }
+        if (ZSTD_isError(initError) || ZSTD_isError(dictModeError)) { job->cSize = initError; goto _endJob; }
         ZSTD_setCCtxParameter(job->cctx, ZSTD_p_forceWindow, 1);
     }
     if (!job->firstChunk) {  /* flush and overwrite frame header when it's not first segment */
diff --git a/lib/zstd.h b/lib/zstd.h
index f35805447..7cca36909 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -462,7 +462,8 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
 ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
 
 typedef enum {
-    ZSTD_p_forceWindow   /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */
+    ZSTD_p_forceWindow,   /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */
+    ZSTD_p_forceRawDict   /* Force loading dictionary in "content-only" mode (no header analysis) */
 } ZSTD_CCtxParameter;
 /*! ZSTD_setCCtxParameter() :
  *  Set advanced parameters, selected through enum ZSTD_CCtxParameter

From df9f9296e3b1eaed41cc0bbc4cf0e5b935d321ed Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Fri, 24 Feb 2017 00:16:05 -0800
Subject: [PATCH 151/223] attempt to fix pthreat linking error

replacing -lpthread by -pthread
---
 tests/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/Makefile b/tests/Makefile
index 4fae769d3..30b2a04a3 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -54,7 +54,7 @@ MULTITHREAD_LD  =
 else
 EXT =
 MULTITHREAD_CPP = -DZSTD_MULTITHREAD
-MULTITHREAD_LD  = -lpthread
+MULTITHREAD_LD  = -pthread
 endif
 MULTITHREAD = $(MULTITHREAD_CPP) $(MULTITHREAD_LD)
 

From 8740d6bcf54671acbcb94db93f8643a9dcdf5aa3 Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Fri, 24 Feb 2017 09:24:55 +0100
Subject: [PATCH 152/223] fix uninitialized value warning

---
 tests/decodecorpus.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index d75025a83..8de5c252b 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -609,7 +609,7 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
 {
     /* The total length of all the matches */
     size_t const remainingMatch = contentSize - literalsSize;
-    size_t excessMatch;
+    size_t excessMatch = 0;
     U32 i;
 
     U32 numSequences;

From a66b764d79e59e48f6e2f06f759e0f87de5b9653 Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Fri, 24 Feb 2017 16:09:17 +0100
Subject: [PATCH 153/223] added tests for gzip

---
 tests/gzip/Makefile               |  42 ++
 tests/gzip/gzip-env.sh            |  43 +++
 tests/gzip/helin-segv.sh          |  31 ++
 tests/gzip/help-version.sh        | 270 +++++++++++++
 tests/gzip/hufts-segv.gz          | Bin 0 -> 425 bytes
 tests/gzip/hufts.sh               |  34 ++
 tests/gzip/init.cfg               |   5 +
 tests/gzip/init.sh                | 616 ++++++++++++++++++++++++++++++
 tests/gzip/keep.sh                |  51 +++
 tests/gzip/list.sh                |  31 ++
 tests/gzip/list2.log              |   7 +
 tests/gzip/memcpy-abuse.sh        |  34 ++
 tests/gzip/mixed.sh               |  68 ++++
 tests/gzip/null-suffix-clobber.sh |  35 ++
 tests/gzip/stdin.sh               |  31 ++
 tests/gzip/test-driver.sh         | 150 ++++++++
 tests/gzip/trailing-nul.sh        |  37 ++
 tests/gzip/unpack-invalid.sh      |  36 ++
 tests/gzip/z-suffix.sh            |  30 ++
 tests/gzip/zdiff.sh               |  48 +++
 tests/gzip/zgrep-context.sh       |  47 +++
 tests/gzip/zgrep-f.sh             |  43 +++
 tests/gzip/zgrep-signal.sh        |  64 ++++
 tests/gzip/znew-k.sh              |  40 ++
 24 files changed, 1793 insertions(+)
 create mode 100644 tests/gzip/Makefile
 create mode 100755 tests/gzip/gzip-env.sh
 create mode 100644 tests/gzip/helin-segv.sh
 create mode 100644 tests/gzip/help-version.sh
 create mode 100644 tests/gzip/hufts-segv.gz
 create mode 100644 tests/gzip/hufts.sh
 create mode 100644 tests/gzip/init.cfg
 create mode 100644 tests/gzip/init.sh
 create mode 100644 tests/gzip/keep.sh
 create mode 100644 tests/gzip/list.sh
 create mode 100644 tests/gzip/list2.log
 create mode 100644 tests/gzip/memcpy-abuse.sh
 create mode 100644 tests/gzip/mixed.sh
 create mode 100644 tests/gzip/null-suffix-clobber.sh
 create mode 100644 tests/gzip/stdin.sh
 create mode 100644 tests/gzip/test-driver.sh
 create mode 100644 tests/gzip/trailing-nul.sh
 create mode 100644 tests/gzip/unpack-invalid.sh
 create mode 100644 tests/gzip/z-suffix.sh
 create mode 100644 tests/gzip/zdiff.sh
 create mode 100644 tests/gzip/zgrep-context.sh
 create mode 100644 tests/gzip/zgrep-f.sh
 create mode 100644 tests/gzip/zgrep-signal.sh
 create mode 100644 tests/gzip/znew-k.sh

diff --git a/tests/gzip/Makefile b/tests/gzip/Makefile
new file mode 100644
index 000000000..5d5480444
--- /dev/null
+++ b/tests/gzip/Makefile
@@ -0,0 +1,42 @@
+# ################################################################
+# Copyright (c) 2017-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+# ################################################################
+
+PRGDIR = ../../programs
+VOID   = /dev/null
+
+
+.PHONY: all
+all: test-gzip-env test-helin-segv test-hufts test-keep test-list test-memcpy-abuse test-mixed test-null-suffix-clobber test-stdin test-trailing-nul test-unpack-invalid test-zdiff test-zgrep-context test-zgrep-f test-zgrep-signal test-znew-k test-z-suffix 
+	@echo Testing completed
+
+.PHONY: zstd
+zstd:
+	$(MAKE) -C $(PRGDIR) zstd
+	#alias gzip='$(PRGDIR)/zstd --format=gzip'
+	#ln -sf /drive_d/GitHub/zstd/programs/zstd /usr/local/bin/gzip
+	gzip --version
+
+.PHONY: clean
+clean:
+	@$(MAKE) -C $(PRGDIR) $@ > $(VOID)
+	@$(RM) *.trs *.log
+	@echo Cleaning completed
+
+
+#------------------------------------------------------------------------------
+# validated only for Linux, OSX, Hurd and some BSD targets
+#------------------------------------------------------------------------------
+ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD))
+#.PHONY: test-gzip-env test-helin-segv test-hufts test-keep test-list test-memcpy-abuse test-mixed test-null-suffix-clobber test-stdin test-trailing-nul test-unpack-invalid test-zdiff test-zgrep-context test-zgrep-f test-zgrep-signal test-znew-k test-z-suffix 
+
+test-%: zstd
+	@./test-driver.sh --test-name $* --log-file $*.log --trs-file $*.trs --expect-failure "no" --color-tests "yes" --enable-hard-errors "yes" ./$* || echo error
+#|| exit 1
+
+endif
diff --git a/tests/gzip/gzip-env.sh b/tests/gzip/gzip-env.sh
new file mode 100755
index 000000000..873d716dc
--- /dev/null
+++ b/tests/gzip/gzip-env.sh
@@ -0,0 +1,43 @@
+#!/bin/sh
+# Test the obsolescent GZIP environment variable.
+
+# Copyright 2015-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ..
+
+echo a >exp || framework_failure_
+gzip in || framework_failure_
+
+fail=0
+GZIP=-qv gzip -d out 2>err || fail=1
+compare exp out || fail=1
+
+for badopt in -- -c --stdout -d --decompress -f --force -h --help -k --keep \
+  -l --list -L --license -r --recursive -Sxxx --suffix=xxx '--suffix xxx' \
+  -t --test -V --version
+do
+  GZIP=$badopt gzip -d out 2>err && fail=1
+done
+
+for goodopt in -n --no-name -N --name -q --quiet -v --verbose \
+  -1 --fast -2 -3 -4 -5 -6 -7 -8 -9 --best
+do
+  GZIP=$goodopt gzip -d out 2>err || fail=1
+  compare exp out || fail=1
+done
+
+Exit $fail
diff --git a/tests/gzip/helin-segv.sh b/tests/gzip/helin-segv.sh
new file mode 100644
index 000000000..0182db33d
--- /dev/null
+++ b/tests/gzip/helin-segv.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+# Before gzip-1.4, gzip -d would segfault on some inputs.
+
+# Copyright (C) 2010-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ..
+
+# This test case was provided by Aki Helin.
+printf '\037\235\220\0\0\0\304' > helin.gz || framework_failure_
+printf '\0\0' > exp || framework_failure_
+
+fail=0
+
+gzip -dc helin.gz > out || fail=1
+compare exp out || fail=1
+
+Exit $fail
diff --git a/tests/gzip/help-version.sh b/tests/gzip/help-version.sh
new file mode 100644
index 000000000..5af9a0509
--- /dev/null
+++ b/tests/gzip/help-version.sh
@@ -0,0 +1,270 @@
+#! /bin/sh
+# Make sure all these programs work properly
+# when invoked with --help or --version.
+
+# Copyright (C) 2000-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+
+# Ensure that $SHELL is set to *some* value and exported.
+# This is required for dircolors, which would fail e.g., when
+# invoked via debuild (which removes SHELL from the environment).
+test "x$SHELL" = x && SHELL=/bin/sh
+export SHELL
+
+. "${srcdir=.}/init.sh"; path_prepend_ ..
+
+expected_failure_status_chroot=125
+expected_failure_status_env=125
+expected_failure_status_nice=125
+expected_failure_status_nohup=125
+expected_failure_status_stdbuf=125
+expected_failure_status_su=125
+expected_failure_status_timeout=125
+expected_failure_status_printenv=2
+expected_failure_status_tty=3
+expected_failure_status_sort=2
+expected_failure_status_expr=3
+expected_failure_status_lbracket=2
+expected_failure_status_dir=2
+expected_failure_status_ls=2
+expected_failure_status_vdir=2
+
+expected_failure_status_cmp=2
+expected_failure_status_zcmp=2
+expected_failure_status_sdiff=2
+expected_failure_status_diff3=2
+expected_failure_status_diff=2
+expected_failure_status_zdiff=2
+expected_failure_status_zgrep=2
+expected_failure_status_zegrep=2
+expected_failure_status_zfgrep=2
+
+expected_failure_status_grep=2
+expected_failure_status_egrep=2
+expected_failure_status_fgrep=2
+
+test "$built_programs" \
+  || fail_ "built_programs not specified!?!"
+
+test "$VERSION" \
+  || fail_ "set envvar VERSION; it is required for a PATH sanity-check"
+
+# Extract version from --version output of the first program
+for i in $built_programs; do
+  v=$(env $i --version | sed -n '1s/.* //p;q')
+  break
+done
+
+# Ensure that it matches $VERSION.
+test "x$v" = "x$VERSION" \
+  || fail_ "--version-\$VERSION mismatch"
+
+for lang in C fr da; do
+  for i in $built_programs; do
+
+    # Skip `test'; it doesn't accept --help or --version.
+    test $i = test && continue;
+
+    # false fails even when invoked with --help or --version.
+    if test $i = false; then
+      env LC_MESSAGES=$lang $i --help >/dev/null && fail=1
+      env LC_MESSAGES=$lang $i --version >/dev/null && fail=1
+      continue
+    fi
+
+    args=
+
+    # The just-built install executable is always named `ginstall'.
+    test $i = install && i=ginstall
+
+    # Make sure they exit successfully, under normal conditions.
+    eval "env \$i $args --help    > h-\$i   " || fail=1
+    eval "env \$i $args --version >/dev/null" || fail=1
+
+    # Make sure they mention the bug-reporting address in --help output.
+    grep "$PACKAGE_BUGREPORT" h-$i > /dev/null || fail=1
+    rm -f h-$i
+
+    # Make sure they fail upon `disk full' error.
+    if test -w /dev/full && test -c /dev/full; then
+      eval "env \$i $args --help    >/dev/full 2>/dev/null" && fail=1
+      eval "env \$i $args --version >/dev/full 2>/dev/null" && fail=1
+      status=$?
+      test $i = [ && prog=lbracket || prog=$i
+      eval "expected=\$expected_failure_status_$prog"
+      test x$expected = x && expected=1
+      if test $status = $expected; then
+        : # ok
+      else
+        fail=1
+        echo "*** $i: bad exit status \`$status' (expected $expected)," 1>&2
+        echo "  with --help or --version output redirected to /dev/full" 1>&2
+      fi
+    fi
+  done
+done
+
+bigZ_in=bigZ-in.Z
+zin=zin.gz
+zin2=zin2.gz
+
+tmp=tmp-$$
+tmp_in=in-$$
+tmp_in2=in2-$$
+tmp_dir=dir-$$
+tmp_out=out-$$
+mkdir $tmp || fail=1
+cd $tmp || fail=1
+
+comm_setup () { args="$tmp_in $tmp_in"; }
+csplit_setup () { args="$tmp_in //"; }
+cut_setup () { args='-f 1'; }
+join_setup () { args="$tmp_in $tmp_in"; }
+tr_setup () { args='a a'; }
+
+chmod_setup () { args="a+x $tmp_in"; }
+# Punt on these.
+chgrp_setup () { args=--version; }
+chown_setup () { args=--version; }
+mkfifo_setup () { args=--version; }
+mknod_setup () { args=--version; }
+# Punt on uptime, since it fails (e.g., failing to get boot time)
+# on some systems, and we shouldn't let that stop `make check'.
+uptime_setup () { args=--version; }
+
+# Create a file in the current directory, not in $TMPDIR.
+mktemp_setup () { args=mktemp.XXXX; }
+
+cmp_setup () { args="$tmp_in $tmp_in2"; }
+
+# Tell dd not to print the line with transfer rate and total.
+# The transfer rate would vary between runs.
+dd_setup () { args=status=noxfer; }
+
+zdiff_setup () { args="$args $zin $zin2"; }
+zcmp_setup () { zdiff_setup; }
+zcat_setup () { args="$args $zin"; }
+gunzip_setup () { zcat_setup; }
+zmore_setup () { zcat_setup; }
+zless_setup () { zcat_setup; }
+znew_setup () { args="$args $bigZ_in"; }
+zforce_setup () { zcat_setup; }
+zgrep_setup () { args="$args z $zin"; }
+zegrep_setup () { zgrep_setup; }
+zfgrep_setup () { zgrep_setup; }
+gzexe_setup () { args="$args $tmp_in"; }
+
+# We know that $tmp_in contains a "0"
+grep_setup () { args="0 $tmp_in"; }
+egrep_setup () { args="0 $tmp_in"; }
+fgrep_setup () { args="0 $tmp_in"; }
+
+diff_setup () { args="$tmp_in $tmp_in2"; }
+sdiff_setup () { args="$tmp_in $tmp_in2"; }
+diff3_setup () { args="$tmp_in $tmp_in2 $tmp_in2"; }
+cp_setup () { args="$tmp_in $tmp_in2"; }
+ln_setup () { args="$tmp_in ln-target"; }
+ginstall_setup () { args="$tmp_in $tmp_in2"; }
+mv_setup () { args="$tmp_in $tmp_in2"; }
+mkdir_setup () { args=$tmp_dir/subdir; }
+rmdir_setup () { args=$tmp_dir; }
+rm_setup () { args=$tmp_in; }
+shred_setup () { args=$tmp_in; }
+touch_setup () { args=$tmp_in2; }
+truncate_setup () { args="--reference=$tmp_in $tmp_in2"; }
+
+basename_setup () { args=$tmp_in; }
+dirname_setup () { args=$tmp_in; }
+expr_setup () { args=foo; }
+
+# Punt, in case GNU `id' hasn't been installed yet.
+groups_setup () { args=--version; }
+
+pathchk_setup () { args=$tmp_in; }
+yes_setup () { args=--version; }
+logname_setup () { args=--version; }
+nohup_setup () { args=--version; }
+printf_setup () { args=foo; }
+seq_setup () { args=10; }
+sleep_setup () { args=0; }
+su_setup () { args=--version; }
+stdbuf_setup () { args="-oL true"; }
+timeout_setup () { args=--version; }
+
+# I'd rather not run sync, since it spins up disks that I've
+# deliberately caused to spin down (but not unmounted).
+sync_setup () { args=--version; }
+
+test_setup () { args=foo; }
+
+# This is necessary in the unusual event that there is
+# no valid entry in /etc/mtab.
+df_setup () { args=/; }
+
+# This is necessary in the unusual event that getpwuid (getuid ()) fails.
+id_setup () { args=-u; }
+
+# Use env to avoid invoking built-in sleep of Solaris 11's /bin/sh.
+kill_setup () {
+  env sleep 10m &
+  args=$!
+}
+
+link_setup () { args="$tmp_in link-target"; }
+unlink_setup () { args=$tmp_in; }
+
+readlink_setup () {
+  ln -s . slink
+  args=slink;
+}
+
+stat_setup () { args=$tmp_in; }
+unlink_setup () { args=$tmp_in; }
+lbracket_setup () { args=": ]"; }
+
+# Ensure that each program "works" (exits successfully) when doing
+# something more than --help or --version.
+for i in $built_programs; do
+  # Skip these.
+  case $i in chroot|stty|tty|false|chcon|runcon) continue;; esac
+
+  rm -rf $tmp_in $tmp_in2 $tmp_dir $tmp_out $bigZ_in $zin $zin2
+  echo z |gzip > $zin
+  cp $zin $zin2
+  cp $zin $bigZ_in
+
+  # This is sort of kludgey: use numbers so this is valid input for factor,
+  # and two tokens so it's valid input for tsort.
+  echo 2147483647 0 > $tmp_in
+  # Make $tmp_in2 identical. Then, using $tmp_in and $tmp_in2 as arguments
+  # to the likes of cmp and diff makes them exit successfully.
+  cp $tmp_in $tmp_in2
+  mkdir $tmp_dir
+  # echo ================== $i
+  test $i = [ && prog=lbracket || prog=$i
+  args=
+  if type ${prog}_setup > /dev/null 2>&1; then
+    ${prog}_setup
+  fi
+  if eval "env \$i $args < \$tmp_in > \$tmp_out"; then
+    : # ok
+  else
+    echo FAIL: $i
+    fail=1
+  fi
+  rm -rf $tmp_in $tmp_in2 $tmp_out $tmp_dir
+done
+
+Exit $fail
diff --git a/tests/gzip/hufts-segv.gz b/tests/gzip/hufts-segv.gz
new file mode 100644
index 0000000000000000000000000000000000000000..32cb2a256844358eca0b5e78e49b96a60724ade5
GIT binary patch
literal 425
zcmb2|=HN(gbM#_zR;$YXIAQJoYKNGs>;`^wWjPwSWK0DAIIeK^>}X`{^m_Ya0lV|&
zYimqIc4~0C7(C836k?vzyhFe|)qL~p+izbn`mWSkaaW*c&z7At??oT@)KUKPT2%d<
z)EQeIf2?}{eeb`jw~6!j-kPPne(twf)$402U+*yJv5h)^x!{$(?)QfF&0Aw-49rzN
z9KMz(+Q^%z{Gk5xGO4HR1^2}Jlecnei(K%M*zoA*!T#x@D@1ZKgufMBd9~s*hvm7?
z`*wDM|Nn}gdTuNE
z^74Dh8h6m?oxxNi?z#yyDK&d-#*~%E5FWbAKwJdFIk)y
zvKJc%RKDu4tN+2PSsDND>xbsH`wUBU8CayepVn=ERlJ?OuKvT>L#r0QX|u3uF?!>$
n&ymZie2wg%?WY1i+b{Sb`0)3Qt-;frrv&UW;`;Jh?Swi2;pf&!

literal 0
HcmV?d00001

diff --git a/tests/gzip/hufts.sh b/tests/gzip/hufts.sh
new file mode 100644
index 000000000..5832a2184
--- /dev/null
+++ b/tests/gzip/hufts.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+# Exercise a bug whereby an invalid input could make gzip -d misbehave.
+
+# Copyright (C) 2009-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ..
+
+printf '\n...: invalid compressed data--format violated\n' > exp \
+  || framework_failure_
+
+fail=0
+gzip -dc "$abs_srcdir/hufts-segv.gz" > out 2> err
+test $? = 1 || fail=1
+
+compare /dev/null out || fail=1
+
+sed 's/.*hufts-segv.gz: /...: /' err > k; mv k err || fail=1
+compare exp err || fail=1
+
+Exit $fail
diff --git a/tests/gzip/init.cfg b/tests/gzip/init.cfg
new file mode 100644
index 000000000..901209cea
--- /dev/null
+++ b/tests/gzip/init.cfg
@@ -0,0 +1,5 @@
+# This file is sourced by init.sh, *before* its initialization.
+
+# This goes hand in hand with the "exec 9>&2;" in Makefile.am's
+# TESTS_ENVIRONMENT definition.
+stderr_fileno_=9
diff --git a/tests/gzip/init.sh b/tests/gzip/init.sh
new file mode 100644
index 000000000..97e4e4ba5
--- /dev/null
+++ b/tests/gzip/init.sh
@@ -0,0 +1,616 @@
+# source this file; set up for tests
+
+# Copyright (C) 2009-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+
+# Using this file in a test
+# =========================
+#
+# The typical skeleton of a test looks like this:
+#
+#   #!/bin/sh
+#   . "${srcdir=.}/init.sh"; path_prepend_ .
+#   Execute some commands.
+#   Note that these commands are executed in a subdirectory, therefore you
+#   need to prepend "../" to relative filenames in the build directory.
+#   Note that the "path_prepend_ ." is useful only if the body of your
+#   test invokes programs residing in the initial directory.
+#   For example, if the programs you want to test are in src/, and this test
+#   script is named tests/test-1, then you would use "path_prepend_ ../src",
+#   or perhaps export PATH='$(abs_top_builddir)/src$(PATH_SEPARATOR)'"$$PATH"
+#   to all tests via automake's TESTS_ENVIRONMENT.
+#   Set the exit code 0 for success, 77 for skipped, or 1 or other for failure.
+#   Use the skip_ and fail_ functions to print a diagnostic and then exit
+#   with the corresponding exit code.
+#   Exit $?
+
+# Executing a test that uses this file
+# ====================================
+#
+# Running a single test:
+#   $ make check TESTS=test-foo.sh
+#
+# Running a single test, with verbose output:
+#   $ make check TESTS=test-foo.sh VERBOSE=yes
+#
+# Running a single test, with single-stepping:
+#   1. Go into a sub-shell:
+#   $ bash
+#   2. Set relevant environment variables from TESTS_ENVIRONMENT in the
+#      Makefile:
+#   $ export srcdir=../../tests # this is an example
+#   3. Execute the commands from the test, copy&pasting them one by one:
+#   $ . "$srcdir/init.sh"; path_prepend_ .
+#   ...
+#   4. Finally
+#   $ exit
+
+ME_=`expr "./$0" : '.*/\(.*\)$'`
+
+# We use a trap below for cleanup.  This requires us to go through
+# hoops to get the right exit status transported through the handler.
+# So use 'Exit STATUS' instead of 'exit STATUS' inside of the tests.
+# Turn off errexit here so that we don't trip the bug with OSF1/Tru64
+# sh inside this function.
+Exit () { set +e; (exit $1); exit $1; }
+
+# Print warnings (e.g., about skipped and failed tests) to this file number.
+# Override by defining to say, 9, in init.cfg, and putting say,
+#   export ...ENVVAR_SETTINGS...; $(SHELL) 9>&2
+# in the definition of TESTS_ENVIRONMENT in your tests/Makefile.am file.
+# This is useful when using automake's parallel tests mode, to print
+# the reason for skip/failure to console, rather than to the .log files.
+: ${stderr_fileno_=2}
+
+# Note that correct expansion of "$*" depends on IFS starting with ' '.
+# Always write the full diagnostic to stderr.
+# When stderr_fileno_ is not 2, also emit the first line of the
+# diagnostic to that file descriptor.
+warn_ ()
+{
+  # If IFS does not start with ' ', set it and emit the warning in a subshell.
+  case $IFS in
+    ' '*) printf '%s\n' "$*" >&2
+          test $stderr_fileno_ = 2 \
+            || { printf '%s\n' "$*" | sed 1q >&$stderr_fileno_ ; } ;;
+    *) (IFS=' '; warn_ "$@");;
+  esac
+}
+fail_ () { warn_ "$ME_: failed test: $@"; Exit 1; }
+skip_ () { warn_ "$ME_: skipped test: $@"; Exit 77; }
+fatal_ () { warn_ "$ME_: hard error: $@"; Exit 99; }
+framework_failure_ () { warn_ "$ME_: set-up failure: $@"; Exit 99; }
+
+# This is used to simplify checking of the return value
+# which is useful when ensuring a command fails as desired.
+# I.e., just doing `command ... &&fail=1` will not catch
+# a segfault in command for example.  With this helper you
+# instead check an explicit exit code like
+#   returns_ 1 command ... || fail
+returns_ () {
+  # Disable tracing so it doesn't interfere with stderr of the wrapped command
+  { set +x; } 2>/dev/null
+
+  local exp_exit="$1"
+  shift
+  "$@"
+  test $? -eq $exp_exit && ret_=0 || ret_=1
+
+  if test "$VERBOSE" = yes && test "$gl_set_x_corrupts_stderr_" = false; then
+    set -x
+  fi
+  { return $ret_; } 2>/dev/null
+}
+
+# Sanitize this shell to POSIX mode, if possible.
+DUALCASE=1; export DUALCASE
+if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
+  emulate sh
+  NULLCMD=:
+  alias -g '${1+"$@"}'='"$@"'
+  setopt NO_GLOB_SUBST
+else
+  case `(set -o) 2>/dev/null` in
+    *posix*) set -o posix ;;
+  esac
+fi
+
+# We require $(...) support unconditionally.
+# We require a few additional shell features only when $EXEEXT is nonempty,
+# in order to support automatic $EXEEXT emulation:
+# - hyphen-containing alias names
+# - we prefer to use ${var#...} substitution, rather than having
+#   to work around lack of support for that feature.
+# The following code attempts to find a shell with support for these features.
+# If the current shell passes the test, we're done.  Otherwise, test other
+# shells until we find one that passes.  If one is found, re-exec it.
+# If no acceptable shell is found, skip the current test.
+#
+# The "...set -x; P=1 true 2>err..." test is to disqualify any shell that
+# emits "P=1" into err, as /bin/sh from SunOS 5.11 and OpenBSD 4.7 do.
+#
+# Use "9" to indicate success (rather than 0), in case some shell acts
+# like Solaris 10's /bin/sh but exits successfully instead of with status 2.
+
+# Eval this code in a subshell to determine a shell's suitability.
+# 10 - passes all tests; ok to use
+#  9 - ok, but enabling "set -x" corrupts app stderr; prefer higher score
+#  ? - not ok
+gl_shell_test_script_='
+test $(echo y) = y || exit 1
+f_local_() { local v=1; }; f_local_ || exit 1
+score_=10
+if test "$VERBOSE" = yes; then
+  test -n "$( (exec 3>&1; set -x; P=1 true 2>&3) 2> /dev/null)" && score_=9
+fi
+test -z "$EXEEXT" && exit $score_
+shopt -s expand_aliases
+alias a-b="echo zoo"
+v=abx
+     test ${v%x} = ab \
+  && test ${v#a} = bx \
+  && test $(a-b) = zoo \
+  && exit $score_
+'
+
+if test "x$1" = "x--no-reexec"; then
+  shift
+else
+  # Assume a working shell.  Export to subshells (setup_ needs this).
+  gl_set_x_corrupts_stderr_=false
+  export gl_set_x_corrupts_stderr_
+
+  # Record the first marginally acceptable shell.
+  marginal_=
+
+  # Search for a shell that meets our requirements.
+  for re_shell_ in __current__ "${CONFIG_SHELL:-no_shell}" \
+      /bin/sh bash dash zsh pdksh fail
+  do
+    test "$re_shell_" = no_shell && continue
+
+    # If we've made it all the way to the sentinel, "fail" without
+    # finding even a marginal shell, skip this test.
+    if test "$re_shell_" = fail; then
+      test -z "$marginal_" && skip_ failed to find an adequate shell
+      re_shell_=$marginal_
+      break
+    fi
+
+    # When testing the current shell, simply "eval" the test code.
+    # Otherwise, run it via $re_shell_ -c ...
+    if test "$re_shell_" = __current__; then
+      # 'eval'ing this code makes Solaris 10's /bin/sh exit with
+      # $? set to 2.  It does not evaluate any of the code after the
+      # "unexpected" first '('.  Thus, we must run it in a subshell.
+      ( eval "$gl_shell_test_script_" ) > /dev/null 2>&1
+    else
+      "$re_shell_" -c "$gl_shell_test_script_" 2>/dev/null
+    fi
+
+    st_=$?
+
+    # $re_shell_ works just fine.  Use it.
+    if test $st_ = 10; then
+      gl_set_x_corrupts_stderr_=false
+      break
+    fi
+
+    # If this is our first marginally acceptable shell, remember it.
+    if test "$st_:$marginal_" = 9: ; then
+      marginal_="$re_shell_"
+      gl_set_x_corrupts_stderr_=true
+    fi
+  done
+
+  if test "$re_shell_" != __current__; then
+    # Found a usable shell.  Preserve -v and -x.
+    case $- in
+      *v*x* | *x*v*) opts_=-vx ;;
+      *v*) opts_=-v ;;
+      *x*) opts_=-x ;;
+      *) opts_= ;;
+    esac
+    re_shell=$re_shell_
+    export re_shell
+    exec "$re_shell_" $opts_ "$0" --no-reexec "$@"
+    echo "$ME_: exec failed" 1>&2
+    exit 127
+  fi
+fi
+
+# If this is bash, turn off all aliases.
+test -n "$BASH_VERSION" && unalias -a
+
+# Note that when supporting $EXEEXT (transparently mapping from PROG_NAME to
+# PROG_NAME.exe), we want to support hyphen-containing names like test-acos.
+# That is part of the shell-selection test above.  Why use aliases rather
+# than functions?  Because support for hyphen-containing aliases is more
+# widespread than that for hyphen-containing function names.
+test -n "$EXEEXT" && shopt -s expand_aliases
+
+# Enable glibc's malloc-perturbing option.
+# This is useful for exposing code that depends on the fact that
+# malloc-related functions often return memory that is mostly zeroed.
+# If you have the time and cycles, use valgrind to do an even better job.
+: ${MALLOC_PERTURB_=87}
+export MALLOC_PERTURB_
+
+# This is a stub function that is run upon trap (upon regular exit and
+# interrupt).  Override it with a per-test function, e.g., to unmount
+# a partition, or to undo any other global state changes.
+cleanup_ () { :; }
+
+# Emit a header similar to that from diff -u;  Print the simulated "diff"
+# command so that the order of arguments is clear.  Don't bother with @@ lines.
+emit_diff_u_header_ ()
+{
+  printf '%s\n' "diff -u $*" \
+    "--- $1	1970-01-01" \
+    "+++ $2	1970-01-01"
+}
+
+# Arrange not to let diff or cmp operate on /dev/null,
+# since on some systems (at least OSF/1 5.1), that doesn't work.
+# When there are not two arguments, or no argument is /dev/null, return 2.
+# When one argument is /dev/null and the other is not empty,
+# cat the nonempty file to stderr and return 1.
+# Otherwise, return 0.
+compare_dev_null_ ()
+{
+  test $# = 2 || return 2
+
+  if test "x$1" = x/dev/null; then
+    test -s "$2" || return 0
+    emit_diff_u_header_ "$@"; sed 's/^/+/' "$2"
+    return 1
+  fi
+
+  if test "x$2" = x/dev/null; then
+    test -s "$1" || return 0
+    emit_diff_u_header_ "$@"; sed 's/^/-/' "$1"
+    return 1
+  fi
+
+  return 2
+}
+
+if diff_out_=`exec 2>/dev/null; diff -u "$0" "$0" < /dev/null` \
+   && diff -u Makefile "$0" 2>/dev/null | grep '^[+]#!' >/dev/null; then
+  # diff accepts the -u option and does not (like AIX 7 'diff') produce an
+  # extra space on column 1 of every content line.
+  if test -z "$diff_out_"; then
+    compare_ () { diff -u "$@"; }
+  else
+    compare_ ()
+    {
+      if diff -u "$@" > diff.out; then
+        # No differences were found, but Solaris 'diff' produces output
+        # "No differences encountered". Hide this output.
+        rm -f diff.out
+        true
+      else
+        cat diff.out
+        rm -f diff.out
+        false
+      fi
+    }
+  fi
+elif
+  for diff_opt_ in -U3 -c '' no; do
+    test "$diff_opt_" = no && break
+    diff_out_=`exec 2>/dev/null; diff $diff_opt_ "$0" "$0"  diff.out; then
+        # No differences were found, but AIX and HP-UX 'diff' produce output
+        # "No differences encountered" or "There are no differences between the
+        # files.". Hide this output.
+        rm -f diff.out
+        true
+      else
+        cat diff.out
+        rm -f diff.out
+        false
+      fi
+    }
+  fi
+elif cmp -s /dev/null /dev/null 2>/dev/null; then
+  compare_ () { cmp -s "$@"; }
+else
+  compare_ () { cmp "$@"; }
+fi
+
+# Usage: compare EXPECTED ACTUAL
+#
+# Given compare_dev_null_'s preprocessing, defer to compare_ if 2 or more.
+# Otherwise, propagate $? to caller: any diffs have already been printed.
+compare ()
+{
+  # This looks like it can be factored to use a simple "case $?"
+  # after unchecked compare_dev_null_ invocation, but that would
+  # fail in a "set -e" environment.
+  if compare_dev_null_ "$@"; then
+    return 0
+  else
+    case $? in
+      1) return 1;;
+      *) compare_ "$@";;
+    esac
+  fi
+}
+
+# An arbitrary prefix to help distinguish test directories.
+testdir_prefix_ () { printf gt; }
+
+# Run the user-overridable cleanup_ function, remove the temporary
+# directory and exit with the incoming value of $?.
+remove_tmp_ ()
+{
+  __st=$?
+  cleanup_
+  # cd out of the directory we're about to remove
+  cd "$initial_cwd_" || cd / || cd /tmp
+  chmod -R u+rwx "$test_dir_"
+  # If removal fails and exit status was to be 0, then change it to 1.
+  rm -rf "$test_dir_" || { test $__st = 0 && __st=1; }
+  exit $__st
+}
+
+# Given a directory name, DIR, if every entry in it that matches *.exe
+# contains only the specified bytes (see the case stmt below), then print
+# a space-separated list of those names and return 0.  Otherwise, don't
+# print anything and return 1.  Naming constraints apply also to DIR.
+find_exe_basenames_ ()
+{
+  feb_dir_=$1
+  feb_fail_=0
+  feb_result_=
+  feb_sp_=
+  for feb_file_ in $feb_dir_/*.exe; do
+    # If there was no *.exe file, or there existed a file named "*.exe" that
+    # was deleted between the above glob expansion and the existence test
+    # below, just skip it.
+    test "x$feb_file_" = "x$feb_dir_/*.exe" && test ! -f "$feb_file_" \
+      && continue
+    # Exempt [.exe, since we can't create a function by that name, yet
+    # we can't invoke [ by PATH search anyways due to shell builtins.
+    test "x$feb_file_" = "x$feb_dir_/[.exe" && continue
+    case $feb_file_ in
+      *[!-a-zA-Z/0-9_.+]*) feb_fail_=1; break;;
+      *) # Remove leading file name components as well as the .exe suffix.
+         feb_file_=${feb_file_##*/}
+         feb_file_=${feb_file_%.exe}
+         feb_result_="$feb_result_$feb_sp_$feb_file_";;
+    esac
+    feb_sp_=' '
+  done
+  test $feb_fail_ = 0 && printf %s "$feb_result_"
+  return $feb_fail_
+}
+
+# Consider the files in directory, $1.
+# For each file name of the form PROG.exe, create an alias named
+# PROG that simply invokes PROG.exe, then return 0.  If any selected
+# file name or the directory name, $1, contains an unexpected character,
+# define no alias and return 1.
+create_exe_shims_ ()
+{
+  case $EXEEXT in
+    '') return 0 ;;
+    .exe) ;;
+    *) echo "$0: unexpected \$EXEEXT value: $EXEEXT" 1>&2; return 1 ;;
+  esac
+
+  base_names_=`find_exe_basenames_ $1` \
+    || { echo "$0 (exe_shim): skipping directory: $1" 1>&2; return 0; }
+
+  if test -n "$base_names_"; then
+    for base_ in $base_names_; do
+      alias "$base_"="$base_$EXEEXT"
+    done
+  fi
+
+  return 0
+}
+
+# Use this function to prepend to PATH an absolute name for each
+# specified, possibly-$initial_cwd_-relative, directory.
+path_prepend_ ()
+{
+  while test $# != 0; do
+    path_dir_=$1
+    case $path_dir_ in
+      '') fail_ "invalid path dir: '$1'";;
+      /*) abs_path_dir_=$path_dir_;;
+      *) abs_path_dir_=$initial_cwd_/$path_dir_;;
+    esac
+    case $abs_path_dir_ in
+      *:*) fail_ "invalid path dir: '$abs_path_dir_'";;
+    esac
+    PATH="$abs_path_dir_:$PATH"
+
+    # Create an alias, FOO, for each FOO.exe in this directory.
+    create_exe_shims_ "$abs_path_dir_" \
+      || fail_ "something failed (above): $abs_path_dir_"
+    shift
+  done
+  export PATH
+}
+
+setup_ ()
+{
+  if test "$VERBOSE" = yes; then
+    # Test whether set -x may cause the selected shell to corrupt an
+    # application's stderr.  Many do, including zsh-4.3.10 and the /bin/sh
+    # from SunOS 5.11, OpenBSD 4.7 and Irix 5.x and 6.5.
+    # If enabling verbose output this way would cause trouble, simply
+    # issue a warning and refrain.
+    if $gl_set_x_corrupts_stderr_; then
+      warn_ "using SHELL=$SHELL with 'set -x' corrupts stderr"
+    else
+      set -x
+    fi
+  fi
+
+  initial_cwd_=$PWD
+
+  pfx_=`testdir_prefix_`
+  test_dir_=`mktempd_ "$initial_cwd_" "$pfx_-$ME_.XXXX"` \
+    || fail_ "failed to create temporary directory in $initial_cwd_"
+  cd "$test_dir_" || fail_ "failed to cd to temporary directory"
+
+  # As autoconf-generated configure scripts do, ensure that IFS
+  # is defined initially, so that saving and restoring $IFS works.
+  gl_init_sh_nl_='
+'
+  IFS=" ""	$gl_init_sh_nl_"
+
+  # This trap statement, along with a trap on 0 below, ensure that the
+  # temporary directory, $test_dir_, is removed upon exit as well as
+  # upon receipt of any of the listed signals.
+  for sig_ in 1 2 3 13 15; do
+    eval "trap 'Exit $(expr $sig_ + 128)' $sig_"
+  done
+}
+
+# Create a temporary directory, much like mktemp -d does.
+# Written by Jim Meyering.
+#
+# Usage: mktempd_ /tmp phoey.XXXXXXXXXX
+#
+# First, try to use the mktemp program.
+# Failing that, we'll roll our own mktemp-like function:
+#  - try to get random bytes from /dev/urandom
+#  - failing that, generate output from a combination of quickly-varying
+#      sources and gzip.  Ignore non-varying gzip header, and extract
+#      "random" bits from there.
+#  - given those bits, map to file-name bytes using tr, and try to create
+#      the desired directory.
+#  - make only $MAX_TRIES_ attempts
+
+# Helper function.  Print $N pseudo-random bytes from a-zA-Z0-9.
+rand_bytes_ ()
+{
+  n_=$1
+
+  # Maybe try openssl rand -base64 $n_prime_|tr '+/=\012' abcd first?
+  # But if they have openssl, they probably have mktemp, too.
+
+  chars_=abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789
+  dev_rand_=/dev/urandom
+  if test -r "$dev_rand_"; then
+    # Note: 256-length($chars_) == 194; 3 copies of $chars_ is 186 + 8 = 194.
+    dd ibs=$n_ count=1 if=$dev_rand_ 2>/dev/null \
+      | LC_ALL=C tr -c $chars_ 01234567$chars_$chars_$chars_
+    return
+  fi
+
+  n_plus_50_=`expr $n_ + 50`
+  cmds_='date; date +%N; free; who -a; w; ps auxww; ps ef; netstat -n'
+  data_=` (eval "$cmds_") 2>&1 | gzip `
+
+  # Ensure that $data_ has length at least 50+$n_
+  while :; do
+    len_=`echo "$data_"|wc -c`
+    test $n_plus_50_ -le $len_ && break;
+    data_=` (echo "$data_"; eval "$cmds_") 2>&1 | gzip `
+  done
+
+  echo "$data_" \
+    | dd bs=1 skip=50 count=$n_ 2>/dev/null \
+    | LC_ALL=C tr -c $chars_ 01234567$chars_$chars_$chars_
+}
+
+mktempd_ ()
+{
+  case $# in
+  2);;
+  *) fail_ "Usage: mktempd_ DIR TEMPLATE";;
+  esac
+
+  destdir_=$1
+  template_=$2
+
+  MAX_TRIES_=4
+
+  # Disallow any trailing slash on specified destdir:
+  # it would subvert the post-mktemp "case"-based destdir test.
+  case $destdir_ in
+  / | //) destdir_slash_=$destdir;;
+  */) fail_ "invalid destination dir: remove trailing slash(es)";;
+  *) destdir_slash_=$destdir_/;;
+  esac
+
+  case $template_ in
+  *XXXX) ;;
+  *) fail_ \
+       "invalid template: $template_ (must have a suffix of at least 4 X's)";;
+  esac
+
+  # First, try to use mktemp.
+  d=`unset TMPDIR; { mktemp -d -t -p "$destdir_" "$template_"; } 2>/dev/null` &&
+
+  # The resulting name must be in the specified directory.
+  case $d in "$destdir_slash_"*) :;; *) false;; esac &&
+
+  # It must have created the directory.
+  test -d "$d" &&
+
+  # It must have 0700 permissions.  Handle sticky "S" bits.
+  perms=`ls -dgo "$d" 2>/dev/null` &&
+  case $perms in drwx--[-S]---*) :;; *) false;; esac && {
+    echo "$d"
+    return
+  }
+
+  # If we reach this point, we'll have to create a directory manually.
+
+  # Get a copy of the template without its suffix of X's.
+  base_template_=`echo "$template_"|sed 's/XX*$//'`
+
+  # Calculate how many X's we've just removed.
+  template_length_=`echo "$template_" | wc -c`
+  nx_=`echo "$base_template_" | wc -c`
+  nx_=`expr $template_length_ - $nx_`
+
+  err_=
+  i_=1
+  while :; do
+    X_=`rand_bytes_ $nx_`
+    candidate_dir_="$destdir_slash_$base_template_$X_"
+    err_=`mkdir -m 0700 "$candidate_dir_" 2>&1` \
+      && { echo "$candidate_dir_"; return; }
+    test $MAX_TRIES_ -le $i_ && break;
+    i_=`expr $i_ + 1`
+  done
+  fail_ "$err_"
+}
+
+# If you want to override the testdir_prefix_ function,
+# or to add more utility functions, use this file.
+test -f "$srcdir/init.cfg" \
+  && . "$srcdir/init.cfg"
+
+setup_ "$@"
+# This trap is here, rather than in the setup_ function, because some
+# shells run the exit trap at shell function exit, rather than script exit.
+trap remove_tmp_ 0
diff --git a/tests/gzip/keep.sh b/tests/gzip/keep.sh
new file mode 100644
index 000000000..105d43e64
--- /dev/null
+++ b/tests/gzip/keep.sh
@@ -0,0 +1,51 @@
+#!/bin/sh
+# Exercise the --keep option.
+
+# Copyright (C) 2013-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ..
+
+echo fooooooooo > in || framework_failure_
+cp in orig || framework_failure_
+
+fail=0
+
+# Compress and decompress both with and without --keep.
+for k in --keep ''; do
+  # With --keep, the source must be retained, otherwise, it must be removed.
+  case $k in --keep) op='||' ;; *) op='&&' ;; esac
+
+  gzip $k in || fail=1
+  eval "test -f in $op fail=1"
+  test -f in.gz || fail=1
+  rm -f in || fail=1
+
+  gzip -d $k in.gz || fail=1
+  eval "test -f in.gz $op fail=1"
+  test -f in || fail=1
+  compare in orig || fail=1
+  rm -f in.gz || fail=1
+done
+
+cp orig in || framework_failure_
+log=$(gzip -kv in 2>&1) || fail=1
+case $log in
+  *'created in.gz'*) ;;
+  *) fail=1;;
+esac
+
+Exit $fail
diff --git a/tests/gzip/list.sh b/tests/gzip/list.sh
new file mode 100644
index 000000000..7576dc3a8
--- /dev/null
+++ b/tests/gzip/list.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+# Exercise the --list option.
+
+# Copyright 2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ..
+
+echo zoology zucchini > in || framework_failure_
+cp in orig || framework_failure_
+
+gzip -l in && fail=1
+gzip -9 in || fail=1
+gzip -l in.gz >out1 || fail=1
+gzip -l in.gz | cat >out2 || fail=1
+compare out1 out2 || fail=1
+
+Exit $fail
diff --git a/tests/gzip/list2.log b/tests/gzip/list2.log
new file mode 100644
index 000000000..86e3a37f4
--- /dev/null
+++ b/tests/gzip/list2.log
@@ -0,0 +1,7 @@
+gzip -l in
+
+gzip: in: not in gzip format
+gzip -9 in
+gzip -l in.gz
+gzip -l in.gz
+PASS list (exit status: 0)
diff --git a/tests/gzip/memcpy-abuse.sh b/tests/gzip/memcpy-abuse.sh
new file mode 100644
index 000000000..970b88523
--- /dev/null
+++ b/tests/gzip/memcpy-abuse.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+# Before gzip-1.4, this the use of memcpy in inflate_codes could
+# mistakenly operate on overlapping regions.  Exercise that code.
+
+# Copyright (C) 2010-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ..
+
+# The input must be larger than 32KiB and slightly
+# less uniform than e.g., all zeros.
+printf wxy%032767d 0 | tee in | gzip > in.gz || framework_failure_
+
+fail=0
+
+# Before the fix, this would call memcpy with overlapping regions.
+gzip -dc in.gz > out || fail=1
+
+compare in out || fail=1
+
+Exit $fail
diff --git a/tests/gzip/mixed.sh b/tests/gzip/mixed.sh
new file mode 100644
index 000000000..50e2537a2
--- /dev/null
+++ b/tests/gzip/mixed.sh
@@ -0,0 +1,68 @@
+#!/bin/sh
+# Ensure that gzip -cdf handles mixed compressed/not-compressed data
+# Before gzip-1.5, it would produce invalid output.
+
+# Copyright (C) 2010-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ..
+
+printf 'xxx\nyyy\n'      > exp2 || framework_failure_
+printf 'aaa\nbbb\nccc\n' > exp3 || framework_failure_
+
+fail=0
+
+(echo xxx; echo yyy) > in || fail=1
+gzip -cdf < in > out || fail=1
+compare exp2 out || fail=1
+
+# Uncompressed input, followed by compressed data.
+# Currently fails, so skip it.
+# (echo xxx; echo yyy|gzip) > in || fail=1
+# gzip -cdf < in > out || fail=1
+# compare exp2 out || fail=1
+
+# Compressed input, followed by regular (not-compressed) data.
+(echo xxx|gzip; echo yyy) > in || fail=1
+gzip -cdf < in > out || fail=1
+compare exp2 out || fail=1
+
+(echo xxx|gzip; echo yyy|gzip) > in || fail=1
+gzip -cdf < in > out || fail=1
+compare exp2 out || fail=1
+
+in_str=0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-+=%
+for i in 0 1 2 3 4 5 6 7 8 9 a; do in_str="$in_str$in_str" ;done
+
+# Start with some small sizes.  $(seq 64)
+sizes=$(i=0; while :; do echo $i; test $i = 64 && break; i=$(expr $i + 1); done)
+
+# gzip's internal buffer size is 32KiB + 64 bytes:
+sizes="$sizes 32831 32832 32833"
+
+# 128KiB, +/- 1
+sizes="$sizes 131071 131072 131073"
+
+# Ensure that "gzip -cdf" acts like cat, for a range of small input files.
+i=0
+for i in $sizes; do
+  echo $i
+  printf %$i.${i}s $in_str > in
+  gzip -cdf < in > out
+  compare in out || fail=1
+done
+
+Exit $fail
diff --git a/tests/gzip/null-suffix-clobber.sh b/tests/gzip/null-suffix-clobber.sh
new file mode 100644
index 000000000..903864ce6
--- /dev/null
+++ b/tests/gzip/null-suffix-clobber.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+# Before gzip-1.5, gzip -d -S '' k.gz would delete F.gz and not create "F"
+
+# Copyright (C) 2010-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ..
+
+printf anything | gzip > F.gz || framework_failure_
+echo y > yes || framework_failure_
+echo "gzip: invalid suffix ''" > expected-err || framework_failure_
+
+fail=0
+
+gzip ---presume-input-tty -d -S '' F.gz < yes > out 2>err && fail=1
+
+compare /dev/null out || fail=1
+compare expected-err err || fail=1
+
+test -f F.gz || fail=1
+
+Exit $fail
diff --git a/tests/gzip/stdin.sh b/tests/gzip/stdin.sh
new file mode 100644
index 000000000..190687f96
--- /dev/null
+++ b/tests/gzip/stdin.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+# Ensure that gzip interprets "-" as stdin.
+
+# Copyright (C) 2009-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ..
+
+printf a | gzip > in || framework_failure_
+printf aaa > exp || framework_failure_
+
+fail=0
+gzip -dc in - in < in > out 2>err || fail=1
+
+compare exp out || fail=1
+compare /dev/null err || fail=1
+
+Exit $fail
diff --git a/tests/gzip/test-driver.sh b/tests/gzip/test-driver.sh
new file mode 100644
index 000000000..649c084e4
--- /dev/null
+++ b/tests/gzip/test-driver.sh
@@ -0,0 +1,150 @@
+#! /bin/sh
+# test-driver - basic testsuite driver script.
+
+scriptversion=2016-01-11.22; # UTC
+
+# Copyright (C) 2011-2015 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# This file is maintained in Automake, please report
+# bugs to  or send patches to
+# .
+
+# Make unconditional expansion of undefined variables an error.  This
+# helps a lot in preventing typo-related bugs.
+set -u
+
+usage_error ()
+{
+  echo "$0: $*" >&2
+  print_usage >&2
+  exit 2
+}
+
+print_usage ()
+{
+  cat <$log_file 2>&1
+estatus=$?
+
+if test $enable_hard_errors = no && test $estatus -eq 99; then
+  tweaked_estatus=1
+else
+  tweaked_estatus=$estatus
+fi
+
+case $tweaked_estatus:$expect_failure in
+  0:yes) col=$red res=XPASS recheck=yes gcopy=yes;;
+  0:*)   col=$grn res=PASS  recheck=no  gcopy=no;;
+  77:*)  col=$blu res=SKIP  recheck=no  gcopy=yes;;
+  99:*)  col=$mgn res=ERROR recheck=yes gcopy=yes;;
+  *:yes) col=$lgn res=XFAIL recheck=no  gcopy=yes;;
+  *:*)   col=$red res=FAIL  recheck=yes gcopy=yes;;
+esac
+
+# Report the test outcome and exit status in the logs, so that one can
+# know whether the test passed or failed simply by looking at the '.log'
+# file, without the need of also peaking into the corresponding '.trs'
+# file (automake bug#11814).
+echo "$res $test_name (exit status: $estatus)" >>$log_file
+
+# Report outcome to console.
+echo "${col}${res}${std}: $test_name"
+
+# Register the test result, and other relevant metadata.
+echo ":test-result: $res" > $trs_file
+echo ":global-test-result: $res" >> $trs_file
+echo ":recheck: $recheck" >> $trs_file
+echo ":copy-in-global-log: $gcopy" >> $trs_file
+
+# Local Variables:
+# mode: shell-script
+# sh-indentation: 2
+# eval: (add-hook 'write-file-hooks 'time-stamp)
+# time-stamp-start: "scriptversion="
+# time-stamp-format: "%:y-%02m-%02d.%02H"
+# time-stamp-time-zone: "UTC0"
+# time-stamp-end: "; # UTC"
+# End:
+
+exit $tweaked_estatus
diff --git a/tests/gzip/trailing-nul.sh b/tests/gzip/trailing-nul.sh
new file mode 100644
index 000000000..b21f76fdc
--- /dev/null
+++ b/tests/gzip/trailing-nul.sh
@@ -0,0 +1,37 @@
+#!/bin/sh
+# gzip accepts trailing NUL bytes; don't fail if there is exactly one.
+# Before gzip-1.4, this would fail.
+
+# Copyright (C) 2009-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ..
+
+(echo 0 | gzip; printf '\0') > 0.gz || framework_failure_
+(echo 00 | gzip; printf '\0\0') > 00.gz || framework_failure_
+(echo 1 | gzip; printf '\1') > 1.gz || framework_failure_
+
+fail=0
+
+for i in 0 00 1; do
+  gzip -d $i.gz; ret=$?
+  test $ret -eq $i || fail=1
+  test $ret = 1 && continue
+  echo $i > exp || fail=1
+  compare exp $i || fail=1
+done
+
+Exit $fail
diff --git a/tests/gzip/unpack-invalid.sh b/tests/gzip/unpack-invalid.sh
new file mode 100644
index 000000000..acea97e83
--- /dev/null
+++ b/tests/gzip/unpack-invalid.sh
@@ -0,0 +1,36 @@
+#!/bin/sh
+# gzip should report invalid 'unpack' input when uncompressing.
+# With gzip-1.5, it would output invalid data instead.
+
+# Copyright (C) 2012-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ..
+
+for input in \
+  '\037\036\000\000\037\213\010\000\000\000\000\000\002\003\036\000\000\000\002\003\037\213\010\000\000\000\000\000\002\003\355\301\001\015\000\000\000\302\240\037\000\302\240\037\213\010\000\000\000\000\000\002\003\355\301' \
+  '\037\213\010\000\000\000\000\000\002\003\355\301\001\015\000\000\000\302\240\076\366\017\370\036\016\030\000\000\000\000\000\000\000\000\000\034\010\105\140\104\025\020\047\000\000\037\036\016\030\000\000\000'; do
+
+  printf "$input" >in || framework_failure_
+
+  if gzip -d out 2>err; then
+    fail=1
+  else
+    fail=0
+  fi
+done
+
+Exit $fail
diff --git a/tests/gzip/z-suffix.sh b/tests/gzip/z-suffix.sh
new file mode 100644
index 000000000..470932060
--- /dev/null
+++ b/tests/gzip/z-suffix.sh
@@ -0,0 +1,30 @@
+#!/bin/sh
+# Check that -Sz works.
+
+# Copyright 2014-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ..
+
+printf anything > F && cp F G || framework_failure_
+gzip -Sz F || fail=1
+test ! -f F || fail=1
+test -f Fz || fail=1
+gzip -dSz F || fail=1
+test ! -f Fz || fail=1
+compare F G || fail\1
+
+Exit $fail
diff --git a/tests/gzip/zdiff.sh b/tests/gzip/zdiff.sh
new file mode 100644
index 000000000..0bb7c7dfd
--- /dev/null
+++ b/tests/gzip/zdiff.sh
@@ -0,0 +1,48 @@
+#!/bin/sh
+# Exercise zdiff with two compressed inputs.
+# Before gzip-1.4, this would fail.
+
+# Copyright (C) 2009-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ..
+
+echo a > a || framework_failure_
+echo b > b || framework_failure_
+gzip a b || framework_failure_
+
+cat < exp
+1c1
+< a
+---
+> b
+EOF
+
+fail=0
+zdiff a.gz b.gz > out 2>&1
+test $? = 1 || fail=1
+
+compare exp out || fail=1
+
+rm -f out
+# expect success, for equal files
+zdiff a.gz a.gz > out 2> err || fail=1
+# expect no output
+test -s out && fail=1
+# expect no stderr
+test -s err && fail=1
+
+Exit $fail
diff --git a/tests/gzip/zgrep-context.sh b/tests/gzip/zgrep-context.sh
new file mode 100644
index 000000000..089d25639
--- /dev/null
+++ b/tests/gzip/zgrep-context.sh
@@ -0,0 +1,47 @@
+#!/bin/sh
+# Ensure that zgrep -15 works.  Before gzip-1.5, it would fail.
+
+# Copyright (C) 2012-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ..
+
+# A limited replacement for seq: handle 1 or 2 args; increment must be 1
+seq()
+{
+  case $# in
+    1) start=1  final=$1;;
+    2) start=$1 final=$2;;
+    *) echo you lose 1>&2; exit 1;;
+  esac
+  awk 'BEGIN{for(i='$start';i<='$final';i++) print i}' < /dev/null
+}
+
+seq 40 > in || framework_failure_
+gzip < in > in.gz || framework_failure_
+seq 2 32 > exp || framework_failure_
+
+: ${GREP=grep}
+$GREP -15 17 - < in > out && compare exp out || {
+  echo >&2 "$0: $GREP does not support context options; skipping this test"
+  exit 77
+}
+
+fail=0
+zgrep -15 17 - < in.gz > out || fail=1
+compare exp out || fail=1
+
+Exit $fail
diff --git a/tests/gzip/zgrep-f.sh b/tests/gzip/zgrep-f.sh
new file mode 100644
index 000000000..1ce8cc293
--- /dev/null
+++ b/tests/gzip/zgrep-f.sh
@@ -0,0 +1,43 @@
+#!/bin/sh
+# Ensure that zgrep -f - works like grep -f -
+# Before gzip-1.4, it would fail.
+
+# Copyright (C) 2009-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ..
+
+printf 'needle\nn2\n' > n || framework_failure_
+cp n haystack || framework_failure_
+gzip haystack || framework_failure_
+
+fail=0
+zgrep -f - haystack.gz < n > out 2>&1 || fail=1
+
+compare out n || fail=1
+
+if ${BASH_VERSION+:} false; then
+  set +o posix
+  # This failed with gzip 1.6.
+  cat n n >nn || framework_failure_
+  eval 'zgrep -h -f <(cat n) haystack.gz haystack.gz' >out || fail=1
+  compare out nn || fail=1
+fi
+
+# This failed with gzip 1.4.
+echo a-b | zgrep -e - > /dev/null || fail=1
+
+Exit $fail
diff --git a/tests/gzip/zgrep-signal.sh b/tests/gzip/zgrep-signal.sh
new file mode 100644
index 000000000..13783ef10
--- /dev/null
+++ b/tests/gzip/zgrep-signal.sh
@@ -0,0 +1,64 @@
+#!/bin/sh
+# Check that zgrep is terminated gracefully by signal when
+# its grep/sed pipeline is terminated by a signal.
+
+# Copyright (C) 2010-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ..
+
+echo a | gzip -c > f.gz || framework_failure_
+
+test "x$PERL" = x && PERL=perl
+("$PERL" -e 'use POSIX qw(dup2)') >/dev/null 2>&1 ||
+   skip_ "no suitable perl found"
+
+# Run the arguments as a command, in a process where stdout is a
+# dangling pipe and SIGPIPE has the default signal-handling action.
+# This can't be done portably in the shell, because if SIGPIPE is
+# ignored when the shell is entered, the shell might refuse to trap
+# it.  Fall back on Perl+POSIX, if available.  Take care to close the
+# pipe's read end before running the program; the equivalent of the
+# shell's "command | :" has a race condition in that COMMAND could
+# write before ":" exits.
+write_to_dangling_pipe () {
+  program=${1?}
+  shift
+  args=
+  for arg; do
+    args="$args, '$arg'"
+  done
+  "$PERL" -e '
+     use POSIX qw(dup2);
+     $SIG{PIPE} = "DEFAULT";
+     pipe my ($read_end, $write_end) or die "pipe: $!\n";
+     dup2 fileno $write_end, 1 or die "dup2: $!\n";
+     close $read_end or die "close: $!\n";
+     exec '"'$program'$args"';
+  '
+}
+
+write_to_dangling_pipe cat f.gz f.gz
+signal_status=$?
+test 128 -lt $signal_status ||
+  framework_failure_ 'signal handling busted on this host'
+
+fail=0
+
+write_to_dangling_pipe zgrep a f.gz f.gz
+test $? -eq $signal_status || fail=1
+
+Exit $fail
diff --git a/tests/gzip/znew-k.sh b/tests/gzip/znew-k.sh
new file mode 100644
index 000000000..98e7fa1f8
--- /dev/null
+++ b/tests/gzip/znew-k.sh
@@ -0,0 +1,40 @@
+#!/bin/sh
+# Check that znew -K works without compress(1).
+
+# Copyright (C) 2010-2016 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+# limit so don't run it by default.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ..
+
+cat <<'EOF' >compress || framework_failure_
+#!/bin/sh
+echo >&2 'compress has been invoked'
+exit 1
+EOF
+chmod +x compress || framework_failure_
+
+# Note that the basename must have a length of 6 or greater.
+# Otherwise, "test -f $name" below would fail.
+name=123456.Z
+
+printf '%1012977s' ' ' | gzip -c > $name || framework_failure_
+
+fail=0
+
+znew -K $name || fail=1
+test -f $name || fail=1
+
+Exit $fail

From 22468b0cc3b1b52cad7c8c465a6158cb278c0caf Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Fri, 24 Feb 2017 10:53:16 -0800
Subject: [PATCH 154/223] Fix some more ARM compile errors

https://travis-ci.org/facebook/zstd/jobs/204807461

Can't get them to reproduce the compile errors locally,
tested fix by forcing that test to run on Travis.
---
 tests/decodecorpus.c | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/tests/decodecorpus.c b/tests/decodecorpus.c
index d75025a83..ecac7b234 100644
--- a/tests/decodecorpus.c
+++ b/tests/decodecorpus.c
@@ -609,17 +609,16 @@ static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore,
 {
     /* The total length of all the matches */
     size_t const remainingMatch = contentSize - literalsSize;
-    size_t excessMatch;
+    size_t excessMatch = 0;
+    U32 numSequences = 0;
+
     U32 i;
 
-    U32 numSequences;
 
     const BYTE* literals = LITERAL_BUFFER;
     BYTE* srcPtr = frame->src;
 
-    if (literalsSize == contentSize) {
-        numSequences = 0;
-    } else {
+    if (literalsSize != contentSize) {
         /* each match must be at least MIN_SEQ_LEN, so this is the maximum
          * number of sequences we can have */
         U32 const maxSequences = (U32)remainingMatch / MIN_SEQ_LEN;

From 8dff956dbf1ad9c280d222a5804e149635c47d7e Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sat, 25 Feb 2017 10:11:15 -0800
Subject: [PATCH 155/223] Added DDict unit test in fuzzer

also : slightly modified loadEntropy :
know src must points at start of dictionary
---
 lib/decompress/zstd_decompress.c | 47 ++++++++++++++++----------------
 tests/fuzzer.c                   |  8 ++++++
 2 files changed, 32 insertions(+), 23 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index e38ef79ba..655296e56 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -97,10 +97,10 @@ struct ZSTD_DCtx_s
     FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
     FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
     HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)];  /* can accommodate HUF_decompress4X */
-    const void* previousDstEnd;
-    const void* base;
-    const void* vBase;
-    const void* dictEnd;
+    const void* previousDstEnd;   /* detect continuity */
+    const void* base;             /* start of current segment */
+    const void* vBase;            /* virtual start of previous segment if it was just before current one */
+    const void* dictEnd;          /* end of previous segment */
     size_t expected;
     U32 rep[ZSTD_REP_NUM];
     ZSTD_frameParams fParams;
@@ -999,9 +999,9 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
 
 FORCE_INLINE
 size_t ZSTD_execSequence(BYTE* op,
-                                BYTE* const oend, seq_t sequence,
-                                const BYTE** litPtr, const BYTE* const litLimit,
-                                const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
+                         BYTE* const oend, seq_t sequence,
+                         const BYTE** litPtr, const BYTE* const litLimit,
+                         const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
 {
     BYTE* const oLitEnd = op + sequence.litLength;
     size_t const sequenceLength = sequence.litLength + sequence.matchLength;
@@ -1579,8 +1579,8 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
 
 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
         if (ZSTD_isLegacy(src, srcSize)) {
-            size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
             size_t decodedSize;
+            size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
             if (ZSTD_isError(frameSize)) return frameSize;
 
             decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize);
@@ -1625,8 +1625,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
         }
         ZSTD_checkContinuity(dctx, dst);
 
-        {
-            const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity,
+        {   const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity,
                                                     &src, &srcSize);
             if (ZSTD_isError(res)) return res;
             /* don't need to bounds check this, ZSTD_decompressFrame will have
@@ -1636,9 +1635,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
         }
     }
 
-    if (srcSize) {
-        return ERROR(srcSize_wrong);
-    }
+    if (srcSize) return ERROR(srcSize_wrong); /* input not entirely consumed */
 
     return (BYTE*)dst - (BYTE*)dststart;
 }
@@ -1835,18 +1832,24 @@ static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dict
     return 0;
 }
 
+/* ZSTD_loadEntropy() :
+ * dict : must point at beginning of dictionary
+ * @return : size of entropy tables read */
 static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t const dictSize)
 {
     const BYTE* dictPtr = (const BYTE*)dict;
     const BYTE* const dictEnd = dictPtr + dictSize;
 
-    {   size_t const hSize = HUF_readDTableX4(dctx->hufTable, dict, dictSize);
+    if (dictSize <= 8) return ERROR(dictionary_corrupted);
+    dictPtr += 8;   /* skip header = magic + dictID */
+
+    {   size_t const hSize = HUF_readDTableX4(dctx->hufTable, dictPtr, dictEnd-dictPtr);
         if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
         dictPtr += hSize;
     }
 
     {   short offcodeNCount[MaxOff+1];
-        U32 offcodeMaxValue=MaxOff, offcodeLog;
+        U32 offcodeMaxValue = MaxOff, offcodeLog;
         size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
         if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
         if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
@@ -1892,8 +1895,6 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
     dctx->dictID = MEM_readLE32((const char*)dict + 4);
 
     /* load entropy tables */
-    dict = (const char*)dict + 8;
-    dictSize -= 8;
     {   size_t const eSize = ZSTD_loadEntropy(dctx, dict, dictSize);
         if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted);
         dict = (const char*)dict + eSize;
@@ -1934,27 +1935,27 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, unsigne
             ZSTD_free(dctx, customMem);
             return NULL;
         }
+        ddict->refContext = dctx;
 
         if ((byReference) || (!dict) || (!dictSize)) {
             ddict->dictBuffer = NULL;
             ddict->dictContent = dict;
         } else {
             void* const internalBuffer = ZSTD_malloc(dictSize, customMem);
-            if (!internalBuffer) { ZSTD_free(dctx, customMem); ZSTD_free(ddict, customMem); return NULL; }
+            if (!internalBuffer) { ZSTD_freeDDict(ddict); return NULL; }
             memcpy(internalBuffer, dict, dictSize);
             ddict->dictBuffer = internalBuffer;
             ddict->dictContent = internalBuffer;
         }
-        {   size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, ddict->dictContent, dictSize);
+        /* parse dictionary content */
+        {   //size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, ddict->dictContent, dictSize);
+            size_t const errorCode = ZSTD_decompress_insertDictionary(dctx, ddict->dictContent, dictSize);
             if (ZSTD_isError(errorCode)) {
-                ZSTD_free(ddict->dictBuffer, customMem);
-                ZSTD_free(ddict, customMem);
-                ZSTD_free(dctx, customMem);
+                ZSTD_freeDDict(ddict);
                 return NULL;
         }   }
 
         ddict->dictSize = dictSize;
-        ddict->refContext = dctx;
         return ddict;
     }
 }
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 033998525..d38ac6f83 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -256,6 +256,14 @@ static int basicUnitTests(U32 seed, double compressibility)
                   if (r != CNBuffSize - dictSize) goto _output_error);
         DISPLAYLEVEL(4, "OK \n");
 
+        DISPLAYLEVEL(4, "test%3i : decompress with DDict : ", testNb++);
+        {   ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, dictSize);
+            size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
+            if (r != CNBuffSize - dictSize) goto _output_error;
+            DISPLAYLEVEL(4, "OK (size of DDict : %u) \n", (U32)ZSTD_sizeof_DDict(ddict));
+            ZSTD_freeDDict(ddict);
+        }
+
         DISPLAYLEVEL(4, "test%3i : check content size on duplicated context : ", testNb++);
         {   size_t const testSize = CNBuffSize / 3;
             {   ZSTD_parameters p = ZSTD_getParams(2, testSize, dictSize);

From 8629f0e41f9e8412dc22b9a5562c82d7fdbacc8e Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sat, 25 Feb 2017 18:33:31 -0800
Subject: [PATCH 156/223] created entropy structure type

---
 lib/decompress/zstd_decompress.c | 82 +++++++++++++++++---------------
 1 file changed, 44 insertions(+), 38 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 655296e56..493d191f1 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -87,22 +87,26 @@ typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
                ZSTDds_decompressLastBlock, ZSTDds_checkChecksum,
                ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
 
+typedef struct {
+    FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
+    FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
+    FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
+    HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)];  /* can accommodate HUF_decompress4X */
+    U32 rep[ZSTD_REP_NUM];
+} ZSTD_entropyTables_t;
+
 struct ZSTD_DCtx_s
 {
     const FSE_DTable* LLTptr;
     const FSE_DTable* MLTptr;
     const FSE_DTable* OFTptr;
     const HUF_DTable* HUFptr;
-    FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
-    FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
-    FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
-    HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)];  /* can accommodate HUF_decompress4X */
+    ZSTD_entropyTables_t entropy;
     const void* previousDstEnd;   /* detect continuity */
     const void* base;             /* start of current segment */
     const void* vBase;            /* virtual start of previous segment if it was just before current one */
     const void* dictEnd;          /* end of previous segment */
     size_t expected;
-    U32 rep[ZSTD_REP_NUM];
     ZSTD_frameParams fParams;
     blockType_e bType;   /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */
     ZSTD_dStage stage;
@@ -131,15 +135,15 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
     dctx->base = NULL;
     dctx->vBase = NULL;
     dctx->dictEnd = NULL;
-    dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);  /* cover both little and big endian */
+    dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);  /* cover both little and big endian */
     dctx->litEntropy = dctx->fseEntropy = 0;
     dctx->dictID = 0;
-    MEM_STATIC_ASSERT(sizeof(dctx->rep) == sizeof(repStartValue));
-    memcpy(dctx->rep, repStartValue, sizeof(repStartValue));  /* initial repcodes */
-    dctx->LLTptr = dctx->LLTable;
-    dctx->MLTptr = dctx->MLTable;
-    dctx->OFTptr = dctx->OFTable;
-    dctx->HUFptr = dctx->hufTable;
+    MEM_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue));
+    memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue));  /* initial repcodes */
+    dctx->LLTptr = dctx->entropy.LLTable;
+    dctx->MLTptr = dctx->entropy.MLTable;
+    dctx->OFTptr = dctx->entropy.OFTable;
+    dctx->HUFptr = dctx->entropy.hufTable;
     return 0;
 }
 
@@ -186,13 +190,13 @@ static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
         dstDCtx->dictID = srcDCtx->dictID;
         dstDCtx->litEntropy = srcDCtx->litEntropy;
         dstDCtx->fseEntropy = srcDCtx->fseEntropy;
-        dstDCtx->LLTptr = srcDCtx->LLTable;
-        dstDCtx->MLTptr = srcDCtx->MLTable;
-        dstDCtx->OFTptr = srcDCtx->OFTable;
-        dstDCtx->HUFptr = srcDCtx->hufTable;
-        dstDCtx->rep[0] = srcDCtx->rep[0];
-        dstDCtx->rep[1] = srcDCtx->rep[1];
-        dstDCtx->rep[2] = srcDCtx->rep[2];
+        dstDCtx->LLTptr = srcDCtx->entropy.LLTable;
+        dstDCtx->MLTptr = srcDCtx->entropy.MLTable;
+        dstDCtx->OFTptr = srcDCtx->entropy.OFTable;
+        dstDCtx->HUFptr = srcDCtx->entropy.hufTable;
+        dstDCtx->entropy.rep[0] = srcDCtx->entropy.rep[0];
+        dstDCtx->entropy.rep[1] = srcDCtx->entropy.rep[1];
+        dstDCtx->entropy.rep[2] = srcDCtx->entropy.rep[2];
     }
 }
 
@@ -506,14 +510,14 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
                                         HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) :
                                         HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) ) :
                                     ( singleStream ?
-                                        HUF_decompress1X2_DCtx(dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) :
-                                        HUF_decompress4X_hufOnly (dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize)) ))
+                                        HUF_decompress1X2_DCtx(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) :
+                                        HUF_decompress4X_hufOnly (dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize)) ))
                     return ERROR(corruption_detected);
 
                 dctx->litPtr = dctx->litBuffer;
                 dctx->litSize = litSize;
                 dctx->litEntropy = 1;
-                if (litEncType==set_compressed) dctx->HUFptr = dctx->hufTable;
+                if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable;
                 memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH);
                 return litCSize + lhSize;
             }
@@ -830,19 +834,19 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
         ip++;
 
         /* Build DTables */
-        {   size_t const llhSize = ZSTD_buildSeqTable(dctx->LLTable, &dctx->LLTptr,
+        {   size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr,
                                                       LLtype, MaxLL, LLFSELog,
                                                       ip, iend-ip, LL_defaultDTable, dctx->fseEntropy);
             if (ZSTD_isError(llhSize)) return ERROR(corruption_detected);
             ip += llhSize;
         }
-        {   size_t const ofhSize = ZSTD_buildSeqTable(dctx->OFTable, &dctx->OFTptr,
+        {   size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr,
                                                       OFtype, MaxOff, OffFSELog,
                                                       ip, iend-ip, OF_defaultDTable, dctx->fseEntropy);
             if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected);
             ip += ofhSize;
         }
-        {   size_t const mlhSize = ZSTD_buildSeqTable(dctx->MLTable, &dctx->MLTptr,
+        {   size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr,
                                                       MLtype, MaxML, MLFSELog,
                                                       ip, iend-ip, ML_defaultDTable, dctx->fseEntropy);
             if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected);
@@ -1104,7 +1108,7 @@ static size_t ZSTD_decompressSequences(
     if (nbSeq) {
         seqState_t seqState;
         dctx->fseEntropy = 1;
-        { U32 i; for (i=0; irep[i]; }
+        { U32 i; for (i=0; ientropy.rep[i]; }
         CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected);
         FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
         FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
@@ -1121,7 +1125,7 @@ static size_t ZSTD_decompressSequences(
         /* check if reached exact end */
         if (nbSeq) return ERROR(corruption_detected);
         /* save reps for next block */
-        { U32 i; for (i=0; irep[i] = (U32)(seqState.prevOffset[i]); }
+        { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); }
     }
 
     /* last literal segment */
@@ -1330,7 +1334,7 @@ static size_t ZSTD_decompressSequencesLong(
         seqState_t seqState;
         int seqNb;
         dctx->fseEntropy = 1;
-        { U32 i; for (i=0; irep[i]; }
+        { U32 i; for (i=0; ientropy.rep[i]; }
         seqState.base = base;
         seqState.pos = (size_t)(op-base);
         seqState.gotoDict = (iPtrDiff)(dictEnd - base);
@@ -1365,7 +1369,7 @@ static size_t ZSTD_decompressSequencesLong(
         }
 
         /* save reps for next block */
-        { U32 i; for (i=0; irep[i] = (U32)(seqState.prevOffset[i]); }
+        { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); }
     }
 
     /* last literal segment */
@@ -1843,7 +1847,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
     if (dictSize <= 8) return ERROR(dictionary_corrupted);
     dictPtr += 8;   /* skip header = magic + dictID */
 
-    {   size_t const hSize = HUF_readDTableX4(dctx->hufTable, dictPtr, dictEnd-dictPtr);
+    {   size_t const hSize = HUF_readDTableX4(dctx->entropy.hufTable, dictPtr, dictEnd-dictPtr);
         if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
         dictPtr += hSize;
     }
@@ -1853,7 +1857,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
         size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
         if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
         if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
-        CHECK_E(FSE_buildDTable(dctx->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted);
+        CHECK_E(FSE_buildDTable(dctx->entropy.OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted);
         dictPtr += offcodeHeaderSize;
     }
 
@@ -1862,7 +1866,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
         size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
         if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
         if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
-        CHECK_E(FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted);
+        CHECK_E(FSE_buildDTable(dctx->entropy.MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted);
         dictPtr += matchlengthHeaderSize;
     }
 
@@ -1871,17 +1875,18 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
         size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
         if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
         if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
-        CHECK_E(FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted);
+        CHECK_E(FSE_buildDTable(dctx->entropy.LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted);
         dictPtr += litlengthHeaderSize;
     }
 
     if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
-    dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] == 0 || dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
-    dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] == 0 || dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
-    dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] == 0 || dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
-    dictPtr += 12;
+    {   int i;
+        for (i=0; i<3; i++) {
+            U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4;
+            if (rep==0 || rep >= dictSize) return ERROR(dictionary_corrupted);
+            dctx->entropy.rep[i] = rep;
+    }   }
 
-    dctx->litEntropy = dctx->fseEntropy = 1;
     return dictPtr - (const BYTE*)dict;
 }
 
@@ -1900,6 +1905,7 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
         dict = (const char*)dict + eSize;
         dictSize -= eSize;
     }
+    dctx->litEntropy = dctx->fseEntropy = 1;
 
     /* reference dictionary content */
     return ZSTD_refDictContent(dctx, dict, dictSize);

From d73eebc00f7bfdc0877974af66dc2945c033f855 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sun, 26 Feb 2017 10:16:42 -0800
Subject: [PATCH 157/223] loadEntropy works on new ZSTD_entropy_t type

---
 lib/decompress/zstd_decompress.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 493d191f1..fc9744002 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1839,7 +1839,7 @@ static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dict
 /* ZSTD_loadEntropy() :
  * dict : must point at beginning of dictionary
  * @return : size of entropy tables read */
-static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t const dictSize)
+static size_t ZSTD_loadEntropy(ZSTD_entropyTables_t* entropy, const void* const dict, size_t const dictSize)
 {
     const BYTE* dictPtr = (const BYTE*)dict;
     const BYTE* const dictEnd = dictPtr + dictSize;
@@ -1847,7 +1847,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
     if (dictSize <= 8) return ERROR(dictionary_corrupted);
     dictPtr += 8;   /* skip header = magic + dictID */
 
-    {   size_t const hSize = HUF_readDTableX4(dctx->entropy.hufTable, dictPtr, dictEnd-dictPtr);
+    {   size_t const hSize = HUF_readDTableX4(entropy->hufTable, dictPtr, dictEnd-dictPtr);
         if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
         dictPtr += hSize;
     }
@@ -1857,7 +1857,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
         size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
         if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
         if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
-        CHECK_E(FSE_buildDTable(dctx->entropy.OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted);
+        CHECK_E(FSE_buildDTable(entropy->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted);
         dictPtr += offcodeHeaderSize;
     }
 
@@ -1866,7 +1866,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
         size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
         if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
         if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
-        CHECK_E(FSE_buildDTable(dctx->entropy.MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted);
+        CHECK_E(FSE_buildDTable(entropy->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted);
         dictPtr += matchlengthHeaderSize;
     }
 
@@ -1875,7 +1875,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
         size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
         if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
         if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
-        CHECK_E(FSE_buildDTable(dctx->entropy.LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted);
+        CHECK_E(FSE_buildDTable(entropy->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted);
         dictPtr += litlengthHeaderSize;
     }
 
@@ -1884,7 +1884,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
         for (i=0; i<3; i++) {
             U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4;
             if (rep==0 || rep >= dictSize) return ERROR(dictionary_corrupted);
-            dctx->entropy.rep[i] = rep;
+            entropy->rep[i] = rep;
     }   }
 
     return dictPtr - (const BYTE*)dict;
@@ -1900,7 +1900,7 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
     dctx->dictID = MEM_readLE32((const char*)dict + 4);
 
     /* load entropy tables */
-    {   size_t const eSize = ZSTD_loadEntropy(dctx, dict, dictSize);
+    {   size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize);
         if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted);
         dict = (const char*)dict + eSize;
         dictSize -= eSize;

From bd7fa21deb289fb252aadb6adb10b635f5d89732 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sun, 26 Feb 2017 14:43:07 -0800
Subject: [PATCH 158/223] added ZSTD_refDDict()

Now DDict does no longer depends on DCtx duplication
---
 lib/decompress/zstd_decompress.c | 79 +++++++++++++++++++++++++-------
 tests/fuzzer.c                   |  2 +-
 tests/playTests.sh               |  7 +--
 3 files changed, 68 insertions(+), 20 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index fc9744002..069c07bf2 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -179,6 +179,8 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
     memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize);  /* no need to copy workspace */
 }
 
+#if 0
+/* deprecated */
 static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
 {
     ZSTD_decompressBegin(dstDCtx);  /* init */
@@ -199,6 +201,9 @@ static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
         dstDCtx->entropy.rep[2] = srcDCtx->entropy.rep[2];
     }
 }
+#endif
+
+static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict);
 
 
 /*-*************************************************************
@@ -1575,7 +1580,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
                                         void* dst, size_t dstCapacity,
                                   const void* src, size_t srcSize,
                                   const void *dict, size_t dictSize,
-                                  const ZSTD_DCtx* refContext)
+                                  const ZSTD_DDict* ddict)
 {
     void* const dststart = dst;
     while (srcSize >= ZSTD_frameHeaderSize_prefix) {
@@ -1619,9 +1624,9 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
             }
         }
 
-        if (refContext) {
+        if (ddict) {
             /* we were called from ZSTD_decompress_usingDDict */
-            ZSTD_refDCtx(dctx, refContext);
+            ZSTD_refDDict(dctx, ddict);
         } else {
             /* this will initialize correctly with no dict if dict == NULL, so
              * use this in all cases but ddict */
@@ -1881,9 +1886,10 @@ static size_t ZSTD_loadEntropy(ZSTD_entropyTables_t* entropy, const void* const
 
     if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
     {   int i;
+        size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12));
         for (i=0; i<3; i++) {
             U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4;
-            if (rep==0 || rep >= dictSize) return ERROR(dictionary_corrupted);
+            if (rep==0 || rep >= dictContentSize) return ERROR(dictionary_corrupted);
             entropy->rep[i] = rep;
     }   }
 
@@ -1926,8 +1932,51 @@ struct ZSTD_DDict_s {
     const void* dictContent;
     size_t dictSize;
     ZSTD_DCtx* refContext;
+    U32 dictID;
+    U32 entropyPresent;
 };  /* typedef'd to ZSTD_DDict within "zstd.h" */
 
+static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict)
+{
+    ZSTD_decompressBegin(dstDCtx);  /* init */
+    if (ddict) {   /* support refDDict on NULL */
+        dstDCtx->dictID = ddict->dictID;
+        dstDCtx->base = ddict->dictContent;
+        dstDCtx->vBase = ddict->dictContent;
+        dstDCtx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize;
+        dstDCtx->previousDstEnd = dstDCtx->dictEnd;
+        if (ddict->entropyPresent) {
+            dstDCtx->litEntropy = 1;
+            dstDCtx->fseEntropy = 1;
+            dstDCtx->LLTptr = ddict->refContext->entropy.LLTable;
+            dstDCtx->MLTptr = ddict->refContext->entropy.MLTable;
+            dstDCtx->OFTptr = ddict->refContext->entropy.OFTable;
+            dstDCtx->HUFptr = ddict->refContext->entropy.hufTable;
+            dstDCtx->entropy.rep[0] = ddict->refContext->entropy.rep[0];
+            dstDCtx->entropy.rep[1] = ddict->refContext->entropy.rep[1];
+            dstDCtx->entropy.rep[2] = ddict->refContext->entropy.rep[2];
+        } else {
+            dstDCtx->litEntropy = 0;
+            dstDCtx->fseEntropy = 0;
+        }
+    }
+}
+
+static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict)
+{
+    ddict->entropyPresent = 0;
+    if (ddict->dictSize < 8) return 0;
+    {   U32 const magic = MEM_readLE32(ddict->dictContent);
+        if (magic != ZSTD_DICT_MAGIC) return 0;   /* pure content mode */
+    }
+    ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + 4);
+
+    /* load entropy tables */
+    CHECK_E( ZSTD_loadEntropy(&ddict->refContext->entropy, ddict->dictContent, ddict->dictSize), dictionary_corrupted );
+    ddict->entropyPresent = 1;
+    return 0;
+}
+
 ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem)
 {
     if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
@@ -1953,22 +2002,22 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, unsigne
             ddict->dictBuffer = internalBuffer;
             ddict->dictContent = internalBuffer;
         }
+        ddict->dictSize = dictSize;
         /* parse dictionary content */
-        {   //size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, ddict->dictContent, dictSize);
-            size_t const errorCode = ZSTD_decompress_insertDictionary(dctx, ddict->dictContent, dictSize);
+        {   size_t const errorCode = ZSTD_loadEntropy_inDDict(ddict);
             if (ZSTD_isError(errorCode)) {
                 ZSTD_freeDDict(ddict);
                 return NULL;
         }   }
 
-        ddict->dictSize = dictSize;
         return ddict;
     }
 }
 
 /*! ZSTD_createDDict() :
-*   Create a digested dictionary, ready to start decompression without startup delay.
-*   `dict` can be released after `ZSTD_DDict` creation */
+*   Create a digested dictionary, to start decompression without startup delay.
+*   `dict` content is copied inside DDict.
+*   Consequently, `dict` can be released after `ZSTD_DDict` creation */
 ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
 {
     ZSTD_customMem const allocator = { NULL, NULL, NULL };
@@ -1977,9 +2026,9 @@ ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
 
 
 /*! ZSTD_createDDict_byReference() :
- *  Create a digested dictionary, ready to start decompression operation without startup delay.
- *  Dictionary content is simply referenced, and therefore stays in dictBuffer.
- *  It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict */
+ *  Create a digested dictionary, to start decompression without startup delay.
+ *  Dictionary content is simply referenced, it will be accessed during decompression.
+ *  Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */
 ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
 {
     ZSTD_customMem const allocator = { NULL, NULL, NULL };
@@ -2055,7 +2104,7 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
     /* pass content and size in case legacy frames are encountered */
     return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize,
                                      ddict->dictContent, ddict->dictSize,
-                                     ddict->refContext);
+                                     ddict);
 }
 
 
@@ -2256,9 +2305,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             }   }
 
             /* Consume header */
-            {   const ZSTD_DCtx* refContext = zds->ddict ? zds->ddict->refContext : NULL;
-                ZSTD_refDCtx(zds->dctx, refContext);
-            }
+            ZSTD_refDDict(zds->dctx, zds->ddict);
             {   size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx);  /* == ZSTD_frameHeaderSize_prefix */
                 CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size));
                 {   size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx);
diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index d38ac6f83..79516b6cc 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -257,7 +257,7 @@ static int basicUnitTests(U32 seed, double compressibility)
         DISPLAYLEVEL(4, "OK \n");
 
         DISPLAYLEVEL(4, "test%3i : decompress with DDict : ", testNb++);
-        {   ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, dictSize);
+        {   ZSTD_DDict* const ddict = ZSTD_createDDict_byReference(CNBuffer, dictSize);
             size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
             if (r != CNBuffSize - dictSize) goto _output_error;
             DISPLAYLEVEL(4, "OK (size of DDict : %u) \n", (U32)ZSTD_sizeof_DDict(ddict));
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 35731f9cf..46ac39a58 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -209,18 +209,19 @@ $ZSTD -f tmp1 notHere tmp2 && die "missing file not detected!"
 
 $ECHO "\n**** dictionary tests **** "
 
-TESTFILE=../programs/zstdcli.c
+$ECHO "- test with raw dict (content only) "
 ./datagen > tmpDict
 ./datagen -g1M | $MD5SUM > tmp1
 ./datagen -g1M | $ZSTD -D tmpDict | $ZSTD -D tmpDict -dvq | $MD5SUM > tmp2
 $DIFF -q tmp1 tmp2
-$ECHO "- Create first dictionary"
+$ECHO "- Create first dictionary "
+TESTFILE=../programs/zstdcli.c
 $ZSTD --train *.c ../programs/*.c -o tmpDict
 cp $TESTFILE tmp
 $ZSTD -f tmp -D tmpDict
 $ZSTD -d tmp.zst -D tmpDict -fo result
 $DIFF $TESTFILE result
-$ECHO "- Create second (different) dictionary"
+$ECHO "- Create second (different) dictionary "
 $ZSTD --train *.c ../programs/*.c ../programs/*.h -o tmpDictC
 $ZSTD -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
 $ECHO "- Create dictionary with short dictID"

From b43d75154da410a4521924c2b0580a3b48c1b312 Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Mon, 27 Feb 2017 09:07:35 +0100
Subject: [PATCH 159/223] update gzip tests

---
 tests/gzip/Makefile  | 3 +--
 tests/gzip/list2.log | 7 -------
 2 files changed, 1 insertion(+), 9 deletions(-)
 delete mode 100644 tests/gzip/list2.log

diff --git a/tests/gzip/Makefile b/tests/gzip/Makefile
index 5d5480444..f87682c7f 100644
--- a/tests/gzip/Makefile
+++ b/tests/gzip/Makefile
@@ -36,7 +36,6 @@ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly
 #.PHONY: test-gzip-env test-helin-segv test-hufts test-keep test-list test-memcpy-abuse test-mixed test-null-suffix-clobber test-stdin test-trailing-nul test-unpack-invalid test-zdiff test-zgrep-context test-zgrep-f test-zgrep-signal test-znew-k test-z-suffix 
 
 test-%: zstd
-	@./test-driver.sh --test-name $* --log-file $*.log --trs-file $*.trs --expect-failure "no" --color-tests "yes" --enable-hard-errors "yes" ./$* || echo error
-#|| exit 1
+	@./test-driver.sh --test-name $* --log-file $*.log --trs-file $*.trs --expect-failure "no" --color-tests "yes" --enable-hard-errors "yes" ./$*.sh
 
 endif
diff --git a/tests/gzip/list2.log b/tests/gzip/list2.log
deleted file mode 100644
index 86e3a37f4..000000000
--- a/tests/gzip/list2.log
+++ /dev/null
@@ -1,7 +0,0 @@
-gzip -l in
-
-gzip: in: not in gzip format
-gzip -9 in
-gzip -l in.gz
-gzip -l in.gz
-PASS list (exit status: 0)

From 0b9b894b2d1adeec658a8c7d3c6b74f1201297b9 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 27 Feb 2017 00:27:30 -0800
Subject: [PATCH 160/223] reduced ZSTD_DDict memory usage

saved 128 KB
---
 lib/common/fse_decompress.c      |  1 -
 lib/decompress/huf_decompress.c  |  2 +-
 lib/decompress/zstd_decompress.c | 39 +++++++++++++++-----------------
 programs/fileio.c                |  2 +-
 4 files changed, 20 insertions(+), 24 deletions(-)

diff --git a/lib/common/fse_decompress.c b/lib/common/fse_decompress.c
index 1479a5e82..8474a4c07 100644
--- a/lib/common/fse_decompress.c
+++ b/lib/common/fse_decompress.c
@@ -59,7 +59,6 @@
 ****************************************************************/
 #include      /* malloc, free, qsort */
 #include      /* memcpy, memset */
-#include       /* printf (debug) */
 #include "bitstream.h"
 #define FSE_STATIC_LINKING_ONLY
 #include "fse.h"
diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c
index 889a22a8c..0f11f5c1b 100644
--- a/lib/decompress/huf_decompress.c
+++ b/lib/decompress/huf_decompress.c
@@ -459,7 +459,7 @@ size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
     void* dtPtr = DTable+1;   /* force compiler to avoid strict-aliasing */
     HUF_DEltX4* const dt = (HUF_DEltX4*)dtPtr;
 
-    HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable));   /* if compilation fails here, assertion is false */
+    HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable));   /* if compiler fails here, assertion is wrong */
     if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
     /* memset(weightList, 0, sizeof(weightList)); */  /* is not necessary, even though some analyzer complain ... */
 
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 069c07bf2..2646c8028 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1842,7 +1842,7 @@ static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dict
 }
 
 /* ZSTD_loadEntropy() :
- * dict : must point at beginning of dictionary
+ * dict : must point at beginning of a valid zstd dictionary
  * @return : size of entropy tables read */
 static size_t ZSTD_loadEntropy(ZSTD_entropyTables_t* entropy, const void* const dict, size_t const dictSize)
 {
@@ -1852,6 +1852,7 @@ static size_t ZSTD_loadEntropy(ZSTD_entropyTables_t* entropy, const void* const
     if (dictSize <= 8) return ERROR(dictionary_corrupted);
     dictPtr += 8;   /* skip header = magic + dictID */
 
+
     {   size_t const hSize = HUF_readDTableX4(entropy->hufTable, dictPtr, dictEnd-dictPtr);
         if (HUF_isError(hSize)) return ERROR(dictionary_corrupted);
         dictPtr += hSize;
@@ -1931,9 +1932,10 @@ struct ZSTD_DDict_s {
     void* dictBuffer;
     const void* dictContent;
     size_t dictSize;
-    ZSTD_DCtx* refContext;
+    ZSTD_entropyTables_t entropy;
     U32 dictID;
     U32 entropyPresent;
+    ZSTD_customMem cMem;
 };  /* typedef'd to ZSTD_DDict within "zstd.h" */
 
 static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict)
@@ -1948,13 +1950,13 @@ static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict)
         if (ddict->entropyPresent) {
             dstDCtx->litEntropy = 1;
             dstDCtx->fseEntropy = 1;
-            dstDCtx->LLTptr = ddict->refContext->entropy.LLTable;
-            dstDCtx->MLTptr = ddict->refContext->entropy.MLTable;
-            dstDCtx->OFTptr = ddict->refContext->entropy.OFTable;
-            dstDCtx->HUFptr = ddict->refContext->entropy.hufTable;
-            dstDCtx->entropy.rep[0] = ddict->refContext->entropy.rep[0];
-            dstDCtx->entropy.rep[1] = ddict->refContext->entropy.rep[1];
-            dstDCtx->entropy.rep[2] = ddict->refContext->entropy.rep[2];
+            dstDCtx->LLTptr = ddict->entropy.LLTable;
+            dstDCtx->MLTptr = ddict->entropy.MLTable;
+            dstDCtx->OFTptr = ddict->entropy.OFTable;
+            dstDCtx->HUFptr = ddict->entropy.hufTable;
+            dstDCtx->entropy.rep[0] = ddict->entropy.rep[0];
+            dstDCtx->entropy.rep[1] = ddict->entropy.rep[1];
+            dstDCtx->entropy.rep[2] = ddict->entropy.rep[2];
         } else {
             dstDCtx->litEntropy = 0;
             dstDCtx->fseEntropy = 0;
@@ -1972,25 +1974,20 @@ static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict)
     ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + 4);
 
     /* load entropy tables */
-    CHECK_E( ZSTD_loadEntropy(&ddict->refContext->entropy, ddict->dictContent, ddict->dictSize), dictionary_corrupted );
+    CHECK_E( ZSTD_loadEntropy(&ddict->entropy, ddict->dictContent, ddict->dictSize), dictionary_corrupted );
     ddict->entropyPresent = 1;
     return 0;
 }
 
+
 ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem)
 {
     if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
     if (!customMem.customAlloc || !customMem.customFree) return NULL;
 
     {   ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
-        ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(customMem);
-
-        if (!ddict || !dctx) {
-            ZSTD_free(ddict, customMem);
-            ZSTD_free(dctx, customMem);
-            return NULL;
-        }
-        ddict->refContext = dctx;
+        if (!ddict) return NULL;
+        ddict->cMem = customMem;
 
         if ((byReference) || (!dict) || (!dictSize)) {
             ddict->dictBuffer = NULL;
@@ -2003,6 +2000,7 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, unsigne
             ddict->dictContent = internalBuffer;
         }
         ddict->dictSize = dictSize;
+        ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001);  /* cover both little and big endian */
         /* parse dictionary content */
         {   size_t const errorCode = ZSTD_loadEntropy_inDDict(ddict);
             if (ZSTD_isError(errorCode)) {
@@ -2039,8 +2037,7 @@ ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize
 size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
 {
     if (ddict==NULL) return 0;   /* support free on NULL */
-    {   ZSTD_customMem const cMem = ddict->refContext->customMem;
-        ZSTD_freeDCtx(ddict->refContext);
+    {   ZSTD_customMem const cMem = ddict->cMem;
         ZSTD_free(ddict->dictBuffer, cMem);
         ZSTD_free(ddict, cMem);
         return 0;
@@ -2050,7 +2047,7 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
 size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
 {
     if (ddict==NULL) return 0;   /* support sizeof on NULL */
-    return sizeof(*ddict) + ZSTD_sizeof_DCtx(ddict->refContext) + (ddict->dictBuffer ? ddict->dictSize : 0) ;
+    return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ;
 }
 
 /*! ZSTD_getDictID_fromDict() :
diff --git a/programs/fileio.c b/programs/fileio.c
index 3cbd83ae0..3b9d51a32 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -905,7 +905,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
     }
 
     srcFile = FIO_openSrcFile(srcFileName);
-    if (srcFile==0) return 1;
+    if (srcFile==NULL) return 1;
 
     /* for each frame */
     for ( ; ; ) {

From 013f8b4c27802bdb1995ce6c91bde739a877a7bf Mon Sep 17 00:00:00 2001
From: Prashant Khandelwal 
Date: Mon, 27 Feb 2017 16:28:22 +0530
Subject: [PATCH 161/223] Fix for a small Typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 229571b3e..ce27056bc 100644
--- a/README.md
+++ b/README.md
@@ -108,7 +108,7 @@ to create `zstd` binary, and `libzstd` dynamic and static libraries.
 
 A Meson project is provided within `contrib/meson`.
 
-#### Visual (Windows)
+#### Visual Studio (Windows)
 
 Going into `build` directory, you will find additional possibilities :
 - Projects for Visual Studio 2005, 2008 and 2010

From 862698f4792007e6c4ff4aa430f0881924184510 Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Mon, 27 Feb 2017 13:21:05 +0100
Subject: [PATCH 162/223] minor tweaks in FIO_decompressGzFrame

---
 programs/fileio.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 3cbd83ae0..ede6e5516 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -385,7 +385,7 @@ static unsigned long long FIO_compressGzFrame(cRess_t* ress, const char* srcFile
     if (ret != Z_OK) EXM_THROW(71, "zstd: %s: deflateInit2 error %d \n", srcFileName, ret);
 
     strm.next_in = 0;
-    strm.avail_in = Z_NULL;
+    strm.avail_in = 0;
     strm.next_out = (Bytef*)ress->dstBuffer;
     strm.avail_out = (uInt)ress->dstBufferSize;
 
@@ -846,12 +846,13 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, co
 {
     unsigned long long outFileSize = 0;
     z_stream strm;
+    int ret;
 
     strm.zalloc = Z_NULL;
     strm.zfree = Z_NULL;
     strm.opaque = Z_NULL;
     strm.next_in = 0;
-    strm.avail_in = Z_NULL;
+    strm.avail_in = 0;
     if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) return 0;  /* see http://www.zlib.net/manual.html */
 
     strm.next_out = (Bytef*)ress->dstBuffer;
@@ -860,7 +861,6 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, co
     strm.next_in = (z_const unsigned char*)ress->srcBuffer;
 
     for ( ; ; ) {
-        int ret;
         if (strm.avail_in == 0) {
             ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile);
             if (ress->srcBufferLoaded == 0) break;
@@ -882,7 +882,8 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, co
 
     if (strm.avail_in > 0) memmove(ress->srcBuffer, strm.next_in, strm.avail_in);
     ress->srcBufferLoaded = strm.avail_in;
-    inflateEnd(&strm);
+    ret = inflateEnd(&strm);
+    if (ret != Z_OK) EXM_THROW(32, "zstd: %s: inflateEnd error %d \n", srcFileName, ret);
     return outFileSize;
 }
 #endif

From 5d848527e6770d715d1de12cc279d7f7f35e9dfe Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Mon, 27 Feb 2017 22:01:03 +0100
Subject: [PATCH 163/223] use "./gzip" for gzip tests

---
 tests/gzip/Makefile               | 10 +++++-----
 tests/gzip/gzip-env.sh            |  8 +++++++-
 tests/gzip/helin-segv.sh          |  2 +-
 tests/gzip/help-version.sh        |  2 +-
 tests/gzip/hufts.sh               |  2 +-
 tests/gzip/keep.sh                |  2 +-
 tests/gzip/list.sh                |  2 +-
 tests/gzip/memcpy-abuse.sh        |  2 +-
 tests/gzip/mixed.sh               |  2 +-
 tests/gzip/null-suffix-clobber.sh |  2 +-
 tests/gzip/stdin.sh               |  2 +-
 tests/gzip/trailing-nul.sh        |  2 +-
 tests/gzip/unpack-invalid.sh      |  2 +-
 tests/gzip/z-suffix.sh            |  2 +-
 tests/gzip/zdiff.sh               |  2 +-
 tests/gzip/zgrep-context.sh       |  2 +-
 tests/gzip/zgrep-f.sh             |  2 +-
 tests/gzip/zgrep-signal.sh        |  2 +-
 tests/gzip/znew-k.sh              |  2 +-
 19 files changed, 29 insertions(+), 23 deletions(-)

diff --git a/tests/gzip/Makefile b/tests/gzip/Makefile
index f87682c7f..ee993934a 100644
--- a/tests/gzip/Makefile
+++ b/tests/gzip/Makefile
@@ -9,7 +9,7 @@
 
 PRGDIR = ../../programs
 VOID   = /dev/null
-
+export PATH := .:$(PATH)
 
 .PHONY: all
 all: test-gzip-env test-helin-segv test-hufts test-keep test-list test-memcpy-abuse test-mixed test-null-suffix-clobber test-stdin test-trailing-nul test-unpack-invalid test-zdiff test-zgrep-context test-zgrep-f test-zgrep-signal test-znew-k test-z-suffix 
@@ -18,8 +18,8 @@ all: test-gzip-env test-helin-segv test-hufts test-keep test-list test-memcpy-ab
 .PHONY: zstd
 zstd:
 	$(MAKE) -C $(PRGDIR) zstd
-	#alias gzip='$(PRGDIR)/zstd --format=gzip'
-	#ln -sf /drive_d/GitHub/zstd/programs/zstd /usr/local/bin/gzip
+	ln -sf $(PRGDIR)/zstd gzip
+	@echo PATH=$(PATH)
 	gzip --version
 
 .PHONY: clean
@@ -33,9 +33,9 @@ clean:
 # validated only for Linux, OSX, Hurd and some BSD targets
 #------------------------------------------------------------------------------
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD))
-#.PHONY: test-gzip-env test-helin-segv test-hufts test-keep test-list test-memcpy-abuse test-mixed test-null-suffix-clobber test-stdin test-trailing-nul test-unpack-invalid test-zdiff test-zgrep-context test-zgrep-f test-zgrep-signal test-znew-k test-z-suffix 
 
 test-%: zstd
-	@./test-driver.sh --test-name $* --log-file $*.log --trs-file $*.trs --expect-failure "no" --color-tests "yes" --enable-hard-errors "yes" ./$*.sh
+	@./test-driver.sh --test-name $* --log-file $*.log --trs-file $*.trs --expect-failure "no" --color-tests "yes" --enable-hard-errors "yes" ./$*.sh 
+	# || echo ignoring error
 
 endif
diff --git a/tests/gzip/gzip-env.sh b/tests/gzip/gzip-env.sh
index 873d716dc..f15a8d476 100755
--- a/tests/gzip/gzip-env.sh
+++ b/tests/gzip/gzip-env.sh
@@ -17,7 +17,13 @@
 # along with this program.  If not, see .
 # limit so don't run it by default.
 
-. "${srcdir=.}/init.sh"; path_prepend_ ..
+echo XXX=$PATH
+gzip --version
+
+. "${srcdir=.}/init.sh"; path_prepend_ .
+
+echo XXX=$PATH
+gzip --version
 
 echo a >exp || framework_failure_
 gzip in || framework_failure_
diff --git a/tests/gzip/helin-segv.sh b/tests/gzip/helin-segv.sh
index 0182db33d..f182c8066 100644
--- a/tests/gzip/helin-segv.sh
+++ b/tests/gzip/helin-segv.sh
@@ -17,7 +17,7 @@
 # along with this program.  If not, see .
 # limit so don't run it by default.
 
-. "${srcdir=.}/init.sh"; path_prepend_ ..
+. "${srcdir=.}/init.sh"; path_prepend_ .
 
 # This test case was provided by Aki Helin.
 printf '\037\235\220\0\0\0\304' > helin.gz || framework_failure_
diff --git a/tests/gzip/help-version.sh b/tests/gzip/help-version.sh
index 5af9a0509..ee0c19f7d 100644
--- a/tests/gzip/help-version.sh
+++ b/tests/gzip/help-version.sh
@@ -23,7 +23,7 @@
 test "x$SHELL" = x && SHELL=/bin/sh
 export SHELL
 
-. "${srcdir=.}/init.sh"; path_prepend_ ..
+. "${srcdir=.}/init.sh"; path_prepend_ .
 
 expected_failure_status_chroot=125
 expected_failure_status_env=125
diff --git a/tests/gzip/hufts.sh b/tests/gzip/hufts.sh
index 5832a2184..9b9576ce3 100644
--- a/tests/gzip/hufts.sh
+++ b/tests/gzip/hufts.sh
@@ -17,7 +17,7 @@
 # along with this program.  If not, see .
 # limit so don't run it by default.
 
-. "${srcdir=.}/init.sh"; path_prepend_ ..
+. "${srcdir=.}/init.sh"; path_prepend_ .
 
 printf '\n...: invalid compressed data--format violated\n' > exp \
   || framework_failure_
diff --git a/tests/gzip/keep.sh b/tests/gzip/keep.sh
index 105d43e64..ab9a21811 100644
--- a/tests/gzip/keep.sh
+++ b/tests/gzip/keep.sh
@@ -17,7 +17,7 @@
 # along with this program.  If not, see .
 # limit so don't run it by default.
 
-. "${srcdir=.}/init.sh"; path_prepend_ ..
+. "${srcdir=.}/init.sh"; path_prepend_ .
 
 echo fooooooooo > in || framework_failure_
 cp in orig || framework_failure_
diff --git a/tests/gzip/list.sh b/tests/gzip/list.sh
index 7576dc3a8..75912e1e2 100644
--- a/tests/gzip/list.sh
+++ b/tests/gzip/list.sh
@@ -17,7 +17,7 @@
 # along with this program.  If not, see .
 # limit so don't run it by default.
 
-. "${srcdir=.}/init.sh"; path_prepend_ ..
+. "${srcdir=.}/init.sh"; path_prepend_ .
 
 echo zoology zucchini > in || framework_failure_
 cp in orig || framework_failure_
diff --git a/tests/gzip/memcpy-abuse.sh b/tests/gzip/memcpy-abuse.sh
index 970b88523..7d5c056de 100644
--- a/tests/gzip/memcpy-abuse.sh
+++ b/tests/gzip/memcpy-abuse.sh
@@ -18,7 +18,7 @@
 # along with this program.  If not, see .
 # limit so don't run it by default.
 
-. "${srcdir=.}/init.sh"; path_prepend_ ..
+. "${srcdir=.}/init.sh"; path_prepend_ .
 
 # The input must be larger than 32KiB and slightly
 # less uniform than e.g., all zeros.
diff --git a/tests/gzip/mixed.sh b/tests/gzip/mixed.sh
index 50e2537a2..383a54f5e 100644
--- a/tests/gzip/mixed.sh
+++ b/tests/gzip/mixed.sh
@@ -18,7 +18,7 @@
 # along with this program.  If not, see .
 # limit so don't run it by default.
 
-. "${srcdir=.}/init.sh"; path_prepend_ ..
+. "${srcdir=.}/init.sh"; path_prepend_ .
 
 printf 'xxx\nyyy\n'      > exp2 || framework_failure_
 printf 'aaa\nbbb\nccc\n' > exp3 || framework_failure_
diff --git a/tests/gzip/null-suffix-clobber.sh b/tests/gzip/null-suffix-clobber.sh
index 903864ce6..0efd0e344 100644
--- a/tests/gzip/null-suffix-clobber.sh
+++ b/tests/gzip/null-suffix-clobber.sh
@@ -17,7 +17,7 @@
 # along with this program.  If not, see .
 # limit so don't run it by default.
 
-. "${srcdir=.}/init.sh"; path_prepend_ ..
+. "${srcdir=.}/init.sh"; path_prepend_ .
 
 printf anything | gzip > F.gz || framework_failure_
 echo y > yes || framework_failure_
diff --git a/tests/gzip/stdin.sh b/tests/gzip/stdin.sh
index 190687f96..eef4cd8b1 100644
--- a/tests/gzip/stdin.sh
+++ b/tests/gzip/stdin.sh
@@ -17,7 +17,7 @@
 # along with this program.  If not, see .
 # limit so don't run it by default.
 
-. "${srcdir=.}/init.sh"; path_prepend_ ..
+. "${srcdir=.}/init.sh"; path_prepend_ .
 
 printf a | gzip > in || framework_failure_
 printf aaa > exp || framework_failure_
diff --git a/tests/gzip/trailing-nul.sh b/tests/gzip/trailing-nul.sh
index b21f76fdc..7b15d5e55 100644
--- a/tests/gzip/trailing-nul.sh
+++ b/tests/gzip/trailing-nul.sh
@@ -18,7 +18,7 @@
 # along with this program.  If not, see .
 # limit so don't run it by default.
 
-. "${srcdir=.}/init.sh"; path_prepend_ ..
+. "${srcdir=.}/init.sh"; path_prepend_ .
 
 (echo 0 | gzip; printf '\0') > 0.gz || framework_failure_
 (echo 00 | gzip; printf '\0\0') > 00.gz || framework_failure_
diff --git a/tests/gzip/unpack-invalid.sh b/tests/gzip/unpack-invalid.sh
index acea97e83..fe8384d73 100644
--- a/tests/gzip/unpack-invalid.sh
+++ b/tests/gzip/unpack-invalid.sh
@@ -18,7 +18,7 @@
 # along with this program.  If not, see .
 # limit so don't run it by default.
 
-. "${srcdir=.}/init.sh"; path_prepend_ ..
+. "${srcdir=.}/init.sh"; path_prepend_ .
 
 for input in \
   '\037\036\000\000\037\213\010\000\000\000\000\000\002\003\036\000\000\000\002\003\037\213\010\000\000\000\000\000\002\003\355\301\001\015\000\000\000\302\240\037\000\302\240\037\213\010\000\000\000\000\000\002\003\355\301' \
diff --git a/tests/gzip/z-suffix.sh b/tests/gzip/z-suffix.sh
index 470932060..a870a5408 100644
--- a/tests/gzip/z-suffix.sh
+++ b/tests/gzip/z-suffix.sh
@@ -17,7 +17,7 @@
 # along with this program.  If not, see .
 # limit so don't run it by default.
 
-. "${srcdir=.}/init.sh"; path_prepend_ ..
+. "${srcdir=.}/init.sh"; path_prepend_ .
 
 printf anything > F && cp F G || framework_failure_
 gzip -Sz F || fail=1
diff --git a/tests/gzip/zdiff.sh b/tests/gzip/zdiff.sh
index 0bb7c7dfd..d62a84606 100644
--- a/tests/gzip/zdiff.sh
+++ b/tests/gzip/zdiff.sh
@@ -18,7 +18,7 @@
 # along with this program.  If not, see .
 # limit so don't run it by default.
 
-. "${srcdir=.}/init.sh"; path_prepend_ ..
+. "${srcdir=.}/init.sh"; path_prepend_ .
 
 echo a > a || framework_failure_
 echo b > b || framework_failure_
diff --git a/tests/gzip/zgrep-context.sh b/tests/gzip/zgrep-context.sh
index 089d25639..c8648b7e4 100644
--- a/tests/gzip/zgrep-context.sh
+++ b/tests/gzip/zgrep-context.sh
@@ -17,7 +17,7 @@
 # along with this program.  If not, see .
 # limit so don't run it by default.
 
-. "${srcdir=.}/init.sh"; path_prepend_ ..
+. "${srcdir=.}/init.sh"; path_prepend_ .
 
 # A limited replacement for seq: handle 1 or 2 args; increment must be 1
 seq()
diff --git a/tests/gzip/zgrep-f.sh b/tests/gzip/zgrep-f.sh
index 1ce8cc293..d0cf27f7e 100644
--- a/tests/gzip/zgrep-f.sh
+++ b/tests/gzip/zgrep-f.sh
@@ -18,7 +18,7 @@
 # along with this program.  If not, see .
 # limit so don't run it by default.
 
-. "${srcdir=.}/init.sh"; path_prepend_ ..
+. "${srcdir=.}/init.sh"; path_prepend_ .
 
 printf 'needle\nn2\n' > n || framework_failure_
 cp n haystack || framework_failure_
diff --git a/tests/gzip/zgrep-signal.sh b/tests/gzip/zgrep-signal.sh
index 13783ef10..a8c53881a 100644
--- a/tests/gzip/zgrep-signal.sh
+++ b/tests/gzip/zgrep-signal.sh
@@ -18,7 +18,7 @@
 # along with this program.  If not, see .
 # limit so don't run it by default.
 
-. "${srcdir=.}/init.sh"; path_prepend_ ..
+. "${srcdir=.}/init.sh"; path_prepend_ .
 
 echo a | gzip -c > f.gz || framework_failure_
 
diff --git a/tests/gzip/znew-k.sh b/tests/gzip/znew-k.sh
index 98e7fa1f8..6c239e28e 100644
--- a/tests/gzip/znew-k.sh
+++ b/tests/gzip/znew-k.sh
@@ -17,7 +17,7 @@
 # along with this program.  If not, see .
 # limit so don't run it by default.
 
-. "${srcdir=.}/init.sh"; path_prepend_ ..
+. "${srcdir=.}/init.sh"; path_prepend_ .
 
 cat <<'EOF' >compress || framework_failure_
 #!/bin/sh

From dccd6b6f653155d6f08d20b64ca6fcbe5a0052fa Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 27 Feb 2017 15:57:50 -0800
Subject: [PATCH 164/223] cli : fix : --rm is silent when input is stdin

previously, app would produce an error message, and stop.
---
 NEWS                         | 1 +
 lib/compress/zstd_compress.c | 2 +-
 programs/fileio.c            | 5 ++++-
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 96ff25fd7..7ede5f70c 100644
--- a/NEWS
+++ b/NEWS
@@ -2,6 +2,7 @@ v1.1.4
 cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski
 cli : new : advanced benchmark command --priority=rt
 cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77
+cli : fix : --rm remains silent when input is stdin
 API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize(), by Sean Purcell
 API : change : ZSTD_compress*() with srcSize==0 create an empty-frame of known size
 build:new : meson build system in contrib/meson, by Dima Krasner
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index d684e6a0d..ec758db00 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -248,7 +248,7 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 fra
 typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_compResetPolicy_e;
 
 /*! ZSTD_resetCCtx_advanced() :
-    note : 'params' must be validated */
+    note : @params must be validated */
 static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
                                        ZSTD_parameters params, U64 frameContentSize,
                                        ZSTD_compResetPolicy_e const crp)
diff --git a/programs/fileio.c b/programs/fileio.c
index 3cbd83ae0..bc202cb24 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -549,7 +549,10 @@ static int FIO_compressFilename_srcFile(cRess_t ress,
     result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, compressionLevel);
 
     fclose(ress.srcFile);
-    if (g_removeSrcFile && !result) { if (remove(srcFileName)) EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno)); } /* remove source file : --rm */
+    if (g_removeSrcFile /* --rm */ && !result && strcmp(srcFileName, stdinmark)) {
+        if (remove(srcFileName))
+            EXM_THROW(1, "zstd: %s: %s", srcFileName, strerror(errno));
+    }
     return result;
 }
 

From 67d86a74a56033d4b5b568c9fddad13afc71af0b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 27 Feb 2017 16:09:20 -0800
Subject: [PATCH 165/223] added test case : --rm on stdin

must remain silent (instead of failing)
---
 tests/playTests.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/playTests.sh b/tests/playTests.sh
index 35731f9cf..c465fee3c 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -101,6 +101,8 @@ $ZSTD -f --rm tmp
 ls tmp && die "tmp should no longer be present"
 $ZSTD -f -d --rm tmp.zst
 ls tmp.zst && die "tmp.zst should no longer be present"
+$ECHO "test : --rm on stdin"
+$ECHO a | $ZSTD --rm > $INTOVOID   # --rm should remain silent
 rm tmp
 $ZSTD -f tmp && die "tmp not present : should have failed"
 ls tmp.zst && die "tmp.zst should not be created"

From 952d06fa9ce0c1fc364b1be93a1aab0abfb6b612 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 27 Feb 2017 17:58:02 -0800
Subject: [PATCH 166/223] fullbench : -i0 displays list of functions to bench

---
 tests/fullbench.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/fullbench.c b/tests/fullbench.c
index 5c7e2cc32..940d315a7 100644
--- a/tests/fullbench.c
+++ b/tests/fullbench.c
@@ -24,7 +24,7 @@
     #define KB *(1 <<10)
     #define MB *(1 <<20)
     #define GB *(1U<<30)
-    typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; 
+    typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
 #endif
 #include "zstd.h"            /* ZSTD_VERSION_STRING */
 #include "datagen.h"
@@ -359,6 +359,7 @@ static size_t benchMem(const void* src, size_t srcSize, U32 benchNb)
     { size_t i; for (i=0; i
Date: Tue, 28 Feb 2017 08:16:49 +0100
Subject: [PATCH 167/223] updated .travis.yml

---
 .travis.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 3c77a17e1..8399bd8a0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,7 +4,7 @@ dist: trusty
 matrix:
   fast_finish: true
   include:
-    # other feature branches => short tests
+    # Ubuntu 14.04
     - env: Cmd="make libc6install && make -C tests test32"
     - env: Cmd='make valgrindinstall arminstall ppcinstall arm-ppc-compilation && make clean lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest'
 
@@ -25,10 +25,11 @@ script:
   #  cron & master          => full tests, as this is the final step towards a Release
   #  pull requests          => normal tests (job numbers 1-3)
   #  other feature branches => short tests (job numbers 1-2)
+  - @echo JOB_NUMBER=$JOB_NUMBER TRAVIS_BRANCH=$TRAVIS_BRANCH TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST
   - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "master" ]; then
         FUZZERTEST=-T7mn sh -c "$Cmd" || travis_terminate 1;
     else
-        if [ "$TRAVIS_PULL_REQUEST" == "true" ] && [ $JOB_NUMBER -lt 4 ]; then
+        if [ "$TRAVIS_PULL_REQUEST" = "true" ] && [ $JOB_NUMBER -lt 4 ]; then
             sh -c "$Cmd" || travis_terminate 1;
         else
             if [ $JOB_NUMBER -lt 3 ]; then

From a3352d06bc51ed76d7e4ca592c878918df93eabd Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Tue, 28 Feb 2017 08:20:53 +0100
Subject: [PATCH 168/223] updated .travis.yml (2)

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 8399bd8a0..7d19e36c9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -25,7 +25,7 @@ script:
   #  cron & master          => full tests, as this is the final step towards a Release
   #  pull requests          => normal tests (job numbers 1-3)
   #  other feature branches => short tests (job numbers 1-2)
-  - @echo JOB_NUMBER=$JOB_NUMBER TRAVIS_BRANCH=$TRAVIS_BRANCH TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST
+  - echo JOB_NUMBER=$JOB_NUMBER TRAVIS_BRANCH=$TRAVIS_BRANCH TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST
   - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "master" ]; then
         FUZZERTEST=-T7mn sh -c "$Cmd" || travis_terminate 1;
     else

From d1760113eca6244d40603997467a22423e2e1a5b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 28 Feb 2017 00:14:28 -0800
Subject: [PATCH 169/223] Improved speed of ZSTD_decompressStream()

When ZSTD_decompressStream() detects
that there is enough space in dst
to complete decompression in a single pass,
delegates to ZSTD_decompress(),
for an extra ~5% speed boost
---
 lib/decompress/zstd_decompress.c | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 2646c8028..305a9a876 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -2100,7 +2100,7 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
 {
     /* pass content and size in case legacy frames are encountered */
     return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize,
-                                     ddict->dictContent, ddict->dictSize,
+                                     NULL, 0,
                                      ddict);
 }
 
@@ -2301,6 +2301,20 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                     break;
             }   }
 
+            /* check for single-pass mode opportunity */
+            if (zds->fParams.frameContentSize
+                && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) {
+                size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart);
+                if (cSize <= (size_t)(iend-istart)) {
+                    size_t const decompressedSize = ZSTD_decompress_usingDDict(zds->dctx, op, oend-op, istart, cSize, zds->ddict);
+                    if (ZSTD_isError(decompressedSize)) return decompressedSize;
+                    ip += cSize;
+                    op += decompressedSize;
+                    zds->stage = zdss_init;
+                    someMoreWork = 0;
+                    break;
+            }   }
+
             /* Consume header */
             ZSTD_refDDict(zds->dctx, zds->ddict);
             {   size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx);  /* == ZSTD_frameHeaderSize_prefix */

From 8b3560e196dc15d64a591fe02c95c414412e9994 Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Tue, 28 Feb 2017 09:41:23 +0100
Subject: [PATCH 170/223] update gzip tests

---
 tests/gzip/Makefile    | 7 +++++--
 tests/gzip/gzip-env.sh | 7 ++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/gzip/Makefile b/tests/gzip/Makefile
index ee993934a..b02fb693f 100644
--- a/tests/gzip/Makefile
+++ b/tests/gzip/Makefile
@@ -12,7 +12,10 @@ VOID   = /dev/null
 export PATH := .:$(PATH)
 
 .PHONY: all
-all: test-gzip-env test-helin-segv test-hufts test-keep test-list test-memcpy-abuse test-mixed test-null-suffix-clobber test-stdin test-trailing-nul test-unpack-invalid test-zdiff test-zgrep-context test-zgrep-f test-zgrep-signal test-znew-k test-z-suffix 
+#all: test-gzip-env 
+all: test-helin-segv test-hufts test-keep test-list test-memcpy-abuse test-mixed
+all: test-null-suffix-clobber test-stdin test-trailing-nul test-unpack-invalid
+all: test-zdiff test-zgrep-context test-zgrep-f test-zgrep-signal test-znew-k test-z-suffix
 	@echo Testing completed
 
 .PHONY: zstd
@@ -35,7 +38,7 @@ clean:
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD))
 
 test-%: zstd
-	@./test-driver.sh --test-name $* --log-file $*.log --trs-file $*.trs --expect-failure "no" --color-tests "yes" --enable-hard-errors "yes" ./$*.sh 
+	@./test-driver.sh --test-name $* --log-file $*.log --trs-file $*.trs --expect-failure "no" --color-tests "yes" --enable-hard-errors "yes" ./$*.sh
 	# || echo ignoring error
 
 endif
diff --git a/tests/gzip/gzip-env.sh b/tests/gzip/gzip-env.sh
index f15a8d476..120e52d78 100755
--- a/tests/gzip/gzip-env.sh
+++ b/tests/gzip/gzip-env.sh
@@ -17,13 +17,10 @@
 # along with this program.  If not, see .
 # limit so don't run it by default.
 
-echo XXX=$PATH
-gzip --version
-
 . "${srcdir=.}/init.sh"; path_prepend_ .
 
-echo XXX=$PATH
-gzip --version
+#echo PATH=$PATH
+#gzip --version
 
 echo a >exp || framework_failure_
 gzip in || framework_failure_

From 8e5032a965c38f54cd4e651a011cefb94b94944b Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Tue, 28 Feb 2017 09:42:37 +0100
Subject: [PATCH 171/223] cli : fix : --rm is silent when input is stdin
 (decompression)

---
 programs/fileio.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/fileio.c b/programs/fileio.c
index 7f076aa15..41daa125e 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -954,7 +954,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
 
     /* Close file */
     if (fclose(srcFile)) EXM_THROW(33, "zstd: %s close error", srcFileName);  /* error should never happen */
-    if (g_removeSrcFile) { if (remove(srcFileName)) EXM_THROW(34, "zstd: %s: %s", srcFileName, strerror(errno)); };
+    if (g_removeSrcFile /* --rm */ && strcmp(srcFileName, stdinmark)) { if (remove(srcFileName)) EXM_THROW(34, "zstd: %s: %s", srcFileName, strerror(errno)); };
     return 0;
 }
 

From c0b1731bce1460df837425e39fc2f3745b2d3908 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 28 Feb 2017 01:02:46 -0800
Subject: [PATCH 172/223] added test for decompression with NULL dict and NULL
 DDict

previous version of ZSTD_decompressMultiFrame() would fail that test
---
 tests/fuzzer.c | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/tests/fuzzer.c b/tests/fuzzer.c
index 79516b6cc..6fb69972a 100644
--- a/tests/fuzzer.c
+++ b/tests/fuzzer.c
@@ -108,6 +108,7 @@ static int basicUnitTests(U32 seed, double compressibility)
     void* const CNBuffer = malloc(CNBuffSize);
     void* const compressedBuffer = malloc(ZSTD_compressBound(CNBuffSize));
     void* const decodedBuffer = malloc(CNBuffSize);
+    ZSTD_DCtx* dctx = ZSTD_createDCtx();
     int testResult = 0;
     U32 testNb=0;
     size_t cSize;
@@ -155,6 +156,16 @@ static int basicUnitTests(U32 seed, double compressibility)
     }   }
     DISPLAYLEVEL(4, "OK \n");
 
+    DISPLAYLEVEL(4, "test%3i : decompress with null dict : ", testNb++);
+    { size_t const r = ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL, 0);
+      if (r != CNBuffSize) goto _output_error; }
+    DISPLAYLEVEL(4, "OK \n");
+
+    DISPLAYLEVEL(4, "test%3i : decompress with null DDict : ", testNb++);
+    { size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, NULL);
+      if (r != CNBuffSize) goto _output_error; }
+    DISPLAYLEVEL(4, "OK \n");
+
     DISPLAYLEVEL(4, "test%3i : decompress with 1 missing byte : ", testNb++);
     { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
       if (!ZSTD_isError(r)) goto _output_error;
@@ -210,7 +221,6 @@ static int basicUnitTests(U32 seed, double compressibility)
     /* Dictionary and CCtx Duplication tests */
     {   ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx();
         ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();
-        ZSTD_DCtx* const dctx = ZSTD_createDCtx();
         static const size_t dictSize = 551;
 
         DISPLAYLEVEL(4, "test%3i : copy context too soon : ", testNb++);
@@ -283,12 +293,10 @@ static int basicUnitTests(U32 seed, double compressibility)
 
         ZSTD_freeCCtx(ctxOrig);
         ZSTD_freeCCtx(ctxDuplicated);
-        ZSTD_freeDCtx(dctx);
     }
 
     /* Dictionary and dictBuilder tests */
     {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();
-        ZSTD_DCtx* const dctx = ZSTD_createDCtx();
         size_t dictSize = 16 KB;
         void* dictBuffer = malloc(dictSize);
         size_t const totalSampleSize = 1 MB;
@@ -370,14 +378,12 @@ static int basicUnitTests(U32 seed, double compressibility)
         DISPLAYLEVEL(4, "OK \n");
 
         ZSTD_freeCCtx(cctx);
-        ZSTD_freeDCtx(dctx);
         free(dictBuffer);
         free(samplesSizes);
     }
 
     /* COVER dictionary builder tests */
     {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();
-        ZSTD_DCtx* const dctx = ZSTD_createDCtx();
         size_t dictSize = 16 KB;
         size_t optDictSize = dictSize;
         void* dictBuffer = malloc(dictSize);
@@ -414,7 +420,7 @@ static int basicUnitTests(U32 seed, double compressibility)
         memset(¶ms, 0, sizeof(params));
         params.steps = 4;
         optDictSize = COVER_optimizeTrainFromBuffer(dictBuffer, optDictSize,
-                                                    CNBuffer, samplesSizes, nbSamples,
+                                                    CNBuffer, samplesSizes, nbSamples / 4,
                                                     ¶ms);
         if (ZDICT_isError(optDictSize)) goto _output_error;
         DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)optDictSize);
@@ -425,7 +431,6 @@ static int basicUnitTests(U32 seed, double compressibility)
         DISPLAYLEVEL(4, "OK : %u \n", dictID);
 
         ZSTD_freeCCtx(cctx);
-        ZSTD_freeDCtx(dctx);
         free(dictBuffer);
         free(samplesSizes);
     }
@@ -445,7 +450,6 @@ static int basicUnitTests(U32 seed, double compressibility)
 
     /* block API tests */
     {   ZSTD_CCtx* const cctx = ZSTD_createCCtx();
-        ZSTD_DCtx* const dctx = ZSTD_createDCtx();
         static const size_t dictSize = 65 KB;
         static const size_t blockSize = 100 KB;   /* won't cause pb with small dict size */
         size_t cSize2;

From a33ae6420491e4aa6819041452a108df96977b02 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 28 Feb 2017 01:15:28 -0800
Subject: [PATCH 173/223] fixed decoding skippable frames

---
 lib/decompress/zstd_decompress.c | 9 ++++-----
 tests/zstreamtest.c              | 2 +-
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 305a9a876..1252b2c37 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1469,8 +1469,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
         if (ZSTD_isError(headerSize)) return headerSize;
 
         /* Frame Header */
-        {
-            size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize);
+        {   size_t const ret = ZSTD_getFrameParams(&fParams, ip, remainingSize);
             if (ZSTD_isError(ret)) return ret;
             if (ret > 0) return ERROR(srcSize_wrong);
         }
@@ -1503,7 +1502,7 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
 }
 
 /*! ZSTD_decompressFrame() :
-*   `dctx` must be properly initialized */
+*   @dctx must be properly initialized */
 static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
                                  void* dst, size_t dstCapacity,
                                  const void** srcPtr, size_t *srcSizePtr)
@@ -1570,7 +1569,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
         remainingSize -= 4;
     }
 
-    // Allow caller to get size read
+    /* Allow caller to get size read */
     *srcPtr = ip;
     *srcSizePtr = remainingSize;
     return op-ostart;
@@ -2302,7 +2301,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             }   }
 
             /* check for single-pass mode opportunity */
-            if (zds->fParams.frameContentSize
+            if (zds->fParams.frameContentSize && zds->fParams.windowSize /* skippable frame if == 0 */
                 && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) {
                 size_t const cSize = ZSTD_findFrameCompressedSize(istart, iend-istart);
                 if (cSize <= (size_t)(iend-istart)) {
diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c
index c22a284c7..54b890266 100644
--- a/tests/zstreamtest.c
+++ b/tests/zstreamtest.c
@@ -218,7 +218,7 @@ static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem custo
     outBuff.pos = 0;
     { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
       if (r != 0) goto _output_error; }
-    if (outBuff.pos != 0) goto _output_error;   /* skippable frame len is 0 */
+    if (outBuff.pos != 0) goto _output_error;   /* skippable frame output len is 0 */
     DISPLAYLEVEL(3, "OK \n");
 
     /* Basic decompression test */

From 59709d97d9541605c166d02ba055e3814ea634cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Milan=20=C5=A0ev=C4=8D=C3=ADk?= 
Date: Thu, 9 Feb 2017 15:10:24 +0100
Subject: [PATCH 174/223] Support building contrib utils from cmake

---
 build/cmake/CMakeLists.txt               |  4 ++++
 build/cmake/contrib/CMakeLists.txt       | 16 ++++++++++++++
 build/cmake/contrib/pzstd/CMakeLists.txt | 28 ++++++++++++++++++++++++
 3 files changed, 48 insertions(+)
 create mode 100644 build/cmake/contrib/CMakeLists.txt
 create mode 100644 build/cmake/contrib/pzstd/CMakeLists.txt

diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt
index b8f5d18e4..f28f74118 100644
--- a/build/cmake/CMakeLists.txt
+++ b/build/cmake/CMakeLists.txt
@@ -11,6 +11,7 @@ PROJECT(zstd)
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8.7)
 
 OPTION(ZSTD_LEGACY_SUPPORT "LEGACY SUPPORT" OFF)
+OPTION(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF)
 
 IF (ZSTD_LEGACY_SUPPORT)
     MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT defined!")
@@ -23,6 +24,9 @@ ENDIF (ZSTD_LEGACY_SUPPORT)
 ADD_SUBDIRECTORY(lib)
 ADD_SUBDIRECTORY(programs)
 ADD_SUBDIRECTORY(tests)
+IF (ZSTD_BUILD_CONTRIB)
+    ADD_SUBDIRECTORY(contrib)
+ENDIF (ZSTD_BUILD_CONTRIB)
 
 #-----------------------------------------------------------------------------
 # Add extra compilation flags
diff --git a/build/cmake/contrib/CMakeLists.txt b/build/cmake/contrib/CMakeLists.txt
new file mode 100644
index 000000000..68e0881c5
--- /dev/null
+++ b/build/cmake/contrib/CMakeLists.txt
@@ -0,0 +1,16 @@
+# ################################################################
+# * Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
+# * All rights reserved.
+# *
+# * This source code is licensed under the BSD-style license found in the
+# * LICENSE file in the root directory of this source tree. An additional grant
+# * of patent rights can be found in the PATENTS file in the same directory.
+#
+# You can contact the author at :
+#  - zstd homepage : http://www.zstd.net/
+# ################################################################
+
+PROJECT(contrib)
+
+ADD_SUBDIRECTORY(pzstd)
+
diff --git a/build/cmake/contrib/pzstd/CMakeLists.txt b/build/cmake/contrib/pzstd/CMakeLists.txt
new file mode 100644
index 000000000..6699fcc87
--- /dev/null
+++ b/build/cmake/contrib/pzstd/CMakeLists.txt
@@ -0,0 +1,28 @@
+# ################################################################
+# * Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
+# * All rights reserved.
+# *
+# * This source code is licensed under the BSD-style license found in the
+# * LICENSE file in the root directory of this source tree. An additional grant
+# * of patent rights can be found in the PATENTS file in the same directory.
+#
+# You can contact the author at :
+#  - zstd homepage : http://www.zstd.net/
+# ################################################################
+
+PROJECT(pzstd)
+
+SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
+
+# Define project root directory
+SET(ROOT_DIR ../../../..)
+
+# Define programs directory, where sources and header files are located
+SET(LIBRARY_DIR ${ROOT_DIR}/lib)
+SET(PROGRAMS_DIR ${ROOT_DIR}/programs)
+SET(PZSTD_DIR ${ROOT_DIR}/contrib/pzstd)
+INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD_DIR})
+
+ADD_EXECUTABLE(pzstd ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp)
+TARGET_LINK_LIBRARIES(pzstd libzstd_static pthread)
+

From bf8a30ce0d0e8f2b45a954fcf396220050f9172d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Milan=20=C5=A0ev=C4=8D=C3=ADk?= 
Date: Thu, 9 Feb 2017 15:11:05 +0100
Subject: [PATCH 175/223] Add zstdmt target in cmake

---
 build/cmake/CMakeLists.txt          |  1 +
 build/cmake/programs/CMakeLists.txt | 10 +++++++++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt
index f28f74118..6b7c28925 100644
--- a/build/cmake/CMakeLists.txt
+++ b/build/cmake/CMakeLists.txt
@@ -11,6 +11,7 @@ PROJECT(zstd)
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8.7)
 
 OPTION(ZSTD_LEGACY_SUPPORT "LEGACY SUPPORT" OFF)
+OPTION(ZSTD_MULTITHREAD_SUPPORT "MULTITHREADING SUPPORT" ON)
 OPTION(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF)
 
 IF (ZSTD_LEGACY_SUPPORT)
diff --git a/build/cmake/programs/CMakeLists.txt b/build/cmake/programs/CMakeLists.txt
index cb3dc6e89..c88ee5cc9 100644
--- a/build/cmake/programs/CMakeLists.txt
+++ b/build/cmake/programs/CMakeLists.txt
@@ -34,9 +34,17 @@ ENDIF (MSVC)
 
 ADD_EXECUTABLE(zstd ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
 TARGET_LINK_LIBRARIES(zstd libzstd_static)
-
 IF (UNIX)
     ADD_EXECUTABLE(zstd-frugal ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c)
     TARGET_LINK_LIBRARIES(zstd-frugal libzstd_static)
     SET_TARGET_PROPERTIES(zstd-frugal PROPERTIES COMPILE_DEFINITIONS "ZSTD_NOBENCH;ZSTD_NODICT")
 ENDIF (UNIX)
+
+IF (ZSTD_MULTITHREAD_SUPPORT)
+    ADD_EXECUTABLE(zstdmt ${PROGRAMS_DIR}/zstdcli.c ${PROGRAMS_DIR}/fileio.c ${PROGRAMS_DIR}/bench.c ${PROGRAMS_DIR}/datagen.c ${PROGRAMS_DIR}/dibio.c ${PlatformDependResources})
+    SET_TARGET_PROPERTIES(zstdmt PROPERTIES COMPILE_DEFINITIONS "ZSTD_MULTITHREAD")
+    TARGET_LINK_LIBRARIES(zstdmt libzstd_static)
+    IF (UNIX)
+        TARGET_LINK_LIBRARIES(zstdmt pthread)
+    ENDIF (UNIX)
+ENDIF (ZSTD_MULTITHREAD_SUPPORT)

From 5a1cc5c22d149135b93c9c48b0c71a8b1f7c0caf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Milan=20=C5=A0ev=C4=8D=C3=ADk?= 
Date: Fri, 10 Feb 2017 12:29:55 +0100
Subject: [PATCH 176/223] Improve handling of library symlinks.

Previous method was failing to remove the symlinks when make clean was
invoked and wasn't portable.
---
 build/cmake/lib/CMakeLists.txt | 22 ++++++++--------------
 1 file changed, 8 insertions(+), 14 deletions(-)

diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt
index 265f7aeb8..1950d97cd 100644
--- a/build/cmake/lib/CMakeLists.txt
+++ b/build/cmake/lib/CMakeLists.txt
@@ -166,23 +166,17 @@ IF (UNIX)
     SET(SHARED_LIBRARY_SYMLINK1_PATH ${CMAKE_CURRENT_BINARY_DIR}/${SHARED_LIBRARY_SYMLINK1})
     SET(SHARED_LIBRARY_SYMLINK2_PATH ${CMAKE_CURRENT_BINARY_DIR}/${SHARED_LIBRARY_SYMLINK2})
 
-    if (EXISTS ${SHARED_LIBRARY_SYMLINK1_PATH})
-        FILE(REMOVE ${SHARED_LIBRARY_SYMLINK1_PATH})
-    endif (EXISTS ${SHARED_LIBRARY_SYMLINK1_PATH})
-
-    if (EXISTS ${SHARED_LIBRARY_SYMLINK2_PATH})
-        FILE(REMOVE ${SHARED_LIBRARY_SYMLINK2_PATH})
-    endif (EXISTS ${SHARED_LIBRARY_SYMLINK2_PATH})
+    ADD_CUSTOM_COMMAND(TARGET libzstd_shared POST_BUILD
+            COMMAND ${CMAKE_COMMAND} -E create_symlink ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK1}
+            DEPENDS  ${SHARED_LIBRARY_LINK_PATH}
+            COMMENT "Generating symbolic link ${SHARED_LIBRARY_LINK} -> ${SHARED_LIBRARY_SYMLINK1}")
 
     ADD_CUSTOM_COMMAND(TARGET libzstd_shared POST_BUILD
-            COMMAND ln -s ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK1}
-            DEPENDS ${SHARED_LIBRARY_LINK_PATH}
-            COMMENT "Generating symbolic link")
+            COMMAND ${CMAKE_COMMAND} -E create_symlink ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK2}
+            DEPENDS  ${SHARED_LIBRARY_LINK_PATH}
+            COMMENT "Generating symbolic link ${SHARED_LIBRARY_LINK} -> ${SHARED_LIBRARY_SYMLINK2}")
 
-    ADD_CUSTOM_COMMAND(TARGET libzstd_shared POST_BUILD
-            COMMAND ln -s ${SHARED_LIBRARY_LINK} ${SHARED_LIBRARY_SYMLINK2}
-            DEPENDS ${SHARED_LIBRARY_LINK_PATH}
-            COMMENT "Generating symbolic link")
+    SET_DIRECTORY_PROPERTIES(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${SHARED_LIBRARY_SYMLINK1};${SHARED_LIBRARY_SYMLINK2}")
 
     INSTALL(FILES ${SHARED_LIBRARY_SYMLINK1_PATH} DESTINATION ${INSTALL_LIBRARY_DIR})
     INSTALL(FILES ${SHARED_LIBRARY_SYMLINK2_PATH} DESTINATION ${INSTALL_LIBRARY_DIR})

From eeb080e6015ecc67a99d912d0376d83e6c81a79c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Milan=20=C5=A0ev=C4=8D=C3=ADk?= 
Date: Mon, 27 Feb 2017 13:56:04 +0100
Subject: [PATCH 177/223] -Wstrict-prototypes is not supported with C++

---
 build/cmake/CMakeModules/AddExtraCompilationFlags.cmake | 2 --
 1 file changed, 2 deletions(-)

diff --git a/build/cmake/CMakeModules/AddExtraCompilationFlags.cmake b/build/cmake/CMakeModules/AddExtraCompilationFlags.cmake
index 2d59fabb0..e480c7ead 100644
--- a/build/cmake/CMakeModules/AddExtraCompilationFlags.cmake
+++ b/build/cmake/CMakeModules/AddExtraCompilationFlags.cmake
@@ -132,10 +132,8 @@ MACRO(ADD_EXTRA_COMPILATION_FLAGS)
         endif (ACTIVATE_WARNING_CAST_ALIGN)
 
         if (ACTIVATE_WARNING_STRICT_PROTOTYPES)
-            list(APPEND CMAKE_CXX_FLAGS ${WARNING_STRICT_PROTOTYPES})
             list(APPEND CMAKE_C_FLAGS ${WARNING_STRICT_PROTOTYPES})
         else ()
-            string(REPLACE ${WARNING_STRICT_PROTOTYPES} "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
             string(REPLACE ${WARNING_STRICT_PROTOTYPES} "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
         endif (ACTIVATE_WARNING_STRICT_PROTOTYPES)
 

From 4b62f419698d313647f3a111caf2c2b804711a05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Milan=20=C5=A0ev=C4=8D=C3=ADk?= 
Date: Mon, 27 Feb 2017 14:44:49 +0100
Subject: [PATCH 178/223] Added compile flags to pzstd

Definition NDEBUG from original Makefile
-Wno-shadow silences shadowing in initializers
---
 build/cmake/contrib/pzstd/CMakeLists.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/build/cmake/contrib/pzstd/CMakeLists.txt b/build/cmake/contrib/pzstd/CMakeLists.txt
index 6699fcc87..2a3663f31 100644
--- a/build/cmake/contrib/pzstd/CMakeLists.txt
+++ b/build/cmake/contrib/pzstd/CMakeLists.txt
@@ -25,4 +25,6 @@ INCLUDE_DIRECTORIES(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD
 
 ADD_EXECUTABLE(pzstd ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp)
 TARGET_LINK_LIBRARIES(pzstd libzstd_static pthread)
+SET_TARGET_PROPERTIES(pzstd PROPERTIES COMPILE_DEFINITIONS "NDEBUG")
+SET_TARGET_PROPERTIES(pzstd PROPERTIES COMPILE_OPTIONS "-Wno-shadow")
 

From 22d79762ef3e5c3760975742af10e8874958c13b Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 28 Feb 2017 02:12:42 -0800
Subject: [PATCH 179/223] fixed multi frames

---
 lib/decompress/zstd_decompress.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 1252b2c37..5aa863afc 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -2307,8 +2307,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                 if (cSize <= (size_t)(iend-istart)) {
                     size_t const decompressedSize = ZSTD_decompress_usingDDict(zds->dctx, op, oend-op, istart, cSize, zds->ddict);
                     if (ZSTD_isError(decompressedSize)) return decompressedSize;
-                    ip += cSize;
+                    ip = istart + cSize;
                     op += decompressedSize;
+                    zds->dctx->expected = 0;
                     zds->stage = zdss_init;
                     someMoreWork = 0;
                     break;

From 5c1c80cbb6fe3c71679303e521ec0ed93053899d Mon Sep 17 00:00:00 2001
From: Przemyslaw Skibinski 
Date: Tue, 28 Feb 2017 17:27:41 +0100
Subject: [PATCH 180/223] travis.yml: fixed pull_request

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 7d19e36c9..82b92c2a2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -29,7 +29,7 @@ script:
   - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "master" ]; then
         FUZZERTEST=-T7mn sh -c "$Cmd" || travis_terminate 1;
     else
-        if [ "$TRAVIS_PULL_REQUEST" = "true" ] && [ $JOB_NUMBER -lt 4 ]; then
+        if [ "$TRAVIS_EVENT_TYPE" = "pull_request" ] && [ $JOB_NUMBER -lt 4 ]; then
             sh -c "$Cmd" || travis_terminate 1;
         else
             if [ $JOB_NUMBER -lt 3 ]; then

From a81d4fee5822ed15e15922334197a1f0bd3e09c1 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Tue, 28 Feb 2017 15:28:29 -0800
Subject: [PATCH 181/223] Check to ensure ddict isn't null before dereference

---
 lib/decompress/zstd_decompress.c | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 2646c8028..0504778e4 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1576,6 +1576,9 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
     return op-ostart;
 }
 
+static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict);
+static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict);
+
 static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
                                         void* dst, size_t dstCapacity,
                                   const void* src, size_t srcSize,
@@ -1583,6 +1586,17 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx,
                                   const ZSTD_DDict* ddict)
 {
     void* const dststart = dst;
+
+    if (ddict) {
+        if (dict) {
+            /* programmer error, these two cases should be mutually exclusive */
+            return ERROR(GENERIC);
+        }
+
+        dict = ZSTD_DDictDictContent(ddict);
+        dictSize = ZSTD_DDictDictSize(ddict);
+    }
+
     while (srcSize >= ZSTD_frameHeaderSize_prefix) {
         U32 magicNumber;
 
@@ -1938,6 +1952,16 @@ struct ZSTD_DDict_s {
     ZSTD_customMem cMem;
 };  /* typedef'd to ZSTD_DDict within "zstd.h" */
 
+static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict)
+{
+    return ddict->dictContent;
+}
+
+static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict)
+{
+    return ddict->dictSize;
+}
+
 static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict)
 {
     ZSTD_decompressBegin(dstDCtx);  /* init */
@@ -2100,7 +2124,7 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
 {
     /* pass content and size in case legacy frames are encountered */
     return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize,
-                                     ddict->dictContent, ddict->dictSize,
+                                     NULL, 0,
                                      ddict);
 }
 

From 43764cdb1dbb2c8889eb5861e8f48ea0a4450990 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 28 Feb 2017 17:44:17 -0800
Subject: [PATCH 182/223] updated NEWS for 1.1.4

cmake, performance
---
 NEWS | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 7ede5f70c..8b7d83c8a 100644
--- a/NEWS
+++ b/NEWS
@@ -3,9 +3,11 @@ cli : new : can compress in *.gz format, using --format=gzip command, by Przemys
 cli : new : advanced benchmark command --priority=rt
 cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77
 cli : fix : --rm remains silent when input is stdin
+speed : improved decompression speed in streaming mode for single shot scenarios (+5%)
+memory : DDict (decompression dictionary) memory usage down from 150 KB to 20 KB
 API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize(), by Sean Purcell
-API : change : ZSTD_compress*() with srcSize==0 create an empty-frame of known size
-build:new : meson build system in contrib/meson, by Dima Krasner
+build: new: meson build system in contrib/meson, by Dima Krasner
+build: improved cmake script, by @Majlen
 doc : new : educational decoder, by Sean Purcell
 
 v1.1.3

From 4bcc69b7616ea7ec39eab75676905a64b2750bed Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 1 Mar 2017 11:33:25 -0800
Subject: [PATCH 183/223] solves warnings when compiling with global
 XXH_STATIC_LINKING_ONLY

XXH_STATIC_LINKING_ONLY protection macro is intended to be triggered just before the include.
The main idea is to keep this setting local :
user module shall explicitly understand and accept the static linking restriction
which becomes transparent when triggering the macro at project level.
Global definition also triggers redefinition warnings for user modules which do locally define the macro.

This new version compiles lib and cli without warning when the macro is set globally.
That's not a scenario to be recommended, since it trades a local effect for a global one,
but it was easy enough to provide from zstd side.
---
 lib/common/xxhash.c              |  4 +++-
 lib/common/zstd_internal.h       |  4 ++++
 lib/compress/zstd_compress.c     |  2 --
 lib/compress/zstdmt_compress.c   |  2 --
 lib/decompress/zstd_decompress.c |  2 --
 lib/legacy/zstd_v07.c            | 13 ++++++++-----
 6 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/lib/common/xxhash.c b/lib/common/xxhash.c
index 29e4fa628..eb44222c5 100644
--- a/lib/common/xxhash.c
+++ b/lib/common/xxhash.c
@@ -104,7 +104,9 @@ static void  XXH_free  (void* p)  { free(p); }
 #include 
 static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
 
-#define XXH_STATIC_LINKING_ONLY
+#ifndef XXH_STATIC_LINKING_ONLY
+#  define XXH_STATIC_LINKING_ONLY
+#endif
 #include "xxhash.h"
 
 
diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h
index 4b56ce1a2..5c5b28732 100644
--- a/lib/common/zstd_internal.h
+++ b/lib/common/zstd_internal.h
@@ -49,6 +49,10 @@
 #include "error_private.h"
 #define ZSTD_STATIC_LINKING_ONLY
 #include "zstd.h"
+#ifndef XXH_STATIC_LINKING_ONLY
+#  define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
+#endif
+#include "xxhash.h"               /* XXH_reset, update, digest */
 
 
 /*-*************************************
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index ec758db00..994e606e6 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -13,8 +13,6 @@
 ***************************************/
 #include          /* memset */
 #include "mem.h"
-#define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
-#include "xxhash.h"               /* XXH_reset, update, digest */
 #define FSE_STATIC_LINKING_ONLY   /* FSE_encodeSymbol */
 #include "fse.h"
 #define HUF_STATIC_LINKING_ONLY
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index 483ea157e..45514a81a 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -25,8 +25,6 @@
 #include "threading.h"  /* mutex */
 #include "zstd_internal.h"   /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
 #include "zstdmt_compress.h"
-#define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
-#include "xxhash.h"
 
 
 /* ======   Debug   ====== */
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 61c7936f8..32b1747c6 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -43,8 +43,6 @@
 *********************************************************/
 #include       /* memcpy, memmove, memset */
 #include "mem.h"         /* low level memory routines */
-#define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
-#include "xxhash.h"      /* XXH64_* */
 #define FSE_STATIC_LINKING_ONLY
 #include "fse.h"
 #define HUF_STATIC_LINKING_ONLY
diff --git a/lib/legacy/zstd_v07.c b/lib/legacy/zstd_v07.c
index e67916b3c..a54ad0ffd 100644
--- a/lib/legacy/zstd_v07.c
+++ b/lib/legacy/zstd_v07.c
@@ -13,12 +13,14 @@
 #include      /* memcpy */
 #include      /* malloc, free, qsort */
 
-#define XXH_STATIC_LINKING_ONLY   /* XXH64_state_t */
-#include "xxhash.h"      /* XXH64_* */
+#ifndef XXH_STATIC_LINKING_ONLY
+#  define XXH_STATIC_LINKING_ONLY    /* XXH64_state_t */
+#endif
+#include "xxhash.h"                  /* XXH64_* */
 #include "zstd_v07.h"
 
-#define FSEv07_STATIC_LINKING_ONLY  /* FSEv07_MIN_TABLELOG */
-#define HUFv07_STATIC_LINKING_ONLY  /* HUFv07_TABLELOG_ABSOLUTEMAX */
+#define FSEv07_STATIC_LINKING_ONLY   /* FSEv07_MIN_TABLELOG */
+#define HUFv07_STATIC_LINKING_ONLY   /* HUFv07_TABLELOG_ABSOLUTEMAX */
 #define ZSTDv07_STATIC_LINKING_ONLY
 
 #include "error_private.h"
@@ -4536,7 +4538,8 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd,
                     if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; }   /* this was just a header */
                     zbd->outEnd = zbd->outStart +  decodedSize;
                     zbd->stage = ZBUFFds_flush;
-                    // break; /* ZBUFFds_flush follows */
+                    /* break; */
+                    /* pass-through */
             }   }
 
         case ZBUFFds_flush:

From 76f04940898bf062ad4504d95fe741c0c784cb24 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 1 Mar 2017 13:29:29 -0800
Subject: [PATCH 184/223] xxhash can be included twice in any order

Previously,

followed by :

would fail to include the static definitions,
because the second include was simply skipped by guard macro.

Now it works as intended :
the missing static part is included during the second include.
---
 lib/common/xxhash.h | 26 +++++++++++---------------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/lib/common/xxhash.h b/lib/common/xxhash.h
index 2c9b7c61b..9bad1f59f 100644
--- a/lib/common/xxhash.h
+++ b/lib/common/xxhash.h
@@ -64,16 +64,12 @@ XXH64       13.8 GB/s            1.9 GB/s
 XXH32        6.8 GB/s            6.0 GB/s
 */
 
-#ifndef XXHASH_H_5627135585666179
-#define XXHASH_H_5627135585666179 1
-
 #if defined (__cplusplus)
 extern "C" {
 #endif
 
-#ifndef XXH_NAMESPACE
-#  define XXH_NAMESPACE ZSTD_  /* Zstandard specific */
-#endif
+#ifndef XXHASH_H_5627135585666179
+#define XXHASH_H_5627135585666179 1
 
 
 /* ****************************
@@ -242,6 +238,11 @@ XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH
 /* **************************
 *  Canonical representation
 ****************************/
+/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
+*  The canonical representation uses human-readable write convention, aka big-endian (large digits first).
+*  These functions allow transformation of hash result into and from its canonical format.
+*  This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
+*/
 typedef struct { unsigned char digest[4]; } XXH32_canonical_t;
 typedef struct { unsigned char digest[8]; } XXH64_canonical_t;
 
@@ -251,14 +252,9 @@ XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t
 XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
 XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
 
-/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
-*  The canonical representation uses human-readable write convention, aka big-endian (large digits first).
-*  These functions allow transformation of hash result into and from its canonical format.
-*  This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
-*/
+#endif /* XXHASH_H_5627135585666179 */
 
 
-#ifdef XXH_STATIC_LINKING_ONLY
 
 /* ================================================================================================
    This section contains definitions which are not guaranteed to remain stable.
@@ -266,6 +262,8 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src
    They shall only be used with static linking.
    Never use these definitions in association with dynamic linking !
 =================================================================================================== */
+#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXH_STATIC_H_3543687687345)
+#define XXH_STATIC_H_3543687687345
 
 /* These definitions are only meant to allow allocation of XXH state
    statically, on stack, or in a struct for example.
@@ -299,11 +297,9 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src
 #    include "xxhash.c"   /* include xxhash functions as `static`, for inlining */
 #  endif
 
-#endif /* XXH_STATIC_LINKING_ONLY */
+#endif /* XXH_STATIC_LINKING_ONLY && XXH_STATIC_H_3543687687345 */
 
 
 #if defined (__cplusplus)
 }
 #endif
-
-#endif /* XXHASH_H_5627135585666179 */

From d44703d145611945d7b92eeb3c93ad9fbe6fd480 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Wed, 1 Mar 2017 14:36:25 -0800
Subject: [PATCH 185/223] Offsets >= 32MB in 32-bits mode

---
 lib/common/bitstream.h           |  3 +++
 lib/compress/zstd_compress.c     | 39 ++++++++++++++++++++++++++++----
 lib/decompress/zstd_decompress.c | 26 +++++++++++++++++----
 lib/zstd.h                       |  2 +-
 4 files changed, 60 insertions(+), 10 deletions(-)

diff --git a/lib/common/bitstream.h b/lib/common/bitstream.h
index 3a45244f8..d3873002e 100644
--- a/lib/common/bitstream.h
+++ b/lib/common/bitstream.h
@@ -60,6 +60,9 @@ extern "C" {
 #  include    /* support for bextr (experimental) */
 #endif
 
+#define STREAM_ACCUMULATOR_MIN_32  25
+#define STREAM_ACCUMULATOR_MIN_64  57
+#define STREAM_ACCUMULATOR_MIN    ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64))
 
 /*-******************************************
 *  bitStream encoding API (write forward)
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index ec758db00..89f6575dc 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -579,9 +579,9 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
 }
 
 
-size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
+FORCE_INLINE size_t ZSTD_compressSequences_generic (ZSTD_CCtx* zc,
                               void* dst, size_t dstCapacity,
-                              size_t srcSize)
+                              size_t srcSize, int const longOffsets)
 {
     const seqStore_t* seqStorePtr = &(zc->seqStore);
     U32 count[MaxSeq+1];
@@ -716,7 +716,18 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
         if (MEM_32bits()) BIT_flushBits(&blockStream);
         BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]);
         if (MEM_32bits()) BIT_flushBits(&blockStream);
-        BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]);
+        if (longOffsets) {
+            U32 const ofBits = ofCodeTable[nbSeq-1];
+            int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
+            if (extraBits) {
+                BIT_addBits(&blockStream, sequences[nbSeq-1].offset, extraBits);
+                BIT_flushBits(&blockStream);
+            }
+            BIT_addBits(&blockStream, sequences[nbSeq-1].offset >> extraBits,
+                        ofBits - extraBits);
+        } else {
+            BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]);
+        }
         BIT_flushBits(&blockStream);
 
         {   size_t n;
@@ -738,7 +749,17 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
                 if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream);
                 BIT_addBits(&blockStream, sequences[n].matchLength, mlBits);
                 if (MEM_32bits()) BIT_flushBits(&blockStream);                  /* (7)*/
-                BIT_addBits(&blockStream, sequences[n].offset, ofBits);         /* 31 */
+                if (longOffsets) {
+                    int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1);
+                    if (extraBits) {
+                        BIT_addBits(&blockStream, sequences[n].offset, extraBits);
+                        BIT_flushBits(&blockStream);                            /* (7)*/
+                    }
+                    BIT_addBits(&blockStream, sequences[n].offset >> extraBits,
+                                ofBits - extraBits);                            /* 31 */
+                } else {
+                    BIT_addBits(&blockStream, sequences[n].offset, ofBits);     /* 31 */
+                }
                 BIT_flushBits(&blockStream);                                    /* (7)*/
         }   }
 
@@ -763,6 +784,16 @@ _check_compressibility:
     return op - ostart;
 }
 
+FORCE_INLINE size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
+                              void* dst, size_t dstCapacity,
+                              size_t srcSize)
+{
+    if (zc->params.cParams.windowLog > STREAM_ACCUMULATOR_MIN) {
+        return ZSTD_compressSequences_generic(zc, dst, dstCapacity, srcSize, 1);
+    } else {
+        return ZSTD_compressSequences_generic(zc, dst, dstCapacity, srcSize, 0);
+    }
+}
 
 #if 0 /* for debug */
 #  define STORESEQ_DEBUG
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 2646c8028..e39bf42bf 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1144,7 +1144,7 @@ static size_t ZSTD_decompressSequences(
 }
 
 
-static seq_t ZSTD_decodeSequenceLong(seqState_t* seqState)
+FORCE_INLINE seq_t ZSTD_decodeSequenceLong_generic(seqState_t* seqState, int const longOffsets)
 {
     seq_t seq;
 
@@ -1179,8 +1179,15 @@ static seq_t ZSTD_decodeSequenceLong(seqState_t* seqState)
         if (!ofCode)
             offset = 0;
         else {
-            offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits);   /* <=  (ZSTD_WINDOWLOG_MAX-1) bits */
-            if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);
+            if (longOffsets) {
+                int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN);
+                offset = OF_base[ofCode] + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits);
+                if (MEM_32bits() || extraBits) BIT_reloadDStream(&seqState->DStream);
+                if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits);
+            } else {
+                offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits);   /* <=  (ZSTD_WINDOWLOG_MAX-1) bits */
+                if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream);
+            }
         }
 
         if (ofCode <= 1) {
@@ -1224,6 +1231,14 @@ static seq_t ZSTD_decodeSequenceLong(seqState_t* seqState)
     return seq;
 }
 
+static seq_t ZSTD_decodeSequenceLong(seqState_t* seqState, unsigned const windowSize) {
+    if (ZSTD_highbit32(windowSize) > STREAM_ACCUMULATOR_MIN) {
+        return ZSTD_decodeSequenceLong_generic(seqState, 1);
+    } else {
+        return ZSTD_decodeSequenceLong_generic(seqState, 0);
+    }
+}
+
 FORCE_INLINE
 size_t ZSTD_execSequenceLong(BYTE* op,
                                 BYTE* const oend, seq_t sequence,
@@ -1321,6 +1336,7 @@ static size_t ZSTD_decompressSequencesLong(
     const BYTE* const base = (const BYTE*) (dctx->base);
     const BYTE* const vBase = (const BYTE*) (dctx->vBase);
     const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
+    unsigned const windowSize = dctx->fParams.windowSize;
     int nbSeq;
 
     /* Build Decoding Tables */
@@ -1350,13 +1366,13 @@ static size_t ZSTD_decompressSequencesLong(
 
         /* prepare in advance */
         for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && seqNb
Date: Wed, 1 Mar 2017 16:49:20 -0800
Subject: [PATCH 186/223] added gzip tests

also : made sure zstd --format=gzip -V
would fail if gzip compatibility is not supported
---
 programs/bench.c   |  1 +
 programs/zstdcli.c |  4 +++-
 tests/playTests.sh | 35 ++++++++++++++++++++++++++++-------
 3 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/programs/bench.c b/programs/bench.c
index 663b30743..2dd1cfb0f 100644
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -581,6 +581,7 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, const char* di
 {
     double const compressibility = (double)g_compressibilityDefault / 100;
 
+    if (cLevel < 1) cLevel = 1;   /* minimum compression level */
     if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel();
     if (cLevelLast > ZSTD_maxCLevel()) cLevelLast = ZSTD_maxCLevel();
     if (cLevelLast < cLevel) cLevelLast = cLevel;
diff --git a/programs/zstdcli.c b/programs/zstdcli.c
index a7b4fddc8..050d7a6a2 100644
--- a/programs/zstdcli.c
+++ b/programs/zstdcli.c
@@ -370,10 +370,12 @@ int main(int argCount, const char* argv[])
                     if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
                     if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
                     if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
+#ifdef ZSTD_GZCOMPRESS
                     if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; }
+#endif
 
                     /* long commands with arguments */
-#ifndef  ZSTD_NODICT
+#ifndef ZSTD_NODICT
                     if (longCommandWArg(&argument, "--cover=")) {
                       cover=1; if (!parseCoverParameters(argument, &coverParams)) CLEAN_RETURN(badusage(programName));
                       continue;
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 88c0ecdfe..5933b14e6 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -304,11 +304,32 @@ $ECHO "\n**** benchmark mode tests **** "
 
 $ECHO "bench one file"
 ./datagen > tmp1
-$ZSTD -bi1 tmp1
+$ZSTD -bi0 tmp1
 $ECHO "bench multiple levels"
-$ZSTD -i1b1e3 tmp1
+$ZSTD -i0b0e3 tmp1
 $ECHO "with recursive and quiet modes"
-$ZSTD -rqi1b1e3 tmp1
+$ZSTD -rqi1b1e2 tmp1
+
+
+$ECHO "\n**** gzip compatibility tests **** "
+
+GZIPMODE=1
+$ZSTD --format=gzip -V || GZIPMODE=0
+if [ $GZIPMODE -eq 1 ]; then
+    GZIPEXE=1
+    which gzip || GZIPEXE=0
+    if [ $GZIPEXE -eq 1 ]; then
+        ./datagen > tmp
+        $ZSTD --format=gzip -f tmp
+        gzip -t -v tmp.gz
+        gzip -f tmp
+        $ZSTD -d -f -v tmp.gz
+    else
+        $ECHO "gzip binary not detected"
+    fi
+else
+    $ECHO "gzip mode not supported"
+fi
 
 
 $ECHO "\n**** zstd round-trip tests **** "
@@ -317,10 +338,10 @@ roundTripTest
 roundTripTest -g15K       # TableID==3
 roundTripTest -g127K      # TableID==2
 roundTripTest -g255K      # TableID==1
-roundTripTest -g513K      # TableID==0
-roundTripTest -g512K 6    # greedy, hash chain
-roundTripTest -g512K 16   # btlazy2
-roundTripTest -g512K 19   # btopt
+roundTripTest -g522K      # TableID==0
+roundTripTest -g519K 6    # greedy, hash chain
+roundTripTest -g517K 16   # btlazy2
+roundTripTest -g516K 19   # btopt
 
 rm tmp*
 

From 27526c7201bc4cf1b5ad66ad746e20671cab3ae1 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 1 Mar 2017 17:02:49 -0800
Subject: [PATCH 187/223] make : added target shortest

shortest only run fast part of playTests.sh .
cc @iburinoc
---
 Makefile           | 6 +++++-
 tests/Makefile     | 3 +++
 tests/playTests.sh | 3 ++-
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index c73393cdd..0f97e6c5b 100644
--- a/Makefile
+++ b/Makefile
@@ -63,6 +63,10 @@ zstdmt:
 zlibwrapper:
 	$(MAKE) -C $(ZWRAPDIR) test
 
+.PHONY: shortest
+shortest:
+	$(MAKE) -C $(TESTDIR) $@
+
 .PHONY: test
 test:
 	$(MAKE) -C $(TESTDIR) $@
@@ -173,7 +177,7 @@ ppcinstall:
 arminstall:
 	APT_PACKAGES="qemu-system-arm qemu-user-static gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross" $(MAKE) apt-install
 
-valgrindinstall: 
+valgrindinstall:
 	APT_PACKAGES="valgrind" $(MAKE) apt-install
 
 libc6install:
diff --git a/tests/Makefile b/tests/Makefile
index 30b2a04a3..88c58fce4 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -253,6 +253,9 @@ zstd-playTests: datagen
 	file $(ZSTD)
 	ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
 
+shortest: ZSTDRTTEST=
+shortest: test-zstd
+
 test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus
 ifeq ($(QEMU_SYS),)
 test: test-pool
diff --git a/tests/playTests.sh b/tests/playTests.sh
index 5933b14e6..653aaf3c8 100755
--- a/tests/playTests.sh
+++ b/tests/playTests.sh
@@ -316,8 +316,9 @@ $ECHO "\n**** gzip compatibility tests **** "
 GZIPMODE=1
 $ZSTD --format=gzip -V || GZIPMODE=0
 if [ $GZIPMODE -eq 1 ]; then
+    $ECHO "gzip support detected"
     GZIPEXE=1
-    which gzip || GZIPEXE=0
+    gzip -V || GZIPEXE=0
     if [ $GZIPEXE -eq 1 ]; then
         ./datagen > tmp
         $ZSTD --format=gzip -f tmp

From 78208bd8be0a7e19a066214c053f52534d2729f4 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Wed, 1 Mar 2017 21:02:06 -0800
Subject: [PATCH 188/223] fixed : build zstd cli after libzstd

---
 programs/Makefile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/programs/Makefile b/programs/Makefile
index db718d14c..407a4f374 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -24,7 +24,9 @@ else
 ALIGN_LOOP =
 endif
 
-CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress -I$(ZSTDDIR)/dictBuilder
+CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
+           -I$(ZSTDDIR)/dictBuilder \
+           -DXXH_NAMESPACE=ZSTD_   # because xxhash.o already compiled with this macro from library
 CFLAGS  ?= -O3
 DEBUGFLAGS = -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
           -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \

From 3475b9b43100a2a8b8e08ae2b09f8aafc2af758a Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Thu, 2 Mar 2017 12:33:02 -0800
Subject: [PATCH 189/223] Set dictID to 0 for content only dictionaries

---
 lib/decompress/zstd_decompress.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 61c7936f8..b7c668d39 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1989,6 +1989,7 @@ static void ZSTD_refDDict(ZSTD_DCtx* dstDCtx, const ZSTD_DDict* ddict)
 
 static size_t ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict)
 {
+    ddict->dictID = 0;
     ddict->entropyPresent = 0;
     if (ddict->dictSize < 8) return 0;
     {   U32 const magic = MEM_readLE32(ddict->dictContent);

From a419777eb12392159aefd6c28fd1e3cc8862ce2e Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 1 Mar 2017 17:51:56 -0800
Subject: [PATCH 190/223] Allow compressor to repeat Huffman tables

* Compressor saves most recently used Huffman table and reuses it
  if it produces better results.
* I attempted to preserve CPU usage profile.
  I intentionally left all of the existing heuristics in place.
  There is only a speed difference on the second block and later.
  When compressing large enough blocks (say >= 4 KiB) there is
  no significant difference in compression speed.
  Dictionary compression of one block is the same speed for blocks
  with literals <= 1 KiB, and after that the difference is not
  very significant.
* In the synthetic data, with blocks 10 KB or smaller, most blocks
  can't use repeated tables because the previous block did not
  contain a symbol that the current block contains.
  Once blocks are about 12 KB or more, most previous blocks have
  valid Huffman tables for the current block, and the compression
  ratio and decompression speed jumped.
* In silesia blocks as small as 4KB can frequently reuse the
  previous Huffman table (85%), but it isn't as profitable, and
  the previous Huffman table only gets used about 3% of the time.
* Microbenchmarks show that `HUF_validateCTable()` takes ~55 ns
  and `HUF_estimateCompressedSize()` takes ~35 ns.
  They are decently well optimized, the first versions took 90 ns
  and 120 ns respectively. `HUF_validateCTable()` could be twice as
  fast, if we cast the `HUF_CElt*` to a `U32*` and compare to 0.
  However, `U32` has an alignment of 4 instead of 2, so I think that
  might be undefined behavior.
* I've ran `zstreamtest` compiled normally, with UASAN and with MSAN
  for 4 hours each.

The worst case for the speed difference is a bunch of small blocks
in the same frame. I modified `bench.c` to compress the input in a
single frame but with blocks of the given block size, set by `-B`.
Benchmarks on level 1:

|  Program  | Block size |   Corpus  | Ratio | Compression MB/s | Decompression MB/s |
|-----------|------------|-----------|-------|------------------|--------------------|
| zstd.base |        256 | synthetic | 2.364 |            110.0 |              297.0 |
|      zstd |        256 | synthetic | 2.367 |            108.9 |              297.0 |
| zstd.base |        256 | silesia   | 2.204 |             93.8 |              415.7 |
|      zstd |        256 | silesia   | 2.204 |             93.4 |              415.7 |
| zstd.base |        512 | synthetic | 2.594 |            144.2 |              420.0 |
|      zstd |        512 | synthetic | 2.599 |            141.5 |              425.7 |
| zstd.base |        512 | silesia   | 2.358 |            118.4 |              432.6 |
|      zstd |        512 | silesia   | 2.358 |            119.8 |              432.6 |
| zstd.base |       1024 | synthetic | 2.790 |            192.3 |              594.1 |
|      zstd |       1024 | synthetic | 2.794 |            192.3 |              600.0 |
| zstd.base |       1024 | silesia   | 2.524 |            148.2 |              464.2 |
|      zstd |       1024 | silesia   | 2.525 |            148.2 |              467.6 |
| zstd.base |       4096 | synthetic | 3.023 |            300.0 |             1000.0 |
|      zstd |       4096 | synthetic | 3.024 |            300.0 |             1010.1 |
| zstd.base |       4096 | silesia   | 2.779 |            223.1 |              623.5 |
|      zstd |       4096 | silesia   | 2.779 |            223.1 |              636.0 |
| zstd.base |      16384 | synthetic | 3.131 |            350.0 |             1150.1 |
|      zstd |      16384 | synthetic | 3.152 |            350.0 |             1630.3 |
| zstd.base |      16384 | silesia   | 2.871 |            296.5 |              883.3 |
|      zstd |      16384 | silesia   | 2.872 |            294.4 |              898.3 |
---
 lib/common/huf.h             |  15 +++++
 lib/compress/huf_compress.c  | 117 +++++++++++++++++++++++++++--------
 lib/compress/zstd_compress.c |  45 +++++++++-----
 3 files changed, 136 insertions(+), 41 deletions(-)

diff --git a/lib/common/huf.h b/lib/common/huf.h
index 9427ae8cb..89a226ea0 100644
--- a/lib/common/huf.h
+++ b/lib/common/huf.h
@@ -168,6 +168,16 @@ size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSym
 size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
 size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
 
+typedef enum {
+   HUF_repeat_none,  /**< Cannot use the previous table */
+   HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */
+   HUF_repeat_valid  /**< Can use the previous table and it is asumed to be valid */
+ } HUF_repeat;
+/** HUF_compress4X_repeat() :
+*   Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
+*   If it uses hufTable it does not modify hufTable or repeat.
+*   If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. */
+size_t HUF_compress4X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat);  /**< `workSpace` must be a table of at least 1024 unsigned */
 
 /** HUF_buildCTable_wksp() :
  *  Same as HUF_buildCTable(), but using externally allocated scratch buffer.
@@ -216,6 +226,11 @@ size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* c
 size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
 size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);  /**< `workSpace` must be a table of at least 1024 unsigned */
 size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
+/** HUF_compress1X_repeat() :
+*   Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
+*   If it uses hufTable it does not modify hufTable or repeat.
+*   If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. */
+size_t HUF_compress1X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat);  /**< `workSpace` must be a table of at least 1024 unsigned */
 
 size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* single-symbol decoder */
 size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* double-symbol decoder */
diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index 7869ccf64..8b914cbc6 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -409,6 +409,25 @@ size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U3
     return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, nodeTable, sizeof(nodeTable));
 }
 
+static size_t HUF_estimateCompressedSize(HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue)
+{
+    size_t nbBits = 0;
+    int s;
+    for (s = 0; s <= (int)maxSymbolValue; ++s) {
+        nbBits += CTable[s].nbBits * count[s];
+    }
+    return nbBits >> 3;
+}
+
+static int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) {
+  int bad = 0;
+  int s;
+  for (s = 0; s <= (int)maxSymbolValue; ++s) {
+    bad |= (count[s] != 0) & (CTable[s].nbBits == 0);
+  }
+  return !bad;
+}
+
 static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable)
 {
     BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits);
@@ -510,22 +529,37 @@ size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, si
 }
 
 
+static size_t HUF_compressCTable_internal(
+                BYTE* const ostart, BYTE* op, BYTE* const oend,
+                const void* src, size_t srcSize,
+                unsigned singleStream, const HUF_CElt* CTable)
+{
+    size_t const cSize = singleStream ?
+                         HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) :
+                         HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
+    if (HUF_isError(cSize)) { return cSize; }
+    if (cSize==0) { return 0; }   /* uncompressible */
+    op += cSize;
+    /* check compressibility */
+    if ((size_t)(op-ostart) >= srcSize-1) { return 0; }
+    return op-ostart;
+}
+
+
 /* `workSpace` must a table of at least 1024 unsigned */
 static size_t HUF_compress_internal (
                 void* dst, size_t dstSize,
                 const void* src, size_t srcSize,
                 unsigned maxSymbolValue, unsigned huffLog,
                 unsigned singleStream,
-                void* workSpace, size_t wkspSize)
+                void* workSpace, size_t wkspSize, HUF_CElt* oldHufTable, HUF_repeat* repeat)
 {
     BYTE* const ostart = (BYTE*)dst;
     BYTE* const oend = ostart + dstSize;
     BYTE* op = ostart;
 
-    union {
-        U32 count[HUF_SYMBOLVALUE_MAX+1];
-        HUF_CElt CTable[HUF_SYMBOLVALUE_MAX+1];
-    } table;   /* `count` can overlap with `CTable`; saves 1 KB */
+    U32 count[HUF_SYMBOLVALUE_MAX+1];
+    HUF_CElt CTable[HUF_SYMBOLVALUE_MAX+1];
 
     /* checks & inits */
     if (wkspSize < sizeof(huffNodeTable)) return ERROR(GENERIC);
@@ -536,38 +570,51 @@ static size_t HUF_compress_internal (
     if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX;
     if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT;
 
+    /* Heuristic : If we don't need to check the validity of the old table use the old table for small inputs */
+    if (srcSize <= 1024 && repeat && *repeat == HUF_repeat_valid) {
+        return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
+    }
+
     /* Scan input and build symbol stats */
-    {   CHECK_V_F(largest, FSE_count_wksp (table.count, &maxSymbolValue, (const BYTE*)src, srcSize, (U32*)workSpace) );
+    {   CHECK_V_F(largest, FSE_count_wksp (count, &maxSymbolValue, (const BYTE*)src, srcSize, (U32*)workSpace) );
         if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; }   /* single symbol, rle */
         if (largest <= (srcSize >> 7)+1) return 0;   /* Fast heuristic : not compressible enough */
     }
 
+    /* Check validity of previous table */
+    if (repeat && *repeat == HUF_repeat_check && !HUF_validateCTable(oldHufTable, count, maxSymbolValue)) {
+        *repeat = HUF_repeat_none;
+    }
+    /* Heuristic : use existing table for small inputs */
+    if (srcSize <= 1024 && repeat && *repeat != HUF_repeat_none) {
+        return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
+    }
+
     /* Build Huffman Tree */
     huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
-    {   CHECK_V_F(maxBits, HUF_buildCTable_wksp (table.CTable, table.count, maxSymbolValue, huffLog, workSpace, wkspSize) );
+    {   CHECK_V_F(maxBits, HUF_buildCTable_wksp (CTable, count, maxSymbolValue, huffLog, workSpace, wkspSize) );
         huffLog = (U32)maxBits;
+        /* Zero the unused symbols so we can check it for validity */
+        memset(CTable + maxSymbolValue + 1, 0, sizeof(CTable) - (maxSymbolValue + 1) * sizeof(HUF_CElt));
     }
 
     /* Write table description header */
-    {   CHECK_V_F(hSize, HUF_writeCTable (op, dstSize, table.CTable, maxSymbolValue, huffLog) );
-        if (hSize + 12 >= srcSize) return 0;   /* not useful to try compression */
+    {   CHECK_V_F(hSize, HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog) );
+        /* Check if using the previous table will be beneficial */
+        if (repeat && *repeat != HUF_repeat_none) {
+            size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, count, maxSymbolValue);
+            size_t const newSize = HUF_estimateCompressedSize(CTable, count, maxSymbolValue);
+            if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) {
+                return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
+            }
+        }
+        /* Use the new table */
+        if (hSize + 12ul >= srcSize) { return 0; }
         op += hSize;
+        if (repeat) { *repeat = HUF_repeat_none; }
+        if (oldHufTable) { memcpy(oldHufTable, CTable, sizeof(CTable)); } /* Save the new table */
     }
-
-    /* Compress */
-    {   size_t const cSize = (singleStream) ?
-                            HUF_compress1X_usingCTable(op, oend - op, src, srcSize, table.CTable) :   /* single segment */
-                            HUF_compress4X_usingCTable(op, oend - op, src, srcSize, table.CTable);
-        if (HUF_isError(cSize)) return cSize;
-        if (cSize==0) return 0;   /* uncompressible */
-        op += cSize;
-    }
-
-    /* check compressibility */
-    if ((size_t)(op-ostart) >= srcSize-1)
-        return 0;
-
-    return op-ostart;
+    return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, CTable);
 }
 
 
@@ -576,7 +623,16 @@ size_t HUF_compress1X_wksp (void* dst, size_t dstSize,
                       unsigned maxSymbolValue, unsigned huffLog,
                       void* workSpace, size_t wkspSize)
 {
-    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize);
+    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, NULL, NULL);
+}
+
+size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
+                      const void* src, size_t srcSize,
+                      unsigned maxSymbolValue, unsigned huffLog,
+                      void* workSpace, size_t wkspSize,
+                      HUF_CElt* hufTable, HUF_repeat* repeat)
+{
+    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, hufTable, repeat);
 }
 
 size_t HUF_compress1X (void* dst, size_t dstSize,
@@ -592,7 +648,16 @@ size_t HUF_compress4X_wksp (void* dst, size_t dstSize,
                       unsigned maxSymbolValue, unsigned huffLog,
                       void* workSpace, size_t wkspSize)
 {
-    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize);
+    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, NULL, NULL);
+}
+
+size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
+                      const void* src, size_t srcSize,
+                      unsigned maxSymbolValue, unsigned huffLog,
+                      void* workSpace, size_t wkspSize,
+                      HUF_CElt* hufTable, HUF_repeat* repeat)
+{
+    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, hufTable, repeat);
 }
 
 size_t HUF_compress2 (void* dst, size_t dstSize,
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index ec758db00..538cb685c 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -81,6 +81,7 @@ struct ZSTD_CCtx_s {
     U32* chainTable;
     HUF_CElt* hufTable;
     U32 flagStaticTables;
+    HUF_repeat flagStaticHufTable;
     FSE_CTable offcodeCTable  [FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
     FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
     FSE_CTable litlengthCTable  [FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
@@ -254,8 +255,11 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
                                        ZSTD_compResetPolicy_e const crp)
 {
     if (crp == ZSTDcrp_continue)
-        if (ZSTD_equivalentParams(params, zc->params))
+        if (ZSTD_equivalentParams(params, zc->params)) {
+            zc->flagStaticTables = 0;
+            zc->flagStaticHufTable = HUF_repeat_none;
             return ZSTD_continueCCtx(zc, params, frameContentSize);
+        }
 
     {   size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog);
         U32    const divider = (params.cParams.searchLength==3) ? 3 : 4;
@@ -289,6 +293,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
         ptr = zc->hashTable3 + h3Size;
         zc->hufTable = (HUF_CElt*)ptr;
         zc->flagStaticTables = 0;
+        zc->flagStaticHufTable = HUF_repeat_none;
         ptr = ((U32*)ptr) + 256;  /* note : HUF_CElt* is incomplete type, size is simulated using U32 */
 
         zc->nextToUpdate = 1;
@@ -374,12 +379,15 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long
 
     /* copy entropy tables */
     dstCCtx->flagStaticTables = srcCCtx->flagStaticTables;
+    dstCCtx->flagStaticHufTable = srcCCtx->flagStaticHufTable;
     if (srcCCtx->flagStaticTables) {
-        memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256*4);
         memcpy(dstCCtx->litlengthCTable, srcCCtx->litlengthCTable, sizeof(dstCCtx->litlengthCTable));
         memcpy(dstCCtx->matchlengthCTable, srcCCtx->matchlengthCTable, sizeof(dstCCtx->matchlengthCTable));
         memcpy(dstCCtx->offcodeCTable, srcCCtx->offcodeCTable, sizeof(dstCCtx->offcodeCTable));
     }
+    if (srcCCtx->flagStaticHufTable) {
+        memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256*4);
+    }
 
     return 0;
 }
@@ -493,24 +501,27 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc,
 
     /* small ? don't even attempt compression (speed opt) */
 #   define LITERAL_NOENTROPY 63
-    {   size_t const minLitSize = zc->flagStaticTables ? 6 : LITERAL_NOENTROPY;
+    {   size_t const minLitSize = zc->flagStaticHufTable == HUF_repeat_valid ? 6 : LITERAL_NOENTROPY;
         if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
     }
 
     if (dstCapacity < lhSize+1) return ERROR(dstSize_tooSmall);   /* not enough space for compression */
-    if (zc->flagStaticTables && (lhSize==3)) {
-        hType = set_repeat;
-        singleStream = 1;
-        cLitSize = HUF_compress1X_usingCTable(ostart+lhSize, dstCapacity-lhSize, src, srcSize, zc->hufTable);
-    } else {
-        cLitSize = singleStream ? HUF_compress1X_wksp(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters))
-                                : HUF_compress4X_wksp(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters));
+    {   HUF_repeat repeat = zc->flagStaticHufTable;
+        if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
+        cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat)
+                                : HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat);
+        if (repeat != HUF_repeat_none) { hType = set_repeat; }    /* reused the existing table */
+        else { zc->flagStaticHufTable = HUF_repeat_check; }       /* now have a table to reuse */
     }
 
-    if ((cLitSize==0) | (cLitSize >= srcSize - minGain))
+    if ((cLitSize==0) | (cLitSize >= srcSize - minGain)) {
+        zc->flagStaticHufTable = HUF_repeat_none;
         return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize);
-    if (cLitSize==1)
+    }
+    if (cLitSize==1) {
+        zc->flagStaticHufTable = HUF_repeat_none;
         return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize);
+    }
 
     /* Build header */
     switch(lhSize)
@@ -753,9 +764,12 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
 
     /* check compressibility */
 _check_compressibility:
-    { size_t const minGain = ZSTD_minGain(srcSize);
-      size_t const maxCSize = srcSize - minGain;
-      if ((size_t)(op-ostart) >= maxCSize) return 0; }
+    {   size_t const minGain = ZSTD_minGain(srcSize);
+        size_t const maxCSize = srcSize - minGain;
+        if ((size_t)(op-ostart) >= maxCSize) {
+            zc->flagStaticHufTable = HUF_repeat_none;
+            return 0;
+    }   }
 
     /* confirm repcodes */
     { int i; for (i=0; irep[i] = zc->repToConfirm[i]; }
@@ -2606,6 +2620,7 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_
     }
 
     cctx->flagStaticTables = 1;
+    cctx->flagStaticHufTable = HUF_repeat_valid;
     return dictPtr - (const BYTE*)dict;
 }
 

From 553f67e0c1f62c06a5e9e6efc1016feb8bb7b398 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Thu, 2 Mar 2017 15:15:31 -0800
Subject: [PATCH 191/223] Remove 'generic' inline strategy

Seems to avoid performance loss for compression.
Same strategy tested on decompression side, did not appear to improve
speed.
---
 lib/compress/zstd_compress.c | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 5d81f8a38..6b8047206 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -576,11 +576,11 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr)
         mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
 }
 
-
-FORCE_INLINE size_t ZSTD_compressSequences_generic (ZSTD_CCtx* zc,
+MEM_STATIC size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
                               void* dst, size_t dstCapacity,
-                              size_t srcSize, int const longOffsets)
+                              size_t srcSize)
 {
+    const int longOffsets = zc->params.cParams.windowLog > STREAM_ACCUMULATOR_MIN;
     const seqStore_t* seqStorePtr = &(zc->seqStore);
     U32 count[MaxSeq+1];
     S16 norm[MaxSeq+1];
@@ -782,17 +782,6 @@ _check_compressibility:
     return op - ostart;
 }
 
-FORCE_INLINE size_t ZSTD_compressSequences (ZSTD_CCtx* zc,
-                              void* dst, size_t dstCapacity,
-                              size_t srcSize)
-{
-    if (zc->params.cParams.windowLog > STREAM_ACCUMULATOR_MIN) {
-        return ZSTD_compressSequences_generic(zc, dst, dstCapacity, srcSize, 1);
-    } else {
-        return ZSTD_compressSequences_generic(zc, dst, dstCapacity, srcSize, 0);
-    }
-}
-
 #if 0 /* for debug */
 #  define STORESEQ_DEBUG
 #include    /* fprintf */

From 976e325b2e4baf69ba1d20013def951f7c7382ab Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Thu, 2 Mar 2017 15:54:39 -0800
Subject: [PATCH 192/223] Fix COVER_optimizeTrainFromBuffer() resource leaks

Thanks to @nemequ for reporting the resource leaks.
---
 lib/dictBuilder/cover.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c
index 1ced645b5..3a7b9f39f 100644
--- a/lib/dictBuilder/cover.c
+++ b/lib/dictBuilder/cover.c
@@ -966,6 +966,7 @@ ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void *dictBuffer,
     if (!COVER_ctx_init(&ctx, samplesBuffer, samplesSizes, nbSamples, d)) {
       LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to initialize context\n");
       COVER_best_destroy(&best);
+      POOL_free(pool);
       return ERROR(GENERIC);
     }
     /* Loop through k reusing the same context */
@@ -978,6 +979,7 @@ ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void *dictBuffer,
         LOCALDISPLAYLEVEL(displayLevel, 1, "Failed to allocate parameters\n");
         COVER_best_destroy(&best);
         COVER_ctx_destroy(&ctx);
+        POOL_free(pool);
         return ERROR(GENERIC);
       }
       data->ctx = &ctx;
@@ -990,6 +992,7 @@ ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void *dictBuffer,
       /* Check the parameters */
       if (!COVER_checkParameters(data->parameters)) {
         DISPLAYLEVEL(1, "Cover parameters incorrect\n");
+        free(data);
         continue;
       }
       /* Call the function and pass ownership of data to it */
@@ -1012,8 +1015,10 @@ ZDICTLIB_API size_t COVER_optimizeTrainFromBuffer(void *dictBuffer,
   {
     const size_t dictSize = best.dictSize;
     if (ZSTD_isError(best.compressedSize)) {
+      const size_t compressedSize = best.compressedSize;
       COVER_best_destroy(&best);
-      return best.compressedSize;
+      POOL_free(pool);
+      return compressedSize;
     }
     *parameters = best.parameters;
     memcpy(dictBuffer, best.dict, dictSize);

From d051cd5b430e60e8ea8edc89ff1f73844378c3ff Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Thu, 2 Mar 2017 16:38:07 -0800
Subject: [PATCH 193/223] Use workspace for count and CTable

---
 lib/common/huf.h             | 12 ++++++++----
 lib/compress/huf_compress.c  | 19 ++++++++++++++-----
 lib/compress/zstd_compress.c |  2 +-
 3 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/lib/common/huf.h b/lib/common/huf.h
index 89a226ea0..1315f1190 100644
--- a/lib/common/huf.h
+++ b/lib/common/huf.h
@@ -91,7 +91,7 @@ size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize
 
 /** HUF_compress4X_wksp() :
 *   Same as HUF_compress2(), but uses externally allocated `workSpace`, which must be a table of >= 1024 unsigned */
-size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);  /**< `workSpace` must be a table of at least 1024 unsigned */
+size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);  /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
 
 
 
@@ -133,6 +133,10 @@ typedef U32 HUF_DTable;
 #define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
         HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) }
 
+/* The workspace must have alignment at least 4 and be at least this large */
+#define HUF_WORKSPACE_SIZE (6 << 10)
+#define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32))
+
 
 /* ****************************************
 *  Advanced decompression functions
@@ -177,7 +181,7 @@ typedef enum {
 *   Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
 *   If it uses hufTable it does not modify hufTable or repeat.
 *   If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. */
-size_t HUF_compress4X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat);  /**< `workSpace` must be a table of at least 1024 unsigned */
+size_t HUF_compress4X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat);  /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
 
 /** HUF_buildCTable_wksp() :
  *  Same as HUF_buildCTable(), but using externally allocated scratch buffer.
@@ -224,13 +228,13 @@ size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* c
 /* single stream variants */
 
 size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
-size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);  /**< `workSpace` must be a table of at least 1024 unsigned */
+size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize);  /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
 size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
 /** HUF_compress1X_repeat() :
 *   Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
 *   If it uses hufTable it does not modify hufTable or repeat.
 *   If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. */
-size_t HUF_compress1X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat);  /**< `workSpace` must be a table of at least 1024 unsigned */
+size_t HUF_compress1X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat);  /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
 
 size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* single-symbol decoder */
 size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* double-symbol decoder */
diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index 8b914cbc6..d103c3b22 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -558,11 +558,13 @@ static size_t HUF_compress_internal (
     BYTE* const oend = ostart + dstSize;
     BYTE* op = ostart;
 
-    U32 count[HUF_SYMBOLVALUE_MAX+1];
-    HUF_CElt CTable[HUF_SYMBOLVALUE_MAX+1];
+    U32* count;
+    size_t const countSize = sizeof(U32) * (HUF_SYMBOLVALUE_MAX + 1);
+    HUF_CElt* CTable;
+    size_t const CTableSize = sizeof(HUF_CElt) * (HUF_SYMBOLVALUE_MAX + 1);
 
     /* checks & inits */
-    if (wkspSize < sizeof(huffNodeTable)) return ERROR(GENERIC);
+    if (wkspSize < sizeof(huffNodeTable) + countSize + CTableSize) return ERROR(GENERIC);
     if (!srcSize) return 0;  /* Uncompressed (note : 1 means rle, so first byte must be correct) */
     if (!dstSize) return 0;  /* cannot fit within dst budget */
     if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong);   /* current block size limit */
@@ -570,6 +572,13 @@ static size_t HUF_compress_internal (
     if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX;
     if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT;
 
+    count = (U32*)workSpace;
+    workSpace = (BYTE*)workSpace + countSize;
+    wkspSize -= countSize;
+    CTable = (HUF_CElt*)workSpace;
+    workSpace = (BYTE*)workSpace + CTableSize;
+    wkspSize -= CTableSize;
+
     /* Heuristic : If we don't need to check the validity of the old table use the old table for small inputs */
     if (srcSize <= 1024 && repeat && *repeat == HUF_repeat_valid) {
         return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
@@ -595,7 +604,7 @@ static size_t HUF_compress_internal (
     {   CHECK_V_F(maxBits, HUF_buildCTable_wksp (CTable, count, maxSymbolValue, huffLog, workSpace, wkspSize) );
         huffLog = (U32)maxBits;
         /* Zero the unused symbols so we can check it for validity */
-        memset(CTable + maxSymbolValue + 1, 0, sizeof(CTable) - (maxSymbolValue + 1) * sizeof(HUF_CElt));
+        memset(CTable + maxSymbolValue + 1, 0, CTableSize - (maxSymbolValue + 1) * sizeof(HUF_CElt));
     }
 
     /* Write table description header */
@@ -612,7 +621,7 @@ static size_t HUF_compress_internal (
         if (hSize + 12ul >= srcSize) { return 0; }
         op += hSize;
         if (repeat) { *repeat = HUF_repeat_none; }
-        if (oldHufTable) { memcpy(oldHufTable, CTable, sizeof(CTable)); } /* Save the new table */
+        if (oldHufTable) { memcpy(oldHufTable, CTable, CTableSize); } /* Save the new table */
     }
     return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, CTable);
 }
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 538cb685c..a77d674b2 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -85,7 +85,7 @@ struct ZSTD_CCtx_s {
     FSE_CTable offcodeCTable  [FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)];
     FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)];
     FSE_CTable litlengthCTable  [FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)];
-    unsigned tmpCounters[1024];
+    unsigned tmpCounters[HUF_WORKSPACE_SIZE_U32];
 };
 
 ZSTD_CCtx* ZSTD_createCCtx(void)

From fe5d27062e83f6f55fdee3c80f31f51fbf10f7bb Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 2 Mar 2017 17:09:21 -0800
Subject: [PATCH 194/223] disable prefetch-decode for 32-bits target

This decoder variant is detrimental to x86 architecture
likely due to register pressure.

Note that the variant is disabled for all 32-bits targets.
It's unclear if it would help for different architectures,
such as ARM, MIPS or PowerPC.
---
 lib/decompress/zstd_decompress.c | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 4dfdf20fb..482c334ff 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1410,13 +1410,18 @@ static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
 
     if (srcSize >= ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(srcSize_wrong);
 
-    /* Decode literals sub-block */
+    /* Decode literals section */
     {   size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize);
         if (ZSTD_isError(litCSize)) return litCSize;
         ip += litCSize;
         srcSize -= litCSize;
     }
-    if (dctx->fParams.windowSize > (1<<23)) return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize);
+    if (sizeof(size_t) > 4)  /* do not enable prefetching on 32-bits x86, as it's performance detrimental */
+                             /* likely because of register pressure */
+                             /* if that's the correct cause, then 32-bits ARM should be affected differently */
+                             /* it would be good to test this on ARM real hardware, to see if prefetch version improves speed */
+        if (dctx->fParams.windowSize > (1<<23))
+            return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize);
     return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize);
 }
 

From e02409fdc3bf56243c7106d45faa1fd02b1dbaf7 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Thu, 2 Mar 2017 17:14:57 -0800
Subject: [PATCH 195/223] update NEWS on @iburinoc's 32-bits version
 improvement

---
 NEWS | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/NEWS b/NEWS
index 8b7d83c8a..760724a25 100644
--- a/NEWS
+++ b/NEWS
@@ -5,7 +5,8 @@ cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77
 cli : fix : --rm remains silent when input is stdin
 speed : improved decompression speed in streaming mode for single shot scenarios (+5%)
 memory : DDict (decompression dictionary) memory usage down from 150 KB to 20 KB
-API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize(), by Sean Purcell
+arch : 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell
+API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize()
 build: new: meson build system in contrib/meson, by Dima Krasner
 build: improved cmake script, by @Majlen
 doc : new : educational decoder, by Sean Purcell

From 54c4babd8f67d40cbc63c2cd2cd77914ec80d4ee Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Fri, 3 Mar 2017 12:30:24 -0800
Subject: [PATCH 196/223] Always check Huffman tables for ZSTD_lazy+

The compressor always reuses the existing Huffman table if the literals
size is at most 1 KiB. If the compression strategy is `ZSTD_lazy` or
stronger always check to see if reusing the previous table or creating
a new table is better.

This doesn't yet weigh in decompression speed. I don't want to add any
heuristics there until I have real data to work with to ensure that the
heuristic works for at least one use case, preferably more.
---
 lib/common/huf.h             | 10 ++++++----
 lib/compress/huf_compress.c  | 19 ++++++++++---------
 lib/compress/zstd_compress.c |  5 +++--
 3 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/lib/common/huf.h b/lib/common/huf.h
index 1315f1190..691f2e764 100644
--- a/lib/common/huf.h
+++ b/lib/common/huf.h
@@ -180,8 +180,9 @@ typedef enum {
 /** HUF_compress4X_repeat() :
 *   Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
 *   If it uses hufTable it does not modify hufTable or repeat.
-*   If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. */
-size_t HUF_compress4X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat);  /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
+*   If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
+*   If preferRepeat then the old table will always be used if valid. */
+size_t HUF_compress4X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat);  /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
 
 /** HUF_buildCTable_wksp() :
  *  Same as HUF_buildCTable(), but using externally allocated scratch buffer.
@@ -233,8 +234,9 @@ size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, si
 /** HUF_compress1X_repeat() :
 *   Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none.
 *   If it uses hufTable it does not modify hufTable or repeat.
-*   If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. */
-size_t HUF_compress1X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat);  /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
+*   If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used.
+*   If preferRepeat then the old table will always be used if valid. */
+size_t HUF_compress1X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat);  /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
 
 size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* single-symbol decoder */
 size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* double-symbol decoder */
diff --git a/lib/compress/huf_compress.c b/lib/compress/huf_compress.c
index d103c3b22..fe11aafb8 100644
--- a/lib/compress/huf_compress.c
+++ b/lib/compress/huf_compress.c
@@ -552,7 +552,8 @@ static size_t HUF_compress_internal (
                 const void* src, size_t srcSize,
                 unsigned maxSymbolValue, unsigned huffLog,
                 unsigned singleStream,
-                void* workSpace, size_t wkspSize, HUF_CElt* oldHufTable, HUF_repeat* repeat)
+                void* workSpace, size_t wkspSize,
+                HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat)
 {
     BYTE* const ostart = (BYTE*)dst;
     BYTE* const oend = ostart + dstSize;
@@ -580,7 +581,7 @@ static size_t HUF_compress_internal (
     wkspSize -= CTableSize;
 
     /* Heuristic : If we don't need to check the validity of the old table use the old table for small inputs */
-    if (srcSize <= 1024 && repeat && *repeat == HUF_repeat_valid) {
+    if (preferRepeat && repeat && *repeat == HUF_repeat_valid) {
         return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
     }
 
@@ -595,7 +596,7 @@ static size_t HUF_compress_internal (
         *repeat = HUF_repeat_none;
     }
     /* Heuristic : use existing table for small inputs */
-    if (srcSize <= 1024 && repeat && *repeat != HUF_repeat_none) {
+    if (preferRepeat && repeat && *repeat != HUF_repeat_none) {
         return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, singleStream, oldHufTable);
     }
 
@@ -632,16 +633,16 @@ size_t HUF_compress1X_wksp (void* dst, size_t dstSize,
                       unsigned maxSymbolValue, unsigned huffLog,
                       void* workSpace, size_t wkspSize)
 {
-    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, NULL, NULL);
+    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, NULL, NULL, 0);
 }
 
 size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
                       const void* src, size_t srcSize,
                       unsigned maxSymbolValue, unsigned huffLog,
                       void* workSpace, size_t wkspSize,
-                      HUF_CElt* hufTable, HUF_repeat* repeat)
+                      HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat)
 {
-    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, hufTable, repeat);
+    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize, hufTable, repeat, preferRepeat);
 }
 
 size_t HUF_compress1X (void* dst, size_t dstSize,
@@ -657,16 +658,16 @@ size_t HUF_compress4X_wksp (void* dst, size_t dstSize,
                       unsigned maxSymbolValue, unsigned huffLog,
                       void* workSpace, size_t wkspSize)
 {
-    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, NULL, NULL);
+    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, NULL, NULL, 0);
 }
 
 size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
                       const void* src, size_t srcSize,
                       unsigned maxSymbolValue, unsigned huffLog,
                       void* workSpace, size_t wkspSize,
-                      HUF_CElt* hufTable, HUF_repeat* repeat)
+                      HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat)
 {
-    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, hufTable, repeat);
+    return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize, hufTable, repeat, preferRepeat);
 }
 
 size_t HUF_compress2 (void* dst, size_t dstSize,
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 432323e21..15f063e18 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -505,9 +505,10 @@ static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc,
 
     if (dstCapacity < lhSize+1) return ERROR(dstSize_tooSmall);   /* not enough space for compression */
     {   HUF_repeat repeat = zc->flagStaticHufTable;
+        int const preferRepeat = zc->params.cParams.strategy < ZSTD_lazy ? srcSize <= 1024 : 0;
         if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1;
-        cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat)
-                                : HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat);
+        cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat)
+                                : HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters), zc->hufTable, &repeat, preferRepeat);
         if (repeat != HUF_repeat_none) { hType = set_repeat; }    /* reused the existing table */
         else { zc->flagStaticHufTable = HUF_repeat_check; }       /* now have a table to reuse */
     }

From 38a3428b37a3b21d2a699b8e1db1f10af57588b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?P=C3=A1draig=20Brady?= 
Date: Sun, 5 Mar 2017 19:36:56 -0800
Subject: [PATCH 197/223] support -Werror=format-security

Fedora now enables this option by default, resulting
in the following build failure:

Logging.h: In instantiation of
'void pzstd::Logger::operator()(int, const char*, Args ...)
Pzstd.cpp:413:48:   required from here
Logging.h:46:17: error: format not a string literal and no format arguments
[-Werror=format-security]
     std::fprintf(out_, fmt, args...);
     ~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~
---
 .buckconfig             | 2 +-
 contrib/pzstd/Pzstd.cpp | 6 +++---
 contrib/pzstd/Pzstd.h   | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.buckconfig b/.buckconfig
index b2b9c036f..d698b35ba 100644
--- a/.buckconfig
+++ b/.buckconfig
@@ -2,7 +2,7 @@
   cppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=1
   cflags = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wpointer-arith
   cxxppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=1
-  cxxflags = -std=c++11 -Wno-format-security -Wno-deprecated-declarations
+  cxxflags = -std=c++11 -Wno-deprecated-declarations
   gtest_dep = //contrib/pzstd:gtest
 
 [httpserver]
diff --git a/contrib/pzstd/Pzstd.cpp b/contrib/pzstd/Pzstd.cpp
index f4cb19d9c..1265b53ef 100644
--- a/contrib/pzstd/Pzstd.cpp
+++ b/contrib/pzstd/Pzstd.cpp
@@ -410,7 +410,7 @@ std::uint64_t asyncCompressChunks(
     });
     // Pass the output queue to the writer thread.
     chunks.push(std::move(out));
-    state.log(VERBOSE, "Starting a new frame\n");
+    state.log(VERBOSE, "%s\n", "Starting a new frame");
     // Fill the input queue for the compression job we just started
     status = readData(*in, ZSTD_CStreamInSize(), step, fd, &bytesRead);
   }
@@ -547,8 +547,8 @@ std::uint64_t asyncDecompressFrames(
     if (frameSize == 0) {
       // We hit a non SkippableFrame ==> not compressed by pzstd or corrupted
       // Pass the rest of the source to this decompression task
-      state.log(VERBOSE,
-          "Input not in pzstd format, falling back to serial decompression\n");
+      state.log(VERBOSE, "%s\n",
+          "Input not in pzstd format, falling back to serial decompression");
       while (status == FileStatus::Continue && !state.errorHolder.hasError()) {
         status = readData(*in, chunkSize, chunkSize, fd, &totalBytesRead);
       }
diff --git a/contrib/pzstd/Pzstd.h b/contrib/pzstd/Pzstd.h
index dc60dd9b8..1e29a7170 100644
--- a/contrib/pzstd/Pzstd.h
+++ b/contrib/pzstd/Pzstd.h
@@ -41,7 +41,7 @@ class SharedState {
       auto parameters = options.determineParameters();
       cStreamPool.reset(new ResourcePool{
           [this, parameters]() -> ZSTD_CStream* {
-            this->log(VERBOSE, "Creating new ZSTD_CStream\n");
+            this->log(VERBOSE, "%s\n", "Creating new ZSTD_CStream");
             auto zcs = ZSTD_createCStream();
             if (zcs) {
               auto err = ZSTD_initCStream_advanced(
@@ -59,7 +59,7 @@ class SharedState {
     } else {
       dStreamPool.reset(new ResourcePool{
           [this]() -> ZSTD_DStream* {
-            this->log(VERBOSE, "Creating new ZSTD_DStream\n");
+            this->log(VERBOSE, "%s\n", "Creating new ZSTD_DStream");
             auto zds = ZSTD_createDStream();
             if (zds) {
               auto err = ZSTD_initDStream(zds);

From 1f2c95c5f3f6995d0efdca18577da74336e8e586 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sun, 5 Mar 2017 21:07:20 -0800
Subject: [PATCH 198/223] minor code refactor in HUF module

---
 lib/common/entropy_common.c     | 14 ++------------
 lib/common/huf.h                |  7 ++++---
 lib/decompress/huf_decompress.c | 10 +++++-----
 3 files changed, 11 insertions(+), 20 deletions(-)

diff --git a/lib/common/entropy_common.c b/lib/common/entropy_common.c
index 72bc398da..b37a082fe 100644
--- a/lib/common/entropy_common.c
+++ b/lib/common/entropy_common.c
@@ -43,25 +43,15 @@
 #include "huf.h"
 
 
-/*-****************************************
-*  Version
-******************************************/
+/*===   Version   ===*/
 unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; }
 
 
-/*-****************************************
-*  FSE Error Management
-******************************************/
+/*===   Error Management   ===*/
 unsigned FSE_isError(size_t code) { return ERR_isError(code); }
-
 const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); }
 
-
-/* **************************************************************
-*  HUF Error Management
-****************************************************************/
 unsigned HUF_isError(size_t code) { return ERR_isError(code); }
-
 const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
 
 
diff --git a/lib/common/huf.h b/lib/common/huf.h
index 691f2e764..e5572760a 100644
--- a/lib/common/huf.h
+++ b/lib/common/huf.h
@@ -102,10 +102,11 @@ size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t s
 
 
 /* *** Constants *** */
-#define HUF_TABLELOG_ABSOLUTEMAX  15   /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
-#define HUF_TABLELOG_MAX  12           /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
+#define HUF_TABLELOG_MAX      12       /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
 #define HUF_TABLELOG_DEFAULT  11       /* tableLog by default, when not specified */
-#define HUF_SYMBOLVALUE_MAX 255
+#define HUF_SYMBOLVALUE_MAX  255
+
+#define HUF_TABLELOG_ABSOLUTEMAX  15   /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
 #if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
 #  error "HUF_TABLELOG_MAX is too large !"
 #endif
diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c
index 0f11f5c1b..6e76cc435 100644
--- a/lib/decompress/huf_decompress.c
+++ b/lib/decompress/huf_decompress.c
@@ -108,10 +108,10 @@ size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize)
         memcpy(DTable, &dtd, sizeof(dtd));
     }
 
-    /* Prepare ranks */
+    /* Calculate starting value for each rank */
     {   U32 n, nextRankStart = 0;
         for (n=1; n> 1;
-            U32 i;
+            U32 u;
             HUF_DEltX2 D;
             D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w);
-            for (i = rankVal[w]; i < rankVal[w] + length; i++)
-                dt[i] = D;
+            for (u = rankVal[w]; u < rankVal[w] + length; u++)
+                dt[u] = D;
             rankVal[w] += length;
     }   }
 

From 8b1d004031b2b78b535b6ce369510e1b91f87cc3 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Sun, 5 Mar 2017 21:17:32 -0800
Subject: [PATCH 199/223] added -Wformat-security flag, as recommended by
 @pixelb

---
 Makefile          | 6 +++++-
 NEWS              | 1 +
 lib/Makefile      | 2 +-
 programs/Makefile | 2 +-
 tests/Makefile    | 5 +++--
 5 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index 0f97e6c5b..f0492d944 100644
--- a/Makefile
+++ b/Makefile
@@ -23,7 +23,7 @@ EXT =
 endif
 
 .PHONY: default
-default: lib zstd-release
+default: lib-release zstd-release
 
 .PHONY: all
 all: | allmost examples manual
@@ -42,6 +42,10 @@ all32:
 
 .PHONY: lib
 lib:
+	@$(MAKE) -C $(ZSTDDIR) $@
+
+.PHONY: lib-release
+lib-release:
 	@$(MAKE) -C $(ZSTDDIR)
 
 .PHONY: zstd
diff --git a/NEWS b/NEWS
index 760724a25..9073a8724 100644
--- a/NEWS
+++ b/NEWS
@@ -9,6 +9,7 @@ arch : 32-bits variant able to generate and decode very long matches (>32 MB), b
 API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize()
 build: new: meson build system in contrib/meson, by Dima Krasner
 build: improved cmake script, by @Majlen
+build: added -Wformat-security flag, as recommended by Padraig Brady
 doc : new : educational decoder, by Sean Purcell
 
 v1.1.3
diff --git a/lib/Makefile b/lib/Makefile
index 05dd2bc9a..58f99baf5 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -24,7 +24,7 @@ CPPFLAGS+= -I. -I./common -DXXH_NAMESPACE=ZSTD_
 CFLAGS  ?= -O3
 DEBUGFLAGS = -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
            -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-           -Wstrict-prototypes -Wundef -Wpointer-arith
+           -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security
 CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
 FLAGS    = $(CPPFLAGS) $(CFLAGS)
 
diff --git a/programs/Makefile b/programs/Makefile
index 407a4f374..5620a25a3 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -30,7 +30,7 @@ CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
 CFLAGS  ?= -O3
 DEBUGFLAGS = -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
           -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
-          -Wstrict-prototypes -Wundef -Wpointer-arith
+          -Wstrict-prototypes -Wundef -Wpointer-arith -Wformat-security
 CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
 FLAGS    = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
 
diff --git a/tests/Makefile b/tests/Makefile
index 88c58fce4..8b19aa3d5 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -28,8 +28,9 @@ TESTARTEFACT := versionsTest namespaceTest
 
 CPPFLAGS+= -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
 CFLAGS  ?= -O3
-CFLAGS  += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
-           -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef
+CFLAGS  += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+           -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
+           -Wstrict-prototypes -Wundef -Wformat-security
 CFLAGS  += $(MOREFLAGS)
 FLAGS    = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
 

From 3437bf2febb7e11be3550d3658cce31c6e10039e Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Wed, 1 Mar 2017 16:10:26 -0800
Subject: [PATCH 200/223] Add build targets to the Makefile, and update
 CircleCI tests

---
 Makefile                     | 71 ++++++++++++++++++++++++++++++------
 circle.yml                   | 43 +++++++++++-----------
 lib/compress/zstd_compress.c |  2 +-
 3 files changed, 82 insertions(+), 34 deletions(-)

diff --git a/Makefile b/Makefile
index f0492d944..e10d29267 100644
--- a/Makefile
+++ b/Makefile
@@ -35,6 +35,13 @@ allmost:
 	$(MAKE) -C $(TESTDIR) all
 	$(MAKE) -C $(ZWRAPDIR) all
 
+#skip zwrapper, can't build that on alternate architectures without the proper zlib installed
+.PHONY: allarch
+allarch:
+	$(MAKE) -C $(ZSTDDIR) all
+	$(MAKE) -C $(PRGDIR) all
+	$(MAKE) -C $(TESTDIR) all
+
 .PHONY: all32
 all32:
 	$(MAKE) -C $(PRGDIR) zstd32
@@ -94,7 +101,6 @@ clean:
 	@$(RM) zstd$(EXT) zstdmt$(EXT) tmp*
 	@echo Cleaning completed
 
-
 #------------------------------------------------------------------------------
 # make install is validated only for Linux, OSX, Hurd and some BSD targets
 #------------------------------------------------------------------------------
@@ -113,9 +119,41 @@ uninstall:
 travis-install:
 	$(MAKE) install PREFIX=~/install_test_dir
 
-gpptest: clean
+gppbuild: clean
+	g++ -v
 	CC=g++ $(MAKE) -C programs all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
 
+gcc5build: clean
+	gcc-5 -v
+	CC=gcc-5 $(MAKE) all MOREFLAGS="-Werror"
+
+gcc6build: clean
+	gcc-6 -v
+	CC=gcc-6 $(MAKE) all MOREFLAGS="-Werror"
+
+clangbuild: clean
+	clang -v
+	CXX=clang++ CC=clang $(MAKE) all MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
+
+m32build: clean
+	gcc -v
+	$(MAKE) all32
+
+armbuild: clean
+	CC=arm-linux-gnueabi-gcc CFLAGS="-Werror" $(MAKE) allarch
+
+aarch64build: clean
+	CC=aarch64-linux-gnu-gcc CFLAGS="-Werror" $(MAKE) allarch
+
+ppcbuild: clean
+	CC=powerpc-linux-gnu-gcc CLAGS="-m32 -Wno-attributes -Werror" $(MAKE) allarch
+
+ppc64build: clean
+	CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allarch
+
+gpptest: clean
+	CC=g++ $(MAKE) -C $(PRGDIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
+
 gcc5test: clean
 	gcc-5 -v
 	$(MAKE) all CC=gcc-5 MOREFLAGS="-Werror"
@@ -126,7 +164,7 @@ gcc6test: clean
 
 clangtest: clean
 	clang -v
-	$(MAKE) all CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
+	$(MAKE) all CXX=clang-++ CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -Wdocumentation"
 
 armtest: clean
 	$(MAKE) -C $(TESTDIR) datagen   # use native, faster
@@ -206,36 +244,45 @@ endif
 #make tests validated only for MSYS, Linux, OSX, kFreeBSD and Hurd targets
 #------------------------------------------------------------------------
 ifneq (,$(filter $(HOST_OS),MSYS POSIX))
-cmaketest:
+cmakebuild:
 	cmake --version
 	$(RM) -r $(BUILDIR)/cmake/build
 	mkdir $(BUILDIR)/cmake/build
 	cd $(BUILDIR)/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall
 
-c90test: clean
+c90build: clean
+	gcc -v
 	CFLAGS="-std=c90" $(MAKE) allmost  # will fail, due to missing support for `long long`
 
-gnu90test: clean
+gnu90build: clean
+	gcc -v
 	CFLAGS="-std=gnu90" $(MAKE) allmost
 
-c99test: clean
+c99build: clean
+	gcc -v
 	CFLAGS="-std=c99" $(MAKE) allmost
 
-gnu99test: clean
+gnu99build: clean
+	gcc -v
 	CFLAGS="-std=gnu99" $(MAKE) allmost
 
-c11test: clean
+c11build: clean
+	gcc -v
 	CFLAGS="-std=c11" $(MAKE) allmost
 
-bmix64test: clean
+bmix64build: clean
+	gcc -v
 	CFLAGS="-O3 -mbmi -Werror" $(MAKE) -C $(TESTDIR) test
 
-bmix32test: clean
+bmix32build: clean
+	gcc -v
 	CFLAGS="-O3 -mbmi -mx32 -Werror" $(MAKE) -C $(TESTDIR) test
 
-bmi32test: clean
+bmi32build: clean
+	gcc -v
 	CFLAGS="-O3 -mbmi -m32 -Werror" $(MAKE) -C $(TESTDIR) test
 
 staticAnalyze: clean
+	gcc -v
 	CPPFLAGS=-g scan-build --status-bugs -v $(MAKE) all
 endif
diff --git a/circle.yml b/circle.yml
index 3102633e6..046a48443 100644
--- a/circle.yml
+++ b/circle.yml
@@ -1,39 +1,40 @@
 dependencies:
   override:
+    - sudo dpkg --add-architecture i386
     - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update
-    #- sudo apt-get -y install qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu valgrind
-    #- sudo apt-get -y install qemu-system-arm gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
-    - sudo apt-get -y install libc6-dev-i386 clang gcc-5 gcc-6
+    - sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
+    - sudo apt-get -y install libstdc++-6-dev clang gcc g++ gcc-5 gcc-6
+    - sudo apt-get -y install linux-libc-dev:i386 libc6-dev-i386
 
   # use default "parallel: true" for commands in the machine, checkout, dependencies and database build phase
   post:
     - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make cmaketest           && make clean; fi
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-invalidDictionaries && make clean; fi
+      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then cc -v; make all   && make clean; fi &&
+      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu90build   && make clean; fi
     - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then g++ -v; make gpptest     && make clean; fi
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-decodecorpus && make clean; fi
+      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make c99build     && make clean; fi &&
+      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu99build   && make clean; fi
     - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make gnu90test   && make clean; fi
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-symbols             && make clean; fi
+      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make c11build     && make clean; fi &&
+      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make cmakebuild   && make clean; fi
     - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make c99test     && make clean; fi
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-longmatch           && make clean; fi
+      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make gppbuild     && make clean; fi &&
+      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc5build    && make clean; fi
     - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc -v; make gnu99test   && make clean; fi
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests dll                      && make clean; fi
+      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make gcc6build    && make clean; fi &&
+      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make clangbuild   && make clean; fi
     - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then clang -v; make clangtest && make clean; fi
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C programs zstd-small zstd-decompress zstd-compress zstd32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean lib && make clean; fi
+      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make m32build     && make clean; fi &&
+      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make armbuild     && make clean; fi
     - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make travis-install      && make clean; fi
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-fullbench           && make clean; fi
+      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make aarch64build && make clean; fi &&
+      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppcbuild     && make clean; fi
     - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then gcc-5 -v; make gcc5test  && gcc-6 -v && make gcc6test && make clean; fi
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-zstream             && make clean; fi
+      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make ppc64build   && make clean; fi &&
+      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then true              && make clean; fi #could add another test here
     - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]; then make -C tests test-zstd  && make clean; fi
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-fuzzer FUZZERTEST=-T4mn && make clean; fi
+      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make shortest     && make clean; fi &&
+      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-longmatch test-symbols && make clean; fi
 
 test:
   override:
diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 15f063e18..308659467 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -247,7 +247,7 @@ static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 fra
 typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_compResetPolicy_e;
 
 /*! ZSTD_resetCCtx_advanced() :
-    note : @params must be validated */
+    note : `params` must be validated */
 static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
                                        ZSTD_parameters params, U64 frameContentSize,
                                        ZSTD_compResetPolicy_e const crp)

From 764c2fdfed11ef8c7bbaf6dab7d7be7d8f1d3be1 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 6 Mar 2017 17:20:44 -0800
Subject: [PATCH 201/223] updated benchmark table

zstd v1.1.3, new station i7-6700K
---
 README.md | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index ce27056bc..9ead400f7 100644
--- a/README.md
+++ b/README.md
@@ -11,23 +11,22 @@ you can consult a list of known ports on [Zstandard homepage](http://www.zstd.ne
 |master      | [![Build Status](https://travis-ci.org/facebook/zstd.svg?branch=master)](https://travis-ci.org/facebook/zstd) |
 |dev         | [![Build Status](https://travis-ci.org/facebook/zstd.svg?branch=dev)](https://travis-ci.org/facebook/zstd) |
 
-As a reference, several fast compression algorithms were tested and compared on a Core i7-3930K CPU @ 4.5GHz, using [lzbench], an open-source in-memory benchmark by @inikep compiled with GCC 5.4.0, with the [Silesia compression corpus].
+As a reference, several fast compression algorithms were tested and compared on a server running Linux Mint Debian Edition (`Linux version 4.8.0-1-amd64`), with a Core i7-6700K CPU @ 4.0GHz, using [lzbench v1.6], an open-source in-memory benchmark by @inikep compiled with GCC 6.3.0, with the [Silesia compression corpus].
 
 [lzbench]: https://github.com/inikep/lzbench
 [Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
 
-
-| Name                    | Ratio | C.speed | D.speed |
-|-------------------------|-------|--------:|--------:|
-|                         |       |   MB/s  |  MB/s   |
-| **zstd 0.8.2 -1**     |**2.877**| **330** | **940** |
-| [zlib] 1.2.8 deflate -1 | 2.730 |    95   |   360   |
-| brotli 0.4 -0           | 2.708 |   320   |   375   |
-| QuickLZ 1.5             | 2.237 |   510   |   605   |
-| LZO 2.09                | 2.106 |   610   |   870   |
-| [LZ4] r131              | 2.101 |   620   |  3100   |
-| Snappy 1.1.3            | 2.091 |   480   |  1600   |
-| LZF 3.6                 | 2.077 |   375   |   790   |
+| Compressor name         | Ratio | Compression| Decompress.|
+| ---------------         | ------| -----------| ---------- |
+| memcpy                  | 1.000 | 13958 MB/s | 13932 MB/s |
+| **zstd 1.1.3 -1**       | 2.877 |   430 MB/s |  1110 MB/s |
+| zlib 1.2.8 -1           | 2.743 |   110 MB/s |   400 MB/s |
+| brotli 0.5.2 -0         | 2.708 |   400 MB/s |   430 MB/s |
+| quicklz 1.5.0 -1        | 2.238 |   550 MB/s |   710 MB/s |
+| lzo1x 2.09 -1           | 2.108 |   650 MB/s |   830 MB/s |
+| lz4 1.7.5               | 2.101 |   720 MB/s |  3600 MB/s |
+| snappy 1.1.3            | 2.091 |   500 MB/s |  1650 MB/s |
+| lzf 3.6 -1              | 2.077 |   400 MB/s |   860 MB/s |
 
 [zlib]:http://www.zlib.net/
 [LZ4]: http://www.lz4.org/

From eeb9758c39e3df79b9ca5868eb2f7ce096fd22c1 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 6 Mar 2017 17:22:47 -0800
Subject: [PATCH 202/223] fix : remove mempcpy line in bench

---
 README.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/README.md b/README.md
index 9ead400f7..d3c677b81 100644
--- a/README.md
+++ b/README.md
@@ -18,7 +18,6 @@ As a reference, several fast compression algorithms were tested and compared on
 
 | Compressor name         | Ratio | Compression| Decompress.|
 | ---------------         | ------| -----------| ---------- |
-| memcpy                  | 1.000 | 13958 MB/s | 13932 MB/s |
 | **zstd 1.1.3 -1**       | 2.877 |   430 MB/s |  1110 MB/s |
 | zlib 1.2.8 -1           | 2.743 |   110 MB/s |   400 MB/s |
 | brotli 0.5.2 -0         | 2.708 |   400 MB/s |   430 MB/s |

From 38ab1db3cd7b8b3b7cd8a26dc70e1d6a41a55863 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Mon, 6 Mar 2017 17:24:34 -0800
Subject: [PATCH 203/223] fixed lzbench link

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d3c677b81..b2aa3d2d5 100644
--- a/README.md
+++ b/README.md
@@ -13,7 +13,7 @@ you can consult a list of known ports on [Zstandard homepage](http://www.zstd.ne
 
 As a reference, several fast compression algorithms were tested and compared on a server running Linux Mint Debian Edition (`Linux version 4.8.0-1-amd64`), with a Core i7-6700K CPU @ 4.0GHz, using [lzbench v1.6], an open-source in-memory benchmark by @inikep compiled with GCC 6.3.0, with the [Silesia compression corpus].
 
-[lzbench]: https://github.com/inikep/lzbench
+[lzbench v1.6]: https://github.com/inikep/lzbench
 [Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
 
 | Compressor name         | Ratio | Compression| Decompress.|

From a1a195044fec16f32c7cfc50f8b0f196f1e56644 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Mon, 6 Mar 2017 16:57:04 -0800
Subject: [PATCH 204/223] Use test section

---
 circle.yml | 77 +++++++++++++++++++++++++++++++++---------------------
 1 file changed, 47 insertions(+), 30 deletions(-)

diff --git a/circle.yml b/circle.yml
index 046a48443..298569d14 100644
--- a/circle.yml
+++ b/circle.yml
@@ -6,38 +6,55 @@ dependencies:
     - sudo apt-get -y install libstdc++-6-dev clang gcc g++ gcc-5 gcc-6
     - sudo apt-get -y install linux-libc-dev:i386 libc6-dev-i386
 
-  # use default "parallel: true" for commands in the machine, checkout, dependencies and database build phase
-  post:
-    - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then cc -v; make all   && make clean; fi &&
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu90build   && make clean; fi
-    - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make c99build     && make clean; fi &&
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu99build   && make clean; fi
-    - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make c11build     && make clean; fi &&
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make cmakebuild   && make clean; fi
-    - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make gppbuild     && make clean; fi &&
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc5build    && make clean; fi
-    - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make gcc6build    && make clean; fi &&
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make clangbuild   && make clean; fi
-    - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make m32build     && make clean; fi &&
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make armbuild     && make clean; fi
-    - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make aarch64build && make clean; fi &&
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppcbuild     && make clean; fi
-    - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make ppc64build   && make clean; fi &&
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then true              && make clean; fi #could add another test here
-    - |
-      if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make shortest     && make clean; fi &&
-      if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-longmatch test-symbols && make clean; fi
-
 test:
   override:
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then cc -v; make all   && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu90build   && make clean; fi
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make c99build     && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gnu99build   && make clean; fi
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make c11build     && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make cmakebuild   && make clean; fi
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make gppbuild     && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make gcc5build    && make clean; fi
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make gcc6build    && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make clangbuild   && make clean; fi
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make m32build     && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make armbuild     && make clean; fi
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make aarch64build && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make ppcbuild     && make clean; fi
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make ppc64build   && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then true              && make clean; fi #could add another test here
+      :
+        parallel: true
+    - ? |
+        if [[ "$CIRCLE_NODE_INDEX" == "0" ]]                                    ; then make shortest     && make clean; fi &&
+        if [[ "$CIRCLE_NODE_TOTAL" < "2" ]] || [[ "$CIRCLE_NODE_INDEX" == "1" ]]; then make -C tests test-legacy test-longmatch test-symbols && make clean; fi
+      :
+        parallel: true
+
+  post:
     - echo Circle CI tests finished
 
   # Longer tests

From d66450fd7dbefc49928ed1445eda73e5d0ee17b5 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Tue, 7 Mar 2017 11:36:19 -0800
Subject: [PATCH 205/223] Fix travis test broken by Makefile change

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 82b92c2a2..9c1e10e15 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,7 +17,7 @@ matrix:
         - export CXX="g++-6" CC="gcc-6"
 
     # OS X Mavericks
-    - env: Cmd="make gnu90test && make clean && make test && make clean && make travis-install"
+    - env: Cmd="make gnu90build && make clean && make test && make clean && make travis-install"
       os: osx
 
 script:

From baa9b114f8fc3bf68ec7c181aef5b668aae6bdb7 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 7 Mar 2017 16:24:54 -0800
Subject: [PATCH 206/223] minor text refactor in readme

---
 README.md | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index b2aa3d2d5..6de5a1079 100644
--- a/README.md
+++ b/README.md
@@ -11,7 +11,12 @@ you can consult a list of known ports on [Zstandard homepage](http://www.zstd.ne
 |master      | [![Build Status](https://travis-ci.org/facebook/zstd.svg?branch=master)](https://travis-ci.org/facebook/zstd) |
 |dev         | [![Build Status](https://travis-ci.org/facebook/zstd.svg?branch=dev)](https://travis-ci.org/facebook/zstd) |
 
-As a reference, several fast compression algorithms were tested and compared on a server running Linux Mint Debian Edition (`Linux version 4.8.0-1-amd64`), with a Core i7-6700K CPU @ 4.0GHz, using [lzbench v1.6], an open-source in-memory benchmark by @inikep compiled with GCC 6.3.0, with the [Silesia compression corpus].
+As a reference, several fast compression algorithms were tested and compared
+on a server running Linux Mint Debian Edition (`Linux version 4.8.0-1-amd64`),
+with a Core i7-6700K CPU @ 4.0GHz,
+using [lzbench v1.6], an open-source in-memory benchmark by @inikep
+compiled with GCC 6.3.0,
+on the [Silesia compression corpus].
 
 [lzbench v1.6]: https://github.com/inikep/lzbench
 [Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia

From 881abe44f17d6fc8ef8805e221e4ae48f4a7d5f0 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Tue, 7 Mar 2017 16:52:23 -0800
Subject: [PATCH 207/223] Reduce point at which we reduce offsets to protect
 against UB

---
 lib/compress/zstd_compress.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 308659467..15a9245e4 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2347,7 +2347,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
         if (remaining < blockSize) blockSize = remaining;
 
         /* preemptive overflow correction */
-        if (cctx->lowLimit > (2U<<30)) {
+        if (cctx->lowLimit > (3U<<29)) {
             U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->params.cParams.hashLog, cctx->params.cParams.strategy)) - 1;
             U32 const current = (U32)(ip - cctx->base);
             U32 const newCurrent = (current & cycleMask) + (1 << cctx->params.cParams.windowLog);

From e06c303475020e7df6400d2be2293533d0415d56 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 8 Mar 2017 13:45:10 -0800
Subject: [PATCH 208/223] Fix ZSTD_sizeof_CStream()

---
 lib/compress/zstd_compress.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 432323e21..85744b753 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -3074,7 +3074,7 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
 size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
 {
     if (zcs==NULL) return 0;   /* support sizeof on NULL */
-    return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdictLocal) + zcs->outBuffSize + zcs->inBuffSize;
+    return sizeof(*zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdictLocal) + zcs->outBuffSize + zcs->inBuffSize;
 }
 
 /*======   Compression   ======*/

From 81512e9ebee5db31ac62eba4e6bc04ea34517729 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 8 Mar 2017 13:45:58 -0800
Subject: [PATCH 209/223] Avoid '#define inline /* ... */'

Take definition of `FORCE_INLINE` from `zstd_internal.h`.
---
 lib/decompress/huf_decompress.c | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/lib/decompress/huf_decompress.c b/lib/decompress/huf_decompress.c
index 0f11f5c1b..1d13150c5 100644
--- a/lib/decompress/huf_decompress.c
+++ b/lib/decompress/huf_decompress.c
@@ -35,16 +35,19 @@
 /* **************************************************************
 *  Compiler specifics
 ****************************************************************/
-#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
-/* inline is defined */
-#elif defined(_MSC_VER) || defined(__GNUC__)
-#  define inline __inline
-#else
-#  define inline /* disable inline */
-#endif
-
 #ifdef _MSC_VER    /* Visual Studio */
+#  define FORCE_INLINE static __forceinline
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
+#else
+#  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#    ifdef __GNUC__
+#      define FORCE_INLINE static inline __attribute__((always_inline))
+#    else
+#      define FORCE_INLINE static inline
+#    endif
+#  else
+#    define FORCE_INLINE static
+#  endif /* __STDC_VERSION__ */
 #endif
 
 
@@ -152,7 +155,7 @@ static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, con
     if (MEM_64bits()) \
         HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
 
-static inline size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX2* const dt, const U32 dtLog)
+FORCE_INLINE size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX2* const dt, const U32 dtLog)
 {
     BYTE* const pStart = p;
 
@@ -559,7 +562,7 @@ static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DE
     if (MEM_64bits()) \
         ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
 
-static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUF_DEltX4* const dt, const U32 dtLog)
+FORCE_INLINE size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUF_DEltX4* const dt, const U32 dtLog)
 {
     BYTE* const pStart = p;
 

From e65aab8e0fa823d91a76490924d1d7a67b485aa2 Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Wed, 8 Mar 2017 15:40:13 -0800
Subject: [PATCH 210/223] Remove 'mem.h' dependency from ZSTD_WINDOWLOG_MAX

---
 lib/zstd.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/zstd.h b/lib/zstd.h
index f462d0997..0b48b7ec6 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -345,7 +345,7 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output
 
 #define ZSTD_WINDOWLOG_MAX_32  27
 #define ZSTD_WINDOWLOG_MAX_64  27
-#define ZSTD_WINDOWLOG_MAX    ((U32)(MEM_32bits() ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
+#define ZSTD_WINDOWLOG_MAX    ((unsigned)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
 #define ZSTD_WINDOWLOG_MIN     10
 #define ZSTD_HASHLOG_MAX       ZSTD_WINDOWLOG_MAX
 #define ZSTD_HASHLOG_MIN        6

From 201e8c815709f306db8227c1054e99888d135c6f Mon Sep 17 00:00:00 2001
From: "Dmitry V. Levin" 
Date: Thu, 9 Mar 2017 02:00:35 +0000
Subject: [PATCH 211/223] programs/Makefile: remove zstd-internal target

zstd-internal was intended to be a helper target, but it doesn't help
at all, what it does in practice is a useless rebuild of zstd every time
"make zstd" is invoked.

Fixes: 030ac243a0f3 ("Changed Makefile to generate zstd with .gz support by default")
---
 programs/Makefile | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/programs/Makefile b/programs/Makefile
index 5620a25a3..a935c744a 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -83,8 +83,11 @@ all: zstd
 
 $(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
 
-zstd-internal : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
-zstd-internal : $(ZSTDLIB_OBJ) zstdcli.o fileio.o bench.o datagen.o dibio.o
+zstd : CPPFLAGS += $(ZLIBCPP)
+zstd : LDFLAGS += $(ZLIBLD)
+zstd-nogz : HAVE_ZLIB=0
+zstd zstd-nogz : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
+zstd zstd-nogz : $(ZSTDLIB_OBJ) zstdcli.o fileio.o bench.o datagen.o dibio.o
 ifeq ($(HAVE_ZLIB), 1)
 	@echo "==> building zstd with .gz decompression support "
 else
@@ -95,13 +98,6 @@ ifneq (,$(filter Windows%,$(OS)))
 endif
 	$(CC) $(FLAGS) $^ $(RES_FILE) -o zstd$(EXT) $(LDFLAGS)
 
-zstd-nogz : HAVE_ZLIB=0
-zstd-nogz : zstd-internal
-
-zstd : CPPFLAGS += $(ZLIBCPP)
-zstd : LDFLAGS += $(ZLIBLD)
-zstd : zstd-internal
-
 zstd-release: DEBUGFLAGS :=
 zstd-release: zstd
 

From daec40db242db71946dbb250b05a88beadb37008 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Tue, 7 Mar 2017 12:08:15 -0800
Subject: [PATCH 212/223] Update .travis.yml and Makefile for medium tests

---
 .travis.yml    | 54 +++++++++++++++++++++++++-------------------------
 Makefile       | 28 ++++++++++++++++++++++----
 appveyor.yml   |  5 +++++
 tests/Makefile |  4 ++--
 4 files changed, 58 insertions(+), 33 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 9c1e10e15..a52d57af3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,39 +1,39 @@
+# Medium Tests: Run on all commits/PRs to dev branch
+
 language: c
 sudo: required
 dist: trusty
 matrix:
-  fast_finish: true
   include:
     # Ubuntu 14.04
-    - env: Cmd="make libc6install && make -C tests test32"
-    - env: Cmd='make valgrindinstall arminstall ppcinstall arm-ppc-compilation && make clean lib && CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make -C tests valgrindTest'
+    - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-test-zstd'
+    - env: Cmd='make gcc6install libc6install && CC=gcc-6 make clean uasan-test-zstd32'
+    - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-test-zstd'
 
-    - env: Cmd='CC=gcc-6 make gcc6install uasan-test'
-    - env: Cmd='CC=gcc-6 make gcc6install uasan-test32'
-    - env: Cmd="make arminstall armtest && make clean && make aarch64test"
-    - env: Cmd='make ppcinstall ppctest && make clean && make ppc64test'
-    - env: Cmd='make gpp6install zlibwrapper && make -C tests clean test-zstd-nolegacy && make -C tests versionsTest && make clean && cd contrib/pzstd && make test-pzstd && make test-pzstd32 && make test-pzstd-tsan && make test-pzstd-asan'
-      install:
-        - export CXX="g++-6" CC="gcc-6"
+    - env: Cmd='make gcc6install && CC=gcc-6 make clean uasan-fuzztest'
+    - env: Cmd='make gcc6install libc6install && CC=gcc-6 CFLAGS=-m32 make clean uasan-fuzztest'
+    - env: Cmd='make clang38install && CC=clang-3.8 make clean msan-fuzztest'
+    - env: Cmd='make clang38install && CC=clang-3.8 make clean tsan-test-zstream'
 
-    # OS X Mavericks
-    - env: Cmd="make gnu90build && make clean && make test && make clean && make travis-install"
-      os: osx
+    - env: Cmd='make valgrindinstall && make -C tests clean valgrindTest'
+
+    - env: Cmd='make arminstall && make armfuzz'
+    - env: Cmd='make arminstall && make aarch64fuzz'
+    - env: Cmd='make ppcinstall && make ppcfuzz'
+    - env: Cmd='make ppcinstall && make ppc64fuzz'
+
+git:
+  depth: 1
+
+branches:
+  only:
+  - dev
+  - master
 
 script:
   - JOB_NUMBER=$(echo $TRAVIS_JOB_NUMBER | sed -e 's:[0-9][0-9]*\.\(.*\):\1:')
-  #  cron & master          => full tests, as this is the final step towards a Release
-  #  pull requests          => normal tests (job numbers 1-3)
-  #  other feature branches => short tests (job numbers 1-2)
   - echo JOB_NUMBER=$JOB_NUMBER TRAVIS_BRANCH=$TRAVIS_BRANCH TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST
-  - if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$TRAVIS_BRANCH" = "master" ]; then
-        FUZZERTEST=-T7mn sh -c "$Cmd" || travis_terminate 1;
-    else
-        if [ "$TRAVIS_EVENT_TYPE" = "pull_request" ] && [ $JOB_NUMBER -lt 4 ]; then
-            sh -c "$Cmd" || travis_terminate 1;
-        else
-            if [ $JOB_NUMBER -lt 3 ]; then
-                sh -c "$Cmd" || travis_terminate 1;
-            fi
-        fi
-    fi
+  - export FUZZERTEST=-T2mn;
+    export ZSTREAM_TESTTIME=-T2mn;
+    export DECODECORPUS_TESTTIME=-T1mn;
+    sh -c "$Cmd" || travis_terminate 1;
diff --git a/Makefile b/Makefile
index e10d29267..0187348ee 100644
--- a/Makefile
+++ b/Makefile
@@ -151,6 +151,18 @@ ppcbuild: clean
 ppc64build: clean
 	CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allarch
 
+armfuzz: clean
+	CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest
+
+aarch64fuzz: clean
+	CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest
+
+ppcfuzz: clean
+	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest
+
+ppc64fuzz: clean
+	CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" $(MAKE) -C $(TESTDIR) fuzztest
+
 gpptest: clean
 	CC=g++ $(MAKE) -C $(PRGDIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror"
 
@@ -189,7 +201,7 @@ arm-ppc-compilation:
 	$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static"
 
 usan: clean
-	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=undefined"
+	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=undefined"
 
 asan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address"
@@ -197,15 +209,20 @@ asan: clean
 msan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer"   # datagen.c fails this test for no obvious reason
 
+msan-%: clean
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer" $(MAKE) -C $(TESTDIR) $*
+
 asan32: clean
 	$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
 
 uasan: clean
-	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -fsanitize=undefined"
+	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address,undefined"
 
 uasan-%: clean
-	LDFLAGS=-fuse-ld=gold CFLAGS="-Og -fsanitize=address -fsanitize=undefined" $(MAKE) -C $(TESTDIR) $*
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-Og -fno-sanitize-recover=all -fsanitize=address,undefined" $(MAKE) -C $(TESTDIR) $*
 
+tsan-%: clean
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread" $(MAKE) -C $(TESTDIR) $*
 apt-install:
 	sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES)
 
@@ -217,7 +234,7 @@ ppcinstall:
 	APT_PACKAGES="qemu-system-ppc qemu-user-static gcc-powerpc-linux-gnu" $(MAKE) apt-install
 
 arminstall:
-	APT_PACKAGES="qemu-system-arm qemu-user-static gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross" $(MAKE) apt-install
+	APT_PACKAGES="qemu-system-arm qemu-user-static gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross" $(MAKE) apt-install
 
 valgrindinstall:
 	APT_PACKAGES="valgrind" $(MAKE) apt-install
@@ -231,6 +248,9 @@ gcc6install: apt-add-repo
 gpp6install: apt-add-repo
 	APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install
 
+clang38install:
+	APT_PACKAGES="clang-3.8" $(MAKE) apt-install
+
 endif
 
 
diff --git a/appveyor.yml b/appveyor.yml
index 51ff488a4..9507fec6e 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -146,6 +146,11 @@ test_script:
       fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST%
     )
 
+branches:
+  only:
+  - dev
+  - master
+
 artifacts:
   - path: bin\zstd.exe
   - path: bin\zstd32.exe
diff --git a/tests/Makefile b/tests/Makefile
index 8b19aa3d5..39e4d1015 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -75,8 +75,6 @@ all32: fullbench32 fuzzer32 zstreamtest32 zbufftest32
 
 dll: fuzzer-dll zstreamtest-dll zbufftest-dll
 
-
-
 zstd:
 	$(MAKE) -C $(PRGDIR) $@
 
@@ -257,6 +255,8 @@ zstd-playTests: datagen
 shortest: ZSTDRTTEST=
 shortest: test-zstd
 
+fuzztest: test-fuzzer test-zstream test-decodecorpus
+
 test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus
 ifeq ($(QEMU_SYS),)
 test: test-pool

From 7c8f5d5bc7ee4c16c5de73cb2e1c0e13f2ffca79 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Thu, 9 Mar 2017 16:05:10 -0800
Subject: [PATCH 213/223] Make test times overwritable

---
 tests/Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/Makefile b/tests/Makefile
index 39e4d1015..59256f841 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -60,10 +60,10 @@ endif
 MULTITHREAD = $(MULTITHREAD_CPP) $(MULTITHREAD_LD)
 
 VOID = /dev/null
-ZSTREAM_TESTTIME = -T2mn
+ZSTREAM_TESTTIME ?= -T2mn
 FUZZERTEST ?= -T5mn
 ZSTDRTTEST = --test-large-data
-DECODECORPUS_TESTTIME = -T30
+DECODECORPUS_TESTTIME ?= -T30
 
 .PHONY: default all all32 dll clean test test32 test-all namespaceTest versionsTest
 

From 2500dcfa5f6f99dc015231d5561c35d9a4016965 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Thu, 9 Mar 2017 13:59:26 -0800
Subject: [PATCH 214/223] Add testing description

---
 TESTING.md | 44 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 TESTING.md

diff --git a/TESTING.md b/TESTING.md
new file mode 100644
index 000000000..1fa5fe8c2
--- /dev/null
+++ b/TESTING.md
@@ -0,0 +1,44 @@
+Testing
+=======
+
+Zstandard CI testing is split up into three sections:
+short, medium, and long tests.
+
+Short Tests
+-----------
+Short tests run on CircleCI for new commits on every branch and pull request.
+They consist of the following tests:
+- Compilation on all supported targets (x86, x86_64, ARM, AArch64, PowerPC, and PowerPC64)
+- Compilation on various versions of gcc, clang, and g++
+- `tests/playTests.sh` on x86_64, without the tests on long data (CLI tests)
+- Small tests (`tests/legacy.c`, `tests/longmatch.c`, `tests/symbols.c`) on x64_64
+
+Medium Tests
+------------
+Medium tests run on every commit and pull request to `dev` branch, on TravisCI.
+They consist of the following tests:
+- The following tests run with UBsan and Asan on x86_64 and x86, as well as with
+  Msan on x86_64
+  - `tests/playTests.sh --test-long-data`
+  - Fuzzer tests: `tests/fuzzer.c`, `tests/zstreamtest.c`, and `tests/decodecorpus.c`
+- `tests/zstreamtest.c` under Tsan (streaming mode, including multithreaded mode)
+- Valgrind Test (`make -C tests valgrindTest`) (testing CLI and fuzzer under valgrind)
+- Fuzzer tests (see above) on ARM, AArch64, PowerPC, and PowerPC64
+
+Long Tests
+----------
+Long tests run on all commits to `master` branch,
+and once a day on the current version of `dev` branch,
+on TravisCI.
+They consist of the following tests:
+- Entire test suite (including fuzzers and some other specialized tests) on:
+  - x86_64 and x86 with UBsan and Asan
+  - x86_64 with Msan
+  - ARM, AArch64, PowerPC, and PowerPC64
+- Streaming mode fuzzer with Tsan (for the `zstdmt` testing)
+- ZlibWrapper tests, including under valgrind
+- Versions test (ensuring `zstd` can decode files from all previous versions)
+- `pzstd` with asan and tsan, as well as in 32-bits mode
+- Testing `zstd` with legacy mode off
+- Testing `zbuff` (old streaming API)
+- Entire test suite and make install on OS X

From caf0ee8d20663b9dce7d0ac4cd048ebc542eb473 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Thu, 9 Mar 2017 17:28:08 -0800
Subject: [PATCH 215/223] Make signed integer overflow recoverable in UBsan

---
 Makefile | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 0187348ee..49f29d782 100644
--- a/Makefile
+++ b/Makefile
@@ -200,12 +200,19 @@ arm-ppc-compilation:
 	$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static"
 	$(MAKE) -C $(PRGDIR) clean zstd CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static"
 
+# run UBsan with -fsanitize-recover=signed-integer-overflow
+# due to a bug in UBsan when doing pointer subtraction
+# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63303
+
 usan: clean
-	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=undefined"
+	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=undefined"
 
 asan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address"
 
+asan-%: clean
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address" $(MAKE) -C $(TESTDIR) $*
+
 msan: clean
 	$(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer"   # datagen.c fails this test for no obvious reason
 
@@ -216,10 +223,10 @@ asan32: clean
 	$(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address"
 
 uasan: clean
-	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address,undefined"
+	$(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined"
 
 uasan-%: clean
-	LDFLAGS=-fuse-ld=gold MOREFLAGS="-Og -fno-sanitize-recover=all -fsanitize=address,undefined" $(MAKE) -C $(TESTDIR) $*
+	LDFLAGS=-fuse-ld=gold MOREFLAGS="-Og -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined" $(MAKE) -C $(TESTDIR) $*
 
 tsan-%: clean
 	LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread" $(MAKE) -C $(TESTDIR) $*

From 8fe5c6862c56f937ad5c3c25d387d4a331b42775 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Thu, 9 Mar 2017 11:54:34 -0800
Subject: [PATCH 216/223] Fix undefined behaviour in decompressor

---
 lib/common/mem.h                 | 18 ++++++++++--------
 lib/decompress/zstd_decompress.c |  6 +++---
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/lib/common/mem.h b/lib/common/mem.h
index 7a3f72141..3cacd216a 100644
--- a/lib/common/mem.h
+++ b/lib/common/mem.h
@@ -48,14 +48,15 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size
 *****************************************************************/
 #if  !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
 # include 
-  typedef  uint8_t BYTE;
-  typedef uint16_t U16;
-  typedef  int16_t S16;
-  typedef uint32_t U32;
-  typedef  int32_t S32;
-  typedef uint64_t U64;
-  typedef  int64_t S64;
-  typedef intptr_t iPtrDiff;
+  typedef   uint8_t BYTE;
+  typedef  uint16_t U16;
+  typedef   int16_t S16;
+  typedef  uint32_t U32;
+  typedef   int32_t S32;
+  typedef  uint64_t U64;
+  typedef   int64_t S64;
+  typedef  intptr_t iPtrDiff;
+  typedef uintptr_t uPtrDiff;
 #else
   typedef unsigned char      BYTE;
   typedef unsigned short      U16;
@@ -65,6 +66,7 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size
   typedef unsigned long long  U64;
   typedef   signed long long  S64;
   typedef ptrdiff_t      iPtrDiff;
+  typedef size_t         uPtrDiff;
 #endif
 
 
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 482c334ff..516edfcc6 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1033,7 +1033,7 @@ size_t ZSTD_execSequence(BYTE* op,
     if (sequence.offset > (size_t)(oLitEnd - base)) {
         /* offset beyond prefix */
         if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected);
-        match += (dictEnd-base);
+        match = dictEnd + (match - base);
         if (match + sequence.matchLength <= dictEnd) {
             memmove(oLitEnd, match, sequence.matchLength);
             return sequenceLength;
@@ -1216,7 +1216,7 @@ FORCE_INLINE seq_t ZSTD_decodeSequenceLong_generic(seqState_t* seqState, int con
 
     {   size_t const pos = seqState->pos + seq.litLength;
         seq.match = seqState->base + pos - seq.offset;    /* single memory segment */
-        if (seq.offset > pos) seq.match += seqState->gotoDict;   /* separate memory segment */
+        if (seq.offset > pos) seq.match += (uPtrDiff)seqState->gotoDict;   /* separate memory segment */
         seqState->pos = pos + seq.matchLength;
     }
 
@@ -1356,7 +1356,7 @@ static size_t ZSTD_decompressSequencesLong(
         { U32 i; for (i=0; ientropy.rep[i]; }
         seqState.base = base;
         seqState.pos = (size_t)(op-base);
-        seqState.gotoDict = (iPtrDiff)(dictEnd - base);
+        seqState.gotoDict = (iPtrDiff)((uPtrDiff)dictEnd - (uPtrDiff)base); /* cast to avoid undefined behaviour */
         CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected);
         FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
         FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);

From 784082f49cfd48c9f77db23ae7df35703b8b68b2 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Fri, 10 Mar 2017 10:34:45 -0800
Subject: [PATCH 217/223] Change gotoDict type to uPtrDiff

---
 lib/decompress/zstd_decompress.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 516edfcc6..943bdf94e 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -876,7 +876,7 @@ typedef struct {
     size_t prevOffset[ZSTD_REP_NUM];
     const BYTE* base;
     size_t pos;
-    iPtrDiff gotoDict;
+    uPtrDiff gotoDict;
 } seqState_t;
 
 
@@ -1216,7 +1216,7 @@ FORCE_INLINE seq_t ZSTD_decodeSequenceLong_generic(seqState_t* seqState, int con
 
     {   size_t const pos = seqState->pos + seq.litLength;
         seq.match = seqState->base + pos - seq.offset;    /* single memory segment */
-        if (seq.offset > pos) seq.match += (uPtrDiff)seqState->gotoDict;   /* separate memory segment */
+        if (seq.offset > pos) seq.match += seqState->gotoDict;   /* separate memory segment */
         seqState->pos = pos + seq.matchLength;
     }
 
@@ -1356,7 +1356,7 @@ static size_t ZSTD_decompressSequencesLong(
         { U32 i; for (i=0; ientropy.rep[i]; }
         seqState.base = base;
         seqState.pos = (size_t)(op-base);
-        seqState.gotoDict = (iPtrDiff)((uPtrDiff)dictEnd - (uPtrDiff)base); /* cast to avoid undefined behaviour */
+        seqState.gotoDict = (uPtrDiff)dictEnd - (uPtrDiff)base; /* cast to avoid undefined behaviour */
         CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected);
         FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
         FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);

From 334cb34edba6da2232701b298b61a62fc26259a4 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Mon, 13 Mar 2017 14:32:30 -0700
Subject: [PATCH 218/223] ZSTD_LEGACY_SUPPORT defines lowest supported version

---
 lib/Makefile                     | 13 +++--
 lib/decompress/zstd_decompress.c |  4 +-
 lib/legacy/zstd_legacy.h         | 97 ++++++++++++++++++++++++++++++++
 programs/Makefile                | 10 ++--
 4 files changed, 113 insertions(+), 11 deletions(-)

diff --git a/lib/Makefile b/lib/Makefile
index 58f99baf5..18b08a11d 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -31,12 +31,15 @@ FLAGS    = $(CPPFLAGS) $(CFLAGS)
 
 ZSTD_FILES := $(wildcard common/*.c compress/*.c decompress/*.c dictBuilder/*.c deprecated/*.c)
 
-ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
-CPPFLAGS  += -DZSTD_LEGACY_SUPPORT=0
-else
-CPPFLAGS  += -I./legacy -DZSTD_LEGACY_SUPPORT=1
-ZSTD_FILES+= $(wildcard legacy/*.c)
+ZSTD_LEGACY_SUPPORT ?= 1
+
+ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
+ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
+	ZSTD_FILES += $(shell ls legacy/*.c | grep 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
 endif
+	CPPFLAGS += -I./legacy
+endif
+CPPFLAGS  += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
 
 ZSTD_OBJ   := $(patsubst %.c,%.o,$(ZSTD_FILES))
 
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 943bdf94e..2aaa4a3df 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -320,7 +320,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
 *             - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */
 unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize)
 {
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
     if (ZSTD_isLegacy(src, srcSize)) {
         unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize);
         return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret;
@@ -1472,7 +1472,7 @@ size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t len
  *  @return : the compressed size of the frame starting at `src` */
 size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize)
 {
-#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
+#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
     if (ZSTD_isLegacy(src, srcSize)) return ZSTD_findFrameCompressedSizeLegacy(src, srcSize);
 #endif
     if (srcSize >= ZSTD_skippableHeaderSize &&
diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h
index 707e76f0a..18e22e651 100644
--- a/lib/legacy/zstd_legacy.h
+++ b/lib/legacy/zstd_legacy.h
@@ -28,6 +28,31 @@ extern "C" {
 #include "zstd_v06.h"
 #include "zstd_v07.h"
 
+#ifndef ZSTD_LEGACY_SUPPORT
+#  define ZSTD_LEGACY_SUPPORT 8
+#endif
+
+#if (ZSTD_LEGACY_SUPPORT <= 1)
+#  include "zstd_v01.h"
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 2)
+#  include "zstd_v02.h"
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 3)
+#  include "zstd_v03.h"
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 4)
+#  include "zstd_v04.h"
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 5)
+#  include "zstd_v05.h"
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
+#  include "zstd_v06.h"
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
+#  include "zstd_v07.h"
+#endif
 
 /** ZSTD_isLegacy() :
     @return : > 0 if supported by legacy decoder. 0 otherwise.
@@ -40,13 +65,27 @@ MEM_STATIC unsigned ZSTD_isLegacy(const void* src, size_t srcSize)
     magicNumberLE = MEM_readLE32(src);
     switch(magicNumberLE)
     {
+#if (ZSTD_LEGACY_SUPPORT <= 1)
         case ZSTDv01_magicNumberLE:return 1;
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 2)
         case ZSTDv02_magicNumber : return 2;
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 3)
         case ZSTDv03_magicNumber : return 3;
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 4)
         case ZSTDv04_magicNumber : return 4;
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 5)
         case ZSTDv05_MAGICNUMBER : return 5;
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
         case ZSTDv06_MAGICNUMBER : return 6;
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
         case ZSTDv07_MAGICNUMBER : return 7;
+#endif
         default : return 0;
     }
 }
@@ -56,24 +95,30 @@ MEM_STATIC unsigned long long ZSTD_getDecompressedSize_legacy(const void* src, s
 {
     U32 const version = ZSTD_isLegacy(src, srcSize);
     if (version < 5) return 0;  /* no decompressed size in frame header, or not a legacy format */
+#if (ZSTD_LEGACY_SUPPORT <= 5)
     if (version==5) {
         ZSTDv05_parameters fParams;
         size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize);
         if (frResult != 0) return 0;
         return fParams.srcSize;
     }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
     if (version==6) {
         ZSTDv06_frameParams fParams;
         size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize);
         if (frResult != 0) return 0;
         return fParams.frameContentSize;
     }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
     if (version==7) {
         ZSTDv07_frameParams fParams;
         size_t const frResult = ZSTDv07_getFrameParams(&fParams, src, srcSize);
         if (frResult != 0) return 0;
         return fParams.frameContentSize;
     }
+#endif
     return 0;   /* should not be possible */
 }
 
@@ -86,14 +131,23 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
     U32 const version = ZSTD_isLegacy(src, compressedSize);
     switch(version)
     {
+#if (ZSTD_LEGACY_SUPPORT <= 1)
         case 1 :
             return ZSTDv01_decompress(dst, dstCapacity, src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 2)
         case 2 :
             return ZSTDv02_decompress(dst, dstCapacity, src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 3)
         case 3 :
             return ZSTDv03_decompress(dst, dstCapacity, src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 4)
         case 4 :
             return ZSTDv04_decompress(dst, dstCapacity, src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 5)
         case 5 :
             {   size_t result;
                 ZSTDv05_DCtx* const zd = ZSTDv05_createDCtx();
@@ -102,6 +156,8 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
                 ZSTDv05_freeDCtx(zd);
                 return result;
             }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
         case 6 :
             {   size_t result;
                 ZSTDv06_DCtx* const zd = ZSTDv06_createDCtx();
@@ -110,6 +166,8 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
                 ZSTDv06_freeDCtx(zd);
                 return result;
             }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
         case 7 :
             {   size_t result;
                 ZSTDv07_DCtx* const zd = ZSTDv07_createDCtx();
@@ -118,6 +176,7 @@ MEM_STATIC size_t ZSTD_decompressLegacy(
                 ZSTDv07_freeDCtx(zd);
                 return result;
             }
+#endif
         default :
             return ERROR(prefix_unknown);
     }
@@ -129,20 +188,34 @@ MEM_STATIC size_t ZSTD_findFrameCompressedSizeLegacy(const void *src,
     U32 const version = ZSTD_isLegacy(src, compressedSize);
     switch(version)
     {
+#if (ZSTD_LEGACY_SUPPORT <= 1)
         case 1 :
             return ZSTDv01_findFrameCompressedSize(src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 2)
         case 2 :
             return ZSTDv02_findFrameCompressedSize(src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 3)
         case 3 :
             return ZSTDv03_findFrameCompressedSize(src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 4)
         case 4 :
             return ZSTDv04_findFrameCompressedSize(src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 5)
         case 5 :
             return ZSTDv05_findFrameCompressedSize(src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
         case 6 :
             return ZSTDv06_findFrameCompressedSize(src, compressedSize);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
         case 7 :
             return ZSTDv07_findFrameCompressedSize(src, compressedSize);
+#endif
         default :
             return ERROR(prefix_unknown);
     }
@@ -157,10 +230,18 @@ MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version)
         case 2 :
         case 3 :
             return ERROR(version_unsupported);
+#if (ZSTD_LEGACY_SUPPORT <= 4)
         case 4 : return ZBUFFv04_freeDCtx((ZBUFFv04_DCtx*)legacyContext);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 5)
         case 5 : return ZBUFFv05_freeDCtx((ZBUFFv05_DCtx*)legacyContext);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
         case 6 : return ZBUFFv06_freeDCtx((ZBUFFv06_DCtx*)legacyContext);
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
         case 7 : return ZBUFFv07_freeDCtx((ZBUFFv07_DCtx*)legacyContext);
+#endif
     }
 }
 
@@ -176,6 +257,7 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
         case 2 :
         case 3 :
             return 0;
+#if (ZSTD_LEGACY_SUPPORT <= 4)
         case 4 :
         {
             ZBUFFv04_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv04_createDCtx() : (ZBUFFv04_DCtx*)*legacyContext;
@@ -185,6 +267,8 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
             *legacyContext = dctx;
             return 0;
         }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 5)
         case 5 :
         {
             ZBUFFv05_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv05_createDCtx() : (ZBUFFv05_DCtx*)*legacyContext;
@@ -193,6 +277,8 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
             *legacyContext = dctx;
             return 0;
         }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
         case 6 :
         {
             ZBUFFv06_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv06_createDCtx() : (ZBUFFv06_DCtx*)*legacyContext;
@@ -201,6 +287,8 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
             *legacyContext = dctx;
             return 0;
         }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
         case 7 :
         {
             ZBUFFv07_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv07_createDCtx() : (ZBUFFv07_DCtx*)*legacyContext;
@@ -209,6 +297,7 @@ MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U
             *legacyContext = dctx;
             return 0;
         }
+#endif
     }
 }
 
@@ -224,6 +313,7 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
         case 2 :
         case 3 :
             return ERROR(version_unsupported);
+#if (ZSTD_LEGACY_SUPPORT <= 4)
         case 4 :
             {
                 ZBUFFv04_DCtx* dctx = (ZBUFFv04_DCtx*) legacyContext;
@@ -236,6 +326,8 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
                 input->pos += readSize;
                 return hintSize;
             }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 5)
         case 5 :
             {
                 ZBUFFv05_DCtx* dctx = (ZBUFFv05_DCtx*) legacyContext;
@@ -248,6 +340,8 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
                 input->pos += readSize;
                 return hintSize;
             }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 6)
         case 6 :
             {
                 ZBUFFv06_DCtx* dctx = (ZBUFFv06_DCtx*) legacyContext;
@@ -260,6 +354,8 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
                 input->pos += readSize;
                 return hintSize;
             }
+#endif
+#if (ZSTD_LEGACY_SUPPORT <= 7)
         case 7 :
             {
                 ZBUFFv07_DCtx* dctx = (ZBUFFv07_DCtx*) legacyContext;
@@ -272,6 +368,7 @@ MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version,
                 input->pos += readSize;
                 return hintSize;
             }
+#endif
     }
 }
 
diff --git a/programs/Makefile b/programs/Makefile
index a935c744a..0bf194260 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -42,12 +42,14 @@ ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
 ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
 ZSTDDECOMP_O = $(ZSTDDIR)/decompress/zstd_decompress.o
 
-ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
+ZSTD_LEGACY_SUPPORT ?= 1
 ZSTDLEGACY_FILES:=
+ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
+ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
+	ZSTDLEGACY_FILES += $(shell ls $(ZSTDDIR)/legacy/*.c | grep 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
+endif
+	CPPFLAGS += -I$(ZSTDDIR)/legacy
 else
-ZSTD_LEGACY_SUPPORT:=1
-CPPFLAGS  += -I$(ZSTDDIR)/legacy
-ZSTDLEGACY_FILES:= $(ZSTDDIR)/legacy/*.c
 endif
 
 ZSTDLIB_FILES := $(wildcard $(ZSTD_FILES)) $(wildcard $(ZSTDLEGACY_FILES)) $(wildcard $(ZDICT_FILES))

From 120df494e9e5df89781307776377a6e1dac16df3 Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Mon, 13 Mar 2017 14:44:08 -0700
Subject: [PATCH 219/223] Update builds to not support legacy v01-v03

---
 .buckconfig                                  | 4 ++--
 build/VS2005/zstd/zstd.vcproj                | 8 ++++----
 build/VS2005/zstdlib/zstdlib.vcproj          | 8 ++++----
 build/VS2008/zstd/zstd.vcproj                | 8 ++++----
 build/VS2008/zstdlib/zstdlib.vcproj          | 8 ++++----
 build/VS2010/libzstd-dll/libzstd-dll.vcxproj | 8 ++++----
 build/VS2010/libzstd/libzstd.vcxproj         | 8 ++++----
 build/VS2010/zstd/zstd.vcxproj               | 8 ++++----
 build/cmake/CMakeLists.txt                   | 2 +-
 contrib/meson/meson.build                    | 2 +-
 lib/Makefile                                 | 2 +-
 programs/Makefile                            | 2 +-
 tests/Makefile                               | 2 +-
 13 files changed, 35 insertions(+), 35 deletions(-)

diff --git a/.buckconfig b/.buckconfig
index d698b35ba..483f6053b 100644
--- a/.buckconfig
+++ b/.buckconfig
@@ -1,7 +1,7 @@
 [cxx]
-  cppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=1
+  cppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4
   cflags = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wpointer-arith
-  cxxppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=1
+  cxxppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4
   cxxflags = -std=c++11 -Wno-deprecated-declarations
   gtest_dep = //contrib/pzstd:gtest
 
diff --git a/build/VS2005/zstd/zstd.vcproj b/build/VS2005/zstd/zstd.vcproj
index 58f254bc8..1f4febead 100644
--- a/build/VS2005/zstd/zstd.vcproj
+++ b/build/VS2005/zstd/zstd.vcproj
@@ -44,7 +44,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -121,7 +121,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -196,7 +196,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -274,7 +274,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
diff --git a/build/VS2005/zstdlib/zstdlib.vcproj b/build/VS2005/zstdlib/zstdlib.vcproj
index f4c9950ff..8da313673 100644
--- a/build/VS2005/zstdlib/zstdlib.vcproj
+++ b/build/VS2005/zstdlib/zstdlib.vcproj
@@ -44,7 +44,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -120,7 +120,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -194,7 +194,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -271,7 +271,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
diff --git a/build/VS2008/zstd/zstd.vcproj b/build/VS2008/zstd/zstd.vcproj
index 2dfaf3937..468d25672 100644
--- a/build/VS2008/zstd/zstd.vcproj
+++ b/build/VS2008/zstd/zstd.vcproj
@@ -45,7 +45,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -122,7 +122,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -197,7 +197,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -275,7 +275,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\lib\dictBuilder;$(SolutionDir)..\..\lib\compress"
-				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
diff --git a/build/VS2008/zstdlib/zstdlib.vcproj b/build/VS2008/zstdlib/zstdlib.vcproj
index cba0ff908..857e1463e 100644
--- a/build/VS2008/zstdlib/zstdlib.vcproj
+++ b/build/VS2008/zstdlib/zstdlib.vcproj
@@ -45,7 +45,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -121,7 +121,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
@@ -195,7 +195,7 @@
 				Name="VCCLCompilerTool"
 				Optimization="0"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE"
 				MinimalRebuild="true"
 				BasicRuntimeChecks="3"
 				RuntimeLibrary="3"
@@ -272,7 +272,7 @@
 				EnableIntrinsicFunctions="true"
 				OmitFramePointers="true"
 				AdditionalIncludeDirectories="$(SolutionDir)..\..\lib;$(SolutionDir)..\..\lib\common;$(SolutionDir)..\..\lib\legacy;$(SolutionDir)..\..\programs\legacy;$(SolutionDir)..\..\lib\dictBuilder"
-				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE"
+				PreprocessorDefinitions="ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE"
 				RuntimeLibrary="0"
 				EnableFunctionLevelLinking="true"
 				UsePrecompiledHeader="0"
diff --git a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
index f78598fb4..866d04a04 100644
--- a/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
+++ b/build/VS2010/libzstd-dll/libzstd-dll.vcxproj
@@ -149,7 +149,7 @@
       
       Level4
       Disabled
-      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       true
       EnableFastChecks
       MultiThreadedDebugDLL
@@ -169,7 +169,7 @@
       
       Level4
       Disabled
-      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       true
       EnableFastChecks
       MultiThreadedDebugDLL
@@ -189,7 +189,7 @@
       MaxSpeed
       true
       true
-      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       false
       MultiThreaded
       ProgramDatabase
@@ -211,7 +211,7 @@
       MaxSpeed
       true
       true
-      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       false
       false
       MultiThreaded
diff --git a/build/VS2010/libzstd/libzstd.vcxproj b/build/VS2010/libzstd/libzstd.vcxproj
index 727795514..186b4c4da 100644
--- a/build/VS2010/libzstd/libzstd.vcxproj
+++ b/build/VS2010/libzstd/libzstd.vcxproj
@@ -146,7 +146,7 @@
       
       Level4
       Disabled
-      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       true
       EnableFastChecks
       MultiThreadedDebugDLL
@@ -166,7 +166,7 @@
       
       Level4
       Disabled
-      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       true
       EnableFastChecks
       MultiThreadedDebugDLL
@@ -186,7 +186,7 @@
       MaxSpeed
       true
       true
-      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       false
       MultiThreaded
       ProgramDatabase
@@ -208,7 +208,7 @@
       MaxSpeed
       true
       true
-      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
+      ZSTD_DLL_EXPORT=1;ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions)
       false
       false
       MultiThreaded
diff --git a/build/VS2010/zstd/zstd.vcxproj b/build/VS2010/zstd/zstd.vcxproj
index 62c0fe10f..7568e4902 100644
--- a/build/VS2010/zstd/zstd.vcxproj
+++ b/build/VS2010/zstd/zstd.vcxproj
@@ -155,7 +155,7 @@
       
       Level4
       Disabled
-      ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
+      ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
       true
       false
     
@@ -171,7 +171,7 @@
       
       Level4
       Disabled
-      ZSTD_LEGACY_SUPPORT=1;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
+      ZSTD_LEGACY_SUPPORT=4;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)
       true
       false
     
@@ -189,7 +189,7 @@
       MaxSpeed
       true
       true
-      ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
+      ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
       false
       false
       MultiThreaded
@@ -210,7 +210,7 @@
       MaxSpeed
       true
       true
-      ZSTD_LEGACY_SUPPORT=1;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
+      ZSTD_LEGACY_SUPPORT=4;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)
       false
       false
       MultiThreaded
diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt
index 6b7c28925..4805cc2c9 100644
--- a/build/cmake/CMakeLists.txt
+++ b/build/cmake/CMakeLists.txt
@@ -16,7 +16,7 @@ OPTION(ZSTD_BUILD_CONTRIB "BUILD CONTRIB" OFF)
 
 IF (ZSTD_LEGACY_SUPPORT)
     MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT defined!")
-    ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=1)
+    ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=4)
 ELSE (ZSTD_LEGACY_SUPPORT)
     MESSAGE(STATUS "ZSTD_LEGACY_SUPPORT not defined!")
     ADD_DEFINITIONS(-DZSTD_LEGACY_SUPPORT=0)
diff --git a/contrib/meson/meson.build b/contrib/meson/meson.build
index 369461335..8cbdcabec 100644
--- a/contrib/meson/meson.build
+++ b/contrib/meson/meson.build
@@ -15,7 +15,7 @@ libzstd_includes = [include_directories(common_dir, dictbuilder_dir, compress_di
 
 if get_option('legacy_support')
     message('Enabling legacy support')
-    libzstd_cflags = ['-DZSTD_LEGACY_SUPPORT=1']
+    libzstd_cflags = ['-DZSTD_LEGACY_SUPPORT=4']
 
     legacy_dir = join_paths(lib_dir, 'legacy')
     libzstd_includes += [include_directories(legacy_dir)]
diff --git a/lib/Makefile b/lib/Makefile
index 18b08a11d..197fdeeea 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -31,7 +31,7 @@ FLAGS    = $(CPPFLAGS) $(CFLAGS)
 
 ZSTD_FILES := $(wildcard common/*.c compress/*.c decompress/*.c dictBuilder/*.c deprecated/*.c)
 
-ZSTD_LEGACY_SUPPORT ?= 1
+ZSTD_LEGACY_SUPPORT ?= 4
 
 ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
 ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
diff --git a/programs/Makefile b/programs/Makefile
index 0bf194260..beeb0711c 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -42,7 +42,7 @@ ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
 ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
 ZSTDDECOMP_O = $(ZSTDDIR)/decompress/zstd_decompress.o
 
-ZSTD_LEGACY_SUPPORT ?= 1
+ZSTD_LEGACY_SUPPORT ?= 4
 ZSTDLEGACY_FILES:=
 ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
 ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
diff --git a/tests/Makefile b/tests/Makefile
index 59256f841..9382fe80d 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -167,7 +167,7 @@ longmatch  : $(ZSTD_FILES) longmatch.c
 invalidDictionaries  : $(ZSTD_FILES) invalidDictionaries.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)
 
-legacy : CFLAGS+= -DZSTD_LEGACY_SUPPORT=1
+legacy : CFLAGS+= -DZSTD_LEGACY_SUPPORT=4
 legacy : CPPFLAGS+= -I$(ZSTDDIR)/legacy
 legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c
 	$(CC)      $(FLAGS) $^ -o $@$(EXT)

From 9830aeeea6458124f6086ae9cdbf79a24244e30c Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Mon, 13 Mar 2017 17:19:37 -0700
Subject: [PATCH 220/223] Fix legacy support=0 case and accidental double
 include of version headers

---
 lib/legacy/zstd_legacy.h | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h
index 18e22e651..3c9798f88 100644
--- a/lib/legacy/zstd_legacy.h
+++ b/lib/legacy/zstd_legacy.h
@@ -20,15 +20,9 @@ extern "C" {
 #include "mem.h"            /* MEM_STATIC */
 #include "error_private.h"  /* ERROR */
 #include "zstd.h"           /* ZSTD_inBuffer, ZSTD_outBuffer */
-#include "zstd_v01.h"
-#include "zstd_v02.h"
-#include "zstd_v03.h"
-#include "zstd_v04.h"
-#include "zstd_v05.h"
-#include "zstd_v06.h"
-#include "zstd_v07.h"
 
-#ifndef ZSTD_LEGACY_SUPPORT
+#if !defined (ZSTD_LEGACY_SUPPORT) || (ZSTD_LEGACY_SUPPORT == 0)
+#  undef ZSTD_LEGACY_SUPPORT
 #  define ZSTD_LEGACY_SUPPORT 8
 #endif
 

From aa8bcf360fe7e4da8b00c250f81893e71aa90c0c Mon Sep 17 00:00:00 2001
From: Nick Terrell 
Date: Mon, 13 Mar 2017 18:11:07 -0700
Subject: [PATCH 221/223] Add xz and lzma support.

Finish feature started by @inikep.

* Add xz and lzma compression and decompression support to target `xzstd`.
* Fix bug in gzip decompression that silently accepted truncated files.
* Add gzip frame composition tests.
* Add xz/lzma compatibility tests.
* Add xz/lzma frame composition tests.
---
 programs/Makefile  |  32 ++++++--
 programs/fileio.c  | 187 +++++++++++++++++++++++++++++++++++++++------
 programs/fileio.h  |   6 +-
 programs/zstdcli.c |  14 +++-
 tests/playTests.sh |  58 ++++++++++++++
 5 files changed, 261 insertions(+), 36 deletions(-)

diff --git a/programs/Makefile b/programs/Makefile
index a935c744a..8b97592d5 100644
--- a/programs/Makefile
+++ b/programs/Makefile
@@ -68,12 +68,27 @@ EXT =
 endif
 
 # zlib detection
+NO_ZLIB_MSG := ==> no zlib, building zstd without .gz support
 VOID = /dev/null
 HAVE_ZLIB := $(shell printf '\#include \nint main(){}' | $(CC) -o have_zlib -x c - -lz 2> $(VOID) && rm have_zlib$(EXT) && echo 1 || echo 0)
 ifeq ($(HAVE_ZLIB), 1)
+ZLIB_MSG := ==> building zstd with .gz compression support
 ZLIBCPP = -DZSTD_GZCOMPRESS -DZSTD_GZDECOMPRESS
 ZLIBLD = -lz
+else
+ZLIB_MSG := $(NO_ZLIB_MSG)
 endif
+# lzma detection
+NO_LZMA_MSG := ==> no liblzma, building zstd without .xz/.lzma support
+HAVE_LZMA := $(shell printf '\#include \nint main(){}' | $(CC) -o have_lzma -x c - -llzma 2> $(VOID) && rm have_lzma$(EXT) && echo 1 || echo 0)
+ifeq ($(HAVE_LZMA), 1)
+LZMA_MSG := ==> building zstd with .xz/.lzma compression support
+LZMACPP = -DZSTD_LZMACOMPRESS -DZSTD_LZMADECOMPRESS
+LZMALD = -llzma
+else
+LZMA_MSG := $(NO_LZMA_MSG)
+endif
+
 
 .PHONY: default all clean clean_decomp_o install uninstall generate_res
 
@@ -85,14 +100,15 @@ $(ZSTDDECOMP_O): CFLAGS += $(ALIGN_LOOP)
 
 zstd : CPPFLAGS += $(ZLIBCPP)
 zstd : LDFLAGS += $(ZLIBLD)
-zstd-nogz : HAVE_ZLIB=0
-zstd zstd-nogz : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
-zstd zstd-nogz : $(ZSTDLIB_OBJ) zstdcli.o fileio.o bench.o datagen.o dibio.o
-ifeq ($(HAVE_ZLIB), 1)
-	@echo "==> building zstd with .gz decompression support "
-else
-	@echo "==> no zlib, building zstd with .zst support only (no .gz support) "
-endif
+zstd : LZMA_MSG := $(NO_LZMA_MSG)
+zstd-nogz : ZLIB_MSG := $(NO_ZLIB_MSG)
+zstd-nogz : LZMA_MSG := $(NO_LZMA_MSG)
+xzstd : CPPFLAGS += $(ZLIBCPP) $(LZMACPP)
+xzstd : LDFLAGS += $(ZLIBLD) $(LZMALD)
+zstd zstd-nogz xzstd : CPPFLAGS += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
+zstd zstd-nogz xzstd : $(ZSTDLIB_OBJ) zstdcli.o fileio.o bench.o datagen.o dibio.o
+	@echo "$(ZLIB_MSG)"
+	@echo "$(LZMA_MSG)"
 ifneq (,$(filter Windows%,$(OS)))
 	windres/generate_res.bat
 endif
diff --git a/programs/fileio.c b/programs/fileio.c
index 41daa125e..e6481f1fa 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -44,6 +44,9 @@
 #    define z_const
 #  endif
 #endif
+#if defined(ZSTD_LZMACOMPRESS) || defined(ZSTD_LZMADECOMPRESS)
+#  include 
+#endif
 
 
 /*-*************************************
@@ -71,7 +74,6 @@
 #define MAX_DICT_SIZE (8 MB)   /* protection against large input (attack scenario) */
 
 #define FNSPACE 30
-#define GZ_EXTENSION ".gz"
 
 
 /*-*************************************
@@ -434,6 +436,65 @@ static unsigned long long FIO_compressGzFrame(cRess_t* ress, const char* srcFile
 #endif
 
 
+#ifdef ZSTD_LZMACOMPRESS
+static unsigned long long FIO_compressLzmaFrame(cRess_t* ress, const char* srcFileName, U64 const srcFileSize, int compressionLevel, U64* readsize, int plain_lzma)
+{
+    unsigned long long inFileSize = 0, outFileSize = 0;
+    lzma_stream strm = LZMA_STREAM_INIT;
+    lzma_action action = LZMA_RUN;
+    lzma_ret ret;
+
+    if (compressionLevel < 0) compressionLevel = 0;
+    if (compressionLevel > 9) compressionLevel = 9;
+
+    if (plain_lzma) {
+        lzma_options_lzma opt_lzma;
+        if (lzma_lzma_preset(&opt_lzma, compressionLevel)) EXM_THROW(71, "zstd: %s: lzma_lzma_preset error", srcFileName);
+        ret = lzma_alone_encoder(&strm, &opt_lzma); /* LZMA */
+        if (ret != LZMA_OK) EXM_THROW(71, "zstd: %s: lzma_alone_encoder error %d", srcFileName, ret);
+    } else {
+        ret = lzma_easy_encoder(&strm, compressionLevel, LZMA_CHECK_CRC64); /* XZ */
+        if (ret != LZMA_OK) EXM_THROW(71, "zstd: %s: lzma_easy_encoder error %d", srcFileName, ret);
+    }
+
+    strm.next_in = 0;
+    strm.avail_in = 0;
+    strm.next_out = ress->dstBuffer;
+    strm.avail_out = ress->dstBufferSize;
+
+    while (1) {
+        if (strm.avail_in == 0) {
+            size_t const inSize = fread(ress->srcBuffer, 1, ress->srcBufferSize, ress->srcFile);
+            if (inSize == 0) action = LZMA_FINISH;
+            inFileSize += inSize;
+            strm.next_in = ress->srcBuffer;
+            strm.avail_in = inSize;
+        }
+
+        ret = lzma_code(&strm, action);
+
+        if (ret != LZMA_OK && ret != LZMA_STREAM_END) EXM_THROW(72, "zstd: %s: lzma_code encoding error %d", srcFileName, ret);
+        {   size_t const compBytes = ress->dstBufferSize - strm.avail_out;
+            if (compBytes) {
+                if (fwrite(ress->dstBuffer, 1, compBytes, ress->dstFile) != compBytes) EXM_THROW(73, "Write error : cannot write to output file");
+                outFileSize += compBytes;
+                strm.next_out = ress->dstBuffer;
+                strm.avail_out = ress->dstBufferSize;
+            }
+        }
+        if (!srcFileSize) DISPLAYUPDATE(2, "\rRead : %u MB ==> %.2f%%", (U32)(inFileSize>>20), (double)outFileSize/inFileSize*100)
+        else DISPLAYUPDATE(2, "\rRead : %u / %u MB ==> %.2f%%", (U32)(inFileSize>>20), (U32)(srcFileSize>>20), (double)outFileSize/inFileSize*100);
+        if (ret == LZMA_STREAM_END) break;
+    }
+
+    lzma_end(&strm);
+    *readsize = inFileSize;
+
+    return outFileSize;
+}
+#endif
+
+
 /*! FIO_compressFilename_internal() :
  *  same as FIO_compressFilename_extRess(), with `ress.desFile` already opened.
  *  @return : 0 : compression completed correctly,
@@ -448,14 +509,26 @@ static int FIO_compressFilename_internal(cRess_t ress,
     U64 compressedfilesize = 0;
     U64 const fileSize = UTIL_getFileSize(srcFileName);
 
-    if (g_compressionType) {
+    switch (g_compressionType) {
+        case FIO_zstdCompression:
+            break;
+        case FIO_gzipCompression:
 #ifdef ZSTD_GZCOMPRESS
-        compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize);
+            compressedfilesize = FIO_compressGzFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize);
 #else
-        (void)compressionLevel;
-        EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", srcFileName);
+            (void)compressionLevel;
+            EXM_THROW(20, "zstd: %s: file cannot be compressed as gzip (zstd compiled without ZSTD_GZCOMPRESS) -- ignored \n", srcFileName);
 #endif
-        goto finish;
+            goto finish;
+        case FIO_xzCompression:
+        case FIO_lzmaCompression:
+#ifdef ZSTD_LZMACOMPRESS
+            compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, g_compressionType==FIO_lzmaCompression);
+#else
+            (void)compressionLevel;
+            EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n", srcFileName);
+#endif
+            goto finish;
     }
 
     /* init */
@@ -763,10 +836,10 @@ static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
 {
     if (storedSkips-->0) {   /* implies g_sparseFileSupport>0 */
         int const seekResult = LONG_SEEK(file, storedSkips, SEEK_CUR);
-        if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)\n");
+        if (seekResult != 0) EXM_THROW(69, "Final skip error (sparse file)");
         {   const char lastZeroByte[1] = { 0 };
             size_t const sizeCheck = fwrite(lastZeroByte, 1, 1, file);
-            if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero\n");
+            if (sizeCheck != 1) EXM_THROW(69, "Write error : cannot write last zero");
     }   }
 }
 
@@ -849,6 +922,7 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, co
 {
     unsigned long long outFileSize = 0;
     z_stream strm;
+    int flush = Z_NO_FLUSH;
     int ret;
 
     strm.zalloc = Z_NULL;
@@ -866,11 +940,12 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, co
     for ( ; ; ) {
         if (strm.avail_in == 0) {
             ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile);
-            if (ress->srcBufferLoaded == 0) break;
+            if (ress->srcBufferLoaded == 0) flush = Z_FINISH;
             strm.next_in = (z_const unsigned char*)ress->srcBuffer;
             strm.avail_in = (uInt)ress->srcBufferLoaded;
         }
-        ret = inflate(&strm, Z_NO_FLUSH);
+        ret = inflate(&strm, flush);
+        if (ret == Z_BUF_ERROR) EXM_THROW(39, "zstd: %s: premature end", srcFileName);
         if (ret != Z_OK && ret != Z_STREAM_END) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; }
         {   size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
             if (decompBytes) {
@@ -886,7 +961,60 @@ static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, co
     if (strm.avail_in > 0) memmove(ress->srcBuffer, strm.next_in, strm.avail_in);
     ress->srcBufferLoaded = strm.avail_in;
     ret = inflateEnd(&strm);
-    if (ret != Z_OK) EXM_THROW(32, "zstd: %s: inflateEnd error %d \n", srcFileName, ret);
+    if (ret != Z_OK) EXM_THROW(32, "zstd: %s: inflateEnd error %d", srcFileName, ret);
+    return outFileSize;
+}
+#endif
+
+
+#ifdef ZSTD_LZMADECOMPRESS
+static unsigned long long FIO_decompressLzmaFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName, int plain_lzma)
+{
+    unsigned long long outFileSize = 0;
+    lzma_stream strm = LZMA_STREAM_INIT;
+    lzma_action action = LZMA_RUN;
+    lzma_ret ret;
+
+    strm.next_in = 0;
+    strm.avail_in = 0;
+    if (plain_lzma) {
+        ret = lzma_alone_decoder(&strm, UINT64_MAX); /* LZMA */
+    } else {
+        ret = lzma_stream_decoder(&strm, UINT64_MAX, 0); /* XZ */
+    }
+
+    if (ret != LZMA_OK) EXM_THROW(71, "zstd: %s: lzma_alone_decoder/lzma_stream_decoder error %d", srcFileName, ret);
+
+    strm.next_out = ress->dstBuffer;
+    strm.avail_out = ress->dstBufferSize;
+    strm.avail_in = ress->srcBufferLoaded;
+    strm.next_in = ress->srcBuffer;
+
+    for ( ; ; ) {
+        if (strm.avail_in == 0) {
+            ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile);
+            if (ress->srcBufferLoaded == 0) action = LZMA_FINISH;
+            strm.next_in = ress->srcBuffer;
+            strm.avail_in = ress->srcBufferLoaded;
+        }
+        ret = lzma_code(&strm, action);
+
+        if (ret == LZMA_BUF_ERROR) EXM_THROW(39, "zstd: %s: premature end", srcFileName);
+        if (ret != LZMA_OK && ret != LZMA_STREAM_END) { DISPLAY("zstd: %s: lzma_code decoding error %d \n", srcFileName, ret); return 0; }
+        {   size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
+            if (decompBytes) {
+                if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(31, "Write error : cannot write to output file");
+                outFileSize += decompBytes;
+                strm.next_out = ress->dstBuffer;
+                strm.avail_out = ress->dstBufferSize;
+            }
+        }
+        if (ret == LZMA_STREAM_END) break;
+    }
+
+    if (strm.avail_in > 0) memmove(ress->srcBuffer, strm.next_in, strm.avail_in);
+    ress->srcBufferLoaded = strm.avail_in;
+    lzma_end(&strm);
     return outFileSize;
 }
 #endif
@@ -924,7 +1052,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
         }
         readSomething = 1;   /* there is at least >= 4 bytes in srcFile */
         if (ress.srcBufferLoaded < toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; }  /* srcFileName is empty */
-        if (buf[0] == 31 && buf[1] == 139) { /* gz header */
+        if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */
 #ifdef ZSTD_GZDECOMPRESS
             unsigned long long const result = FIO_decompressGzFrame(&ress, srcFile, srcFileName);
             if (result == 0) return 1;
@@ -932,6 +1060,16 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
 #else
             DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without ZSTD_GZDECOMPRESS) -- ignored \n", srcFileName);
             return 1;
+#endif
+        } else if ((buf[0] == 0xFD && buf[1] == 0x37)  /* xz magic number */
+                || (buf[0] == 0x5D && buf[1] == 0x00)) { /* lzma header (no magic number) */
+#ifdef ZSTD_LZMADECOMPRESS
+            unsigned long long const result = FIO_decompressLzmaFrame(&ress, srcFile, srcFileName, buf[0] != 0xFD);
+            if (result == 0) return 1;
+            filesize += result;
+#else
+            DISPLAYLEVEL(1, "zstd: %s: xz/lzma file cannot be uncompressed (zstd compiled without ZSTD_LZMADECOMPRESS) -- ignored \n", srcFileName);
+            return 1;
 #endif
         } else {
             if (!ZSTD_isFrame(ress.srcBuffer, toRead)) {
@@ -1020,32 +1158,31 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
             missingFiles += FIO_decompressSrcFile(ress, suffix, srcNamesTable[u]);
         if (fclose(ress.dstFile)) EXM_THROW(72, "Write error : cannot properly close stdout");
     } else {
-        size_t const suffixSize = strlen(suffix);
-        size_t const gzipSuffixSize = strlen(GZ_EXTENSION);
+        size_t suffixSize;
         size_t dfnSize = FNSPACE;
         unsigned u;
         char* dstFileName = (char*)malloc(FNSPACE);
         if (dstFileName==NULL) EXM_THROW(73, "not enough memory for dstFileName");
         for (u=0; u tmp
+    $ZSTD -f --format=gzip tmp
+    $ZSTD -f tmp
+    cat tmp.gz tmp.zst tmp.gz tmp.zst | $ZSTD -d -f -o tmp
+    head -c -1 tmp.gz | $ZSTD -t && die "incomplete frame not detected !"
+    rm tmp*
+else
+    $ECHO "gzip mode not supported"
+fi
+
+
+$ECHO "\n**** xz compatibility tests **** "
+
+LZMAMODE=1
+$ZSTD --format=xz -V || LZMAMODE=0
+if [ $LZMAMODE -eq 1 ]; then
+    $ECHO "xz support detected"
+    XZEXE=1
+    xz -V && lzma -V || XZEXE=0
+    if [ $XZEXE -eq 1 ]; then
+        ./datagen > tmp
+        $ZSTD --format=lzma -f tmp
+        $ZSTD --format=xz -f tmp
+        xz -t -v tmp.xz
+        xz -t -v tmp.lzma
+        xz -f -k tmp
+        lzma -f -k --lzma1 tmp
+        $ZSTD -d -f -v tmp.xz
+        $ZSTD -d -f -v tmp.lzma
+        rm tmp*
+    else
+        $ECHO "xz binary not detected"
+    fi
+else
+    $ECHO "xz mode not supported"
+fi
+
+
+$ECHO "\n**** xz frame tests **** "
+
+if [ $LZMAMODE -eq 1 ]; then
+    ./datagen > tmp
+    $ZSTD -f --format=xz tmp
+    $ZSTD -f --format=lzma tmp
+    $ZSTD -f tmp
+    cat tmp.xz tmp.lzma tmp.zst tmp.lzma tmp.xz tmp.zst | $ZSTD -d -f -o tmp
+    head -c -1 tmp.xz | $ZSTD -t && die "incomplete frame not detected !"
+    head -c -1 tmp.lzma | $ZSTD -t && die "incomplete frame not detected !"
+    rm tmp*
+else
+    $ECHO "xz mode not supported"
+fi
+
+
 $ECHO "\n**** zstd round-trip tests **** "
 
 roundTripTest

From 7ae3039f41c3277a659e828752a3fe84f2a5e221 Mon Sep 17 00:00:00 2001
From: Yann Collet 
Date: Tue, 14 Mar 2017 04:19:51 -0700
Subject: [PATCH 222/223] updated NEWS for v1.1.4

---
 NEWS | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/NEWS b/NEWS
index 9073a8724..bbd9e1688 100644
--- a/NEWS
+++ b/NEWS
@@ -3,10 +3,12 @@ cli : new : can compress in *.gz format, using --format=gzip command, by Przemys
 cli : new : advanced benchmark command --priority=rt
 cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77
 cli : fix : --rm remains silent when input is stdin
+cli : experimental : xzstd, with support for xz/lzma decoding, by Przemyslaw Skibinski
 speed : improved decompression speed in streaming mode for single shot scenarios (+5%)
 memory : DDict (decompression dictionary) memory usage down from 150 KB to 20 KB
 arch : 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell
 API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize()
+API : changed : dropped support of legacy versions <= v0.3 (can be changed by modifying ZSTD_LEGACY_SUPPORT value)
 build: new: meson build system in contrib/meson, by Dima Krasner
 build: improved cmake script, by @Majlen
 build: added -Wformat-security flag, as recommended by Padraig Brady

From dec2b96536893f16923f149d3eab9c8474e8dd4b Mon Sep 17 00:00:00 2001
From: Sean Purcell 
Date: Tue, 14 Mar 2017 11:24:09 -0700
Subject: [PATCH 223/223] Add functions missing from manual, and fix parameter
 alignment

---
 contrib/gen_html/gen_html.cpp | 12 ++++---
 doc/zstd_manual.html          | 67 +++++++++++++++++++++++------------
 lib/zstd.h                    | 12 ++++++-
 3 files changed, 64 insertions(+), 27 deletions(-)

diff --git a/contrib/gen_html/gen_html.cpp b/contrib/gen_html/gen_html.cpp
index 22ff65b10..e5261c086 100644
--- a/contrib/gen_html/gen_html.cpp
+++ b/contrib/gen_html/gen_html.cpp
@@ -19,7 +19,7 @@ void trim(string& s, string characters)
 {
     size_t p = s.find_first_not_of(characters);
     s.erase(0, p);
- 
+
     p = s.find_last_not_of(characters);
     if (string::npos != p)
        s.erase(p+1);
@@ -48,7 +48,7 @@ vector get_lines(vector& input, int& linenum, string terminator)
         line = input[linenum];
 
         if (terminator.empty() && line.empty()) { linenum--; break; }
-        
+
         epos = line.find(terminator);
         if (!terminator.empty() && epos!=string::npos) {
             out.push_back(line);
@@ -168,7 +168,11 @@ int main(int argc, char *argv[]) {
             sout << "
";
             for (l=0; l
";
             for (l=0; l" << endl << "" << endl;
 
     return 0;
-}
\ No newline at end of file
+}
diff --git a/doc/zstd_manual.html b/doc/zstd_manual.html
index 77e8974de..204f56ea5 100644
--- a/doc/zstd_manual.html
+++ b/doc/zstd_manual.html
@@ -55,8 +55,8 @@
 
Simple API

 
 
size_t ZSTD_compress( void* dst, size_t dstCapacity,
-                            const void* src, size_t srcSize,
-                                  int compressionLevel);
+                const void* src, size_t srcSize,
+                      int compressionLevel);
 
    Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
     Hint : compression runs faster if `dstCapacity` >=  `ZSTD_compressBound(srcSize)`.
     @return : compressed size written into `dst` (<= `dstCapacity),
@@ -64,7 +64,7 @@
 


 
 
size_t ZSTD_decompress( void* dst, size_t dstCapacity,
-                              const void* src, size_t compressedSize);
+                  const void* src, size_t compressedSize);
 
    `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
     `dstCapacity` is an upper bound of originalSize.
     If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
@@ -118,7 +118,11 @@ size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
 
    Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). 
 


 
-
Decompression context
typedef struct ZSTD_DCtx_s ZSTD_DCtx;
+
Decompression context
   When decompressing many times,
+   it is recommended to allocate a context just once, and re-use it for each successive compression operation.
+   This will make workload friendlier for system's memory.
+   Use one context per thread for parallel execution in multi-threaded environments. 
+
typedef struct ZSTD_DCtx_s ZSTD_DCtx;
 ZSTD_DCtx* ZSTD_createDCtx(void);
 size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 


@@ -129,19 +133,19 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 
Simple dictionary API

 
 
size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx,
-                                           void* dst, size_t dstCapacity,
-                                     const void* src, size_t srcSize,
-                                     const void* dict,size_t dictSize,
-                                           int compressionLevel);
+                               void* dst, size_t dstCapacity,
+                         const void* src, size_t srcSize,
+                         const void* dict,size_t dictSize,
+                               int compressionLevel);
 
   Compression using a predefined Dictionary (see dictBuilder/zdict.h).
    Note : This function loads the dictionary, resulting in significant startup delay.
    Note : When `dict == NULL || dictSize < 8` no dictionary is used. 
 


 
 
size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
-                                             void* dst, size_t dstCapacity,
-                                       const void* src, size_t srcSize,
-                                       const void* dict,size_t dictSize);
+                                 void* dst, size_t dstCapacity,
+                           const void* src, size_t srcSize,
+                           const void* dict,size_t dictSize);
 
   Decompression using a predefined Dictionary (see dictBuilder/zdict.h).
    Dictionary must be identical to the one used during compression.
    Note : This function loads the dictionary, resulting in significant startup delay.
@@ -162,9 +166,9 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 


 
 
size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
-                                            void* dst, size_t dstCapacity,
-                                      const void* src, size_t srcSize,
-                                      const ZSTD_CDict* cdict);
+                                void* dst, size_t dstCapacity,
+                          const void* src, size_t srcSize,
+                          const ZSTD_CDict* cdict);
 
   Compression using a digested Dictionary.
    Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
    Note that compression level is decided during dictionary creation. 
@@ -180,9 +184,9 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
 


 
 
size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
-                                              void* dst, size_t dstCapacity,
-                                        const void* src, size_t srcSize,
-                                        const ZSTD_DDict* ddict);
+                                  void* dst, size_t dstCapacity,
+                            const void* src, size_t srcSize,
+                            const ZSTD_DDict* ddict);
 
   Decompression using a digested Dictionary.
    Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. 
 


@@ -239,6 +243,14 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
  
 

 
+
ZSTD_CStream management functions
ZSTD_CStream* ZSTD_createCStream(void);
+size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
+


+
Streaming compression functions
size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
+size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
+


 
size_t ZSTD_CStreamInSize(void);    /**< recommended size for input buffer */
 


 
size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
@@ -264,6 +276,12 @@ size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
  
 

 
+
ZSTD_DStream management functions
ZSTD_DStream* ZSTD_createDStream(void);
+size_t ZSTD_freeDStream(ZSTD_DStream* zds);
+


+
Streaming decompression functions
size_t ZSTD_initDStream(ZSTD_DStream* zds);
+size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
+


 
size_t ZSTD_DStreamInSize(void);    /*!< recommended size for input buffer */
 


 
size_t ZSTD_DStreamOutSize(void);   /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
@@ -381,7 +399,7 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 


 
 
ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
-                                                  ZSTD_parameters params, ZSTD_customMem customMem);
+                                      ZSTD_parameters params, ZSTD_customMem customMem);
 
  Create a ZSTD_CDict using external alloc and free, and customized compression parameters 
 


 
@@ -409,10 +427,10 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
 


 
 
size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
-                                           void* dst, size_t dstCapacity,
-                                     const void* src, size_t srcSize,
-                                     const void* dict,size_t dictSize,
-                                           ZSTD_parameters params);
+                               void* dst, size_t dstCapacity,
+                         const void* src, size_t srcSize,
+                         const void* dict,size_t dictSize,
+                               ZSTD_parameters params);
 
   Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter 
 


 
@@ -443,6 +461,11 @@ typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; v
   It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict 
 


 
+
ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
+                                      unsigned byReference, ZSTD_customMem customMem);
+
  Create a ZSTD_DDict using external alloc and free, optionally by reference 
+


+
 
size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
 
  Gives the amount of memory used by a given ZSTD_DDict 
 


diff --git a/lib/zstd.h b/lib/zstd.h
index 0b48b7ec6..a3237c77e 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -139,7 +139,11 @@ ZSTDLIB_API size_t     ZSTD_freeCCtx(ZSTD_CCtx* cctx);
     Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). */
 ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
 
-/*= Decompression context */
+/*= Decompression context
+*   When decompressing many times,
+*   it is recommended to allocate a context just once, and re-use it for each successive compression operation.
+*   This will make workload friendlier for system's memory.
+*   Use one context per thread for parallel execution in multi-threaded environments. */
 typedef struct ZSTD_DCtx_s ZSTD_DCtx;
 ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
 ZSTDLIB_API size_t     ZSTD_freeDCtx(ZSTD_DCtx* dctx);
@@ -277,9 +281,11 @@ typedef struct ZSTD_outBuffer_s {
 * *******************************************************************/
 
 typedef struct ZSTD_CStream_s ZSTD_CStream;
+/*===== ZSTD_CStream management functions =====*/
 ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
 ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
 
+/*===== Streaming compression functions =====*/
 ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
 ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
 ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
@@ -313,9 +319,11 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void);   /**< recommended size for output
 * *******************************************************************************/
 
 typedef struct ZSTD_DStream_s ZSTD_DStream;
+/*===== ZSTD_DStream management functions =====*/
 ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
 ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
 
+/*===== Streaming decompression functions =====*/
 ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
 ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
 
@@ -540,6 +548,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
  *  It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict */
 ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
 
+/*! ZSTD_createDDict_advanced() :
+ *  Create a ZSTD_DDict using external alloc and free, optionally by reference */
 ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
                                                   unsigned byReference, ZSTD_customMem customMem);