From 736a28d835799495fa1e9a0092edcaf3912a770f Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Fri, 11 Aug 2017 14:34:49 -0700 Subject: [PATCH 01/14] reduce educational decoder to single frame decompression --- doc/educational_decoder/zstd_decompress.c | 102 +++------------------- 1 file changed, 13 insertions(+), 89 deletions(-) diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c index 7c8d8114d..93c346312 100644 --- a/doc/educational_decoder/zstd_decompress.c +++ b/doc/educational_decoder/zstd_decompress.c @@ -28,6 +28,7 @@ size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len, /// Get the decompressed size of an input stream so memory can be allocated in /// advance /// Returns -1 if the size can't be determined +/// Assumes decompression of a single frame size_t ZSTD_get_decompressed_size(const void *const src, const size_t src_len); /******* UTILITY MACROS AND TYPES *********************************************/ @@ -396,9 +397,9 @@ size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len, // Multiple frames can be appended into a single file or stream. A frame is // totally independent, has a defined beginning and end, and a set of // parameters which tells the decoder how to decompress it." - while (IO_istream_len(&in) > 0) { - decode_frame(&out, &in, &parsed_dict); - } + + /* this decoder assumes decompression of a single frame */ + decode_frame(&out, &in, &parsed_dict); free_dictionary(&parsed_dict); @@ -424,30 +425,6 @@ static void decompress_data(frame_context_t *const ctx, ostream_t *const out, static void decode_frame(ostream_t *const out, istream_t *const in, const dictionary_t *const dict) { const u32 magic_number = IO_read_bits(in, 32); - - // Skippable frame - // - // "Magic_Number - // - // 4 Bytes, little-endian format. Value : 0x184D2A5?, which means any value - // from 0x184D2A50 to 0x184D2A5F. All 16 values are valid to identify a - // skippable frame." - if ((magic_number & ~0xFU) == 0x184D2A50U) { - // "Skippable frames allow the insertion of user-defined data into a - // flow of concatenated frames. Its design is pretty straightforward, - // with the sole objective to allow the decoder to quickly skip over - // user-defined data and continue decoding. - // - // Skippable frames defined in this specification are compatible with - // LZ4 ones." - const size_t frame_size = IO_read_bits(in, 32); - - // skip over frame - IO_advance_input(in, frame_size); - - return; - } - // Zstandard frame // // "Magic_Number @@ -460,8 +437,8 @@ static void decode_frame(ostream_t *const out, istream_t *const in, return; } - // not a real frame - ERROR("Invalid magic number"); + // not a real frame or a skippable frame + ERROR("Tried to decode non-ZSTD frame"); } /// Decode a frame that contains compressed data. Not all frames do as there @@ -1420,28 +1397,17 @@ static void execute_sequences(frame_context_t *const ctx, ostream_t *const out, /******* END SEQUENCE EXECUTION ***********************************************/ /******* OUTPUT SIZE COUNTING *************************************************/ -static void traverse_frame(const frame_header_t *const header, istream_t *const in); - /// Get the decompressed size of an input stream so memory can be allocated in /// advance. -/// This is more complex than the implementation in the reference -/// implementation, as this API allows for the decompression of multiple -/// concatenated frames. +/// This implementation assumes `src` points to a single ZSTD-compressed frame size_t ZSTD_get_decompressed_size(const void *src, const size_t src_len) { istream_t in = IO_make_istream(src, src_len); - size_t dst_size = 0; - // Each frame header only gives us the size of its frame, so iterate over - // all - // frames - while (IO_istream_len(&in) > 0) { + // get decompressed size from ZSTD frame header + { const u32 magic_number = IO_read_bits(&in, 32); - if ((magic_number & ~0xFU) == 0x184D2A50U) { - // skippable frame, this has no impact on output size - const size_t frame_size = IO_read_bits(&in, 32); - IO_advance_input(&in, frame_size); - } else if (magic_number == 0xFD2FB528U) { + if (magic_number == 0xFD2FB528U) { // ZSTD frame frame_header_t header; parse_frame_header(&header, &in); @@ -1451,54 +1417,13 @@ size_t ZSTD_get_decompressed_size(const void *src, const size_t src_len) { return -1; } - dst_size += header.frame_content_size; - - // Consume the input from the frame to reach the start of the next - traverse_frame(&header, &in); + return header.frame_content_size; } else { - // not a real frame - ERROR("Invalid magic number"); + // not a real frame or skippable frame + ERROR("ZSTD frame magic number did not match"); } } - - return dst_size; } - -/// Iterate over each block in a frame to find the end of it, to get to the -/// start of the next frame -static void traverse_frame(const frame_header_t *const header, istream_t *const in) { - int last_block = 0; - - do { - // Parse the block header - last_block = IO_read_bits(in, 1); - const int block_type = IO_read_bits(in, 2); - const size_t block_len = IO_read_bits(in, 21); - - switch (block_type) { - case 0: // Raw block, block_len bytes - IO_advance_input(in, block_len); - break; - case 1: // RLE block, 1 byte - IO_advance_input(in, 1); - break; - case 2: // Compressed block, compressed size is block_len - IO_advance_input(in, block_len); - break; - case 3: - // Reserved block type - CORRUPTION(); - break; - default: - IMPOSSIBLE(); - } - } while (!last_block); - - if (header->content_checksum_flag) { - IO_advance_input(in, 4); - } -} - /******* END OUTPUT SIZE COUNTING *********************************************/ /******* DICTIONARY PARSING ***************************************************/ @@ -2355,4 +2280,3 @@ static void FSE_copy_dtable(FSE_dtable *const dst, const FSE_dtable *const src) memcpy(dst->new_state_base, src->new_state_base, size * sizeof(u16)); } /******* END FSE PRIMITIVES ***************************************************/ - From d0dc675596e5d8caa337494b4a3857389b7a448e Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Fri, 11 Aug 2017 14:35:13 -0700 Subject: [PATCH 02/14] add makefile --- doc/educational_decoder/Makefile | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 doc/educational_decoder/Makefile diff --git a/doc/educational_decoder/Makefile b/doc/educational_decoder/Makefile new file mode 100644 index 000000000..27b184154 --- /dev/null +++ b/doc/educational_decoder/Makefile @@ -0,0 +1,21 @@ +HARNESS_FILES=*.c + +MULTITHREAD_LDFLAGS = -pthread +DEBUGFLAGS= -g -DZSTD_DEBUG=1 +CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ + -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) +CFLAGS ?= -O3 +CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ + -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ + -Wstrict-prototypes -Wundef -Wformat-security \ + -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ + -Wredundant-decls +CFLAGS += $(DEBUGFLAGS) +CFLAGS += $(MOREFLAGS) +FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MULTITHREAD_LDFLAGS) + +harness: $(HARNESS_FILES) + $(CC) $(FLAGS) $^ -o $@ + +clean: + @$(RM) -f harness From 9f67e8652e6e3703b9794672df768e842535bc8e Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Fri, 11 Aug 2017 14:41:44 -0700 Subject: [PATCH 03/14] fixed warnings shown by compiler --- doc/educational_decoder/harness.c | 3 +-- doc/educational_decoder/zstd_decompress.c | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/doc/educational_decoder/harness.c b/doc/educational_decoder/harness.c index 683278dfc..31e1c2924 100644 --- a/doc/educational_decoder/harness.c +++ b/doc/educational_decoder/harness.c @@ -87,7 +87,7 @@ int main(int argc, char **argv) { } size_t decompressed_size = ZSTD_get_decompressed_size(input, input_size); - if (decompressed_size == -1) { + if (decompressed_size == (size_t)-1) { decompressed_size = MAX_COMPRESSION_RATIO * input_size; fprintf(stderr, "WARNING: Compressed data does not contain " "decompressed size, going to assume the compression " @@ -117,4 +117,3 @@ int main(int argc, char **argv) { free(dict); input = output = dict = NULL; } - diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c index 93c346312..236ad58aa 100644 --- a/doc/educational_decoder/zstd_decompress.c +++ b/doc/educational_decoder/zstd_decompress.c @@ -1374,7 +1374,7 @@ static void execute_sequences(frame_context_t *const ctx, ostream_t *const out, // than the offset // ex: if the output so far was "abc", a command with offset=3 and // match_length=6 would produce "abcabcabc" as the new output - for (size_t i = 0; i < match_length; i++) { + for (size_t j = 0; j < match_length; j++) { *write_ptr = *(write_ptr - offset); write_ptr++; } @@ -2117,7 +2117,7 @@ static void FSE_init_dtable(FSE_dtable *const dtable, } // Now we can fill baseline and num bits - for (int i = 0; i < size; i++) { + for (size_t i = 0; i < size; i++) { u8 symbol = dtable->symbols[i]; u16 next_state_desc = state_desc[symbol]++; // Fills in the table appropriately, next_state_desc increases by symbol From bd308d806b8949928afab440e6148fdc728b4bd9 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Fri, 11 Aug 2017 14:42:15 -0700 Subject: [PATCH 04/14] remove debug symbols when cleaning, added a simple test --- doc/educational_decoder/Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/educational_decoder/Makefile b/doc/educational_decoder/Makefile index 27b184154..451c2351b 100644 --- a/doc/educational_decoder/Makefile +++ b/doc/educational_decoder/Makefile @@ -19,3 +19,11 @@ harness: $(HARNESS_FILES) clean: @$(RM) -f harness + @$(RM) -rf harness.dSYM + +test: harness + @zstd README.md -o tmp.zst + @./harness tmp.zst tmp + @diff -s tmp README.md + @$(RM) -f tmp* + @make clean From bfc6db8d6aabe8e7b0470128620d7db091fc0580 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Fri, 11 Aug 2017 17:53:37 -0700 Subject: [PATCH 05/14] exposed dictionary functions/types --- doc/educational_decoder/Makefile | 5 +++ doc/educational_decoder/zstd_decompress.c | 34 ++++------------- doc/educational_decoder/zstd_decompress.h | 46 ++++++++++++++++++++--- 3 files changed, 53 insertions(+), 32 deletions(-) diff --git a/doc/educational_decoder/Makefile b/doc/educational_decoder/Makefile index 451c2351b..ace1294f8 100644 --- a/doc/educational_decoder/Makefile +++ b/doc/educational_decoder/Makefile @@ -26,4 +26,9 @@ test: harness @./harness tmp.zst tmp @diff -s tmp README.md @$(RM) -f tmp* + @zstd --train harness.c zstd_decompress.c zstd_decompress.h README.md + @zstd -D dictionary README.md -o tmp.zst + @./harness tmp.zst tmp dictionary + @diff -s tmp README.md + @$(RM) -f tmp* dictionary @make clean diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c index 236ad58aa..fe8770924 100644 --- a/doc/educational_decoder/zstd_decompress.c +++ b/doc/educational_decoder/zstd_decompress.c @@ -14,22 +14,7 @@ #include #include #include - -/// Zstandard decompression functions. -/// `dst` must point to a space at least as large as the reconstructed output. -size_t ZSTD_decompress(void *const dst, const size_t dst_len, - const void *const src, const size_t src_len); -/// If `dict != NULL` and `dict_len >= 8`, does the same thing as -/// `ZSTD_decompress` but uses the provided dict -size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len, - const void *const src, const size_t src_len, - const void *const dict, const size_t dict_len); - -/// Get the decompressed size of an input stream so memory can be allocated in -/// advance -/// Returns -1 if the size can't be determined -/// Assumes decompression of a single frame -size_t ZSTD_get_decompressed_size(const void *const src, const size_t src_len); +#include "zstd_decompress.h" /******* UTILITY MACROS AND TYPES *********************************************/ // Max block size decompressed size is 128 KB and literal blocks can't be @@ -308,7 +293,7 @@ typedef struct { /// The decoded contents of a dictionary so that it doesn't have to be repeated /// for each frame that uses it -typedef struct { +struct dictionary_s { // Entropy tables HUF_dtable literals_dtable; FSE_dtable ll_dtable; @@ -323,7 +308,7 @@ typedef struct { u64 previous_offsets[3]; u32 dictionary_id; -} dictionary_t; +}; /// A tuple containing the parts necessary to decode and execute a ZSTD sequence /// command @@ -368,10 +353,6 @@ static void execute_sequences(frame_context_t *const ctx, ostream_t *const out, const sequence_command_t *const sequences, const size_t num_sequences); -// Parse a provided dictionary blob for use in decompression -static void parse_dictionary(dictionary_t *const dict, const u8 *src, - size_t src_len); -static void free_dictionary(dictionary_t *const dict); /******* END ZSTD HELPER STRUCTS AND PROTOTYPES *******************************/ size_t ZSTD_decompress(void *const dst, const size_t dst_len, @@ -387,7 +368,7 @@ size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len, memset(&parsed_dict, 0, sizeof(dictionary_t)); // dict_len < 8 is not a valid dictionary if (dict && dict_len > 8) { - parse_dictionary(&parsed_dict, (const u8 *)dict, dict_len); + parse_dictionary(&parsed_dict, dict, dict_len); } istream_t in = IO_make_istream(src, src_len); @@ -1430,14 +1411,15 @@ size_t ZSTD_get_decompressed_size(const void *src, const size_t src_len) { static void init_dictionary_content(dictionary_t *const dict, istream_t *const in); -static void parse_dictionary(dictionary_t *const dict, const u8 *src, +void parse_dictionary(dictionary_t *const dict, const void *src, size_t src_len) { + const u8 *byte_src = (const u8 *)src; memset(dict, 0, sizeof(dictionary_t)); if (src_len < 8) { INP_SIZE(); } - istream_t in = IO_make_istream(src, src_len); + istream_t in = IO_make_istream(byte_src, src_len); const u32 magic_number = IO_read_bits(&in, 32); if (magic_number != 0xEC30A437) { @@ -1495,7 +1477,7 @@ static void init_dictionary_content(dictionary_t *const dict, } /// Free an allocated dictionary -static void free_dictionary(dictionary_t *const dict) { +void free_dictionary(dictionary_t *const dict) { HUF_free_dtable(&dict->literals_dtable); FSE_free_dtable(&dict->ll_dtable); FSE_free_dtable(&dict->of_dtable); diff --git a/doc/educational_decoder/zstd_decompress.h b/doc/educational_decoder/zstd_decompress.h index 16f4da3eb..2ac903075 100644 --- a/doc/educational_decoder/zstd_decompress.h +++ b/doc/educational_decoder/zstd_decompress.h @@ -7,10 +7,44 @@ * of patent rights can be found in the PATENTS file in the same directory. */ -size_t ZSTD_decompress(void *const dst, const size_t dst_len, - const void *const src, const size_t src_len); -size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len, - const void *const src, const size_t src_len, - const void *const dict, const size_t dict_len); -size_t ZSTD_get_decompressed_size(const void *const src, const size_t src_len); +/******* DECOMPRESSION FUNCTIONS **********************************************/ +/// Zstandard decompression functions. +/// `dst` must point to a space at least as large as the reconstructed output. +size_t ZSTD_decompress(void *const dst, const size_t dst_len, + const void *const src, const size_t src_len); + +/// If `dict != NULL` and `dict_len >= 8`, does the same thing as +/// `ZSTD_decompress` but uses the provided dict +size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len, + const void *const src, const size_t src_len, + const void *const dict, const size_t dict_len); + +/// Get the decompressed size of an input stream so memory can be allocated in +/// advance +/// Returns -1 if the size can't be determined +/// Assumes decompression of a single frame +size_t ZSTD_get_decompressed_size(const void *const src, const size_t src_len); +/******* END DECOMPRESSION FUNCTIONS ******************************************/ + +/******* DICTIONARY MANAGEMENT ***********************************************/ +/* + * Contains the parsed contents of a dictionary + * This includes Huffman and FSE tables used for decoding and data on offsets + */ +typedef struct dictionary_s dictionary_t; + +/* + * Parse a provided dictionary blob for use in decompression + * `src` -- must point to memory space representing the dictionary + * `src_len` -- must provide the dictionary size + * `dict` -- will contain the parsed contents of the dictionary and + * can be used for decompression + */ +void parse_dictionary(dictionary_t *const dict, const void *src, + size_t src_len); +/* + * Free internal Huffman tables, FSE tables, and dictionary content + */ +void free_dictionary(dictionary_t *const dict); +/******* END DICTIONARY MANAGEMENT *******************************************/ From 7ef9c6f4b2675d4297f2ad4c2a3291a56d33f6c1 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Fri, 11 Aug 2017 18:40:19 -0700 Subject: [PATCH 06/14] made separate API for dictionary management --- doc/educational_decoder/harness.c | 8 +++++- doc/educational_decoder/zstd_decompress.c | 31 +++++++++++++---------- doc/educational_decoder/zstd_decompress.h | 16 +++++++++--- 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/doc/educational_decoder/harness.c b/doc/educational_decoder/harness.c index 31e1c2924..43d6df7fc 100644 --- a/doc/educational_decoder/harness.c +++ b/doc/educational_decoder/harness.c @@ -106,9 +106,15 @@ int main(int argc, char **argv) { return 1; } + dictionary_t* parsed_dict = create_dictionary(); + if (dict) { + parse_dictionary(parsed_dict, dict, dict_size); + } size_t decompressed = ZSTD_decompress_with_dict(output, decompressed_size, - input, input_size, dict, dict_size); + input, input_size, parsed_dict); + + free_dictionary(parsed_dict); write_file(argv[2], output, decompressed); diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c index fe8770924..e22df1c10 100644 --- a/doc/educational_decoder/zstd_decompress.c +++ b/doc/educational_decoder/zstd_decompress.c @@ -357,19 +357,16 @@ static void execute_sequences(frame_context_t *const ctx, ostream_t *const out, size_t ZSTD_decompress(void *const dst, const size_t dst_len, const void *const src, const size_t src_len) { - return ZSTD_decompress_with_dict(dst, dst_len, src, src_len, NULL, 0); + dictionary_t* uninit_dict = create_dictionary(); + size_t const decomp_size = ZSTD_decompress_with_dict(dst, dst_len, src, + src_len, uninit_dict); + free_dictionary(uninit_dict); + return decomp_size; } size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len, const void *const src, const size_t src_len, - const void *const dict, - const size_t dict_len) { - dictionary_t parsed_dict; - memset(&parsed_dict, 0, sizeof(dictionary_t)); - // dict_len < 8 is not a valid dictionary - if (dict && dict_len > 8) { - parse_dictionary(&parsed_dict, dict, dict_len); - } + dictionary_t* parsed_dict) { istream_t in = IO_make_istream(src, src_len); ostream_t out = IO_make_ostream(dst, dst_len); @@ -380,9 +377,7 @@ size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len, // parameters which tells the decoder how to decompress it." /* this decoder assumes decompression of a single frame */ - decode_frame(&out, &in, &parsed_dict); - - free_dictionary(&parsed_dict); + decode_frame(&out, &in, parsed_dict); return out.ptr - (u8 *)dst; } @@ -1408,6 +1403,16 @@ size_t ZSTD_get_decompressed_size(const void *src, const size_t src_len) { /******* END OUTPUT SIZE COUNTING *********************************************/ /******* DICTIONARY PARSING ***************************************************/ +#define DICT_SIZE_ERROR() ERROR("Dictionary size cannot be less than 8 bytes") + +dictionary_t* create_dictionary() { + dictionary_t* dict = calloc(1, sizeof(dictionary_t)); + if (!dict) { + BAD_ALLOC(); + } + return dict; +} + static void init_dictionary_content(dictionary_t *const dict, istream_t *const in); @@ -1416,7 +1421,7 @@ void parse_dictionary(dictionary_t *const dict, const void *src, const u8 *byte_src = (const u8 *)src; memset(dict, 0, sizeof(dictionary_t)); if (src_len < 8) { - INP_SIZE(); + DICT_SIZE_ERROR(); } istream_t in = IO_make_istream(byte_src, src_len); diff --git a/doc/educational_decoder/zstd_decompress.h b/doc/educational_decoder/zstd_decompress.h index 2ac903075..41009909b 100644 --- a/doc/educational_decoder/zstd_decompress.h +++ b/doc/educational_decoder/zstd_decompress.h @@ -7,6 +7,13 @@ * of patent rights can be found in the PATENTS file in the same directory. */ +/******* EXPOSED TYPES ********************************************************/ +/* +* Contains the parsed contents of a dictionary +* This includes Huffman and FSE tables used for decoding and data on offsets +*/ +typedef struct dictionary_s dictionary_t; +/******* END EXPOSED TYPES ****************************************************/ /******* DECOMPRESSION FUNCTIONS **********************************************/ /// Zstandard decompression functions. @@ -18,7 +25,7 @@ size_t ZSTD_decompress(void *const dst, const size_t dst_len, /// `ZSTD_decompress` but uses the provided dict size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len, const void *const src, const size_t src_len, - const void *const dict, const size_t dict_len); + dictionary_t* parsed_dict); /// Get the decompressed size of an input stream so memory can be allocated in /// advance @@ -29,10 +36,10 @@ size_t ZSTD_get_decompressed_size(const void *const src, const size_t src_len); /******* DICTIONARY MANAGEMENT ***********************************************/ /* - * Contains the parsed contents of a dictionary - * This includes Huffman and FSE tables used for decoding and data on offsets + * Return a valid dictionary_t pointer for use with dictionary initialization + * or decompression */ -typedef struct dictionary_s dictionary_t; +dictionary_t* create_dictionary(); /* * Parse a provided dictionary blob for use in decompression @@ -43,6 +50,7 @@ typedef struct dictionary_s dictionary_t; */ void parse_dictionary(dictionary_t *const dict, const void *src, size_t src_len); + /* * Free internal Huffman tables, FSE tables, and dictionary content */ From 38f4e433814158dde6039317188793d637e29310 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 14 Aug 2017 09:41:04 -0700 Subject: [PATCH 07/14] added error checking for dictionary initialized with null src --- doc/educational_decoder/zstd_decompress.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c index e22df1c10..961d27a5b 100644 --- a/doc/educational_decoder/zstd_decompress.c +++ b/doc/educational_decoder/zstd_decompress.c @@ -1404,6 +1404,7 @@ size_t ZSTD_get_decompressed_size(const void *src, const size_t src_len) { /******* DICTIONARY PARSING ***************************************************/ #define DICT_SIZE_ERROR() ERROR("Dictionary size cannot be less than 8 bytes") +#define NULL_SRC() ERROR("Tried to create dictionary with pointer to null src"); dictionary_t* create_dictionary() { dictionary_t* dict = calloc(1, sizeof(dictionary_t)); @@ -1420,6 +1421,9 @@ void parse_dictionary(dictionary_t *const dict, const void *src, size_t src_len) { const u8 *byte_src = (const u8 *)src; memset(dict, 0, sizeof(dictionary_t)); + if (src == NULL) { /* cannot initialize dictionary with null src */ + NULL_SRC(); + } if (src_len < 8) { DICT_SIZE_ERROR(); } From 93c1309fd432aece3daef715d4f9677864b011c0 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 14 Aug 2017 13:08:30 -0700 Subject: [PATCH 08/14] added free to free_dictionary() --- doc/educational_decoder/zstd_decompress.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c index 961d27a5b..f45091b16 100644 --- a/doc/educational_decoder/zstd_decompress.c +++ b/doc/educational_decoder/zstd_decompress.c @@ -1495,6 +1495,8 @@ void free_dictionary(dictionary_t *const dict) { free(dict->content); memset(dict, 0, sizeof(dictionary_t)); + + free(dict); } /******* END DICTIONARY PARSING ***********************************************/ From b6d6be58c9b377f48e9d6959c1f9f8a3cf73b21f Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 14 Aug 2017 14:05:16 -0700 Subject: [PATCH 09/14] created separate function for copying literals during sequence execution --- doc/educational_decoder/zstd_decompress.c | 37 ++++++++++++++--------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c index f45091b16..5b88e4c97 100644 --- a/doc/educational_decoder/zstd_decompress.c +++ b/doc/educational_decoder/zstd_decompress.c @@ -353,6 +353,9 @@ static void execute_sequences(frame_context_t *const ctx, ostream_t *const out, const sequence_command_t *const sequences, const size_t num_sequences); +static u32 copy_literals(sequence_command_t seq, istream_t *litstream, + ostream_t *const out); + /******* END ZSTD HELPER STRUCTS AND PROTOTYPES *******************************/ size_t ZSTD_decompress(void *const dst, const size_t dst_len, @@ -1256,23 +1259,10 @@ static void execute_sequences(frame_context_t *const ctx, ostream_t *const out, for (size_t i = 0; i < num_sequences; i++) { const sequence_command_t seq = sequences[i]; - { - // If the sequence asks for more literals than are left, the - // sequence must be corrupted - if (seq.literal_length > IO_istream_len(&litstream)) { - CORRUPTION(); - } - - u8 *const write_ptr = IO_write_bytes(out, seq.literal_length); - const u8 *const read_ptr = - IO_read_bytes(&litstream, seq.literal_length); - // Copy literals to output - memcpy(write_ptr, read_ptr, seq.literal_length); - - total_output += seq.literal_length; + const u32 literals_size = copy_literals(seq, &litstream, out); + total_output += literals_size; } - size_t offset; // Offsets are special, we need to handle the repeat offsets @@ -1370,6 +1360,23 @@ static void execute_sequences(frame_context_t *const ctx, ostream_t *const out, ctx->current_total_output = total_output; } + +static u32 copy_literals(const sequence_command_t seq, istream_t *litstream, + ostream_t *const out) { + // If the sequence asks for more literals than are left, the + // sequence must be corrupted + if (seq.literal_length > IO_istream_len(litstream)) { + CORRUPTION(); + } + + u8 *const write_ptr = IO_write_bytes(out, seq.literal_length); + const u8 *const read_ptr = + IO_read_bytes(litstream, seq.literal_length); + // Copy literals to output + memcpy(write_ptr, read_ptr, seq.literal_length); + + return seq.literal_length; +} /******* END SEQUENCE EXECUTION ***********************************************/ /******* OUTPUT SIZE COUNTING *************************************************/ From d3e57db0bd1b1e801986be02a34aeb9416892df2 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 14 Aug 2017 14:20:12 -0700 Subject: [PATCH 10/14] created separate function for offset computation --- doc/educational_decoder/zstd_decompress.c | 96 ++++++++++++----------- 1 file changed, 51 insertions(+), 45 deletions(-) diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c index 5b88e4c97..1a999fd92 100644 --- a/doc/educational_decoder/zstd_decompress.c +++ b/doc/educational_decoder/zstd_decompress.c @@ -356,6 +356,8 @@ static void execute_sequences(frame_context_t *const ctx, ostream_t *const out, static u32 copy_literals(sequence_command_t seq, istream_t *litstream, ostream_t *const out); +static size_t compute_offset(sequence_command_t seq, u64 *const offset_hist); + /******* END ZSTD HELPER STRUCTS AND PROTOTYPES *******************************/ size_t ZSTD_decompress(void *const dst, const size_t dst_len, @@ -1263,51 +1265,7 @@ static void execute_sequences(frame_context_t *const ctx, ostream_t *const out, const u32 literals_size = copy_literals(seq, &litstream, out); total_output += literals_size; } - size_t offset; - - // Offsets are special, we need to handle the repeat offsets - if (seq.offset <= 3) { - // "The first 3 values define a repeated offset and we will call - // them Repeated_Offset1, Repeated_Offset2, and Repeated_Offset3. - // They are sorted in recency order, with Repeated_Offset1 meaning - // 'most recent one'". - - // Use 0 indexing for the array - u32 idx = seq.offset - 1; - if (seq.literal_length == 0) { - // "There is an exception though, when current sequence's - // literals length is 0. In this case, repeated offsets are - // shifted by one, so Repeated_Offset1 becomes Repeated_Offset2, - // Repeated_Offset2 becomes Repeated_Offset3, and - // Repeated_Offset3 becomes Repeated_Offset1 - 1_byte." - idx++; - } - - if (idx == 0) { - offset = offset_hist[0]; - } else { - // If idx == 3 then literal length was 0 and the offset was 3, - // as per the exception listed above - offset = idx < 3 ? offset_hist[idx] : offset_hist[0] - 1; - - // If idx == 1 we don't need to modify offset_hist[2], since - // we're using the second-most recent code - if (idx > 1) { - offset_hist[2] = offset_hist[1]; - } - offset_hist[1] = offset_hist[0]; - offset_hist[0] = offset; - } - } else { - // When it's not a repeat offset: - // "if (Offset_Value > 3) offset = Offset_Value - 3;" - offset = seq.offset - 3; - - // Shift back history - offset_hist[2] = offset_hist[1]; - offset_hist[1] = offset_hist[0]; - offset_hist[0] = offset; - } + size_t offset = compute_offset(seq, offset_hist); size_t match_length = seq.match_length; @@ -1377,6 +1335,54 @@ static u32 copy_literals(const sequence_command_t seq, istream_t *litstream, return seq.literal_length; } + +static size_t compute_offset(sequence_command_t seq, u64 *const offset_hist) { + size_t offset; + // Offsets are special, we need to handle the repeat offsets + if (seq.offset <= 3) { + // "The first 3 values define a repeated offset and we will call + // them Repeated_Offset1, Repeated_Offset2, and Repeated_Offset3. + // They are sorted in recency order, with Repeated_Offset1 meaning + // 'most recent one'". + + // Use 0 indexing for the array + u32 idx = seq.offset - 1; + if (seq.literal_length == 0) { + // "There is an exception though, when current sequence's + // literals length is 0. In this case, repeated offsets are + // shifted by one, so Repeated_Offset1 becomes Repeated_Offset2, + // Repeated_Offset2 becomes Repeated_Offset3, and + // Repeated_Offset3 becomes Repeated_Offset1 - 1_byte." + idx++; + } + + if (idx == 0) { + offset = offset_hist[0]; + } else { + // If idx == 3 then literal length was 0 and the offset was 3, + // as per the exception listed above + offset = idx < 3 ? offset_hist[idx] : offset_hist[0] - 1; + + // If idx == 1 we don't need to modify offset_hist[2], since + // we're using the second-most recent code + if (idx > 1) { + offset_hist[2] = offset_hist[1]; + } + offset_hist[1] = offset_hist[0]; + offset_hist[0] = offset; + } + } else { + // When it's not a repeat offset: + // "if (Offset_Value > 3) offset = Offset_Value - 3;" + offset = seq.offset - 3; + + // Shift back history + offset_hist[2] = offset_hist[1]; + offset_hist[1] = offset_hist[0]; + offset_hist[0] = offset; + } + return offset; +} /******* END SEQUENCE EXECUTION ***********************************************/ /******* OUTPUT SIZE COUNTING *************************************************/ From 9d56c2127942ed89f8fab2128b0a25422735374f Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 14 Aug 2017 15:06:03 -0700 Subject: [PATCH 11/14] added separate function for executing match copy command --- doc/educational_decoder/zstd_decompress.c | 78 +++++++++++++---------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c index 1a999fd92..cbfeaa166 100644 --- a/doc/educational_decoder/zstd_decompress.c +++ b/doc/educational_decoder/zstd_decompress.c @@ -358,6 +358,10 @@ static u32 copy_literals(sequence_command_t seq, istream_t *litstream, static size_t compute_offset(sequence_command_t seq, u64 *const offset_hist); +static void execute_match_copy(frame_context_t *const ctx, size_t offset, + size_t match_length, size_t total_output, + ostream_t *const out); + /******* END ZSTD HELPER STRUCTS AND PROTOTYPES *******************************/ size_t ZSTD_decompress(void *const dst, const size_t dst_len, @@ -1269,41 +1273,9 @@ static void execute_sequences(frame_context_t *const ctx, ostream_t *const out, size_t match_length = seq.match_length; - u8 *write_ptr = IO_write_bytes(out, match_length); - if (total_output <= ctx->header.window_size) { - // In this case offset might go back into the dictionary - if (offset > total_output + ctx->dict_content_len) { - // The offset goes beyond even the dictionary - CORRUPTION(); - } + execute_match_copy(ctx, offset, match_length, total_output, out); - if (offset > total_output) { - // "The rest of the dictionary is its content. The content act - // as a "past" in front of data to compress or decompress, so it - // can be referenced in sequence commands." - const size_t dict_copy = - MIN(offset - total_output, match_length); - const size_t dict_offset = - ctx->dict_content_len - (offset - total_output); - - memcpy(write_ptr, ctx->dict_content + dict_offset, dict_copy); - write_ptr += dict_copy; - match_length -= dict_copy; - } - } else if (offset > ctx->header.window_size) { - CORRUPTION(); - } - - // We must copy byte by byte because the match length might be larger - // than the offset - // ex: if the output so far was "abc", a command with offset=3 and - // match_length=6 would produce "abcabcabc" as the new output - for (size_t j = 0; j < match_length; j++) { - *write_ptr = *(write_ptr - offset); - write_ptr++; - } - - total_output += seq.match_length; + total_output += match_length; } // Copy any leftover literals @@ -1383,6 +1355,44 @@ static size_t compute_offset(sequence_command_t seq, u64 *const offset_hist) { } return offset; } + +static void execute_match_copy(frame_context_t *const ctx, size_t offset, + size_t match_length, size_t total_output, + ostream_t *const out) { + u8 *write_ptr = IO_write_bytes(out, match_length); + if (total_output <= ctx->header.window_size) { + // In this case offset might go back into the dictionary + if (offset > total_output + ctx->dict_content_len) { + // The offset goes beyond even the dictionary + CORRUPTION(); + } + + if (offset > total_output) { + // "The rest of the dictionary is its content. The content act + // as a "past" in front of data to compress or decompress, so it + // can be referenced in sequence commands." + const size_t dict_copy = + MIN(offset - total_output, match_length); + const size_t dict_offset = + ctx->dict_content_len - (offset - total_output); + + memcpy(write_ptr, ctx->dict_content + dict_offset, dict_copy); + write_ptr += dict_copy; + match_length -= dict_copy; + } + } else if (offset > ctx->header.window_size) { + CORRUPTION(); + } + + // We must copy byte by byte because the match length might be larger + // than the offset + // ex: if the output so far was "abc", a command with offset=3 and + // match_length=6 would produce "abcabcabc" as the new output + for (size_t j = 0; j < match_length; j++) { + *write_ptr = *(write_ptr - offset); + write_ptr++; + } +} /******* END SEQUENCE EXECUTION ***********************************************/ /******* OUTPUT SIZE COUNTING *************************************************/ From 6aebcfa0bca90b98ba5a4dd718c0d5e45f31a975 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 14 Aug 2017 15:11:01 -0700 Subject: [PATCH 12/14] added comments for new functions --- doc/educational_decoder/zstd_decompress.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c index cbfeaa166..f2e9a95a3 100644 --- a/doc/educational_decoder/zstd_decompress.c +++ b/doc/educational_decoder/zstd_decompress.c @@ -353,11 +353,18 @@ static void execute_sequences(frame_context_t *const ctx, ostream_t *const out, const sequence_command_t *const sequences, const size_t num_sequences); +// Copies literals and returns the total literal length that was copied static u32 copy_literals(sequence_command_t seq, istream_t *litstream, ostream_t *const out); +// Given an offset code from a sequence command (either an actual offset value +// or an index for previous offset), computes the correct offset and udpates +// the offset history static size_t compute_offset(sequence_command_t seq, u64 *const offset_hist); +// Given an offset, match length, and total output, as well as the frame +// context for the dictionary, determines if the dictionary is used and +// executes the copy operation static void execute_match_copy(frame_context_t *const ctx, size_t offset, size_t match_length, size_t total_output, ostream_t *const out); From 8d3f18af2c73523bc4751512eeb4dc88cd412b96 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Mon, 14 Aug 2017 17:51:51 -0700 Subject: [PATCH 13/14] renamed IO functions for clarity --- doc/educational_decoder/zstd_decompress.c | 44 +++++++++++------------ 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c index f2e9a95a3..0fa30b274 100644 --- a/doc/educational_decoder/zstd_decompress.c +++ b/doc/educational_decoder/zstd_decompress.c @@ -94,10 +94,10 @@ static inline size_t IO_istream_len(const istream_t *const in); /// Advances the stream by `len` bytes, and returns a pointer to the chunk that /// was skipped. The stream must be byte aligned. -static inline const u8 *IO_read_bytes(istream_t *const in, size_t len); +static inline const u8 *IO_get_read_ptr(istream_t *const in, size_t len); /// Advances the stream by `len` bytes, and returns a pointer to the chunk that /// was skipped so it can be written to. -static inline u8 *IO_write_bytes(ostream_t *const out, size_t len); +static inline u8 *IO_get_write_ptr(ostream_t *const out, size_t len); /// Advance the inner state by `len` bytes. The stream must be byte aligned. static inline void IO_advance_input(istream_t *const in, size_t len); @@ -641,8 +641,8 @@ static void decompress_data(frame_context_t *const ctx, ostream_t *const out, case 0: { // "Raw_Block - this is an uncompressed block. Block_Size is the // number of bytes to read and copy." - const u8 *const read_ptr = IO_read_bytes(in, block_len); - u8 *const write_ptr = IO_write_bytes(out, block_len); + const u8 *const read_ptr = IO_get_read_ptr(in, block_len); + u8 *const write_ptr = IO_get_write_ptr(out, block_len); // Copy the raw data into the output memcpy(write_ptr, read_ptr, block_len); @@ -654,8 +654,8 @@ static void decompress_data(frame_context_t *const ctx, ostream_t *const out, // "RLE_Block - this is a single byte, repeated N times. In which // case, Block_Size is the size to regenerate, while the // "compressed" block is just 1 byte (the byte to repeat)." - const u8 *const read_ptr = IO_read_bytes(in, 1); - u8 *const write_ptr = IO_write_bytes(out, block_len); + const u8 *const read_ptr = IO_get_read_ptr(in, 1); + u8 *const write_ptr = IO_get_write_ptr(out, block_len); // Copy `block_len` copies of `read_ptr[0]` to the output memset(write_ptr, read_ptr[0], block_len); @@ -801,13 +801,13 @@ static size_t decode_literals_simple(istream_t *const in, u8 **const literals, switch (block_type) { case 0: { // "Raw_Literals_Block - Literals are stored uncompressed." - const u8 *const read_ptr = IO_read_bytes(in, size); + const u8 *const read_ptr = IO_get_read_ptr(in, size); memcpy(*literals, read_ptr, size); break; } case 1: { // "RLE_Literals_Block - Literals consist of a single byte value repeated N times." - const u8 *const read_ptr = IO_read_bytes(in, 1); + const u8 *const read_ptr = IO_get_read_ptr(in, 1); memset(*literals, read_ptr[0], size); break; } @@ -918,7 +918,7 @@ static void decode_huf_table(HUF_dtable *const dtable, istream_t *const in) { num_symbs = header - 127; const size_t bytes = (num_symbs + 1) / 2; - const u8 *const weight_src = IO_read_bytes(in, bytes); + const u8 *const weight_src = IO_get_read_ptr(in, bytes); for (int i = 0; i < num_symbs; i++) { // "They are encoded forward, 2 @@ -1126,7 +1126,7 @@ static void decompress_sequences(frame_context_t *const ctx, istream_t *in, } const size_t len = IO_istream_len(in); - const u8 *const src = IO_read_bytes(in, len); + const u8 *const src = IO_get_read_ptr(in, len); // "After writing the last bit containing information, the compressor writes // a single 1-bit and then fills the byte with 0-7 0 bits of padding." @@ -1231,7 +1231,7 @@ static void decode_seq_table(FSE_dtable *const table, istream_t *const in, } case seq_rle: { // "RLE_Mode : it's a single code, repeated Number_of_Sequences times." - const u8 symb = IO_read_bytes(in, 1)[0]; + const u8 symb = IO_get_read_ptr(in, 1)[0]; FSE_init_dtable_rle(table, symb); break; } @@ -1288,8 +1288,8 @@ static void execute_sequences(frame_context_t *const ctx, ostream_t *const out, // Copy any leftover literals { size_t len = IO_istream_len(&litstream); - u8 *const write_ptr = IO_write_bytes(out, len); - const u8 *const read_ptr = IO_read_bytes(&litstream, len); + u8 *const write_ptr = IO_get_write_ptr(out, len); + const u8 *const read_ptr = IO_get_read_ptr(&litstream, len); memcpy(write_ptr, read_ptr, len); total_output += len; @@ -1306,9 +1306,9 @@ static u32 copy_literals(const sequence_command_t seq, istream_t *litstream, CORRUPTION(); } - u8 *const write_ptr = IO_write_bytes(out, seq.literal_length); + u8 *const write_ptr = IO_get_write_ptr(out, seq.literal_length); const u8 *const read_ptr = - IO_read_bytes(litstream, seq.literal_length); + IO_get_read_ptr(litstream, seq.literal_length); // Copy literals to output memcpy(write_ptr, read_ptr, seq.literal_length); @@ -1366,7 +1366,7 @@ static size_t compute_offset(sequence_command_t seq, u64 *const offset_hist) { static void execute_match_copy(frame_context_t *const ctx, size_t offset, size_t match_length, size_t total_output, ostream_t *const out) { - u8 *write_ptr = IO_write_bytes(out, match_length); + u8 *write_ptr = IO_get_write_ptr(out, match_length); if (total_output <= ctx->header.window_size) { // In this case offset might go back into the dictionary if (offset > total_output + ctx->dict_content_len) { @@ -1510,7 +1510,7 @@ static void init_dictionary_content(dictionary_t *const dict, BAD_ALLOC(); } - const u8 *const content = IO_read_bytes(in, dict->content_size); + const u8 *const content = IO_get_read_ptr(in, dict->content_size); memcpy(dict->content, content, dict->content_size); } @@ -1605,7 +1605,7 @@ static inline size_t IO_istream_len(const istream_t *const in) { /// Returns a pointer where `len` bytes can be read, and advances the internal /// state. The stream must be byte aligned. -static inline const u8 *IO_read_bytes(istream_t *const in, size_t len) { +static inline const u8 *IO_get_read_ptr(istream_t *const in, size_t len) { if (len > in->len) { INP_SIZE(); } @@ -1619,7 +1619,7 @@ static inline const u8 *IO_read_bytes(istream_t *const in, size_t len) { return ptr; } /// Returns a pointer to write `len` bytes to, and advances the internal state -static inline u8 *IO_write_bytes(ostream_t *const out, size_t len) { +static inline u8 *IO_get_write_ptr(ostream_t *const out, size_t len) { if (len > out->len) { OUT_SIZE(); } @@ -1658,7 +1658,7 @@ static inline istream_t IO_make_istream(const u8 *in, size_t len) { /// `in` must be byte aligned static inline istream_t IO_make_sub_istream(istream_t *const in, size_t len) { // Consume `len` bytes of the parent stream - const u8 *const ptr = IO_read_bytes(in, len); + const u8 *const ptr = IO_get_read_ptr(in, len); // Make a substream using the pointer to those `len` bytes return IO_make_istream(ptr, len); @@ -1762,7 +1762,7 @@ static size_t HUF_decompress_1stream(const HUF_dtable *const dtable, if (len == 0) { INP_SIZE(); } - const u8 *const src = IO_read_bytes(in, len); + const u8 *const src = IO_get_read_ptr(in, len); // "Each bitstream must be read backward, that is starting from the end down // to the beginning. Therefore it's necessary to know the size of each @@ -2013,7 +2013,7 @@ static size_t FSE_decompress_interleaved2(const FSE_dtable *const dtable, if (len == 0) { INP_SIZE(); } - const u8 *const src = IO_read_bytes(in, len); + const u8 *const src = IO_get_read_ptr(in, len); // "Each bitstream must be read backward, that is starting from the end down // to the beginning. Therefore it's necessary to know the size of each From bef5eda8d9d9179af6165eaff512bc843cc96fc2 Mon Sep 17 00:00:00 2001 From: Paul Cruz Date: Wed, 16 Aug 2017 11:11:52 -0700 Subject: [PATCH 14/14] const vars, change copy_literals() to only take size_t literal_length --- doc/educational_decoder/harness.c | 2 +- doc/educational_decoder/zstd_decompress.c | 28 +++++++++++------------ 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/doc/educational_decoder/harness.c b/doc/educational_decoder/harness.c index 43d6df7fc..982e066e2 100644 --- a/doc/educational_decoder/harness.c +++ b/doc/educational_decoder/harness.c @@ -106,7 +106,7 @@ int main(int argc, char **argv) { return 1; } - dictionary_t* parsed_dict = create_dictionary(); + dictionary_t* const parsed_dict = create_dictionary(); if (dict) { parse_dictionary(parsed_dict, dict, dict_size); } diff --git a/doc/educational_decoder/zstd_decompress.c b/doc/educational_decoder/zstd_decompress.c index 0fa30b274..af10db528 100644 --- a/doc/educational_decoder/zstd_decompress.c +++ b/doc/educational_decoder/zstd_decompress.c @@ -354,7 +354,7 @@ static void execute_sequences(frame_context_t *const ctx, ostream_t *const out, const size_t num_sequences); // Copies literals and returns the total literal length that was copied -static u32 copy_literals(sequence_command_t seq, istream_t *litstream, +static u32 copy_literals(const size_t seq, istream_t *litstream, ostream_t *const out); // Given an offset code from a sequence command (either an actual offset value @@ -1273,12 +1273,13 @@ static void execute_sequences(frame_context_t *const ctx, ostream_t *const out, for (size_t i = 0; i < num_sequences; i++) { const sequence_command_t seq = sequences[i]; { - const u32 literals_size = copy_literals(seq, &litstream, out); + const u32 literals_size = copy_literals(seq.literal_length, &litstream, out); total_output += literals_size; } - size_t offset = compute_offset(seq, offset_hist); - size_t match_length = seq.match_length; + size_t const offset = compute_offset(seq, offset_hist); + + size_t const match_length = seq.match_length; execute_match_copy(ctx, offset, match_length, total_output, out); @@ -1288,31 +1289,28 @@ static void execute_sequences(frame_context_t *const ctx, ostream_t *const out, // Copy any leftover literals { size_t len = IO_istream_len(&litstream); - u8 *const write_ptr = IO_get_write_ptr(out, len); - const u8 *const read_ptr = IO_get_read_ptr(&litstream, len); - memcpy(write_ptr, read_ptr, len); - + copy_literals(len, &litstream, out); total_output += len; } ctx->current_total_output = total_output; } -static u32 copy_literals(const sequence_command_t seq, istream_t *litstream, +static u32 copy_literals(const size_t literal_length, istream_t *litstream, ostream_t *const out) { // If the sequence asks for more literals than are left, the // sequence must be corrupted - if (seq.literal_length > IO_istream_len(litstream)) { - CORRUPTION(); + if (literal_length > IO_istream_len(litstream)) { + CORRUPTION(); } - u8 *const write_ptr = IO_get_write_ptr(out, seq.literal_length); + u8 *const write_ptr = IO_get_write_ptr(out, literal_length); const u8 *const read_ptr = - IO_get_read_ptr(litstream, seq.literal_length); + IO_get_read_ptr(litstream, literal_length); // Copy literals to output - memcpy(write_ptr, read_ptr, seq.literal_length); + memcpy(write_ptr, read_ptr, literal_length); - return seq.literal_length; + return literal_length; } static size_t compute_offset(sequence_command_t seq, u64 *const offset_hist) {