Merge pull request #789 from paulcruz74/educational-decoder-2

Educational decoder 2

doc/educational_decoder/Makefile

@@ -0,0 +1,34 @@
+HARNESS_FILES=*.c
+
+MULTITHREAD_LDFLAGS = -pthread
+DEBUGFLAGS= -g -DZSTD_DEBUG=1
+CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \
+            -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR)
+CFLAGS   ?= -O3
+CFLAGS   += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow                 \
+            -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
+            -Wstrict-prototypes -Wundef -Wformat-security                   \
+            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings      \
+            -Wredundant-decls
+CFLAGS   += $(DEBUGFLAGS)
+CFLAGS   += $(MOREFLAGS)
+FLAGS     = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MULTITHREAD_LDFLAGS)
+
+harness: $(HARNESS_FILES)
+	$(CC) $(FLAGS) $^ -o $@
+
+clean:
+	@$(RM) -f harness
+	@$(RM) -rf harness.dSYM
+
+test: harness
+	@zstd README.md -o tmp.zst
+	@./harness tmp.zst tmp
+	@diff -s tmp README.md
+	@$(RM) -f tmp*
+	@zstd --train harness.c zstd_decompress.c zstd_decompress.h README.md
+	@zstd -D dictionary README.md -o tmp.zst
+	@./harness tmp.zst tmp dictionary
+	@diff -s tmp README.md
+	@$(RM) -f tmp* dictionary
+	@make clean

doc/educational_decoder/harness.c

@@ -87,7 +87,7 @@ int main(int argc, char **argv) {
     }
 
     size_t decompressed_size = ZSTD_get_decompressed_size(input, input_size);
-    if (decompressed_size == -1) {
+    if (decompressed_size == (size_t)-1) {
         decompressed_size = MAX_COMPRESSION_RATIO * input_size;
         fprintf(stderr, "WARNING: Compressed data does not contain "
                         "decompressed size, going to assume the compression "
@@ -106,9 +106,15 @@ int main(int argc, char **argv) {
         return 1;
     }
 
+    dictionary_t* const parsed_dict = create_dictionary();
+    if (dict) {
+        parse_dictionary(parsed_dict, dict, dict_size);
+    }
     size_t decompressed =
         ZSTD_decompress_with_dict(output, decompressed_size,
-                                  input, input_size, dict, dict_size);
+                                  input, input_size, parsed_dict);
+
+    free_dictionary(parsed_dict);
 
     write_file(argv[2], output, decompressed);
 
@@ -117,4 +123,3 @@ int main(int argc, char **argv) {
     free(dict);
     input = output = dict = NULL;
 }
-

doc/educational_decoder/zstd_decompress.c

@@ -14,21 +14,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-
+#include "zstd_decompress.h"
-/// Zstandard decompression functions.
-/// `dst` must point to a space at least as large as the reconstructed output.
-size_t ZSTD_decompress(void *const dst, const size_t dst_len,
-                       const void *const src, const size_t src_len);
-/// If `dict != NULL` and `dict_len >= 8`, does the same thing as
-/// `ZSTD_decompress` but uses the provided dict
-size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len,
-                                 const void *const src, const size_t src_len,
-                                 const void *const dict, const size_t dict_len);
-
-/// Get the decompressed size of an input stream so memory can be allocated in
-/// advance
-/// Returns -1 if the size can't be determined
-size_t ZSTD_get_decompressed_size(const void *const src, const size_t src_len);
 
 /******* UTILITY MACROS AND TYPES *********************************************/
 // Max block size decompressed size is 128 KB and literal blocks can't be
@@ -108,10 +94,10 @@ static inline size_t IO_istream_len(const istream_t *const in);
 
 /// Advances the stream by `len` bytes, and returns a pointer to the chunk that
 /// was skipped.  The stream must be byte aligned.
-static inline const u8 *IO_read_bytes(istream_t *const in, size_t len);
+static inline const u8 *IO_get_read_ptr(istream_t *const in, size_t len);
 /// Advances the stream by `len` bytes, and returns a pointer to the chunk that
 /// was skipped so it can be written to.
-static inline u8 *IO_write_bytes(ostream_t *const out, size_t len);
+static inline u8 *IO_get_write_ptr(ostream_t *const out, size_t len);
 
 /// Advance the inner state by `len` bytes.  The stream must be byte aligned.
 static inline void IO_advance_input(istream_t *const in, size_t len);
@@ -307,7 +293,7 @@ typedef struct {
 
 /// The decoded contents of a dictionary so that it doesn't have to be repeated
 /// for each frame that uses it
-typedef struct {
+struct dictionary_s {
     // Entropy tables
     HUF_dtable literals_dtable;
     FSE_dtable ll_dtable;
@@ -322,7 +308,7 @@ typedef struct {
     u64 previous_offsets[3];
 
     u32 dictionary_id;
-} dictionary_t;
+};
 
 /// A tuple containing the parts necessary to decode and execute a ZSTD sequence
 /// command
@@ -367,27 +353,36 @@ static void execute_sequences(frame_context_t *const ctx, ostream_t *const out,
                               const sequence_command_t *const sequences,
                               const size_t num_sequences);
 
-// Parse a provided dictionary blob for use in decompression
+// Copies literals and returns the total literal length that was copied
-static void parse_dictionary(dictionary_t *const dict, const u8 *src,
+static u32 copy_literals(const size_t seq, istream_t *litstream,
-                             size_t src_len);
+                         ostream_t *const out);
-static void free_dictionary(dictionary_t *const dict);
+
+// Given an offset code from a sequence command (either an actual offset value
+// or an index for previous offset), computes the correct offset and udpates
+// the offset history
+static size_t compute_offset(sequence_command_t seq, u64 *const offset_hist);
+
+// Given an offset, match length, and total output, as well as the frame
+// context for the dictionary, determines if the dictionary is used and
+// executes the copy operation
+static void execute_match_copy(frame_context_t *const ctx, size_t offset,
+                              size_t match_length, size_t total_output,
+                              ostream_t *const out);
+
 /******* END ZSTD HELPER STRUCTS AND PROTOTYPES *******************************/
 
 size_t ZSTD_decompress(void *const dst, const size_t dst_len,
                        const void *const src, const size_t src_len) {
-    return ZSTD_decompress_with_dict(dst, dst_len, src, src_len, NULL, 0);
+    dictionary_t* uninit_dict = create_dictionary();
+    size_t const decomp_size = ZSTD_decompress_with_dict(dst, dst_len, src,
+                                                         src_len, uninit_dict);
+    free_dictionary(uninit_dict);
+    return decomp_size;
 }
 
 size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len,
                                  const void *const src, const size_t src_len,
-                                 const void *const dict,
+                                 dictionary_t* parsed_dict) {
-                                 const size_t dict_len) {
-    dictionary_t parsed_dict;
-    memset(&parsed_dict, 0, sizeof(dictionary_t));
-    // dict_len < 8 is not a valid dictionary
-    if (dict && dict_len > 8) {
-        parse_dictionary(&parsed_dict, (const u8 *)dict, dict_len);
-    }
 
     istream_t in = IO_make_istream(src, src_len);
     ostream_t out = IO_make_ostream(dst, dst_len);
@@ -396,11 +391,9 @@ size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len,
     // Multiple frames can be appended into a single file or stream. A frame is
     // totally independent, has a defined beginning and end, and a set of
     // parameters which tells the decoder how to decompress it."
-    while (IO_istream_len(&in) > 0) {
-        decode_frame(&out, &in, &parsed_dict);
-    }
 
-    free_dictionary(&parsed_dict);
+    /* this decoder assumes decompression of a single frame */
+    decode_frame(&out, &in, parsed_dict);
 
     return out.ptr - (u8 *)dst;
 }
@@ -424,30 +417,6 @@ static void decompress_data(frame_context_t *const ctx, ostream_t *const out,
 static void decode_frame(ostream_t *const out, istream_t *const in,
                          const dictionary_t *const dict) {
     const u32 magic_number = IO_read_bits(in, 32);
-
-    // Skippable frame
-    //
-    // "Magic_Number
-    //
-    // 4 Bytes, little-endian format. Value : 0x184D2A5?, which means any value
-    // from 0x184D2A50 to 0x184D2A5F. All 16 values are valid to identify a
-    // skippable frame."
-    if ((magic_number & ~0xFU) == 0x184D2A50U) {
-        // "Skippable frames allow the insertion of user-defined data into a
-        // flow of concatenated frames. Its design is pretty straightforward,
-        // with the sole objective to allow the decoder to quickly skip over
-        // user-defined data and continue decoding.
-        //
-        // Skippable frames defined in this specification are compatible with
-        // LZ4 ones."
-        const size_t frame_size = IO_read_bits(in, 32);
-
-        // skip over frame
-        IO_advance_input(in, frame_size);
-
-        return;
-    }
-
     // Zstandard frame
     //
     // "Magic_Number
@@ -460,8 +429,8 @@ static void decode_frame(ostream_t *const out, istream_t *const in,
         return;
     }
 
-    // not a real frame
+    // not a real frame or a skippable frame
-    ERROR("Invalid magic number");
+    ERROR("Tried to decode non-ZSTD frame");
 }
 
 /// Decode a frame that contains compressed data.  Not all frames do as there
@@ -672,8 +641,8 @@ static void decompress_data(frame_context_t *const ctx, ostream_t *const out,
         case 0: {
             // "Raw_Block - this is an uncompressed block. Block_Size is the
             // number of bytes to read and copy."
-            const u8 *const read_ptr = IO_read_bytes(in, block_len);
+            const u8 *const read_ptr = IO_get_read_ptr(in, block_len);
-            u8 *const write_ptr = IO_write_bytes(out, block_len);
+            u8 *const write_ptr = IO_get_write_ptr(out, block_len);
 
             // Copy the raw data into the output
             memcpy(write_ptr, read_ptr, block_len);
@@ -685,8 +654,8 @@ static void decompress_data(frame_context_t *const ctx, ostream_t *const out,
             // "RLE_Block - this is a single byte, repeated N times. In which
             // case, Block_Size is the size to regenerate, while the
             // "compressed" block is just 1 byte (the byte to repeat)."
-            const u8 *const read_ptr = IO_read_bytes(in, 1);
+            const u8 *const read_ptr = IO_get_read_ptr(in, 1);
-            u8 *const write_ptr = IO_write_bytes(out, block_len);
+            u8 *const write_ptr = IO_get_write_ptr(out, block_len);
 
             // Copy `block_len` copies of `read_ptr[0]` to the output
             memset(write_ptr, read_ptr[0], block_len);
@@ -832,13 +801,13 @@ static size_t decode_literals_simple(istream_t *const in, u8 **const literals,
     switch (block_type) {
     case 0: {
         // "Raw_Literals_Block - Literals are stored uncompressed."
-        const u8 *const read_ptr = IO_read_bytes(in, size);
+        const u8 *const read_ptr = IO_get_read_ptr(in, size);
         memcpy(*literals, read_ptr, size);
         break;
     }
     case 1: {
         // "RLE_Literals_Block - Literals consist of a single byte value repeated N times."
-        const u8 *const read_ptr = IO_read_bytes(in, 1);
+        const u8 *const read_ptr = IO_get_read_ptr(in, 1);
         memset(*literals, read_ptr[0], size);
         break;
     }
@@ -949,7 +918,7 @@ static void decode_huf_table(HUF_dtable *const dtable, istream_t *const in) {
         num_symbs = header - 127;
         const size_t bytes = (num_symbs + 1) / 2;
 
-        const u8 *const weight_src = IO_read_bytes(in, bytes);
+        const u8 *const weight_src = IO_get_read_ptr(in, bytes);
 
         for (int i = 0; i < num_symbs; i++) {
             // "They are encoded forward, 2
@@ -1157,7 +1126,7 @@ static void decompress_sequences(frame_context_t *const ctx, istream_t *in,
     }
 
     const size_t len = IO_istream_len(in);
-    const u8 *const src = IO_read_bytes(in, len);
+    const u8 *const src = IO_get_read_ptr(in, len);
 
     // "After writing the last bit containing information, the compressor writes
     // a single 1-bit and then fills the byte with 0-7 0 bits of padding."
@@ -1262,7 +1231,7 @@ static void decode_seq_table(FSE_dtable *const table, istream_t *const in,
     }
     case seq_rle: {
         // "RLE_Mode : it's a single code, repeated Number_of_Sequences times."
-        const u8 symb = IO_read_bytes(in, 1)[0];
+        const u8 symb = IO_get_read_ptr(in, 1)[0];
         FSE_init_dtable_rle(table, symb);
         break;
     }
@@ -1303,25 +1272,49 @@ static void execute_sequences(frame_context_t *const ctx, ostream_t *const out,
 
     for (size_t i = 0; i < num_sequences; i++) {
         const sequence_command_t seq = sequences[i];
-
         {
+            const u32 literals_size = copy_literals(seq.literal_length, &litstream, out);
+            total_output += literals_size;
+        }
+
+        size_t const offset = compute_offset(seq, offset_hist);
+
+        size_t const match_length = seq.match_length;
+
+        execute_match_copy(ctx, offset, match_length, total_output, out);
+
+        total_output += match_length;
+    }
+
+    // Copy any leftover literals
+    {
+        size_t len = IO_istream_len(&litstream);
+        copy_literals(len, &litstream, out); 
+        total_output += len;
+    }
+
+    ctx->current_total_output = total_output;
+}
+
+static u32 copy_literals(const size_t literal_length, istream_t *litstream,
+                         ostream_t *const out) {
     // If the sequence asks for more literals than are left, the
     // sequence must be corrupted
-            if (seq.literal_length > IO_istream_len(&litstream)) {
+    if (literal_length > IO_istream_len(litstream)) {
         CORRUPTION();
     }
 
-            u8 *const write_ptr = IO_write_bytes(out, seq.literal_length);
+    u8 *const write_ptr = IO_get_write_ptr(out, literal_length);
     const u8 *const read_ptr =
-                    IO_read_bytes(&litstream, seq.literal_length);
+         IO_get_read_ptr(litstream, literal_length);
     // Copy literals to output
-            memcpy(write_ptr, read_ptr, seq.literal_length);
+    memcpy(write_ptr, read_ptr, literal_length);
 
-            total_output += seq.literal_length;
+    return literal_length;
 }
 
+static size_t compute_offset(sequence_command_t seq, u64 *const offset_hist) {
     size_t offset;
-
     // Offsets are special, we need to handle the repeat offsets
     if (seq.offset <= 3) {
         // "The first 3 values define a repeated offset and we will call
@@ -1365,10 +1358,13 @@ static void execute_sequences(frame_context_t *const ctx, ostream_t *const out,
         offset_hist[1] = offset_hist[0];
         offset_hist[0] = offset;
     }
+    return offset;
+}
 
-        size_t match_length = seq.match_length;
+static void execute_match_copy(frame_context_t *const ctx, size_t offset,
-
+                              size_t match_length, size_t total_output,
-        u8 *write_ptr = IO_write_bytes(out, match_length);
+                              ostream_t *const out) {
+    u8 *write_ptr = IO_get_write_ptr(out, match_length);
     if (total_output <= ctx->header.window_size) {
         // In this case offset might go back into the dictionary
         if (offset > total_output + ctx->dict_content_len) {
@@ -1397,51 +1393,25 @@ static void execute_sequences(frame_context_t *const ctx, ostream_t *const out,
     // than the offset
     // ex: if the output so far was "abc", a command with offset=3 and
     // match_length=6 would produce "abcabcabc" as the new output
-        for (size_t i = 0; i < match_length; i++) {
+    for (size_t j = 0; j < match_length; j++) {
         *write_ptr = *(write_ptr - offset);
         write_ptr++;
     }
-
-        total_output += seq.match_length;
-    }
-
-    // Copy any leftover literals
-    {
-        size_t len = IO_istream_len(&litstream);
-        u8 *const write_ptr = IO_write_bytes(out, len);
-        const u8 *const read_ptr = IO_read_bytes(&litstream, len);
-        memcpy(write_ptr, read_ptr, len);
-
-        total_output += len;
-    }
-
-    ctx->current_total_output = total_output;
 }
 /******* END SEQUENCE EXECUTION ***********************************************/
 
 /******* OUTPUT SIZE COUNTING *************************************************/
-static void traverse_frame(const frame_header_t *const header, istream_t *const in);
-
 /// Get the decompressed size of an input stream so memory can be allocated in
 /// advance.
-/// This is more complex than the implementation in the reference
+/// This implementation assumes `src` points to a single ZSTD-compressed frame
-/// implementation, as this API allows for the decompression of multiple
-/// concatenated frames.
 size_t ZSTD_get_decompressed_size(const void *src, const size_t src_len) {
     istream_t in = IO_make_istream(src, src_len);
-    size_t dst_size = 0;
 
-    // Each frame header only gives us the size of its frame, so iterate over
+    // get decompressed size from ZSTD frame header
-    // all
+    {
-    // frames
-    while (IO_istream_len(&in) > 0) {
         const u32 magic_number = IO_read_bits(&in, 32);
 
-        if ((magic_number & ~0xFU) == 0x184D2A50U) {
+        if (magic_number == 0xFD2FB528U) {
-            // skippable frame, this has no impact on output size
-            const size_t frame_size = IO_read_bits(&in, 32);
-            IO_advance_input(&in, frame_size);
-        } else if (magic_number == 0xFD2FB528U) {
             // ZSTD frame
             frame_header_t header;
             parse_frame_header(&header, &in);
@@ -1451,68 +1421,42 @@ size_t ZSTD_get_decompressed_size(const void *src, const size_t src_len) {
                 return -1;
             }
 
-            dst_size += header.frame_content_size;
+            return header.frame_content_size;
-
-            // Consume the input from the frame to reach the start of the next
-            traverse_frame(&header, &in);
         } else {
-            // not a real frame
+            // not a real frame or skippable frame
-            ERROR("Invalid magic number");
+            ERROR("ZSTD frame magic number did not match");
         }
     }
-
-    return dst_size;
 }
-
-/// Iterate over each block in a frame to find the end of it, to get to the
-/// start of the next frame
-static void traverse_frame(const frame_header_t *const header, istream_t *const in) {
-    int last_block = 0;
-
-    do {
-        // Parse the block header
-        last_block = IO_read_bits(in, 1);
-        const int block_type = IO_read_bits(in, 2);
-        const size_t block_len = IO_read_bits(in, 21);
-
-        switch (block_type) {
-        case 0: // Raw block, block_len bytes
-            IO_advance_input(in, block_len);
-            break;
-        case 1: // RLE block, 1 byte
-            IO_advance_input(in, 1);
-            break;
-        case 2: // Compressed block, compressed size is block_len
-            IO_advance_input(in, block_len);
-            break;
-        case 3:
-            // Reserved block type
-            CORRUPTION();
-            break;
-        default:
-            IMPOSSIBLE();
-        }
-    } while (!last_block);
-
-    if (header->content_checksum_flag) {
-        IO_advance_input(in, 4);
-    }
-}
-
 /******* END OUTPUT SIZE COUNTING *********************************************/
 
 /******* DICTIONARY PARSING ***************************************************/
+#define DICT_SIZE_ERROR() ERROR("Dictionary size cannot be less than 8 bytes")
+#define NULL_SRC() ERROR("Tried to create dictionary with pointer to null src");
+
+dictionary_t* create_dictionary() {
+    dictionary_t* dict = calloc(1, sizeof(dictionary_t));
+    if (!dict) {
+        BAD_ALLOC();
+    }
+    return dict;
+}
+
 static void init_dictionary_content(dictionary_t *const dict,
                                     istream_t *const in);
 
-static void parse_dictionary(dictionary_t *const dict, const u8 *src,
+void parse_dictionary(dictionary_t *const dict, const void *src,
                              size_t src_len) {
+    const u8 *byte_src = (const u8 *)src;
     memset(dict, 0, sizeof(dictionary_t));
+    if (src == NULL) { /* cannot initialize dictionary with null src */
+        NULL_SRC();
+    }
     if (src_len < 8) {
-        INP_SIZE();
+        DICT_SIZE_ERROR();
     }
 
-    istream_t in = IO_make_istream(src, src_len);
+    istream_t in = IO_make_istream(byte_src, src_len);
 
     const u32 magic_number = IO_read_bits(&in, 32);
     if (magic_number != 0xEC30A437) {
@@ -1564,13 +1508,13 @@ static void init_dictionary_content(dictionary_t *const dict,
         BAD_ALLOC();
     }
 
-    const u8 *const content = IO_read_bytes(in, dict->content_size);
+    const u8 *const content = IO_get_read_ptr(in, dict->content_size);
 
     memcpy(dict->content, content, dict->content_size);
 }
 
 /// Free an allocated dictionary
-static void free_dictionary(dictionary_t *const dict) {
+void free_dictionary(dictionary_t *const dict) {
     HUF_free_dtable(&dict->literals_dtable);
     FSE_free_dtable(&dict->ll_dtable);
     FSE_free_dtable(&dict->of_dtable);
@@ -1579,6 +1523,8 @@ static void free_dictionary(dictionary_t *const dict) {
     free(dict->content);
 
     memset(dict, 0, sizeof(dictionary_t));
+
+    free(dict);
 }
 /******* END DICTIONARY PARSING ***********************************************/
 
@@ -1657,7 +1603,7 @@ static inline size_t IO_istream_len(const istream_t *const in) {
 
 /// Returns a pointer where `len` bytes can be read, and advances the internal
 /// state.  The stream must be byte aligned.
-static inline const u8 *IO_read_bytes(istream_t *const in, size_t len) {
+static inline const u8 *IO_get_read_ptr(istream_t *const in, size_t len) {
     if (len > in->len) {
         INP_SIZE();
     }
@@ -1671,7 +1617,7 @@ static inline const u8 *IO_read_bytes(istream_t *const in, size_t len) {
     return ptr;
 }
 /// Returns a pointer to write `len` bytes to, and advances the internal state
-static inline u8 *IO_write_bytes(ostream_t *const out, size_t len) {
+static inline u8 *IO_get_write_ptr(ostream_t *const out, size_t len) {
     if (len > out->len) {
         OUT_SIZE();
     }
@@ -1710,7 +1656,7 @@ static inline istream_t IO_make_istream(const u8 *in, size_t len) {
 /// `in` must be byte aligned
 static inline istream_t IO_make_sub_istream(istream_t *const in, size_t len) {
     // Consume `len` bytes of the parent stream
-    const u8 *const ptr = IO_read_bytes(in, len);
+    const u8 *const ptr = IO_get_read_ptr(in, len);
 
     // Make a substream using the pointer to those `len` bytes
     return IO_make_istream(ptr, len);
@@ -1814,7 +1760,7 @@ static size_t HUF_decompress_1stream(const HUF_dtable *const dtable,
     if (len == 0) {
         INP_SIZE();
     }
-    const u8 *const src = IO_read_bytes(in, len);
+    const u8 *const src = IO_get_read_ptr(in, len);
 
     // "Each bitstream must be read backward, that is starting from the end down
     // to the beginning. Therefore it's necessary to know the size of each
@@ -2065,7 +2011,7 @@ static size_t FSE_decompress_interleaved2(const FSE_dtable *const dtable,
     if (len == 0) {
         INP_SIZE();
     }
-    const u8 *const src = IO_read_bytes(in, len);
+    const u8 *const src = IO_get_read_ptr(in, len);
 
     // "Each bitstream must be read backward, that is starting from the end down
     // to the beginning. Therefore it's necessary to know the size of each
@@ -2192,7 +2138,7 @@ static void FSE_init_dtable(FSE_dtable *const dtable,
     }
 
     // Now we can fill baseline and num bits
-    for (int i = 0; i < size; i++) {
+    for (size_t i = 0; i < size; i++) {
         u8 symbol = dtable->symbols[i];
         u16 next_state_desc = state_desc[symbol]++;
         // Fills in the table appropriately, next_state_desc increases by symbol
@@ -2355,4 +2301,3 @@ static void FSE_copy_dtable(FSE_dtable *const dst, const FSE_dtable *const src)
     memcpy(dst->new_state_base, src->new_state_base, size * sizeof(u16));
 }
 /******* END FSE PRIMITIVES ***************************************************/
-

doc/educational_decoder/zstd_decompress.h

@@ -7,10 +7,52 @@
  * of patent rights can be found in the PATENTS file in the same directory.
  */
 
+/******* EXPOSED TYPES ********************************************************/
+/*
+* Contains the parsed contents of a dictionary
+* This includes Huffman and FSE tables used for decoding and data on offsets
+*/
+typedef struct dictionary_s dictionary_t;
+/******* END EXPOSED TYPES ****************************************************/
+
+/******* DECOMPRESSION FUNCTIONS **********************************************/
+/// Zstandard decompression functions.
+/// `dst` must point to a space at least as large as the reconstructed output.
 size_t ZSTD_decompress(void *const dst, const size_t dst_len,
                     const void *const src, const size_t src_len);
+
+/// If `dict != NULL` and `dict_len >= 8`, does the same thing as
+/// `ZSTD_decompress` but uses the provided dict
 size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len,
                               const void *const src, const size_t src_len,
-                                 const void *const dict, const size_t dict_len);
+                              dictionary_t* parsed_dict);
-size_t ZSTD_get_decompressed_size(const void *const src, const size_t src_len);
 
+/// Get the decompressed size of an input stream so memory can be allocated in
+/// advance
+/// Returns -1 if the size can't be determined
+/// Assumes decompression of a single frame
+size_t ZSTD_get_decompressed_size(const void *const src, const size_t src_len);
+/******* END DECOMPRESSION FUNCTIONS ******************************************/
+
+/******* DICTIONARY MANAGEMENT ***********************************************/
+/*
+ * Return a valid dictionary_t pointer for use with dictionary initialization
+ * or decompression
+ */
+dictionary_t* create_dictionary();
+
+/*
+ * Parse a provided dictionary blob for use in decompression
+ * `src` -- must point to memory space representing the dictionary
+ * `src_len` -- must provide the dictionary size
+ * `dict` -- will contain the parsed contents of the dictionary and
+ *        can be used for decompression
+ */
+void parse_dictionary(dictionary_t *const dict, const void *src,
+                             size_t src_len);
+
+/*
+ * Free internal Huffman tables, FSE tables, and dictionary content
+ */
+void free_dictionary(dictionary_t *const dict);
+/******* END DICTIONARY MANAGEMENT *******************************************/