ddidderr/zstd-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix eductional decoder

Browse Source 
fix #1774
also :
- fix minor compilation warnings
- make sure the `test` is run during CI tests
This commit is contained in:
Yann Collet
2019-09-06 14:30:13 -07:00
parent b8327ac4c6
commit 2b0a271ed2
5 changed files with 24 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+1
View File
@@ -69,6 +69,7 @@ test: MOREFLAGS += -g -DDEBUGLEVEL=$(DEBUGLEVEL) -Werror
test: test:
MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants MOREFLAGS="$(MOREFLAGS)" $(MAKE) -j -C $(PRGDIR) allVariants
$(MAKE) -C $(TESTDIR) $@ $(MAKE) -C $(TESTDIR) $@
$(MAKE) -C doc/educational_decoder test
## shortest: same as `make check` ## shortest: same as `make check`
.PHONY: shortest .PHONY: shortest
doc/educational_decoder/Makefile
+4 -1
View File
@@ -26,7 +26,10 @@ test: harness
@./harness tmp.zst tmp @./harness tmp.zst tmp
@diff -s tmp README.md @diff -s tmp README.md
@$(RM) -f tmp* @$(RM) -f tmp*
@zstd --train harness.c zstd_decompress.c zstd_decompress.h README.md # present files for training multiple times, to reach minimum threshold
@zstd --train harness.c zstd_decompress.c zstd_decompress.h README.md \
harness.c zstd_decompress.c zstd_decompress.h README.md \
harness.c zstd_decompress.c zstd_decompress.h README.md
@zstd -D dictionary README.md -o tmp.zst @zstd -D dictionary README.md -o tmp.zst
@./harness tmp.zst tmp dictionary @./harness tmp.zst tmp dictionary
@diff -s tmp README.md @diff -s tmp README.md
doc/educational_decoder/zstd_decompress.c
+14 -14
View File
@@ -395,7 +395,7 @@ size_t ZSTD_decompress_with_dict(void *const dst, const size_t dst_len,
/* this decoder assumes decompression of a single frame */ /* this decoder assumes decompression of a single frame */
decode_frame(&out, &in, parsed_dict); decode_frame(&out, &in, parsed_dict);
return out.ptr - (u8 *)dst; return (size_t)(out.ptr - (u8 *)dst);
} }
/******* FRAME DECODING ******************************************************/ /******* FRAME DECODING ******************************************************/
@@ -416,7 +416,7 @@ static void decompress_data(frame_context_t *const ctx, ostream_t *const out,
static void decode_frame(ostream_t *const out, istream_t *const in, static void decode_frame(ostream_t *const out, istream_t *const in,
const dictionary_t *const dict) { const dictionary_t *const dict) {
const u32 magic_number = IO_read_bits(in, 32); const u32 magic_number = (u32)IO_read_bits(in, 32);
// Zstandard frame // Zstandard frame
// //
// "Magic_Number // "Magic_Number
@@ -497,7 +497,7 @@ static void parse_frame_header(frame_header_t *const header,
// 3 Reserved_bit // 3 Reserved_bit
// 2 Content_Checksum_flag // 2 Content_Checksum_flag
// 1-0 Dictionary_ID_flag" // 1-0 Dictionary_ID_flag"
const u8 descriptor = IO_read_bits(in, 8); const u8 descriptor = (u8)IO_read_bits(in, 8);
// decode frame header descriptor into flags // decode frame header descriptor into flags
const u8 frame_content_size_flag = descriptor >> 6; const u8 frame_content_size_flag = descriptor >> 6;
@@ -521,7 +521,7 @@ static void parse_frame_header(frame_header_t *const header,
// //
// Bit numbers 7-3 2-0 // Bit numbers 7-3 2-0
// Field name Exponent Mantissa" // Field name Exponent Mantissa"
u8 window_descriptor = IO_read_bits(in, 8); u8 window_descriptor = (u8)IO_read_bits(in, 8);
u8 exponent = window_descriptor >> 3; u8 exponent = window_descriptor >> 3;
u8 mantissa = window_descriptor & 7; u8 mantissa = window_descriptor & 7;
@@ -541,7 +541,7 @@ static void parse_frame_header(frame_header_t *const header,
const int bytes_array[] = {0, 1, 2, 4}; const int bytes_array[] = {0, 1, 2, 4};
const int bytes = bytes_array[dictionary_id_flag]; const int bytes = bytes_array[dictionary_id_flag];
header->dictionary_id = IO_read_bits(in, bytes * 8); header->dictionary_id = (u32)IO_read_bits(in, bytes * 8);
} else { } else {
header->dictionary_id = 0; header->dictionary_id = 0;
} }
@@ -633,8 +633,8 @@ static void decompress_data(frame_context_t *const ctx, ostream_t *const out,
// //
// The next 2 bits represent the Block_Type, while the remaining 21 bits // The next 2 bits represent the Block_Type, while the remaining 21 bits
// represent the Block_Size. Format is little-endian." // represent the Block_Size. Format is little-endian."
last_block = IO_read_bits(in, 1); last_block = (int)IO_read_bits(in, 1);
const int block_type = IO_read_bits(in, 2); const int block_type = (int)IO_read_bits(in, 2);
const size_t block_len = IO_read_bits(in, 21); const size_t block_len = IO_read_bits(in, 21);
switch (block_type) { switch (block_type) {
@@ -748,8 +748,8 @@ static size_t decode_literals(frame_context_t *const ctx, istream_t *const in,
// types" // types"
// //
// size_format takes between 1 and 2 bits // size_format takes between 1 and 2 bits
int block_type = IO_read_bits(in, 2); int block_type = (int)IO_read_bits(in, 2);
int size_format = IO_read_bits(in, 2); int size_format = (int)IO_read_bits(in, 2);
if (block_type <= 1) { if (block_type <= 1) {
// Raw or RLE literals block // Raw or RLE literals block
@@ -1005,7 +1005,7 @@ static const i16 SEQ_MATCH_LENGTH_DEFAULT_DIST[53] = {
static const u32 SEQ_LITERAL_LENGTH_BASELINES[36] = { static const u32 SEQ_LITERAL_LENGTH_BASELINES[36] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
12, 13, 14, 15, 16, 18, 20, 22, 24, 28, 32, 40, 12, 13, 14, 15, 16, 18, 20, 22, 24, 28, 32, 40,
48, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65538}; 48, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536};
static const u8 SEQ_LITERAL_LENGTH_EXTRA_BITS[36] = { static const u8 SEQ_LITERAL_LENGTH_EXTRA_BITS[36] = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1,
1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
@@ -1021,7 +1021,7 @@ static const u8 SEQ_MATCH_LENGTH_EXTRA_BITS[53] = {
2, 2, 3, 3, 4, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; 2, 2, 3, 3, 4, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
/// Offset decoding is simpler so we just need a maximum code value /// Offset decoding is simpler so we just need a maximum code value
static const u8 SEQ_MAX_CODES[3] = {35, -1, 52}; static const u8 SEQ_MAX_CODES[3] = {35, (u8)-1, 52};
static void decompress_sequences(frame_context_t *const ctx, static void decompress_sequences(frame_context_t *const ctx,
istream_t *const in, istream_t *const in,
@@ -1132,7 +1132,7 @@ static void decompress_sequences(frame_context_t *const ctx, istream_t *in,
// a single 1-bit and then fills the byte with 0-7 0 bits of padding." // a single 1-bit and then fills the byte with 0-7 0 bits of padding."
const int padding = 8 - highest_set_bit(src[len - 1]); const int padding = 8 - highest_set_bit(src[len - 1]);
// The offset starts at the end because FSE streams are read backwards // The offset starts at the end because FSE streams are read backwards
i64 bit_offset = len * 8 - padding; i64 bit_offset = (i64)(len * 8 - (size_t)padding);
// "The bitstream starts with initial state values, each using the required // "The bitstream starts with initial state values, each using the required
// number of bits in their respective accuracy, decoded previously from // number of bits in their respective accuracy, decoded previously from
@@ -1409,7 +1409,7 @@ size_t ZSTD_get_decompressed_size(const void *src, const size_t src_len) {
// get decompressed size from ZSTD frame header // get decompressed size from ZSTD frame header
{ {
const u32 magic_number = IO_read_bits(&in, 32); const u32 magic_number = (u32)IO_read_bits(&in, 32);
if (magic_number == 0xFD2FB528U) { if (magic_number == 0xFD2FB528U) {
// ZSTD frame // ZSTD frame
@@ -1418,7 +1418,7 @@ size_t ZSTD_get_decompressed_size(const void *src, const size_t src_len) {
if (header.frame_content_size == 0 && !header.single_segment_flag) { if (header.frame_content_size == 0 && !header.single_segment_flag) {
// Content size not provided, we can't tell // Content size not provided, we can't tell
return -1; return (size_t)-1;
} }
return header.frame_content_size; return header.frame_content_size;
doc/educational_decoder/zstd_decompress.h
+3 -1
View File
@@ -7,6 +7,8 @@
* in the COPYING file in the root directory of this source tree). * in the COPYING file in the root directory of this source tree).
*/ */
#include <stddef.h> /* size_t */
/******* EXPOSED TYPES ********************************************************/ /******* EXPOSED TYPES ********************************************************/
/* /*
* Contains the parsed contents of a dictionary * Contains the parsed contents of a dictionary
@@ -39,7 +41,7 @@ size_t ZSTD_get_decompressed_size(const void *const src, const size_t src_len);
* Return a valid dictionary_t pointer for use with dictionary initialization * Return a valid dictionary_t pointer for use with dictionary initialization
* or decompression * or decompression
*/ */
dictionary_t* create_dictionary(); dictionary_t* create_dictionary(void);
/* /*
* Parse a provided dictionary blob for use in decompression * Parse a provided dictionary blob for use in decompression
lib/dictBuilder/cover.c
+2 -2
View File
@@ -638,8 +638,8 @@ void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers, int displayLeve
"compared to the source size %u! " "compared to the source size %u! "
"size(source)/size(dictionary) = %f, but it should be >= " "size(source)/size(dictionary) = %f, but it should be >= "
"10! This may lead to a subpar dictionary! We recommend " "10! This may lead to a subpar dictionary! We recommend "
"training on sources at least 10x, and up to 100x the " "training on sources at least 10x, and preferably 100x "
"size of the dictionary!\n", (U32)maxDictSize, "the size of the dictionary! \n", (U32)maxDictSize,
(U32)nbDmers, ratio); (U32)nbDmers, ratio);
} }