From 88220a75916cdfb75214ae072a3fb64b929982b5 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Fri, 9 Aug 2019 21:44:14 +1000 Subject: [PATCH] Add "RAC + Zeroes" codec --- doc/spec/rac-spec.md | 12 +++++++++--- lib/rac/chunk_writer.go | 8 ++------ lib/rac/parser.go | 5 ----- lib/rac/rac.go | 1 + lib/rac/rac_test.go | 38 +++++++++++++++++++++++++++++++++++++- lib/rac/reader.go | 29 +++++++++++++++++++++++++++++ 6 files changed, 78 insertions(+), 15 deletions(-) diff --git a/doc/spec/rac-spec.md b/doc/spec/rac-spec.md index 828f9c3c..4f4d47bf 100644 --- a/doc/spec/rac-spec.md +++ b/doc/spec/rac-spec.md @@ -346,7 +346,7 @@ file, even if it is a valid RAC file. Recall that RAC is just a container, and not tied to any particular compression codec. For the `Codec` byte in a `Branch Node`: - - `0x00` is reserved. + - `0x00` means "RAC + Zeroes". - `0x01` means "RAC + Zlib". - `0x02` means "RAC + Brotli". - `0x04` means "RAC + ZStandard". @@ -515,6 +515,12 @@ if they are `Branch Node`s. # Specific Codecs +## RAC + Zeroes + +The `CRange`s are ignored. The `DRange` is filled with NUL bytes (`memset` to +zero). + + ## RAC + Zlib The `CFile` data in the `Leaf Node`'s `Primary CRange` is decompressed as Zlib @@ -567,8 +573,8 @@ As per the RFC section 3.1.1.1.2, the Zstandard format allows for a window size up to 3.75 TiB, which is more than the maximum `uint32_t` value. The `Dictionary Length` field here is therefore 8 bytes, not 4. However, the RFC acknowledges that "a decoder is allowed to reject a compressed frame that -requests a memory size beyond decoder's authorized range", and likewise, a RAC -+ Zstandard decoder is allowed to reject a `Dictionary Length` that it +requests a memory size beyond decoder's authorized range", and likewise, a +RAC + Zstandard decoder is allowed to reject a `Dictionary Length` that it considers too large. The RFC goes on to say "it's recommended for decoders to support values of Window Size up to 8 MiB and for encoders not to generate frames requiring a Window Size larger than 8 MiB", and that recommendation diff --git a/lib/rac/chunk_writer.go b/lib/rac/chunk_writer.go index 8e43c50f..e852a5d0 100644 --- a/lib/rac/chunk_writer.go +++ b/lib/rac/chunk_writer.go @@ -357,10 +357,6 @@ func (w *ChunkWriter) AddChunk( w.err = errTooManyChunks return w.err } - if codec == 0 { - w.err = errInvalidCodec - return w.err - } if err := w.initialize(); err != nil { return err @@ -383,8 +379,8 @@ func (w *ChunkWriter) AddChunk( } var emptyRACFile = [32]byte{ - 0x72, 0xC3, 0x63, 0x01, 0xE8, 0xB4, 0x00, 0xFF, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x72, 0xC3, 0x63, 0x01, 0x0D, 0xF8, 0x00, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xFF, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, } diff --git a/lib/rac/parser.go b/lib/rac/parser.go index 8625d63a..29942743 100644 --- a/lib/rac/parser.go +++ b/lib/rac/parser.go @@ -194,11 +194,6 @@ func (b *pNode) valid() bool { return false } - // Check that the Codec is non-zero. - if b[(8*arity)+7] == 0 { - return false - } - // Check that the DPtr values are non-decreasing. The first DPtr value is // implicitly zero. prev := u48LE(b[8*1:]) diff --git a/lib/rac/rac.go b/lib/rac/rac.go index cb7ed474..0bdbc80b 100644 --- a/lib/rac/rac.go +++ b/lib/rac/rac.go @@ -43,6 +43,7 @@ const ( type Codec uint8 const ( + CodecZeroes = Codec(0x00) CodecZlib = Codec(0x01) CodecBrotli = Codec(0x02) CodecZstandard = Codec(0x04) diff --git a/lib/rac/rac_test.go b/lib/rac/rac_test.go index 265f7818..a68d681c 100644 --- a/lib/rac/rac_test.go +++ b/lib/rac/rac_test.go @@ -60,7 +60,7 @@ func undoHexDump(s string) (ret []byte) { } const writerWantEmpty = "" + - "00000000 72 c3 63 01 e8 b4 00 ff 00 00 00 00 00 00 00 01 |r.c.............|\n" + + "00000000 72 c3 63 01 0d f8 00 ff 00 00 00 00 00 00 00 00 |r.c.............|\n" + "00000010 20 00 00 00 00 00 01 ff 20 00 00 00 00 00 01 01 | ....... .......|\n" const writerWantILAEnd = "" + @@ -548,3 +548,39 @@ loop: } } } + +func TestReaderEmpty(t *testing.T) { + got, err := ioutil.ReadAll(&Reader{ + ReadSeeker: bytes.NewReader(undoHexDump(writerWantEmpty)), + }) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if len(got) != 0 { + t.Fatalf("got %q, want %q", got, []byte(nil)) + } +} + +func TestReaderZeroes(t *testing.T) { + const dSize = 7 + buf := &bytes.Buffer{} + w := &ChunkWriter{ + Writer: buf, + } + if err := w.AddChunk(dSize, CodecZeroes, nil, 0, 0); err != nil { + t.Fatalf("AddChunk: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + got, err := ioutil.ReadAll(&Reader{ + ReadSeeker: bytes.NewReader(buf.Bytes()), + }) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + want := make([]byte, dSize) + if !bytes.Equal(got, want) { + t.Fatalf("got %q, want %q", got, want) + } +} diff --git a/lib/rac/reader.go b/lib/rac/reader.go index 2f9672a7..c6c512fa 100644 --- a/lib/rac/reader.go +++ b/lib/rac/reader.go @@ -19,6 +19,24 @@ import ( "io" ) +// zeroesReader is an io.Reader that serves up a finite number of '\x00' bytes. +type zeroesReader int64 + +// Read implements io.Reader. +func (z *zeroesReader) Read(p []byte) (int, error) { + if int64(len(p)) > int64(*z) { + p = p[:*z] + } + for i := range p { + p[i] = 0 + } + *z -= zeroesReader(len(p)) + if *z == 0 { + return len(p), io.EOF + } + return len(p), nil +} + // ReaderContext contains the decoded Codec-specific metadata (non-primary // data) associated with a RAC chunk. type ReaderContext struct { @@ -125,6 +143,9 @@ type Reader struct { // In "State A", the dRange is empty and unused, other than trivially // maintaining the invariant. dRange Range + + // zeroes serves the Zeroes Codec. + zeroes zeroesReader } func (r *Reader) initialize() error { @@ -299,6 +320,14 @@ func (r *Reader) nextChunk() error { return r.err } + if chunk.Codec == 0 { + r.currChunk.N = 0 + r.dRange = chunk.DRange + r.zeroes = zeroesReader(r.dRange.Size()) + r.decompressor = &r.zeroes + return nil + } + codecReader := CodecReader(nil) for _, cr := range r.CodecReaders { if cr.Accepts(chunk.Codec) {