Add "RAC + Zeroes" codec

This commit is contained in:
Nigel Tao committed 2019-08-09 21:46:28 +10:00
1 parent 1a76fab33e
commit 88220a7591
6 files changed
+78 -15

No files matched your search

+9 -3
View File
@@ -346,7 +346,7 @@ file, even if it is a valid RAC file. Recall that RAC is just a container, and
not tied to any particular compression codec. For the `Codec` byte in a `Branch
Node`:
- `0x00` is reserved.
- `0x00` means "RAC + Zeroes".
- `0x01` means "RAC + Zlib".
- `0x02` means "RAC + Brotli".
- `0x04` means "RAC + ZStandard".
@@ -515,6 +515,12 @@ if they are `Branch Node`s.
# Specific Codecs
## RAC + Zeroes
The `CRange`s are ignored. The `DRange` is filled with NUL bytes (`memset` to
zero).
## RAC + Zlib
The `CFile` data in the `Leaf Node`'s `Primary CRange` is decompressed as Zlib
@@ -567,8 +573,8 @@ As per the RFC section 3.1.1.1.2, the Zstandard format allows for a window size
up to 3.75 TiB, which is more than the maximum `uint32_t` value. The
`Dictionary Length` field here is therefore 8 bytes, not 4. However, the RFC
acknowledges that "a decoder is allowed to reject a compressed frame that
requests a memory size beyond decoder's authorized range", and likewise, a RAC
+ Zstandard decoder is allowed to reject a `Dictionary Length` that it
requests a memory size beyond decoder's authorized range", and likewise, a
RAC + Zstandard decoder is allowed to reject a `Dictionary Length` that it
considers too large. The RFC goes on to say "it's recommended for decoders to
support values of Window Size up to 8 MiB and for encoders not to generate
frames requiring a Window Size larger than 8 MiB", and that recommendation
+2 -6
View File
@@ -357,10 +357,6 @@ func (w *ChunkWriter) AddChunk(
w.err = errTooManyChunks
return w.err
}
if codec == 0 {
w.err = errInvalidCodec
return w.err
}
if err := w.initialize(); err != nil {
return err
@@ -383,8 +379,8 @@ func (w *ChunkWriter) AddChunk(
}
var emptyRACFile = [32]byte{
0x72, 0xC3, 0x63, 0x01, 0xE8, 0xB4, 0x00, 0xFF,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
0x72, 0xC3, 0x63, 0x01, 0x0D, 0xF8, 0x00, 0xFF,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xFF,
0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01,
}
-5
View File
@@ -194,11 +194,6 @@ func (b *pNode) valid() bool {
return false
}
// Check that the Codec is non-zero.
if b[(8*arity)+7] == 0 {
return false
}
// Check that the DPtr values are non-decreasing. The first DPtr value is
// implicitly zero.
prev := u48LE(b[8*1:])
+1
View File
@@ -43,6 +43,7 @@ const (
type Codec uint8
const (
CodecZeroes = Codec(0x00)
CodecZlib = Codec(0x01)
CodecBrotli = Codec(0x02)
CodecZstandard = Codec(0x04)
+37 -1
View File
@@ -60,7 +60,7 @@ func undoHexDump(s string) (ret []byte) {
}
const writerWantEmpty = "" +
"00000000 72 c3 63 01 e8 b4 00 ff 00 00 00 00 00 00 00 01 |r.c.............|\n" +
"00000000 72 c3 63 01 0d f8 00 ff 00 00 00 00 00 00 00 00 |r.c.............|\n" +
"00000010 20 00 00 00 00 00 01 ff 20 00 00 00 00 00 01 01 | ....... .......|\n"
const writerWantILAEnd = "" +
@@ -548,3 +548,39 @@ loop:
}
}
}
func TestReaderEmpty(t *testing.T) {
got, err := ioutil.ReadAll(&Reader{
ReadSeeker: bytes.NewReader(undoHexDump(writerWantEmpty)),
})
if err != nil {
t.Fatalf("ReadAll: %v", err)
}
if len(got) != 0 {
t.Fatalf("got %q, want %q", got, []byte(nil))
}
}
func TestReaderZeroes(t *testing.T) {
const dSize = 7
buf := &bytes.Buffer{}
w := &ChunkWriter{
Writer: buf,
}
if err := w.AddChunk(dSize, CodecZeroes, nil, 0, 0); err != nil {
t.Fatalf("AddChunk: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("Close: %v", err)
}
got, err := ioutil.ReadAll(&Reader{
ReadSeeker: bytes.NewReader(buf.Bytes()),
})
if err != nil {
t.Fatalf("ReadAll: %v", err)
}
want := make([]byte, dSize)
if !bytes.Equal(got, want) {
t.Fatalf("got %q, want %q", got, want)
}
}
+29
View File
@@ -19,6 +19,24 @@ import (
"io"
)
// zeroesReader is an io.Reader that serves up a finite number of '\x00' bytes.
type zeroesReader int64
// Read implements io.Reader.
func (z *zeroesReader) Read(p []byte) (int, error) {
if int64(len(p)) > int64(*z) {
p = p[:*z]
}
for i := range p {
p[i] = 0
}
*z -= zeroesReader(len(p))
if *z == 0 {
return len(p), io.EOF
}
return len(p), nil
}
// ReaderContext contains the decoded Codec-specific metadata (non-primary
// data) associated with a RAC chunk.
type ReaderContext struct {
@@ -125,6 +143,9 @@ type Reader struct {
// In "State A", the dRange is empty and unused, other than trivially
// maintaining the invariant.
dRange Range
// zeroes serves the Zeroes Codec.
zeroes zeroesReader
}
func (r *Reader) initialize() error {
@@ -299,6 +320,14 @@ func (r *Reader) nextChunk() error {
return r.err
}
if chunk.Codec == 0 {
r.currChunk.N = 0
r.dRange = chunk.DRange
r.zeroes = zeroesReader(r.dRange.Size())
r.decompressor = &r.zeroes
return nil
}
codecReader := CodecReader(nil)
for _, cr := range r.CodecReaders {
if cr.Accepts(chunk.Codec) {