From 861ae95f8c0aae1226a9e035cf0ca6076d5fb3da Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Sun, 4 Aug 2019 22:07:43 +1000 Subject: [PATCH] Add a rac.Reader --- lib/rac/example_test.go | 77 +++++- lib/rac/rac.go | 2 + lib/rac/reader.go | 545 ++++++++++++++++++++++++++++++++++++++++ lib/rac/writer.go | 6 +- lib/rac/writer_test.go | 213 ++++++++++++++-- 5 files changed, 819 insertions(+), 24 deletions(-) create mode 100644 lib/rac/reader.go diff --git a/lib/rac/example_test.go b/lib/rac/example_test.go index 26156492..455d79f1 100644 --- a/lib/rac/example_test.go +++ b/lib/rac/example_test.go @@ -17,11 +17,13 @@ package rac_test import ( "bytes" "compress/zlib" + "encoding/binary" "encoding/hex" "fmt" "hash/adler32" "io" "log" + "os" "github.com/google/wuffs/lib/rac" ) @@ -29,8 +31,8 @@ import ( // ExampleILAEnd demonstrates using the low level "rac" package to encode a // RAC+Zlib formatted file with IndexLocationAtEnd. // -// The sibling "raczlib" package (TODO) provides a higher level API that is -// easier to use. +// The sibling "raczlib" package provides a higher level API that is easier to +// use. // // See the RAC specification for an explanation of the file format. func ExampleILAEnd() { @@ -91,8 +93,8 @@ func ExampleILAEnd() { // ExampleILAStart demonstrates using the low level "rac" package to encode and // then decode a RAC+Zlib formatted file with IndexLocationAtStart. // -// The sibling "raczlib" package (TODO) provides a higher level API that is -// easier to use. +// The sibling "raczlib" package provides a higher level API that is easier to +// use. // // See the RAC specification for an explanation of the file format. func ExampleILAStart() { @@ -151,11 +153,68 @@ func ExampleILAStart() { log.Fatalf("Close: %v", err) } - fmt.Printf("RAC file:\n%s", hex.Dump(buf.Bytes())) + encoded := buf.Bytes() + fmt.Printf("RAC file:\n%s\n", hex.Dump(encoded)) - // TODO: decode the encoded bytes (the RAC-formatted bytes) to recover the + // Decode the encoded bytes (the RAC-formatted bytes) to recover the // original "One sheep.\nTwo sheep\.Three sheep.\n" source. + fmt.Printf("Decoded:\n") + r := &rac.Reader{ + ReadSeeker: bytes.NewReader(encoded), + CompressedSize: int64(len(encoded)), + } + zr := io.ReadCloser(nil) + for { + chunk, err := r.NextChunk() + if err == io.EOF { + break + } else if err != nil { + log.Fatalf("NextChunk: %v", err) + } + if chunk.Codec != rac.CodecZlib { + log.Fatalf("unexpected chunk codec") + } + fmt.Printf("[%2d, %2d): ", chunk.DRange[0], chunk.DRange[1]) + + // Parse the RAC+Zlib secondary data. For details, see + // https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md#rac--zlib + dict := []byte(nil) + if secondary := encoded[chunk.CSecondary[0]:chunk.CSecondary[1]]; len(secondary) > 0 { + if len(secondary) < 6 { + log.Fatalf("invalid secondary data") + } + dictLen := int(binary.LittleEndian.Uint16(secondary)) + secondary = secondary[2:] + if (dictLen + 4) > len(secondary) { + log.Fatalf("invalid secondary data") + } + checksum := binary.BigEndian.Uint32(secondary[dictLen:]) + dict = secondary[:dictLen] + if checksum != adler32.Checksum(dict) { + log.Fatalf("invalid checksum") + } + } + + // Decompress the Zlib-encoded primary payload. + primary := encoded[chunk.CPrimary[0]:chunk.CPrimary[1]] + if zr == nil { + if zr, err = zlib.NewReaderDict(bytes.NewReader(primary), dict); err != nil { + log.Fatalf("zlib.NewReader: %v", err) + } + } else if err := zr.(zlib.Resetter).Reset(bytes.NewReader(primary), dict); err != nil { + log.Fatalf("zlib.Reader.Reset: %v", err) + } + if n, err := io.Copy(os.Stdout, zr); err != nil { + log.Fatalf("io.Copy: %v", err) + } else if n != chunk.DRange.Size() { + log.Fatalf("inconsistent DRange size") + } + if err := zr.Close(); err != nil { + log.Fatalf("zlib.Reader.Close: %v", err) + } + } + // Note that these exact bytes depends on the zlib encoder's algorithm, but // there is more than one valid zlib encoding of any given input. This // "compare to golden output" test is admittedly brittle, as the standard @@ -189,5 +248,9 @@ func ExampleILAStart() { // 00000070 21 03 90 78 f9 0b e0 02 6e 0a 29 cf 87 31 01 01 |!..x....n.)..1..| // 00000080 00 00 ff ff 18 0c 03 a8 78 f9 0b e0 02 6e 0a c9 |........x....n..| // 00000090 28 4a 4d 85 71 00 01 00 00 ff ff 21 6e 04 66 |(JM.q......!n.f| - + // + // Decoded: + // [ 0, 11): One sheep. + // [11, 22): Two sheep. + // [22, 35): Three sheep. } diff --git a/lib/rac/rac.go b/lib/rac/rac.go index 03b9ffb9..09dc8d59 100644 --- a/lib/rac/rac.go +++ b/lib/rac/rac.go @@ -49,6 +49,8 @@ const ( // See the RAC specification for further discussion. type IndexLocation uint8 +const magic = "\x72\xC3\x63" + const ( IndexLocationAtEnd = IndexLocation(0) IndexLocationAtStart = IndexLocation(1) diff --git a/lib/rac/reader.go b/lib/rac/reader.go new file mode 100644 index 00000000..6afee387 --- /dev/null +++ b/lib/rac/reader.go @@ -0,0 +1,545 @@ +// Copyright 2019 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rac + +import ( + "errors" + "hash/crc32" + "io" +) + +var ( + errInvalidIndexNode = errors.New("rac: invalid index node") +) + +func u48LE(b []byte) int64 { + _ = b[7] // Early bounds check to guarantee safety of reads below. + u := uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | + uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 + return int64(u & 0xFFFFFFFFFFFF) +} + +// readAt calls ReadAt if it is available, otherwise it falls back to calling +// Seek and then ReadFull. Calling ReadAt is presumably slightly more +// efficient, e.g. one syscall instead of two. +func readAt(r io.ReadSeeker, p []byte, offset int64) error { + if a, ok := r.(io.ReaderAt); ok && false { + n, err := a.ReadAt(p, offset) + if (n == len(p)) && (err == io.EOF) { + err = nil + } + return err + } + if _, err := r.Seek(offset, io.SeekStart); err != nil { + return err + } + _, err := io.ReadFull(r, p) + return err +} + +// bitwiseSubset returns whether every 1-bit in inner is also set in outer. +func bitwiseSubset(outer Codec, inner Codec) bool { + return outer == (outer | inner) +} + +// Range is the half-open range [low, high). It is invalid for low to be +// greater than high. +type Range [2]int64 + +func (r *Range) Empty() bool { return r[0] == r[1] } +func (r *Range) Size() int64 { return r[1] - r[0] } + +// Chunk is a compressed chunk returned by a Reader. +// +// See the RAC specification for further discussion. +type Chunk struct { + DRange Range + CPrimary Range + CSecondary Range + CTertiary Range + STag uint8 + TTag uint8 + Codec Codec +} + +// nodeSize returns the size (in CSpace) that a node with the given arity +// occupies. +func nodeSize(arity uint8) int { + return (16 * int(arity)) + 16 +} + +// rNode is the Reader's representation of a node. +// +// None of its methods, other than valid, should be called unless valid returns +// true. +type rNode [4096]byte + +func (b *rNode) arity() int { return int(b[3]) } +func (b *rNode) codec() Codec { return Codec(b[(8*int(b[3]))+7]) } +func (b *rNode) cPtrMax() int64 { return u48LE(b[(16*int(b[3]))+8:]) } +func (b *rNode) dPtrMax() int64 { return u48LE(b[8*int(b[3]):]) } +func (b *rNode) version() uint8 { return b[(16*int(b[3]))+14] } + +func (b *rNode) cLen(i int) uint8 { + base := (8 * int(b[3])) + 14 + return b[(8*i)+base] +} + +func (b *rNode) cOff(i int, cBias int64) int64 { + base := (8 * int(b[3])) + 8 + return cBias + u48LE(b[(8*i)+base:]) +} + +func (b *rNode) cOffRange(i int, cBias int64) Range { + m := cBias + b.cPtrMax() + if i >= b.arity() { + return Range{m, m} + } + cOff := b.cOff(i, cBias) + if cLen := b.cLen(i); cLen != 0 { + if n := cOff + (int64(cLen) * 1024); m > n { + m = n + } + } + return Range{cOff, m} +} + +func (b *rNode) dOff(i int, dBias int64) int64 { + if i == 0 { + return dBias + } + return dBias + u48LE(b[8*i:]) +} + +func (b *rNode) dOffRange(i int, dBias int64) Range { + return Range{b.dOff(i, dBias), b.dOff(i+1, dBias)} +} + +func (b *rNode) dSize(i int) int64 { + x := int64(0) + if i > 0 { + x = u48LE(b[8*i:]) + } + return u48LE(b[(8*i)+8:]) - x +} + +func (b *rNode) sTag(i int) uint8 { + base := (8 * int(b[3])) + 15 + return b[(8*i)+base] +} + +func (b *rNode) tTag(i int) uint8 { + return b[(8*i)+7] +} + +func (b *rNode) isLeaf(i int) bool { + return b[(8*i)+7] != 0xFE +} + +func (b *rNode) findChunkContaining(dOff int64, dBias int64) int { + // TODO: binary search instead of linear search. + for i, n := 0, b.arity(); i < n; i++ { + if dOff < b.dOff(i+1, dBias) { + return i + } + } + // We shouldn't get here, since we validate each node, and don't call this + // function for a DOff greater or equal to DOffMax. + panic("rac: internal error: could not find containing chunk") +} + +func (b *rNode) chunk(i int, cBias int64, dBias int64) Chunk { + sTag := b.sTag(i) + tTag := b.tTag(i) + return Chunk{ + DRange: b.dOffRange(i, dBias), + CPrimary: b.cOffRange(i, cBias), + CSecondary: b.cOffRange(int(sTag), cBias), + CTertiary: b.cOffRange(int(tTag), cBias), + STag: sTag, + TTag: tTag, + Codec: b.codec(), + } +} + +func (b *rNode) valid() bool { + // Check the magic and arity. + if (b[0] != magic[0]) || (b[1] != magic[1]) || (b[2] != magic[2]) || (b[3] == 0) { + return false + } + arity := int(b[3]) + size := (16 * arity) + 16 + if b[3] != b[size-1] { + return false + } + + // Check that the "Reserved (0)" bytes are zero and that the TTag values + // aren't in the reserved range [0xC0, 0xFE). + for i := 0; i < arity; i++ { + if b[(8*i)+6] != 0 { + return false + } + if tTag := b[(8*i)+7]; (0xC0 <= tTag) && (tTag < 0xFE) { + return false + } + } + if b[(8*arity)+6] != 0 { + return false + } + + // Check that the Codec is non-zero. + if b[(8*arity)+7] == 0 { + return false + } + + // Check that the DPtr values are non-decreasing. The first DPtr value is + // implicitly zero. + prev := u48LE(b[8*1:]) + for i := 2; i <= arity; i++ { + curr := u48LE(b[8*i:]) + if curr < prev { + return false + } + prev = curr + } + + // Check that no CPtr value exceeds CPtrMax (the final CPtr value). + base := (8 * arity) + 8 + cPtrMax := u48LE(b[size-8:]) + for i := 0; i < arity; i++ { + if cPtr := u48LE(b[(8*i)+base:]); cPtr > cPtrMax { + return false + } + } + + // Check the the version is non-zero. + if b[(16*arity)+14] == 0 { + return false + } + + // Check the checksum. + checksum := crc32.ChecksumIEEE(b[6:size]) + checksum ^= checksum >> 16 + if (b[4] != uint8(checksum>>0)) || (b[5] != uint8(checksum>>8)) { + return false + } + + // Further checking of the codec, version, COffMax and DOffMax requires + // more context, and is done in loadAndValidate. + return true +} + +// Reader reads a RAC file. +// +// Do not modify its exported fields after calling any of its methods. +type Reader struct { + // ReadSeeker is where the RAC-encoded data is read from. + // + // Nil is an invalid value. + ReadSeeker io.ReadSeeker + + // CompressedSize is the size of the RAC file. + // + // Zero is an invalid value, as an empty file is not a valid RAC file. + CompressedSize int64 + + // initialized is set true after the first call on this Reader. + initialized bool + + // rootNodeArity is the root node's arity. + rootNodeArity uint8 + + // needToResolveSeekPosition is whether NextChunk will need to resolve + // seekPosition. + needToResolveSeekPosition bool + + // err is the first error encountered. It is sticky: once a non-nil error + // occurs, all public methods will return that error. + err error + + // decompressedSize is the size of the RAC file in DSpace. + decompressedSize int64 + + // rootNodeCOffset is the position of the root node in the RAC file. + rootNodeCOffset int64 + + // seekPosition gives, if needToResolveSeekPosition is true, the position + // in DSpace that NextChunk needs to find. + seekPosition int64 + + // The i (as in "the i'th child of currNode") that denotes the next chunk + // to be returned by NextChunk, if a non-empty chunk. If nextChunk equals + // currNode's arity, then currNode is exhausted and calling NextChunk will + // seek to the next node. + nextChunk int32 + + // The CBias and DBias of currNode. + currNodeCBias int64 + currNodeDBias int64 + + // currNode is the 4096 byte buffer to hold the current node. + currNode rNode +} + +func (r *Reader) checkParameters() error { + if r.ReadSeeker == nil { + r.err = errors.New("rac: invalid ReadSeeker") + return r.err + } + if r.CompressedSize < 32 { + r.err = errors.New("rac: invalid CompressedSize") + return r.err + } + return nil +} + +func (r *Reader) initialize() error { + if r.err != nil { + return r.err + } + if r.initialized { + return nil + } + r.initialized = true + + if err := r.checkParameters(); err != nil { + return err + } + + if err := r.findRootNode(); err != nil { + return err + } + if r.currNode.version() != 1 { + r.err = errors.New("rac: unsupported RAC file version") + return r.err + } + return nil +} + +func (r *Reader) findRootNode() error { + // Look at the start of the compressed file. + if err := readAt(r.ReadSeeker, r.currNode[:4], 0); err != nil { + r.err = err + return r.err + } + if (r.currNode[0] != magic[0]) || + (r.currNode[1] != magic[1]) || + (r.currNode[2] != magic[2]) { + r.err = errors.New("rac: invalid input: missing magic bytes at the start of file") + return r.err + } + if found, err := r.tryRootNode(r.currNode[3], false); err != nil { + return err + } else if found { + return nil + } + + // Look at the end of the compressed file. + if err := readAt(r.ReadSeeker, r.currNode[:1], r.CompressedSize-1); err != nil { + r.err = err + return r.err + } + if found, err := r.tryRootNode(r.currNode[0], true); err != nil { + return err + } else if found { + return nil + } + + return errors.New("rac: invalid input: missing index root node") +} + +func (r *Reader) tryRootNode(arity uint8, fromEnd bool) (found bool, ioErr error) { + if arity == 0 { + return false, nil + } + size := int64(nodeSize(arity)) + if r.CompressedSize < size { + return false, nil + } + cOffset := int64(0) + if fromEnd { + cOffset = r.CompressedSize - size + } + if err := r.load(cOffset, arity); err != nil { + return false, err + } + if !r.currNode.valid() { + return false, nil + } + if r.currNode.cPtrMax() != r.CompressedSize { + return false, nil + } + r.needToResolveSeekPosition = true + r.rootNodeCOffset = cOffset + r.rootNodeArity = arity + r.decompressedSize = r.currNode.dPtrMax() + return true, nil +} + +// load loads a node from the RAC file into r.currNode. It does not check that +// the result is valid, and the caller should do so if it doesn't already know +// that it is valid. +func (r *Reader) load(cOffset int64, arity uint8) error { + if arity == 0 { + r.err = errors.New("rac: internal error: inconsistent arity") + return r.err + } + size := nodeSize(arity) + if err := readAt(r.ReadSeeker, r.currNode[:size], cOffset); err != nil { + r.err = err + return r.err + } + return nil +} + +func (r *Reader) loadAndValidate(cOffset int64, + parentCodec Codec, parentVersion uint8, parentCOffMax int64, + childCBias int64, childDSize int64) error { + + if (cOffset < 0) || ((r.CompressedSize - 4) < cOffset) { + r.err = errInvalidIndexNode + return r.err + } + if err := readAt(r.ReadSeeker, r.currNode[:4], cOffset); err != nil { + r.err = err + return r.err + } + arity := r.currNode[3] + if arity == 0 { + r.err = errInvalidIndexNode + return r.err + } + size := int64(nodeSize(arity)) + if (r.CompressedSize < size) || ((r.CompressedSize - size) < cOffset) { + r.err = errInvalidIndexNode + return r.err + } + if err := r.load(cOffset, arity); err != nil { + return err + } + + if !r.currNode.valid() { + r.err = errInvalidIndexNode + return r.err + } + + // Validate the parent and child codec, version, COffMax and DOffMax. + childVersion := r.currNode.version() + if !bitwiseSubset(parentCodec, r.currNode.codec()) || + (parentVersion < childVersion) || + (parentCOffMax < (childCBias + r.currNode.cPtrMax())) || + (childDSize != r.currNode.dPtrMax()) { + r.err = errInvalidIndexNode + return r.err + } + return nil +} + +// DecompressedSize returns the total size of the decompressed data. +func (r *Reader) DecompressedSize() (int64, error) { + if err := r.initialize(); err != nil { + return 0, err + } + return r.decompressedSize, nil +} + +// SeekToChunkContaining sets up NextChunk to return the chunk containing +// dSpaceOffset. That chunk does not necessarily start at dSpaceOffset. +// +// It is an error to seek to a negative value. +func (r *Reader) SeekToChunkContaining(dSpaceOffset int64) error { + if err := r.initialize(); err != nil { + return err + } + if dSpaceOffset < 0 { + r.err = errors.New("rac: seek: negative position") + return r.err + } + r.needToResolveSeekPosition = true + r.seekPosition = dSpaceOffset + return nil +} + +// NextChunk returns the next independently compressed chunk, or io.EOF if +// there are no more chunks. +// +// Empty chunks (those that contain no decompressed data, only metadata) are +// skipped. +func (r *Reader) NextChunk() (Chunk, error) { + if err := r.initialize(); err != nil { + return Chunk{}, err + } + for { + if r.needToResolveSeekPosition { + if r.seekPosition >= r.decompressedSize { + return Chunk{}, io.EOF + } + r.needToResolveSeekPosition = false + if err := r.resolveSeekPosition(); err != nil { + return Chunk{}, err + } + } + for n := int32(r.currNode.arity()); r.nextChunk < n; { + c := r.currNode.chunk(int(r.nextChunk), r.currNodeCBias, r.currNodeDBias) + r.nextChunk++ + r.seekPosition = c.DRange[1] + if !c.DRange.Empty() { + return c, nil + } + } + r.needToResolveSeekPosition = true + } +} + +func (r *Reader) resolveSeekPosition() error { + // Load the root node. It has already been validated, during initialize. + if err := r.load(r.rootNodeCOffset, r.rootNodeArity); err != nil { + return err + } + + // Walk the branch nodes until we find the leaf node containing the + // seekPosition. + cBias := int64(0) + dBias := int64(0) + for { + i := r.currNode.findChunkContaining(r.seekPosition, dBias) + if r.currNode.isLeaf(i) { + r.nextChunk = int32(i) + r.currNodeCBias = cBias + r.currNodeDBias = dBias + return nil + } + + parentCodec := r.currNode.codec() + parentVersion := r.currNode.version() + parentCOffMax := cBias + r.currNode.cPtrMax() + childCOffset := r.currNode.cOff(i, cBias) + childCBias := cBias + if sTag := int(r.currNode.sTag(i)); sTag < r.currNode.arity() { + childCBias = r.currNode.cOff(sTag, cBias) + } + childDBias := r.currNode.dOff(i, dBias) + childDSize := r.currNode.dSize(i) + + if err := r.loadAndValidate(childCOffset, + parentCodec, parentVersion, parentCOffMax, + childCBias, childDSize); err != nil { + return err + } + + cBias = childCBias + dBias = childDBias + } +} diff --git a/lib/rac/writer.go b/lib/rac/writer.go index 64417210..9cb4f8da 100644 --- a/lib/rac/writer.go +++ b/lib/rac/writer.go @@ -65,9 +65,6 @@ type Writer struct { // Zero is an invalid value. Codec Codec - // initialized is set true after the first AddXxx call. - initialized bool - // IndexLocation is whether the index is at the start or end of the RAC // file. // @@ -127,6 +124,9 @@ type Writer struct { // each chunk's starting offset will be aligned to a page boundary. CPageSize uint64 + // initialized is set true after the first AddXxx call. + initialized bool + // err is the first error encountered. It is sticky: once a non-nil error // occurs, all public methods will return that error. err error diff --git a/lib/rac/writer_test.go b/lib/rac/writer_test.go index b5a01842..bf54deb4 100644 --- a/lib/rac/writer_test.go +++ b/lib/rac/writer_test.go @@ -29,10 +29,35 @@ const bytesPerHexDumpLine = 79 const fakeCodec = Codec(0xEE) -// Note that these exact bytes depends on the zlib encoder's algorithm, but -// there is more than one valid zlib encoding of any given input. This "compare -// to golden output" test is admittedly brittle, as the standard library's zlib -// package's output isn't necessarily stable across Go releases. +var unhex = [256]uint8{ + '0': 0x00, '1': 0x01, '2': 0x02, '3': 0x03, '4': 0x04, + '5': 0x05, '6': 0x06, '7': 0x07, '8': 0x08, '9': 0x09, + 'A': 0x0A, 'B': 0x0B, 'C': 0x0C, 'D': 0x0D, 'E': 0x0E, 'F': 0x0F, + 'a': 0x0A, 'b': 0x0B, 'c': 0x0C, 'd': 0x0D, 'e': 0x0E, 'f': 0x0F, +} + +func undoHexDump(s string) (ret []byte) { + for s != "" { + for i := 0; i < 16; i++ { + pos := 10 + 3*i + if i > 7 { + pos++ + } + c0 := s[pos+0] + c1 := s[pos+1] + if c0 == ' ' { + break + } + ret = append(ret, (unhex[c0]<<4)|unhex[c1]) + } + if n := strings.IndexByte(s, '\n'); n >= 0 { + s = s[n+1:] + } else { + break + } + } + return ret +} const writerWantEmpty = "" + "00000000 72 c3 63 01 30 14 00 ff 00 00 00 00 00 00 00 ee |r.c.0...........|\n" + @@ -179,6 +204,8 @@ func testWriter(iloc IndexLocation, tempFile io.ReadWriter, cPageSize uint64, em if !empty { // We ignore errors (assigning them to _) from the AddXxx calls. Any // non-nil errors are sticky, and should be returned by Close. + // + // These {Aa,Bb,Cc} chunks are also used in the reader test. res0, _ := w.AddResource([]byte("Rrr")) res1, _ := w.AddResource([]byte("Ss")) _ = w.AddChunk(0x11, []byte("Aaa"), 0, 0) @@ -215,7 +242,7 @@ func testWriter(iloc IndexLocation, tempFile io.ReadWriter, cPageSize uint64, em return nil } -func TestWriterMultiLevelIndex(t *testing.T) { +func TestMultiLevelIndex(t *testing.T) { buf := &bytes.Buffer{} w := &Writer{ Writer: buf, @@ -232,6 +259,7 @@ func TestWriterMultiLevelIndex(t *testing.T) { xRes := OptResource(0) yRes := OptResource(0) zRes := OptResource(0) + primaries := []byte(nil) for i := 0; i < 260; i++ { secondary := OptResource(0) tertiary := OptResource(0) @@ -253,6 +281,7 @@ func TestWriterMultiLevelIndex(t *testing.T) { } primary := []byte(fmt.Sprintf("p%02x", i&0xFF)) + primaries = append(primaries, primary...) _ = w.AddChunk(0x10000, primary, secondary, tertiary) } @@ -265,17 +294,17 @@ func TestWriterMultiLevelIndex(t *testing.T) { t.Fatalf("len(encoded): got 0x%X, want 0x%X", got, want) } - got := hex.Dump(encoded) - got = "" + - got[0x000*bytesPerHexDumpLine:0x008*bytesPerHexDumpLine] + + gotHexDump := hex.Dump(encoded) + gotHexDump = "" + + gotHexDump[0x000*bytesPerHexDumpLine:0x008*bytesPerHexDumpLine] + "...\n" + - got[0x080*bytesPerHexDumpLine:0x088*bytesPerHexDumpLine] + + gotHexDump[0x080*bytesPerHexDumpLine:0x088*bytesPerHexDumpLine] + "...\n" + - got[0x100*bytesPerHexDumpLine:0x110*bytesPerHexDumpLine] + + gotHexDump[0x100*bytesPerHexDumpLine:0x110*bytesPerHexDumpLine] + "...\n" + - got[0x13C*bytesPerHexDumpLine:] + gotHexDump[0x13C*bytesPerHexDumpLine:] - const want = "" + + const wantHexDump = "" + "00000000 72 c3 63 02 17 fa 00 fe 00 00 fc 00 00 00 00 fe |r.c.............|\n" + "00000010 00 00 04 01 00 00 00 ee 30 00 00 00 00 00 04 ff |........0.......|\n" + "00000020 30 10 00 00 00 00 01 ff e2 13 00 00 00 00 01 02 |0...............|\n" + @@ -315,8 +344,33 @@ func TestWriterMultiLevelIndex(t *testing.T) { "000013d0 70 66 65 70 66 66 70 30 30 70 30 31 70 30 32 70 |pfepffp00p01p02p|\n" + "000013e0 30 33 |03|\n" - if got != want { - t.Fatalf("\ngot:\n%s\nwant:\n%s", got, want) + if gotHexDump != wantHexDump { + t.Fatalf("\ngot:\n%s\nwant:\n%s", gotHexDump, wantHexDump) + } + + r := &Reader{ + ReadSeeker: bytes.NewReader(encoded), + CompressedSize: int64(len(encoded)), + } + gotPrimaries := []byte(nil) + for { + c, err := r.NextChunk() + if err == io.EOF { + break + } else if err != nil { + t.Fatalf("NextChunk: %v", err) + } + p0, p1 := c.CPrimary[0], c.CPrimary[1] + if (0 <= p0) && (p0 <= p1) && (p1 <= int64(len(encoded))) { + primary := encoded[p0:p1] + if len(primary) > 3 { + primary = primary[:3] + } + gotPrimaries = append(gotPrimaries, primary...) + } + } + if !bytes.Equal(gotPrimaries, primaries) { + t.Fatalf("\ngot:\n%s\nwant:\n%s", gotPrimaries, primaries) } } @@ -338,3 +392,134 @@ func TestWriter1000Chunks(t *testing.T) { t.Fatalf("Close: %v", err) } } + +func printChunks(chunks []Chunk) string { + ss := make([]string, len(chunks)) + for i, c := range chunks { + ss[i] = fmt.Sprintf("D:%#x, C0:%#x, C1:%#x, C2:%#x, S:%#x, T:%#x, C:%#x", + c.DRange, c.CPrimary, c.CSecondary, c.CTertiary, c.STag, c.TTag, c.Codec) + } + return strings.Join(ss, "\n") +} + +func TestReader(t *testing.T) { + makeChunk := func() Chunk { + return Chunk{} + } + _ = makeChunk + + testCases := []struct { + name string + compressed []byte + }{ + {"Empty", undoHexDump(writerWantEmpty)}, + {"ILAEnd", undoHexDump(writerWantILAEnd)}, + {"ILAEndCPageSize8", undoHexDump(writerWantILAEndCPageSize8)}, + {"ILAStart", undoHexDump(writerWantILAStart)}, + {"ILAStartCPageSize4", undoHexDump(writerWantILAStartCPageSize4)}, + {"ILAStartCPageSize128", undoHexDump(writerWantILAStartCPageSize128)}, + } + +loop: + for _, tc := range testCases { + snippet := func(r Range) string { + if r.Empty() { + return "" + } + if r[1] > int64(len(tc.compressed)) { + return "CRange goes beyond COffMax" + } + s := tc.compressed[r[0]:r[1]] + if len(s) > 2 { + return string(s[:2]) + "..." + } + return string(s) + } + + wantDecompressedSize := int64(0) + wantDescription := "" + if tc.name != "Empty" { + // These {Aa,Bb,Cc} chunks are also used in the writer test. + wantDecompressedSize = 0x11 + 0x22 + 0x44 + wantDescription = `` + + `DRangeSize:0x11, C0:"Aa...", C1:"", C2:""` + "\n" + + `DRangeSize:0x22, C0:"Bb...", C1:"Rr...", C2:""` + "\n" + + `DRangeSize:0x44, C0:"Cc...", C1:"Rr...", C2:"Ss..."` + } + + r := &Reader{ + ReadSeeker: bytes.NewReader(tc.compressed), + CompressedSize: int64(len(tc.compressed)), + } + + if gotDecompressedSize, err := r.DecompressedSize(); err != nil { + t.Errorf("%q test case: %v", tc.name, err) + continue loop + } else if gotDecompressedSize != wantDecompressedSize { + t.Errorf("%q test case: DecompressedSize: got %d, want %d", + tc.name, gotDecompressedSize, wantDecompressedSize) + continue loop + } + + gotChunks := []Chunk(nil) + description := &bytes.Buffer{} + prevDRange1 := int64(0) + for { + c, err := r.NextChunk() + if err == io.EOF { + break + } else if err != nil { + t.Errorf("%q test case: NextChunk: %v", tc.name, err) + continue loop + } + + if c.DRange[0] != prevDRange1 { + t.Errorf("%q test case: NextChunk: DRange[0]: got %d, want %d", + tc.name, c.DRange[0], prevDRange1) + continue loop + } + prevDRange1 = c.DRange[1] + + gotChunks = append(gotChunks, c) + if description.Len() > 0 { + description.WriteByte('\n') + } + fmt.Fprintf(description, "DRangeSize:0x%X, C0:%q, C1:%q, C2:%q", + c.DRange.Size(), snippet(c.CPrimary), snippet(c.CSecondary), snippet(c.CTertiary)) + } + + if tc.name == "Empty" { + if len(gotChunks) != 0 { + t.Errorf("%q test case: NextChunk: got non-empty, want empty", tc.name) + } + continue loop + } + + gotDescription := description.String() + if gotDescription != wantDescription { + t.Errorf("%q test case: NextChunk:\n got\n%s\n which is\n%s\n want\n%s", + tc.name, printChunks(gotChunks), gotDescription, wantDescription) + continue loop + } + + // NextChunk should return io.EOF. + if _, err := r.NextChunk(); err != io.EOF { + t.Errorf("%q test case: NextChunk: got %v, want io.EOF", tc.name, err) + continue loop + } + + if err := r.SeekToChunkContaining(0x30); err != nil { + t.Errorf("%q test case: SeekToChunkContaining: %v", tc.name, err) + continue loop + } + + // NextChunk should return the "Bb..." chunk. + if c, err := r.NextChunk(); err != nil { + t.Errorf("%q test case: NextChunk: %v", tc.name, err) + continue loop + } else if got, want := snippet(c.CPrimary), "Bb..."; got != want { + t.Errorf("%q test case: NextChunk: got %q, want %q", tc.name, got, want) + continue loop + } + } +}