Add a rac.Reader

This commit is contained in:
Nigel Tao committed 2019-08-04 22:07:43 +10:00
1 parent e292a22ccb
commit 861ae95f8c
5 files changed
+819 -24

No files matched your search

+70 -7
View File
@@ -17,11 +17,13 @@ package rac_test
import (
"bytes"
"compress/zlib"
"encoding/binary"
"encoding/hex"
"fmt"
"hash/adler32"
"io"
"log"
"os"
"github.com/google/wuffs/lib/rac"
)
@@ -29,8 +31,8 @@ import (
// ExampleILAEnd demonstrates using the low level "rac" package to encode a
// RAC+Zlib formatted file with IndexLocationAtEnd.
//
// The sibling "raczlib" package (TODO) provides a higher level API that is
// easier to use.
// The sibling "raczlib" package provides a higher level API that is easier to
// use.
//
// See the RAC specification for an explanation of the file format.
func ExampleILAEnd() {
@@ -91,8 +93,8 @@ func ExampleILAEnd() {
// ExampleILAStart demonstrates using the low level "rac" package to encode and
// then decode a RAC+Zlib formatted file with IndexLocationAtStart.
//
// The sibling "raczlib" package (TODO) provides a higher level API that is
// easier to use.
// The sibling "raczlib" package provides a higher level API that is easier to
// use.
//
// See the RAC specification for an explanation of the file format.
func ExampleILAStart() {
@@ -151,11 +153,68 @@ func ExampleILAStart() {
log.Fatalf("Close: %v", err)
}
fmt.Printf("RAC file:\n%s", hex.Dump(buf.Bytes()))
encoded := buf.Bytes()
fmt.Printf("RAC file:\n%s\n", hex.Dump(encoded))
// TODO: decode the encoded bytes (the RAC-formatted bytes) to recover the
// Decode the encoded bytes (the RAC-formatted bytes) to recover the
// original "One sheep.\nTwo sheep\.Three sheep.\n" source.
fmt.Printf("Decoded:\n")
r := &rac.Reader{
ReadSeeker: bytes.NewReader(encoded),
CompressedSize: int64(len(encoded)),
}
zr := io.ReadCloser(nil)
for {
chunk, err := r.NextChunk()
if err == io.EOF {
break
} else if err != nil {
log.Fatalf("NextChunk: %v", err)
}
if chunk.Codec != rac.CodecZlib {
log.Fatalf("unexpected chunk codec")
}
fmt.Printf("[%2d, %2d): ", chunk.DRange[0], chunk.DRange[1])
// Parse the RAC+Zlib secondary data. For details, see
// https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md#rac--zlib
dict := []byte(nil)
if secondary := encoded[chunk.CSecondary[0]:chunk.CSecondary[1]]; len(secondary) > 0 {
if len(secondary) < 6 {
log.Fatalf("invalid secondary data")
}
dictLen := int(binary.LittleEndian.Uint16(secondary))
secondary = secondary[2:]
if (dictLen + 4) > len(secondary) {
log.Fatalf("invalid secondary data")
}
checksum := binary.BigEndian.Uint32(secondary[dictLen:])
dict = secondary[:dictLen]
if checksum != adler32.Checksum(dict) {
log.Fatalf("invalid checksum")
}
}
// Decompress the Zlib-encoded primary payload.
primary := encoded[chunk.CPrimary[0]:chunk.CPrimary[1]]
if zr == nil {
if zr, err = zlib.NewReaderDict(bytes.NewReader(primary), dict); err != nil {
log.Fatalf("zlib.NewReader: %v", err)
}
} else if err := zr.(zlib.Resetter).Reset(bytes.NewReader(primary), dict); err != nil {
log.Fatalf("zlib.Reader.Reset: %v", err)
}
if n, err := io.Copy(os.Stdout, zr); err != nil {
log.Fatalf("io.Copy: %v", err)
} else if n != chunk.DRange.Size() {
log.Fatalf("inconsistent DRange size")
}
if err := zr.Close(); err != nil {
log.Fatalf("zlib.Reader.Close: %v", err)
}
}
// Note that these exact bytes depends on the zlib encoder's algorithm, but
// there is more than one valid zlib encoding of any given input. This
// "compare to golden output" test is admittedly brittle, as the standard
@@ -189,5 +248,9 @@ func ExampleILAStart() {
// 00000070 21 03 90 78 f9 0b e0 02 6e 0a 29 cf 87 31 01 01 |!..x....n.)..1..|
// 00000080 00 00 ff ff 18 0c 03 a8 78 f9 0b e0 02 6e 0a c9 |........x....n..|
// 00000090 28 4a 4d 85 71 00 01 00 00 ff ff 21 6e 04 66 |(JM.q......!n.f|
//
// Decoded:
// [ 0, 11): One sheep.
// [11, 22): Two sheep.
// [22, 35): Three sheep.
}
+2
View File
@@ -49,6 +49,8 @@ const (
// See the RAC specification for further discussion.
type IndexLocation uint8
const magic = "\x72\xC3\x63"
const (
IndexLocationAtEnd = IndexLocation(0)
IndexLocationAtStart = IndexLocation(1)
+545
View File
@@ -0,0 +1,545 @@
// Copyright 2019 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rac
import (
"errors"
"hash/crc32"
"io"
)
var (
errInvalidIndexNode = errors.New("rac: invalid index node")
)
func u48LE(b []byte) int64 {
_ = b[7] // Early bounds check to guarantee safety of reads below.
u := uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 |
uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
return int64(u & 0xFFFFFFFFFFFF)
}
// readAt calls ReadAt if it is available, otherwise it falls back to calling
// Seek and then ReadFull. Calling ReadAt is presumably slightly more
// efficient, e.g. one syscall instead of two.
func readAt(r io.ReadSeeker, p []byte, offset int64) error {
if a, ok := r.(io.ReaderAt); ok && false {
n, err := a.ReadAt(p, offset)
if (n == len(p)) && (err == io.EOF) {
err = nil
}
return err
}
if _, err := r.Seek(offset, io.SeekStart); err != nil {
return err
}
_, err := io.ReadFull(r, p)
return err
}
// bitwiseSubset returns whether every 1-bit in inner is also set in outer.
func bitwiseSubset(outer Codec, inner Codec) bool {
return outer == (outer | inner)
}
// Range is the half-open range [low, high). It is invalid for low to be
// greater than high.
type Range [2]int64
func (r *Range) Empty() bool { return r[0] == r[1] }
func (r *Range) Size() int64 { return r[1] - r[0] }
// Chunk is a compressed chunk returned by a Reader.
//
// See the RAC specification for further discussion.
type Chunk struct {
DRange Range
CPrimary Range
CSecondary Range
CTertiary Range
STag uint8
TTag uint8
Codec Codec
}
// nodeSize returns the size (in CSpace) that a node with the given arity
// occupies.
func nodeSize(arity uint8) int {
return (16 * int(arity)) + 16
}
// rNode is the Reader's representation of a node.
//
// None of its methods, other than valid, should be called unless valid returns
// true.
type rNode [4096]byte
func (b *rNode) arity() int { return int(b[3]) }
func (b *rNode) codec() Codec { return Codec(b[(8*int(b[3]))+7]) }
func (b *rNode) cPtrMax() int64 { return u48LE(b[(16*int(b[3]))+8:]) }
func (b *rNode) dPtrMax() int64 { return u48LE(b[8*int(b[3]):]) }
func (b *rNode) version() uint8 { return b[(16*int(b[3]))+14] }
func (b *rNode) cLen(i int) uint8 {
base := (8 * int(b[3])) + 14
return b[(8*i)+base]
}
func (b *rNode) cOff(i int, cBias int64) int64 {
base := (8 * int(b[3])) + 8
return cBias + u48LE(b[(8*i)+base:])
}
func (b *rNode) cOffRange(i int, cBias int64) Range {
m := cBias + b.cPtrMax()
if i >= b.arity() {
return Range{m, m}
}
cOff := b.cOff(i, cBias)
if cLen := b.cLen(i); cLen != 0 {
if n := cOff + (int64(cLen) * 1024); m > n {
m = n
}
}
return Range{cOff, m}
}
func (b *rNode) dOff(i int, dBias int64) int64 {
if i == 0 {
return dBias
}
return dBias + u48LE(b[8*i:])
}
func (b *rNode) dOffRange(i int, dBias int64) Range {
return Range{b.dOff(i, dBias), b.dOff(i+1, dBias)}
}
func (b *rNode) dSize(i int) int64 {
x := int64(0)
if i > 0 {
x = u48LE(b[8*i:])
}
return u48LE(b[(8*i)+8:]) - x
}
func (b *rNode) sTag(i int) uint8 {
base := (8 * int(b[3])) + 15
return b[(8*i)+base]
}
func (b *rNode) tTag(i int) uint8 {
return b[(8*i)+7]
}
func (b *rNode) isLeaf(i int) bool {
return b[(8*i)+7] != 0xFE
}
func (b *rNode) findChunkContaining(dOff int64, dBias int64) int {
// TODO: binary search instead of linear search.
for i, n := 0, b.arity(); i < n; i++ {
if dOff < b.dOff(i+1, dBias) {
return i
}
}
// We shouldn't get here, since we validate each node, and don't call this
// function for a DOff greater or equal to DOffMax.
panic("rac: internal error: could not find containing chunk")
}
func (b *rNode) chunk(i int, cBias int64, dBias int64) Chunk {
sTag := b.sTag(i)
tTag := b.tTag(i)
return Chunk{
DRange: b.dOffRange(i, dBias),
CPrimary: b.cOffRange(i, cBias),
CSecondary: b.cOffRange(int(sTag), cBias),
CTertiary: b.cOffRange(int(tTag), cBias),
STag: sTag,
TTag: tTag,
Codec: b.codec(),
}
}
func (b *rNode) valid() bool {
// Check the magic and arity.
if (b[0] != magic[0]) || (b[1] != magic[1]) || (b[2] != magic[2]) || (b[3] == 0) {
return false
}
arity := int(b[3])
size := (16 * arity) + 16
if b[3] != b[size-1] {
return false
}
// Check that the "Reserved (0)" bytes are zero and that the TTag values
// aren't in the reserved range [0xC0, 0xFE).
for i := 0; i < arity; i++ {
if b[(8*i)+6] != 0 {
return false
}
if tTag := b[(8*i)+7]; (0xC0 <= tTag) && (tTag < 0xFE) {
return false
}
}
if b[(8*arity)+6] != 0 {
return false
}
// Check that the Codec is non-zero.
if b[(8*arity)+7] == 0 {
return false
}
// Check that the DPtr values are non-decreasing. The first DPtr value is
// implicitly zero.
prev := u48LE(b[8*1:])
for i := 2; i <= arity; i++ {
curr := u48LE(b[8*i:])
if curr < prev {
return false
}
prev = curr
}
// Check that no CPtr value exceeds CPtrMax (the final CPtr value).
base := (8 * arity) + 8
cPtrMax := u48LE(b[size-8:])
for i := 0; i < arity; i++ {
if cPtr := u48LE(b[(8*i)+base:]); cPtr > cPtrMax {
return false
}
}
// Check the the version is non-zero.
if b[(16*arity)+14] == 0 {
return false
}
// Check the checksum.
checksum := crc32.ChecksumIEEE(b[6:size])
checksum ^= checksum >> 16
if (b[4] != uint8(checksum>>0)) || (b[5] != uint8(checksum>>8)) {
return false
}
// Further checking of the codec, version, COffMax and DOffMax requires
// more context, and is done in loadAndValidate.
return true
}
// Reader reads a RAC file.
//
// Do not modify its exported fields after calling any of its methods.
type Reader struct {
// ReadSeeker is where the RAC-encoded data is read from.
//
// Nil is an invalid value.
ReadSeeker io.ReadSeeker
// CompressedSize is the size of the RAC file.
//
// Zero is an invalid value, as an empty file is not a valid RAC file.
CompressedSize int64
// initialized is set true after the first call on this Reader.
initialized bool
// rootNodeArity is the root node's arity.
rootNodeArity uint8
// needToResolveSeekPosition is whether NextChunk will need to resolve
// seekPosition.
needToResolveSeekPosition bool
// err is the first error encountered. It is sticky: once a non-nil error
// occurs, all public methods will return that error.
err error
// decompressedSize is the size of the RAC file in DSpace.
decompressedSize int64
// rootNodeCOffset is the position of the root node in the RAC file.
rootNodeCOffset int64
// seekPosition gives, if needToResolveSeekPosition is true, the position
// in DSpace that NextChunk needs to find.
seekPosition int64
// The i (as in "the i'th child of currNode") that denotes the next chunk
// to be returned by NextChunk, if a non-empty chunk. If nextChunk equals
// currNode's arity, then currNode is exhausted and calling NextChunk will
// seek to the next node.
nextChunk int32
// The CBias and DBias of currNode.
currNodeCBias int64
currNodeDBias int64
// currNode is the 4096 byte buffer to hold the current node.
currNode rNode
}
func (r *Reader) checkParameters() error {
if r.ReadSeeker == nil {
r.err = errors.New("rac: invalid ReadSeeker")
return r.err
}
if r.CompressedSize < 32 {
r.err = errors.New("rac: invalid CompressedSize")
return r.err
}
return nil
}
func (r *Reader) initialize() error {
if r.err != nil {
return r.err
}
if r.initialized {
return nil
}
r.initialized = true
if err := r.checkParameters(); err != nil {
return err
}
if err := r.findRootNode(); err != nil {
return err
}
if r.currNode.version() != 1 {
r.err = errors.New("rac: unsupported RAC file version")
return r.err
}
return nil
}
func (r *Reader) findRootNode() error {
// Look at the start of the compressed file.
if err := readAt(r.ReadSeeker, r.currNode[:4], 0); err != nil {
r.err = err
return r.err
}
if (r.currNode[0] != magic[0]) ||
(r.currNode[1] != magic[1]) ||
(r.currNode[2] != magic[2]) {
r.err = errors.New("rac: invalid input: missing magic bytes at the start of file")
return r.err
}
if found, err := r.tryRootNode(r.currNode[3], false); err != nil {
return err
} else if found {
return nil
}
// Look at the end of the compressed file.
if err := readAt(r.ReadSeeker, r.currNode[:1], r.CompressedSize-1); err != nil {
r.err = err
return r.err
}
if found, err := r.tryRootNode(r.currNode[0], true); err != nil {
return err
} else if found {
return nil
}
return errors.New("rac: invalid input: missing index root node")
}
func (r *Reader) tryRootNode(arity uint8, fromEnd bool) (found bool, ioErr error) {
if arity == 0 {
return false, nil
}
size := int64(nodeSize(arity))
if r.CompressedSize < size {
return false, nil
}
cOffset := int64(0)
if fromEnd {
cOffset = r.CompressedSize - size
}
if err := r.load(cOffset, arity); err != nil {
return false, err
}
if !r.currNode.valid() {
return false, nil
}
if r.currNode.cPtrMax() != r.CompressedSize {
return false, nil
}
r.needToResolveSeekPosition = true
r.rootNodeCOffset = cOffset
r.rootNodeArity = arity
r.decompressedSize = r.currNode.dPtrMax()
return true, nil
}
// load loads a node from the RAC file into r.currNode. It does not check that
// the result is valid, and the caller should do so if it doesn't already know
// that it is valid.
func (r *Reader) load(cOffset int64, arity uint8) error {
if arity == 0 {
r.err = errors.New("rac: internal error: inconsistent arity")
return r.err
}
size := nodeSize(arity)
if err := readAt(r.ReadSeeker, r.currNode[:size], cOffset); err != nil {
r.err = err
return r.err
}
return nil
}
func (r *Reader) loadAndValidate(cOffset int64,
parentCodec Codec, parentVersion uint8, parentCOffMax int64,
childCBias int64, childDSize int64) error {
if (cOffset < 0) || ((r.CompressedSize - 4) < cOffset) {
r.err = errInvalidIndexNode
return r.err
}
if err := readAt(r.ReadSeeker, r.currNode[:4], cOffset); err != nil {
r.err = err
return r.err
}
arity := r.currNode[3]
if arity == 0 {
r.err = errInvalidIndexNode
return r.err
}
size := int64(nodeSize(arity))
if (r.CompressedSize < size) || ((r.CompressedSize - size) < cOffset) {
r.err = errInvalidIndexNode
return r.err
}
if err := r.load(cOffset, arity); err != nil {
return err
}
if !r.currNode.valid() {
r.err = errInvalidIndexNode
return r.err
}
// Validate the parent and child codec, version, COffMax and DOffMax.
childVersion := r.currNode.version()
if !bitwiseSubset(parentCodec, r.currNode.codec()) ||
(parentVersion < childVersion) ||
(parentCOffMax < (childCBias + r.currNode.cPtrMax())) ||
(childDSize != r.currNode.dPtrMax()) {
r.err = errInvalidIndexNode
return r.err
}
return nil
}
// DecompressedSize returns the total size of the decompressed data.
func (r *Reader) DecompressedSize() (int64, error) {
if err := r.initialize(); err != nil {
return 0, err
}
return r.decompressedSize, nil
}
// SeekToChunkContaining sets up NextChunk to return the chunk containing
// dSpaceOffset. That chunk does not necessarily start at dSpaceOffset.
//
// It is an error to seek to a negative value.
func (r *Reader) SeekToChunkContaining(dSpaceOffset int64) error {
if err := r.initialize(); err != nil {
return err
}
if dSpaceOffset < 0 {
r.err = errors.New("rac: seek: negative position")
return r.err
}
r.needToResolveSeekPosition = true
r.seekPosition = dSpaceOffset
return nil
}
// NextChunk returns the next independently compressed chunk, or io.EOF if
// there are no more chunks.
//
// Empty chunks (those that contain no decompressed data, only metadata) are
// skipped.
func (r *Reader) NextChunk() (Chunk, error) {
if err := r.initialize(); err != nil {
return Chunk{}, err
}
for {
if r.needToResolveSeekPosition {
if r.seekPosition >= r.decompressedSize {
return Chunk{}, io.EOF
}
r.needToResolveSeekPosition = false
if err := r.resolveSeekPosition(); err != nil {
return Chunk{}, err
}
}
for n := int32(r.currNode.arity()); r.nextChunk < n; {
c := r.currNode.chunk(int(r.nextChunk), r.currNodeCBias, r.currNodeDBias)
r.nextChunk++
r.seekPosition = c.DRange[1]
if !c.DRange.Empty() {
return c, nil
}
}
r.needToResolveSeekPosition = true
}
}
func (r *Reader) resolveSeekPosition() error {
// Load the root node. It has already been validated, during initialize.
if err := r.load(r.rootNodeCOffset, r.rootNodeArity); err != nil {
return err
}
// Walk the branch nodes until we find the leaf node containing the
// seekPosition.
cBias := int64(0)
dBias := int64(0)
for {
i := r.currNode.findChunkContaining(r.seekPosition, dBias)
if r.currNode.isLeaf(i) {
r.nextChunk = int32(i)
r.currNodeCBias = cBias
r.currNodeDBias = dBias
return nil
}
parentCodec := r.currNode.codec()
parentVersion := r.currNode.version()
parentCOffMax := cBias + r.currNode.cPtrMax()
childCOffset := r.currNode.cOff(i, cBias)
childCBias := cBias
if sTag := int(r.currNode.sTag(i)); sTag < r.currNode.arity() {
childCBias = r.currNode.cOff(sTag, cBias)
}
childDBias := r.currNode.dOff(i, dBias)
childDSize := r.currNode.dSize(i)
if err := r.loadAndValidate(childCOffset,
parentCodec, parentVersion, parentCOffMax,
childCBias, childDSize); err != nil {
return err
}
cBias = childCBias
dBias = childDBias
}
}
+3 -3
View File
@@ -65,9 +65,6 @@ type Writer struct {
// Zero is an invalid value.
Codec Codec
// initialized is set true after the first AddXxx call.
initialized bool
// IndexLocation is whether the index is at the start or end of the RAC
// file.
//
@@ -127,6 +124,9 @@ type Writer struct {
// each chunk's starting offset will be aligned to a page boundary.
CPageSize uint64
// initialized is set true after the first AddXxx call.
initialized bool
// err is the first error encountered. It is sticky: once a non-nil error
// occurs, all public methods will return that error.
err error
+199 -14
View File
@@ -29,10 +29,35 @@ const bytesPerHexDumpLine = 79
const fakeCodec = Codec(0xEE)
// Note that these exact bytes depends on the zlib encoder's algorithm, but
// there is more than one valid zlib encoding of any given input. This "compare
// to golden output" test is admittedly brittle, as the standard library's zlib
// package's output isn't necessarily stable across Go releases.
var unhex = [256]uint8{
'0': 0x00, '1': 0x01, '2': 0x02, '3': 0x03, '4': 0x04,
'5': 0x05, '6': 0x06, '7': 0x07, '8': 0x08, '9': 0x09,
'A': 0x0A, 'B': 0x0B, 'C': 0x0C, 'D': 0x0D, 'E': 0x0E, 'F': 0x0F,
'a': 0x0A, 'b': 0x0B, 'c': 0x0C, 'd': 0x0D, 'e': 0x0E, 'f': 0x0F,
}
func undoHexDump(s string) (ret []byte) {
for s != "" {
for i := 0; i < 16; i++ {
pos := 10 + 3*i
if i > 7 {
pos++
}
c0 := s[pos+0]
c1 := s[pos+1]
if c0 == ' ' {
break
}
ret = append(ret, (unhex[c0]<<4)|unhex[c1])
}
if n := strings.IndexByte(s, '\n'); n >= 0 {
s = s[n+1:]
} else {
break
}
}
return ret
}
const writerWantEmpty = "" +
"00000000 72 c3 63 01 30 14 00 ff 00 00 00 00 00 00 00 ee |r.c.0...........|\n" +
@@ -179,6 +204,8 @@ func testWriter(iloc IndexLocation, tempFile io.ReadWriter, cPageSize uint64, em
if !empty {
// We ignore errors (assigning them to _) from the AddXxx calls. Any
// non-nil errors are sticky, and should be returned by Close.
//
// These {Aa,Bb,Cc} chunks are also used in the reader test.
res0, _ := w.AddResource([]byte("Rrr"))
res1, _ := w.AddResource([]byte("Ss"))
_ = w.AddChunk(0x11, []byte("Aaa"), 0, 0)
@@ -215,7 +242,7 @@ func testWriter(iloc IndexLocation, tempFile io.ReadWriter, cPageSize uint64, em
return nil
}
func TestWriterMultiLevelIndex(t *testing.T) {
func TestMultiLevelIndex(t *testing.T) {
buf := &bytes.Buffer{}
w := &Writer{
Writer: buf,
@@ -232,6 +259,7 @@ func TestWriterMultiLevelIndex(t *testing.T) {
xRes := OptResource(0)
yRes := OptResource(0)
zRes := OptResource(0)
primaries := []byte(nil)
for i := 0; i < 260; i++ {
secondary := OptResource(0)
tertiary := OptResource(0)
@@ -253,6 +281,7 @@ func TestWriterMultiLevelIndex(t *testing.T) {
}
primary := []byte(fmt.Sprintf("p%02x", i&0xFF))
primaries = append(primaries, primary...)
_ = w.AddChunk(0x10000, primary, secondary, tertiary)
}
@@ -265,17 +294,17 @@ func TestWriterMultiLevelIndex(t *testing.T) {
t.Fatalf("len(encoded): got 0x%X, want 0x%X", got, want)
}
got := hex.Dump(encoded)
got = "" +
got[0x000*bytesPerHexDumpLine:0x008*bytesPerHexDumpLine] +
gotHexDump := hex.Dump(encoded)
gotHexDump = "" +
gotHexDump[0x000*bytesPerHexDumpLine:0x008*bytesPerHexDumpLine] +
"...\n" +
got[0x080*bytesPerHexDumpLine:0x088*bytesPerHexDumpLine] +
gotHexDump[0x080*bytesPerHexDumpLine:0x088*bytesPerHexDumpLine] +
"...\n" +
got[0x100*bytesPerHexDumpLine:0x110*bytesPerHexDumpLine] +
gotHexDump[0x100*bytesPerHexDumpLine:0x110*bytesPerHexDumpLine] +
"...\n" +
got[0x13C*bytesPerHexDumpLine:]
gotHexDump[0x13C*bytesPerHexDumpLine:]
const want = "" +
const wantHexDump = "" +
"00000000 72 c3 63 02 17 fa 00 fe 00 00 fc 00 00 00 00 fe |r.c.............|\n" +
"00000010 00 00 04 01 00 00 00 ee 30 00 00 00 00 00 04 ff |........0.......|\n" +
"00000020 30 10 00 00 00 00 01 ff e2 13 00 00 00 00 01 02 |0...............|\n" +
@@ -315,8 +344,33 @@ func TestWriterMultiLevelIndex(t *testing.T) {
"000013d0 70 66 65 70 66 66 70 30 30 70 30 31 70 30 32 70 |pfepffp00p01p02p|\n" +
"000013e0 30 33 |03|\n"
if got != want {
t.Fatalf("\ngot:\n%s\nwant:\n%s", got, want)
if gotHexDump != wantHexDump {
t.Fatalf("\ngot:\n%s\nwant:\n%s", gotHexDump, wantHexDump)
}
r := &Reader{
ReadSeeker: bytes.NewReader(encoded),
CompressedSize: int64(len(encoded)),
}
gotPrimaries := []byte(nil)
for {
c, err := r.NextChunk()
if err == io.EOF {
break
} else if err != nil {
t.Fatalf("NextChunk: %v", err)
}
p0, p1 := c.CPrimary[0], c.CPrimary[1]
if (0 <= p0) && (p0 <= p1) && (p1 <= int64(len(encoded))) {
primary := encoded[p0:p1]
if len(primary) > 3 {
primary = primary[:3]
}
gotPrimaries = append(gotPrimaries, primary...)
}
}
if !bytes.Equal(gotPrimaries, primaries) {
t.Fatalf("\ngot:\n%s\nwant:\n%s", gotPrimaries, primaries)
}
}
@@ -338,3 +392,134 @@ func TestWriter1000Chunks(t *testing.T) {
t.Fatalf("Close: %v", err)
}
}
func printChunks(chunks []Chunk) string {
ss := make([]string, len(chunks))
for i, c := range chunks {
ss[i] = fmt.Sprintf("D:%#x, C0:%#x, C1:%#x, C2:%#x, S:%#x, T:%#x, C:%#x",
c.DRange, c.CPrimary, c.CSecondary, c.CTertiary, c.STag, c.TTag, c.Codec)
}
return strings.Join(ss, "\n")
}
func TestReader(t *testing.T) {
makeChunk := func() Chunk {
return Chunk{}
}
_ = makeChunk
testCases := []struct {
name string
compressed []byte
}{
{"Empty", undoHexDump(writerWantEmpty)},
{"ILAEnd", undoHexDump(writerWantILAEnd)},
{"ILAEndCPageSize8", undoHexDump(writerWantILAEndCPageSize8)},
{"ILAStart", undoHexDump(writerWantILAStart)},
{"ILAStartCPageSize4", undoHexDump(writerWantILAStartCPageSize4)},
{"ILAStartCPageSize128", undoHexDump(writerWantILAStartCPageSize128)},
}
loop:
for _, tc := range testCases {
snippet := func(r Range) string {
if r.Empty() {
return ""
}
if r[1] > int64(len(tc.compressed)) {
return "CRange goes beyond COffMax"
}
s := tc.compressed[r[0]:r[1]]
if len(s) > 2 {
return string(s[:2]) + "..."
}
return string(s)
}
wantDecompressedSize := int64(0)
wantDescription := ""
if tc.name != "Empty" {
// These {Aa,Bb,Cc} chunks are also used in the writer test.
wantDecompressedSize = 0x11 + 0x22 + 0x44
wantDescription = `` +
`DRangeSize:0x11, C0:"Aa...", C1:"", C2:""` + "\n" +
`DRangeSize:0x22, C0:"Bb...", C1:"Rr...", C2:""` + "\n" +
`DRangeSize:0x44, C0:"Cc...", C1:"Rr...", C2:"Ss..."`
}
r := &Reader{
ReadSeeker: bytes.NewReader(tc.compressed),
CompressedSize: int64(len(tc.compressed)),
}
if gotDecompressedSize, err := r.DecompressedSize(); err != nil {
t.Errorf("%q test case: %v", tc.name, err)
continue loop
} else if gotDecompressedSize != wantDecompressedSize {
t.Errorf("%q test case: DecompressedSize: got %d, want %d",
tc.name, gotDecompressedSize, wantDecompressedSize)
continue loop
}
gotChunks := []Chunk(nil)
description := &bytes.Buffer{}
prevDRange1 := int64(0)
for {
c, err := r.NextChunk()
if err == io.EOF {
break
} else if err != nil {
t.Errorf("%q test case: NextChunk: %v", tc.name, err)
continue loop
}
if c.DRange[0] != prevDRange1 {
t.Errorf("%q test case: NextChunk: DRange[0]: got %d, want %d",
tc.name, c.DRange[0], prevDRange1)
continue loop
}
prevDRange1 = c.DRange[1]
gotChunks = append(gotChunks, c)
if description.Len() > 0 {
description.WriteByte('\n')
}
fmt.Fprintf(description, "DRangeSize:0x%X, C0:%q, C1:%q, C2:%q",
c.DRange.Size(), snippet(c.CPrimary), snippet(c.CSecondary), snippet(c.CTertiary))
}
if tc.name == "Empty" {
if len(gotChunks) != 0 {
t.Errorf("%q test case: NextChunk: got non-empty, want empty", tc.name)
}
continue loop
}
gotDescription := description.String()
if gotDescription != wantDescription {
t.Errorf("%q test case: NextChunk:\n got\n%s\n which is\n%s\n want\n%s",
tc.name, printChunks(gotChunks), gotDescription, wantDescription)
continue loop
}
// NextChunk should return io.EOF.
if _, err := r.NextChunk(); err != io.EOF {
t.Errorf("%q test case: NextChunk: got %v, want io.EOF", tc.name, err)
continue loop
}
if err := r.SeekToChunkContaining(0x30); err != nil {
t.Errorf("%q test case: SeekToChunkContaining: %v", tc.name, err)
continue loop
}
// NextChunk should return the "Bb..." chunk.
if c, err := r.NextChunk(); err != nil {
t.Errorf("%q test case: NextChunk: %v", tc.name, err)
continue loop
} else if got, want := snippet(c.CPrimary), "Bb..."; got != want {
t.Errorf("%q test case: NextChunk: got %q, want %q", tc.name, got, want)
continue loop
}
}
}