mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-18 00:32:11 +01:00
Add a rac.Reader
This commit is contained in:
1 parent
e292a22ccb
commit
861ae95f8c
5 files changed
+819
-24
No files matched your search
+70
-7
@@ -17,11 +17,13 @@ package rac_test
|
||||
import (
|
||||
"bytes"
|
||||
"compress/zlib"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"hash/adler32"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/google/wuffs/lib/rac"
|
||||
)
|
||||
@@ -29,8 +31,8 @@ import (
|
||||
// ExampleILAEnd demonstrates using the low level "rac" package to encode a
|
||||
// RAC+Zlib formatted file with IndexLocationAtEnd.
|
||||
//
|
||||
// The sibling "raczlib" package (TODO) provides a higher level API that is
|
||||
// easier to use.
|
||||
// The sibling "raczlib" package provides a higher level API that is easier to
|
||||
// use.
|
||||
//
|
||||
// See the RAC specification for an explanation of the file format.
|
||||
func ExampleILAEnd() {
|
||||
@@ -91,8 +93,8 @@ func ExampleILAEnd() {
|
||||
// ExampleILAStart demonstrates using the low level "rac" package to encode and
|
||||
// then decode a RAC+Zlib formatted file with IndexLocationAtStart.
|
||||
//
|
||||
// The sibling "raczlib" package (TODO) provides a higher level API that is
|
||||
// easier to use.
|
||||
// The sibling "raczlib" package provides a higher level API that is easier to
|
||||
// use.
|
||||
//
|
||||
// See the RAC specification for an explanation of the file format.
|
||||
func ExampleILAStart() {
|
||||
@@ -151,11 +153,68 @@ func ExampleILAStart() {
|
||||
log.Fatalf("Close: %v", err)
|
||||
}
|
||||
|
||||
fmt.Printf("RAC file:\n%s", hex.Dump(buf.Bytes()))
|
||||
encoded := buf.Bytes()
|
||||
fmt.Printf("RAC file:\n%s\n", hex.Dump(encoded))
|
||||
|
||||
// TODO: decode the encoded bytes (the RAC-formatted bytes) to recover the
|
||||
// Decode the encoded bytes (the RAC-formatted bytes) to recover the
|
||||
// original "One sheep.\nTwo sheep\.Three sheep.\n" source.
|
||||
|
||||
fmt.Printf("Decoded:\n")
|
||||
r := &rac.Reader{
|
||||
ReadSeeker: bytes.NewReader(encoded),
|
||||
CompressedSize: int64(len(encoded)),
|
||||
}
|
||||
zr := io.ReadCloser(nil)
|
||||
for {
|
||||
chunk, err := r.NextChunk()
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
log.Fatalf("NextChunk: %v", err)
|
||||
}
|
||||
if chunk.Codec != rac.CodecZlib {
|
||||
log.Fatalf("unexpected chunk codec")
|
||||
}
|
||||
fmt.Printf("[%2d, %2d): ", chunk.DRange[0], chunk.DRange[1])
|
||||
|
||||
// Parse the RAC+Zlib secondary data. For details, see
|
||||
// https://github.com/google/wuffs/blob/master/doc/spec/rac-spec.md#rac--zlib
|
||||
dict := []byte(nil)
|
||||
if secondary := encoded[chunk.CSecondary[0]:chunk.CSecondary[1]]; len(secondary) > 0 {
|
||||
if len(secondary) < 6 {
|
||||
log.Fatalf("invalid secondary data")
|
||||
}
|
||||
dictLen := int(binary.LittleEndian.Uint16(secondary))
|
||||
secondary = secondary[2:]
|
||||
if (dictLen + 4) > len(secondary) {
|
||||
log.Fatalf("invalid secondary data")
|
||||
}
|
||||
checksum := binary.BigEndian.Uint32(secondary[dictLen:])
|
||||
dict = secondary[:dictLen]
|
||||
if checksum != adler32.Checksum(dict) {
|
||||
log.Fatalf("invalid checksum")
|
||||
}
|
||||
}
|
||||
|
||||
// Decompress the Zlib-encoded primary payload.
|
||||
primary := encoded[chunk.CPrimary[0]:chunk.CPrimary[1]]
|
||||
if zr == nil {
|
||||
if zr, err = zlib.NewReaderDict(bytes.NewReader(primary), dict); err != nil {
|
||||
log.Fatalf("zlib.NewReader: %v", err)
|
||||
}
|
||||
} else if err := zr.(zlib.Resetter).Reset(bytes.NewReader(primary), dict); err != nil {
|
||||
log.Fatalf("zlib.Reader.Reset: %v", err)
|
||||
}
|
||||
if n, err := io.Copy(os.Stdout, zr); err != nil {
|
||||
log.Fatalf("io.Copy: %v", err)
|
||||
} else if n != chunk.DRange.Size() {
|
||||
log.Fatalf("inconsistent DRange size")
|
||||
}
|
||||
if err := zr.Close(); err != nil {
|
||||
log.Fatalf("zlib.Reader.Close: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Note that these exact bytes depends on the zlib encoder's algorithm, but
|
||||
// there is more than one valid zlib encoding of any given input. This
|
||||
// "compare to golden output" test is admittedly brittle, as the standard
|
||||
@@ -189,5 +248,9 @@ func ExampleILAStart() {
|
||||
// 00000070 21 03 90 78 f9 0b e0 02 6e 0a 29 cf 87 31 01 01 |!..x....n.)..1..|
|
||||
// 00000080 00 00 ff ff 18 0c 03 a8 78 f9 0b e0 02 6e 0a c9 |........x....n..|
|
||||
// 00000090 28 4a 4d 85 71 00 01 00 00 ff ff 21 6e 04 66 |(JM.q......!n.f|
|
||||
|
||||
//
|
||||
// Decoded:
|
||||
// [ 0, 11): One sheep.
|
||||
// [11, 22): Two sheep.
|
||||
// [22, 35): Three sheep.
|
||||
}
|
||||
@@ -49,6 +49,8 @@ const (
|
||||
// See the RAC specification for further discussion.
|
||||
type IndexLocation uint8
|
||||
|
||||
const magic = "\x72\xC3\x63"
|
||||
|
||||
const (
|
||||
IndexLocationAtEnd = IndexLocation(0)
|
||||
IndexLocationAtStart = IndexLocation(1)
|
||||
|
||||
@@ -0,0 +1,545 @@
|
||||
// Copyright 2019 The Wuffs Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package rac
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
)
|
||||
|
||||
var (
|
||||
errInvalidIndexNode = errors.New("rac: invalid index node")
|
||||
)
|
||||
|
||||
func u48LE(b []byte) int64 {
|
||||
_ = b[7] // Early bounds check to guarantee safety of reads below.
|
||||
u := uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 |
|
||||
uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
|
||||
return int64(u & 0xFFFFFFFFFFFF)
|
||||
}
|
||||
|
||||
// readAt calls ReadAt if it is available, otherwise it falls back to calling
|
||||
// Seek and then ReadFull. Calling ReadAt is presumably slightly more
|
||||
// efficient, e.g. one syscall instead of two.
|
||||
func readAt(r io.ReadSeeker, p []byte, offset int64) error {
|
||||
if a, ok := r.(io.ReaderAt); ok && false {
|
||||
n, err := a.ReadAt(p, offset)
|
||||
if (n == len(p)) && (err == io.EOF) {
|
||||
err = nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if _, err := r.Seek(offset, io.SeekStart); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := io.ReadFull(r, p)
|
||||
return err
|
||||
}
|
||||
|
||||
// bitwiseSubset returns whether every 1-bit in inner is also set in outer.
|
||||
func bitwiseSubset(outer Codec, inner Codec) bool {
|
||||
return outer == (outer | inner)
|
||||
}
|
||||
|
||||
// Range is the half-open range [low, high). It is invalid for low to be
|
||||
// greater than high.
|
||||
type Range [2]int64
|
||||
|
||||
func (r *Range) Empty() bool { return r[0] == r[1] }
|
||||
func (r *Range) Size() int64 { return r[1] - r[0] }
|
||||
|
||||
// Chunk is a compressed chunk returned by a Reader.
|
||||
//
|
||||
// See the RAC specification for further discussion.
|
||||
type Chunk struct {
|
||||
DRange Range
|
||||
CPrimary Range
|
||||
CSecondary Range
|
||||
CTertiary Range
|
||||
STag uint8
|
||||
TTag uint8
|
||||
Codec Codec
|
||||
}
|
||||
|
||||
// nodeSize returns the size (in CSpace) that a node with the given arity
|
||||
// occupies.
|
||||
func nodeSize(arity uint8) int {
|
||||
return (16 * int(arity)) + 16
|
||||
}
|
||||
|
||||
// rNode is the Reader's representation of a node.
|
||||
//
|
||||
// None of its methods, other than valid, should be called unless valid returns
|
||||
// true.
|
||||
type rNode [4096]byte
|
||||
|
||||
func (b *rNode) arity() int { return int(b[3]) }
|
||||
func (b *rNode) codec() Codec { return Codec(b[(8*int(b[3]))+7]) }
|
||||
func (b *rNode) cPtrMax() int64 { return u48LE(b[(16*int(b[3]))+8:]) }
|
||||
func (b *rNode) dPtrMax() int64 { return u48LE(b[8*int(b[3]):]) }
|
||||
func (b *rNode) version() uint8 { return b[(16*int(b[3]))+14] }
|
||||
|
||||
func (b *rNode) cLen(i int) uint8 {
|
||||
base := (8 * int(b[3])) + 14
|
||||
return b[(8*i)+base]
|
||||
}
|
||||
|
||||
func (b *rNode) cOff(i int, cBias int64) int64 {
|
||||
base := (8 * int(b[3])) + 8
|
||||
return cBias + u48LE(b[(8*i)+base:])
|
||||
}
|
||||
|
||||
func (b *rNode) cOffRange(i int, cBias int64) Range {
|
||||
m := cBias + b.cPtrMax()
|
||||
if i >= b.arity() {
|
||||
return Range{m, m}
|
||||
}
|
||||
cOff := b.cOff(i, cBias)
|
||||
if cLen := b.cLen(i); cLen != 0 {
|
||||
if n := cOff + (int64(cLen) * 1024); m > n {
|
||||
m = n
|
||||
}
|
||||
}
|
||||
return Range{cOff, m}
|
||||
}
|
||||
|
||||
func (b *rNode) dOff(i int, dBias int64) int64 {
|
||||
if i == 0 {
|
||||
return dBias
|
||||
}
|
||||
return dBias + u48LE(b[8*i:])
|
||||
}
|
||||
|
||||
func (b *rNode) dOffRange(i int, dBias int64) Range {
|
||||
return Range{b.dOff(i, dBias), b.dOff(i+1, dBias)}
|
||||
}
|
||||
|
||||
func (b *rNode) dSize(i int) int64 {
|
||||
x := int64(0)
|
||||
if i > 0 {
|
||||
x = u48LE(b[8*i:])
|
||||
}
|
||||
return u48LE(b[(8*i)+8:]) - x
|
||||
}
|
||||
|
||||
func (b *rNode) sTag(i int) uint8 {
|
||||
base := (8 * int(b[3])) + 15
|
||||
return b[(8*i)+base]
|
||||
}
|
||||
|
||||
func (b *rNode) tTag(i int) uint8 {
|
||||
return b[(8*i)+7]
|
||||
}
|
||||
|
||||
func (b *rNode) isLeaf(i int) bool {
|
||||
return b[(8*i)+7] != 0xFE
|
||||
}
|
||||
|
||||
func (b *rNode) findChunkContaining(dOff int64, dBias int64) int {
|
||||
// TODO: binary search instead of linear search.
|
||||
for i, n := 0, b.arity(); i < n; i++ {
|
||||
if dOff < b.dOff(i+1, dBias) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
// We shouldn't get here, since we validate each node, and don't call this
|
||||
// function for a DOff greater or equal to DOffMax.
|
||||
panic("rac: internal error: could not find containing chunk")
|
||||
}
|
||||
|
||||
func (b *rNode) chunk(i int, cBias int64, dBias int64) Chunk {
|
||||
sTag := b.sTag(i)
|
||||
tTag := b.tTag(i)
|
||||
return Chunk{
|
||||
DRange: b.dOffRange(i, dBias),
|
||||
CPrimary: b.cOffRange(i, cBias),
|
||||
CSecondary: b.cOffRange(int(sTag), cBias),
|
||||
CTertiary: b.cOffRange(int(tTag), cBias),
|
||||
STag: sTag,
|
||||
TTag: tTag,
|
||||
Codec: b.codec(),
|
||||
}
|
||||
}
|
||||
|
||||
func (b *rNode) valid() bool {
|
||||
// Check the magic and arity.
|
||||
if (b[0] != magic[0]) || (b[1] != magic[1]) || (b[2] != magic[2]) || (b[3] == 0) {
|
||||
return false
|
||||
}
|
||||
arity := int(b[3])
|
||||
size := (16 * arity) + 16
|
||||
if b[3] != b[size-1] {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check that the "Reserved (0)" bytes are zero and that the TTag values
|
||||
// aren't in the reserved range [0xC0, 0xFE).
|
||||
for i := 0; i < arity; i++ {
|
||||
if b[(8*i)+6] != 0 {
|
||||
return false
|
||||
}
|
||||
if tTag := b[(8*i)+7]; (0xC0 <= tTag) && (tTag < 0xFE) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if b[(8*arity)+6] != 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check that the Codec is non-zero.
|
||||
if b[(8*arity)+7] == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check that the DPtr values are non-decreasing. The first DPtr value is
|
||||
// implicitly zero.
|
||||
prev := u48LE(b[8*1:])
|
||||
for i := 2; i <= arity; i++ {
|
||||
curr := u48LE(b[8*i:])
|
||||
if curr < prev {
|
||||
return false
|
||||
}
|
||||
prev = curr
|
||||
}
|
||||
|
||||
// Check that no CPtr value exceeds CPtrMax (the final CPtr value).
|
||||
base := (8 * arity) + 8
|
||||
cPtrMax := u48LE(b[size-8:])
|
||||
for i := 0; i < arity; i++ {
|
||||
if cPtr := u48LE(b[(8*i)+base:]); cPtr > cPtrMax {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Check the the version is non-zero.
|
||||
if b[(16*arity)+14] == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check the checksum.
|
||||
checksum := crc32.ChecksumIEEE(b[6:size])
|
||||
checksum ^= checksum >> 16
|
||||
if (b[4] != uint8(checksum>>0)) || (b[5] != uint8(checksum>>8)) {
|
||||
return false
|
||||
}
|
||||
|
||||
// Further checking of the codec, version, COffMax and DOffMax requires
|
||||
// more context, and is done in loadAndValidate.
|
||||
return true
|
||||
}
|
||||
|
||||
// Reader reads a RAC file.
|
||||
//
|
||||
// Do not modify its exported fields after calling any of its methods.
|
||||
type Reader struct {
|
||||
// ReadSeeker is where the RAC-encoded data is read from.
|
||||
//
|
||||
// Nil is an invalid value.
|
||||
ReadSeeker io.ReadSeeker
|
||||
|
||||
// CompressedSize is the size of the RAC file.
|
||||
//
|
||||
// Zero is an invalid value, as an empty file is not a valid RAC file.
|
||||
CompressedSize int64
|
||||
|
||||
// initialized is set true after the first call on this Reader.
|
||||
initialized bool
|
||||
|
||||
// rootNodeArity is the root node's arity.
|
||||
rootNodeArity uint8
|
||||
|
||||
// needToResolveSeekPosition is whether NextChunk will need to resolve
|
||||
// seekPosition.
|
||||
needToResolveSeekPosition bool
|
||||
|
||||
// err is the first error encountered. It is sticky: once a non-nil error
|
||||
// occurs, all public methods will return that error.
|
||||
err error
|
||||
|
||||
// decompressedSize is the size of the RAC file in DSpace.
|
||||
decompressedSize int64
|
||||
|
||||
// rootNodeCOffset is the position of the root node in the RAC file.
|
||||
rootNodeCOffset int64
|
||||
|
||||
// seekPosition gives, if needToResolveSeekPosition is true, the position
|
||||
// in DSpace that NextChunk needs to find.
|
||||
seekPosition int64
|
||||
|
||||
// The i (as in "the i'th child of currNode") that denotes the next chunk
|
||||
// to be returned by NextChunk, if a non-empty chunk. If nextChunk equals
|
||||
// currNode's arity, then currNode is exhausted and calling NextChunk will
|
||||
// seek to the next node.
|
||||
nextChunk int32
|
||||
|
||||
// The CBias and DBias of currNode.
|
||||
currNodeCBias int64
|
||||
currNodeDBias int64
|
||||
|
||||
// currNode is the 4096 byte buffer to hold the current node.
|
||||
currNode rNode
|
||||
}
|
||||
|
||||
func (r *Reader) checkParameters() error {
|
||||
if r.ReadSeeker == nil {
|
||||
r.err = errors.New("rac: invalid ReadSeeker")
|
||||
return r.err
|
||||
}
|
||||
if r.CompressedSize < 32 {
|
||||
r.err = errors.New("rac: invalid CompressedSize")
|
||||
return r.err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) initialize() error {
|
||||
if r.err != nil {
|
||||
return r.err
|
||||
}
|
||||
if r.initialized {
|
||||
return nil
|
||||
}
|
||||
r.initialized = true
|
||||
|
||||
if err := r.checkParameters(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := r.findRootNode(); err != nil {
|
||||
return err
|
||||
}
|
||||
if r.currNode.version() != 1 {
|
||||
r.err = errors.New("rac: unsupported RAC file version")
|
||||
return r.err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) findRootNode() error {
|
||||
// Look at the start of the compressed file.
|
||||
if err := readAt(r.ReadSeeker, r.currNode[:4], 0); err != nil {
|
||||
r.err = err
|
||||
return r.err
|
||||
}
|
||||
if (r.currNode[0] != magic[0]) ||
|
||||
(r.currNode[1] != magic[1]) ||
|
||||
(r.currNode[2] != magic[2]) {
|
||||
r.err = errors.New("rac: invalid input: missing magic bytes at the start of file")
|
||||
return r.err
|
||||
}
|
||||
if found, err := r.tryRootNode(r.currNode[3], false); err != nil {
|
||||
return err
|
||||
} else if found {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Look at the end of the compressed file.
|
||||
if err := readAt(r.ReadSeeker, r.currNode[:1], r.CompressedSize-1); err != nil {
|
||||
r.err = err
|
||||
return r.err
|
||||
}
|
||||
if found, err := r.tryRootNode(r.currNode[0], true); err != nil {
|
||||
return err
|
||||
} else if found {
|
||||
return nil
|
||||
}
|
||||
|
||||
return errors.New("rac: invalid input: missing index root node")
|
||||
}
|
||||
|
||||
func (r *Reader) tryRootNode(arity uint8, fromEnd bool) (found bool, ioErr error) {
|
||||
if arity == 0 {
|
||||
return false, nil
|
||||
}
|
||||
size := int64(nodeSize(arity))
|
||||
if r.CompressedSize < size {
|
||||
return false, nil
|
||||
}
|
||||
cOffset := int64(0)
|
||||
if fromEnd {
|
||||
cOffset = r.CompressedSize - size
|
||||
}
|
||||
if err := r.load(cOffset, arity); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !r.currNode.valid() {
|
||||
return false, nil
|
||||
}
|
||||
if r.currNode.cPtrMax() != r.CompressedSize {
|
||||
return false, nil
|
||||
}
|
||||
r.needToResolveSeekPosition = true
|
||||
r.rootNodeCOffset = cOffset
|
||||
r.rootNodeArity = arity
|
||||
r.decompressedSize = r.currNode.dPtrMax()
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// load loads a node from the RAC file into r.currNode. It does not check that
|
||||
// the result is valid, and the caller should do so if it doesn't already know
|
||||
// that it is valid.
|
||||
func (r *Reader) load(cOffset int64, arity uint8) error {
|
||||
if arity == 0 {
|
||||
r.err = errors.New("rac: internal error: inconsistent arity")
|
||||
return r.err
|
||||
}
|
||||
size := nodeSize(arity)
|
||||
if err := readAt(r.ReadSeeker, r.currNode[:size], cOffset); err != nil {
|
||||
r.err = err
|
||||
return r.err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) loadAndValidate(cOffset int64,
|
||||
parentCodec Codec, parentVersion uint8, parentCOffMax int64,
|
||||
childCBias int64, childDSize int64) error {
|
||||
|
||||
if (cOffset < 0) || ((r.CompressedSize - 4) < cOffset) {
|
||||
r.err = errInvalidIndexNode
|
||||
return r.err
|
||||
}
|
||||
if err := readAt(r.ReadSeeker, r.currNode[:4], cOffset); err != nil {
|
||||
r.err = err
|
||||
return r.err
|
||||
}
|
||||
arity := r.currNode[3]
|
||||
if arity == 0 {
|
||||
r.err = errInvalidIndexNode
|
||||
return r.err
|
||||
}
|
||||
size := int64(nodeSize(arity))
|
||||
if (r.CompressedSize < size) || ((r.CompressedSize - size) < cOffset) {
|
||||
r.err = errInvalidIndexNode
|
||||
return r.err
|
||||
}
|
||||
if err := r.load(cOffset, arity); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !r.currNode.valid() {
|
||||
r.err = errInvalidIndexNode
|
||||
return r.err
|
||||
}
|
||||
|
||||
// Validate the parent and child codec, version, COffMax and DOffMax.
|
||||
childVersion := r.currNode.version()
|
||||
if !bitwiseSubset(parentCodec, r.currNode.codec()) ||
|
||||
(parentVersion < childVersion) ||
|
||||
(parentCOffMax < (childCBias + r.currNode.cPtrMax())) ||
|
||||
(childDSize != r.currNode.dPtrMax()) {
|
||||
r.err = errInvalidIndexNode
|
||||
return r.err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DecompressedSize returns the total size of the decompressed data.
|
||||
func (r *Reader) DecompressedSize() (int64, error) {
|
||||
if err := r.initialize(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return r.decompressedSize, nil
|
||||
}
|
||||
|
||||
// SeekToChunkContaining sets up NextChunk to return the chunk containing
|
||||
// dSpaceOffset. That chunk does not necessarily start at dSpaceOffset.
|
||||
//
|
||||
// It is an error to seek to a negative value.
|
||||
func (r *Reader) SeekToChunkContaining(dSpaceOffset int64) error {
|
||||
if err := r.initialize(); err != nil {
|
||||
return err
|
||||
}
|
||||
if dSpaceOffset < 0 {
|
||||
r.err = errors.New("rac: seek: negative position")
|
||||
return r.err
|
||||
}
|
||||
r.needToResolveSeekPosition = true
|
||||
r.seekPosition = dSpaceOffset
|
||||
return nil
|
||||
}
|
||||
|
||||
// NextChunk returns the next independently compressed chunk, or io.EOF if
|
||||
// there are no more chunks.
|
||||
//
|
||||
// Empty chunks (those that contain no decompressed data, only metadata) are
|
||||
// skipped.
|
||||
func (r *Reader) NextChunk() (Chunk, error) {
|
||||
if err := r.initialize(); err != nil {
|
||||
return Chunk{}, err
|
||||
}
|
||||
for {
|
||||
if r.needToResolveSeekPosition {
|
||||
if r.seekPosition >= r.decompressedSize {
|
||||
return Chunk{}, io.EOF
|
||||
}
|
||||
r.needToResolveSeekPosition = false
|
||||
if err := r.resolveSeekPosition(); err != nil {
|
||||
return Chunk{}, err
|
||||
}
|
||||
}
|
||||
for n := int32(r.currNode.arity()); r.nextChunk < n; {
|
||||
c := r.currNode.chunk(int(r.nextChunk), r.currNodeCBias, r.currNodeDBias)
|
||||
r.nextChunk++
|
||||
r.seekPosition = c.DRange[1]
|
||||
if !c.DRange.Empty() {
|
||||
return c, nil
|
||||
}
|
||||
}
|
||||
r.needToResolveSeekPosition = true
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Reader) resolveSeekPosition() error {
|
||||
// Load the root node. It has already been validated, during initialize.
|
||||
if err := r.load(r.rootNodeCOffset, r.rootNodeArity); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Walk the branch nodes until we find the leaf node containing the
|
||||
// seekPosition.
|
||||
cBias := int64(0)
|
||||
dBias := int64(0)
|
||||
for {
|
||||
i := r.currNode.findChunkContaining(r.seekPosition, dBias)
|
||||
if r.currNode.isLeaf(i) {
|
||||
r.nextChunk = int32(i)
|
||||
r.currNodeCBias = cBias
|
||||
r.currNodeDBias = dBias
|
||||
return nil
|
||||
}
|
||||
|
||||
parentCodec := r.currNode.codec()
|
||||
parentVersion := r.currNode.version()
|
||||
parentCOffMax := cBias + r.currNode.cPtrMax()
|
||||
childCOffset := r.currNode.cOff(i, cBias)
|
||||
childCBias := cBias
|
||||
if sTag := int(r.currNode.sTag(i)); sTag < r.currNode.arity() {
|
||||
childCBias = r.currNode.cOff(sTag, cBias)
|
||||
}
|
||||
childDBias := r.currNode.dOff(i, dBias)
|
||||
childDSize := r.currNode.dSize(i)
|
||||
|
||||
if err := r.loadAndValidate(childCOffset,
|
||||
parentCodec, parentVersion, parentCOffMax,
|
||||
childCBias, childDSize); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cBias = childCBias
|
||||
dBias = childDBias
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -65,9 +65,6 @@ type Writer struct {
|
||||
// Zero is an invalid value.
|
||||
Codec Codec
|
||||
|
||||
// initialized is set true after the first AddXxx call.
|
||||
initialized bool
|
||||
|
||||
// IndexLocation is whether the index is at the start or end of the RAC
|
||||
// file.
|
||||
//
|
||||
@@ -127,6 +124,9 @@ type Writer struct {
|
||||
// each chunk's starting offset will be aligned to a page boundary.
|
||||
CPageSize uint64
|
||||
|
||||
// initialized is set true after the first AddXxx call.
|
||||
initialized bool
|
||||
|
||||
// err is the first error encountered. It is sticky: once a non-nil error
|
||||
// occurs, all public methods will return that error.
|
||||
err error
|
||||
|
||||
+199
-14
@@ -29,10 +29,35 @@ const bytesPerHexDumpLine = 79
|
||||
|
||||
const fakeCodec = Codec(0xEE)
|
||||
|
||||
// Note that these exact bytes depends on the zlib encoder's algorithm, but
|
||||
// there is more than one valid zlib encoding of any given input. This "compare
|
||||
// to golden output" test is admittedly brittle, as the standard library's zlib
|
||||
// package's output isn't necessarily stable across Go releases.
|
||||
var unhex = [256]uint8{
|
||||
'0': 0x00, '1': 0x01, '2': 0x02, '3': 0x03, '4': 0x04,
|
||||
'5': 0x05, '6': 0x06, '7': 0x07, '8': 0x08, '9': 0x09,
|
||||
'A': 0x0A, 'B': 0x0B, 'C': 0x0C, 'D': 0x0D, 'E': 0x0E, 'F': 0x0F,
|
||||
'a': 0x0A, 'b': 0x0B, 'c': 0x0C, 'd': 0x0D, 'e': 0x0E, 'f': 0x0F,
|
||||
}
|
||||
|
||||
func undoHexDump(s string) (ret []byte) {
|
||||
for s != "" {
|
||||
for i := 0; i < 16; i++ {
|
||||
pos := 10 + 3*i
|
||||
if i > 7 {
|
||||
pos++
|
||||
}
|
||||
c0 := s[pos+0]
|
||||
c1 := s[pos+1]
|
||||
if c0 == ' ' {
|
||||
break
|
||||
}
|
||||
ret = append(ret, (unhex[c0]<<4)|unhex[c1])
|
||||
}
|
||||
if n := strings.IndexByte(s, '\n'); n >= 0 {
|
||||
s = s[n+1:]
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
const writerWantEmpty = "" +
|
||||
"00000000 72 c3 63 01 30 14 00 ff 00 00 00 00 00 00 00 ee |r.c.0...........|\n" +
|
||||
@@ -179,6 +204,8 @@ func testWriter(iloc IndexLocation, tempFile io.ReadWriter, cPageSize uint64, em
|
||||
if !empty {
|
||||
// We ignore errors (assigning them to _) from the AddXxx calls. Any
|
||||
// non-nil errors are sticky, and should be returned by Close.
|
||||
//
|
||||
// These {Aa,Bb,Cc} chunks are also used in the reader test.
|
||||
res0, _ := w.AddResource([]byte("Rrr"))
|
||||
res1, _ := w.AddResource([]byte("Ss"))
|
||||
_ = w.AddChunk(0x11, []byte("Aaa"), 0, 0)
|
||||
@@ -215,7 +242,7 @@ func testWriter(iloc IndexLocation, tempFile io.ReadWriter, cPageSize uint64, em
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestWriterMultiLevelIndex(t *testing.T) {
|
||||
func TestMultiLevelIndex(t *testing.T) {
|
||||
buf := &bytes.Buffer{}
|
||||
w := &Writer{
|
||||
Writer: buf,
|
||||
@@ -232,6 +259,7 @@ func TestWriterMultiLevelIndex(t *testing.T) {
|
||||
xRes := OptResource(0)
|
||||
yRes := OptResource(0)
|
||||
zRes := OptResource(0)
|
||||
primaries := []byte(nil)
|
||||
for i := 0; i < 260; i++ {
|
||||
secondary := OptResource(0)
|
||||
tertiary := OptResource(0)
|
||||
@@ -253,6 +281,7 @@ func TestWriterMultiLevelIndex(t *testing.T) {
|
||||
}
|
||||
|
||||
primary := []byte(fmt.Sprintf("p%02x", i&0xFF))
|
||||
primaries = append(primaries, primary...)
|
||||
_ = w.AddChunk(0x10000, primary, secondary, tertiary)
|
||||
}
|
||||
|
||||
@@ -265,17 +294,17 @@ func TestWriterMultiLevelIndex(t *testing.T) {
|
||||
t.Fatalf("len(encoded): got 0x%X, want 0x%X", got, want)
|
||||
}
|
||||
|
||||
got := hex.Dump(encoded)
|
||||
got = "" +
|
||||
got[0x000*bytesPerHexDumpLine:0x008*bytesPerHexDumpLine] +
|
||||
gotHexDump := hex.Dump(encoded)
|
||||
gotHexDump = "" +
|
||||
gotHexDump[0x000*bytesPerHexDumpLine:0x008*bytesPerHexDumpLine] +
|
||||
"...\n" +
|
||||
got[0x080*bytesPerHexDumpLine:0x088*bytesPerHexDumpLine] +
|
||||
gotHexDump[0x080*bytesPerHexDumpLine:0x088*bytesPerHexDumpLine] +
|
||||
"...\n" +
|
||||
got[0x100*bytesPerHexDumpLine:0x110*bytesPerHexDumpLine] +
|
||||
gotHexDump[0x100*bytesPerHexDumpLine:0x110*bytesPerHexDumpLine] +
|
||||
"...\n" +
|
||||
got[0x13C*bytesPerHexDumpLine:]
|
||||
gotHexDump[0x13C*bytesPerHexDumpLine:]
|
||||
|
||||
const want = "" +
|
||||
const wantHexDump = "" +
|
||||
"00000000 72 c3 63 02 17 fa 00 fe 00 00 fc 00 00 00 00 fe |r.c.............|\n" +
|
||||
"00000010 00 00 04 01 00 00 00 ee 30 00 00 00 00 00 04 ff |........0.......|\n" +
|
||||
"00000020 30 10 00 00 00 00 01 ff e2 13 00 00 00 00 01 02 |0...............|\n" +
|
||||
@@ -315,8 +344,33 @@ func TestWriterMultiLevelIndex(t *testing.T) {
|
||||
"000013d0 70 66 65 70 66 66 70 30 30 70 30 31 70 30 32 70 |pfepffp00p01p02p|\n" +
|
||||
"000013e0 30 33 |03|\n"
|
||||
|
||||
if got != want {
|
||||
t.Fatalf("\ngot:\n%s\nwant:\n%s", got, want)
|
||||
if gotHexDump != wantHexDump {
|
||||
t.Fatalf("\ngot:\n%s\nwant:\n%s", gotHexDump, wantHexDump)
|
||||
}
|
||||
|
||||
r := &Reader{
|
||||
ReadSeeker: bytes.NewReader(encoded),
|
||||
CompressedSize: int64(len(encoded)),
|
||||
}
|
||||
gotPrimaries := []byte(nil)
|
||||
for {
|
||||
c, err := r.NextChunk()
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
t.Fatalf("NextChunk: %v", err)
|
||||
}
|
||||
p0, p1 := c.CPrimary[0], c.CPrimary[1]
|
||||
if (0 <= p0) && (p0 <= p1) && (p1 <= int64(len(encoded))) {
|
||||
primary := encoded[p0:p1]
|
||||
if len(primary) > 3 {
|
||||
primary = primary[:3]
|
||||
}
|
||||
gotPrimaries = append(gotPrimaries, primary...)
|
||||
}
|
||||
}
|
||||
if !bytes.Equal(gotPrimaries, primaries) {
|
||||
t.Fatalf("\ngot:\n%s\nwant:\n%s", gotPrimaries, primaries)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -338,3 +392,134 @@ func TestWriter1000Chunks(t *testing.T) {
|
||||
t.Fatalf("Close: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func printChunks(chunks []Chunk) string {
|
||||
ss := make([]string, len(chunks))
|
||||
for i, c := range chunks {
|
||||
ss[i] = fmt.Sprintf("D:%#x, C0:%#x, C1:%#x, C2:%#x, S:%#x, T:%#x, C:%#x",
|
||||
c.DRange, c.CPrimary, c.CSecondary, c.CTertiary, c.STag, c.TTag, c.Codec)
|
||||
}
|
||||
return strings.Join(ss, "\n")
|
||||
}
|
||||
|
||||
func TestReader(t *testing.T) {
|
||||
makeChunk := func() Chunk {
|
||||
return Chunk{}
|
||||
}
|
||||
_ = makeChunk
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
compressed []byte
|
||||
}{
|
||||
{"Empty", undoHexDump(writerWantEmpty)},
|
||||
{"ILAEnd", undoHexDump(writerWantILAEnd)},
|
||||
{"ILAEndCPageSize8", undoHexDump(writerWantILAEndCPageSize8)},
|
||||
{"ILAStart", undoHexDump(writerWantILAStart)},
|
||||
{"ILAStartCPageSize4", undoHexDump(writerWantILAStartCPageSize4)},
|
||||
{"ILAStartCPageSize128", undoHexDump(writerWantILAStartCPageSize128)},
|
||||
}
|
||||
|
||||
loop:
|
||||
for _, tc := range testCases {
|
||||
snippet := func(r Range) string {
|
||||
if r.Empty() {
|
||||
return ""
|
||||
}
|
||||
if r[1] > int64(len(tc.compressed)) {
|
||||
return "CRange goes beyond COffMax"
|
||||
}
|
||||
s := tc.compressed[r[0]:r[1]]
|
||||
if len(s) > 2 {
|
||||
return string(s[:2]) + "..."
|
||||
}
|
||||
return string(s)
|
||||
}
|
||||
|
||||
wantDecompressedSize := int64(0)
|
||||
wantDescription := ""
|
||||
if tc.name != "Empty" {
|
||||
// These {Aa,Bb,Cc} chunks are also used in the writer test.
|
||||
wantDecompressedSize = 0x11 + 0x22 + 0x44
|
||||
wantDescription = `` +
|
||||
`DRangeSize:0x11, C0:"Aa...", C1:"", C2:""` + "\n" +
|
||||
`DRangeSize:0x22, C0:"Bb...", C1:"Rr...", C2:""` + "\n" +
|
||||
`DRangeSize:0x44, C0:"Cc...", C1:"Rr...", C2:"Ss..."`
|
||||
}
|
||||
|
||||
r := &Reader{
|
||||
ReadSeeker: bytes.NewReader(tc.compressed),
|
||||
CompressedSize: int64(len(tc.compressed)),
|
||||
}
|
||||
|
||||
if gotDecompressedSize, err := r.DecompressedSize(); err != nil {
|
||||
t.Errorf("%q test case: %v", tc.name, err)
|
||||
continue loop
|
||||
} else if gotDecompressedSize != wantDecompressedSize {
|
||||
t.Errorf("%q test case: DecompressedSize: got %d, want %d",
|
||||
tc.name, gotDecompressedSize, wantDecompressedSize)
|
||||
continue loop
|
||||
}
|
||||
|
||||
gotChunks := []Chunk(nil)
|
||||
description := &bytes.Buffer{}
|
||||
prevDRange1 := int64(0)
|
||||
for {
|
||||
c, err := r.NextChunk()
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
t.Errorf("%q test case: NextChunk: %v", tc.name, err)
|
||||
continue loop
|
||||
}
|
||||
|
||||
if c.DRange[0] != prevDRange1 {
|
||||
t.Errorf("%q test case: NextChunk: DRange[0]: got %d, want %d",
|
||||
tc.name, c.DRange[0], prevDRange1)
|
||||
continue loop
|
||||
}
|
||||
prevDRange1 = c.DRange[1]
|
||||
|
||||
gotChunks = append(gotChunks, c)
|
||||
if description.Len() > 0 {
|
||||
description.WriteByte('\n')
|
||||
}
|
||||
fmt.Fprintf(description, "DRangeSize:0x%X, C0:%q, C1:%q, C2:%q",
|
||||
c.DRange.Size(), snippet(c.CPrimary), snippet(c.CSecondary), snippet(c.CTertiary))
|
||||
}
|
||||
|
||||
if tc.name == "Empty" {
|
||||
if len(gotChunks) != 0 {
|
||||
t.Errorf("%q test case: NextChunk: got non-empty, want empty", tc.name)
|
||||
}
|
||||
continue loop
|
||||
}
|
||||
|
||||
gotDescription := description.String()
|
||||
if gotDescription != wantDescription {
|
||||
t.Errorf("%q test case: NextChunk:\n got\n%s\n which is\n%s\n want\n%s",
|
||||
tc.name, printChunks(gotChunks), gotDescription, wantDescription)
|
||||
continue loop
|
||||
}
|
||||
|
||||
// NextChunk should return io.EOF.
|
||||
if _, err := r.NextChunk(); err != io.EOF {
|
||||
t.Errorf("%q test case: NextChunk: got %v, want io.EOF", tc.name, err)
|
||||
continue loop
|
||||
}
|
||||
|
||||
if err := r.SeekToChunkContaining(0x30); err != nil {
|
||||
t.Errorf("%q test case: SeekToChunkContaining: %v", tc.name, err)
|
||||
continue loop
|
||||
}
|
||||
|
||||
// NextChunk should return the "Bb..." chunk.
|
||||
if c, err := r.NextChunk(); err != nil {
|
||||
t.Errorf("%q test case: NextChunk: %v", tc.name, err)
|
||||
continue loop
|
||||
} else if got, want := snippet(c.CPrimary), "Bb..."; got != want {
|
||||
t.Errorf("%q test case: NextChunk: got %q, want %q", tc.name, got, want)
|
||||
continue loop
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user