Files
wuffs/lib/rac/chunk_writer.go
T
Nigel Tao 74a6ea2fcd all: relicense from Apache2 to dual Apache2/MIT
The LICENSE has changed from a single license (Apache2) to a dual
license (Apache2 or MIT, at your option).

If you've previously used Wuffs under the Apache2 license, you are able
to keep doing so. No further action required by you.

The two files listed below (previous releases) are exceptions, as they
are older snapshots, frozen in time. Their license has not changed:
release/c/wuffs-v0.2.c
release/c/wuffs-v0.3.c

Every name in the AUTHORS file, other than Google Inc., has signed
Google's CLA (Contributor License Agreement).
2023-10-06 10:00:59 +11:00

802 lines
22 KiB
Go

// Copyright 2019 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
package rac
import (
"errors"
"hash/crc32"
"io"
"sort"
)
func btoi(b bool) int {
if b {
return 1
}
return 0
}
func isZeroOrAPowerOf2(x uint64) bool {
return (x & (x - 1)) == 0
}
func putU64LE(b []byte, v uint64) {
_ = b[7] // Early bounds check to guarantee safety of writes below.
b[0] = byte(v)
b[1] = byte(v >> 8)
b[2] = byte(v >> 16)
b[3] = byte(v >> 24)
b[4] = byte(v >> 32)
b[5] = byte(v >> 40)
b[6] = byte(v >> 48)
b[7] = byte(v >> 56)
}
// OptResource is an option type, optionally holding a ChunkWriter-specific
// identifier for a shared resource.
//
// Zero means that the option is not taken: no shared resource is used.
type OptResource uint32
// ChunkWriter provides a relatively simple way to write a RAC file - one that
// is created starting from nothing, as opposed to incrementally modifying an
// existing RAC file.
//
// Other packages may provide a more flexible (and more complicated) way to
// write or append to RAC files, but that is out of scope of this package.
//
// Do not modify its exported fields after calling any of its methods.
type ChunkWriter struct {
// Writer is where the RAC-encoded data is written to.
//
// Nil is an invalid value.
Writer io.Writer
// IndexLocation is whether the index is at the start or end of the RAC
// file.
//
// See the RAC specification for further discussion.
//
// The zero value of this field is IndexLocationAtEnd: a one pass encoding.
IndexLocation IndexLocation
// TempFile is a temporary file to use for a two pass encoding. The field
// name says "file" but it need not be a real file, in the operating system
// sense.
//
// For IndexLocationAtEnd, this must be nil. For IndexLocationAtStart, this
// must be non-nil.
//
// It does not need to implement io.Seeker, if it supports separate read
// and write positions (e.g. it is a bytes.Buffer). If it does implement
// io.Seeker (e.g. it is an os.File), its current position will be noted
// (via SeekCurrent), then it will be written to (via the
// ChunkWriter.AddXxx methods), then its position will be reset (via
// SeekSet), then it will be read from (via the ChunkWriter.Close method).
//
// The ChunkWriter does not call TempFile.Close even if that method exists
// (e.g. the TempFile is an os.File). In that case, closing the temporary
// file (and deleting it) after the ChunkWriter is closed is the
// responsibility of the ChunkWriter caller, not the ChunkWriter itself.
TempFile io.ReadWriter
// CPageSize guides padding the output to minimize the number of pages that
// each chunk occupies (in what the RAC spec calls CSpace).
//
// It must either be zero (which means no padding is inserted) or a power
// of 2, and no larger than MaxSize.
//
// For example, suppose that CSpace is partitioned into 1024-byte pages,
// that 1000 bytes have already been written to the output, and the next
// chunk is 1500 bytes long.
//
// - With no padding (i.e. with CPageSize set to 0), this chunk will
// occupy the half-open range [1000, 2500) in CSpace, which straddles
// three 1024-byte pages: the page [0, 1024), the page [1024, 2048) and
// the page [2048, 3072). Call those pages p0, p1 and p2.
//
// - With padding (i.e. with CPageSize set to 1024), 24 bytes of zeroes
// are first inserted so that this chunk occupies the half-open range
// [1024, 2524) in CSpace, which straddles only two pages (p1 and p2).
//
// This concept is similar, but not identical, to alignment. Even with a
// non-zero CPageSize, chunk start offsets are not necessarily aligned to
// page boundaries. For example, suppose that the chunk size was only 1040
// bytes, not 1500 bytes. No padding will be inserted, as both [1000, 2040)
// and [1024, 2064) straddle two pages: either pages p0 and p1, or pages p1
// and p2.
//
// Nonetheless, if all chunks (or all but the final chunk) have a size
// equal to (or just under) a multiple of the page size, then in practice,
// each chunk's starting offset will be aligned to a page boundary.
CPageSize uint64
// initialized is set true after the first AddXxx call.
initialized bool
// codec is the compression codec for the RAC file.
codec Codec
// err is the first error encountered. It is sticky: once a non-nil error
// occurs, all public methods will return that error.
err error
// tempFileSeekStart is the TempFile's starting position, if the TempFile
// is an io.Seeker.
tempFileSeekStart int64
// dataSize is the total size (so far) of the data portion (as opposed to
// the index portion) of the compressed file. For a two pass encoding, this
// should equal the size of the TempFile.
dataSize uint64
// dFileSize is the total size (so far) of the decompressed file. It is the
// sum of all the dRangeSize arguments passed to AddChunk.
dFileSize uint64
// resourcesCOffCLens is the COffset and CLength values for each shared
// resource. Those values are for the compressed file (what the RAC spec
// calls CSpace).
//
// The first element (if it exists) is an unused placeholder, as a zero
// OptResource means unused.
resourcesCOffCLens []uint64
// leafNodes are the non-resource leaf nodes of the hierarchical index.
leafNodes []wNode
// log2CPageSize is the base-2 logarithm of CPageSize, or zero if CPageSize
// is zero.
log2CPageSize uint32
// padding is a slice of zeroes to pad output to CPageSize boundaries.
padding []byte
}
func (w *ChunkWriter) checkParameters() error {
if w.Writer == nil {
w.err = errInvalidWriter
return w.err
}
if !isZeroOrAPowerOf2(w.CPageSize) || (w.CPageSize > MaxSize) {
w.err = errInvalidCPageSize
return w.err
} else if w.CPageSize > 0 {
for w.log2CPageSize = 1; w.CPageSize != (1 << w.log2CPageSize); w.log2CPageSize++ {
}
}
return nil
}
func (w *ChunkWriter) initialize() error {
if w.err != nil {
return w.err
}
if w.initialized {
return nil
}
w.initialized = true
if err := w.checkParameters(); err != nil {
return err
}
if s, ok := w.TempFile.(io.Seeker); ok {
if n, err := s.Seek(0, io.SeekCurrent); err != nil {
w.err = err
return err
} else {
w.tempFileSeekStart = n
}
}
switch w.IndexLocation {
case IndexLocationAtEnd:
if w.TempFile != nil {
w.err = errILAEndTempFile
return w.err
}
return w.write(indexLocationAtEndMagic)
case IndexLocationAtStart:
if w.TempFile == nil {
w.err = errILAStartTempFile
return w.err
}
}
return nil
}
func (w *ChunkWriter) write(data []byte) error {
ioWriter := w.Writer
if w.TempFile != nil {
ioWriter = w.TempFile
}
if uint64(len(data)) > MaxSize {
w.err = errTooMuchInput
return w.err
}
if w.CPageSize > 0 {
if err := w.writePadding(ioWriter, uint64(len(data))); err != nil {
return err
}
}
if _, err := ioWriter.Write(data); err != nil {
w.err = err
return err
}
w.dataSize += uint64(len(data))
if w.dataSize > MaxSize {
w.err = errTooMuchInput
return w.err
}
return nil
}
func (w *ChunkWriter) writePadding(ioWriter io.Writer, lenData uint64) error {
if lenData == 0 {
return nil
}
offset0 := w.dataSize & (w.CPageSize - 1)
offset1WithPadding := (0 * offset0) + lenData
offset1SansPadding := (1 * offset0) + lenData
numPagesWithPadding := (offset1WithPadding + (w.CPageSize - 1)) >> w.log2CPageSize
numPagesSansPadding := (offset1SansPadding + (w.CPageSize - 1)) >> w.log2CPageSize
if numPagesSansPadding == numPagesWithPadding {
return nil
}
return w.padToPageSize(ioWriter, w.dataSize)
}
func (w *ChunkWriter) padToPageSize(ioWriter io.Writer, offset uint64) error {
offset &= w.CPageSize - 1
if (offset == 0) || (w.CPageSize == 0) {
return nil
}
if w.padding == nil {
n := w.CPageSize
if n > 4096 {
n = 4096
}
w.padding = make([]byte, n)
}
for remaining := w.CPageSize - offset; remaining > 0; {
padding := w.padding
if remaining < uint64(len(padding)) {
padding = padding[:remaining]
}
n, err := ioWriter.Write(padding)
if err != nil {
w.err = err
return err
}
remaining -= uint64(n)
w.dataSize += uint64(n)
if w.dataSize > MaxSize {
w.err = errTooMuchInput
return w.err
}
}
return nil
}
func (w *ChunkWriter) roundUpToCPageBoundary(x uint64) uint64 {
if w.CPageSize == 0 {
return x
}
return (x + w.CPageSize - 1) &^ (w.CPageSize - 1)
}
// AddResource adds a shared resource to the RAC file. It returns a non-zero
// OptResource that identifies the resource's bytes. Future calls to AddChunk
// can pass these identifiers to indicate that decompressing a chunk depends on
// up to two shared resources.
//
// The caller may modify resource's contents after this method returns.
func (w *ChunkWriter) AddResource(resource []byte) (OptResource, error) {
if len(w.resourcesCOffCLens) >= (1 << 30) {
w.err = errTooManyResources
return 0, w.err
}
if err := w.initialize(); err != nil {
return 0, err
}
if err := w.write(resource); err != nil {
return 0, err
}
if len(w.resourcesCOffCLens) == 0 {
w.resourcesCOffCLens = make([]uint64, 1, 8)
}
id := OptResource(len(w.resourcesCOffCLens))
cOffset := w.dataSize - uint64(len(resource))
cLength := calcCLength(len(resource))
w.resourcesCOffCLens = append(w.resourcesCOffCLens, cOffset|(cLength<<48))
return id, nil
}
// AddChunk adds a chunk of compressed data - the (primary, secondary,
// tertiary) tuple - to the RAC file. Decompressing that chunk should produce
// dRangeSize bytes, although the ChunkWriter does not attempt to verify that.
//
// The OptResource arguments may be zero, meaning that no resource is used. In
// that case, the corresponding STag or TTag (see the RAC specification for
// further discussion) will be 0xFF.
//
// The caller may modify primary's contents after this method returns.
func (w *ChunkWriter) AddChunk(
dRangeSize uint64, codec Codec, primary []byte,
secondary OptResource, tertiary OptResource) error {
if w.err != nil {
return w.err
}
if dRangeSize == 0 {
return nil
}
if (dRangeSize > MaxSize) || ((w.dFileSize + dRangeSize) > MaxSize) {
w.err = errTooMuchInput
return w.err
}
if len(w.leafNodes) >= (1 << 30) {
w.err = errTooManyChunks
return w.err
}
if err := w.initialize(); err != nil {
return err
}
if len(w.leafNodes) == 0 {
if !codec.Valid() {
return errInvalidCodec
}
w.codec = codec
} else if w.codec != codec {
w.err = errors.New("rac: TODO: support writing multiple Codecs")
return w.err
}
if err := w.write(primary); err != nil {
return err
}
cOffset := w.dataSize - uint64(len(primary))
cLength := calcCLength(len(primary))
w.dFileSize += dRangeSize
w.leafNodes = append(w.leafNodes, wNode{
dRangeSize: dRangeSize,
cOffsetCLength: cOffset | (cLength << 48),
secondary: secondary,
tertiary: tertiary,
codec: codec,
})
return nil
}
var emptyRACFile = [32]byte{
0x72, 0xC3, 0x63, 0x01, 0x0D, 0xF8, 0x00, 0xFF,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xFF,
0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01,
}
// Close writes the RAC index to w.Writer and marks that w accepts no further
// method calls.
//
// For a one pass encoding, no further action is taken. For a two pass encoding
// (i.e. IndexLocationAtStart), it then copies w.TempFile to w.Writer. Either
// way, if this method returns nil error, the entirety of what was written to
// w.Writer constitutes a valid RAC file.
//
// Closing the underlying w.Writer, w.TempFile or both is the responsibility of
// the ChunkWriter caller, not the ChunkWriter itself.
//
// It is not necessary to call Close, e.g. if an earlier AddXxx call returned a
// non-nil error. Unlike an os.File, failing to call ChunkWriter.Close will not
// leak resources such as file descriptors.
func (w *ChunkWriter) Close() error {
if w.err != nil {
return w.err
}
if !w.initialized {
if err := w.checkParameters(); err != nil {
return err
}
}
if len(w.leafNodes) == 0 {
_, err := w.Writer.Write(emptyRACFile[:])
return err
}
rootNode := gather(w.leafNodes, w.codec.isLong())
indexSize := rootNode.calcEncodedSize(0, w.IndexLocation == IndexLocationAtEnd)
nw := &nodeWriter{
w: w.Writer,
resourcesCOffCLens: w.resourcesCOffCLens,
}
if w.IndexLocation == IndexLocationAtEnd {
nw.indexCOffset = w.roundUpToCPageBoundary(w.dataSize)
nw.cFileSize = nw.indexCOffset + indexSize
} else {
nw.dataCOffset = w.roundUpToCPageBoundary(indexSize)
nw.cFileSize = nw.dataCOffset + w.dataSize
}
if nw.cFileSize > MaxSize {
w.err = errTooMuchInput
return w.err
}
if w.IndexLocation == IndexLocationAtEnd {
// Write the align-to-CPageSize padding.
if w.CPageSize > 0 {
if err := w.padToPageSize(w.Writer, w.dataSize); err != nil {
return err
}
}
// Write the index. The compressed data has already been written.
if err := nw.writeIndex(&rootNode, true); err != nil {
w.err = err
return err
}
} else {
expectedTempFileSize := w.dataSize
// Write the index.
if err := nw.writeIndex(&rootNode, false); err != nil {
w.err = err
return err
}
// Write the align-to-CPageSize padding.
if w.CPageSize > 0 {
if err := w.padToPageSize(w.Writer, indexSize); err != nil {
return err
}
}
// Write the compressed data.
if s, ok := w.TempFile.(io.Seeker); ok {
if _, err := s.Seek(w.tempFileSeekStart, io.SeekStart); err != nil {
w.err = err
return err
}
}
if n, err := io.Copy(w.Writer, w.TempFile); err != nil {
w.err = err
return err
} else if uint64(n) != expectedTempFileSize {
w.err = errInconsistentCompressedSize
return w.err
}
}
w.err = errAlreadyClosed
return nil
}
// wNode is the ChunkWriter's representation of a node.
type wNode struct {
dRangeSize uint64
children []wNode // Excludes resource-node chidren.
resources []int
cOffsetCLength uint64
secondary OptResource
tertiary OptResource
codec Codec
}
// calcEncodedSize accumulates the encoded size of n and its children,
// traversing the nodes in depth-first order.
//
// As a side effect, it also sets n.cOffsetCLength for branch nodes.
func (n *wNode) calcEncodedSize(accumulator uint64, rootAndIsAtEnd bool) (newAccumulator uint64) {
arity := len(n.children) + len(n.resources)
if arity == 0 {
return accumulator
}
if n.codec.isLong() {
arity++
}
size := (arity * 16) + 16
cLength := calcCLength(size)
if rootAndIsAtEnd {
for i := range n.children {
accumulator = n.children[i].calcEncodedSize(accumulator, false)
}
}
n.cOffsetCLength = accumulator | (cLength << 48)
accumulator += uint64(size)
if !rootAndIsAtEnd {
for i := range n.children {
accumulator = n.children[i].calcEncodedSize(accumulator, false)
}
}
return accumulator
}
type nodeWriter struct {
w io.Writer
cFileSize uint64
dataCOffset uint64
indexCOffset uint64
resourcesCOffCLens []uint64
// A wNode's maximum arity is 255, so a wNode's maximum size in bytes is
// ((255 * 16) + 16), which is 4096.
buffer [4096]byte
}
func (w *nodeWriter) writeIndex(n *wNode, rootAndIsAtEnd bool) error {
const tagFF = 0xFF << 56
if rootAndIsAtEnd {
for i, o := range n.children {
if len(o.children) != 0 {
if err := w.writeIndex(&n.children[i], false); err != nil {
return err
}
}
}
}
if !n.codec.Valid() {
return errors.New("rac: TODO: support writing multiple Codecs")
}
buf, dPtr := w.buffer[:], uint64(0)
arity := uint64(len(n.children) + len(n.resources))
if n.codec.isLong() {
arity++
}
if arity > 0xFF {
return errInternalArityIsTooLarge
}
// Codec Element 'DPtr|Reserved0|TTag', if present.
if n.codec.isLong() {
putU64LE(buf, 0xFD00000000000000)
buf = buf[8:]
}
// DPtr's. We write resources before regular children (non-resources), so
// that any TTag that refers to a resource always avoids the [0xC0, 0xFD]
// reserved zone. The ratio of resources to regulars is at most 2:1, as
// every chunk uses exactly 1 regular slot and up to 2 resource slots, and
// that reserved zone is less than 33% of the [0x00, 0xFF] space.
for i := range n.resources {
putU64LE(buf[8*i:], dPtr|tagFF)
}
buf = buf[8*len(n.resources):]
for i, o := range n.children {
tag := uint64(0xFE << 56)
if len(o.children) == 0 {
tag = resourceToTag(n.resources, o.tertiary)
}
putU64LE(buf[8*i:], dPtr|tag)
dPtr += o.dRangeSize
}
buf = buf[8*len(n.children):]
// DPtrMax and the CodecByte. By construction, a Long Codec's 'c64' value
// is always 0, as the Codec Element is always in position 0.
codecHighByte := uint64(n.codec) & 0xFF00000000000000
putU64LE(buf, dPtr|codecHighByte)
buf = buf[8:]
// Codec Element 'CPtr|CLen|STag', if present.
if n.codec.isLong() {
putU64LE(buf, uint64(n.codec)&0x00FFFFFFFFFFFFFF)
buf = buf[8:]
}
// CPtr's. While the RAC format allows otherwise, this nodeWriter always
// keeps the CBias at zero, so that a CPtr equals a COffset.
for i, res := range n.resources {
cOffsetCLength := w.resourcesCOffCLens[res] + w.dataCOffset
putU64LE(buf[8*i:], cOffsetCLength|tagFF)
}
buf = buf[8*len(n.resources):]
for i, o := range n.children {
cOffsetCLength := o.cOffsetCLength
if len(o.children) == 0 {
cOffsetCLength += w.dataCOffset
} else {
cOffsetCLength += w.indexCOffset
}
putU64LE(buf[8*i:], cOffsetCLength|resourceToTag(n.resources, o.secondary))
}
buf = buf[8*len(n.children):]
// CPtrMax.
const version = 0x01
putU64LE(buf, w.cFileSize|(version<<48)|(arity<<56))
// Magic and Arity.
w.buffer[0] = 0x72
w.buffer[1] = 0xC3
w.buffer[2] = 0x63
w.buffer[3] = uint8(arity)
// Checksum.
size := (arity * 16) + 16
checksum := crc32.ChecksumIEEE(w.buffer[6:size])
checksum ^= checksum >> 16
w.buffer[4] = uint8(checksum >> 0)
w.buffer[5] = uint8(checksum >> 8)
if _, err := w.w.Write(w.buffer[:size]); err != nil {
return err
}
if !rootAndIsAtEnd {
for i, o := range n.children {
if len(o.children) != 0 {
if err := w.writeIndex(&n.children[i], false); err != nil {
return err
}
}
}
}
return nil
}
func calcCLength(primarySize int) uint64 {
if primarySize <= 0 {
return 1
}
// Divide by 1024, rounding up.
primarySize = ((primarySize - 1) >> 10) + 1
if primarySize > 255 {
return 0
}
return uint64(primarySize)
}
func resourceToTag(resources []int, r OptResource) uint64 {
if r != 0 {
for i, res := range resources {
if res == int(r) {
return uint64(i) << 56
}
}
}
return 0xFF << 56
}
// gather brings the given nodes into a tree, such that every branch node's
// arity (the count of its resource and non-resource child nodes) is at most
// 0xFF. It returns the root of that tree.
//
// TODO: it currently builds a tree where all leaf nodes have equal depth. For
// small RAC files (with only one branch node: the mandatory root node), this
// doesn't matter. For larger RAC files, it might be better to build an uneven
// tree to minimize average leaf node depth, with some branch nodes having both
// branch node children and leaf node children. It might even be worth ensuring
// that the root node always directly contains the first and last leaf nodes as
// children, as those leaves presumably contain the most commonly accessed
// parts of the decompressed file.
//
// If doing this TODO, we'd also have to change the "codec = codecMixBit |
// CodecZeroes" line below, as it assumes that no branch nodes have both branch
// node children and leaf node children.
func gather(nodes []wNode, codecIsLong bool) wNode {
if len(nodes) == 0 {
panic("gather: no nodes")
}
resources := map[OptResource]bool{}
arityBudget := 0xFF
if codecIsLong {
arityBudget = 0xFE
}
for {
i, j, arity, newNodes := 0, 0, 0, []wNode(nil)
for ; j < len(nodes); j++ {
o := &nodes[j]
new2 := (o.secondary != 0) && !resources[o.secondary]
new3 := (o.tertiary != 0) && !resources[o.tertiary]
arity += 1 + btoi(new2) + btoi(new3)
if arity <= arityBudget {
if new2 {
resources[o.secondary] = true
}
if new3 {
resources[o.tertiary] = true
}
continue
}
newNodes = append(newNodes, makeBranch(nodes[i:j], resources))
if len(resources) != 0 {
resources = map[OptResource]bool{}
}
i = j
arity = 1
if o.secondary != 0 {
resources[o.secondary] = true
arity++
}
if o.tertiary != 0 {
resources[o.tertiary] = true
arity++
}
}
if i == 0 {
return makeBranch(nodes, resources)
}
newNodes = append(newNodes, makeBranch(nodes[i:], resources))
if len(resources) != 0 {
resources = map[OptResource]bool{}
}
nodes, newNodes = newNodes, nil
}
}
func makeBranch(children []wNode, resMap map[OptResource]bool) wNode {
dRangeSize, codec := uint64(0), Codec(0)
for i, c := range children {
dRangeSize += c.dRangeSize
if i == 0 {
codec = c.codec
} else if codec != c.codec {
// We construct the node tree so that a branch node's children are
// either all branch nodes or all leaf nodes. If they are all leaf
// nodes (with the same Codec), or all branch nodes with the same
// Codec, the parent uses the same Codec. If they are all branch
// nodes, with different Codecs, we set the Mix bit and since it
// has no leaf nodes, we might as well use CodecZeroes.
codec = codecMixBit | CodecZeroes
}
}
resList := []int(nil)
if len(resMap) != 0 {
resList = make([]int, 0, len(resMap))
for res := range resMap {
resList = append(resList, int(res))
}
sort.Ints(resList)
}
return wNode{
dRangeSize: dRangeSize,
children: children,
resources: resList,
cOffsetCLength: invalidCOffsetCLength,
codec: codec,
}
}