mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-19 08:38:14 +01:00
495 lines
14 KiB
Go
495 lines
14 KiB
Go
// Copyright 2019 The Wuffs Authors.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// https://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package rac
|
|
|
|
import (
|
|
"errors"
|
|
"hash/crc32"
|
|
"io"
|
|
"sort"
|
|
)
|
|
|
|
func putU64LE(b []byte, v uint64) {
|
|
_ = b[7] // Early bounds check to guarantee safety of writes below.
|
|
b[0] = byte(v)
|
|
b[1] = byte(v >> 8)
|
|
b[2] = byte(v >> 16)
|
|
b[3] = byte(v >> 24)
|
|
b[4] = byte(v >> 32)
|
|
b[5] = byte(v >> 40)
|
|
b[6] = byte(v >> 48)
|
|
b[7] = byte(v >> 56)
|
|
}
|
|
|
|
// Writer provides a relatively simple way to write a RAC file - one that is
|
|
// created starting from nothing, as opposed to incrementally modifying an
|
|
// existing RAC file.
|
|
//
|
|
// Other packages may provide a more flexible (and more complicated) way to
|
|
// write or append to RAC files, but that is out of scope of this package.
|
|
//
|
|
// Do not modify its exported fields after calling any of its methods.
|
|
type Writer struct {
|
|
// Writer is where the RAC-encoded data is written to.
|
|
//
|
|
// Nil is an invalid value.
|
|
Writer io.Writer
|
|
|
|
// Codec is the compression codec for the RAC file.
|
|
//
|
|
// See the RAC specification for further discussion.
|
|
//
|
|
// Zero is an invalid value.
|
|
Codec Codec
|
|
|
|
// initialized is set true after the first AddXxx call.
|
|
initialized bool
|
|
|
|
// IndexLocation is whether the index is at the start or end of the RAC
|
|
// file.
|
|
//
|
|
// See the RAC specification for further discussion.
|
|
//
|
|
// The zero value of this field is IndexLocationAtEnd: a one pass encoding.
|
|
IndexLocation IndexLocation
|
|
|
|
// TempFile is a temporary file to use for a two pass encoding. The field
|
|
// name says "file" but it need not be a real file, in the operating system
|
|
// sense.
|
|
//
|
|
// For IndexLocationAtEnd, this must be nil. For IndexLocationAtStart, this
|
|
// must be non-nil.
|
|
//
|
|
// It does not need to implement io.Seeker, if it supports separate read
|
|
// and write positions (e.g. it is a bytes.Buffer). If it does implement
|
|
// io.Seeker (e.g. it is an os.File), its current position will be noted
|
|
// (via SeekCurrent), then it will be written to (via the rac.Writer.AddXxx
|
|
// methods), then its position will be reset (via SeekSet), then it will be
|
|
// read from (via the rac.Writer.Close method).
|
|
//
|
|
// The rac.Writer does not call TempFile.Close even if that method exists
|
|
// (e.g. the TempFile is an os.File). In that case, closing the temporary
|
|
// file (and deleting it) after the rac.Writer is closed is the
|
|
// responsibility of the rac.Writer caller, not the rac.Writer itself.
|
|
TempFile io.ReadWriter
|
|
|
|
// err is the first error encountered. It is sticky: once a non-nil error
|
|
// occurs, all public methods will return that error.
|
|
err error
|
|
|
|
// tempFileSeekStart is the TempFile's starting position, if the TempFile
|
|
// is an io.Seeker.
|
|
tempFileSeekStart int64
|
|
|
|
// dataSize is the total size (so far) of the data portion (as opposed to
|
|
// the index portion) of the compressed file. For a two pass encoding, this
|
|
// should equal the size of the TempFile.
|
|
dataSize uint64
|
|
|
|
// dFileSize is the total size (so far) of the decompressed file. It is the
|
|
// sum of all the dRangeSize arguments passed to AddChunk.
|
|
dFileSize uint64
|
|
|
|
// resourcesCOffsets is the file offsets of each shared resource. Those
|
|
// offsets are for the compressed file (what the RAC spec calls CSpace).
|
|
//
|
|
// The first element (if it exists) is an unused placeholder, as a zero
|
|
// OptResource means unused.
|
|
resourcesCOffsets []uint64
|
|
|
|
// leafNodes are the non-resource leaf nodes of the hierarchical index.
|
|
leafNodes []node
|
|
}
|
|
|
|
func (w *Writer) initialize() error {
|
|
if w.err != nil {
|
|
return w.err
|
|
}
|
|
if w.initialized {
|
|
return nil
|
|
}
|
|
w.initialized = true
|
|
|
|
if w.Writer == nil {
|
|
w.err = errors.New("rac: invalid Writer")
|
|
return w.err
|
|
}
|
|
if w.Codec == 0 {
|
|
w.err = errors.New("rac: invalid Codec")
|
|
return w.err
|
|
}
|
|
|
|
if s, ok := w.TempFile.(io.Seeker); ok {
|
|
if n, err := s.Seek(0, io.SeekCurrent); err != nil {
|
|
w.err = err
|
|
return err
|
|
} else {
|
|
w.tempFileSeekStart = n
|
|
}
|
|
}
|
|
|
|
switch w.IndexLocation {
|
|
case IndexLocationAtEnd:
|
|
if w.TempFile != nil {
|
|
w.err = errors.New("rac: IndexLocationAtEnd requires a nil TempFile")
|
|
return w.err
|
|
}
|
|
return w.write(indexLocationAtEndMagic)
|
|
|
|
case IndexLocationAtStart:
|
|
if w.TempFile == nil {
|
|
w.err = errors.New("rac: IndexLocationAtStart requires a non-nil TempFile")
|
|
return w.err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (w *Writer) write(data []byte) error {
|
|
ioWriter := w.Writer
|
|
if w.TempFile != nil {
|
|
ioWriter = w.TempFile
|
|
}
|
|
|
|
if _, err := ioWriter.Write(data); err != nil {
|
|
w.err = err
|
|
return err
|
|
}
|
|
w.dataSize += uint64(len(data))
|
|
return nil
|
|
}
|
|
|
|
// AddResource adds a shared resource to the RAC file. It returns a non-zero
|
|
// OptResource that identifies the resource's bytes. Future calls to AddChunk
|
|
// can pass these identifiers to indicate that decompressing a chunk depends on
|
|
// up to two shared resources.
|
|
//
|
|
// The caller may modify resource's contents after this method returns.
|
|
func (w *Writer) AddResource(resource []byte) (OptResource, error) {
|
|
if err := w.initialize(); err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
if len(w.resourcesCOffsets) >= (1 << 30) {
|
|
w.err = errors.New("rac: too many shared resources")
|
|
return 0, w.err
|
|
}
|
|
if len(w.resourcesCOffsets) == 0 {
|
|
w.resourcesCOffsets = make([]uint64, 1, 8)
|
|
}
|
|
id := OptResource(len(w.resourcesCOffsets))
|
|
w.resourcesCOffsets = append(w.resourcesCOffsets, w.dataSize)
|
|
if err := w.write(resource); err != nil {
|
|
return 0, err
|
|
}
|
|
return id, nil
|
|
}
|
|
|
|
// AddChunk adds a chunk of compressed data (the [primary, secondary, tertiary]
|
|
// tuple) to the RAC file. Decompressing that chunk should produce dRangeSize
|
|
// bytes, although the rac.Writer does not attempt to verify that.
|
|
//
|
|
// The OptResource arguments may be zero, meaning that no resource is used.
|
|
//
|
|
// The caller may modify primary's contents after this method returns.
|
|
func (w *Writer) AddChunk(dRangeSize uint64, primary []byte, secondary OptResource, tertiary OptResource) error {
|
|
if err := w.initialize(); err != nil {
|
|
return err
|
|
}
|
|
|
|
if dRangeSize == 0 {
|
|
w.err = errors.New("rac: dRangeSize is too small")
|
|
return w.err
|
|
}
|
|
if (dRangeSize > MaxSize) || ((w.dFileSize + dRangeSize) > MaxSize) {
|
|
w.err = errors.New("rac: dRangeSize is too large")
|
|
return w.err
|
|
}
|
|
w.dFileSize += dRangeSize
|
|
w.leafNodes = append(w.leafNodes, node{
|
|
dRangeSize: dRangeSize,
|
|
cOffset: w.dataSize,
|
|
secondary: secondary,
|
|
tertiary: tertiary,
|
|
})
|
|
return w.write(primary)
|
|
}
|
|
|
|
// Close writes the RAC index to w.Writer and marks that w accepts no further
|
|
// method calls.
|
|
//
|
|
// For a one pass encoding, no further action is taken. For a two pass encoding
|
|
// (i.e. IndexLocationAtStart), it then copies w.TempFile to w.Writer. Either
|
|
// way, if this method returns nil error, the entirety of what was written to
|
|
// w.Writer constitutes a valid RAC file.
|
|
//
|
|
// Closing the underlying w.Writer, w.TempFile or both is the responsibility of
|
|
// the rac.Writer caller, not the rac.Writer itself.
|
|
//
|
|
// It is not necessary to call Close, e.g. if an earlier AddXxx call returned a
|
|
// non-nil error. Unlike an os.File, failing to call rac.Writer.Close will not
|
|
// leak resources such as file descriptors.
|
|
func (w *Writer) Close() error {
|
|
if w.err != nil {
|
|
return w.err
|
|
}
|
|
|
|
rootNode := node{}
|
|
if len(w.leafNodes) == 0 {
|
|
rootNode.children = []node{{}}
|
|
} else {
|
|
rootNode = gather(w.leafNodes)
|
|
}
|
|
indexSize := rootNode.calcEncodedSize(0)
|
|
|
|
nw := &nodeWriter{
|
|
w: w.Writer,
|
|
resourcesCOffsets: w.resourcesCOffsets,
|
|
cFileSize: indexSize + w.dataSize,
|
|
codec: w.Codec,
|
|
}
|
|
|
|
if w.IndexLocation == IndexLocationAtEnd {
|
|
// Write the index. The compressed data has already been written.
|
|
nw.indexCOffset = w.dataSize
|
|
if err := nw.writeIndex(&rootNode); err != nil {
|
|
w.err = err
|
|
return err
|
|
}
|
|
|
|
} else {
|
|
// Write the index.
|
|
nw.dataCOffset = indexSize
|
|
if err := nw.writeIndex(&rootNode); err != nil {
|
|
w.err = err
|
|
return err
|
|
}
|
|
|
|
// Write the compressed data.
|
|
if s, ok := w.TempFile.(io.Seeker); ok {
|
|
if _, err := s.Seek(w.tempFileSeekStart, io.SeekStart); err != nil {
|
|
w.err = err
|
|
return err
|
|
}
|
|
}
|
|
if n, err := io.Copy(w.Writer, w.TempFile); err != nil {
|
|
w.err = err
|
|
return err
|
|
} else if uint64(n) != w.dataSize {
|
|
w.err = errors.New("rac: inconsistent compressed data size")
|
|
return w.err
|
|
}
|
|
}
|
|
|
|
w.err = errors.New("rac: Writer is closed")
|
|
return nil
|
|
}
|
|
|
|
type node struct {
|
|
dRangeSize uint64
|
|
|
|
children []node // Excludes resource-node chidren.
|
|
resources []int
|
|
|
|
cOffset uint64
|
|
secondary OptResource
|
|
tertiary OptResource
|
|
}
|
|
|
|
// calcEncodedSize accumulates the encoded size of n and its children,
|
|
// traversing the nodes in depth-first order.
|
|
//
|
|
// As a side effect, it also sets n.cOffset for branch nodes.
|
|
func (n *node) calcEncodedSize(accumulator uint64) (newAccumulator uint64) {
|
|
if len(n.children)+len(n.resources) == 0 {
|
|
return accumulator
|
|
}
|
|
n.cOffset = accumulator
|
|
accumulator += (uint64(len(n.children)+len(n.resources)) * 16) + 16
|
|
for i := range n.children {
|
|
accumulator = n.children[i].calcEncodedSize(accumulator)
|
|
}
|
|
return accumulator
|
|
}
|
|
|
|
type nodeWriter struct {
|
|
w io.Writer
|
|
|
|
cFileSize uint64
|
|
dataCOffset uint64
|
|
indexCOffset uint64
|
|
resourcesCOffsets []uint64
|
|
|
|
codec Codec
|
|
|
|
// A node's maximum arity is 255, so a node's maximum size in bytes is
|
|
// ((255 * 16) + 16), which is 4096.
|
|
buffer [4096]byte
|
|
}
|
|
|
|
func (w *nodeWriter) writeIndex(n *node) error {
|
|
const tagFF = 0xFF << 56
|
|
|
|
buf, dPtr := w.buffer[:], uint64(0)
|
|
arity := uint64(len(n.children) + len(n.resources))
|
|
if arity > 0xFF {
|
|
return errors.New("rac: internal error: arity is too large")
|
|
}
|
|
|
|
// DPtr's. We write resources before regular children (non-resources), so
|
|
// that any TTag that refers to a resource always avoids the [0xC0, 0xFD]
|
|
// reserved zone. The ratio of resources to regulars is at most 2:1, as
|
|
// every chunk uses exactly 1 regular slot and up to 2 resource slots, and
|
|
// that reserved zone is less than 33% of the [0x00, 0xFF] space.
|
|
for i := range n.resources {
|
|
putU64LE(buf[8*i:], dPtr|tagFF)
|
|
}
|
|
buf = buf[8*len(n.resources):]
|
|
for i, o := range n.children {
|
|
putU64LE(buf[8*i:], dPtr|resourceToTag(n.resources, o.tertiary))
|
|
dPtr += o.dRangeSize
|
|
}
|
|
buf = buf[8*len(n.children):]
|
|
|
|
// DPtrMax.
|
|
putU64LE(buf, dPtr|(uint64(w.codec)<<56))
|
|
buf = buf[8:]
|
|
|
|
// CPtr's. While the RAC format allows otherwise, this Writer always keeps
|
|
// the CBias at zero, so that a CPtr equals a COffset.
|
|
for i, res := range n.resources {
|
|
cOffset := w.resourcesCOffsets[res] + w.dataCOffset
|
|
putU64LE(buf[8*i:], cOffset|tagFF)
|
|
}
|
|
buf = buf[8*len(n.resources):]
|
|
for i, o := range n.children {
|
|
cOffset := o.cOffset
|
|
if len(o.children) == 0 {
|
|
cOffset += w.dataCOffset
|
|
} else {
|
|
cOffset += w.indexCOffset
|
|
}
|
|
putU64LE(buf[8*i:], cOffset|resourceToTag(n.resources, o.secondary))
|
|
}
|
|
buf = buf[8*len(n.children):]
|
|
|
|
// CPtrMax.
|
|
putU64LE(buf, w.cFileSize|(arity<<56))
|
|
|
|
// Magic, Arity, (Checksum is skipped), Version.
|
|
w.buffer[0] = 0x72
|
|
w.buffer[1] = 0xC3
|
|
w.buffer[2] = 0x63
|
|
w.buffer[3] = uint8(arity)
|
|
w.buffer[6] = 0x01
|
|
|
|
// Checksum (which depends on the Version).
|
|
size := (arity * 16) + 16
|
|
checksum := crc32.ChecksumIEEE(w.buffer[6:size])
|
|
checksum ^= checksum >> 16
|
|
w.buffer[4] = uint8(checksum >> 0)
|
|
w.buffer[5] = uint8(checksum >> 8)
|
|
|
|
_, err := w.w.Write(w.buffer[:size])
|
|
return err
|
|
}
|
|
|
|
func resourceToTag(resources []int, r OptResource) uint64 {
|
|
if r != 0 {
|
|
for i, res := range resources {
|
|
if res == int(r) {
|
|
return uint64(i) << 56
|
|
}
|
|
}
|
|
}
|
|
return 0xFF << 56
|
|
}
|
|
|
|
// gather brings the given nodes into a tree, such that every branch node's
|
|
// arity (the count of its resource and non-resource child nodes) is at most
|
|
// 0xFF. It returns the root of that tree.
|
|
//
|
|
// TODO: it currently builds a tree where all leaf nodes have equal depth. For
|
|
// small RAC files (with only one branch node: the mandatory root node), this
|
|
// doesn't matter. For larger RAC files, it might be better to build an uneven
|
|
// tree to minimize average leaf node depth, with some branch nodes having both
|
|
// branch node children and leaf node children. It might even be worth ensuring
|
|
// that the root node always directly contains the first and last leaf nodes as
|
|
// children, as those leaves presumably contain the most commonly accessed
|
|
// parts of the decompressed file.
|
|
func gather(nodes []node) node {
|
|
if len(nodes) == 0 {
|
|
panic("gather: no nodes")
|
|
}
|
|
|
|
resources := map[OptResource]bool{}
|
|
arity := 0
|
|
|
|
for {
|
|
i, j, newNodes := 0, 0, []node(nil)
|
|
for ; j < len(nodes); j++ {
|
|
o := &nodes[j]
|
|
|
|
arity++
|
|
if (o.secondary != 0) && !resources[o.secondary] {
|
|
resources[o.secondary] = true
|
|
arity++
|
|
}
|
|
if (o.tertiary != 0) && !resources[o.tertiary] {
|
|
resources[o.tertiary] = true
|
|
arity++
|
|
}
|
|
if arity <= 0xFF {
|
|
continue
|
|
}
|
|
|
|
newNodes = append(newNodes, makeBranch(nodes[i:j], resources))
|
|
i = j
|
|
arity = 0
|
|
if len(resources) != 0 {
|
|
resources = map[OptResource]bool{}
|
|
}
|
|
}
|
|
|
|
if i == 0 {
|
|
return makeBranch(nodes, resources)
|
|
}
|
|
nodes, newNodes = newNodes, nil
|
|
}
|
|
}
|
|
|
|
func makeBranch(children []node, resMap map[OptResource]bool) node {
|
|
dRangeSize := uint64(0)
|
|
for _, c := range children {
|
|
dRangeSize += c.dRangeSize
|
|
}
|
|
|
|
resList := []int(nil)
|
|
if len(resMap) != 0 {
|
|
resList = make([]int, 0, len(resMap))
|
|
for res := range resMap {
|
|
resList = append(resList, int(res))
|
|
}
|
|
sort.Ints(resList)
|
|
}
|
|
|
|
return node{
|
|
dRangeSize: dRangeSize,
|
|
children: children,
|
|
resources: resList,
|
|
cOffset: invalidCOffset,
|
|
}
|
|
}
|