// Copyright 2019 The Wuffs Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // https://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package rac import ( "errors" "hash/crc32" "io" "sort" ) func btoi(b bool) int { if b { return 1 } return 0 } func isZeroOrAPowerOf2(x uint64) bool { return (x & (x - 1)) == 0 } func putU64LE(b []byte, v uint64) { _ = b[7] // Early bounds check to guarantee safety of writes below. b[0] = byte(v) b[1] = byte(v >> 8) b[2] = byte(v >> 16) b[3] = byte(v >> 24) b[4] = byte(v >> 32) b[5] = byte(v >> 40) b[6] = byte(v >> 48) b[7] = byte(v >> 56) } // OptResource is an option type, optionally holding a ChunkWriter-specific // identifier for a shared resource. // // Zero means that the option is not taken: no shared resource is used. type OptResource uint32 // ChunkWriter provides a relatively simple way to write a RAC file - one that // is created starting from nothing, as opposed to incrementally modifying an // existing RAC file. // // Other packages may provide a more flexible (and more complicated) way to // write or append to RAC files, but that is out of scope of this package. // // Do not modify its exported fields after calling any of its methods. type ChunkWriter struct { // Writer is where the RAC-encoded data is written to. // // Nil is an invalid value. Writer io.Writer // IndexLocation is whether the index is at the start or end of the RAC // file. // // See the RAC specification for further discussion. // // The zero value of this field is IndexLocationAtEnd: a one pass encoding. IndexLocation IndexLocation // TempFile is a temporary file to use for a two pass encoding. The field // name says "file" but it need not be a real file, in the operating system // sense. // // For IndexLocationAtEnd, this must be nil. For IndexLocationAtStart, this // must be non-nil. // // It does not need to implement io.Seeker, if it supports separate read // and write positions (e.g. it is a bytes.Buffer). If it does implement // io.Seeker (e.g. it is an os.File), its current position will be noted // (via SeekCurrent), then it will be written to (via the // ChunkWriter.AddXxx methods), then its position will be reset (via // SeekSet), then it will be read from (via the ChunkWriter.Close method). // // The ChunkWriter does not call TempFile.Close even if that method exists // (e.g. the TempFile is an os.File). In that case, closing the temporary // file (and deleting it) after the ChunkWriter is closed is the // responsibility of the ChunkWriter caller, not the ChunkWriter itself. TempFile io.ReadWriter // CPageSize guides padding the output to minimize the number of pages that // each chunk occupies (in what the RAC spec calls CSpace). // // It must either be zero (which means no padding is inserted) or a power // of 2, and no larger than MaxSize. // // For example, suppose that CSpace is partitioned into 1024-byte pages, // that 1000 bytes have already been written to the output, and the next // chunk is 1500 bytes long. // // - With no padding (i.e. with CPageSize set to 0), this chunk will // occupy the half-open range [1000, 2500) in CSpace, which straddles // three 1024-byte pages: the page [0, 1024), the page [1024, 2048) and // the page [2048, 3072). Call those pages p0, p1 and p2. // // - With padding (i.e. with CPageSize set to 1024), 24 bytes of zeroes // are first inserted so that this chunk occupies the half-open range // [1024, 2524) in CSpace, which straddles only two pages (p1 and p2). // // This concept is similar, but not identical, to alignment. Even with a // non-zero CPageSize, chunk start offsets are not necessarily aligned to // page boundaries. For example, suppose that the chunk size was only 1040 // bytes, not 1500 bytes. No padding will be inserted, as both [1000, 2040) // and [1024, 2064) straddle two pages: either pages p0 and p1, or pages p1 // and p2. // // Nonetheless, if all chunks (or all but the final chunk) have a size // equal to (or just under) a multiple of the page size, then in practice, // each chunk's starting offset will be aligned to a page boundary. CPageSize uint64 // initialized is set true after the first AddXxx call. initialized bool // codec is the compression codec for the RAC file. codec Codec // err is the first error encountered. It is sticky: once a non-nil error // occurs, all public methods will return that error. err error // tempFileSeekStart is the TempFile's starting position, if the TempFile // is an io.Seeker. tempFileSeekStart int64 // dataSize is the total size (so far) of the data portion (as opposed to // the index portion) of the compressed file. For a two pass encoding, this // should equal the size of the TempFile. dataSize uint64 // dFileSize is the total size (so far) of the decompressed file. It is the // sum of all the dRangeSize arguments passed to AddChunk. dFileSize uint64 // resourcesCOffCLens is the COffset and CLength values for each shared // resource. Those values are for the compressed file (what the RAC spec // calls CSpace). // // The first element (if it exists) is an unused placeholder, as a zero // OptResource means unused. resourcesCOffCLens []uint64 // leafNodes are the non-resource leaf nodes of the hierarchical index. leafNodes []wNode // log2CPageSize is the base-2 logarithm of CPageSize, or zero if CPageSize // is zero. log2CPageSize uint32 // padding is a slice of zeroes to pad output to CPageSize boundaries. padding []byte } func (w *ChunkWriter) checkParameters() error { if w.Writer == nil { w.err = errInvalidWriter return w.err } if !isZeroOrAPowerOf2(w.CPageSize) || (w.CPageSize > MaxSize) { w.err = errInvalidCPageSize return w.err } else if w.CPageSize > 0 { for w.log2CPageSize = 1; w.CPageSize != (1 << w.log2CPageSize); w.log2CPageSize++ { } } return nil } func (w *ChunkWriter) initialize() error { if w.err != nil { return w.err } if w.initialized { return nil } w.initialized = true if err := w.checkParameters(); err != nil { return err } if s, ok := w.TempFile.(io.Seeker); ok { if n, err := s.Seek(0, io.SeekCurrent); err != nil { w.err = err return err } else { w.tempFileSeekStart = n } } switch w.IndexLocation { case IndexLocationAtEnd: if w.TempFile != nil { w.err = errILAEndTempFile return w.err } return w.write(indexLocationAtEndMagic) case IndexLocationAtStart: if w.TempFile == nil { w.err = errILAStartTempFile return w.err } } return nil } func (w *ChunkWriter) write(data []byte) error { ioWriter := w.Writer if w.TempFile != nil { ioWriter = w.TempFile } if uint64(len(data)) > MaxSize { w.err = errTooMuchInput return w.err } if w.CPageSize > 0 { if err := w.writePadding(ioWriter, uint64(len(data))); err != nil { return err } } if _, err := ioWriter.Write(data); err != nil { w.err = err return err } w.dataSize += uint64(len(data)) if w.dataSize > MaxSize { w.err = errTooMuchInput return w.err } return nil } func (w *ChunkWriter) writePadding(ioWriter io.Writer, lenData uint64) error { if lenData == 0 { return nil } offset0 := w.dataSize & (w.CPageSize - 1) offset1WithPadding := (0 * offset0) + lenData offset1SansPadding := (1 * offset0) + lenData numPagesWithPadding := (offset1WithPadding + (w.CPageSize - 1)) >> w.log2CPageSize numPagesSansPadding := (offset1SansPadding + (w.CPageSize - 1)) >> w.log2CPageSize if numPagesSansPadding == numPagesWithPadding { return nil } return w.padToPageSize(ioWriter, w.dataSize) } func (w *ChunkWriter) padToPageSize(ioWriter io.Writer, offset uint64) error { offset &= w.CPageSize - 1 if (offset == 0) || (w.CPageSize == 0) { return nil } if w.padding == nil { n := w.CPageSize if n > 4096 { n = 4096 } w.padding = make([]byte, n) } for remaining := w.CPageSize - offset; remaining > 0; { padding := w.padding if remaining < uint64(len(padding)) { padding = padding[:remaining] } n, err := ioWriter.Write(padding) if err != nil { w.err = err return err } remaining -= uint64(n) w.dataSize += uint64(n) if w.dataSize > MaxSize { w.err = errTooMuchInput return w.err } } return nil } func (w *ChunkWriter) roundUpToCPageBoundary(x uint64) uint64 { if w.CPageSize == 0 { return x } return (x + w.CPageSize - 1) &^ (w.CPageSize - 1) } // AddResource adds a shared resource to the RAC file. It returns a non-zero // OptResource that identifies the resource's bytes. Future calls to AddChunk // can pass these identifiers to indicate that decompressing a chunk depends on // up to two shared resources. // // The caller may modify resource's contents after this method returns. func (w *ChunkWriter) AddResource(resource []byte) (OptResource, error) { if len(w.resourcesCOffCLens) >= (1 << 30) { w.err = errTooManyResources return 0, w.err } if err := w.initialize(); err != nil { return 0, err } if err := w.write(resource); err != nil { return 0, err } if len(w.resourcesCOffCLens) == 0 { w.resourcesCOffCLens = make([]uint64, 1, 8) } id := OptResource(len(w.resourcesCOffCLens)) cOffset := w.dataSize - uint64(len(resource)) cLength := calcCLength(len(resource)) w.resourcesCOffCLens = append(w.resourcesCOffCLens, cOffset|(cLength<<48)) return id, nil } // AddChunk adds a chunk of compressed data - the (primary, secondary, // tertiary) tuple - to the RAC file. Decompressing that chunk should produce // dRangeSize bytes, although the ChunkWriter does not attempt to verify that. // // The OptResource arguments may be zero, meaning that no resource is used. In // that case, the corresponding STag or TTag (see the RAC specification for // further discussion) will be 0xFF. // // The caller may modify primary's contents after this method returns. func (w *ChunkWriter) AddChunk( dRangeSize uint64, codec Codec, primary []byte, secondary OptResource, tertiary OptResource) error { if w.err != nil { return w.err } if dRangeSize == 0 { return nil } if (dRangeSize > MaxSize) || ((w.dFileSize + dRangeSize) > MaxSize) { w.err = errTooMuchInput return w.err } if len(w.leafNodes) >= (1 << 30) { w.err = errTooManyChunks return w.err } if err := w.initialize(); err != nil { return err } if len(w.leafNodes) == 0 { if !codec.Valid() { return errInvalidCodec } w.codec = codec } else if w.codec != codec { w.err = errors.New("rac: TODO: support writing multiple Codecs") return w.err } if err := w.write(primary); err != nil { return err } cOffset := w.dataSize - uint64(len(primary)) cLength := calcCLength(len(primary)) w.dFileSize += dRangeSize w.leafNodes = append(w.leafNodes, wNode{ dRangeSize: dRangeSize, cOffsetCLength: cOffset | (cLength << 48), secondary: secondary, tertiary: tertiary, codec: codec, }) return nil } var emptyRACFile = [32]byte{ 0x72, 0xC3, 0x63, 0x01, 0x0D, 0xF8, 0x00, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xFF, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, } // Close writes the RAC index to w.Writer and marks that w accepts no further // method calls. // // For a one pass encoding, no further action is taken. For a two pass encoding // (i.e. IndexLocationAtStart), it then copies w.TempFile to w.Writer. Either // way, if this method returns nil error, the entirety of what was written to // w.Writer constitutes a valid RAC file. // // Closing the underlying w.Writer, w.TempFile or both is the responsibility of // the ChunkWriter caller, not the ChunkWriter itself. // // It is not necessary to call Close, e.g. if an earlier AddXxx call returned a // non-nil error. Unlike an os.File, failing to call ChunkWriter.Close will not // leak resources such as file descriptors. func (w *ChunkWriter) Close() error { if w.err != nil { return w.err } if !w.initialized { if err := w.checkParameters(); err != nil { return err } } if len(w.leafNodes) == 0 { _, err := w.Writer.Write(emptyRACFile[:]) return err } rootNode := gather(w.leafNodes, w.codec.isLong()) indexSize := rootNode.calcEncodedSize(0, w.IndexLocation == IndexLocationAtEnd) nw := &nodeWriter{ w: w.Writer, resourcesCOffCLens: w.resourcesCOffCLens, } if w.IndexLocation == IndexLocationAtEnd { nw.indexCOffset = w.roundUpToCPageBoundary(w.dataSize) nw.cFileSize = nw.indexCOffset + indexSize } else { nw.dataCOffset = w.roundUpToCPageBoundary(indexSize) nw.cFileSize = nw.dataCOffset + w.dataSize } if nw.cFileSize > MaxSize { w.err = errTooMuchInput return w.err } if w.IndexLocation == IndexLocationAtEnd { // Write the align-to-CPageSize padding. if w.CPageSize > 0 { if err := w.padToPageSize(w.Writer, w.dataSize); err != nil { return err } } // Write the index. The compressed data has already been written. if err := nw.writeIndex(&rootNode, true); err != nil { w.err = err return err } } else { expectedTempFileSize := w.dataSize // Write the index. if err := nw.writeIndex(&rootNode, false); err != nil { w.err = err return err } // Write the align-to-CPageSize padding. if w.CPageSize > 0 { if err := w.padToPageSize(w.Writer, indexSize); err != nil { return err } } // Write the compressed data. if s, ok := w.TempFile.(io.Seeker); ok { if _, err := s.Seek(w.tempFileSeekStart, io.SeekStart); err != nil { w.err = err return err } } if n, err := io.Copy(w.Writer, w.TempFile); err != nil { w.err = err return err } else if uint64(n) != expectedTempFileSize { w.err = errInconsistentCompressedSize return w.err } } w.err = errAlreadyClosed return nil } // wNode is the ChunkWriter's representation of a node. type wNode struct { dRangeSize uint64 children []wNode // Excludes resource-node chidren. resources []int cOffsetCLength uint64 secondary OptResource tertiary OptResource codec Codec } // calcEncodedSize accumulates the encoded size of n and its children, // traversing the nodes in depth-first order. // // As a side effect, it also sets n.cOffsetCLength for branch nodes. func (n *wNode) calcEncodedSize(accumulator uint64, rootAndIsAtEnd bool) (newAccumulator uint64) { arity := len(n.children) + len(n.resources) if arity == 0 { return accumulator } if n.codec.isLong() { arity++ } size := (arity * 16) + 16 cLength := calcCLength(size) if rootAndIsAtEnd { for i := range n.children { accumulator = n.children[i].calcEncodedSize(accumulator, false) } } n.cOffsetCLength = accumulator | (cLength << 48) accumulator += uint64(size) if !rootAndIsAtEnd { for i := range n.children { accumulator = n.children[i].calcEncodedSize(accumulator, false) } } return accumulator } type nodeWriter struct { w io.Writer cFileSize uint64 dataCOffset uint64 indexCOffset uint64 resourcesCOffCLens []uint64 // A wNode's maximum arity is 255, so a wNode's maximum size in bytes is // ((255 * 16) + 16), which is 4096. buffer [4096]byte } func (w *nodeWriter) writeIndex(n *wNode, rootAndIsAtEnd bool) error { const tagFF = 0xFF << 56 if rootAndIsAtEnd { for i, o := range n.children { if len(o.children) != 0 { if err := w.writeIndex(&n.children[i], false); err != nil { return err } } } } if !n.codec.Valid() { return errors.New("rac: TODO: support writing multiple Codecs") } buf, dPtr := w.buffer[:], uint64(0) arity := uint64(len(n.children) + len(n.resources)) if n.codec.isLong() { arity++ } if arity > 0xFF { return errInternalArityIsTooLarge } // Codec Element 'DPtr|Reserved0|TTag', if present. if n.codec.isLong() { putU64LE(buf, 0xFD00000000000000) buf = buf[8:] } // DPtr's. We write resources before regular children (non-resources), so // that any TTag that refers to a resource always avoids the [0xC0, 0xFD] // reserved zone. The ratio of resources to regulars is at most 2:1, as // every chunk uses exactly 1 regular slot and up to 2 resource slots, and // that reserved zone is less than 33% of the [0x00, 0xFF] space. for i := range n.resources { putU64LE(buf[8*i:], dPtr|tagFF) } buf = buf[8*len(n.resources):] for i, o := range n.children { tag := uint64(0xFE << 56) if len(o.children) == 0 { tag = resourceToTag(n.resources, o.tertiary) } putU64LE(buf[8*i:], dPtr|tag) dPtr += o.dRangeSize } buf = buf[8*len(n.children):] // DPtrMax and the CodecByte. By construction, a Long Codec's 'c64' value // is always 0, as the Codec Element is always in position 0. codecHighByte := uint64(n.codec) & 0xFF00000000000000 putU64LE(buf, dPtr|codecHighByte) buf = buf[8:] // Codec Element 'CPtr|CLen|STag', if present. if n.codec.isLong() { putU64LE(buf, uint64(n.codec)&0x00FFFFFFFFFFFFFF) buf = buf[8:] } // CPtr's. While the RAC format allows otherwise, this nodeWriter always // keeps the CBias at zero, so that a CPtr equals a COffset. for i, res := range n.resources { cOffsetCLength := w.resourcesCOffCLens[res] + w.dataCOffset putU64LE(buf[8*i:], cOffsetCLength|tagFF) } buf = buf[8*len(n.resources):] for i, o := range n.children { cOffsetCLength := o.cOffsetCLength if len(o.children) == 0 { cOffsetCLength += w.dataCOffset } else { cOffsetCLength += w.indexCOffset } putU64LE(buf[8*i:], cOffsetCLength|resourceToTag(n.resources, o.secondary)) } buf = buf[8*len(n.children):] // CPtrMax. const version = 0x01 putU64LE(buf, w.cFileSize|(version<<48)|(arity<<56)) // Magic and Arity. w.buffer[0] = 0x72 w.buffer[1] = 0xC3 w.buffer[2] = 0x63 w.buffer[3] = uint8(arity) // Checksum. size := (arity * 16) + 16 checksum := crc32.ChecksumIEEE(w.buffer[6:size]) checksum ^= checksum >> 16 w.buffer[4] = uint8(checksum >> 0) w.buffer[5] = uint8(checksum >> 8) if _, err := w.w.Write(w.buffer[:size]); err != nil { return err } if !rootAndIsAtEnd { for i, o := range n.children { if len(o.children) != 0 { if err := w.writeIndex(&n.children[i], false); err != nil { return err } } } } return nil } func calcCLength(primarySize int) uint64 { if primarySize <= 0 { return 1 } // Divide by 1024, rounding up. primarySize = ((primarySize - 1) >> 10) + 1 if primarySize > 255 { return 0 } return uint64(primarySize) } func resourceToTag(resources []int, r OptResource) uint64 { if r != 0 { for i, res := range resources { if res == int(r) { return uint64(i) << 56 } } } return 0xFF << 56 } // gather brings the given nodes into a tree, such that every branch node's // arity (the count of its resource and non-resource child nodes) is at most // 0xFF. It returns the root of that tree. // // TODO: it currently builds a tree where all leaf nodes have equal depth. For // small RAC files (with only one branch node: the mandatory root node), this // doesn't matter. For larger RAC files, it might be better to build an uneven // tree to minimize average leaf node depth, with some branch nodes having both // branch node children and leaf node children. It might even be worth ensuring // that the root node always directly contains the first and last leaf nodes as // children, as those leaves presumably contain the most commonly accessed // parts of the decompressed file. // // If doing this TODO, we'd also have to change the "codec = codecMixBit | // CodecZeroes" line below, as it assumes that no branch nodes have both branch // node children and leaf node children. func gather(nodes []wNode, codecIsLong bool) wNode { if len(nodes) == 0 { panic("gather: no nodes") } resources := map[OptResource]bool{} arityBudget := 0xFF if codecIsLong { arityBudget = 0xFE } for { i, j, arity, newNodes := 0, 0, 0, []wNode(nil) for ; j < len(nodes); j++ { o := &nodes[j] new2 := (o.secondary != 0) && !resources[o.secondary] new3 := (o.tertiary != 0) && !resources[o.tertiary] arity += 1 + btoi(new2) + btoi(new3) if arity <= arityBudget { if new2 { resources[o.secondary] = true } if new3 { resources[o.tertiary] = true } continue } newNodes = append(newNodes, makeBranch(nodes[i:j], resources)) if len(resources) != 0 { resources = map[OptResource]bool{} } i = j arity = 1 if o.secondary != 0 { resources[o.secondary] = true arity++ } if o.tertiary != 0 { resources[o.tertiary] = true arity++ } } if i == 0 { return makeBranch(nodes, resources) } newNodes = append(newNodes, makeBranch(nodes[i:], resources)) if len(resources) != 0 { resources = map[OptResource]bool{} } nodes, newNodes = newNodes, nil } } func makeBranch(children []wNode, resMap map[OptResource]bool) wNode { dRangeSize, codec := uint64(0), Codec(0) for i, c := range children { dRangeSize += c.dRangeSize if i == 0 { codec = c.codec } else if codec != c.codec { // We construct the node tree so that a branch node's children are // either all branch nodes or all leaf nodes. If they are all leaf // nodes (with the same Codec), or all branch nodes with the same // Codec, the parent uses the same Codec. If they are all branch // nodes, with different Codecs, we set the Mix bit and since it // has no leaf nodes, we might as well use CodecZeroes. codec = codecMixBit | CodecZeroes } } resList := []int(nil) if len(resMap) != 0 { resList = make([]int, 0, len(resMap)) for res := range resMap { resList = append(resList, int(res)) } sort.Ints(resList) } return wNode{ dRangeSize: dRangeSize, children: children, resources: resList, cOffsetCLength: invalidCOffsetCLength, codec: codec, } }