Make the rac codec per-chunk, not per-chunkWriter

This commit is contained in:
Nigel Tao committed 2019-08-08 14:39:18 +10:00
1 parent 2f1d6c77c9
commit fe2b4e236d
4 files changed
+34 -51

No files matched your search

+22 -33
View File
@@ -58,13 +58,6 @@ type ChunkWriter struct {
// Nil is an invalid value.
Writer io.Writer
// Codec is the compression codec for the RAC file.
//
// See the RAC specification for further discussion.
//
// Zero is an invalid value.
Codec Codec
// IndexLocation is whether the index is at the start or end of the RAC
// file.
//
@@ -127,6 +120,9 @@ type ChunkWriter struct {
// initialized is set true after the first AddXxx call.
initialized bool
// codec is the compression codec for the RAC file.
codec Codec
// err is the first error encountered. It is sticky: once a non-nil error
// occurs, all public methods will return that error.
err error
@@ -168,10 +164,6 @@ func (w *ChunkWriter) checkParameters() error {
w.err = errors.New("rac: invalid Writer")
return w.err
}
if w.Codec == 0 {
w.err = errors.New("rac: invalid Codec")
return w.err
}
if !isZeroOrAPowerOf2(w.CPageSize) || (w.CPageSize > MaxSize) {
w.err = errors.New("rac: invalid CPageSize")
return w.err
@@ -346,7 +338,8 @@ func (w *ChunkWriter) AddResource(resource []byte) (OptResource, error) {
//
// The caller may modify primary's contents after this method returns.
func (w *ChunkWriter) AddChunk(
dRangeSize uint64, primary []byte, secondary OptResource, tertiary OptResource) error {
dRangeSize uint64, primary []byte,
secondary OptResource, tertiary OptResource, codec Codec) error {
if dRangeSize == 0 {
return nil
@@ -359,6 +352,10 @@ func (w *ChunkWriter) AddChunk(
w.err = errors.New("rac: too many chunks")
return w.err
}
if codec == 0 {
w.err = errors.New("rac: invalid Codec")
return w.err
}
if err := w.initialize(); err != nil {
return err
@@ -375,25 +372,16 @@ func (w *ChunkWriter) AddChunk(
cOffsetCLength: cOffset | (cLength << 48),
secondary: secondary,
tertiary: tertiary,
codec: codec,
})
return nil
}
func writeEmpty(w io.Writer, codec Codec) error {
buf := [32]byte{
0x72, 0xC3, 0x63, 0x01, 0x00, 0x00, 0x00, 0xFF,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, uint8(codec),
0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xFF,
0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01,
}
checksum := crc32.ChecksumIEEE(buf[6:])
checksum ^= checksum >> 16
buf[4] = uint8(checksum >> 0)
buf[5] = uint8(checksum >> 8)
_, err := w.Write(buf[:])
return err
var emptyRACFile = [32]byte{
0x72, 0xC3, 0x63, 0x01, 0xE8, 0xB4, 0x00, 0xFF,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xFF,
0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01,
}
// Close writes the RAC index to w.Writer and marks that w accepts no further
@@ -421,7 +409,8 @@ func (w *ChunkWriter) Close() error {
}
if len(w.leafNodes) == 0 {
return writeEmpty(w.Writer, w.Codec)
_, err := w.Writer.Write(emptyRACFile[:])
return err
}
rootNode := gather(w.leafNodes)
indexSize := rootNode.calcEncodedSize(0, w.IndexLocation == IndexLocationAtEnd)
@@ -429,7 +418,6 @@ func (w *ChunkWriter) Close() error {
nw := &nodeWriter{
w: w.Writer,
resourcesCOffCLens: w.resourcesCOffCLens,
codec: w.Codec,
}
if w.IndexLocation == IndexLocationAtEnd {
@@ -504,6 +492,7 @@ type wNode struct {
cOffsetCLength uint64
secondary OptResource
tertiary OptResource
codec Codec
}
// calcEncodedSize accumulates the encoded size of n and its children,
@@ -544,8 +533,6 @@ type nodeWriter struct {
indexCOffset uint64
resourcesCOffCLens []uint64
codec Codec
// A wNode's maximum arity is 255, so a wNode's maximum size in bytes is
// ((255 * 16) + 16), which is 4096.
buffer [4096]byte
@@ -590,7 +577,7 @@ func (w *nodeWriter) writeIndex(n *wNode, rootAndIsAtEnd bool) error {
buf = buf[8*len(n.children):]
// DPtrMax.
putU64LE(buf, dPtr|(uint64(w.codec)<<56))
putU64LE(buf, dPtr|(uint64(n.codec)<<56))
buf = buf[8:]
// CPtr's. While the RAC format allows otherwise, this nodeWriter always
@@ -738,9 +725,10 @@ func gather(nodes []wNode) wNode {
}
func makeBranch(children []wNode, resMap map[OptResource]bool) wNode {
dRangeSize := uint64(0)
dRangeSize, codec := uint64(0), Codec(0)
for _, c := range children {
dRangeSize += c.dRangeSize
codec |= c.codec
}
resList := []int(nil)
@@ -757,5 +745,6 @@ func makeBranch(children []wNode, resMap map[OptResource]bool) wNode {
children: children,
resources: resList,
cOffsetCLength: invalidCOffsetCLength,
codec: codec,
}
}
+2 -4
View File
@@ -71,9 +71,8 @@ func Example_indexLocationAtEnd() {
buf := &bytes.Buffer{}
w := &rac.ChunkWriter{
Writer: buf,
Codec: rac.CodecZlib,
}
if err := w.AddChunk(uint64(len(src)), enc, 0, 0); err != nil {
if err := w.AddChunk(uint64(len(src)), enc, 0, 0, rac.CodecZlib); err != nil {
log.Fatalf("AddChunk: %v", err)
}
if err := w.Close(); err != nil {
@@ -102,7 +101,6 @@ func Example_indexLocationAtStart() {
buf := &bytes.Buffer{}
w := &rac.ChunkWriter{
Writer: buf,
Codec: rac.CodecZlib,
IndexLocation: rac.IndexLocationAtStart,
TempFile: &bytes.Buffer{},
}
@@ -143,7 +141,7 @@ func Example_indexLocationAtStart() {
}
encodedChunk := b.Bytes()
if err := w.AddChunk(uint64(len(chunk)), encodedChunk, dictResource, 0); err != nil {
if err := w.AddChunk(uint64(len(chunk)), encodedChunk, dictResource, 0, rac.CodecZlib); err != nil {
log.Fatalf("AddChunk: %v", err)
}
+7 -10
View File
@@ -60,7 +60,7 @@ func undoHexDump(s string) (ret []byte) {
}
const writerWantEmpty = "" +
"00000000 72 c3 63 01 30 14 00 ff 00 00 00 00 00 00 00 ee |r.c.0...........|\n" +
"00000000 72 c3 63 01 e8 b4 00 ff 00 00 00 00 00 00 00 01 |r.c.............|\n" +
"00000010 20 00 00 00 00 00 01 ff 20 00 00 00 00 00 01 01 | ....... .......|\n"
const writerWantILAEnd = "" +
@@ -195,7 +195,6 @@ func testWriter(iloc IndexLocation, tempFile io.ReadWriter, cPageSize uint64, em
buf := &bytes.Buffer{}
w := &ChunkWriter{
Writer: buf,
Codec: fakeCodec,
IndexLocation: iloc,
TempFile: tempFile,
CPageSize: cPageSize,
@@ -208,9 +207,9 @@ func testWriter(iloc IndexLocation, tempFile io.ReadWriter, cPageSize uint64, em
// These {Aa,Bb,Cc} chunks are also used in the reader test.
res0, _ := w.AddResource([]byte("Rrr"))
res1, _ := w.AddResource([]byte("Ss"))
_ = w.AddChunk(0x11, []byte("Aaa"), 0, 0)
_ = w.AddChunk(0x22, []byte("Bbbb"), res0, 0)
_ = w.AddChunk(0x44, []byte("Cccccccccc12"), res0, res1)
_ = w.AddChunk(0x11, []byte("Aaa"), 0, 0, fakeCodec)
_ = w.AddChunk(0x22, []byte("Bbbb"), res0, 0, fakeCodec)
_ = w.AddChunk(0x44, []byte("Cccccccccc12"), res0, res1, fakeCodec)
}
if err := w.Close(); err != nil {
@@ -246,7 +245,6 @@ func TestMultiLevelIndex(t *testing.T) {
buf := &bytes.Buffer{}
w := &ChunkWriter{
Writer: buf,
Codec: fakeCodec,
IndexLocation: IndexLocationAtStart,
TempFile: &bytes.Buffer{},
}
@@ -285,7 +283,7 @@ func TestMultiLevelIndex(t *testing.T) {
primary[0] = 'q'
}
primaries = append(primaries, primary...)
_ = w.AddChunk(0x10000, primary, secondary, tertiary)
_ = w.AddChunk(0x10000, primary, secondary, tertiary, fakeCodec)
}
if err := w.Close(); err != nil {
@@ -383,7 +381,6 @@ loop:
buf := &bytes.Buffer{}
w := &ChunkWriter{
Writer: buf,
Codec: fakeCodec,
}
if i > 0 {
w.IndexLocation = IndexLocationAtStart
@@ -394,9 +391,9 @@ loop:
res, _ := w.AddResource(data)
for i := 0; i < 1000; i++ {
if i == 2*255 {
_ = w.AddChunk(1, data, res, 0)
_ = w.AddChunk(1, data, res, 0, fakeCodec)
} else {
_ = w.AddChunk(1, data, 0, 0)
_ = w.AddChunk(1, data, 0, 0, fakeCodec)
}
}
if err := w.Close(); err != nil {
+3 -4
View File
@@ -276,7 +276,6 @@ func (w *Writer) initialize() error {
}
w.racWriter.Writer = w.Writer
w.racWriter.Codec = rac.CodecZlib
w.racWriter.IndexLocation = w.IndexLocation
w.racWriter.TempFile = w.TempFile
w.racWriter.CPageSize = w.CPageSize
@@ -348,7 +347,7 @@ func (w *Writer) writeDChunks(eof bool) error {
return err
}
if err := w.racWriter.AddChunk(dSize, cBytes, 0, 0); err != nil {
if err := w.racWriter.AddChunk(dSize, cBytes, 0, 0, rac.CodecZlib); err != nil {
w.err = err
return err
}
@@ -415,7 +414,7 @@ func (w *Writer) tryCChunk(targetDChunkSize uint64, force bool) error {
}
fallthrough
case uint64(len(cBytes)) == w.cChunkSize:
if err := w.racWriter.AddChunk(dSize, cBytes, 0, 0); err != nil {
if err := w.racWriter.AddChunk(dSize, cBytes, 0, 0, rac.CodecZlib); err != nil {
w.err = err
return err
}
@@ -432,7 +431,7 @@ func (w *Writer) tryCChunk(targetDChunkSize uint64, force bool) error {
w.err = errCChunkSizeIsTooSmall
return w.err
}
if err := w.racWriter.AddChunk(uint64(dLen), cBytes[:eLen], 0, 0); err != nil {
if err := w.racWriter.AddChunk(uint64(dLen), cBytes[:eLen], 0, 0, rac.CodecZlib); err != nil {
w.err = err
return err
}