Encode RAC implicit zeroes when CChunkSize set

This commit is contained in:
Nigel Tao committed 2019-08-09 15:10:43 +10:00
1 parent ac4fed2275
commit 706e7a2474
2 files changed
+73 -29

No files matched your search

+31 -3
View File
@@ -115,6 +115,27 @@ func (b *writeBuffer) advance(n uint64) {
b.p = len(b.prev)
}
// advancePastLeadingZeroes consumes the next n bytes, all of which are '\x00'.
func (b *writeBuffer) advancePastLeadingZeroes() (n uint64) {
// Consume zeroes from b.prev.
i := b.p
for ; (i < len(b.prev)) && (b.prev[i] == '\x00'); i++ {
}
if i == b.p {
return 0
}
n = uint64(i - b.p)
b.p = i
// Consume zeroes from b.curr.
i = 0
for ; (i < len(b.curr)) && (b.curr[i] == '\x00'); i++ {
}
n += uint64(i)
b.curr = b.curr[i:]
return n
}
// compact moves and copies any unprocessed bytes in b.prev and b.curr to be at
// the start of b.prev.
func (b *writeBuffer) compact() {
@@ -371,6 +392,10 @@ func (w *Writer) Write(p []byte) (int, error) {
if err := w.initialize(); err != nil {
return 0, err
}
if n := uint64(len(p)); (n > MaxSize) || (w.uncompressed.length() > (MaxSize - n)) {
w.err = errTooMuchInput
return 0, w.err
}
w.uncompressed.extend(p)
n, err := len(p), w.write(false)
w.uncompressed.compact()
@@ -494,11 +519,12 @@ func (w *Writer) tryCChunk(targetDChunkSize uint64, force bool) error {
}
fallthrough
case uint64(len(cBytes)) == w.cChunkSize:
w.uncompressed.advance(dSize)
dSize += w.uncompressed.advancePastLeadingZeroes()
if err := w.chunkWriter.AddChunk(dSize, cBytes, res2, res3, codec); err != nil {
w.err = err
return err
}
w.uncompressed.advance(dSize)
return nil
}
@@ -511,11 +537,13 @@ func (w *Writer) tryCChunk(targetDChunkSize uint64, force bool) error {
w.err = errCChunkSizeIsTooSmall
return w.err
}
if err := w.chunkWriter.AddChunk(uint64(dLen), cBytes[:eLen], res2, res3, codec); err != nil {
dSize, cBytes = uint64(dLen), cBytes[:eLen]
w.uncompressed.advance(dSize)
dSize += w.uncompressed.advancePastLeadingZeroes()
if err := w.chunkWriter.AddChunk(dSize, cBytes, res2, res3, codec); err != nil {
w.err = err
return err
}
w.uncompressed.advance(uint64(dLen))
return nil
}
+42 -26
View File
@@ -56,11 +56,12 @@ const (
"\x28\x4A\x4D\x85\x71\x00\x01\x00\x00\xFF\xFF\x21\x6E\x04\x66"
)
func racCompress(original []byte, dChunkSize uint64, resourcesData [][]byte) ([]byte, error) {
func racCompress(original []byte, cChunkSize uint64, dChunkSize uint64, resourcesData [][]byte) ([]byte, error) {
buf := &bytes.Buffer{}
w := &rac.Writer{
Writer: buf,
CodecWriter: &CodecWriter{},
CChunkSize: cChunkSize,
DChunkSize: dChunkSize,
ResourcesData: resourcesData,
}
@@ -175,35 +176,50 @@ func TestReaderConcatenation(t *testing.T) {
}
func TestZeroedBytes(t *testing.T) {
original := []byte("abcde\x00\x00\x00\x00j")
compressed, err := racCompress(original, 8, nil)
if err != nil {
t.Fatal(err)
}
original := make([]byte, 32)
original[0] = 'a'
original[1] = 'b'
original[2] = 'c'
original[20] = 'm'
original[31] = 'z'
r := &rac.Reader{
ReadSeeker: bytes.NewReader(compressed),
CompressedSize: int64(len(compressed)),
CodecReaders: []rac.CodecReader{&CodecReader{}},
}
for i := 0; i <= len(original); i++ {
want := original[i:]
got := make([]byte, len(want))
for j := range got {
got[j] = '?'
for i := 0; i < 2; i++ {
cChunkSize, dChunkSize := uint64(0), uint64(0)
if i == 0 {
cChunkSize = 10
} else {
dChunkSize = 8
}
if _, err := r.Seek(int64(i), io.SeekStart); err != nil {
t.Errorf("i=%d: Seek: %v", i, err)
continue
compressed, err := racCompress(original, cChunkSize, dChunkSize, nil)
if err != nil {
t.Fatalf("i=%d: racCompress: %v", i, err)
}
if _, err := io.ReadFull(r, got); err != nil {
t.Errorf("i=%d: ReadFull: %v", i, err)
continue
r := &rac.Reader{
ReadSeeker: bytes.NewReader(compressed),
CompressedSize: int64(len(compressed)),
CodecReaders: []rac.CodecReader{&CodecReader{}},
}
if !bytes.Equal(got, want) {
t.Errorf("i=%d: got\n% 02x\nwant\n% 02x", i, got, want)
continue
for j := 0; j <= len(original); j++ {
want := original[j:]
got := make([]byte, len(want))
for j := range got {
got[j] = '?'
}
if _, err := r.Seek(int64(j), io.SeekStart); err != nil {
t.Errorf("i=%d, j=%d: Seek: %v", i, j, err)
continue
}
if _, err := io.ReadFull(r, got); err != nil {
t.Errorf("i=%d, j=%d: ReadFull: %v", i, j, err)
continue
}
if !bytes.Equal(got, want) {
t.Errorf("i=%d, j=%d: got\n% 02x\nwant\n% 02x", i, j, got, want)
continue
}
}
}
}
@@ -233,7 +249,7 @@ func TestSharedDictionary(t *testing.T) {
}
// Compress.
compressed, err := racCompress(original, n, resourcesData)
compressed, err := racCompress(original, 0, n, resourcesData)
if err != nil {
t.Fatalf("i=%d: racCompress: %v", i, err)
}