Have flatecut.Cut take an optional io.Writer arg

This commit is contained in:
Nigel Tao committed 2019-04-20 10:35:58 +10:00
1 parent b54f3b4df3
commit a79eb83f65
3 files changed
+61 -22

No files matched your search

+12 -6
View File
@@ -43,7 +43,7 @@ func clone(b []byte) []byte {
func Test(t *testing.T,
smallestValidMaxEncodedLen int,
cut func([]byte, int) (int, int, error),
cut func(io.Writer, []byte, int) (int, int, error),
newReader func(io.Reader) (io.ReadCloser, error),
filenames []string) {
@@ -90,8 +90,9 @@ func Test(t *testing.T,
if maxEncodedLen < smallestValidMaxEncodedLen {
continue
}
w0 := &bytes.Buffer{}
encoded := clone(full)
encLen, decLen, err := cut(encoded, maxEncodedLen)
encLen, decLen, err := cut(w0, encoded, maxEncodedLen)
if err != nil {
t.Errorf("f=%q, mEL=%d: cut: %v", filename, maxEncodedLen, err)
continue
@@ -107,13 +108,13 @@ func Test(t *testing.T,
continue
}
w := &bytes.Buffer{}
w1 := &bytes.Buffer{}
r, err := newReader(bytes.NewReader(encoded[:encLen]))
if err != nil {
t.Errorf("f=%q, mEL=%d: newReader: %v", filename, maxEncodedLen, err)
continue
}
if n, err := io.Copy(w, r); err != nil {
if n, err := io.Copy(w1, r); err != nil {
t.Errorf("f=%q, mEL=%d: io.Copy: %v", filename, maxEncodedLen, err)
continue
} else if n != int64(decLen) {
@@ -122,6 +123,11 @@ func Test(t *testing.T,
continue
}
if !bytes.Equal(w0.Bytes(), w1.Bytes()) {
t.Errorf("f=%q, mEL=%d: decoded bytes were not equal", filename, maxEncodedLen)
continue
}
if (maxEncodedLen == len(encoded)) && (int64(decLen) != fullDecodedLen) {
t.Errorf("f=%q, mEL=%d: full decode: got %d, want %d",
filename, maxEncodedLen, decLen, fullDecodedLen)
@@ -138,7 +144,7 @@ func Test(t *testing.T,
func Benchmark(b *testing.B,
smallestValidMaxEncodedLen int,
cut func([]byte, int) (int, int, error),
cut func(io.Writer, []byte, int) (int, int, error),
newReader func(io.Reader) (io.ReadCloser, error),
filename string,
trimPrefix int,
@@ -168,7 +174,7 @@ func Benchmark(b *testing.B,
continue
}
encoded := clone(full)
if _, _, err := cut(encoded, maxEncodedLen); err != nil {
if _, _, err := cut(nil, encoded, maxEncodedLen); err != nil {
b.Fatalf("cut: %v", err)
}
if maxEncodedLen >= len(full) {
+34 -4
View File
@@ -23,14 +23,18 @@
package flatecut
import (
"bytes"
"compress/flate"
"errors"
"io"
)
var (
errMaxEncodedLenTooSmall = errors.New("flatecut: maxEncodedLen is too small")
errInternalNoProgress = errors.New("flatecut: internal: no progress")
errInternalSomeProgress = errors.New("flatecut: internal: some progress")
errInternalInconsistentDecodedLen = errors.New("flatecut: internal: inconsistent decodedLen")
errInternalNoProgress = errors.New("flatecut: internal: no progress")
errInternalSomeProgress = errors.New("flatecut: internal: some progress")
errInvalidBadBlockLength = errors.New("flatecut: invalid input: bad block length")
errInvalidBadBlockType = errors.New("flatecut: invalid input: bad block type")
@@ -335,11 +339,16 @@ func cutEmpty(encoded []byte, maxEncodedLen int) (encodedLen int, decodedLen int
// DEFLATE-compressed data, assuming that encoded starts off containing valid
// DEFLATE-compressed data.
//
// If a nil error is returned, then encodedLen <= maxEncodedLen will hold.
//
// Decompressing that modified, shorter byte slice produces a prefix (of length
// decodedLen) of the decompression of the original, longer byte slice.
//
// If w is non-nil, that prefix is also written to w. If a non-nil error is
// returned, incomplete data might still be written to w.
//
// It does not necessarily return the largest possible decodedLen.
func Cut(encoded []byte, maxEncodedLen int) (encodedLen int, decodedLen int, retErr error) {
func Cut(w io.Writer, encoded []byte, maxEncodedLen int) (encodedLen int, decodedLen int, retErr error) {
if maxEncodedLen < SmallestValidMaxEncodedLen {
return 0, 0, errMaxEncodedLenTooSmall
}
@@ -360,7 +369,28 @@ func Cut(encoded []byte, maxEncodedLen int) (encodedLen int, decodedLen int, ret
},
maxEncodedLen: maxEncodedLen,
}
return c.cut()
encodedLen, decodedLen, err := c.cut()
if err != nil {
return 0, 0, err
}
if w != nil {
// TODO: writing to w directly, in cutter's doStored and doHuffman,
// might be faster than re-walking the bitstream with compress/flate.
r := flate.NewReader(bytes.NewReader(encoded[:encodedLen]))
if n, err := io.Copy(w, r); err != nil {
r.Close()
return 0, 0, err
} else if n != int64(decodedLen) {
r.Close()
return 0, 0, errInternalInconsistentDecodedLen
}
if err := r.Close(); err != nil {
return 0, 0, err
}
}
return encodedLen, decodedLen, nil
}
type cutter struct {
+15 -12
View File
@@ -23,8 +23,6 @@
package zlibcut
import (
"bytes"
"compress/flate"
"errors"
"hash/adler32"
"io"
@@ -52,11 +50,16 @@ const (
// zlib-compressed data, assuming that encoded starts off containing valid
// zlib-compressed data.
//
// If a nil error is returned, then encodedLen <= maxEncodedLen will hold.
//
// Decompressing that modified, shorter byte slice produces a prefix (of length
// decodedLen) of the decompression of the original, longer byte slice.
//
// If w is non-nil, that prefix is also written to w. If a non-nil error is
// returned, incomplete data might still be written to w.
//
// It does not necessarily return the largest possible decodedLen.
func Cut(encoded []byte, maxEncodedLen int) (encodedLen int, decodedLen int, retErr error) {
func Cut(w io.Writer, encoded []byte, maxEncodedLen int) (encodedLen int, decodedLen int, retErr error) {
if len(encoded) < 2 {
return 0, 0, errInvalidNotEnoughData
}
@@ -85,7 +88,15 @@ func Cut(encoded []byte, maxEncodedLen int) (encodedLen int, decodedLen int, ret
return 0, 0, errMaxEncodedLenTooSmall
}
hasher := adler32.New()
if w == nil {
w = hasher
} else {
w = io.MultiWriter(w, hasher)
}
encodedLen, decodedLen, err := flatecut.Cut(
w,
encoded[payloadStart:len(encoded)-4],
maxEncodedLen-payloadStart-4,
)
@@ -93,15 +104,7 @@ func Cut(encoded []byte, maxEncodedLen int) (encodedLen int, decodedLen int, ret
return 0, 0, err
}
w := adler32.New()
r := bytes.NewReader(encoded[payloadStart : payloadStart+encodedLen])
if n, err := io.Copy(w, flate.NewReader(r)); err != nil {
return 0, 0, err
} else if n != int64(decodedLen) {
return 0, 0, errInternalInconsistentDecodedLen
}
hash := w.Sum32()
hash := hasher.Sum32()
hashBytes := encoded[payloadStart+encodedLen : payloadStart+encodedLen+4]
hashBytes[0] = uint8(hash >> 24)
hashBytes[1] = uint8(hash >> 16)