diff --git a/internal/testcut/testcut.go b/internal/testcut/testcut.go index ba2d0ca0..f15b8a53 100644 --- a/internal/testcut/testcut.go +++ b/internal/testcut/testcut.go @@ -43,7 +43,7 @@ func clone(b []byte) []byte { func Test(t *testing.T, smallestValidMaxEncodedLen int, - cut func([]byte, int) (int, int, error), + cut func(io.Writer, []byte, int) (int, int, error), newReader func(io.Reader) (io.ReadCloser, error), filenames []string) { @@ -90,8 +90,9 @@ func Test(t *testing.T, if maxEncodedLen < smallestValidMaxEncodedLen { continue } + w0 := &bytes.Buffer{} encoded := clone(full) - encLen, decLen, err := cut(encoded, maxEncodedLen) + encLen, decLen, err := cut(w0, encoded, maxEncodedLen) if err != nil { t.Errorf("f=%q, mEL=%d: cut: %v", filename, maxEncodedLen, err) continue @@ -107,13 +108,13 @@ func Test(t *testing.T, continue } - w := &bytes.Buffer{} + w1 := &bytes.Buffer{} r, err := newReader(bytes.NewReader(encoded[:encLen])) if err != nil { t.Errorf("f=%q, mEL=%d: newReader: %v", filename, maxEncodedLen, err) continue } - if n, err := io.Copy(w, r); err != nil { + if n, err := io.Copy(w1, r); err != nil { t.Errorf("f=%q, mEL=%d: io.Copy: %v", filename, maxEncodedLen, err) continue } else if n != int64(decLen) { @@ -122,6 +123,11 @@ func Test(t *testing.T, continue } + if !bytes.Equal(w0.Bytes(), w1.Bytes()) { + t.Errorf("f=%q, mEL=%d: decoded bytes were not equal", filename, maxEncodedLen) + continue + } + if (maxEncodedLen == len(encoded)) && (int64(decLen) != fullDecodedLen) { t.Errorf("f=%q, mEL=%d: full decode: got %d, want %d", filename, maxEncodedLen, decLen, fullDecodedLen) @@ -138,7 +144,7 @@ func Test(t *testing.T, func Benchmark(b *testing.B, smallestValidMaxEncodedLen int, - cut func([]byte, int) (int, int, error), + cut func(io.Writer, []byte, int) (int, int, error), newReader func(io.Reader) (io.ReadCloser, error), filename string, trimPrefix int, @@ -168,7 +174,7 @@ func Benchmark(b *testing.B, continue } encoded := clone(full) - if _, _, err := cut(encoded, maxEncodedLen); err != nil { + if _, _, err := cut(nil, encoded, maxEncodedLen); err != nil { b.Fatalf("cut: %v", err) } if maxEncodedLen >= len(full) { diff --git a/lib/flatecut/flatecut.go b/lib/flatecut/flatecut.go index f359a524..714dcc6b 100644 --- a/lib/flatecut/flatecut.go +++ b/lib/flatecut/flatecut.go @@ -23,14 +23,18 @@ package flatecut import ( + "bytes" + "compress/flate" "errors" + "io" ) var ( errMaxEncodedLenTooSmall = errors.New("flatecut: maxEncodedLen is too small") - errInternalNoProgress = errors.New("flatecut: internal: no progress") - errInternalSomeProgress = errors.New("flatecut: internal: some progress") + errInternalInconsistentDecodedLen = errors.New("flatecut: internal: inconsistent decodedLen") + errInternalNoProgress = errors.New("flatecut: internal: no progress") + errInternalSomeProgress = errors.New("flatecut: internal: some progress") errInvalidBadBlockLength = errors.New("flatecut: invalid input: bad block length") errInvalidBadBlockType = errors.New("flatecut: invalid input: bad block type") @@ -335,11 +339,16 @@ func cutEmpty(encoded []byte, maxEncodedLen int) (encodedLen int, decodedLen int // DEFLATE-compressed data, assuming that encoded starts off containing valid // DEFLATE-compressed data. // +// If a nil error is returned, then encodedLen <= maxEncodedLen will hold. +// // Decompressing that modified, shorter byte slice produces a prefix (of length // decodedLen) of the decompression of the original, longer byte slice. // +// If w is non-nil, that prefix is also written to w. If a non-nil error is +// returned, incomplete data might still be written to w. +// // It does not necessarily return the largest possible decodedLen. -func Cut(encoded []byte, maxEncodedLen int) (encodedLen int, decodedLen int, retErr error) { +func Cut(w io.Writer, encoded []byte, maxEncodedLen int) (encodedLen int, decodedLen int, retErr error) { if maxEncodedLen < SmallestValidMaxEncodedLen { return 0, 0, errMaxEncodedLenTooSmall } @@ -360,7 +369,28 @@ func Cut(encoded []byte, maxEncodedLen int) (encodedLen int, decodedLen int, ret }, maxEncodedLen: maxEncodedLen, } - return c.cut() + encodedLen, decodedLen, err := c.cut() + if err != nil { + return 0, 0, err + } + + if w != nil { + // TODO: writing to w directly, in cutter's doStored and doHuffman, + // might be faster than re-walking the bitstream with compress/flate. + r := flate.NewReader(bytes.NewReader(encoded[:encodedLen])) + if n, err := io.Copy(w, r); err != nil { + r.Close() + return 0, 0, err + } else if n != int64(decodedLen) { + r.Close() + return 0, 0, errInternalInconsistentDecodedLen + } + if err := r.Close(); err != nil { + return 0, 0, err + } + } + + return encodedLen, decodedLen, nil } type cutter struct { diff --git a/lib/zlibcut/zlibcut.go b/lib/zlibcut/zlibcut.go index 4f7da1c6..6554ea11 100644 --- a/lib/zlibcut/zlibcut.go +++ b/lib/zlibcut/zlibcut.go @@ -23,8 +23,6 @@ package zlibcut import ( - "bytes" - "compress/flate" "errors" "hash/adler32" "io" @@ -52,11 +50,16 @@ const ( // zlib-compressed data, assuming that encoded starts off containing valid // zlib-compressed data. // +// If a nil error is returned, then encodedLen <= maxEncodedLen will hold. +// // Decompressing that modified, shorter byte slice produces a prefix (of length // decodedLen) of the decompression of the original, longer byte slice. // +// If w is non-nil, that prefix is also written to w. If a non-nil error is +// returned, incomplete data might still be written to w. +// // It does not necessarily return the largest possible decodedLen. -func Cut(encoded []byte, maxEncodedLen int) (encodedLen int, decodedLen int, retErr error) { +func Cut(w io.Writer, encoded []byte, maxEncodedLen int) (encodedLen int, decodedLen int, retErr error) { if len(encoded) < 2 { return 0, 0, errInvalidNotEnoughData } @@ -85,7 +88,15 @@ func Cut(encoded []byte, maxEncodedLen int) (encodedLen int, decodedLen int, ret return 0, 0, errMaxEncodedLenTooSmall } + hasher := adler32.New() + if w == nil { + w = hasher + } else { + w = io.MultiWriter(w, hasher) + } + encodedLen, decodedLen, err := flatecut.Cut( + w, encoded[payloadStart:len(encoded)-4], maxEncodedLen-payloadStart-4, ) @@ -93,15 +104,7 @@ func Cut(encoded []byte, maxEncodedLen int) (encodedLen int, decodedLen int, ret return 0, 0, err } - w := adler32.New() - r := bytes.NewReader(encoded[payloadStart : payloadStart+encodedLen]) - if n, err := io.Copy(w, flate.NewReader(r)); err != nil { - return 0, 0, err - } else if n != int64(decodedLen) { - return 0, 0, errInternalInconsistentDecodedLen - } - - hash := w.Sum32() + hash := hasher.Sum32() hashBytes := encoded[payloadStart+encodedLen : payloadStart+encodedLen+4] hashBytes[0] = uint8(hash >> 24) hashBytes[1] = uint8(hash >> 16)