mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-17 16:32:11 +01:00
Add a ractool -tmpdir flag
This commit is contained in:
1 parent
18763813d5
commit
b4808596ac
2 files changed
+89
-15
No files matched your search
@@ -82,6 +82,8 @@ General Flags:
|
||||
whether to decode the input
|
||||
-encode
|
||||
whether to encode the input
|
||||
-quiet
|
||||
whether to suppress messages
|
||||
|
||||
Decode-Related Flags:
|
||||
|
||||
@@ -104,6 +106,8 @@ Encode-Related Flags:
|
||||
the index location, "start" or "end" (default "start")
|
||||
-resources
|
||||
comma-separated list of resource files, such as shared dictionaries
|
||||
-tmpdir
|
||||
directory (e.g. $TMPDIR) for intermediate work; empty means in-memory
|
||||
|
||||
Codecs:
|
||||
|
||||
|
||||
+85
-15
@@ -83,6 +83,8 @@ General Flags:
|
||||
whether to decode the input
|
||||
-encode
|
||||
whether to encode the input
|
||||
-quiet
|
||||
whether to suppress messages
|
||||
|
||||
Decode-Related Flags:
|
||||
|
||||
@@ -105,6 +107,8 @@ Encode-Related Flags:
|
||||
the index location, "start" or "end" (default "start")
|
||||
-resources
|
||||
comma-separated list of resource files, such as shared dictionaries
|
||||
-tmpdir
|
||||
directory (e.g. $TMPDIR) for intermediate work; empty means in-memory
|
||||
|
||||
Codecs:
|
||||
|
||||
@@ -253,11 +257,10 @@ import (
|
||||
"github.com/google/wuffs/lib/raczstd"
|
||||
)
|
||||
|
||||
// TODO: a flag to use a disk-backed (not memory-backed) TempFile.
|
||||
|
||||
var (
|
||||
decodeFlag = flag.Bool("decode", false, "whether to decode the input")
|
||||
encodeFlag = flag.Bool("encode", false, "whether to encode the input")
|
||||
quietFlag = flag.Bool("quiet", false, "whether to suppress messages")
|
||||
|
||||
// Decode-related flags.
|
||||
drangeFlag = flag.String("drange", "..",
|
||||
@@ -274,6 +277,8 @@ var (
|
||||
"the index location, \"start\" or \"end\"")
|
||||
resourcesFlag = flag.String("resources", "",
|
||||
"comma-separated list of resource files, such as shared dictionaries")
|
||||
tmpdirFlag = flag.String("tmpdir", "",
|
||||
"directory (e.g. $TMPDIR) for intermediate work; empty means in-memory")
|
||||
)
|
||||
|
||||
func usage() {
|
||||
@@ -395,7 +400,7 @@ func parseRange(s string) (i int64, j int64, ok bool) {
|
||||
func decode(inFile *os.File) error {
|
||||
i, j, ok := parseRange(*drangeFlag)
|
||||
if !ok {
|
||||
return fmt.Errorf("invalid -drange")
|
||||
return errors.New("invalid -drange")
|
||||
}
|
||||
|
||||
rs := io.ReadSeeker(inFile)
|
||||
@@ -465,14 +470,20 @@ func decode(inFile *os.File) error {
|
||||
}
|
||||
|
||||
func encode(r io.Reader) error {
|
||||
indexLocation := rac.IndexLocation(0)
|
||||
indexLocation, tempFile := rac.IndexLocation(0), io.ReadWriter(nil)
|
||||
switch *indexlocationFlag {
|
||||
case "start":
|
||||
indexLocation = rac.IndexLocationAtStart
|
||||
case "end":
|
||||
indexLocation = rac.IndexLocationAtEnd
|
||||
default:
|
||||
return errors.New("invalid -indexlocation")
|
||||
case "end":
|
||||
indexLocation = rac.IndexLocationAtEnd
|
||||
case "start":
|
||||
indexLocation = rac.IndexLocationAtStart
|
||||
|
||||
tf, err := makeTempFile()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tempFile = tf
|
||||
}
|
||||
|
||||
cchunksize := parseNumber(*cchunksizeFlag)
|
||||
@@ -494,21 +505,21 @@ func encode(r io.Reader) error {
|
||||
dchunksize = 65536 // 64 KiB.
|
||||
}
|
||||
|
||||
w := &rac.Writer{
|
||||
rw := &rac.Writer{
|
||||
Writer: os.Stdout,
|
||||
IndexLocation: indexLocation,
|
||||
TempFile: &bytes.Buffer{},
|
||||
TempFile: tempFile,
|
||||
CPageSize: uint64(cpagesize),
|
||||
CChunkSize: uint64(cchunksize),
|
||||
DChunkSize: uint64(dchunksize),
|
||||
}
|
||||
switch *codecFlag {
|
||||
case "lz4":
|
||||
w.CodecWriter = &raclz4.CodecWriter{}
|
||||
rw.CodecWriter = &raclz4.CodecWriter{}
|
||||
case "zlib":
|
||||
w.CodecWriter = &raczlib.CodecWriter{}
|
||||
rw.CodecWriter = &raczlib.CodecWriter{}
|
||||
case "zstd":
|
||||
w.CodecWriter = &raczstd.CodecWriter{}
|
||||
rw.CodecWriter = &raczstd.CodecWriter{}
|
||||
default:
|
||||
return errors.New("unsupported -codec")
|
||||
}
|
||||
@@ -519,12 +530,71 @@ func encode(r io.Reader) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.ResourcesData = append(w.ResourcesData, resource)
|
||||
rw.ResourcesData = append(rw.ResourcesData, resource)
|
||||
}
|
||||
}
|
||||
|
||||
const warn1g = "" +
|
||||
"ractool: encoding 1 GiB or more with -indexlocation=start. Set -tmpdir to\n" +
|
||||
"store intermediate work on disk instead of in memory.\n"
|
||||
w := io.Writer(rw)
|
||||
if !*quietFlag && (*indexlocationFlag == "start") && (*tmpdirFlag == "") {
|
||||
w = &ifNBytesWriter{
|
||||
w: w,
|
||||
n: 1 << 30,
|
||||
f: func() { fmt.Fprintf(os.Stderr, warn1g) },
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := io.Copy(w, r); err != nil {
|
||||
return err
|
||||
}
|
||||
return w.Close()
|
||||
return rw.Close()
|
||||
}
|
||||
|
||||
// ifNBytesWriter wraps w, calling f once if n or more bytes are written.
|
||||
type ifNBytesWriter struct {
|
||||
w io.Writer
|
||||
n int64
|
||||
f func()
|
||||
}
|
||||
|
||||
func (t *ifNBytesWriter) Write(p []byte) (int, error) {
|
||||
if t.n <= 0 {
|
||||
return t.w.Write(p)
|
||||
}
|
||||
if t.n > int64(len(p)) {
|
||||
t.n -= int64(len(p))
|
||||
return t.w.Write(p)
|
||||
}
|
||||
prefix, suffix := p[:t.n], p[t.n:]
|
||||
t.n = 0
|
||||
|
||||
ret0, err := t.w.Write(prefix)
|
||||
t.f()
|
||||
if err != nil {
|
||||
return ret0, err
|
||||
}
|
||||
ret1, err := t.w.Write(suffix)
|
||||
return ret0 + ret1, err
|
||||
}
|
||||
|
||||
func makeTempFile() (io.ReadWriter, error) {
|
||||
if *tmpdirFlag == "" {
|
||||
return &bytes.Buffer{}, nil
|
||||
}
|
||||
|
||||
f, err := ioutil.TempFile(*tmpdirFlag, "ractool-")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Delete the file while it's still open, so it will be cleaned up on exit.
|
||||
// No other process can find it by name, but that's fine. We can still read
|
||||
// from and write to it.
|
||||
if err := os.Remove(f.Name()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return f, nil
|
||||
}
|
||||
Reference in new issue
Block a user