From 1aa501632f40701c8f86dc6b608a7297ca0ce6c1 Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Wed, 5 Feb 2020 17:17:40 +1100 Subject: [PATCH] Add gif.config_decoder (part 1 of 2) This commit (part 1) adds script/preprocess-wuffs.go but running it is deferred for part 2. Part 1 makes no change to the C code (release/c/** and test/c/**). --- doc/changelog.md | 2 + script/preprocess-wuffs.go | 507 +++++++++++++++++++++++++++++++++++++ std/gif/decode_gif.wuffs | 52 +++- 3 files changed, 560 insertions(+), 1 deletion(-) create mode 100644 script/preprocess-wuffs.go diff --git a/doc/changelog.md b/doc/changelog.md index a15b685e..4062f838 100644 --- a/doc/changelog.md +++ b/doc/changelog.md @@ -4,10 +4,12 @@ ## Work In Progress - Added `example/imageviewer`. +- Added `std/gif.config_decoder`. - Added `std/wbmp`. - Added `WUFFS_BASE__PIXEL_BLEND__SRC_OVER`. - Added `WUFFS_BASE__PIXEL_FORMAT__BGR_565`. - Added interfaces. +- Added preprocessor. - Made `wuffs_base__pixel_format` a struct. - Made `wuffs_base__pixel_subsampling` a struct. - Made `wuffs_base__status` a struct. diff --git a/script/preprocess-wuffs.go b/script/preprocess-wuffs.go new file mode 100644 index 00000000..fd3aee76 --- /dev/null +++ b/script/preprocess-wuffs.go @@ -0,0 +1,507 @@ +// Copyright 2020 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build ignore + +// TODO: consider renaming this from script/preprocess-wuffs.go to +// cmd/wuffspreprocess, making it a "go install"able command line tool. + +package main + +// preprocess-wuffs.go generates target Wuffs files based on source Wuffs +// files. It is conceptually similar to, but weaker than, the C language's +// preprocessor's #ifdef mechanism. It is a stand-alone tool, not built into +// the Wuffs compiler. It runs relatively infrequently (not at every compile) +// and preprocessed output is checked into the repository. +// +// Preprocessing is separate from compilation, unlike C, for multiple reasons: +// +// - It simplifies the Wuffs language per se, which makes it easier to write +// other tools for Wuffs programs, such as an independent implementation of +// the type checker or a tool that converts from Wuffs programs to the input +// format of a formal verifier. +// +// - Having an explicit file containing the preprocessed output helps keep the +// programmer aware of the cost (increased source size is correlated with +// increased binary size) of generating code. Other programming languages +// make it very easy (in terms of lines of code written and checked in), +// possibly too easy, to produce lots of object code, especiallly when +// monomorphizing favors run-time performance over binary size. +// +// - Writing the generated code to disk can help debug that generated code. +// +// It is the programmer's responsibility to re-run the preprocessor to +// re-generate the target files whenever the source file changes, similar to +// the Go language's "go generate" (https://blog.golang.org/generate). +// Naturally, this can be automated to some extent, e.g. through Makefiles or +// git hooks (when combined with the -fail-on-change flag). +// +// -------- +// +// Usage: +// +// go run preprocess-wuffs.go a.wuffs b*.wuffs dir3 dir4 +// +// This scans all of the files or directories (recursively, albeit skipping +// dot-files) passed for Wuffs-preprocessor directives (see below). If no files +// or directories are passed, it scans ".", the current working directory. +// +// The optional -fail-on-change flag means to fail (with a non-zero exit code) +// if the target files' contents would change. +// +// -------- +// +// Directives: +// +// Preprocessing is line-based, and lines of interest all start with optional +// whitespace and then "//#", slash slash hash, e.g. "//#USE etc". +// +// The first directive must be #USE, which mentions the name of this program +// and then lists the files to generate. +// +// Other directives are grouped into blocks: +// - One or more "#WHEN FOO filename1 filename2" lines, and then +// - One "#DONE FOO" line. +// +// The "FOO" names are arbitrary but must be unique (in a file), preventing +// nested blocks. A good text editor can also quickly cycle through the #WHEN +// and #DONE directives for any given block by searching for that unique name. +// By convention, the names look like "PREPROC123". The "123" suffix is for +// uniqueness. The names' ordering, number-suffixed or not, does not matter. +// +// A #WHEN's filenames detail which target files are active: the subset of the +// #USE directive's filenames that the subsequent lines (up until the next +// #WHEN or #DONE) apply to. A #WHEN's filenames may be empty, in which case +// the subsequent lines are part of the source file but none of the generated +// target files. +// +// A #REPLACE directive adds a simple find/replace filter to the active +// targets, applied to every subsequent generated line. A target may have +// multiple filters, which are applied sequentially. Filters are conceptually +// similar to a sed script, but the mechanism is trivial: for each input line, +// the first exact sub-string match (if any) is replaced. +// +// Lines that aren't directives (that don't start with whitespace then "//#") +// are simply copied (after per-target filtering) to either all active targets +// (when within a block) or to all targets (otherwise). +// +// The ## directive (e.g. "//## apple banana") is, like all directives, a "//" +// comment in the source file, but the "//##" is stripped and the remainder +// ("apple banana") is treated as a non-directive line, copied and filtered per +// the previous paragraph. +// +// For an example, look for "PREPROC" in the std/gif/decode_gif.wuffs file, and +// try "diff std/gif/decode_{gif,config}.wuffs". + +import ( + "bytes" + "flag" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + + "github.com/google/wuffs/lang/render" + + t "github.com/google/wuffs/lang/token" +) + +var ( + focFlag = flag.Bool("fail-on-change", false, + "fail (with a non-zero exit code) if the target files' contents would change") +) + +func main() { + if err := main1(); err != nil { + os.Stderr.WriteString(err.Error() + "\n") + os.Exit(1) + } +} + +func main1() error { + flag.Parse() + + if flag.NArg() == 0 { + if err := filepath.Walk(".", walk); err != nil { + return err + } + } else { + for i := 0; i < flag.NArg(); i++ { + arg := flag.Arg(i) + switch dir, err := os.Stat(arg); { + case err != nil: + return err + case dir.IsDir(): + if err := filepath.Walk(arg, walk); err != nil { + return err + } + default: + if err := do(arg); err != nil { + return err + } + } + } + } + + sortedFilenames := []string(nil) + for filename := range globalTargets { + sortedFilenames = append(sortedFilenames, filename) + } + sort.Strings(sortedFilenames) + for _, filename := range sortedFilenames { + contents := globalTargets[filename] + if x, err := ioutil.ReadFile(filename); (err == nil) && bytes.Equal(x, contents) { + fmt.Printf("gen unchanged: %s\n", filename) + + continue + } + if *focFlag { + return fmt.Errorf("fail-on-change: %s\n", filename) + } + if err := writeFile(filename, contents); err != nil { + return fmt.Errorf("writing %s: %v", filename, err) + } + fmt.Printf("gen wrote: %s\n", filename) + } + + return nil +} + +func isWuffsFile(info os.FileInfo) bool { + name := info.Name() + return !info.IsDir() && !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".wuffs") +} + +func walk(filename string, info os.FileInfo, err error) error { + if (err == nil) && isWuffsFile(info) { + err = do(filename) + } + // Don't complain if a file was deleted in the meantime (i.e. the directory + // changed concurrently while running this program). + if (err != nil) && !os.IsNotExist(err) { + return err + } + return nil +} + +var ( + directiveDone = []byte(`//#DONE `) + directiveHash = []byte(`//## `) + directiveReplace = []byte(`//#REPLACE `) + directiveUse = []byte(`//#USE "go run preprocess-wuffs.go" TO MAKE `) + directiveWhen = []byte(`//#WHEN `) + + _with_ = []byte(" WITH ") + space = []byte(" ") + + // globalTargets map from filenames to contents. + globalTargets = map[string][]byte{} +) + +type target struct { + buffer *bytes.Buffer + filters []filter +} + +func (t *target) write(s []byte) { + for _, f := range t.filters { + i := bytes.Index(s, f.find) + if i < 0 { + continue + } + x := []byte(nil) + x = append(x, s[:i]...) + x = append(x, f.replace...) + x = append(x, s[i+len(f.find):]...) + s = x + } + t.buffer.Write(s) +} + +type filter struct { + find []byte + replace []byte +} + +func do(filename string) error { + src, err := ioutil.ReadFile(filename) + if err != nil { + return err + } + if !bytes.Contains(src, directiveUse) { + return nil + } + localTargets := map[string]*target(nil) + activeTargets := []*target(nil) + usedBlockNames := map[string]bool{} + blockName := []byte(nil) // Typically something like "PREPROC123". + prefix := []byte(nil) // Source file contents up to the "//#USE" directive. + + for remaining := src; len(remaining) > 0; { + line := remaining + if i := bytes.IndexByte(remaining, '\n'); i >= 0 { + line, remaining = remaining[:i+1], remaining[i+1:] + } else { + remaining = nil + } + + ppLine := parsePreprocessorLine(line) + if ppLine == nil { + if localTargets == nil { + prefix = append(prefix, line...) + } else { + for _, t := range activeTargets { + t.write(line) + } + } + continue + } + + if bytes.HasPrefix(ppLine, directiveUse) { + if localTargets != nil { + return fmt.Errorf("multiple #USE directives") + } + err := error(nil) + localTargets, err = parseUse(filename, ppLine[len(directiveUse):]) + if err != nil { + return err + } + + activeTargets = activeTargets[:0] + for _, t := range localTargets { + activeTargets = append(activeTargets, t) + t.write(prefix) + } + prefix = nil + continue + } + + if localTargets == nil { + return fmt.Errorf("missing #USE directive") + } + + switch { + case bytes.HasPrefix(ppLine, directiveDone): + arg := ppLine[len(directiveDone):] + if blockName == nil { + return fmt.Errorf("bad #DONE directive without #WHEN directive") + } else if !bytes.Equal(blockName, arg) { + return fmt.Errorf("bad directive name: %q", arg) + } + activeTargets = activeTargets[:0] + for _, t := range localTargets { + activeTargets = append(activeTargets, t) + } + blockName = nil + + case bytes.HasPrefix(ppLine, directiveHash): + indent := []byte(nil) + if i := bytes.IndexByte(line, '/'); i >= 0 { + indent = line[:i] + } + if blockName == nil { + for _, t := range localTargets { + t.buffer.Write(indent) + t.write(ppLine[len(directiveHash):]) + t.buffer.WriteByte('\n') + } + } else { + for _, t := range activeTargets { + t.buffer.Write(indent) + t.write(ppLine[len(directiveHash):]) + t.buffer.WriteByte('\n') + } + } + + case bytes.HasPrefix(ppLine, directiveReplace): + f := parseReplace(ppLine[len(directiveReplace):]) + if (f.find == nil) || (f.replace == nil) { + return fmt.Errorf("bad #REPLACE directive: %q", ppLine) + } + for _, t := range activeTargets { + t.filters = append(t.filters, f) + } + + case bytes.HasPrefix(ppLine, directiveWhen): + args := bytes.Split(ppLine[len(directiveWhen):], space) + if len(args) == 0 { + return fmt.Errorf("bad #WHEN directive: %q", ppLine) + } + if blockName == nil { + blockName = args[0] + if bn := string(blockName); usedBlockNames[bn] { + return fmt.Errorf("duplicate directive name: %q", bn) + } else { + usedBlockNames[bn] = true + } + } else if !bytes.Equal(blockName, args[0]) { + return fmt.Errorf("bad directive name: %q", args[0]) + } + + dir := filepath.Dir(filename) + activeTargets = activeTargets[:0] + for _, arg := range args[1:] { + t := localTargets[filepath.Join(dir, string(arg))] + if t == nil { + return fmt.Errorf("bad #WHEN filename: %q", arg) + } + activeTargets = append(activeTargets, t) + } + + default: + return fmt.Errorf("bad directive: %q", ppLine) + } + } + + if blockName != nil { + return fmt.Errorf("missing #DONE directive: %q", blockName) + } + + for absFilename, t := range localTargets { + globalTargets[absFilename] = wuffsfmt(t.buffer.Bytes()) + } + return nil +} + +func wuffsfmt(src []byte) []byte { + tm := &t.Map{} + tokens, comments, err := t.Tokenize(tm, "placeholder.filename", src) + if err != nil { + return src + } + dst := &bytes.Buffer{} + if err := render.Render(dst, tm, tokens, comments); err != nil { + return src + } + return dst.Bytes() +} + +const chmodSupported = runtime.GOOS != "windows" + +func writeFile(filename string, b []byte) error { + f, err := ioutil.TempFile(filepath.Dir(filename), filepath.Base(filename)) + if err != nil { + return err + } + if chmodSupported { + if info, err := os.Stat(filename); err == nil { + f.Chmod(info.Mode().Perm()) + } + } + _, werr := f.Write(b) + cerr := f.Close() + if werr != nil { + os.Remove(f.Name()) + return werr + } + if cerr != nil { + os.Remove(f.Name()) + return cerr + } + return os.Rename(f.Name(), filename) +} + +func parsePreprocessorLine(line []byte) []byte { + // Look for "//#", slash slash hash. + line = stripLeadingWhitespace(line) + if (len(line) >= 3) && (line[0] == '/') && (line[1] == '/') && (line[2] == '#') { + return bytes.TrimSpace(line) + } + return nil +} + +func parseReplace(ppLine []byte) filter { + s0, ppLine := parseString(ppLine) + if s0 == nil { + return filter{} + } + if !bytes.HasPrefix(ppLine, _with_) { + return filter{} + } + ppLine = ppLine[len(_with_):] + s1, ppLine := parseString(ppLine) + if (s1 == nil) || (len(ppLine) != 0) { + return filter{} + } + return filter{ + find: s0, + replace: s1, + } +} + +func parseString(line []byte) (s []byte, remaining []byte) { + line = stripLeadingWhitespace(line) + if (len(line) == 0) || (line[0] != '"') { + return nil, line + } + line = line[1:] + i := bytes.IndexByte(line, '"') + if i < 0 { + return nil, line + } + if bytes.IndexByte(line[:i], '\\') >= 0 { + return nil, line + } + return line[:i], line[i+1:] +} + +func parseUse(srcFilename string, ppLine []byte) (map[string]*target, error) { + absSrcFilename := filepath.Clean(srcFilename) + localTargets := map[string]*target{} + dir := filepath.Dir(srcFilename) + for _, relFilename := range bytes.Split(ppLine, space) { + if len(relFilename) == 0 { + continue + } + if !validFilename(relFilename) { + return nil, fmt.Errorf("invalid filename: %q", string(relFilename)) + } + absFilename := filepath.Join(dir, string(relFilename)) + if _, ok := globalTargets[absFilename]; ok { + return nil, fmt.Errorf("duplicate filename: %q", absFilename) + } + if absFilename == absSrcFilename { + return nil, fmt.Errorf("self-referential filename: %q", absFilename) + } + + buf := &bytes.Buffer{} + buf.WriteString( + "// This file was automatically generated by \"preprocess-wuffs.go\".\n\n") + buf.WriteString("// --------\n\n") + localTargets[absFilename] = &target{buffer: buf} + } + return localTargets, nil +} + +func stripLeadingWhitespace(s []byte) []byte { + for (len(s) > 0) && (s[0] <= ' ') { + s = s[1:] + } + return s +} + +func validFilename(s []byte) bool { + if (len(s) == 0) || (s[0] == '.') { + return false + } + for _, c := range s { + if (c <= ' ') || (c == '/') || (c == '\\') || (c == ':') { + return false + } + } + return true +} diff --git a/std/gif/decode_gif.wuffs b/std/gif/decode_gif.wuffs index acb72630..a07bd7bb 100644 --- a/std/gif/decode_gif.wuffs +++ b/std/gif/decode_gif.wuffs @@ -12,6 +12,29 @@ // See the License for the specific language governing permissions and // limitations under the License. +// -------- + +//#USE "go run preprocess-wuffs.go" TO MAKE decode_config.wuffs +//#WHEN PREPROC000 +// +// This file defines "struct decoder". It is also preprocessed (copied with +// automatic edits) to generate the decode_config.wuffs file that defines +// "struct config_decoder". See preprocess-wuffs.go for more detail about the +// preprocessor language (the //#FOOBAR comments). +// +// config_decoder does not implement decode_frame, only decode_frame_config and +// decode_image_config (and other miscellaneous methods). The sizeof the +// config_decoder can be substantially smaller than the sizeof the full +// decoder, as it does not need any internal buffers for understanding the +// underlying compression format. +// +//#WHEN PREPROC000 decode_config.wuffs +//#REPLACE "func decoder" WITH "func config_decoder" +//#REPLACE "struct decoder" WITH "struct config_decoder" +//#DONE PREPROC000 + +// -------- + pub struct decoder? implements base.image_decoder( width : base.u32, height : base.u32, @@ -101,6 +124,7 @@ pub struct decoder? implements base.image_decoder( frame_rect_x1 : base.u32, frame_rect_y1 : base.u32, + //#WHEN PREPROC100 // The dst_etc fields are the output cursor during copy_to_image_buffer. dst_x : base.u32, dst_y : base.u32, @@ -111,8 +135,11 @@ pub struct decoder? implements base.image_decoder( compressed_wi : base.u64, swizzler : base.pixel_swizzler, - util : base.utility, + //#DONE PREPROC100 + + util : base.utility, )( + //#WHEN PREPROC101 compressed : array[4096] base.u8, // palettes[0] and palettes[1] are the Global and Local Color Table. @@ -121,6 +148,9 @@ pub struct decoder? implements base.image_decoder( dst_palette : array[4 * 256] base.u8, lzw : lzw.decoder, + //#WHEN PREPROC101 decode_config.wuffs + //## palettes : array[1] array[4 * 256] base.u8, + //#DONE PREPROC101 ) pub func decoder.set_quirk_enabled!(quirk: base.u32, enabled: base.bool) { @@ -260,6 +290,7 @@ pub func decoder.num_decoded_frames() base.u64 { } pub func decoder.frame_dirty_rect() base.rect_ie_u32 { + //#WHEN PREPROC200 // The "foo.min(a:this.width_or_height)" calls clip the nominal frame_rect // to the image_rect. return this.util.make_rect_ie_u32( @@ -267,12 +298,19 @@ pub func decoder.frame_dirty_rect() base.rect_ie_u32 { min_incl_y: this.frame_rect_y0.min(a: this.height), max_excl_x: this.frame_rect_x1.min(a: this.width), max_excl_y: this.dirty_max_excl_y.min(a: this.height)) + //#WHEN PREPROC200 decode_config.wuffs + //## return this.util.empty_rect_ie_u32() + //#DONE PREPROC200 } pub func decoder.workbuf_len() base.range_ii_u64 { + //#WHEN PREPROC201 return this.util.make_range_ii_u64( min_incl: decoder_workbuf_len_max_incl_worst_case, max_incl: decoder_workbuf_len_max_incl_worst_case) + //#WHEN PREPROC201 decode_config.wuffs + //## return this.util.empty_range_ii_u64() + //#DONE PREPROC201 } pub func decoder.restart_frame!(index: base.u64, io_position: base.u64) base.status { @@ -294,7 +332,10 @@ pub func decoder.decode_frame_config?(dst: nptr base.frame_config, src: base.io_ var flags : base.u8 this.ignore_metadata = true + + //#WHEN PREPROC202 this.dirty_max_excl_y = 0 + //#DONE PREPROC202 if not this.end_of_data { if this.call_sequence == 0 { @@ -384,6 +425,7 @@ pri func decoder.skip_frame?(src: base.io_reader) { // TODO: honor args.opts. pub func decoder.decode_frame?(dst: ptr base.pixel_buffer, src: base.io_reader, blend: base.pixel_blend, workbuf: slice base.u8, opts: nptr base.decode_frame_options) { + //#WHEN PREPROC300 this.ignore_metadata = true if this.call_sequence <> 4 { this.decode_frame_config?(dst: nullptr, src: args.src) @@ -397,6 +439,9 @@ pub func decoder.decode_frame?(dst: ptr base.pixel_buffer, src: base.io_reader, this.num_decoded_frames_value ~sat+= 1 this.reset_gc!() + //#WHEN PREPROC300 decode_config.wuffs + //## return base."#unsupported method" + //#DONE PREPROC300 } pri func decoder.reset_gc!() { @@ -740,8 +785,11 @@ pri func decoder.decode_id_part0?(src: base.io_reader) { this.frame_rect_x1 ~mod+= this.frame_rect_x0 this.frame_rect_y1 = args.src.read_u16le_as_u32?() this.frame_rect_y1 ~mod+= this.frame_rect_y0 + + //#WHEN PREPROC400 this.dst_x = this.frame_rect_x0 this.dst_y = this.frame_rect_y0 + //#DONE PREPROC400 // Set the image's overall width and height to be the maximum of the // nominal image width and height (given in the Logical Screen Descriptor) @@ -754,6 +802,7 @@ pri func decoder.decode_id_part0?(src: base.io_reader) { } } +//#WHEN PREPROC900 pri func decoder.decode_id_part1?(dst: ptr base.pixel_buffer, src: base.io_reader, blend: base.pixel_blend) { var flags : base.u8 var which_palette : base.u8[..= 1] @@ -1098,3 +1147,4 @@ pri func decoder.copy_to_image_buffer!(pb: ptr base.pixel_buffer, src: slice bas } return ok } +//#DONE PREPROC900