diff --git a/README.md b/README.md index d4cfa99c..d52b469e 100644 --- a/README.md +++ b/README.md @@ -290,6 +290,7 @@ background. - `gen` holds the transpiled editions of that standard library. The initial focus is generating C code. Later on, the repository might include generated Go and Rust code. +- `release` holds the releases of the Wuffs standard library. - `test` holds the regular tests for the Wuffs standard library. - `fuzz` holds the fuzz tests for the Wuffs standard library. - `script` holds miscellaneous utility programs. diff --git a/build-all.sh b/build-all.sh index a4180cbf..ef444320 100755 --- a/build-all.sh +++ b/build-all.sh @@ -13,8 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This script is a simple sanity check that the tests pass and the example and -# fuzz programs compile. +# This script, build-all.sh, is a simple sanity check that the tests pass and +# the example and fuzz programs compile. Despite the "all" in the script name, +# it does not build the release editions, as building a release is a separate +# process from e.g. checking that all tests pass before pushing commits. # # If you're just looking to get started with Wuffs, running this script isn't # necessary (as Wuffs doesn't have the "configure; make; make install" dance or diff --git a/cmd/commonflags/commonflags.go b/cmd/commonflags/commonflags.go index 1dfe226f..e7c51b21 100644 --- a/cmd/commonflags/commonflags.go +++ b/cmd/commonflags/commonflags.go @@ -19,7 +19,9 @@ package commonflags import ( + "fmt" "path" + "strings" ) const ( @@ -44,13 +46,16 @@ const ( RepsMin = 0 RepsMax = 1000000 RepsUsage = `the number of repetitions per benchmark` + + VersionDefault = "0.0.0" + VersionUsage = `version string, e.g. "1.2.3-beta.4"` ) // TODO: do IsAlphaNumericIsh and IsValidUsePath belong in a separate package, // such as lang/validate? Perhaps together with token.Unescape? // IsAlphaNumericIsh returns whether s contains only ASCII alpha-numerics and a -// limited set of punctuation such as commas and slahes, but not containing +// limited set of punctuation such as commas and slashes, but not containing // e.g. spaces, semi-colons, colons or backslashes. // // The intent is that if s is alpha-numeric-ish, then it should not need @@ -70,3 +75,35 @@ func IsAlphaNumericIsh(s string) bool { func IsValidUsePath(s string) bool { return s == path.Clean(s) && s != "" && s[0] != '.' && s[0] != '/' } + +type Version struct { + Major uint32 + Minor uint16 + Patch uint16 + Extension string +} + +func ParseVersion(s string) (v Version, ok bool) { + if !IsAlphaNumericIsh(s) { + return Version{}, false + } + if i := strings.IndexByte(s, '-'); i >= 0 { + s, v.Extension = s[:i], s[i+1:] + } + if _, err := fmt.Sscanf(s, "%d.%d.%d", &v.Major, &v.Minor, &v.Patch); err != nil { + return Version{}, false + } + return v, true +} + +func (v Version) String() string { + s := fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch) + if v.Extension != "" { + s += "-" + v.Extension + } + return s +} + +func (v Version) Uint64() uint64 { + return (uint64(v.Major) << 32) | (uint64(v.Minor) << 16) | (uint64(v.Patch) << 0) +} diff --git a/cmd/wuffs-c/main.go b/cmd/wuffs-c/main.go index 2c8eb102..14946741 100644 --- a/cmd/wuffs-c/main.go +++ b/cmd/wuffs-c/main.go @@ -41,6 +41,8 @@ func main1() error { return cgen.Do(args) case "genlib": return doGenlib(args) + case "genrelease": + return doGenrelease(args) case "test": return doTest(args) } diff --git a/cmd/wuffs-c/release.go b/cmd/wuffs-c/release.go new file mode 100644 index 00000000..5f98e81a --- /dev/null +++ b/cmd/wuffs-c/release.go @@ -0,0 +1,230 @@ +// Copyright 2018 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "flag" + "fmt" + "io/ioutil" + "os" + "os/exec" + "time" + + cf "github.com/google/wuffs/cmd/commonflags" +) + +func doGenrelease(args []string) error { + flags := flag.FlagSet{} + cformatterFlag := flags.String("cformatter", cf.CformatterDefault, cf.CformatterUsage) + revisionFlag := flags.String("revision", "", "git revision the release was built from") + versionFlag := flags.String("version", cf.VersionDefault, cf.VersionUsage) + + if err := flags.Parse(args); err != nil { + return err + } + if !cf.IsAlphaNumericIsh(*cformatterFlag) { + return fmt.Errorf("bad -cformatter flag value %q", *cformatterFlag) + } + if !cf.IsAlphaNumericIsh(*revisionFlag) { + return fmt.Errorf("bad -revision flag value %q", *revisionFlag) + } + v, ok := cf.ParseVersion(*versionFlag) + if !ok { + return fmt.Errorf("bad -version flag value %q", *versionFlag) + } + args = flags.Args() + + h := &genReleaseHelper{ + seen: map[string]bool{}, + revision: *revisionFlag, + version: v, + } + + unformatted := bytes.NewBuffer(nil) + unformatted.WriteString("#ifndef WUFFS_INCLUDE_GUARD\n") + unformatted.WriteString("#define WUFFS_INCLUDE_GUARD\n\n") + + // First, cat all of the headers together, filtering out duplicate + // WUFFS_INCLUDE_GUARD__FOO sections and overriding WUFFS_VERSION. + for _, filename := range args { + s, err := ioutil.ReadFile(filename) + if err != nil { + return err + } + + if i := bytes.Index(s, grCHeader); i >= 0 { + s = s[:i] + } + + if err := h.gen(unformatted, s); err != nil { + return fmt.Errorf("%v in %s", err, filename) + } + } + + unformatted.WriteString("\n\n#endif // WUFFS_INCLUDE_GUARD\n\n") + + // Then, cat all of the implementations together, filtering out duplicate + // WUFFS_INCLUDE_GUARD__BASE_PRIVATE sections. + for _, filename := range args { + s, err := ioutil.ReadFile(filename) + if err != nil { + return err + } + + if i := bytes.Index(s, grCHeader); i >= 0 { + s = s[i+len(grCHeader):] + } else { + continue + } + + if err := h.gen(unformatted, s); err != nil { + return fmt.Errorf("%v in %s", err, filename) + } + } + + cmd := exec.Command(*cformatterFlag, "-style=Chromium") + cmd.Stdin = unformatted + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +var ( + grCHeader = []byte("// !! C HEADER ENDS HERE.\n") + grNN = []byte("\n\n") + grVOverride = []byte("// !! Some code generation programs can override WUFFS_VERSION.\n") + grVString = []byte(`#define WUFFS_VERSION_STRING "0.0.0"`) + + grDefine = []byte("#define WUFFS_INCLUDE_GUARD__") + grEndif = []byte("#endif // WUFFS_INCLUDE_GUARD__") + grIfndef = []byte("#ifndef WUFFS_INCLUDE_GUARD__") +) + +type genReleaseHelper struct { + seen map[string]bool + revision string + version cf.Version +} + +func (h *genReleaseHelper) gen(w *bytes.Buffer, s []byte) error { + skipping := 0 + stack := []string{} + + for remaining := []byte(nil); len(s) > 0; s, remaining = remaining, nil { + if i := bytes.IndexByte(s, '\n'); i >= 0 { + s, remaining = s[:i+1], s[i+1:] + } + if len(s) == 0 { + continue + } + + if s[0] == '#' { + switch { + case bytes.HasPrefix(s, grIfndef): + pkg := string(s[len(grIfndef):]) + for _, p := range stack { + if p == pkg { + return fmt.Errorf("unexpected %q", s) + } + } + + if h.seen[pkg] { + skipping++ + } + + stack = append(stack, pkg) + continue + + case bytes.HasPrefix(s, grDefine): + pkg := string(s[len(grDefine):]) + if (len(stack) == 0) || (stack[len(stack)-1] != pkg) { + return fmt.Errorf("unexpected %q", s) + } + continue + + case bytes.HasPrefix(s, grEndif): + pkg := string(s[len(grEndif):]) + if (len(stack) == 0) || (stack[len(stack)-1] != pkg) { + return fmt.Errorf("unexpected %q", s) + } + + if h.seen[pkg] { + skipping-- + } else { + h.seen[pkg] = true + } + + stack = stack[:len(stack)-1] + continue + } + } + + if skipping > 0 { + continue + } + + if s[0] == '/' { + switch { + case bytes.Equal(s, grVOverride): + var err error + remaining, err = h.genWuffsVersion(w, remaining) + if err != nil { + return err + } + continue + } + } + + w.Write(s) + } + + if len(stack) != 0 { + return fmt.Errorf("unmatched %q", string(grIfndef)+stack[len(stack)-1]) + } + return nil +} + +func (h *genReleaseHelper) genWuffsVersion(w *bytes.Buffer, s []byte) (remaining []byte, err error) { + cut := []byte(nil) + if i := bytes.Index(s, grNN); i >= 0 { + cut, s = s[:i], s[i+len(grNN):] + } else { + return nil, fmt.Errorf(`could not find "\n\n" near WUFFS_VERSION`) + } + + if !bytes.HasSuffix(cut, grVString) { + return nil, fmt.Errorf("%q did not end with %q", cut, grVString) + } + + fmt.Fprintf(w, "// WUFFS_VERSION was overridden by \"wuffs genrelease\" on %v UTC", + time.Now().UTC().Format("2006-01-02")) + if h.revision != "" { + fmt.Fprintf(w, ",\n// based on revision %s", h.revision) + } + + fmt.Fprintf(w, `. + #define WUFFS_VERSION ((uint64_t)0x%016X) + #define WUFFS_VERSION_MAJOR ((uint64_t)0x%08X) + #define WUFFS_VERSION_MINOR ((uint64_t)0x%04X) + #define WUFFS_VERSION_PATCH ((uint64_t)0x%04X) + #define WUFFS_VERSION_EXTENSION %q + #define WUFFS_VERSION_STRING %q + + `, h.version.Uint64(), h.version.Major, h.version.Minor, h.version.Patch, + h.version.Extension, h.version) + + return s, nil +} diff --git a/cmd/wuffs/gen.go b/cmd/wuffs/gen.go index 147e4774..19043c2d 100644 --- a/cmd/wuffs/gen.go +++ b/cmd/wuffs/gen.go @@ -18,7 +18,6 @@ import ( "bytes" "flag" "fmt" - "io/ioutil" "os" "os/exec" "path" @@ -119,12 +118,13 @@ func (h *genHelper) gen(dirname string, recursive bool) error { return fmt.Errorf("invalid package path %q", dirname) } - filenames, dirnames, err := listDir(h.wuffsRoot, dirname, recursive) + qualFilenames, dirnames, err := listDir( + filepath.Join(h.wuffsRoot, filepath.FromSlash(dirname)), ".wuffs", recursive) if err != nil { return err } - if len(filenames) > 0 { - if err := h.genDir(dirname, filenames); err != nil { + if len(qualFilenames) > 0 { + if err := h.genDir(dirname, qualFilenames); err != nil { return err } h.affected = append(h.affected, dirname) @@ -139,7 +139,7 @@ func (h *genHelper) gen(dirname string, recursive bool) error { return nil } -func (h *genHelper) genDir(dirname string, filenames []string) error { +func (h *genHelper) genDir(dirname string, qualFilenames []string) error { // TODO: skip the generation if the output file already exists and its // mtime is newer than all inputs and the wuffs-gen-foo command. @@ -147,12 +147,9 @@ func (h *genHelper) genDir(dirname string, filenames []string) error { if !validName(packageName) { return fmt.Errorf(`invalid package %q, not in [a-z0-9]+`, packageName) } - qualifiedFilenames := make([]string, len(filenames)) - for i, filename := range filenames { - qualifiedFilenames[i] = filepath.Join(h.wuffsRoot, filepath.FromSlash(dirname), filename) - } + if !h.skipgendeps { - if err := h.genDirDependencies(qualifiedFilenames); err != nil { + if err := h.genDirDependencies(qualFilenames); err != nil { return err } } @@ -163,7 +160,7 @@ func (h *genHelper) genDir(dirname string, filenames []string) error { if lang == "c" { cmdArgs = append(cmdArgs, fmt.Sprintf("-cformatter=%s", h.cformatter)) } - cmdArgs = append(cmdArgs, qualifiedFilenames...) + cmdArgs = append(cmdArgs, qualFilenames...) stdout := &bytes.Buffer{} cmd := exec.Command(command, cmdArgs...) @@ -196,7 +193,7 @@ func (h *genHelper) genDir(dirname string, filenames []string) error { } } if len(h.langs) > 0 && packageName != "base" { - if err := h.genWuffs(dirname, qualifiedFilenames); err != nil { + if err := h.genWuffs(dirname, qualFilenames); err != nil { return err } } @@ -226,19 +223,10 @@ func (h *genHelper) genDirDependencies(qualifiedFilenames []string) error { } func (h *genHelper) genFile(dirname string, lang string, out []byte) error { - outFilename := filepath.Join(h.wuffsRoot, "gen", lang, filepath.FromSlash(dirname)+"."+lang) - if existing, err := ioutil.ReadFile(outFilename); err == nil && bytes.Equal(existing, out) { - fmt.Println("gen unchanged: ", outFilename) - return nil - } - if err := os.MkdirAll(filepath.Dir(outFilename), 0755); err != nil { - return err - } - if err := ioutil.WriteFile(outFilename, out, 0644); err != nil { - return err - } - fmt.Println("gen wrote: ", outFilename) - return nil + return writeFile( + filepath.Join(h.wuffsRoot, "gen", lang, filepath.FromSlash(dirname)+"."+lang), + out, + ) } func (h *genHelper) genWuffs(dirname string, qualifiedFilenames []string) error { diff --git a/cmd/wuffs/main.go b/cmd/wuffs/main.go index 9ccd05fc..fd48b04c 100644 --- a/cmd/wuffs/main.go +++ b/cmd/wuffs/main.go @@ -16,8 +16,10 @@ package main import ( + "bytes" "flag" "fmt" + "io/ioutil" "os" "path/filepath" "sort" @@ -33,6 +35,7 @@ var commands = []struct { {"bench", doBench}, {"gen", doGen}, {"genlib", doGenlib}, + {"genrelease", doGenrelease}, {"test", doTest}, } @@ -48,6 +51,7 @@ The commands are: bench benchmark packages gen generate code for packages and dependencies genlib generate software libraries + genrelease generate a stand-alone release test test packages `) } @@ -79,8 +83,41 @@ func main1() error { return nil } -func listDir(wuffsRoot string, dirname string, returnSubdirs bool) (filenames []string, dirnames []string, err error) { - f, err := os.Open(filepath.Join(wuffsRoot, filepath.FromSlash(dirname))) +func findFiles(qualDirname string, suffix string) (filenames []string, err error) { + filenames, err = findFiles1(nil, qualDirname, suffix) + if err != nil { + return nil, err + } + sort.Strings(filenames) + return filenames, nil +} + +func findFiles1(dstQF []string, qualDirname string, suffix string) (qualFilenames []string, err error) { + dstQF, relDirnames, err := appendDir(dstQF, qualDirname, suffix, true) + if err != nil { + return nil, err + } + for _, d := range relDirnames { + dstQF, err = findFiles1(dstQF, filepath.Join(qualDirname, d), suffix) + if err != nil { + return nil, err + } + } + return dstQF, nil +} + +func listDir(qualDirname string, suffix string, returnSubdirs bool) (qualFilenames []string, relDirnames []string, err error) { + qualFilenames, relDirnames, err = appendDir(nil, qualDirname, suffix, returnSubdirs) + if err != nil { + return nil, nil, err + } + sort.Strings(qualFilenames) + sort.Strings(relDirnames) + return qualFilenames, relDirnames, nil +} + +func appendDir(dstQF []string, qualDirname string, suffix string, returnSubdirs bool) (qualFilenames []string, relDirnames []string, err error) { + f, err := os.Open(qualDirname) if err != nil { return nil, nil, err } @@ -94,16 +131,29 @@ func listDir(wuffsRoot string, dirname string, returnSubdirs bool) (filenames [] name := o.Name() if o.IsDir() { if returnSubdirs { - dirnames = append(dirnames, name) + relDirnames = append(relDirnames, name) } - } else if strings.HasSuffix(name, ".wuffs") { - filenames = append(filenames, name) + } else if strings.HasSuffix(name, suffix) { + dstQF = append(dstQF, filepath.Join(qualDirname, name)) } } - sort.Strings(filenames) - sort.Strings(dirnames) - return filenames, dirnames, nil + return dstQF, relDirnames, nil +} + +func writeFile(filename string, contents []byte) error { + if existing, err := ioutil.ReadFile(filename); err == nil && bytes.Equal(existing, contents) { + fmt.Println("gen unchanged: ", filename) + return nil + } + if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil { + return err + } + if err := ioutil.WriteFile(filename, contents, 0644); err != nil { + return err + } + fmt.Println("gen wrote: ", filename) + return nil } const ( diff --git a/cmd/wuffs/release.go b/cmd/wuffs/release.go new file mode 100644 index 00000000..f7af9efc --- /dev/null +++ b/cmd/wuffs/release.go @@ -0,0 +1,130 @@ +// Copyright 2018 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "flag" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + + cf "github.com/google/wuffs/cmd/commonflags" +) + +func doGenrelease(wuffsRoot string, args []string) error { + flags := flag.NewFlagSet("genrelease", flag.ExitOnError) + langsFlag := flags.String("langs", langsDefault, langsUsage) + skipgenFlag := flags.Bool("skipgen", skipgenDefault, skipgenUsage) + versionFlag := flags.String("version", cf.VersionDefault, cf.VersionUsage) + + if err := flags.Parse(args); err != nil { + return err + } + langs, err := parseLangs(*langsFlag) + if err != nil { + return err + } + v, ok := cf.ParseVersion(*versionFlag) + if !ok { + return fmt.Errorf("bad -version flag value %q", *versionFlag) + } + + if !*skipgenFlag { + gh := genHelper{ + wuffsRoot: wuffsRoot, + langs: langs, + cformatter: cf.CformatterDefault, + } + if err := gh.gen("std", true); err != nil { + return err + } + } + + revision := findRevision(wuffsRoot) + for _, lang := range langs { + if err := doGenrelease1(wuffsRoot, revision, v, lang, lang); err != nil { + return err + } + + if lang == "c" { + if err := doGenrelease1(wuffsRoot, revision, v, "c", "h"); err != nil { + return err + } + } + } + return nil +} + +func doGenrelease1(wuffsRoot string, revision string, v cf.Version, lang string, suffix string) error { + qualFilenames, err := findFiles(filepath.Join(wuffsRoot, "gen", suffix), "."+suffix) + if err != nil { + return err + } + + command := "wuffs-" + lang + args := []string(nil) + args = append(args, "genrelease", "-revision", revision, "-version", v.String()) + args = append(args, qualFilenames...) + stdout := &bytes.Buffer{} + + cmd := exec.Command(command, args...) + cmd.Stdout = stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err == nil { + // No-op. + } else if _, ok := err.(*exec.ExitError); ok { + return fmt.Errorf("%s: failed", command) + } else { + return err + } + + wv := fmt.Sprintf("wuffs-v%d.%d", v.Major, v.Minor) + return writeFile( + filepath.Join(wuffsRoot, "release", lang, wv, wv+"."+suffix), + stdout.Bytes(), + ) +} + +func findRevision(wuffsRoot string) string { + // Assume that we're using git. + + head, err := ioutil.ReadFile(filepath.Join(wuffsRoot, ".git", "HEAD")) + if err != nil { + return "" + } + refPrefix := []byte("ref: ") + if !bytes.HasPrefix(head, refPrefix) { + return "" + } + head = head[len(refPrefix):] + if len(head) == 0 || head[len(head)-1] != '\n' { + return "" + } + head = head[:len(head)-1] + + ref, err := ioutil.ReadFile(filepath.Join(wuffsRoot, ".git", string(head))) + if err != nil { + return "" + } + if len(ref) == 0 || ref[len(ref)-1] != '\n' { + return "" + } + ref = ref[:len(ref)-1] + + return string(ref) +} diff --git a/cmd/wuffs/test.go b/cmd/wuffs/test.go index 1a53dbd3..cea2280c 100644 --- a/cmd/wuffs/test.go +++ b/cmd/wuffs/test.go @@ -139,11 +139,12 @@ type testHelper struct { } func (h *testHelper) benchTest(dirname string, recursive bool) (failed bool, err error) { - filenames, dirnames, err := listDir(h.wuffsRoot, dirname, recursive) + qualFilenames, dirnames, err := listDir( + filepath.Join(h.wuffsRoot, filepath.FromSlash(dirname)), ".wuffs", recursive) if err != nil { return false, err } - if len(filenames) > 0 { + if len(qualFilenames) > 0 { f, err := h.benchTestDir(dirname) if err != nil { return false, err