Add a "wuffs genrelease" command

This commit is contained in:
Nigel Tao committed 2018-06-16 11:47:35 +10:00
1 parent beed8a1c52
commit dccf0c60d7
9 files changed
+479 -38

No files matched your search

+1
View File
@@ -290,6 +290,7 @@ background.
- `gen` holds the transpiled editions of that standard library. The initial
focus is generating C code. Later on, the repository might include generated
Go and Rust code.
- `release` holds the releases of the Wuffs standard library.
- `test` holds the regular tests for the Wuffs standard library.
- `fuzz` holds the fuzz tests for the Wuffs standard library.
- `script` holds miscellaneous utility programs.
+4 -2
View File
@@ -13,8 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# This script is a simple sanity check that the tests pass and the example and
# fuzz programs compile.
# This script, build-all.sh, is a simple sanity check that the tests pass and
# the example and fuzz programs compile. Despite the "all" in the script name,
# it does not build the release editions, as building a release is a separate
# process from e.g. checking that all tests pass before pushing commits.
#
# If you're just looking to get started with Wuffs, running this script isn't
# necessary (as Wuffs doesn't have the "configure; make; make install" dance or
+38 -1
View File
@@ -19,7 +19,9 @@
package commonflags
import (
"fmt"
"path"
"strings"
)
const (
@@ -44,13 +46,16 @@ const (
RepsMin = 0
RepsMax = 1000000
RepsUsage = `the number of repetitions per benchmark`
VersionDefault = "0.0.0"
VersionUsage = `version string, e.g. "1.2.3-beta.4"`
)
// TODO: do IsAlphaNumericIsh and IsValidUsePath belong in a separate package,
// such as lang/validate? Perhaps together with token.Unescape?
// IsAlphaNumericIsh returns whether s contains only ASCII alpha-numerics and a
// limited set of punctuation such as commas and slahes, but not containing
// limited set of punctuation such as commas and slashes, but not containing
// e.g. spaces, semi-colons, colons or backslashes.
//
// The intent is that if s is alpha-numeric-ish, then it should not need
@@ -70,3 +75,35 @@ func IsAlphaNumericIsh(s string) bool {
func IsValidUsePath(s string) bool {
return s == path.Clean(s) && s != "" && s[0] != '.' && s[0] != '/'
}
type Version struct {
Major uint32
Minor uint16
Patch uint16
Extension string
}
func ParseVersion(s string) (v Version, ok bool) {
if !IsAlphaNumericIsh(s) {
return Version{}, false
}
if i := strings.IndexByte(s, '-'); i >= 0 {
s, v.Extension = s[:i], s[i+1:]
}
if _, err := fmt.Sscanf(s, "%d.%d.%d", &v.Major, &v.Minor, &v.Patch); err != nil {
return Version{}, false
}
return v, true
}
func (v Version) String() string {
s := fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch)
if v.Extension != "" {
s += "-" + v.Extension
}
return s
}
func (v Version) Uint64() uint64 {
return (uint64(v.Major) << 32) | (uint64(v.Minor) << 16) | (uint64(v.Patch) << 0)
}
+2
View File
@@ -41,6 +41,8 @@ func main1() error {
return cgen.Do(args)
case "genlib":
return doGenlib(args)
case "genrelease":
return doGenrelease(args)
case "test":
return doTest(args)
}
+230
View File
@@ -0,0 +1,230 @@
// Copyright 2018 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"bytes"
"flag"
"fmt"
"io/ioutil"
"os"
"os/exec"
"time"
cf "github.com/google/wuffs/cmd/commonflags"
)
func doGenrelease(args []string) error {
flags := flag.FlagSet{}
cformatterFlag := flags.String("cformatter", cf.CformatterDefault, cf.CformatterUsage)
revisionFlag := flags.String("revision", "", "git revision the release was built from")
versionFlag := flags.String("version", cf.VersionDefault, cf.VersionUsage)
if err := flags.Parse(args); err != nil {
return err
}
if !cf.IsAlphaNumericIsh(*cformatterFlag) {
return fmt.Errorf("bad -cformatter flag value %q", *cformatterFlag)
}
if !cf.IsAlphaNumericIsh(*revisionFlag) {
return fmt.Errorf("bad -revision flag value %q", *revisionFlag)
}
v, ok := cf.ParseVersion(*versionFlag)
if !ok {
return fmt.Errorf("bad -version flag value %q", *versionFlag)
}
args = flags.Args()
h := &genReleaseHelper{
seen: map[string]bool{},
revision: *revisionFlag,
version: v,
}
unformatted := bytes.NewBuffer(nil)
unformatted.WriteString("#ifndef WUFFS_INCLUDE_GUARD\n")
unformatted.WriteString("#define WUFFS_INCLUDE_GUARD\n\n")
// First, cat all of the headers together, filtering out duplicate
// WUFFS_INCLUDE_GUARD__FOO sections and overriding WUFFS_VERSION.
for _, filename := range args {
s, err := ioutil.ReadFile(filename)
if err != nil {
return err
}
if i := bytes.Index(s, grCHeader); i >= 0 {
s = s[:i]
}
if err := h.gen(unformatted, s); err != nil {
return fmt.Errorf("%v in %s", err, filename)
}
}
unformatted.WriteString("\n\n#endif // WUFFS_INCLUDE_GUARD\n\n")
// Then, cat all of the implementations together, filtering out duplicate
// WUFFS_INCLUDE_GUARD__BASE_PRIVATE sections.
for _, filename := range args {
s, err := ioutil.ReadFile(filename)
if err != nil {
return err
}
if i := bytes.Index(s, grCHeader); i >= 0 {
s = s[i+len(grCHeader):]
} else {
continue
}
if err := h.gen(unformatted, s); err != nil {
return fmt.Errorf("%v in %s", err, filename)
}
}
cmd := exec.Command(*cformatterFlag, "-style=Chromium")
cmd.Stdin = unformatted
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
return cmd.Run()
}
var (
grCHeader = []byte("// !! C HEADER ENDS HERE.\n")
grNN = []byte("\n\n")
grVOverride = []byte("// !! Some code generation programs can override WUFFS_VERSION.\n")
grVString = []byte(`#define WUFFS_VERSION_STRING "0.0.0"`)
grDefine = []byte("#define WUFFS_INCLUDE_GUARD__")
grEndif = []byte("#endif // WUFFS_INCLUDE_GUARD__")
grIfndef = []byte("#ifndef WUFFS_INCLUDE_GUARD__")
)
type genReleaseHelper struct {
seen map[string]bool
revision string
version cf.Version
}
func (h *genReleaseHelper) gen(w *bytes.Buffer, s []byte) error {
skipping := 0
stack := []string{}
for remaining := []byte(nil); len(s) > 0; s, remaining = remaining, nil {
if i := bytes.IndexByte(s, '\n'); i >= 0 {
s, remaining = s[:i+1], s[i+1:]
}
if len(s) == 0 {
continue
}
if s[0] == '#' {
switch {
case bytes.HasPrefix(s, grIfndef):
pkg := string(s[len(grIfndef):])
for _, p := range stack {
if p == pkg {
return fmt.Errorf("unexpected %q", s)
}
}
if h.seen[pkg] {
skipping++
}
stack = append(stack, pkg)
continue
case bytes.HasPrefix(s, grDefine):
pkg := string(s[len(grDefine):])
if (len(stack) == 0) || (stack[len(stack)-1] != pkg) {
return fmt.Errorf("unexpected %q", s)
}
continue
case bytes.HasPrefix(s, grEndif):
pkg := string(s[len(grEndif):])
if (len(stack) == 0) || (stack[len(stack)-1] != pkg) {
return fmt.Errorf("unexpected %q", s)
}
if h.seen[pkg] {
skipping--
} else {
h.seen[pkg] = true
}
stack = stack[:len(stack)-1]
continue
}
}
if skipping > 0 {
continue
}
if s[0] == '/' {
switch {
case bytes.Equal(s, grVOverride):
var err error
remaining, err = h.genWuffsVersion(w, remaining)
if err != nil {
return err
}
continue
}
}
w.Write(s)
}
if len(stack) != 0 {
return fmt.Errorf("unmatched %q", string(grIfndef)+stack[len(stack)-1])
}
return nil
}
func (h *genReleaseHelper) genWuffsVersion(w *bytes.Buffer, s []byte) (remaining []byte, err error) {
cut := []byte(nil)
if i := bytes.Index(s, grNN); i >= 0 {
cut, s = s[:i], s[i+len(grNN):]
} else {
return nil, fmt.Errorf(`could not find "\n\n" near WUFFS_VERSION`)
}
if !bytes.HasSuffix(cut, grVString) {
return nil, fmt.Errorf("%q did not end with %q", cut, grVString)
}
fmt.Fprintf(w, "// WUFFS_VERSION was overridden by \"wuffs genrelease\" on %v UTC",
time.Now().UTC().Format("2006-01-02"))
if h.revision != "" {
fmt.Fprintf(w, ",\n// based on revision %s", h.revision)
}
fmt.Fprintf(w, `.
#define WUFFS_VERSION ((uint64_t)0x%016X)
#define WUFFS_VERSION_MAJOR ((uint64_t)0x%08X)
#define WUFFS_VERSION_MINOR ((uint64_t)0x%04X)
#define WUFFS_VERSION_PATCH ((uint64_t)0x%04X)
#define WUFFS_VERSION_EXTENSION %q
#define WUFFS_VERSION_STRING %q
`, h.version.Uint64(), h.version.Major, h.version.Minor, h.version.Patch,
h.version.Extension, h.version)
return s, nil
}
+13 -25
View File
@@ -18,7 +18,6 @@ import (
"bytes"
"flag"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path"
@@ -119,12 +118,13 @@ func (h *genHelper) gen(dirname string, recursive bool) error {
return fmt.Errorf("invalid package path %q", dirname)
}
filenames, dirnames, err := listDir(h.wuffsRoot, dirname, recursive)
qualFilenames, dirnames, err := listDir(
filepath.Join(h.wuffsRoot, filepath.FromSlash(dirname)), ".wuffs", recursive)
if err != nil {
return err
}
if len(filenames) > 0 {
if err := h.genDir(dirname, filenames); err != nil {
if len(qualFilenames) > 0 {
if err := h.genDir(dirname, qualFilenames); err != nil {
return err
}
h.affected = append(h.affected, dirname)
@@ -139,7 +139,7 @@ func (h *genHelper) gen(dirname string, recursive bool) error {
return nil
}
func (h *genHelper) genDir(dirname string, filenames []string) error {
func (h *genHelper) genDir(dirname string, qualFilenames []string) error {
// TODO: skip the generation if the output file already exists and its
// mtime is newer than all inputs and the wuffs-gen-foo command.
@@ -147,12 +147,9 @@ func (h *genHelper) genDir(dirname string, filenames []string) error {
if !validName(packageName) {
return fmt.Errorf(`invalid package %q, not in [a-z0-9]+`, packageName)
}
qualifiedFilenames := make([]string, len(filenames))
for i, filename := range filenames {
qualifiedFilenames[i] = filepath.Join(h.wuffsRoot, filepath.FromSlash(dirname), filename)
}
if !h.skipgendeps {
if err := h.genDirDependencies(qualifiedFilenames); err != nil {
if err := h.genDirDependencies(qualFilenames); err != nil {
return err
}
}
@@ -163,7 +160,7 @@ func (h *genHelper) genDir(dirname string, filenames []string) error {
if lang == "c" {
cmdArgs = append(cmdArgs, fmt.Sprintf("-cformatter=%s", h.cformatter))
}
cmdArgs = append(cmdArgs, qualifiedFilenames...)
cmdArgs = append(cmdArgs, qualFilenames...)
stdout := &bytes.Buffer{}
cmd := exec.Command(command, cmdArgs...)
@@ -196,7 +193,7 @@ func (h *genHelper) genDir(dirname string, filenames []string) error {
}
}
if len(h.langs) > 0 && packageName != "base" {
if err := h.genWuffs(dirname, qualifiedFilenames); err != nil {
if err := h.genWuffs(dirname, qualFilenames); err != nil {
return err
}
}
@@ -226,19 +223,10 @@ func (h *genHelper) genDirDependencies(qualifiedFilenames []string) error {
}
func (h *genHelper) genFile(dirname string, lang string, out []byte) error {
outFilename := filepath.Join(h.wuffsRoot, "gen", lang, filepath.FromSlash(dirname)+"."+lang)
if existing, err := ioutil.ReadFile(outFilename); err == nil && bytes.Equal(existing, out) {
fmt.Println("gen unchanged: ", outFilename)
return nil
}
if err := os.MkdirAll(filepath.Dir(outFilename), 0755); err != nil {
return err
}
if err := ioutil.WriteFile(outFilename, out, 0644); err != nil {
return err
}
fmt.Println("gen wrote: ", outFilename)
return nil
return writeFile(
filepath.Join(h.wuffsRoot, "gen", lang, filepath.FromSlash(dirname)+"."+lang),
out,
)
}
func (h *genHelper) genWuffs(dirname string, qualifiedFilenames []string) error {
+58 -8
View File
@@ -16,8 +16,10 @@
package main
import (
"bytes"
"flag"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"sort"
@@ -33,6 +35,7 @@ var commands = []struct {
{"bench", doBench},
{"gen", doGen},
{"genlib", doGenlib},
{"genrelease", doGenrelease},
{"test", doTest},
}
@@ -48,6 +51,7 @@ The commands are:
bench benchmark packages
gen generate code for packages and dependencies
genlib generate software libraries
genrelease generate a stand-alone release
test test packages
`)
}
@@ -79,8 +83,41 @@ func main1() error {
return nil
}
func listDir(wuffsRoot string, dirname string, returnSubdirs bool) (filenames []string, dirnames []string, err error) {
f, err := os.Open(filepath.Join(wuffsRoot, filepath.FromSlash(dirname)))
func findFiles(qualDirname string, suffix string) (filenames []string, err error) {
filenames, err = findFiles1(nil, qualDirname, suffix)
if err != nil {
return nil, err
}
sort.Strings(filenames)
return filenames, nil
}
func findFiles1(dstQF []string, qualDirname string, suffix string) (qualFilenames []string, err error) {
dstQF, relDirnames, err := appendDir(dstQF, qualDirname, suffix, true)
if err != nil {
return nil, err
}
for _, d := range relDirnames {
dstQF, err = findFiles1(dstQF, filepath.Join(qualDirname, d), suffix)
if err != nil {
return nil, err
}
}
return dstQF, nil
}
func listDir(qualDirname string, suffix string, returnSubdirs bool) (qualFilenames []string, relDirnames []string, err error) {
qualFilenames, relDirnames, err = appendDir(nil, qualDirname, suffix, returnSubdirs)
if err != nil {
return nil, nil, err
}
sort.Strings(qualFilenames)
sort.Strings(relDirnames)
return qualFilenames, relDirnames, nil
}
func appendDir(dstQF []string, qualDirname string, suffix string, returnSubdirs bool) (qualFilenames []string, relDirnames []string, err error) {
f, err := os.Open(qualDirname)
if err != nil {
return nil, nil, err
}
@@ -94,16 +131,29 @@ func listDir(wuffsRoot string, dirname string, returnSubdirs bool) (filenames []
name := o.Name()
if o.IsDir() {
if returnSubdirs {
dirnames = append(dirnames, name)
relDirnames = append(relDirnames, name)
}
} else if strings.HasSuffix(name, ".wuffs") {
filenames = append(filenames, name)
} else if strings.HasSuffix(name, suffix) {
dstQF = append(dstQF, filepath.Join(qualDirname, name))
}
}
sort.Strings(filenames)
sort.Strings(dirnames)
return filenames, dirnames, nil
return dstQF, relDirnames, nil
}
func writeFile(filename string, contents []byte) error {
if existing, err := ioutil.ReadFile(filename); err == nil && bytes.Equal(existing, contents) {
fmt.Println("gen unchanged: ", filename)
return nil
}
if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil {
return err
}
if err := ioutil.WriteFile(filename, contents, 0644); err != nil {
return err
}
fmt.Println("gen wrote: ", filename)
return nil
}
const (
+130
View File
@@ -0,0 +1,130 @@
// Copyright 2018 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"bytes"
"flag"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
cf "github.com/google/wuffs/cmd/commonflags"
)
func doGenrelease(wuffsRoot string, args []string) error {
flags := flag.NewFlagSet("genrelease", flag.ExitOnError)
langsFlag := flags.String("langs", langsDefault, langsUsage)
skipgenFlag := flags.Bool("skipgen", skipgenDefault, skipgenUsage)
versionFlag := flags.String("version", cf.VersionDefault, cf.VersionUsage)
if err := flags.Parse(args); err != nil {
return err
}
langs, err := parseLangs(*langsFlag)
if err != nil {
return err
}
v, ok := cf.ParseVersion(*versionFlag)
if !ok {
return fmt.Errorf("bad -version flag value %q", *versionFlag)
}
if !*skipgenFlag {
gh := genHelper{
wuffsRoot: wuffsRoot,
langs: langs,
cformatter: cf.CformatterDefault,
}
if err := gh.gen("std", true); err != nil {
return err
}
}
revision := findRevision(wuffsRoot)
for _, lang := range langs {
if err := doGenrelease1(wuffsRoot, revision, v, lang, lang); err != nil {
return err
}
if lang == "c" {
if err := doGenrelease1(wuffsRoot, revision, v, "c", "h"); err != nil {
return err
}
}
}
return nil
}
func doGenrelease1(wuffsRoot string, revision string, v cf.Version, lang string, suffix string) error {
qualFilenames, err := findFiles(filepath.Join(wuffsRoot, "gen", suffix), "."+suffix)
if err != nil {
return err
}
command := "wuffs-" + lang
args := []string(nil)
args = append(args, "genrelease", "-revision", revision, "-version", v.String())
args = append(args, qualFilenames...)
stdout := &bytes.Buffer{}
cmd := exec.Command(command, args...)
cmd.Stdout = stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err == nil {
// No-op.
} else if _, ok := err.(*exec.ExitError); ok {
return fmt.Errorf("%s: failed", command)
} else {
return err
}
wv := fmt.Sprintf("wuffs-v%d.%d", v.Major, v.Minor)
return writeFile(
filepath.Join(wuffsRoot, "release", lang, wv, wv+"."+suffix),
stdout.Bytes(),
)
}
func findRevision(wuffsRoot string) string {
// Assume that we're using git.
head, err := ioutil.ReadFile(filepath.Join(wuffsRoot, ".git", "HEAD"))
if err != nil {
return ""
}
refPrefix := []byte("ref: ")
if !bytes.HasPrefix(head, refPrefix) {
return ""
}
head = head[len(refPrefix):]
if len(head) == 0 || head[len(head)-1] != '\n' {
return ""
}
head = head[:len(head)-1]
ref, err := ioutil.ReadFile(filepath.Join(wuffsRoot, ".git", string(head)))
if err != nil {
return ""
}
if len(ref) == 0 || ref[len(ref)-1] != '\n' {
return ""
}
ref = ref[:len(ref)-1]
return string(ref)
}
+3 -2
View File
@@ -139,11 +139,12 @@ type testHelper struct {
}
func (h *testHelper) benchTest(dirname string, recursive bool) (failed bool, err error) {
filenames, dirnames, err := listDir(h.wuffsRoot, dirname, recursive)
qualFilenames, dirnames, err := listDir(
filepath.Join(h.wuffsRoot, filepath.FromSlash(dirname)), ".wuffs", recursive)
if err != nil {
return false, err
}
if len(filenames) > 0 {
if len(qualFilenames) > 0 {
f, err := h.benchTestDir(dirname)
if err != nil {
return false, err