hack/update-vendor.sh

This commit is contained in:
Jordan Liggitt
2019-11-05 14:11:10 -05:00
parent 9a5b7c24ad
commit 297570e06a
932 changed files with 77190 additions and 28219 deletions

View File

@@ -15,38 +15,119 @@ distributed under the License is distributed on an "AS IS" BASIS,
limitations under the License.
"""
load(
"@io_bazel_rules_go//go/private:providers.bzl",
"GoSource",
)
_GO_YACC_TOOL = "@org_golang_x_tools//cmd/goyacc"
def go_yacc(src, out, visibility=None):
"""Runs go tool yacc -o $out $src."""
native.genrule(
name = src + ".go_yacc",
srcs = [src],
outs = [out],
tools = [_GO_YACC_TOOL],
cmd = ("export GOROOT=$$(dirname $(location " + _GO_YACC_TOOL + "))/..;" +
" $(location " + _GO_YACC_TOOL + ") " +
" -o $(location " + out + ") $(SRCS)"),
visibility = visibility,
local = 1,
)
def go_yacc(src, out, visibility = None):
"""Runs go tool yacc -o $out $src."""
native.genrule(
name = src + ".go_yacc",
srcs = [src],
outs = [out],
tools = [_GO_YACC_TOOL],
cmd = ("export GOROOT=$$(dirname $(location " + _GO_YACC_TOOL + "))/..;" +
" $(location " + _GO_YACC_TOOL + ") " +
" -o $(location " + out + ") $(SRCS) > /dev/null"),
visibility = visibility,
)
def _extract_go_src(ctx):
"""Thin rule that exposes the GoSource from a go_library."""
return [DefaultInfo(files = depset(ctx.attr.library[GoSource].srcs))]
extract_go_src = rule(
implementation = _extract_go_src,
attrs = {
"library": attr.label(
providers = [GoSource],
),
},
)
def genfile_check_test(src, gen):
"""Asserts that any checked-in generated code matches regen."""
if not src:
fail("src is required", "src")
if not gen:
fail("gen is required", "gen")
native.genrule(
name = src + "_checksh",
outs = [src + "_check.sh"],
cmd = "echo 'diff $$@' > $@",
)
native.sh_test(
name = src + "_checkshtest",
size = "small",
srcs = [src + "_check.sh"],
data = [src, gen],
args = ["$(location " + src + ")", "$(location " + gen + ")"],
)
"""Asserts that any checked-in generated code matches bazel gen."""
if not src:
fail("src is required", "src")
if not gen:
fail("gen is required", "gen")
native.genrule(
name = src + "_checksh",
outs = [src + "_check.sh"],
cmd = r"""cat >$@ <<'eof'
#!/bin/bash
# Script generated by @com_github_bazelbuild_buildtools//build:build_defs.bzl
# --- begin runfiles.bash initialization ---
# Copy-pasted from Bazel's Bash runfiles library (tools/bash/runfiles/runfiles.bash).
set -euo pipefail
if [[ ! -d "$${RUNFILES_DIR:-/dev/null}" && ! -f "$${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
if [[ -f "$$0.runfiles_manifest" ]]; then
export RUNFILES_MANIFEST_FILE="$$0.runfiles_manifest"
elif [[ -f "$$0.runfiles/MANIFEST" ]]; then
export RUNFILES_MANIFEST_FILE="$$0.runfiles/MANIFEST"
elif [[ -f "$$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
export RUNFILES_DIR="$$0.runfiles"
fi
fi
if [[ -f "$${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
source "$${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash"
elif [[ -f "$${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
source "$$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " \
"$$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)"
else
echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash"
exit 1
fi
# --- end runfiles.bash initialization ---
[[ "$$1" = external/* ]] && F1="$${1#external/}" || F1="$$TEST_WORKSPACE/$$1"
[[ "$$2" = external/* ]] && F2="$${2#external/}" || F2="$$TEST_WORKSPACE/$$2"
F1="$$(rlocation "$$F1")"
F2="$$(rlocation "$$F2")"
diff -q "$$F1" "$$F2"
eof
""",
)
native.sh_test(
name = src + "_checkshtest",
size = "small",
srcs = [src + "_check.sh"],
deps = ["@bazel_tools//tools/bash/runfiles"],
data = [src, gen],
args = ["$(location " + src + ")", "$(location " + gen + ")"],
)
# magic copy rule used to update the checked-in version
native.genrule(
name = src + "_copysh",
srcs = [gen],
outs = [src + "copy.sh"],
cmd = "echo 'cp $${BUILD_WORKSPACE_DIRECTORY}/$(location " + gen +
") $${BUILD_WORKSPACE_DIRECTORY}/" + native.package_name() + "/" + src + "' > $@",
)
native.sh_binary(
name = src + "_copy",
srcs = [src + "_copysh"],
data = [gen],
)
def go_proto_checkedin_test(src, proto = "go_default_library"):
"""Asserts that any checked-in .pb.go code matches bazel gen."""
genfile = src + "_genfile"
extract_go_src(
name = genfile + "go",
library = proto,
)
# TODO(pmbethe09): why is the extra copy needed?
native.genrule(
name = genfile,
srcs = [genfile + "go"],
outs = [genfile + ".go"],
cmd = "cp $< $@",
)
genfile_check_test(src, genfile)

View File

@@ -20,18 +20,146 @@ package build
import (
"bytes"
"fmt"
"path/filepath"
"sort"
"strings"
"unicode/utf8"
"github.com/bazelbuild/buildtools/tables"
)
// FileType represents a type of a file (default (for .bzl files), BUILD, or WORKSPACE).
// Certain formatting or refactoring rules can be applied to several file types, so they support
// bitwise operations: `type1 | type2` can represent a scope (e.g. BUILD and WORKSPACE files) and
// `scope & fileType` can be used to check whether a file type belongs to a scope.
type FileType int
const (
// TypeDefault represents general Starlark files
TypeDefault FileType = 1 << iota
// TypeBuild represents BUILD files
TypeBuild
// TypeWorkspace represents WORKSPACE files
TypeWorkspace
// TypeBzl represents .bzl files
TypeBzl
)
func (t FileType) String() string {
switch t {
case TypeDefault:
return "default"
case TypeBuild:
return "BUILD"
case TypeWorkspace:
return "WORKSPACE"
case TypeBzl:
return ".bzl"
}
return "unknown"
}
// ParseBuild parses a file, marks it as a BUILD file and returns the corresponding parse tree.
//
// The filename is used only for generating error messages.
func ParseBuild(filename string, data []byte) (*File, error) {
in := newInput(filename, data)
f, err := in.parse()
if f != nil {
f.Type = TypeBuild
}
return f, err
}
// ParseWorkspace parses a file, marks it as a WORKSPACE file and returns the corresponding parse tree.
//
// The filename is used only for generating error messages.
func ParseWorkspace(filename string, data []byte) (*File, error) {
in := newInput(filename, data)
f, err := in.parse()
if f != nil {
f.Type = TypeWorkspace
}
return f, err
}
// ParseBzl parses a file, marks it as a .bzl file and returns the corresponding parse tree.
//
// The filename is used only for generating error messages.
func ParseBzl(filename string, data []byte) (*File, error) {
in := newInput(filename, data)
f, err := in.parse()
if f != nil {
f.Type = TypeBzl
}
return f, err
}
// ParseDefault parses a file, marks it as a generic Starlark file and returns the corresponding parse tree.
//
// The filename is used only for generating error messages.
func ParseDefault(filename string, data []byte) (*File, error) {
in := newInput(filename, data)
f, err := in.parse()
if f != nil {
f.Type = TypeDefault
}
return f, err
}
func getFileType(filename string) FileType {
if filename == "" { // stdin
return TypeDefault
}
basename := strings.ToLower(filepath.Base(filename))
if strings.HasSuffix(basename, ".oss") {
basename = basename[:len(basename)-4]
}
ext := filepath.Ext(basename)
switch ext {
case ".bzl":
return TypeBzl
case ".sky":
return TypeDefault
}
base := basename[:len(basename)-len(ext)]
switch {
case ext == ".build" || base == "build" || strings.HasPrefix(base, "build."):
return TypeBuild
case ext == ".workspace" || base == "workspace" || strings.HasPrefix(base, "workspace."):
return TypeWorkspace
}
return TypeDefault
}
// Parse parses the input data and returns the corresponding parse tree.
//
// The filename is used only for generating error messages.
// Uses the filename to detect the formatting type (build, workspace, or default) and calls
// ParseBuild, ParseWorkspace, or ParseDefault correspondingly.
func Parse(filename string, data []byte) (*File, error) {
in := newInput(filename, data)
return in.parse()
switch getFileType(filename) {
case TypeBuild:
return ParseBuild(filename, data)
case TypeWorkspace:
return ParseWorkspace(filename, data)
case TypeBzl:
return ParseBzl(filename, data)
}
return ParseDefault(filename, data)
}
// ParseError contains information about the error encountered during parsing.
type ParseError struct {
Message string
Filename string
Pos Position
}
// Error returns a string representation of the parse error.
func (e ParseError) Error() string {
filename := e.Filename
if filename == "" {
filename = "<stdin>"
}
return fmt.Sprintf("%s:%d:%d: %v", filename, e.Pos.Line, e.Pos.LineRune, e.Message)
}
// An input represents a single input file being parsed.
@@ -45,7 +173,6 @@ type input struct {
pos Position // current input position
lineComments []Comment // accumulated line comments
suffixComments []Comment // accumulated suffix comments
endStmt int // position of the end of the current statement
depth int // nesting of [ ] { } ( )
cleanLine bool // true if the current line only contains whitespace before the current position
indent int // current line indentation in spaces
@@ -73,7 +200,6 @@ func newInput(filename string, data []byte) *input {
pos: Position{Line: 1, LineRune: 1, Byte: 0},
cleanLine: true,
indents: []int{0},
endStmt: -1, // -1 denotes it's not inside a statement
}
}
@@ -92,7 +218,7 @@ func (in *input) parse() (f *File, err error) {
if e == in.parseError {
err = in.parseError
} else {
err = fmt.Errorf("%s:%d:%d: internal error: %v", in.filename, in.pos.Line, in.pos.LineRune, e)
err = ParseError{Message: fmt.Sprintf("internal error: %v", e), Filename: in.filename, Pos: in.pos}
}
}
}()
@@ -117,7 +243,7 @@ func (in *input) Error(s string) {
if s == "syntax error" && in.lastToken != "" {
s += " near " + in.lastToken
}
in.parseError = fmt.Errorf("%s:%d:%d: %v", in.filename, in.pos.Line, in.pos.LineRune, s)
in.parseError = ParseError{Message: s, Filename: in.filename, Pos: in.pos}
panic(in.parseError)
}
@@ -187,25 +313,6 @@ func (in *input) Lex(val *yySymType) int {
// Skip past spaces, stopping at non-space or EOF.
countNL := 0 // number of newlines we've skipped past
for !in.eof() {
// If a single statement is split into multiple lines, we don't need
// to track indentations and unindentations within these lines. For example:
//
// def f(
// # This indentation should be ignored
// x):
// # This unindentation should be ignored
// # Actual indentation is from 0 to 2 spaces here
// return x
//
// To handle this case, when we reach the beginning of a statement we scan forward to see where
// it should end and record the number of input bytes remaining at that endpoint.
//
// If --format_bzl is set to false, top level blocks (e.g. an entire function definition)
// is considered as a single statement.
if in.endStmt != -1 && len(in.remaining) == in.endStmt {
in.endStmt = -1
}
// Skip over spaces. Count newlines so we can give the parser
// information about where top-level blank lines are,
// for top-level comment assignment.
@@ -214,7 +321,7 @@ func (in *input) Lex(val *yySymType) int {
if c == '\n' {
in.indent = 0
in.cleanLine = true
if in.endStmt == -1 {
if in.depth == 0 {
// Not in a statememt. Tell parser about top-level blank line.
in.startToken(val)
in.readRune()
@@ -234,18 +341,23 @@ func (in *input) Lex(val *yySymType) int {
// If a line contains just a comment its indentation level doesn't matter.
// Reset it to zero.
in.indent = 0
isLineComment := in.cleanLine
in.cleanLine = true
// Is this comment the only thing on its line?
// Find the last \n before this # and see if it's all
// spaces from there to here.
// If it's a suffix comment but the last non-space symbol before
// it is one of (, [, or {, treat it as a line comment that should be
// it is one of (, [, or {, or it's a suffix comment to "):"
// (e.g. trailing closing bracket or a function definition),
// treat it as a line comment that should be
// put inside the corresponding block.
i := bytes.LastIndex(in.complete[:in.pos.Byte], []byte("\n"))
prefix := bytes.TrimSpace(in.complete[i+1 : in.pos.Byte])
prefix = bytes.Replace(prefix, []byte{' '}, []byte{}, -1)
isSuffix := true
if len(prefix) == 0 ||
(len(prefix) == 2 && prefix[0] == ')' && prefix[1] == ':') ||
prefix[len(prefix)-1] == '[' ||
prefix[len(prefix)-1] == '(' ||
prefix[len(prefix)-1] == '{' {
@@ -266,7 +378,7 @@ func (in *input) Lex(val *yySymType) int {
// If we are at top level (not in a rule), hand the comment to
// the parser as a _COMMENT token. The grammar is written
// to handle top-level comments itself.
if in.endStmt == -1 {
if in.depth == 0 && isLineComment {
// Not in a statement. Tell parser about top-level comment.
return _COMMENT
}
@@ -296,9 +408,9 @@ func (in *input) Lex(val *yySymType) int {
}
// Check for changes in indentation
// Skip if --format_bzl is set to false, if we're inside a statement, or if there were non-space
// Skip if we're inside a statement, or if there were non-space
// characters before in the current line.
if tables.FormatBzlFiles && in.endStmt == -1 && in.cleanLine {
if in.depth == 0 && in.cleanLine {
if in.indent > in.currentIndent() {
// A new indentation block starts
in.indents = append(in.indents, in.indent)
@@ -340,11 +452,6 @@ func (in *input) Lex(val *yySymType) int {
return _EOF
}
// If endStmt is 0, we need to recompute where the end of the next statement is.
if in.endStmt == -1 {
in.endStmt = len(in.skipStmt(in.remaining))
}
// Punctuation tokens.
switch c := in.peekRune(); c {
case '[', '(', '{':
@@ -361,11 +468,35 @@ func (in *input) Lex(val *yySymType) int {
in.readRune()
return c
case '<', '>', '=', '!', '+', '-', '*', '/', '%': // possibly followed by =
case '<', '>', '=', '!', '+', '-', '*', '/', '%', '|', '&', '~', '^': // possibly followed by =
in.readRune()
if c == '/' && in.peekRune() == '/' {
// integer division
if c == '~' {
// unary bitwise not, shouldn't be followed by anything
return c
}
if c == '*' && in.peekRune() == '*' {
// double asterisk
in.readRune()
return _STAR_STAR
}
if c == in.peekRune() {
switch c {
case '/':
// integer division
in.readRune()
c = _INT_DIV
case '<':
// left shift
in.readRune()
c = _BIT_LSH
case '>':
// right shift
in.readRune()
c = _BIT_RSH
}
}
if in.peekRune() == '=' {
@@ -442,7 +573,7 @@ func (in *input) Lex(val *yySymType) int {
}
}
in.endToken(val)
s, triple, err := unquote(val.tok)
s, triple, err := Unquote(val.tok)
if err != nil {
in.Error(fmt.Sprint(err))
}
@@ -456,19 +587,6 @@ func (in *input) Lex(val *yySymType) int {
in.Error(fmt.Sprintf("unexpected input character %#q", c))
}
if !tables.FormatBzlFiles {
// Look for raw Python block (class, def, if, etc at beginning of line) and pass through.
if in.depth == 0 && in.pos.LineRune == 1 && hasPythonPrefix(in.remaining) {
// Find end of Python block and advance input beyond it.
// Have to loop calling readRune in order to maintain line number info.
rest := in.skipStmt(in.remaining)
for len(in.remaining) > len(rest) {
in.readRune()
}
return _PYTHON
}
}
// Scan over alphanumeric identifier.
for {
c := in.peekRune()
@@ -484,7 +602,17 @@ func (in *input) Lex(val *yySymType) int {
if k := keywordToken[val.tok]; k != 0 {
return k
}
switch val.tok {
case "pass":
return _PASS
case "break":
return _BREAK
case "continue":
return _CONTINUE
}
if len(val.tok) > 0 && val.tok[0] >= '0' && val.tok[0] <= '9' {
return _NUMBER
}
return _IDENT
}
@@ -516,164 +644,6 @@ var keywordToken = map[string]int{
"return": _RETURN,
}
// Python scanning.
// About 1% of BUILD files embed arbitrary Python into the file.
// We do not attempt to parse it. Instead, we lex just enough to scan
// beyond it, treating the Python block as an unintepreted blob.
// hasPythonPrefix reports whether p begins with a keyword that would
// introduce an uninterpreted Python block.
func hasPythonPrefix(p []byte) bool {
if tables.FormatBzlFiles {
return false
}
for _, pre := range prefixes {
if hasPrefixSpace(p, pre) {
return true
}
}
return false
}
// These keywords introduce uninterpreted Python blocks.
var prefixes = []string{
"assert",
"class",
"def",
"del",
"for",
"if",
"try",
"else",
"elif",
"except",
}
// hasPrefixSpace reports whether p begins with pre followed by a space or colon.
func hasPrefixSpace(p []byte, pre string) bool {
if len(p) <= len(pre) || p[len(pre)] != ' ' && p[len(pre)] != '\t' && p[len(pre)] != ':' {
return false
}
for i := range pre {
if p[i] != pre[i] {
return false
}
}
return true
}
// A utility function for the legacy formatter.
// Returns whether a given code starts with a top-level statement (maybe with some preceeding
// comments and blank lines)
func isOutsideBlock(b []byte) bool {
isBlankLine := true
isComment := false
for _, c := range b {
switch {
case c == ' ' || c == '\t' || c == '\r':
isBlankLine = false
case c == '#':
isBlankLine = false
isComment = true
case c == '\n':
isBlankLine = true
isComment = false
default:
if !isComment {
return isBlankLine
}
}
}
return true
}
// skipStmt returns the data remaining after the statement beginning at p.
// It does not advance the input position.
// (The only reason for the input receiver is to be able to call in.Error.)
func (in *input) skipStmt(p []byte) []byte {
quote := byte(0) // if non-zero, the kind of quote we're in
tripleQuote := false // if true, the quote is a triple quote
depth := 0 // nesting depth for ( ) [ ] { }
var rest []byte // data after the Python block
defer func() {
if quote != 0 {
in.Error("EOF scanning Python quoted string")
}
}()
// Scan over input one byte at a time until we find
// an unindented, non-blank, non-comment line
// outside quoted strings and brackets.
for i := 0; i < len(p); i++ {
c := p[i]
if quote != 0 && c == quote && !tripleQuote {
quote = 0
continue
}
if quote != 0 && c == quote && tripleQuote && i+2 < len(p) && p[i+1] == quote && p[i+2] == quote {
i += 2
quote = 0
tripleQuote = false
continue
}
if quote != 0 {
if c == '\\' {
i++ // skip escaped char
}
continue
}
if c == '\'' || c == '"' {
if i+2 < len(p) && p[i+1] == c && p[i+2] == c {
quote = c
tripleQuote = true
i += 2
continue
}
quote = c
continue
}
if depth == 0 && i > 0 && p[i] == '\n' && p[i-1] != '\\' {
// Possible stopping point. Save the earliest one we find.
if rest == nil {
rest = p[i:]
}
if tables.FormatBzlFiles {
// In the bzl files mode we only care about the end of the statement, we've found it.
return rest
}
// In the legacy mode we need to find where the current block ends
if isOutsideBlock(p[i+1:]) {
return rest
}
// Not a stopping point after all.
rest = nil
}
switch c {
case '#':
// Skip comment.
for i < len(p) && p[i] != '\n' {
i++
}
// Rewind 1 position back because \n should be handled at the next iteration
i--
case '(', '[', '{':
depth++
case ')', ']', '}':
depth--
}
}
return rest
}
// Comment assignment.
// We build two lists of all subexpressions, preorder and postorder.
// The preorder list is ordered by start location, with outer expressions first.
@@ -707,12 +677,21 @@ func (in *input) order(v Expr) {
in.order(x)
}
in.order(&v.End)
case *PythonBlock:
// nothing
case *LoadStmt:
in.order(v.Module)
for i := range v.From {
in.order(v.To[i])
in.order(v.From[i])
}
in.order(&v.Rparen)
case *LiteralExpr:
// nothing
case *StringExpr:
// nothing
case *Ident:
// nothing
case *BranchStmt:
// nothing
case *DotExpr:
in.order(v.X)
case *ListExpr:
@@ -720,9 +699,9 @@ func (in *input) order(v Expr) {
in.order(x)
}
in.order(&v.End)
case *ListForExpr:
in.order(v.X)
for _, c := range v.For {
case *Comprehension:
in.order(v.Body)
for _, c := range v.Clauses {
in.order(c)
}
in.order(&v.End)
@@ -731,16 +710,9 @@ func (in *input) order(v Expr) {
in.order(x)
}
in.order(&v.End)
case *ForClauseWithIfClausesOpt:
in.order(v.For)
for _, c := range v.Ifs {
in.order(c)
}
case *ForClause:
for _, name := range v.Var {
in.order(name)
}
in.order(v.Expr)
in.order(v.Vars)
in.order(v.X)
case *IfClause:
in.order(v.Cond)
case *KeyValueExpr:
@@ -755,12 +727,17 @@ func (in *input) order(v Expr) {
for _, x := range v.List {
in.order(x)
}
in.order(&v.End)
if !v.NoBrackets {
in.order(&v.End)
}
case *UnaryExpr:
in.order(v.X)
case *BinaryExpr:
in.order(v.X)
in.order(v.Y)
case *AssignExpr:
in.order(v.LHS)
in.order(v.RHS)
case *ConditionalExpr:
in.order(v.Then)
in.order(v.Test)
@@ -777,35 +754,39 @@ func (in *input) order(v Expr) {
in.order(v.X)
in.order(v.Y)
case *LambdaExpr:
for _, name := range v.Var {
in.order(name)
for _, param := range v.Params {
in.order(param)
}
in.order(v.Expr)
case *ReturnExpr:
if v.X != nil {
in.order(v.X)
for _, expr := range v.Body {
in.order(expr)
}
case *FuncDef:
for _, x := range v.Args {
case *ReturnStmt:
if v.Result != nil {
in.order(v.Result)
}
case *DefStmt:
for _, x := range v.Params {
in.order(x)
}
for _, x := range v.Body.Statements {
for _, x := range v.Body {
in.order(x)
}
case *ForLoop:
for _, x := range v.LoopVars {
case *ForStmt:
in.order(v.Vars)
in.order(v.X)
for _, x := range v.Body {
in.order(x)
}
in.order(v.Iterable)
for _, x := range v.Body.Statements {
in.order(x)
case *IfStmt:
in.order(v.Cond)
for _, s := range v.True {
in.order(s)
}
case *IfElse:
for _, condition := range v.Conditions {
in.order(condition.If)
for _, x := range condition.Then.Statements {
in.order(x)
}
if len(v.False) > 0 {
in.order(&v.ElsePos)
}
for _, s := range v.False {
in.order(s)
}
}
if v != nil {
@@ -817,29 +798,19 @@ func (in *input) order(v Expr) {
func (in *input) assignComments() {
// Generate preorder and postorder lists.
in.order(in.file)
in.assignSuffixComments()
in.assignLineComments()
}
// Assign line comments to syntax immediately following.
line := in.lineComments
for _, x := range in.pre {
start, _ := x.Span()
xcom := x.Comment()
for len(line) > 0 && start.Byte >= line[0].Start.Byte {
xcom.Before = append(xcom.Before, line[0])
line = line[1:]
}
}
// Remaining line comments go at end of file.
in.file.After = append(in.file.After, line...)
func (in *input) assignSuffixComments() {
// Assign suffix comments to syntax immediately before.
suffix := in.suffixComments
for i := len(in.post) - 1; i >= 0; i-- {
x := in.post[i]
// Do not assign suffix comments to file
// Do not assign suffix comments to file or to block statements
switch x.(type) {
case *File:
case *File, *DefStmt, *IfStmt, *ForStmt, *CommentBlock:
continue
}
@@ -862,6 +833,27 @@ func (in *input) assignComments() {
in.file.Before = append(in.file.Before, suffix...)
}
func (in *input) assignLineComments() {
// Assign line comments to syntax immediately following.
line := in.lineComments
for _, x := range in.pre {
start, _ := x.Span()
xcom := x.Comment()
for len(line) > 0 && start.Byte >= line[0].Start.Byte {
xcom.Before = append(xcom.Before, line[0])
line = line[1:]
}
// Line comments can be sorted in a wrong order because they get assigned from different
// parts of the lexer and the parser. Restore the original order.
sort.SliceStable(xcom.Before, func(i, j int) bool {
return xcom.Before[i].Start.Byte < xcom.Before[j].Start.Byte
})
}
// Remaining line comments go at end of file.
in.file.After = append(in.file.After, line...)
}
// reverseComments reverses the []Comment list.
func reverseComments(list []Comment) {
for i, j := 0, len(list)-1; i < j; i, j = i+1, j-1 {

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -23,19 +23,27 @@ import (
"strings"
)
const nestedIndentation = 2 // Indentation of nested blocks
const listIndentation = 4 // Indentation of multiline expressions
const (
nestedIndentation = 4 // Indentation of nested blocks
listIndentation = 4 // Indentation of multiline expressions
defIndentation = 8 // Indentation of multiline function definitions
)
// Format returns the formatted form of the given BUILD file.
// Format returns the formatted form of the given BUILD or bzl file.
func Format(f *File) []byte {
pr := &printer{}
pr := &printer{fileType: f.Type}
pr.file(f)
return pr.Bytes()
}
// FormatString returns the string form of the given expression.
func FormatString(x Expr) string {
pr := &printer{}
fileType := TypeBuild // for compatibility
if file, ok := x.(*File); ok {
fileType = file.Type
}
pr := &printer{fileType: fileType}
switch x := x.(type) {
case *File:
pr.file(x)
@@ -47,10 +55,13 @@ func FormatString(x Expr) string {
// A printer collects the state during printing of a file or expression.
type printer struct {
fileType FileType // different rules can be applied to different file types.
bytes.Buffer // output buffer
comment []Comment // pending end-of-line comments
margin int // left margin (indent), a number of spaces
depth int // nesting depth inside ( ) [ ] { }
level int // nesting level of def-, if-else- and for-blocks
needsNewLine bool // true if the next statement needs a new line before it
}
// printf prints to the buffer.
@@ -74,6 +85,7 @@ func (p *printer) indent() int {
// To break a line inside an expression that might not be enclosed
// in brackets of some kind, use breakline instead.
func (p *printer) newline() {
p.needsNewLine = false
if len(p.comment) > 0 {
p.printf(" ")
for i, com := range p.comment {
@@ -90,6 +102,28 @@ func (p *printer) newline() {
p.printf("\n%*s", p.margin, "")
}
// softNewline postpones a call to newline to the next call of p.newlineIfNeeded()
// If softNewline is called several times, just one newline is printed.
// Usecase: if there are several nested blocks ending at the same time, for instance
//
// if True:
// for a in b:
// pass
// foo()
//
// the last statement (`pass`) doesn't end with a newline, each block ends with a lazy newline
// which actually gets printed only once when right before the next statement (`foo()`) is printed.
func (p *printer) softNewline() {
p.needsNewLine = true
}
// newlineIfNeeded calls newline if softNewline() has previously been called
func (p *printer) newlineIfNeeded() {
if p.needsNewLine == true {
p.newline()
}
}
// breakline breaks the current line, inserting a continuation \ if needed.
// If no continuation \ is needed, breakline flushes end-of-line comments.
func (p *printer) breakline() {
@@ -128,40 +162,59 @@ func (p *printer) file(f *File) {
p.newline()
}
// If the last expression is in an indented code block there can be spaces in the last line.
p.trim()
p.newlineIfNeeded()
}
func (p *printer) statements(stmts []Expr) {
func (p *printer) nestedStatements(stmts []Expr) {
p.margin += nestedIndentation
p.level++
p.newline()
p.statements(stmts)
p.margin -= nestedIndentation
p.level--
}
func (p *printer) statements(rawStmts []Expr) {
// rawStmts may contain nils if a refactoring tool replaces an actual statement with nil.
// It means the statements don't exist anymore, just ignore them.
stmts := []Expr{}
for _, stmt := range rawStmts {
if stmt != nil {
stmts = append(stmts, stmt)
}
}
for i, stmt := range stmts {
switch stmt := stmt.(type) {
case *CommentBlock:
// comments already handled
case *PythonBlock:
for _, com := range stmt.Before {
p.printf("%s", strings.TrimSpace(com.Token))
p.newline()
}
p.printf("%s", stmt.Token)
p.newline()
default:
p.expr(stmt, precLow)
}
// Print an empty line break after the expression unless it's a code block.
// For a code block, the line break is generated by its last statement.
if !isCodeBlock(stmt) {
p.newline()
}
// A CommentBlock is an empty statement without a body,
// it doesn't need an line break after the body
if _, ok := stmt.(*CommentBlock); !ok {
p.softNewline()
}
for _, com := range stmt.Comment().After {
p.newlineIfNeeded()
p.printf("%s", strings.TrimSpace(com.Token))
p.softNewline()
}
// Print an empty line break after the statement unless it's the last statement in the sequence.
// In that case a line break should be printed when the block or the file ends.
if i < len(stmts)-1 {
p.newline()
}
if i+1 < len(stmts) && !compactStmt(stmt, stmts[i+1], p.margin == 0) {
if i+1 < len(stmts) && !p.compactStmt(stmt, stmts[i+1]) {
p.newline()
}
}
@@ -171,43 +224,58 @@ func (p *printer) statements(stmts []Expr) {
// should be printed without an intervening blank line.
// We omit the blank line when both are subinclude statements
// and the second one has no leading comments.
func compactStmt(s1, s2 Expr, isTopLevel bool) bool {
func (p *printer) compactStmt(s1, s2 Expr) bool {
if len(s2.Comment().Before) > 0 {
return false
}
if isTopLevel {
return isCall(s1, "load") && isCall(s2, "load")
} else if isLoad(s1) && isLoad(s2) {
// Load statements should be compact
return true
} else if isLoad(s1) || isLoad(s2) {
// Load statements should be separated from anything else
return false
} else if isCommentBlock(s1) || isCommentBlock(s2) {
// Standalone comment blocks shouldn't be attached to other statements
return false
} else if (p.fileType == TypeBuild || p.fileType == TypeWorkspace) && p.level == 0 {
// Top-level statements in a BUILD or WORKSPACE file
return false
} else if isFunctionDefinition(s1) || isFunctionDefinition(s2) {
// On of the statements is a function definition
return false
} else {
return !(isCodeBlock(s1) || isCodeBlock(s2))
// Depend on how the statements have been printed in the original file
_, end := s1.Span()
start, _ := s2.Span()
return start.Line-end.Line <= 1
}
}
// isCall reports whether x is a call to a function with the given name.
func isCall(x Expr, name string) bool {
c, ok := x.(*CallExpr)
if !ok {
return false
}
nam, ok := c.X.(*LiteralExpr)
if !ok {
return false
}
return nam.Token == name
// isLoad reports whether x is a load statement.
func isLoad(x Expr) bool {
_, ok := x.(*LoadStmt)
return ok
}
// isCodeBlock checks if the statement is a code block (def, if, for, etc.)
func isCodeBlock(x Expr) bool {
switch x.(type) {
case *FuncDef:
return true
case *ForLoop:
return true
case *IfElse:
return true
default:
// isCommentBlock reports whether x is a comment block node.
func isCommentBlock(x Expr) bool {
_, ok := x.(*CommentBlock)
return ok
}
// isFunctionDefinition checks if the statement is a def code block
func isFunctionDefinition(x Expr) bool {
_, ok := x.(*DefStmt)
return ok
}
// isDifferentLines reports whether two positions belong to different lines.
// If one of the positions is null (Line == 0), it's not a real position but probably an indicator
// of manually inserted node. Return false in this case
func isDifferentLines(p1, p2 *Position) bool {
if p1.Line == 0 || p2.Line == 0 {
return false
}
return p1.Line != p2.Line
}
// Expression formatting.
@@ -236,42 +304,44 @@ func isCodeBlock(x Expr) bool {
const (
precLow = iota
precAssign
precComma
precColon
precIn
precIfElse
precOr
precAnd
precCmp
precBitwiseOr
precBitwiseXor
precBitwiseAnd
precBitwiseShift
precAdd
precMultiply
precSuffix
precUnary
precConcat
precSuffix
)
// opPrec gives the precedence for operators found in a BinaryExpr.
var opPrec = map[string]int{
"=": precAssign,
"+=": precAssign,
"-=": precAssign,
"*=": precAssign,
"/=": precAssign,
"//=": precAssign,
"%=": precAssign,
"or": precOr,
"and": precAnd,
"<": precCmp,
">": precCmp,
"==": precCmp,
"!=": precCmp,
"<=": precCmp,
">=": precCmp,
"+": precAdd,
"-": precAdd,
"*": precMultiply,
"/": precMultiply,
"//": precMultiply,
"%": precMultiply,
"or": precOr,
"and": precAnd,
"in": precCmp,
"not in": precCmp,
"<": precCmp,
">": precCmp,
"==": precCmp,
"!=": precCmp,
"<=": precCmp,
">=": precCmp,
"+": precAdd,
"-": precAdd,
"*": precMultiply,
"/": precMultiply,
"//": precMultiply,
"%": precMultiply,
"|": precBitwiseOr,
"&": precBitwiseAnd,
"^": precBitwiseXor,
"<<": precBitwiseShift,
">>": precBitwiseShift,
}
// expr prints the expression v to the print buffer.
@@ -291,6 +361,8 @@ func (p *printer) expr(v Expr, outerPrec int) {
// TODO(bazel-team): Check whether it is valid to emit comments right now,
// and if not, insert them earlier in the output instead, at the most
// recent \n not following a \ line.
p.newlineIfNeeded()
if before := v.Comment().Before; len(before) > 0 {
// Want to print a line comment.
// Line comments must be at the current margin.
@@ -330,14 +402,19 @@ func (p *printer) expr(v Expr, outerPrec int) {
case *LiteralExpr:
p.printf("%s", v.Token)
case *Ident:
p.printf("%s", v.Name)
case *BranchStmt:
p.printf("%s", v.Token)
case *StringExpr:
// If the Token is a correct quoting of Value, use it.
// This preserves the specific escaping choices that
// BUILD authors have made, and it also works around
// b/7272572.
if strings.HasPrefix(v.Token, `"`) {
s, triple, err := unquote(v.Token)
if s == v.Value && triple == v.TripleQuote && err == nil {
// If the Token is a correct quoting of Value and has double quotes, use it,
// also use it if it has single quotes and the value itself contains a double quote symbol.
// This preserves the specific escaping choices that BUILD authors have made.
s, triple, err := Unquote(v.Token)
if s == v.Value && triple == v.TripleQuote && err == nil {
if strings.HasPrefix(v.Token, `"`) || strings.ContainsRune(v.Value, '"') {
p.printf("%s", v.Token)
break
}
@@ -348,7 +425,16 @@ func (p *printer) expr(v Expr, outerPrec int) {
case *DotExpr:
addParen(precSuffix)
p.expr(v.X, precSuffix)
_, xEnd := v.X.Span()
isMultiline := isDifferentLines(&v.NamePos, &xEnd)
if isMultiline {
p.margin += listIndentation
p.breakline()
}
p.printf(".%s", v.Name)
if isMultiline {
p.margin -= listIndentation
}
case *IndexExpr:
addParen(precSuffix)
@@ -388,19 +474,23 @@ func (p *printer) expr(v Expr, outerPrec int) {
} else {
p.printf("%s", v.Op)
}
p.expr(v.X, precUnary)
// Use the next precedence level (precSuffix), so that nested unary expressions are parenthesized,
// for example: `not (-(+(~foo)))` instead of `not -+~foo`
if v.X != nil {
p.expr(v.X, precSuffix)
}
case *LambdaExpr:
addParen(precColon)
p.printf("lambda ")
for i, name := range v.Var {
for i, param := range v.Params {
if i > 0 {
p.printf(", ")
}
p.expr(name, precLow)
p.expr(param, precLow)
}
p.printf(": ")
p.expr(v.Expr, precColon)
p.expr(v.Body[0], precLow) // lambdas should have exactly one statement
case *BinaryExpr:
// Precedence: use the precedence of the operator.
@@ -423,9 +513,6 @@ func (p *printer) expr(v Expr, outerPrec int) {
m := p.margin
if v.LineBreak {
p.margin = p.indent()
if v.Op == "=" {
p.margin += listIndentation
}
}
p.expr(v.X, prec)
@@ -438,95 +525,165 @@ func (p *printer) expr(v Expr, outerPrec int) {
p.expr(v.Y, prec+1)
p.margin = m
case *AssignExpr:
addParen(precAssign)
m := p.margin
if v.LineBreak {
p.margin = p.indent() + listIndentation
}
p.expr(v.LHS, precAssign)
p.printf(" %s", v.Op)
if v.LineBreak {
p.breakline()
} else {
p.printf(" ")
}
p.expr(v.RHS, precAssign+1)
p.margin = m
case *ParenExpr:
p.seq("()", []Expr{v.X}, &v.End, modeParen, false, v.ForceMultiLine)
p.seq("()", &v.Start, &[]Expr{v.X}, &v.End, modeParen, false, v.ForceMultiLine)
case *CallExpr:
addParen(precSuffix)
p.expr(v.X, precSuffix)
p.seq("()", v.List, &v.End, modeCall, v.ForceCompact, v.ForceMultiLine)
p.seq("()", &v.ListStart, &v.List, &v.End, modeCall, v.ForceCompact, v.ForceMultiLine)
case *LoadStmt:
addParen(precSuffix)
p.printf("load")
args := []Expr{v.Module}
for i := range v.From {
from := v.From[i]
to := v.To[i]
var arg Expr
if from.Name == to.Name {
// Suffix comments are attached to the `to` token,
// Before comments are attached to the `from` token,
// they need to be combined.
arg = from.asString()
arg.Comment().Before = to.Comment().Before
} else {
arg = &AssignExpr{
LHS: to,
Op: "=",
RHS: from.asString(),
}
}
args = append(args, arg)
}
p.seq("()", &v.Load, &args, &v.Rparen, modeLoad, v.ForceCompact, false)
case *ListExpr:
p.seq("[]", v.List, &v.End, modeList, false, v.ForceMultiLine)
p.seq("[]", &v.Start, &v.List, &v.End, modeList, false, v.ForceMultiLine)
case *SetExpr:
p.seq("{}", v.List, &v.End, modeList, false, v.ForceMultiLine)
p.seq("{}", &v.Start, &v.List, &v.End, modeList, false, v.ForceMultiLine)
case *TupleExpr:
p.seq("()", v.List, &v.End, modeTuple, v.ForceCompact, v.ForceMultiLine)
mode := modeTuple
if v.NoBrackets {
mode = modeSeq
}
p.seq("()", &v.Start, &v.List, &v.End, mode, v.ForceCompact, v.ForceMultiLine)
case *DictExpr:
var list []Expr
for _, x := range v.List {
list = append(list, x)
}
p.seq("{}", list, &v.End, modeDict, false, v.ForceMultiLine)
p.seq("{}", &v.Start, &list, &v.End, modeDict, false, v.ForceMultiLine)
case *ListForExpr:
case *Comprehension:
p.listFor(v)
case *ConditionalExpr:
addParen(precSuffix)
p.expr(v.Then, precSuffix)
p.expr(v.Then, precIfElse)
p.printf(" if ")
p.expr(v.Test, precSuffix)
p.expr(v.Test, precIfElse)
p.printf(" else ")
p.expr(v.Else, precSuffix)
p.expr(v.Else, precIfElse)
case *ReturnExpr:
case *ReturnStmt:
p.printf("return")
if v.X != nil {
if v.Result != nil {
p.printf(" ")
p.expr(v.X, precSuffix)
p.expr(v.Result, precLow)
}
case *FuncDef:
case *DefStmt:
p.printf("def ")
p.printf(v.Name)
p.seq("()", v.Args, &v.End, modeCall, v.ForceCompact, v.ForceMultiLine)
p.seq("()", &v.StartPos, &v.Params, nil, modeDef, v.ForceCompact, v.ForceMultiLine)
p.printf(":")
p.margin += nestedIndentation
p.newline()
p.statements(v.Body.Statements)
p.margin -= nestedIndentation
p.nestedStatements(v.Body)
case *ForLoop:
case *ForStmt:
p.printf("for ")
for i, loopVar := range v.LoopVars {
if i > 0 {
p.printf(", ")
}
p.expr(loopVar, precLow)
}
p.expr(v.Vars, precLow)
p.printf(" in ")
p.expr(v.Iterable, precLow)
p.expr(v.X, precLow)
p.printf(":")
p.margin += nestedIndentation
p.newline()
p.statements(v.Body.Statements)
p.margin -= nestedIndentation
p.nestedStatements(v.Body)
case *IfElse:
for i, block := range v.Conditions {
if i == 0 {
p.printf("if ")
} else if block.If == nil {
p.newline()
p.printf("else")
} else {
p.newline()
p.printf("elif ")
}
if block.If != nil {
p.expr(block.If, precLow)
case *IfStmt:
block := v
isFirst := true
needsEmptyLine := false
for {
p.newlineIfNeeded()
if !isFirst {
if needsEmptyLine {
p.newline()
}
p.printf("el")
}
p.printf("if ")
p.expr(block.Cond, precLow)
p.printf(":")
p.margin += nestedIndentation
p.newline()
p.statements(block.Then.Statements)
p.margin -= nestedIndentation
p.nestedStatements(block.True)
isFirst = false
_, end := block.True[len(block.True)-1].Span()
needsEmptyLine = block.ElsePos.Pos.Line-end.Line > 1
// If the else-block contains just one statement which is an IfStmt, flatten it as a part
// of if-elif chain.
// Don't do it if the "else" statement has a suffix comment or if the next "if" statement
// has a before-comment.
if len(block.False) != 1 {
break
}
next, ok := block.False[0].(*IfStmt)
if !ok {
break
}
if len(block.ElsePos.Comment().Suffix) == 0 && len(next.Comment().Before) == 0 {
block = next
continue
}
break
}
if len(block.False) > 0 {
p.newlineIfNeeded()
if needsEmptyLine {
p.newline()
}
p.printf("else:")
p.comment = append(p.comment, block.ElsePos.Comment().Suffix...)
p.nestedStatements(block.False)
}
case *ForClause:
p.printf("for ")
p.expr(v.Vars, precLow)
p.printf(" in ")
p.expr(v.X, precLow)
case *IfClause:
p.printf("if ")
p.expr(v.Cond, precLow)
}
// Add closing parenthesis if needed.
@@ -553,87 +710,159 @@ const (
modeParen // (x)
modeDict // {x:y}
modeSeq // x, y
modeDef // def f(x, y)
modeLoad // load(a, b, c)
)
// useCompactMode reports whether a sequence should be formatted in a compact mode
func (p *printer) useCompactMode(start *Position, list *[]Expr, end *End, mode seqMode, forceCompact, forceMultiLine bool) bool {
// If there are line comments, use multiline
// so we can print the comments before the closing bracket.
for _, x := range *list {
if len(x.Comment().Before) > 0 || (len(x.Comment().Suffix) > 0 && mode != modeDef) {
return false
}
}
if end != nil && len(end.Before) > 0 {
return false
}
// Implicit tuples are always compact
if mode == modeSeq {
return true
}
// In the Default and .bzl printing modes try to keep the original printing style.
// Non-top-level statements and lists of arguments of a function definition
// should also keep the original style regardless of the mode.
if (p.level != 0 || p.fileType == TypeDefault || p.fileType == TypeBzl || mode == modeDef) && mode != modeLoad {
// If every element (including the brackets) ends on the same line where the next element starts,
// use the compact mode, otherwise use multiline mode.
// If an node's line number is 0, it means it doesn't appear in the original file,
// its position shouldn't be taken into account. Unless a sequence is new,
// then use multiline mode if ForceMultiLine mode was set.
previousEnd := start
isNewSeq := start.Line == 0
for _, x := range *list {
start, end := x.Span()
isNewSeq = isNewSeq && start.Line == 0
if isDifferentLines(&start, previousEnd) {
return false
}
if end.Line != 0 {
previousEnd = &end
}
}
if end != nil {
isNewSeq = isNewSeq && end.Pos.Line == 0
if isDifferentLines(previousEnd, &end.Pos) {
return false
}
}
if !isNewSeq {
return true
}
// Use the forceMultiline value for new sequences.
return !forceMultiLine
}
// In Build mode, use the forceMultiline and forceCompact values
if forceMultiLine {
return false
}
if forceCompact {
return true
}
// If neither of the flags are set, use compact mode only for empty or 1-element sequences
return len(*list) <= 1
}
// seq formats a list of values inside a given bracket pair (brack = "()", "[]", "{}").
// The end node holds any trailing comments to be printed just before the
// closing bracket.
// The mode parameter specifies the sequence mode (see above).
// If multiLine is true, seq avoids the compact form even
// for 0- and 1-element sequences.
func (p *printer) seq(brack string, list []Expr, end *End, mode seqMode, forceCompact, forceMultiLine bool) {
p.printf("%s", brack[:1])
func (p *printer) seq(brack string, start *Position, list *[]Expr, end *End, mode seqMode, forceCompact, forceMultiLine bool) {
if mode != modeSeq {
p.printf("%s", brack[:1])
}
p.depth++
// If there are line comments, force multiline
// so we can print the comments before the closing bracket.
for _, x := range list {
if len(x.Comment().Before) > 0 {
forceMultiLine = true
defer func() {
p.depth--
if mode != modeSeq {
p.printf("%s", brack[1:])
}
}
if len(end.Before) > 0 {
forceMultiLine = true
}
}()
// Resolve possibly ambiguous call arguments explicitly
// instead of depending on implicit resolution in logic below.
if forceMultiLine {
forceCompact = false
}
switch {
case len(list) == 0 && !forceMultiLine:
// Compact form: print nothing.
case len(list) == 1 && !forceMultiLine:
// Compact form.
p.expr(list[0], precLow)
// Tuple must end with comma, to mark it as a tuple.
if mode == modeTuple {
p.printf(",")
}
case forceCompact:
// Compact form but multiple elements.
for i, x := range list {
if p.useCompactMode(start, list, end, mode, forceCompact, forceMultiLine) {
for i, x := range *list {
if i > 0 {
p.printf(", ")
}
p.expr(x, precLow)
}
default:
// Multi-line form.
p.margin += listIndentation
for i, x := range list {
// If we are about to break the line before the first
// element and there are trailing end-of-line comments
// waiting to be printed, delay them and print them as
// whole-line comments preceding that element.
// Do this by printing a newline ourselves and positioning
// so that the end-of-line comment, with the two spaces added,
// will line up with the current margin.
if i == 0 && len(p.comment) > 0 {
p.printf("\n%*s", p.margin-2, "")
}
p.newline()
p.expr(x, precLow)
if mode != modeParen || i+1 < len(list) {
p.printf(",")
}
// Single-element tuple must end with comma, to mark it as a tuple.
if len(*list) == 1 && mode == modeTuple {
p.printf(",")
}
// Final comments.
return
}
// Multi-line form.
indentation := listIndentation
if mode == modeDef {
indentation = defIndentation
}
p.margin += indentation
for i, x := range *list {
// If we are about to break the line before the first
// element and there are trailing end-of-line comments
// waiting to be printed, delay them and print them as
// whole-line comments preceding that element.
// Do this by printing a newline ourselves and positioning
// so that the end-of-line comment, with the two spaces added,
// will line up with the current margin.
if i == 0 && len(p.comment) > 0 {
p.printf("\n%*s", p.margin-2, "")
}
p.newline()
p.expr(x, precLow)
if i+1 < len(*list) || needsTrailingComma(mode, x) {
p.printf(",")
}
}
// Final comments.
if end != nil {
for _, com := range end.Before {
p.newline()
p.printf("%s", strings.TrimSpace(com.Token))
}
p.margin -= listIndentation
}
p.margin -= indentation
// in modeDef print the closing bracket on the same line
if mode != modeDef {
p.newline()
}
p.depth--
p.printf("%s", brack[1:])
}
func needsTrailingComma(mode seqMode, v Expr) bool {
switch mode {
case modeDef:
return false
case modeParen:
return false
case modeCall:
// *args and **kwargs in fn calls
switch v := v.(type) {
case *UnaryExpr:
if v.Op == "*" || v.Op == "**" {
return false
}
}
}
return true
}
// listFor formats a ListForExpr (list comprehension).
@@ -647,7 +876,7 @@ func (p *printer) seq(brack string, list []Expr, end *End, mode seqMode, forceCo
// if c
// ]
//
func (p *printer) listFor(v *ListForExpr) {
func (p *printer) listFor(v *Comprehension) {
multiLine := v.ForceMultiLine || len(v.End.Before) > 0
// space breaks the line in multiline mode
@@ -660,41 +889,23 @@ func (p *printer) listFor(v *ListForExpr) {
}
}
if v.Brack != "" {
p.depth++
p.printf("%s", v.Brack[:1])
open, close := "[", "]"
if v.Curly {
open, close = "{", "}"
}
p.depth++
p.printf("%s", open)
if multiLine {
if v.Brack != "" {
p.margin += listIndentation
}
p.margin += listIndentation
p.newline()
}
p.expr(v.X, precLow)
p.expr(v.Body, precLow)
for _, c := range v.For {
for _, c := range v.Clauses {
space()
p.printf("for ")
for i, name := range c.For.Var {
if i > 0 {
p.printf(", ")
}
p.expr(name, precLow)
}
p.printf(" in ")
p.expr(c.For.Expr, precLow)
p.comment = append(p.comment, c.For.Comment().Suffix...)
for _, i := range c.Ifs {
space()
p.printf("if ")
p.expr(i.Cond, precLow)
p.comment = append(p.comment, i.Comment().Suffix...)
}
p.comment = append(p.comment, c.Comment().Suffix...)
p.expr(c, precLow)
}
if multiLine {
@@ -702,16 +913,12 @@ func (p *printer) listFor(v *ListForExpr) {
p.newline()
p.printf("%s", strings.TrimSpace(com.Token))
}
if v.Brack != "" {
p.margin -= listIndentation
}
p.margin -= listIndentation
p.newline()
}
if v.Brack != "" {
p.printf("%s", v.Brack[1:])
p.depth--
}
p.printf("%s", close)
p.depth--
}
func (p *printer) isTopLevel() bool {

View File

@@ -59,10 +59,10 @@ var esc = [256]byte{
// being used as shell arguments containing regular expressions.
const notEsc = " !#$%&()*+,-./:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ{|}~"
// unquote unquotes the quoted string, returning the actual
// Unquote unquotes the quoted string, returning the actual
// string value, whether the original was triple-quoted, and
// an error describing invalid input.
func unquote(quoted string) (s string, triple bool, err error) {
func Unquote(quoted string) (s string, triple bool, err error) {
// Check for raw prefix: means don't interpret the inner \.
raw := false
if strings.HasPrefix(quoted, "r") {

View File

@@ -18,12 +18,12 @@ distributed under the License is distributed on an "AS IS" BASIS,
package build
import (
"github.com/bazelbuild/buildtools/tables"
"path"
"path/filepath"
"regexp"
"sort"
"strings"
"github.com/bazelbuild/buildtools/tables"
)
// For debugging: flag to disable certain rewrites.
@@ -62,55 +62,78 @@ func Rewrite(f *File, info *RewriteInfo) {
for _, r := range rewrites {
if !disabled(r.name) {
r.fn(f, info)
if f.Type&r.scope != 0 {
r.fn(f, info)
}
}
}
}
// RewriteInfo collects information about what Rewrite did.
type RewriteInfo struct {
EditLabel int // number of label strings edited
NameCall int // number of calls with argument names added
SortCall int // number of call argument lists sorted
SortStringList int // number of string lists sorted
UnsafeSort int // number of unsafe string lists sorted
Log []string // log entries - may change
EditLabel int // number of label strings edited
NameCall int // number of calls with argument names added
SortCall int // number of call argument lists sorted
SortStringList int // number of string lists sorted
UnsafeSort int // number of unsafe string lists sorted
SortLoad int // number of load argument lists sorted
FormatDocstrings int // number of reindented docstrings
ReorderArguments int // number of reordered function call arguments
EditOctal int // number of edited octals
Log []string // log entries - may change
}
func (info *RewriteInfo) String() string {
s := ""
if info.EditLabel > 0 {
s += " label"
// Stats returns a map with statistics about applied rewrites
func (info *RewriteInfo) Stats() map[string]int {
return map[string]int{
"label": info.EditLabel,
"callname": info.NameCall,
"callsort": info.SortCall,
"listsort": info.SortStringList,
"unsafesort": info.UnsafeSort,
"sortload": info.SortLoad,
"formatdocstrings": info.FormatDocstrings,
"reorderarguments": info.ReorderArguments,
"editoctal": info.EditOctal,
}
if info.NameCall > 0 {
s += " callname"
}
if info.SortCall > 0 {
s += " callsort"
}
if info.SortStringList > 0 {
s += " listsort"
}
if info.UnsafeSort > 0 {
s += " unsafesort"
}
if s != "" {
s = s[1:]
}
return s
}
// Each rewrite function can be either applied for BUILD files, other files (such as .bzl),
// or all files.
const (
scopeDefault = TypeDefault | TypeBzl // .bzl and generic Starlark files
scopeBuild = TypeBuild | TypeWorkspace // BUILD and WORKSPACE files
scopeBoth = scopeDefault | scopeBuild
)
// rewrites is the list of all Buildifier rewrites, in the order in which they are applied.
// The order here matters: for example, label canonicalization must happen
// before sorting lists of strings.
var rewrites = []struct {
name string
fn func(*File, *RewriteInfo)
name string
fn func(*File, *RewriteInfo)
scope FileType
}{
{"callsort", sortCallArgs},
{"label", fixLabels},
{"listsort", sortStringLists},
{"multiplus", fixMultilinePlus},
{"callsort", sortCallArgs, scopeBuild},
{"label", fixLabels, scopeBuild},
{"listsort", sortStringLists, scopeBoth},
{"multiplus", fixMultilinePlus, scopeBuild},
{"loadsort", sortAllLoadArgs, scopeBoth},
{"formatdocstrings", formatDocstrings, scopeBoth},
{"reorderarguments", reorderArguments, scopeBoth},
{"editoctal", editOctals, scopeBoth},
}
// DisableLoadSortForBuildFiles disables the loadsort transformation for BUILD files.
// This is a temporary function for backward compatibility, can be called if there's plenty of
// already formatted BUILD files that shouldn't be changed by the transformation.
func DisableLoadSortForBuildFiles() {
for i := range rewrites {
if rewrites[i].name == "loadsort" {
rewrites[i].scope = scopeDefault
break
}
}
}
// leaveAlone reports whether any of the nodes on the stack are marked
@@ -212,14 +235,19 @@ func fixLabels(f *File, info *RewriteInfo) {
editPerformed := false
if tables.StripLabelLeadingSlashes && strings.HasPrefix(str.Value, "//") {
if path.Dir(f.Path) == "." || !strings.HasPrefix(str.Value, "//:") {
if filepath.Dir(f.Path) == "." || !strings.HasPrefix(str.Value, "//:") {
editPerformed = true
str.Value = str.Value[2:]
}
}
if tables.ShortenAbsoluteLabelsToRelative {
thisPackage := labelPrefix + path.Dir(f.Path)
thisPackage := labelPrefix + filepath.Dir(f.Path)
// filepath.Dir on Windows uses backslashes as separators, while labels always have slashes.
if filepath.Separator != '/' {
thisPackage = strings.Replace(thisPackage, string(filepath.Separator), "/", -1)
}
if str.Value == thisPackage {
editPerformed = true
str.Value = ":" + path.Base(str.Value)
@@ -255,18 +283,18 @@ func fixLabels(f *File, info *RewriteInfo) {
if leaveAlone1(v.List[i]) {
continue
}
as, ok := v.List[i].(*BinaryExpr)
if !ok || as.Op != "=" {
as, ok := v.List[i].(*AssignExpr)
if !ok {
continue
}
key, ok := as.X.(*LiteralExpr)
if !ok || !tables.IsLabelArg[key.Token] || tables.LabelBlacklist[callName(v)+"."+key.Token] {
key, ok := as.LHS.(*Ident)
if !ok || !tables.IsLabelArg[key.Name] || tables.LabelBlacklist[callName(v)+"."+key.Name] {
continue
}
if leaveAlone1(as.Y) {
if leaveAlone1(as.RHS) {
continue
}
if list, ok := as.Y.(*ListExpr); ok {
if list, ok := as.RHS.(*ListExpr); ok {
for i := range list.List {
if leaveAlone1(list.List[i]) {
continue
@@ -275,7 +303,7 @@ func fixLabels(f *File, info *RewriteInfo) {
shortenLabel(list.List[i])
}
}
if set, ok := as.Y.(*SetExpr); ok {
if set, ok := as.RHS.(*SetExpr); ok {
for i := range set.List {
if leaveAlone1(set.List[i]) {
continue
@@ -284,8 +312,8 @@ func fixLabels(f *File, info *RewriteInfo) {
shortenLabel(set.List[i])
}
} else {
joinLabel(&as.Y)
shortenLabel(as.Y)
joinLabel(&as.RHS)
shortenLabel(as.RHS)
}
}
}
@@ -295,11 +323,11 @@ func fixLabels(f *File, info *RewriteInfo) {
// callName returns the name of the rule being called by call.
// If the call is not to a literal rule name, callName returns "".
func callName(call *CallExpr) string {
rule, ok := call.X.(*LiteralExpr)
rule, ok := call.X.(*Ident)
if !ok {
return ""
}
return rule.Token
return rule.Name
}
// sortCallArgs sorts lists of named arguments to a call.
@@ -368,9 +396,9 @@ func ruleNamePriority(rule, arg string) int {
// If x is of the form key=value, argName returns the string key.
// Otherwise argName returns "".
func argName(x Expr) string {
if as, ok := x.(*BinaryExpr); ok && as.Op == "=" {
if id, ok := as.X.(*LiteralExpr); ok {
return id.Token
if as, ok := x.(*AssignExpr); ok {
if id, ok := as.LHS.(*Ident); ok {
return id.Name
}
}
return ""
@@ -416,31 +444,31 @@ func sortStringLists(f *File, info *RewriteInfo) {
if leaveAlone1(arg) {
continue
}
as, ok := arg.(*BinaryExpr)
if !ok || as.Op != "=" || leaveAlone1(as) || doNotSort(as) {
as, ok := arg.(*AssignExpr)
if !ok || leaveAlone1(as) || doNotSort(as) {
continue
}
key, ok := as.X.(*LiteralExpr)
key, ok := as.LHS.(*Ident)
if !ok {
continue
}
context := rule + "." + key.Token
if !tables.IsSortableListArg[key.Token] || tables.SortableBlacklist[context] {
context := rule + "." + key.Name
if !tables.IsSortableListArg[key.Name] || tables.SortableBlacklist[context] || f.Type == TypeDefault || f.Type == TypeBzl {
continue
}
if disabled("unsafesort") && !tables.SortableWhitelist[context] && !allowedSort(context) {
continue
}
sortStringList(as.Y, info, context)
sortStringList(as.RHS, info, context)
}
case *BinaryExpr:
case *AssignExpr:
if disabled("unsafesort") {
return
}
// "keep sorted" comment on x = list forces sorting of list.
as := v
if as.Op == "=" && keepSorted(as) {
sortStringList(as.Y, info, "?")
if keepSorted(as) {
sortStringList(as.RHS, info, "?")
}
case *KeyValueExpr:
if disabled("unsafesort") {
@@ -455,7 +483,7 @@ func sortStringLists(f *File, info *RewriteInfo) {
return
}
// "keep sorted" comment above first list element also forces sorting of list.
if len(v.List) > 0 && keepSorted(v.List[0]) {
if len(v.List) > 0 && (keepSorted(v) || keepSorted(v.List[0])) {
sortStringList(v, info, "?")
}
}
@@ -476,7 +504,7 @@ func sortStringList(x Expr, info *RewriteInfo, context string) {
return
}
forceSort := keepSorted(list.List[0])
forceSort := keepSorted(list) || keepSorted(list.List[0])
// TODO(bazel-team): Decide how to recognize lists that cannot
// be sorted. Avoiding all lists with comments avoids sorting
@@ -569,11 +597,11 @@ func callArgName(stk []Expr) string {
if !ok {
return ""
}
rule, ok := call.X.(*LiteralExpr)
rule, ok := call.X.(*Ident)
if !ok {
return ""
}
return rule.Token + "." + arg
return rule.Name + "." + arg
}
// A stringSortKey records information about a single string literal to be
@@ -794,6 +822,17 @@ func fixMultilinePlus(f *File, info *RewriteInfo) {
})
}
// sortAllLoadArgs sorts all load arguments in the file
func sortAllLoadArgs(f *File, info *RewriteInfo) {
Walk(f, func(v Expr, stk []Expr) {
if load, ok := v.(*LoadStmt); ok {
if SortLoadArgs(load) {
info.SortLoad++
}
}
})
}
// hasComments reports whether any comments are associated with
// the list or its elements.
func hasComments(list *ListExpr) (line, suffix bool) {
@@ -815,3 +854,149 @@ func hasComments(list *ListExpr) (line, suffix bool) {
}
return
}
// A wrapper for a LoadStmt's From and To slices for consistent sorting of their contents.
// It's assumed that the following slices have the same length. The contents are sorted by
// the `To` attribute, but all items with equal "From" and "To" parts are placed before the items
// with different parts.
type loadArgs struct {
From []*Ident
To []*Ident
modified bool
}
func (args loadArgs) Len() int {
return len(args.From)
}
func (args loadArgs) Swap(i, j int) {
args.From[i], args.From[j] = args.From[j], args.From[i]
args.To[i], args.To[j] = args.To[j], args.To[i]
args.modified = true
}
func (args loadArgs) Less(i, j int) bool {
// Arguments with equal "from" and "to" parts are prioritized
equalI := args.From[i].Name == args.To[i].Name
equalJ := args.From[j].Name == args.To[j].Name
if equalI != equalJ {
// If equalI and !equalJ, return true, otherwise false.
// Equivalently, return equalI.
return equalI
}
return args.To[i].Name < args.To[j].Name
}
// SortLoadArgs sorts a load statement arguments (lexicographically, but positional first)
func SortLoadArgs(load *LoadStmt) bool {
args := loadArgs{From: load.From, To: load.To}
sort.Sort(args)
return args.modified
}
// formatDocstrings fixes the indentation and trailing whitespace of docstrings
func formatDocstrings(f *File, info *RewriteInfo) {
Walk(f, func(v Expr, stk []Expr) {
def, ok := v.(*DefStmt)
if !ok || len(def.Body) == 0 {
return
}
docstring, ok := def.Body[0].(*StringExpr)
if !ok || !docstring.TripleQuote {
return
}
oldIndentation := docstring.Start.LineRune - 1 // LineRune starts with 1
newIndentation := nestedIndentation * len(stk)
// Operate on Token, not Value, because their line breaks can be different if a line ends with
// a backslash.
updatedToken := formatString(docstring.Token, oldIndentation, newIndentation)
if updatedToken != docstring.Token {
docstring.Token = updatedToken
// Update the value to keep it consistent with Token
docstring.Value, _, _ = Unquote(updatedToken)
info.FormatDocstrings++
}
})
}
// formatString modifies a string value of a docstring to match the new indentation level and
// to remove trailing whitespace from its lines.
func formatString(value string, oldIndentation, newIndentation int) string {
difference := newIndentation - oldIndentation
lines := strings.Split(value, "\n")
for i, line := range lines {
if i == 0 {
// The first line shouldn't be touched because it starts right after ''' or """
continue
}
if difference > 0 {
line = strings.Repeat(" ", difference) + line
} else {
for i, rune := range line {
if i == -difference || rune != ' ' {
line = line[i:]
break
}
}
}
if i != len(lines)-1 {
// Remove trailing space from the line unless it's the last line that's responsible
// for the indentation of the closing `"""`
line = strings.TrimRight(line, " ")
}
lines[i] = line
}
return strings.Join(lines, "\n")
}
// argumentType returns an integer by which funcall arguments can be sorted:
// 1 for positional, 2 for named, 3 for *args, 4 for **kwargs
func argumentType(expr Expr) int {
switch expr := expr.(type) {
case *UnaryExpr:
switch expr.Op {
case "**":
return 4
case "*":
return 3
}
case *AssignExpr:
return 2
}
return 1
}
// reorderArguments fixes the order of arguments of a function call
// (positional, named, *args, **kwargs)
func reorderArguments(f *File, info *RewriteInfo) {
Walk(f, func(expr Expr, stack []Expr) {
call, ok := expr.(*CallExpr)
if !ok {
return
}
compare := func(i, j int) bool {
return argumentType(call.List[i]) < argumentType(call.List[j])
}
if !sort.SliceIsSorted(call.List, compare) {
sort.SliceStable(call.List, compare)
info.ReorderArguments++
}
})
}
// editOctals inserts 'o' into octal numbers to make it more obvious they are octal
// 0123 -> 0o123
func editOctals(f *File, info *RewriteInfo) {
Walk(f, func(expr Expr, stack []Expr) {
l, ok := expr.(*LiteralExpr)
if !ok {
return
}
if len(l.Token) > 1 && l.Token[0] == '0' && l.Token[1] >= '0' && l.Token[1] <= '9' {
l.Token = "0o" + l.Token[1:]
info.EditOctal++
}
})
}

View File

@@ -19,8 +19,8 @@ distributed under the License is distributed on an "AS IS" BASIS,
package build
import (
"strings"
"path/filepath"
"strings"
)
// A Rule represents a single BUILD rule.
@@ -29,6 +29,11 @@ type Rule struct {
ImplicitName string // The name which should be used if the name attribute is not set. See the comment on File.implicitRuleName.
}
// NewRule is a simple constructor for Rule.
func NewRule(call *CallExpr) *Rule {
return &Rule{call, ""}
}
func (f *File) Rule(call *CallExpr) *Rule {
r := &Rule{call, ""}
if r.AttrString("name") == "" {
@@ -43,15 +48,26 @@ func (f *File) Rules(kind string) []*Rule {
var all []*Rule
for _, stmt := range f.Stmt {
call, ok := stmt.(*CallExpr)
if !ok {
continue
}
rule := f.Rule(call)
if kind != "" && rule.Kind() != kind {
continue
}
all = append(all, rule)
Walk(stmt, func(x Expr, stk []Expr) {
call, ok := x.(*CallExpr)
if !ok {
return
}
// Skip nested calls.
for _, frame := range stk {
if _, ok := frame.(*CallExpr); ok {
return
}
}
// Check if the rule kind is correct.
rule := f.Rule(call)
if kind != "" && rule.Kind() != kind {
return
}
all = append(all, rule)
})
}
return all
@@ -145,11 +161,11 @@ func (r *Rule) Kind() string {
names = append(names, x.Name)
expr = x.X
}
x, ok := expr.(*LiteralExpr)
x, ok := expr.(*Ident)
if !ok {
return ""
}
names = append(names, x.Token)
names = append(names, x.Name)
// Reverse the elements since the deepest expression contains the leading literal
for l, r := 0, len(names)-1; l < r; l, r = l+1, r-1 {
names[l], names[r] = names[r], names[l]
@@ -161,18 +177,23 @@ func (r *Rule) Kind() string {
func (r *Rule) SetKind(kind string) {
names := strings.Split(kind, ".")
var expr Expr
expr = &LiteralExpr{Token: names[0]}
expr = &Ident{Name: names[0]}
for _, name := range names[1:] {
expr = &DotExpr{X: expr, Name: name}
}
r.Call.X = expr
}
// ExplicitName returns the rule's target name if it's explicitly provided as a string value, "" otherwise.
func (r *Rule) ExplicitName() string {
return r.AttrString("name")
}
// Name returns the rule's target name.
// If the rule has no explicit target name, Name returns the implicit name if there is one, else the empty string.
func (r *Rule) Name() string {
explicitName := r.AttrString("name")
if explicitName == "" {
explicitName := r.ExplicitName()
if explicitName == "" && r.Kind() != "package" {
return r.ImplicitName
}
return explicitName
@@ -182,26 +203,25 @@ func (r *Rule) Name() string {
func (r *Rule) AttrKeys() []string {
var keys []string
for _, expr := range r.Call.List {
if binExpr, ok := expr.(*BinaryExpr); ok && binExpr.Op == "=" {
if keyExpr, ok := binExpr.X.(*LiteralExpr); ok {
keys = append(keys, keyExpr.Token)
if as, ok := expr.(*AssignExpr); ok {
if keyExpr, ok := as.LHS.(*Ident); ok {
keys = append(keys, keyExpr.Name)
}
}
}
return keys
}
// AttrDefn returns the BinaryExpr defining the rule's attribute with the given key.
// That is, the result is a *BinaryExpr with Op == "=".
// AttrDefn returns the AssignExpr defining the rule's attribute with the given key.
// If the rule has no such attribute, AttrDefn returns nil.
func (r *Rule) AttrDefn(key string) *BinaryExpr {
func (r *Rule) AttrDefn(key string) *AssignExpr {
for _, kv := range r.Call.List {
as, ok := kv.(*BinaryExpr)
if !ok || as.Op != "=" {
as, ok := kv.(*AssignExpr)
if !ok {
continue
}
k, ok := as.X.(*LiteralExpr)
if !ok || k.Token != key {
k, ok := as.LHS.(*Ident)
if !ok || k.Name != key {
continue
}
return as
@@ -217,7 +237,7 @@ func (r *Rule) Attr(key string) Expr {
if as == nil {
return nil
}
return as.Y
return as.RHS
}
// DelAttr deletes the rule's attribute with the named key.
@@ -225,17 +245,17 @@ func (r *Rule) Attr(key string) Expr {
func (r *Rule) DelAttr(key string) Expr {
list := r.Call.List
for i, kv := range list {
as, ok := kv.(*BinaryExpr)
if !ok || as.Op != "=" {
as, ok := kv.(*AssignExpr)
if !ok {
continue
}
k, ok := as.X.(*LiteralExpr)
if !ok || k.Token != key {
k, ok := as.LHS.(*Ident)
if !ok || k.Name != key {
continue
}
copy(list[i:], list[i+1:])
r.Call.List = list[:len(list)-1]
return as.Y
return as.RHS
}
return nil
}
@@ -246,15 +266,15 @@ func (r *Rule) DelAttr(key string) Expr {
func (r *Rule) SetAttr(key string, val Expr) {
as := r.AttrDefn(key)
if as != nil {
as.Y = val
as.RHS = val
return
}
r.Call.List = append(r.Call.List,
&BinaryExpr{
X: &LiteralExpr{Token: key},
Op: "=",
Y: val,
&AssignExpr{
LHS: &Ident{Name: key},
Op: "=",
RHS: val,
},
)
}
@@ -265,11 +285,14 @@ func (r *Rule) SetAttr(key string, val Expr) {
// If the rule has no such attribute or the attribute is not an identifier or number,
// AttrLiteral returns "".
func (r *Rule) AttrLiteral(key string) string {
lit, ok := r.Attr(key).(*LiteralExpr)
if !ok {
return ""
value := r.Attr(key)
if ident, ok := value.(*Ident); ok {
return ident.Name
}
return lit.Token
if literal, ok := value.(*LiteralExpr); ok {
return literal.Token
}
return ""
}
// AttrString returns the value of the rule's attribute

View File

@@ -79,19 +79,41 @@ func (c *Comments) Comment() *Comments {
return c
}
// A File represents an entire BUILD file.
// stmtsEnd returns the end position of the last non-nil statement
func stmtsEnd(stmts []Expr) Position {
for i := len(stmts) - 1; i >= 0; i-- {
if stmts[i] != nil {
_, end := stmts[i].Span()
return end
}
}
return Position{}
}
// A File represents an entire BUILD or .bzl file.
type File struct {
Path string // file path, relative to workspace directory
Pkg string // optional; the package of the file
Type FileType
Comments
Stmt []Expr
}
// DisplayPath returns the filename if it's not empty, "<stdin>" otherwise
func (f *File) DisplayPath() string {
if f.Path == "" {
return "<stdin>"
}
return f.Path
}
func (f *File) Span() (start, end Position) {
if len(f.Stmt) == 0 {
return
p := Position{Line: 1, LineRune: 1}
return p, p
}
start, _ = f.Stmt[0].Span()
_, end = f.Stmt[len(f.Stmt)-1].Span()
start = Position{}
end = stmtsEnd(f.Stmt)
return start, end
}
@@ -106,18 +128,39 @@ func (x *CommentBlock) Span() (start, end Position) {
return x.Start, x.Start
}
// A PythonBlock represents a blob of Python code, typically a def or for loop.
type PythonBlock struct {
// An Ident represents an identifier.
type Ident struct {
Comments
Start Position
Token string // raw Python code, including final newline
NamePos Position
Name string
}
func (x *PythonBlock) Span() (start, end Position) {
return x.Start, x.Start.add(x.Token)
func (x *Ident) Span() (start, end Position) {
return x.NamePos, x.NamePos.add(x.Name)
}
// A LiteralExpr represents a literal identifier or number.
// BranchStmt represents a `pass`, `break`, or `continue` statement.
type BranchStmt struct {
Comments
Token string // pass, break, continue
TokenPos Position
}
func (x *BranchStmt) Span() (start, end Position) {
return x.TokenPos, x.TokenPos.add(x.Token)
}
func (x *Ident) asString() *StringExpr {
_, end := x.Span()
return &StringExpr{
Comments: x.Comments,
Start: x.NamePos,
Value: x.Name,
End: end,
}
}
// A LiteralExpr represents a literal number.
type LiteralExpr struct {
Comments
Start Position
@@ -188,32 +231,32 @@ func (x *DotExpr) Span() (start, end Position) {
return start, x.NamePos.add(x.Name)
}
// A ListForExpr represents a list comprehension expression: [X for ... if ...].
type ListForExpr struct {
// A Comprehension represents a list comprehension expression: [X for ... if ...].
type Comprehension struct {
Comments
Curly bool // curly braces (as opposed to square brackets)
Lbrack Position
Body Expr
Clauses []Expr // = *ForClause | *IfClause
ForceMultiLine bool // split expression across multiple lines
Brack string // "", "()", or "[]"
Start Position
X Expr
For []*ForClauseWithIfClausesOpt
End
}
func (x *ListForExpr) Span() (start, end Position) {
return x.Start, x.End.Pos.add("]")
func (x *Comprehension) Span() (start, end Position) {
return x.Lbrack, x.End.Pos.add("]")
}
// A ForClause represents a for clause in a list comprehension: for Var in Expr.
type ForClause struct {
Comments
For Position
Var []Expr
Vars Expr
In Position
Expr Expr
X Expr
}
func (x *ForClause) Span() (start, end Position) {
_, end = x.Expr.Span()
_, end = x.X.Span()
return x.For, end
}
@@ -229,23 +272,6 @@ func (x *IfClause) Span() (start, end Position) {
return x.If, end
}
// A ForClauseWithIfClausesOpt represents a for clause in a list comprehension followed by optional
// if expressions: for ... in ... [if ... if ...]
type ForClauseWithIfClausesOpt struct {
Comments
For *ForClause
Ifs []*IfClause
}
func (x *ForClauseWithIfClausesOpt) Span() (start, end Position) {
start, end = x.For.Span()
if len(x.Ifs) > 0 {
_, end = x.Ifs[len(x.Ifs)-1].Span()
}
return start, end
}
// A KeyValueExpr represents a dictionary entry: Key: Value.
type KeyValueExpr struct {
Comments
@@ -264,8 +290,7 @@ func (x *KeyValueExpr) Span() (start, end Position) {
type DictExpr struct {
Comments
Start Position
List []Expr // all *KeyValueExprs
Comma Position // position of trailing comma, if any
List []Expr // all *KeyValueExprs
End
ForceMultiLine bool // force multiline form when printing
}
@@ -279,7 +304,6 @@ type ListExpr struct {
Comments
Start Position
List []Expr
Comma Position // position of trailing comma, if any
End
ForceMultiLine bool // force multiline form when printing
}
@@ -293,7 +317,6 @@ type SetExpr struct {
Comments
Start Position
List []Expr
Comma Position // position of trailing comma, if any
End
ForceMultiLine bool // force multiline form when printing
}
@@ -305,16 +328,21 @@ func (x *SetExpr) Span() (start, end Position) {
// A TupleExpr represents a tuple literal: (List)
type TupleExpr struct {
Comments
Start Position
List []Expr
Comma Position // position of trailing comma, if any
NoBrackets bool // true if a tuple has no brackets, e.g. `a, b = x`
Start Position
List []Expr
End
ForceCompact bool // force compact (non-multiline) form when printing
ForceMultiLine bool // force multiline form when printing
}
func (x *TupleExpr) Span() (start, end Position) {
return x.Start, x.End.Pos.add(")")
if !x.NoBrackets {
return x.Start, x.End.Pos.add(")")
}
start, _ = x.List[0].Span()
_, end = x.List[len(x.List)-1].Span()
return start, end
}
// A UnaryExpr represents a unary expression: Op X.
@@ -326,6 +354,9 @@ type UnaryExpr struct {
}
func (x *UnaryExpr) Span() (start, end Position) {
if x.X == nil {
return x.OpStart, x.OpStart
}
_, end = x.X.Span()
return x.OpStart, end
}
@@ -346,6 +377,22 @@ func (x *BinaryExpr) Span() (start, end Position) {
return start, end
}
// An AssignExpr represents a binary expression with `=`: LHS = RHS.
type AssignExpr struct {
Comments
LHS Expr
OpPos Position
Op string
LineBreak bool // insert line break between Op and RHS
RHS Expr
}
func (x *AssignExpr) Span() (start, end Position) {
start, _ = x.LHS.Span()
_, end = x.RHS.Span()
return start, end
}
// A ParenExpr represents a parenthesized expression: (X).
type ParenExpr struct {
Comments
@@ -374,7 +421,7 @@ type SliceExpr struct {
func (x *SliceExpr) Span() (start, end Position) {
start, _ = x.X.Span()
return start, x.End
return start, x.End.add("]")
}
// An IndexExpr represents an index expression: X[Y].
@@ -388,21 +435,30 @@ type IndexExpr struct {
func (x *IndexExpr) Span() (start, end Position) {
start, _ = x.X.Span()
return start, x.End
return start, x.End.add("]")
}
// A Function represents the common parts of LambdaExpr and DefStmt
type Function struct {
Comments
StartPos Position // position of DEF or LAMBDA token
Params []Expr
Body []Expr
}
func (x *Function) Span() (start, end Position) {
_, end = x.Body[len(x.Body)-1].Span()
return x.StartPos, end
}
// A LambdaExpr represents a lambda expression: lambda Var: Expr.
type LambdaExpr struct {
Comments
Lambda Position
Var []Expr
Colon Position
Expr Expr
Function
}
func (x *LambdaExpr) Span() (start, end Position) {
_, end = x.Expr.Span()
return x.Lambda, end
return x.Function.Span()
}
// ConditionalExpr represents the conditional: X if TEST else ELSE.
@@ -423,73 +479,93 @@ func (x *ConditionalExpr) Span() (start, end Position) {
return start, end
}
// A CodeBlock represents an indented code block.
type CodeBlock struct {
Statements []Expr
Start Position
End
}
func (x *CodeBlock) Span() (start, end Position) {
return x.Start, x.End.Pos
}
// A FuncDef represents a function definition expression: def foo(List):.
type FuncDef struct {
// A LoadStmt loads another module and binds names from it:
// load(Module, "x", y="foo").
//
// The AST is slightly unfaithful to the concrete syntax here because
// Skylark's load statement, so that it can be implemented in Python,
// binds some names (like y above) with an identifier and some (like x)
// without. For consistency we create fake identifiers for all the
// strings.
type LoadStmt struct {
Comments
Start Position // position of def
Load Position
Module *StringExpr
From []*Ident // name defined in loading module
To []*Ident // name in loaded module
Rparen End
ForceCompact bool // force compact (non-multiline) form when printing
}
func (x *LoadStmt) Span() (start, end Position) {
return x.Load, x.Rparen.Pos.add(")")
}
// A DefStmt represents a function definition expression: def foo(List):.
type DefStmt struct {
Comments
Function
Name string
ListStart Position // position of (
Args []Expr
Body CodeBlock
End // position of the end
ForceCompact bool // force compact (non-multiline) form when printing
ForceMultiLine bool // force multiline form when printing
ColonPos Position // position of the ":"
ForceCompact bool // force compact (non-multiline) form when printing the arguments
ForceMultiLine bool // force multiline form when printing the arguments
}
func (x *FuncDef) Span() (start, end Position) {
return x.Start, x.End.Pos
func (x *DefStmt) Span() (start, end Position) {
return x.Function.Span()
}
// A ReturnExpr represents a return statement: return f(x).
type ReturnExpr struct {
// HeaderSpan returns the span of the function header `def f(...):`
func (x *DefStmt) HeaderSpan() (start, end Position) {
return x.Function.StartPos, x.ColonPos
}
// A ReturnStmt represents a return statement: return f(x).
type ReturnStmt struct {
Comments
Start Position
X Expr
End Position
Return Position
Result Expr // may be nil
}
func (x *ReturnExpr) Span() (start, end Position) {
return x.Start, x.End
func (x *ReturnStmt) Span() (start, end Position) {
if x.Result == nil {
return x.Return, x.Return.add("return")
}
_, end = x.Result.Span()
return x.Return, end
}
// A ForLoop represents a for loop block: for x in range(10):.
type ForLoop struct {
// A ForStmt represents a for loop block: for x in range(10):.
type ForStmt struct {
Comments
Start Position // position of for
LoopVars []Expr
Iterable Expr
Body CodeBlock
End // position of the end
Function
For Position // position of for
Vars Expr
X Expr
Body []Expr
}
func (x *ForLoop) Span() (start, end Position) {
return x.Start, x.End.Pos
func (x *ForStmt) Span() (start, end Position) {
end = stmtsEnd(x.Body)
return x.For, end
}
// An IfElse represents an if-else blocks sequence: if x: ... elif y: ... else: ... .
type IfElse struct {
// An IfStmt represents an if-else block: if x: ... else: ... .
// `elif`s are treated as a chain of `IfStmt`s.
type IfStmt struct {
Comments
Start Position // position of if
Conditions []Condition
End // position of the end
If Position // position of if
Cond Expr
True []Expr
ElsePos End // position of else or elif
False []Expr // optional
}
type Condition struct {
If Expr
Then CodeBlock
}
func (x *IfElse) Span() (start, end Position) {
return x.Start, x.End.Pos
func (x *IfStmt) Span() (start, end Position) {
body := x.False
if body == nil {
body = x.True
}
end = stmtsEnd(body)
return x.If, end
}

View File

@@ -24,13 +24,22 @@ package build
//
func Walk(v Expr, f func(x Expr, stk []Expr)) {
var stack []Expr
walk1(&v, &stack, func(x Expr, stk []Expr) Expr {
walk1(&v, &stack, func(x *Expr, stk []Expr) Expr {
f(*x, stk)
return nil
})
}
// WalkPointers is the same as Walk but calls the callback function with pointers to nodes.
func WalkPointers(v Expr, f func(x *Expr, stk []Expr)) {
var stack []Expr
walk1(&v, &stack, func(x *Expr, stk []Expr) Expr {
f(x, stk)
return nil
})
}
// WalkAndUpdate walks the expression tree v, calling f on all subexpressions
// Edit walks the expression tree v, calling f on all subexpressions
// in a preorder traversal. If f returns a non-nil value, the tree is mutated.
// The new value replaces the old one.
//
@@ -39,97 +48,199 @@ func Walk(v Expr, f func(x Expr, stk []Expr)) {
//
func Edit(v Expr, f func(x Expr, stk []Expr) Expr) Expr {
var stack []Expr
return walk1(&v, &stack, f)
return walk1(&v, &stack, func(x *Expr, stk []Expr) Expr {
return f(*x, stk)
})
}
// walk1 is the actual implementation of Walk and WalkAndUpdate.
// It has the same signature and meaning as Walk,
// except that it maintains in *stack the current stack
// of nodes. Using a pointer to a slice here ensures that
// as the stack grows and shrinks the storage can be
// reused for the next growth.
func walk1(v *Expr, stack *[]Expr, f func(x Expr, stk []Expr) Expr) Expr {
// EditChildren is similar to Edit but doesn't visit the initial node, instead goes
// directly to its children.
func EditChildren(v Expr, f func(x Expr, stk []Expr) Expr) {
stack := []Expr{v}
WalkOnce(v, func(x *Expr) {
walk1(x, &stack, func(x *Expr, stk []Expr) Expr {
return f(*x, stk)
})
})
}
// walk1 is a helper function for Walk, WalkWithPostfix, and Edit.
func walk1(v *Expr, stack *[]Expr, f func(x *Expr, stk []Expr) Expr) Expr {
if v == nil {
return nil
}
if res := f(*v, *stack); res != nil {
if res := f(v, *stack); res != nil {
*v = res
}
*stack = append(*stack, *v)
switch v := (*v).(type) {
case *File:
for _, stmt := range v.Stmt {
walk1(&stmt, stack, f)
}
case *DotExpr:
walk1(&v.X, stack, f)
case *IndexExpr:
walk1(&v.X, stack, f)
walk1(&v.Y, stack, f)
case *KeyValueExpr:
walk1(&v.Key, stack, f)
walk1(&v.Value, stack, f)
case *SliceExpr:
walk1(&v.X, stack, f)
if v.From != nil {
walk1(&v.From, stack, f)
}
if v.To != nil {
walk1(&v.To, stack, f)
}
if v.Step != nil {
walk1(&v.Step, stack, f)
}
case *ParenExpr:
walk1(&v.X, stack, f)
case *UnaryExpr:
walk1(&v.X, stack, f)
case *BinaryExpr:
walk1(&v.X, stack, f)
walk1(&v.Y, stack, f)
case *LambdaExpr:
for i := range v.Var {
walk1(&v.Var[i], stack, f)
}
walk1(&v.Expr, stack, f)
case *CallExpr:
walk1(&v.X, stack, f)
for i := range v.List {
walk1(&v.List[i], stack, f)
}
case *ListExpr:
for i := range v.List {
walk1(&v.List[i], stack, f)
}
case *SetExpr:
for i := range v.List {
walk1(&v.List[i], stack, f)
}
case *TupleExpr:
for i := range v.List {
walk1(&v.List[i], stack, f)
}
case *DictExpr:
for i := range v.List {
walk1(&v.List[i], stack, f)
}
case *ListForExpr:
walk1(&v.X, stack, f)
for _, c := range v.For {
for j := range c.For.Var {
walk1(&c.For.Var[j], stack, f)
}
walk1(&c.For.Expr, stack, f)
for _, i := range c.Ifs {
walk1(&i.Cond, stack, f)
}
}
case *ConditionalExpr:
walk1(&v.Then, stack, f)
walk1(&v.Test, stack, f)
walk1(&v.Else, stack, f)
}
WalkOnce(*v, func(x *Expr) {
walk1(x, stack, f)
})
*stack = (*stack)[:len(*stack)-1]
return *v
}
// WalkOnce calls f on every child of v.
func WalkOnce(v Expr, f func(x *Expr)) {
switch v := v.(type) {
case *File:
for i := range v.Stmt {
f(&v.Stmt[i])
}
case *DotExpr:
f(&v.X)
case *IndexExpr:
f(&v.X)
f(&v.Y)
case *KeyValueExpr:
f(&v.Key)
f(&v.Value)
case *SliceExpr:
f(&v.X)
if v.From != nil {
f(&v.From)
}
if v.To != nil {
f(&v.To)
}
if v.Step != nil {
f(&v.Step)
}
case *ParenExpr:
f(&v.X)
case *UnaryExpr:
f(&v.X)
case *BinaryExpr:
f(&v.X)
f(&v.Y)
case *AssignExpr:
f(&v.LHS)
f(&v.RHS)
case *LambdaExpr:
for i := range v.Params {
f(&v.Params[i])
}
for i := range v.Body {
f(&v.Body[i])
}
case *CallExpr:
f(&v.X)
for i := range v.List {
f(&v.List[i])
}
case *ListExpr:
for i := range v.List {
f(&v.List[i])
}
case *SetExpr:
for i := range v.List {
f(&v.List[i])
}
case *TupleExpr:
for i := range v.List {
f(&v.List[i])
}
case *DictExpr:
for i := range v.List {
f(&v.List[i])
}
case *Comprehension:
f(&v.Body)
for _, c := range v.Clauses {
f(&c)
}
case *IfClause:
f(&v.Cond)
case *ForClause:
f(&v.Vars)
f(&v.X)
case *ConditionalExpr:
f(&v.Then)
f(&v.Test)
f(&v.Else)
case *LoadStmt:
module := (Expr)(v.Module)
f(&module)
v.Module = module.(*StringExpr)
for i := range v.From {
from := (Expr)(v.From[i])
f(&from)
v.From[i] = from.(*Ident)
to := (Expr)(v.To[i])
f(&to)
v.To[i] = to.(*Ident)
}
case *DefStmt:
for i := range v.Params {
f(&v.Params[i])
}
for i := range v.Body {
f(&v.Body[i])
}
case *IfStmt:
f(&v.Cond)
for i := range v.True {
f(&v.True[i])
}
for i := range v.False {
f(&v.False[i])
}
case *ForStmt:
f(&v.Vars)
f(&v.X)
for i := range v.Body {
f(&v.Body[i])
}
case *ReturnStmt:
if v.Result != nil {
f(&v.Result)
}
}
}
// walkStatements is a helper function for WalkStatements
func walkStatements(v Expr, stack *[]Expr, f func(x Expr, stk []Expr)) {
if v == nil {
return
}
f(v, *stack)
*stack = append(*stack, v)
traverse := func(x Expr) {
walkStatements(x, stack, f)
}
switch expr := v.(type) {
case *File:
for _, s := range expr.Stmt {
traverse(s)
}
case *DefStmt:
for _, s := range expr.Body {
traverse(s)
}
case *IfStmt:
for _, s := range expr.True {
traverse(s)
}
for _, s := range expr.False {
traverse(s)
}
case *ForStmt:
for _, s := range expr.Body {
traverse(s)
}
}
*stack = (*stack)[:len(*stack)-1]
}
// WalkStatements traverses sub statements (not all nodes)
func WalkStatements(v Expr, f func(x Expr, stk []Expr)) {
var stack []Expr
walkStatements(v, &stack, f)
}