hack/update-vendor.sh

This commit is contained in:
Jordan Liggitt
2019-11-05 14:11:10 -05:00
parent 9a5b7c24ad
commit 297570e06a
932 changed files with 77190 additions and 28219 deletions

View File

@@ -10,7 +10,8 @@ distributed under the License is distributed on an "AS IS" BASIS,
See the License for the specific language governing permissions and
limitations under the License.
*/
// Buildozer is a tool for programatically editing BUILD files.
// Buildozer is a tool for programmatically editing BUILD files.
package edit
@@ -58,7 +59,7 @@ func NewOpts() *Options {
return &Options{NumIO: 200, PreferEOLComments: true}
}
// Usage is a user-overriden func to print the program usage.
// Usage is a user-overridden func to print the program usage.
var Usage = func() {}
var fileModified = false // set to true when a file has been fixed
@@ -69,7 +70,7 @@ const stdinPackageName = "-" // the special package name to represent stdin
type CmdEnvironment struct {
File *build.File // the AST
Rule *build.Rule // the rule to modify
Vars map[string]*build.BinaryExpr // global variables set in the build file
Vars map[string]*build.AssignExpr // global variables set in the build file
Pkg string // the full package name
Args []string // the command-line arguments
output *apipb.Output_Record // output proto, stores whatever a command wants to print
@@ -84,9 +85,10 @@ func cmdAdd(opts *Options, env CmdEnvironment) (*build.File, error) {
AddValueToListAttribute(env.Rule, attr, env.Pkg, &build.LiteralExpr{Token: val}, &env.Vars)
continue
}
strVal := &build.StringExpr{Value: ShortenLabel(val, env.Pkg)}
strVal := getStringExpr(val, env.Pkg)
AddValueToListAttribute(env.Rule, attr, env.Pkg, strVal, &env.Vars)
}
ResolveAttr(env.Rule, attr, env.Pkg)
return env.File, nil
}
@@ -96,8 +98,10 @@ func cmdComment(opts *Options, env CmdEnvironment) (*build.File, error) {
str = strings.Replace(str, "\\n", "\n", -1)
// Multiline comments should go on a separate line.
fullLine := !opts.PreferEOLComments || strings.Contains(str, "\n")
str = strings.Replace("# "+str, "\n", "\n# ", -1)
comment := []build.Comment{{Token: str}}
comment := []build.Comment{}
for _, line := range strings.Split(str, "\n") {
comment = append(comment, build.Comment{Token: "# " + line})
}
// The comment might be attached to a rule, an attribute, or a value in a list,
// depending on how many arguments are passed.
@@ -107,9 +111,9 @@ func cmdComment(opts *Options, env CmdEnvironment) (*build.File, error) {
case 2: // Attach to an attribute
if attr := env.Rule.AttrDefn(env.Args[0]); attr != nil {
if fullLine {
attr.X.Comment().Before = comment
attr.LHS.Comment().Before = comment
} else {
attr.Y.Comment().Suffix = comment
attr.RHS.Comment().Suffix = comment
}
}
case 3: // Attach to a specific value in a list
@@ -218,7 +222,7 @@ func cmdNew(opts *Options, env CmdEnvironment) (*build.File, error) {
return nil, fmt.Errorf("rule '%s' already exists", name)
}
call := &build.CallExpr{X: &build.LiteralExpr{Token: kind}}
call := &build.CallExpr{X: &build.Ident{Name: kind}}
rule := &build.Rule{call, ""}
rule.SetAttr("name", &build.StringExpr{Value: name})
@@ -253,7 +257,16 @@ func findInsertionIndex(env CmdEnvironment) (bool, int, error) {
}
func cmdNewLoad(opts *Options, env CmdEnvironment) (*build.File, error) {
env.File.Stmt = InsertLoad(env.File.Stmt, env.Args)
from := env.Args[1:]
to := append([]string{}, from...)
for i := range from {
if s := strings.SplitN(from[i], "=", 2); len(s) == 2 {
to[i] = s[0]
from[i] = s[1]
}
}
env.File.Stmt = InsertLoad(env.File.Stmt, env.Args[0], from, to)
return env.File, nil
}
@@ -290,6 +303,8 @@ func cmdPrint(opts *Options, env CmdEnvironment) (*build.File, error) {
fields[i] = &apipb.Output_Record_Field{Value: &apipb.Output_Record_Field_Error{Error: apipb.Output_Record_Field_MISSING}}
} else if lit, ok := value.(*build.LiteralExpr); ok {
fields[i] = &apipb.Output_Record_Field{Value: &apipb.Output_Record_Field_Text{lit.Token}}
} else if lit, ok := value.(*build.Ident); ok {
fields[i] = &apipb.Output_Record_Field{Value: &apipb.Output_Record_Field_Text{lit.Name}}
} else if string, ok := value.(*build.StringExpr); ok {
fields[i] = &apipb.Output_Record_Field{
Value: &apipb.Output_Record_Field_Text{string.Value},
@@ -326,6 +341,7 @@ func cmdRemove(opts *Options, env CmdEnvironment) (*build.File, error) {
ListAttributeDelete(env.Rule, key, val, env.Pkg)
fixed = true
}
ResolveAttr(env.Rule, key, env.Pkg)
}
if fixed {
return env.File, nil
@@ -334,6 +350,38 @@ func cmdRemove(opts *Options, env CmdEnvironment) (*build.File, error) {
return nil, nil
}
func cmdRemoveComment(opts *Options, env CmdEnvironment) (*build.File, error) {
switch len(env.Args) {
case 0: // Remove comment attached to rule
env.Rule.Call.Comments.Before = nil
env.Rule.Call.Comments.Suffix = nil
env.Rule.Call.Comments.After = nil
case 1: // Remove comment attached to attr
if attr := env.Rule.AttrDefn(env.Args[0]); attr != nil {
attr.Comments.Before = nil
attr.Comments.Suffix = nil
attr.Comments.After = nil
attr.LHS.Comment().Before = nil
attr.LHS.Comment().Suffix = nil
attr.LHS.Comment().After = nil
attr.RHS.Comment().Before = nil
attr.RHS.Comment().Suffix = nil
attr.RHS.Comment().After = nil
}
case 2: // Remove comment attached to value
if attr := env.Rule.Attr(env.Args[0]); attr != nil {
if expr := ListFind(attr, env.Args[1], env.Pkg); expr != nil {
expr.Comments.Before = nil
expr.Comments.Suffix = nil
expr.Comments.After = nil
}
}
default:
panic("cmdRemoveComment")
}
return env.File, nil
}
func cmdRename(opts *Options, env CmdEnvironment) (*build.File, error) {
oldAttr := env.Args[0]
newAttr := env.Args[1]
@@ -350,7 +398,7 @@ func cmdReplace(opts *Options, env CmdEnvironment) (*build.File, error) {
attr := env.Rule.Attr(key)
if e, ok := attr.(*build.StringExpr); ok {
if LabelsEqual(e.Value, oldV, env.Pkg) {
env.Rule.SetAttr(key, getAttrValueExpr(key, []string{newV}))
env.Rule.SetAttr(key, getAttrValueExpr(key, []string{newV}, env))
}
} else {
ListReplace(attr, oldV, newV, env.Pkg)
@@ -373,7 +421,7 @@ func cmdSubstitute(opts *Options, env CmdEnvironment) (*build.File, error) {
continue
}
if newValue, ok := stringSubstitute(e.Value, oldRegexp, newTemplate); ok {
env.Rule.SetAttr(key, getAttrValueExpr(key, []string{newValue}))
env.Rule.SetAttr(key, getAttrValueExpr(key, []string{newValue}, env))
}
}
return env.File, nil
@@ -385,7 +433,7 @@ func cmdSet(opts *Options, env CmdEnvironment) (*build.File, error) {
if attr == "kind" {
env.Rule.SetKind(args[0])
} else {
env.Rule.SetAttr(attr, getAttrValueExpr(attr, args))
env.Rule.SetAttr(attr, getAttrValueExpr(attr, args, env))
}
return env.File, nil
}
@@ -397,12 +445,12 @@ func cmdSetIfAbsent(opts *Options, env CmdEnvironment) (*build.File, error) {
return nil, fmt.Errorf("setting 'kind' is not allowed for set_if_absent. Got %s", env.Args)
}
if env.Rule.Attr(attr) == nil {
env.Rule.SetAttr(attr, getAttrValueExpr(attr, args))
env.Rule.SetAttr(attr, getAttrValueExpr(attr, args, env))
}
return env.File, nil
}
func getAttrValueExpr(attr string, args []string) build.Expr {
func getAttrValueExpr(attr string, args []string, env CmdEnvironment) build.Expr {
switch {
case attr == "kind":
return nil
@@ -414,17 +462,28 @@ func getAttrValueExpr(attr string, args []string) build.Expr {
return &build.ListExpr{List: list}
case IsList(attr) && !(len(args) == 1 && strings.HasPrefix(args[0], "glob(")):
var list []build.Expr
for _, i := range args {
list = append(list, &build.StringExpr{Value: i})
for _, arg := range args {
list = append(list, getStringExpr(arg, env.Pkg))
}
return &build.ListExpr{List: list}
case len(args) == 0:
// Expected a non-list argument, nothing provided
return &build.Ident{Name: "None"}
case IsString(attr):
return &build.StringExpr{Value: args[0]}
return getStringExpr(args[0], env.Pkg)
default:
return &build.LiteralExpr{Token: args[0]}
return &build.Ident{Name: args[0]}
}
}
func getStringExpr(value, pkg string) build.Expr {
unquoted, triple, err := build.Unquote(value)
if err == nil {
return &build.StringExpr{Value: ShortenLabel(unquoted, pkg), TripleQuote: triple}
}
return &build.StringExpr{Value: ShortenLabel(value, pkg)}
}
func cmdCopy(opts *Options, env CmdEnvironment) (*build.File, error) {
attrName := env.Args[0]
from := env.Args[1]
@@ -443,6 +502,77 @@ func cmdCopyNoOverwrite(opts *Options, env CmdEnvironment) (*build.File, error)
return copyAttributeBetweenRules(env, attrName, from)
}
// cmdDictAdd adds a key to a dict, if that key does _not_ exit already.
func cmdDictAdd(opts *Options, env CmdEnvironment) (*build.File, error) {
attr := env.Args[0]
args := env.Args[1:]
dict := &build.DictExpr{}
currDict, ok := env.Rule.Attr(attr).(*build.DictExpr)
if ok {
dict = currDict
}
for _, x := range args {
kv := strings.Split(x, ":")
expr := getStringExpr(kv[1], env.Pkg)
prev := DictionaryGet(dict, kv[0])
if prev == nil {
// Only set the value if the value is currently unset.
DictionarySet(dict, kv[0], expr)
}
}
env.Rule.SetAttr(attr, dict)
return env.File, nil
}
// cmdDictSet adds a key to a dict, overwriting any previous values.
func cmdDictSet(opts *Options, env CmdEnvironment) (*build.File, error) {
attr := env.Args[0]
args := env.Args[1:]
dict := &build.DictExpr{}
currDict, ok := env.Rule.Attr(attr).(*build.DictExpr)
if ok {
dict = currDict
}
for _, x := range args {
kv := strings.Split(x, ":")
expr := getStringExpr(kv[1], env.Pkg)
// Set overwrites previous values.
DictionarySet(dict, kv[0], expr)
}
env.Rule.SetAttr(attr, dict)
return env.File, nil
}
// cmdDictRemove removes a key from a dict.
func cmdDictRemove(opts *Options, env CmdEnvironment) (*build.File, error) {
attr := env.Args[0]
args := env.Args[1:]
thing := env.Rule.Attr(attr)
dictAttr, ok := thing.(*build.DictExpr)
if !ok {
return env.File, nil
}
for _, x := range args {
// should errors here be flagged?
DictionaryDelete(dictAttr, x)
env.Rule.SetAttr(attr, dictAttr)
}
// If the removal results in the dict having no contents, delete the attribute (stay clean!)
if dictAttr == nil || len(dictAttr.List) == 0 {
env.Rule.DelAttr(attr)
}
return env.File, nil
}
func copyAttributeBetweenRules(env CmdEnvironment, attrName string, from string) (*build.File, error) {
fromRule := FindRuleByName(env.File, from)
if fromRule == nil {
@@ -453,7 +583,7 @@ func copyAttributeBetweenRules(env CmdEnvironment, attrName string, from string)
return nil, fmt.Errorf("rule '%s' does not have attribute '%s'", from, attrName)
}
ast, err := build.Parse("" /* filename */, []byte(build.FormatString(attr)))
ast, err := build.ParseBuild("" /* filename */, []byte(build.FormatString(attr)))
if err != nil {
return nil, fmt.Errorf("could not parse attribute value %v", build.FormatString(attr))
}
@@ -474,6 +604,7 @@ func cmdFix(opts *Options, env CmdEnvironment) (*build.File, error) {
// CommandInfo provides a command function and info on incoming arguments.
type CommandInfo struct {
Fn func(*Options, CmdEnvironment) (*build.File, error)
PerRule bool
MinArg int
MaxArg int
Template string
@@ -482,23 +613,27 @@ type CommandInfo struct {
// AllCommands associates the command names with their function and number
// of arguments.
var AllCommands = map[string]CommandInfo{
"add": {cmdAdd, 2, -1, "<attr> <value(s)>"},
"new_load": {cmdNewLoad, 1, -1, "<path> <symbol(s)>"},
"comment": {cmdComment, 1, 3, "<attr>? <value>? <comment>"},
"print_comment": {cmdPrintComment, 0, 2, "<attr>? <value>?"},
"delete": {cmdDelete, 0, 0, ""},
"fix": {cmdFix, 0, -1, "<fix(es)>?"},
"move": {cmdMove, 3, -1, "<old_attr> <new_attr> <value(s)>"},
"new": {cmdNew, 2, 4, "<rule_kind> <rule_name> [(before|after) <relative_rule_name>]"},
"print": {cmdPrint, 0, -1, "<attribute(s)>"},
"remove": {cmdRemove, 1, -1, "<attr> <value(s)>"},
"rename": {cmdRename, 2, 2, "<old_attr> <new_attr>"},
"replace": {cmdReplace, 3, 3, "<attr> <old_value> <new_value>"},
"substitute": {cmdSubstitute, 3, 3, "<attr> <old_regexp> <new_template>"},
"set": {cmdSet, 2, -1, "<attr> <value(s)>"},
"set_if_absent": {cmdSetIfAbsent, 2, -1, "<attr> <value(s)>"},
"copy": {cmdCopy, 2, 2, "<attr> <from_rule>"},
"copy_no_overwrite": {cmdCopyNoOverwrite, 2, 2, "<attr> <from_rule>"},
"add": {cmdAdd, true, 2, -1, "<attr> <value(s)>"},
"new_load": {cmdNewLoad, false, 1, -1, "<path> <[to=]from(s)>"},
"comment": {cmdComment, true, 1, 3, "<attr>? <value>? <comment>"},
"print_comment": {cmdPrintComment, true, 0, 2, "<attr>? <value>?"},
"delete": {cmdDelete, true, 0, 0, ""},
"fix": {cmdFix, true, 0, -1, "<fix(es)>?"},
"move": {cmdMove, true, 3, -1, "<old_attr> <new_attr> <value(s)>"},
"new": {cmdNew, false, 2, 4, "<rule_kind> <rule_name> [(before|after) <relative_rule_name>]"},
"print": {cmdPrint, true, 0, -1, "<attribute(s)>"},
"remove": {cmdRemove, true, 1, -1, "<attr> <value(s)>"},
"remove_comment": {cmdRemoveComment, true, 0, 2, "<attr>? <value>?"},
"rename": {cmdRename, true, 2, 2, "<old_attr> <new_attr>"},
"replace": {cmdReplace, true, 3, 3, "<attr> <old_value> <new_value>"},
"substitute": {cmdSubstitute, true, 3, 3, "<attr> <old_regexp> <new_template>"},
"set": {cmdSet, true, 1, -1, "<attr> <value(s)>"},
"set_if_absent": {cmdSetIfAbsent, true, 1, -1, "<attr> <value(s)>"},
"copy": {cmdCopy, true, 2, 2, "<attr> <from_rule>"},
"copy_no_overwrite": {cmdCopyNoOverwrite, true, 2, 2, "<attr> <from_rule>"},
"dict_add": {cmdDictAdd, true, 2, -1, "<attr> <(key:value)(s)>"},
"dict_set": {cmdDictSet, true, 2, -1, "<attr> <(key:value)(s)>"},
"dict_remove": {cmdDictRemove, true, 2, -1, "<attr> <key(s)>"},
}
func expandTargets(f *build.File, rule string) ([]*build.Rule, error) {
@@ -530,16 +665,12 @@ func filterRules(opts *Options, rules []*build.Rule) (result []*build.Rule) {
return rules
}
for _, rule := range rules {
acceptableType := false
for _, filterType := range opts.FilterRuleTypes {
if rule.Kind() == filterType {
acceptableType = true
result = append(result, rule)
break
}
}
if acceptableType || rule.Kind() == "package" {
result = append(result, rule)
}
}
return
}
@@ -564,6 +695,7 @@ func checkCommandUsage(name string, cmd CommandInfo, count int) {
name, cmd.MaxArg)
}
Usage()
os.Exit(1)
}
// Match text that only contains spaces if they're escaped with '\'.
@@ -630,16 +762,13 @@ type rewriteResult struct {
// getGlobalVariables returns the global variable assignments in the provided list of expressions.
// That is, for each variable assignment of the form
// a = v
// vars["a"] will contain the BinaryExpr whose Y value is the assignment "a = v".
func getGlobalVariables(exprs []build.Expr) (vars map[string]*build.BinaryExpr) {
vars = make(map[string]*build.BinaryExpr)
// vars["a"] will contain the AssignExpr whose RHS value is the assignment "a = v".
func getGlobalVariables(exprs []build.Expr) (vars map[string]*build.AssignExpr) {
vars = make(map[string]*build.AssignExpr)
for _, expr := range exprs {
if binExpr, ok := expr.(*build.BinaryExpr); ok {
if binExpr.Op != "=" {
continue
}
if lhs, ok := binExpr.X.(*build.LiteralExpr); ok {
vars[lhs.Token] = binExpr
if as, ok := expr.(*build.AssignExpr); ok {
if lhs, ok := as.LHS.(*build.Ident); ok {
vars[lhs.Name] = as
}
}
}
@@ -695,12 +824,12 @@ func rewrite(opts *Options, commandsForFile commandsForFile) *rewriteResult {
}
}
f, err := build.Parse(name, data)
f, err := build.ParseBuild(name, data)
if err != nil {
return &rewriteResult{file: name, errs: []error{err}}
}
vars := map[string]*build.BinaryExpr{}
vars := map[string]*build.AssignExpr{}
if opts.EditVariables {
vars = getGlobalVariables(f.Stmt)
}
@@ -726,8 +855,14 @@ func rewrite(opts *Options, commandsForFile commandsForFile) *rewriteResult {
}
targets = filterRules(opts, targets)
for _, cmd := range commands {
for _, r := range targets {
cmdInfo := AllCommands[cmd.tokens[0]]
cmdInfo := AllCommands[cmd.tokens[0]]
// Depending on whether a transformation is rule-specific or not, it should be applied to
// every rule that satisfies the filter or just once to the file.
cmdTargets := targets
if !cmdInfo.PerRule {
cmdTargets = []*build.Rule{nil}
}
for _, r := range cmdTargets {
record := &apipb.Output_Record{}
newf, err := cmdInfo.Fn(opts, CmdEnvironment{f, r, vars, absPkg, cmd.tokens[1:], record})
if len(record.Fields) != 0 {
@@ -793,7 +928,7 @@ func runBuildifier(opts *Options, f *build.File) ([]byte, error) {
return build.Format(f), nil
}
cmd := exec.Command(opts.Buildifier)
cmd := exec.Command(opts.Buildifier, "--type=build")
data := build.Format(f)
cmd.Stdin = bytes.NewBuffer(data)
stdout := bytes.NewBuffer(nil)
@@ -881,18 +1016,29 @@ func appendCommandsFromFile(opts *Options, commandsByFile map[string][]commandsF
reader = rc
defer rc.Close()
}
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
line := scanner.Text()
appendCommandsFromReader(opts, reader, commandsByFile)
}
func appendCommandsFromReader(opts *Options, reader io.Reader, commandsByFile map[string][]commandsForTarget) {
r := bufio.NewReader(reader)
atEof := false
for !atEof {
line, err := r.ReadString('\n')
if err == io.EOF {
atEof = true
err = nil
}
if err != nil {
fmt.Fprintf(os.Stderr, "Error while reading commands file: %v", err)
return
}
line = strings.TrimSuffix(line, "\n")
if line == "" {
continue
}
args := strings.Split(line, "|")
appendCommands(opts, commandsByFile, args)
}
if err := scanner.Err(); err != nil {
fmt.Fprintf(os.Stderr, "Error while reading commands file: %v", scanner.Err())
}
}
func printRecord(writer io.Writer, record *apipb.Output_Record) {

View File

@@ -75,7 +75,7 @@ func ShortenLabel(label string, pkg string) string {
if !ShortenLabelsFlag {
return label
}
if !strings.HasPrefix(label, "//") {
if !strings.Contains(label, "//") {
// It doesn't look like a long label, so we preserve it.
return label
}
@@ -85,7 +85,13 @@ func ShortenLabel(label string, pkg string) string {
}
slash := strings.LastIndex(labelPkg, "/")
if (slash >= 0 && labelPkg[slash+1:] == rule) || labelPkg == rule {
return "//" + labelPkg
if repo == "" {
return "//" + labelPkg
}
return "@" + repo + "//" + labelPkg
}
if strings.HasPrefix(label, "@") && repo == rule && labelPkg == "" {
return "@" + repo
}
return label
}
@@ -162,8 +168,8 @@ func ExprToRule(expr build.Expr, kind string) (*build.Rule, bool) {
if !ok {
return nil, false
}
k, ok := call.X.(*build.LiteralExpr)
if !ok || k.Token != kind {
k, ok := call.X.(*build.Ident)
if !ok || k.Name != kind {
return nil, false
}
return &build.Rule{call, ""}, true
@@ -180,21 +186,25 @@ func ExistingPackageDeclaration(f *build.File) *build.Rule {
}
// PackageDeclaration returns the package declaration. If it doesn't
// exist, it is created at the top of the BUILD file, after leading
// comments.
// exist, it is created at the top of the BUILD file, after optional
// docstring, comments, and load statements.
func PackageDeclaration(f *build.File) *build.Rule {
if pkg := ExistingPackageDeclaration(f); pkg != nil {
return pkg
}
all := []build.Expr{}
added := false
call := &build.CallExpr{X: &build.LiteralExpr{Token: "package"}}
// Skip CommentBlocks and find a place to insert the package declaration.
call := &build.CallExpr{X: &build.Ident{Name: "package"}}
for _, stmt := range f.Stmt {
_, ok := stmt.(*build.CommentBlock)
if !ok && !added {
all = append(all, call)
added = true
switch stmt.(type) {
case *build.CommentBlock, *build.LoadStmt, *build.StringExpr:
// Skip docstring, comments, and load statements to
// find a place to insert the package declaration.
default:
if !added {
all = append(all, call)
added = true
}
}
all = append(all, stmt)
}
@@ -213,14 +223,14 @@ func RemoveEmptyPackage(f *build.File) *build.File {
var all []build.Expr
for _, stmt := range f.Stmt {
if call, ok := stmt.(*build.CallExpr); ok {
functionName, ok := call.X.(*build.LiteralExpr)
if ok && functionName.Token == "package" && len(call.List) == 0 {
functionName, ok := call.X.(*build.Ident)
if ok && functionName.Name == "package" && len(call.List) == 0 {
continue
}
}
all = append(all, stmt)
}
return &build.File{Path: f.Path, Comments: f.Comments, Stmt: all}
return &build.File{Path: f.Path, Comments: f.Comments, Stmt: all, Type: build.TypeBuild}
}
// InsertAfter inserts an expression after index i.
@@ -241,8 +251,8 @@ func IndexOfLast(stmt []build.Expr, Kind string) int {
if !ok {
continue
}
literal, ok := sAsCallExpr.X.(*build.LiteralExpr)
if ok && literal.Token == Kind {
literal, ok := sAsCallExpr.X.(*build.Ident)
if ok && literal.Name == Kind {
lastIndex = i
}
}
@@ -251,7 +261,7 @@ func IndexOfLast(stmt []build.Expr, Kind string) int {
// InsertAfterLastOfSameKind inserts an expression after the last expression of the same kind.
func InsertAfterLastOfSameKind(stmt []build.Expr, expr *build.CallExpr) []build.Expr {
index := IndexOfLast(stmt, expr.X.(*build.LiteralExpr).Token)
index := IndexOfLast(stmt, expr.X.(*build.Ident).Name)
if index == -1 {
return InsertAtEnd(stmt, expr)
}
@@ -329,7 +339,7 @@ func DeleteRule(f *build.File, rule *build.Rule) *build.File {
}
all = append(all, stmt)
}
return &build.File{Path: f.Path, Comments: f.Comments, Stmt: all}
return &build.File{Path: f.Path, Comments: f.Comments, Stmt: all, Type: build.TypeBuild}
}
// DeleteRuleByName returns the AST without the rules that have the
@@ -347,7 +357,7 @@ func DeleteRuleByName(f *build.File, name string) *build.File {
all = append(all, stmt)
}
}
return &build.File{Path: f.Path, Comments: f.Comments, Stmt: all}
return &build.File{Path: f.Path, Comments: f.Comments, Stmt: all, Type: build.TypeBuild}
}
// DeleteRuleByKind removes the rules of the specified kind from the AST.
@@ -360,12 +370,12 @@ func DeleteRuleByKind(f *build.File, kind string) *build.File {
all = append(all, stmt)
continue
}
k, ok := call.X.(*build.LiteralExpr)
if !ok || k.Token != kind {
k, ok := call.X.(*build.Ident)
if !ok || k.Name != kind {
all = append(all, stmt)
}
}
return &build.File{Path: f.Path, Comments: f.Comments, Stmt: all}
return &build.File{Path: f.Path, Comments: f.Comments, Stmt: all, Type: build.TypeBuild}
}
// AllLists returns all the lists concatenated in an expression.
@@ -383,6 +393,21 @@ func AllLists(e build.Expr) []*build.ListExpr {
return nil
}
// AllSelects returns all the selects concatenated in an expression.
func AllSelects(e build.Expr) []*build.CallExpr {
switch e := e.(type) {
case *build.BinaryExpr:
if e.Op == "+" {
return append(AllSelects(e.X), AllSelects(e.Y)...)
}
case *build.CallExpr:
if x, ok := e.X.(*build.Ident); ok && x.Name == "select" {
return []*build.CallExpr{e}
}
}
return nil
}
// FirstList works in the same way as AllLists, except that it
// returns only one list, or nil.
func FirstList(e build.Expr) *build.ListExpr {
@@ -451,24 +476,208 @@ func ContainsComments(expr build.Expr, str string) bool {
return false
}
// RemoveEmptySelectsAndConcatLists iterates the tree in order to turn
// empty selects into empty lists and adjacent lists are concatenated
func RemoveEmptySelectsAndConcatLists(e build.Expr) build.Expr {
switch e := e.(type) {
case *build.BinaryExpr:
if e.Op == "+" {
e.X = RemoveEmptySelectsAndConcatLists(e.X)
e.Y = RemoveEmptySelectsAndConcatLists(e.Y)
x, xIsList := e.X.(*build.ListExpr)
y, yIsList := e.Y.(*build.ListExpr)
if xIsList && yIsList {
return &build.ListExpr{List: append(x.List, y.List...)}
}
if xIsList && len(x.List) == 0 {
return e.Y
}
if yIsList && len(y.List) == 0 {
return e.X
}
}
case *build.CallExpr:
if x, ok := e.X.(*build.Ident); ok && x.Name == "select" {
if len(e.List) == 0 {
return &build.ListExpr{List: []build.Expr{}}
}
if dict, ok := e.List[0].(*build.DictExpr); ok {
for _, keyVal := range dict.List {
if keyVal, ok := keyVal.(*build.KeyValueExpr); ok {
val, ok := keyVal.Value.(*build.ListExpr)
if !ok || len(val.List) > 0 {
return e
}
} else {
return e
}
}
return &build.ListExpr{List: []build.Expr{}}
}
}
}
return e
}
// ComputeIntersection returns the intersection of the two lists given as parameters;
// if the containing elements are not build.StringExpr, the result will be nil.
func ComputeIntersection(list1, list2 []build.Expr) []build.Expr {
if list1 == nil || list2 == nil {
return nil
}
if len(list2) == 0 {
return []build.Expr{}
}
i := 0
for j, common := range list1 {
if common, ok := common.(*build.StringExpr); ok {
found := false
for _, elem := range list2 {
if str, ok := elem.(*build.StringExpr); ok {
if str.Value == common.Value {
found = true
break
}
} else {
return nil
}
}
if found {
list1[i] = list1[j]
i++
}
} else {
return nil
}
}
return list1[:i]
}
// SelectListsIntersection returns the intersection of the lists of strings inside
// the dictionary argument of the select expression given as a parameter
func SelectListsIntersection(sel *build.CallExpr, pkg string) (intersection []build.Expr) {
if len(sel.List) == 0 || len(sel.List) > 1 {
return nil
}
dict, ok := sel.List[0].(*build.DictExpr)
if !ok || len(dict.List) == 0 {
return nil
}
if keyVal, ok := dict.List[0].(*build.KeyValueExpr); ok {
if val, ok := keyVal.Value.(*build.ListExpr); ok {
intersection = make([]build.Expr, len(val.List))
copy(intersection, val.List)
}
}
for _, keyVal := range dict.List[1:] {
if keyVal, ok := keyVal.(*build.KeyValueExpr); ok {
if val, ok := keyVal.Value.(*build.ListExpr); ok {
intersection = ComputeIntersection(intersection, val.List)
if len(intersection) == 0 {
return intersection
}
} else {
return nil
}
} else {
return nil
}
}
return intersection
}
// ResolveAttr extracts common elements of the lists inside select dictionaries
// and adds them at attribute level rather than select level, as well as turns
// empty selects into empty lists and concatenates adjacent lists
func ResolveAttr(r *build.Rule, attr, pkg string) {
var toExtract []build.Expr
e := r.Attr(attr)
if e == nil {
return
}
for _, sel := range AllSelects(e) {
intersection := SelectListsIntersection(sel, pkg)
if intersection != nil {
toExtract = append(toExtract, intersection...)
}
}
for _, common := range toExtract {
e = AddValueToList(e, pkg, common, false) // this will also remove them from selects
}
r.SetAttr(attr, RemoveEmptySelectsAndConcatLists(e))
}
// SelectDelete removes the item from all the lists which are values
// in the dictionary of every select
func SelectDelete(e build.Expr, item, pkg string, deleted **build.StringExpr) {
for _, sel := range AllSelects(e) {
if len(sel.List) == 0 {
continue
}
if dict, ok := sel.List[0].(*build.DictExpr); ok {
for _, keyVal := range dict.List {
if keyVal, ok := keyVal.(*build.KeyValueExpr); ok {
if val, ok := keyVal.Value.(*build.ListExpr); ok {
RemoveFromList(val, item, pkg, deleted)
}
}
}
}
}
}
// RemoveFromList removes one element from a ListExpr and stores
// the deleted StringExpr at the address pointed by the last parameter
func RemoveFromList(li *build.ListExpr, item, pkg string, deleted **build.StringExpr) {
var all []build.Expr
for _, elem := range li.List {
if str, ok := elem.(*build.StringExpr); ok {
if LabelsEqual(str.Value, item, pkg) && (DeleteWithComments || !hasComments(str)) {
if deleted != nil {
*deleted = str
}
continue
}
}
all = append(all, elem)
}
li.List = all
}
// ListDelete deletes the item from a list expression in e and returns
// the StringExpr deleted, or nil otherwise.
func ListDelete(e build.Expr, item, pkg string) (deleted *build.StringExpr) {
if unquoted, _, err := build.Unquote(item); err == nil {
item = unquoted
}
deleted = nil
item = ShortenLabel(item, pkg)
for _, li := range AllLists(e) {
var all []build.Expr
for _, elem := range li.List {
if str, ok := elem.(*build.StringExpr); ok {
if LabelsEqual(str.Value, item, pkg) && (DeleteWithComments || !hasComments(str)) {
deleted = str
continue
}
}
all = append(all, elem)
}
li.List = all
RemoveFromList(li, item, pkg, &deleted)
}
SelectDelete(e, item, pkg, &deleted)
return deleted
}
@@ -582,13 +791,13 @@ func attributeMustNotBeSorted(rule, attr string) bool {
// getVariable returns the binary expression that assignes a variable to expr, if expr is
// an identifier of a variable that vars contains a mapping for.
func getVariable(expr build.Expr, vars *map[string]*build.BinaryExpr) (varAssignment *build.BinaryExpr) {
func getVariable(expr build.Expr, vars *map[string]*build.AssignExpr) (varAssignment *build.AssignExpr) {
if vars == nil {
return nil
}
if literal, ok := expr.(*build.LiteralExpr); ok {
if varAssignment = (*vars)[literal.Token]; varAssignment != nil {
if literal, ok := expr.(*build.Ident); ok {
if varAssignment = (*vars)[literal.Name]; varAssignment != nil {
return varAssignment
}
}
@@ -604,10 +813,14 @@ func AddValueToList(oldList build.Expr, pkg string, item build.Expr, sorted bool
}
str, ok := item.(*build.StringExpr)
if ok && ListFind(oldList, str.Value, pkg) != nil {
// The value is already in the list.
return oldList
if ok {
if ListFind(oldList, str.Value, pkg) != nil {
// The value is already in the list.
return oldList
}
SelectDelete(oldList, str.Value, pkg, nil)
}
li := FirstList(oldList)
if li != nil {
if sorted {
@@ -623,11 +836,11 @@ func AddValueToList(oldList build.Expr, pkg string, item build.Expr, sorted bool
}
// AddValueToListAttribute adds the given item to the list attribute identified by name and pkg.
func AddValueToListAttribute(r *build.Rule, name string, pkg string, item build.Expr, vars *map[string]*build.BinaryExpr) {
func AddValueToListAttribute(r *build.Rule, name string, pkg string, item build.Expr, vars *map[string]*build.AssignExpr) {
old := r.Attr(name)
sorted := !attributeMustNotBeSorted(r.Kind(), name)
if varAssignment := getVariable(old, vars); varAssignment != nil {
varAssignment.Y = AddValueToList(varAssignment.Y, pkg, item, sorted)
varAssignment.RHS = AddValueToList(varAssignment.RHS, pkg, item, sorted)
} else {
r.SetAttr(name, AddValueToList(old, pkg, item, sorted))
}
@@ -635,7 +848,7 @@ func AddValueToListAttribute(r *build.Rule, name string, pkg string, item build.
// MoveAllListAttributeValues moves all values from list attribute oldAttr to newAttr,
// and deletes oldAttr.
func MoveAllListAttributeValues(rule *build.Rule, oldAttr, newAttr, pkg string, vars *map[string]*build.BinaryExpr) error {
func MoveAllListAttributeValues(rule *build.Rule, oldAttr, newAttr, pkg string, vars *map[string]*build.AssignExpr) error {
if rule.Attr(oldAttr) == nil {
return fmt.Errorf("no attribute %s found in %s", oldAttr, rule.Name())
}
@@ -672,21 +885,58 @@ func DictionarySet(dict *build.DictExpr, key string, value build.Expr) build.Exp
return nil
}
// DictionaryGet looks for the key in the dictionary expression, and returns the
// current value. If it is unset, it returns nil.
func DictionaryGet(dict *build.DictExpr, key string) build.Expr {
for _, e := range dict.List {
kv, ok := e.(*build.KeyValueExpr)
if !ok {
continue
}
if k, ok := kv.Key.(*build.StringExpr); ok && k.Value == key {
return kv.Value
}
}
return nil
}
// DictionaryDelete looks for the key in the dictionary expression. If the key exists,
// it removes the key-value pair and returns it. Otherwise it returns nil.
func DictionaryDelete(dict *build.DictExpr, key string) (deleted build.Expr) {
if unquoted, _, err := build.Unquote(key); err == nil {
key = unquoted
}
deleted = nil
var all []build.Expr
for _, e := range dict.List {
kv, _ := e.(*build.KeyValueExpr)
if k, ok := kv.Key.(*build.StringExpr); ok {
if k.Value == key {
deleted = kv
} else {
all = append(all, e)
}
}
}
dict.List = all
return deleted
}
// RenameAttribute renames an attribute in a rule.
func RenameAttribute(r *build.Rule, oldName, newName string) error {
if r.Attr(newName) != nil {
return fmt.Errorf("attribute %s already exists in rule %s", newName, r.Name())
}
for _, kv := range r.Call.List {
as, ok := kv.(*build.BinaryExpr)
if !ok || as.Op != "=" {
as, ok := kv.(*build.AssignExpr)
if !ok {
continue
}
k, ok := as.X.(*build.LiteralExpr)
if !ok || k.Token != oldName {
k, ok := as.LHS.(*build.Ident)
if !ok || k.Name != oldName {
continue
}
k.Token = newName
k.Name = newName
return nil
}
return fmt.Errorf("no attribute %s found in rule %s", oldName, r.Name())
@@ -700,8 +950,8 @@ func EditFunction(v build.Expr, name string, f func(x *build.CallExpr, stk []bui
if !ok {
return nil
}
fct, ok := call.X.(*build.LiteralExpr)
if !ok || fct.Token != name {
fct, ok := call.X.(*build.Ident)
if !ok || fct.Name != name {
return nil
}
return f(call, stk)
@@ -709,107 +959,137 @@ func EditFunction(v build.Expr, name string, f func(x *build.CallExpr, stk []bui
}
// UsedSymbols returns the set of symbols used in the BUILD file (variables, function names).
func UsedSymbols(f *build.File) map[string]bool {
func UsedSymbols(stmt build.Expr) map[string]bool {
symbols := make(map[string]bool)
build.Walk(f, func(expr build.Expr, stack []build.Expr) {
literal, ok := expr.(*build.LiteralExpr)
build.Walk(stmt, func(expr build.Expr, stack []build.Expr) {
// Don't traverse inside load statements
if len(stack) > 0 {
if _, ok := stack[len(stack)-1].(*build.LoadStmt); ok {
return
}
}
literal, ok := expr.(*build.Ident)
if !ok {
return
}
// Check if we are on the left-side of an assignment
for _, e := range stack {
if as, ok := e.(*build.BinaryExpr); ok {
if as.Op == "=" && as.X == expr {
if as, ok := e.(*build.AssignExpr); ok {
if as.LHS == expr {
return
}
}
}
symbols[literal.Token] = true
symbols[literal.Name] = true
})
return symbols
}
func newLoad(args []string) *build.CallExpr {
load := &build.CallExpr{
X: &build.LiteralExpr{
Token: "load",
// NewLoad creates a new LoadStmt node
func NewLoad(location string, from, to []string) *build.LoadStmt {
load := &build.LoadStmt{
Module: &build.StringExpr{
Value: location,
},
List: []build.Expr{},
ForceCompact: true,
}
for _, a := range args {
load.List = append(load.List, &build.StringExpr{Value: a})
for i := range from {
load.From = append(load.From, &build.Ident{Name: from[i]})
load.To = append(load.To, &build.Ident{Name: to[i]})
}
return load
}
// appendLoad tries to find an existing load location and append symbols to it.
func appendLoad(stmts []build.Expr, args []string) bool {
if len(args) == 0 {
return false
// AppendToLoad appends symbols to an existing load statement
// Returns true if the statement was acually edited (if the required symbols haven't been
// loaded yet)
func AppendToLoad(load *build.LoadStmt, from, to []string) bool {
symbolsToLoad := make(map[string]string)
for i, s := range to {
symbolsToLoad[s] = from[i]
}
location := args[0]
symbolsToLoad := make(map[string]bool)
for _, s := range args[1:] {
symbolsToLoad[s] = true
}
var lastLoad *build.CallExpr
for _, s := range stmts {
call, ok := s.(*build.CallExpr)
if !ok {
continue
}
if l, ok := call.X.(*build.LiteralExpr); !ok || l.Token != "load" {
continue
}
if len(call.List) < 2 {
continue
}
if s, ok := call.List[0].(*build.StringExpr); !ok || s.Value != location {
continue // Loads a different file.
}
for _, arg := range call.List[1:] {
if s, ok := arg.(*build.StringExpr); ok {
delete(symbolsToLoad, s.Value) // Already loaded.
}
}
// Remember the last insert location, but potentially remove more symbols
// that are already loaded in other subsequent calls.
lastLoad = call
for _, ident := range load.To {
delete(symbolsToLoad, ident.Name) // Already loaded.
}
if lastLoad == nil {
if len(symbolsToLoad) == 0 {
return false
}
// Append the remaining loads to the last load location.
// Append the remaining loads to the load statement.
sortedSymbols := []string{}
for s := range symbolsToLoad {
sortedSymbols = append(sortedSymbols, s)
}
sort.Strings(sortedSymbols)
for _, s := range sortedSymbols {
lastLoad.List = append(lastLoad.List, &build.StringExpr{Value: s})
load.From = append(load.From, &build.Ident{Name: symbolsToLoad[s]})
load.To = append(load.To, &build.Ident{Name: s})
}
return true
}
// appendLoad tries to find an existing load location and append symbols to it.
func appendLoad(stmts []build.Expr, location string, from, to []string) bool {
symbolsToLoad := make(map[string]string)
for i, s := range to {
symbolsToLoad[s] = from[i]
}
var lastLoad *build.LoadStmt
for _, s := range stmts {
load, ok := s.(*build.LoadStmt)
if !ok {
continue
}
if load.Module.Value != location {
continue // Loads a different file.
}
for _, ident := range load.To {
delete(symbolsToLoad, ident.Name) // Already loaded.
}
// Remember the last insert location, but potentially remove more symbols
// that are already loaded in other subsequent calls.
lastLoad = load
}
if lastLoad == nil {
return false
}
// Append the remaining loads to the last load location.
from = []string{}
to = []string{}
for t, f := range symbolsToLoad {
from = append(from, f)
to = append(to, t)
}
AppendToLoad(lastLoad, from, to)
return true
}
// InsertLoad inserts a load statement at the top of the list of statements.
// The load statement is constructed using args. Symbols that are already loaded
// The load statement is constructed using a string location and two slices of from- and to-symbols.
// The function panics if the slices aren't of the same lentgh. Symbols that are already loaded
// from the given filepath are ignored. If stmts already contains a load for the
// location in arguments, appends the symbols to load to it.
func InsertLoad(stmts []build.Expr, args []string) []build.Expr {
if appendLoad(stmts, args) {
func InsertLoad(stmts []build.Expr, location string, from, to []string) []build.Expr {
if len(from) != len(to) {
panic(fmt.Errorf("length mismatch: %v (from) and %v (to)", len(from), len(to)))
}
if appendLoad(stmts, location, from, to) {
return stmts
}
load := newLoad(args)
load := NewLoad(location, from, to)
var all []build.Expr
added := false
for _, stmt := range stmts {
for i, stmt := range stmts {
_, isComment := stmt.(*build.CommentBlock)
if isComment || added {
_, isString := stmt.(*build.StringExpr)
isDocString := isString && i == 0
if isComment || isDocString || added {
all = append(all, stmt)
continue
}

View File

@@ -315,17 +315,17 @@ func usePlusEqual(f *build.File) bool {
if !ok || len(call.List) != 1 {
continue
}
obj, ok := dot.X.(*build.LiteralExpr)
obj, ok := dot.X.(*build.Ident)
if !ok {
continue
}
var fix *build.BinaryExpr
var fix *build.AssignExpr
if dot.Name == "extend" {
fix = &build.BinaryExpr{X: obj, Op: "+=", Y: call.List[0]}
fix = &build.AssignExpr{LHS: obj, Op: "+=", RHS: call.List[0]}
} else if dot.Name == "append" {
list := &build.ListExpr{List: []build.Expr{call.List[0]}}
fix = &build.BinaryExpr{X: obj, Op: "+=", Y: list}
fix = &build.AssignExpr{LHS: obj, Op: "+=", RHS: list}
} else {
continue
}
@@ -340,13 +340,17 @@ func isNonemptyComment(comment *build.Comments) bool {
return len(comment.Before)+len(comment.Suffix)+len(comment.After) > 0
}
// Checks whether a call or any of its arguments have a comment
func hasComment(call *build.CallExpr) bool {
if isNonemptyComment(call.Comment()) {
// Checks whether a load statement or any of its arguments have a comment
func hasComment(load *build.LoadStmt) bool {
if isNonemptyComment(load.Comment()) {
return true
}
for _, arg := range call.List {
if isNonemptyComment(arg.Comment()) {
if isNonemptyComment(load.Module.Comment()) {
return true
}
for i := range load.From {
if isNonemptyComment(load.From[i].Comment()) || isNonemptyComment(load.To[i].Comment()) {
return true
}
}
@@ -357,40 +361,35 @@ func hasComment(call *build.CallExpr) bool {
// It also cleans symbols loaded multiple times, sorts symbol list, and removes load
// statements when the list is empty.
func cleanUnusedLoads(f *build.File) bool {
// If the file needs preprocessing, leave it alone.
for _, stmt := range f.Stmt {
if _, ok := stmt.(*build.PythonBlock); ok {
return false
}
}
symbols := UsedSymbols(f)
fixed := false
var all []build.Expr
for _, stmt := range f.Stmt {
rule, ok := ExprToRule(stmt, "load")
if !ok || len(rule.Call.List) == 0 || hasComment(rule.Call) {
load, ok := stmt.(*build.LoadStmt)
if !ok || hasComment(load) {
all = append(all, stmt)
continue
}
var args []build.Expr
for _, arg := range rule.Call.List[1:] { // first argument is the path, we keep it
symbol, ok := loadedSymbol(arg)
if !ok || symbols[symbol] {
args = append(args, arg)
if ok {
// If the same symbol is loaded twice, we'll remove it.
delete(symbols, symbol)
}
var fromSymbols, toSymbols []*build.Ident
for i := range load.From {
fromSymbol := load.From[i]
toSymbol := load.To[i]
if symbols[toSymbol.Name] {
// The symbol is actually used
fromSymbols = append(fromSymbols, fromSymbol)
toSymbols = append(toSymbols, toSymbol)
// If the same symbol is loaded twice, we'll remove it.
delete(symbols, toSymbol.Name)
} else {
fixed = true
}
}
if len(args) > 0 { // Keep the load statement if it loads at least one symbol.
li := &build.ListExpr{List: args}
build.SortStringList(li)
rule.Call.List = append(rule.Call.List[:1], li.List...)
all = append(all, rule.Call)
if len(toSymbols) > 0 { // Keep the load statement if it loads at least one symbol.
sort.Sort(loadArgs{fromSymbols, toSymbols})
load.From = fromSymbols
load.To = toSymbols
all = append(all, load)
} else {
fixed = true
}
@@ -399,22 +398,6 @@ func cleanUnusedLoads(f *build.File) bool {
return fixed
}
// loadedSymbol parses the symbol token from a load statement argument,
// supporting aliases.
func loadedSymbol(arg build.Expr) (string, bool) {
symbol, ok := arg.(*build.StringExpr)
if ok {
return symbol.Value, ok
}
// try an aliased symbol
if binExpr, ok := arg.(*build.BinaryExpr); ok && binExpr.Op == "=" {
if keyExpr, ok := binExpr.X.(*build.LiteralExpr); ok {
return keyExpr.Token, ok
}
}
return "", false
}
// movePackageDeclarationToTheTop ensures that the call to package() is done
// before everything else (except comments).
func movePackageDeclarationToTheTop(f *build.File) bool {
@@ -426,9 +409,10 @@ func movePackageDeclarationToTheTop(f *build.File) bool {
inserted := false // true when the package declaration has been inserted
for _, stmt := range f.Stmt {
_, isComment := stmt.(*build.CommentBlock)
_, isBinaryExpr := stmt.(*build.BinaryExpr) // e.g. variable declaration
_, isLoad := ExprToRule(stmt, "load")
if isComment || isBinaryExpr || isLoad {
_, isString := stmt.(*build.StringExpr) // typically a docstring
_, isAssignExpr := stmt.(*build.AssignExpr) // e.g. variable declaration
_, isLoad := stmt.(*build.LoadStmt)
if isComment || isString || isAssignExpr || isLoad {
all = append(all, stmt)
continue
}
@@ -567,3 +551,24 @@ func FixFile(f *build.File, pkg string, fixes []string) *build.File {
}
return f
}
// A wrapper for a LoadStmt's From and To slices for consistent sorting of their contents.
// It's assumed that the following slices have the same length, the contents are sorted by
// the `To` attribute, the items of `From` are swapped exactly the same way as the items of `To`.
type loadArgs struct {
From []*build.Ident
To []*build.Ident
}
func (args loadArgs) Len() int {
return len(args.From)
}
func (args loadArgs) Swap(i, j int) {
args.From[i], args.From[j] = args.From[j], args.From[i]
args.To[i], args.To[j] = args.To[j], args.To[i]
}
func (args loadArgs) Less(i, j int) bool {
return args.To[i].Name < args.To[j].Name
}