Update release tool

Allow inclusion of sub-project changes
Order contributors by number of contributions
Add mailmap

Signed-off-by: Derek McGowan <derek@mcgstyle.net>

.mailmap

@@ -0,0 +1,17 @@
+Abhinandan Prativadi <abhi@docker.com> Abhinandan Prativadi <aprativadi@gmail.com>
+Abhinandan Prativadi <abhi@docker.com> abhi <abhi@docker.com>
+Akihiro Suda <suda.akihiro@lab.ntt.co.jp> Akihiro Suda <suda.kyoto@gmail.com>
+Andrei Vagin <avagin@virtuozzo.com> Andrei Vagin <avagin@openvz.org>
+Frank Yang <yyb196@gmail.com> frank yang <yyb196@gmail.com>
+Justin Terry <juterry@microsoft.com> Justin Terry (VM) <juterry@microsoft.com>
+Justin Terry <juterry@microsoft.com> Justin <jterry75@users.noreply.github.com>
+Kenfe-Mickaël Laventure <mickael.laventure@gmail.com> Kenfe-Mickael Laventure <mickael.laventure@gmail.com>
+Kevin Xu <cming.xu@gmail.com> kevin.xu <cming.xu@gmail.com>
+Lu Jingxiao <lujingxiao@huawei.com> l00397676 <lujingxiao@huawei.com>
+Lantao Liu <lantaol@google.com> Lantao Liu <taotaotheripper@gmail.com>
+Phil Estes <estesp@gmail.com> Phil Estes <estesp@linux.vnet.ibm.com>
+Stephen J Day <stevvooe@gmail.com> Stephen J Day <stephen.day@docker.com>
+Stephen J Day <stevvooe@gmail.com> Stephen Day <stevvooe@users.noreply.github.com>
+Stephen J Day <stevvooe@gmail.com> Stephen Day <stephen.day@getcruise.com>
+Sudeesh John <sudeesh@linux.vnet.ibm.com> sudeesh john <sudeesh@linux.vnet.ibm.com>
+Tõnis Tiigi <tonistiigi@gmail.com> Tonis Tiigi <tonistiigi@gmail.com>

cmd/containerd-release/main.go

@@ -18,11 +18,16 @@ package main
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
+	"path"
+	"path/filepath"
+	"regexp"
 	"sort"
 	"text/tabwriter"
 	"text/template"
 
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
@@ -43,6 +48,7 @@ type dependency struct {
 	Name     string
 	Commit   string
 	Previous string
+	CloneURL string
 }
 
 type download struct {
@@ -50,6 +56,16 @@ type download struct {
 	Hash     string
 }
 
+type projectChange struct {
+	Name    string
+	Changes []change
+}
+
+type projectRename struct {
+	Old string `toml:"old"`
+	New string `toml:"new"`
+}
+
 type release struct {
 	ProjectName     string            `toml:"project_name"`
 	GithubRepo      string            `toml:"github_repo"`
@@ -59,8 +75,13 @@ type release struct {
 	Preface         string            `toml:"preface"`
 	Notes           map[string]note   `toml:"notes"`
 	BreakingChanges map[string]change `toml:"breaking"`
+
+	// dependency options
+	MatchDeps  string                   `toml:"match_deps"`
+	RenameDeps map[string]projectRename `toml:"rename_deps"`
+
 	// generated fields
-	Changes      []change
+	Changes      []projectChange
 	Contributors []string
 	Dependencies []dependency
 	Version      string
@@ -79,6 +100,10 @@ This tool should be ran from the root of the project repository for a new releas
 			Name:  "dry,n",
 			Usage: "run the release tooling as a dry run to print the release notes to stdout",
 		},
+		cli.BoolFlag{
+			Name:  "debug,d",
+			Usage: "show debug output",
+		},
 		cli.StringFlag{
 			Name:  "template,t",
 			Usage: "template filepath to use in place of the default",
@@ -87,45 +112,120 @@ This tool should be ran from the root of the project repository for a new releas
 	}
 	app.Action = func(context *cli.Context) error {
 		var (
-			path = context.Args().First()
-			tag  = parseTag(path)
+			releasePath = context.Args().First()
+			tag         = parseTag(releasePath)
 		)
-		r, err := loadRelease(path)
+		if context.Bool("debug") {
+			logrus.SetLevel(logrus.DebugLevel)
+		}
+		r, err := loadRelease(releasePath)
 		if err != nil {
 			return err
 		}
 		logrus.Infof("Welcome to the %s release tool...", r.ProjectName)
-		previous, err := getPreviousDeps(r.Previous)
+
+		mailmapPath, err := filepath.Abs(".mailmap")
 		if err != nil {
-			return err
+			return errors.Wrap(err, "failed to resolve mailmap")
 		}
+		gitConfigs["mailmap.file"] = mailmapPath
+
+		var (
+			contributors   = map[contributor]int{}
+			projectChanges = []projectChange{}
+		)
+
 		changes, err := changelog(r.Previous, r.Commit)
 		if err != nil {
 			return err
 		}
+		if err := addContributors(r.Previous, r.Commit, contributors); err != nil {
+			return err
+		}
+		projectChanges = append(projectChanges, projectChange{
+			Name:    "",
+			Changes: changes,
+		})
+
 		logrus.Infof("creating new release %s with %d new changes...", tag, len(changes))
 		rd, err := fileFromRev(r.Commit, vendorConf)
 		if err != nil {
 			return err
 		}
+		previous, err := getPreviousDeps(r.Previous)
+		if err != nil {
+			return err
+		}
 		deps, err := parseDependencies(rd)
 		if err != nil {
 			return err
 		}
+		renameDependencies(previous, r.RenameDeps)
 		updatedDeps := updatedDeps(previous, deps)
-		contributors, err := getContributors(r.Previous, r.Commit)
-		if err != nil {
-			return err
-		}
 
 		sort.Slice(updatedDeps, func(i, j int) bool {
 			return updatedDeps[i].Name < updatedDeps[j].Name
 		})
 
+		if r.MatchDeps != "" && len(updatedDeps) > 0 {
+			re, err := regexp.Compile(r.MatchDeps)
+			if err != nil {
+				return errors.Wrap(err, "unable to compile 'match_deps' regexp")
+			}
+			td, err := ioutil.TempDir("", "tmp-clone-")
+			if err != nil {
+				return errors.Wrap(err, "unable to create temp clone directory")
+			}
+			defer os.RemoveAll(td)
+
+			cwd, err := os.Getwd()
+			if err != nil {
+				return errors.Wrap(err, "unable to get cwd")
+			}
+			for _, dep := range updatedDeps {
+				matches := re.FindStringSubmatch(dep.Name)
+				if matches == nil {
+					continue
+				}
+				logrus.Debugf("Matched dependency %s with %s", dep.Name, r.MatchDeps)
+				var name string
+				if len(matches) < 2 {
+					name = path.Base(dep.Name)
+				} else {
+					name = matches[1]
+				}
+				if err := os.Chdir(td); err != nil {
+					return errors.Wrap(err, "unable to chdir to temp clone directory")
+				}
+				git("clone", dep.CloneURL, name)
+
+				if err := os.Chdir(name); err != nil {
+					return errors.Wrapf(err, "unable to chdir to cloned %s directory", name)
+				}
+
+				changes, err := changelog(dep.Previous, dep.Commit)
+				if err != nil {
+					return errors.Wrapf(err, "failed to get changelog for %s", name)
+				}
+				if err := addContributors(dep.Previous, dep.Commit, contributors); err != nil {
+					return errors.Wrapf(err, "failed to get authors for %s", name)
+				}
+
+				projectChanges = append(projectChanges, projectChange{
+					Name:    name,
+					Changes: changes,
+				})
+
+			}
+			if err := os.Chdir(cwd); err != nil {
+				return errors.Wrap(err, "unable to chdir to previous cwd")
+			}
+		}
+
 		// update the release fields with generated data
-		r.Contributors = contributors
+		r.Contributors = orderContributors(contributors)
 		r.Dependencies = updatedDeps
-		r.Changes = changes
+		r.Changes = projectChanges
 		r.Version = tag
 
 		tmpl, err := getTemplate(context)

cmd/containerd-release/template.go

@@ -28,7 +28,8 @@ const (
 Please try out the release binaries and report any issues at
 https://github.com/{{.GithubRepo}}/issues.
 
-{{range  $note := .Notes}}
+{{- range  $note := .Notes}}
+
 ### {{$note.Title}}
 
 {{$note.Description}}
@@ -37,12 +38,15 @@ https://github.com/{{.GithubRepo}}/issues.
 ### Contributors
 {{range $contributor := .Contributors}}
 * {{$contributor}}
-{{- end}}
+{{- end -}}
 
-### Changes
-{{range $change := .Changes}}
+{{range $project := .Changes}}
+
+### Changes{{if $project.Name}} from {{$project.Name}}{{end}}
+{{range $change := $project.Changes }}
 * {{$change.Commit}} {{$change.Description}}
 {{- end}}
+{{- end}}
 
 ### Dependency Changes
 

cmd/containerd-release/util.go

@@ -19,7 +19,6 @@ package main
 import (
 	"bufio"
 	"bytes"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -30,6 +29,8 @@ import (
 	"strings"
 
 	"github.com/BurntSushi/toml"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
@@ -65,9 +66,18 @@ func parseDependencies(r io.Reader) ([]dependency, error) {
 		if len(parts) != 2 && len(parts) != 3 {
 			return nil, fmt.Errorf("invalid config format: %s", ln)
 		}
+
+		var cloneURL string
+		if len(parts) == 3 {
+			cloneURL = parts[2]
+		} else {
+			cloneURL = "git://" + parts[0]
+		}
+
 		deps = append(deps, dependency{
-			Name:   parts[0],
-			Commit: parts[1],
+			Name:     parts[0],
+			Commit:   parts[1],
+			CloneURL: cloneURL,
 		})
 	}
 	if err := s.Err(); err != nil {
@@ -92,8 +102,15 @@ func changelog(previous, commit string) ([]change, error) {
 	return parseChangelog(raw)
 }
 
+func gitChangeDiff(previous, commit string) string {
+	if previous != "" {
+		return fmt.Sprintf("%s..%s", previous, commit)
+	}
+	return commit
+}
+
 func getChangelog(previous, commit string) ([]byte, error) {
-	return git("log", "--oneline", fmt.Sprintf("%s..%s", previous, commit))
+	return git("log", "--oneline", gitChangeDiff(previous, commit))
 }
 
 func parseChangelog(changelog []byte) ([]change, error) {
@@ -123,14 +140,44 @@ func fileFromRev(rev, file string) (io.Reader, error) {
 	return bytes.NewReader(p), nil
 }
 
+var gitConfigs = map[string]string{}
+
 func git(args ...string) ([]byte, error) {
-	o, err := exec.Command("git", args...).CombinedOutput()
+	var gitArgs []string
+	for k, v := range gitConfigs {
+		gitArgs = append(gitArgs, "-c", fmt.Sprintf("%s=%s", k, v))
+	}
+	gitArgs = append(gitArgs, args...)
+	o, err := exec.Command("git", gitArgs...).CombinedOutput()
 	if err != nil {
 		return nil, fmt.Errorf("%s: %s", err, o)
 	}
 	return o, nil
 }
 
+func renameDependencies(deps []dependency, renames map[string]projectRename) {
+	if len(renames) == 0 {
+		return
+	}
+	type dep struct {
+		shortname string
+		name      string
+	}
+	renameMap := map[string]dep{}
+	for shortname, rename := range renames {
+		renameMap[rename.Old] = dep{
+			shortname: shortname,
+			name:      rename.New,
+		}
+	}
+	for i := range deps {
+		if updated, ok := renameMap[deps[i].Name]; ok {
+			logrus.Debugf("Renamed %s from %s to %s", updated.shortname, deps[i].Name, updated.name)
+			deps[i].Name = updated.name
+		}
+	}
+}
+
 func updatedDeps(previous, deps []dependency) []dependency {
 	var updated []dependency
 	pm, cm := toDepMap(previous), toDepMap(deps)
@@ -159,27 +206,61 @@ func toDepMap(deps []dependency) map[string]dependency {
 	return out
 }
 
-func getContributors(previous, commit string) ([]string, error) {
-	raw, err := git("log", "--format=%aN", fmt.Sprintf("%s..%s", previous, commit))
+type contributor struct {
+	name  string
+	email string
+}
+
+func addContributors(previous, commit string, contributors map[contributor]int) error {
+	raw, err := git("log", `--format=%aE %aN`, gitChangeDiff(previous, commit))
 	if err != nil {
-		return nil, err
+		return err
 	}
-	var (
-		set = make(map[string]struct{})
-		s   = bufio.NewScanner(bytes.NewReader(raw))
-		out []string
-	)
+	s := bufio.NewScanner(bytes.NewReader(raw))
 	for s.Scan() {
-		set[s.Text()] = struct{}{}
+		p := strings.SplitN(s.Text(), " ", 2)
+		if len(p) != 2 {
+			return errors.Errorf("invalid author line: %q", s.Text())
+		}
+		c := contributor{
+			name:  p[1],
+			email: p[0],
+		}
+		contributors[c] = contributors[c] + 1
 	}
 	if err := s.Err(); err != nil {
-		return nil, err
+		return err
 	}
-	for name := range set {
-		out = append(out, name)
+	return nil
+}
+
+func orderContributors(contributors map[contributor]int) []string {
+	type contribstat struct {
+		name  string
+		email string
+		count int
 	}
-	sort.Strings(out)
-	return out, nil
+	all := make([]contribstat, 0, len(contributors))
+	for c, count := range contributors {
+		all = append(all, contribstat{
+			name:  c.name,
+			email: c.email,
+			count: count,
+		})
+	}
+	sort.Slice(all, func(i, j int) bool {
+		if all[i].count == all[j].count {
+			return all[i].name < all[j].name
+		}
+		return all[i].count > all[j].count
+	})
+	names := make([]string, len(all))
+	for i := range names {
+		logrus.Debugf("Contributor: %s <%s> with %d commits", all[i].name, all[i].email, all[i].count)
+		names[i] = all[i].name
+	}
+
+	return names
 }
 
 // getTemplate will use a builtin template if the template is not specified on the cli