From 6c925924dbf1f557def8f3207eace095640735c5 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Fri, 14 Jul 2017 13:58:36 -0700 Subject: [PATCH] protobuild: replace protobuild command To make the protobuild tool broadly useful, it has been broken out into a separate project. This PR replaces the command with a configuration file. Signed-off-by: Stephen J Day --- Makefile | 7 +- Protobuild.toml | 29 ++++ cmd/ctrd-protobuild/main.go | 260 ------------------------------------ 3 files changed, 33 insertions(+), 263 deletions(-) create mode 100644 Protobuild.toml delete mode 100644 cmd/ctrd-protobuild/main.go diff --git a/Makefile b/Makefile index 0f8d172cd..352928fd1 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ INTEGRATION_PACKAGE=${PKG} TEST_REQUIRES_ROOT_PACKAGES=$(shell for f in $$(git grep -l testutil.RequiresRoot | grep -v Makefile);do echo "${PKG}/$$(dirname $$f)"; done) # Project binaries. -COMMANDS=ctr containerd protoc-gen-gogoctrd dist ctrd-protobuild +COMMANDS=ctr containerd protoc-gen-gogoctrd dist ifneq ("$(GOOS)", "windows") COMMANDS += containerd-shim endif @@ -61,14 +61,15 @@ setup: ## install dependencies @go get -u github.com/golang/lint/golint #@go get -u github.com/kisielk/errcheck @go get -u github.com/gordonklaus/ineffassign + @go get -u github.com/stevvooe/protobuild generate: protos @echo "$(WHALE) $@" @PATH=${ROOTDIR}/bin:${PATH} go generate -x ${PACKAGES} -protos: bin/protoc-gen-gogoctrd bin/ctrd-protobuild ## generate protobuf +protos: bin/protoc-gen-gogoctrd ## generate protobuf @echo "$(WHALE) $@" - @PATH=${ROOTDIR}/bin:${PATH} ctrd-protobuild ${PACKAGES} + @PATH=${ROOTDIR}/bin:${PATH} protobuild ${PACKAGES} checkprotos: protos ## check if protobufs needs to be generated again @echo "$(WHALE) $@" diff --git a/Protobuild.toml b/Protobuild.toml new file mode 100644 index 000000000..973b9253d --- /dev/null +++ b/Protobuild.toml @@ -0,0 +1,29 @@ +version = "unstable" +generator = "gogoctrd" +plugins = ["grpc"] + +# Control protoc include paths. Below are usually some good defaults, but feel +# free to try it without them if it works for your project. +[includes] + # Include paths that will be added before all others. Typically, you want to + # treat the root of the project as an include, but this may not be necessary. + before = ["."] + + # Paths that should be treated as include roots in relation to the vendor + # directory. These will be calculated with the vendor directory nearest the + # target package. + vendored = ["github.com/gogo/protobuf"] + + # Paths that will be added untouched to the end of the includes. We use + # `/usr/local/include` to pickup the common install location of protobuf. + # This is the default. + after = ["/usr/local/include"] + +# This section let's us map protobuf imports to Go packages. These will become +# `-M` directives in the call to the go protobuf generator. +[packages] + "gogoproto/gogo.proto" = "github.com/gogo/protobuf/gogoproto" + "google/protobuf/any.proto" = "github.com/gogo/protobuf/types" + "google/protobuf/descriptor.proto" = "github.com/gogo/protobuf/protoc-gen-gogo/descriptor" + "google/protobuf/field_mask.proto" = "github.com/gogo/protobuf/types" + "google/protobuf/timestamp.proto" = "github.com/gogo/protobuf/types" diff --git a/cmd/ctrd-protobuild/main.go b/cmd/ctrd-protobuild/main.go deleted file mode 100644 index 207e0d49b..000000000 --- a/cmd/ctrd-protobuild/main.go +++ /dev/null @@ -1,260 +0,0 @@ -package main - -import ( - "bytes" - "flag" - "fmt" - "log" - "os" - "os/exec" - "path/filepath" - "strings" - "text/template" -) - -// defines several variables for parameterizing the protoc command. We can pull -// this out into a toml files in cases where we to vary this per package. -var ( - generationPlugin = "gogoctrd" - - preIncludePaths = []string{ - ".", - } - - // vendoredIncludes is used for packages that should be included as vendor - // directories. We don't differentiate between packages and projects here. - // This should just be the root of where the protos are included from. - vendoredIncludes = []string{ - "github.com/gogo/protobuf", - } - - // postIncludePaths defines untouched include paths to be added untouched - // to the protoc command. - postIncludePaths = []string{ - "/usr/local/include", // common location for protoc installation of WKTs - } - - plugins = []string{ - "grpc", - } - - // packageMap allows us to map protofile imports to specific Go packages. These - // becomes the M declarations at the end of the declaration. - packageMap = map[string]string{ - "google/protobuf/timestamp.proto": "github.com/gogo/protobuf/types", - "google/protobuf/any.proto": "github.com/gogo/protobuf/types", - "google/protobuf/field_mask.proto": "github.com/gogo/protobuf/types", - "google/protobuf/descriptor.proto": "github.com/gogo/protobuf/protoc-gen-gogo/descriptor", - "gogoproto/gogo.proto": "github.com/gogo/protobuf/gogoproto", - } - - tmpl = template.Must(template.New("protoc").Parse(`protoc -I - {{- range $index, $include := .Includes -}} - {{if $index}}:{{end -}} - {{.}} - {{- end }} -- - {{- .Name -}}_out=plugins={{- range $index, $plugin := .Plugins -}} - {{- if $index}}+{{end}} - {{- $plugin}} - {{- end -}} - ,import_path={{.ImportPath}} - {{- range $proto, $gopkg := .PackageMap -}},M - {{- $proto}}={{$gopkg -}} - {{- end -}} - :{{- .OutputDir }} - {{- range .Files}} {{.}}{{end -}} -`)) -) - -// Protoc defines inputs to a protoc command string. -type Protoc struct { - Name string // backend name - Includes []string - Plugins []string - ImportPath string - PackageMap map[string]string - Files []string - OutputDir string -} - -func (p *Protoc) mkcmd() (string, error) { - var buf bytes.Buffer - if err := tmpl.Execute(&buf, p); err != nil { - return "", err - } - - return buf.String(), nil -} - -func main() { - flag.Parse() - - pkgInfos, err := goPkgInfo(flag.Args()...) - if err != nil { - log.Fatalln(err) - } - - gopath, err := gopathSrc() - if err != nil { - log.Fatalln(err) - } - - gopathCurrent, err := gopathCurrent() - if err != nil { - log.Fatalln(err) - } - - // For some reason, the golang protobuf generator makes the god awful - // decision to output the files relative to the gopath root. It doesn't do - // this only in the case where you give it ".". - outputDir := filepath.Join(gopathCurrent, "src") - - for _, pkg := range pkgInfos { - var includes []string - includes = append(includes, preIncludePaths...) - - vendor, err := closestVendorDir(pkg.Dir) - if err != nil { - log.Fatalln(err) - } - - // we also special case the inclusion of gogoproto in the vendor dir. - // We could parameterize this better if we find it to be a common case. - var vendoredIncludesResolved []string - for _, vendoredInclude := range vendoredIncludes { - vendoredIncludesResolved = append(vendoredIncludesResolved, - filepath.Join(vendor, vendoredInclude)) - } - - includes = append(includes, vendoredIncludesResolved...) - includes = append(includes, vendor, gopath) - includes = append(includes, postIncludePaths...) - - protoc := Protoc{ - Name: generationPlugin, - ImportPath: pkg.GoImportPath, - PackageMap: packageMap, - Plugins: plugins, - Files: pkg.ProtoFiles, - OutputDir: outputDir, - Includes: includes, - } - - arg, err := protoc.mkcmd() - if err != nil { - log.Fatalln(err) - } - - fmt.Println(arg) - - // pass to sh -c so we don't need to re-split here. - args := []string{"-c", arg} - cmd := exec.Command("sh", args...) - out, err := cmd.CombinedOutput() - if err != nil { - log.Fatalf("%s %s\n", out, err) - } - } -} - -type protoGoPkgInfo struct { - Dir string - GoImportPath string - ProtoFiles []string -} - -func goPkgInfo(golistpath ...string) ([]protoGoPkgInfo, error) { - args := []string{ - "list", "-e", "-f", "{{.ImportPath}} {{.Dir}}"} - args = append(args, golistpath...) - cmd := exec.Command("go", args...) - - p, err := cmd.Output() - if err != nil { - return nil, err - } - - var pkgInfos []protoGoPkgInfo - lines := bytes.Split(p, []byte("\n")) - for _, line := range lines { - if len(line) == 0 { - continue - } - parts := bytes.Fields(line) - if len(parts) != 2 { - return nil, fmt.Errorf("bad output from command: %s", p) - } - - pkgInfo := protoGoPkgInfo{ - Dir: string(parts[1]), - GoImportPath: string(parts[0]), - } - - protoFiles, err := filepath.Glob(filepath.Join(pkgInfo.Dir, "*.proto")) - if err != nil { - return nil, err - } - if len(protoFiles) == 0 { - continue // not a proto directory, skip - } - - pkgInfo.ProtoFiles = protoFiles - pkgInfos = append(pkgInfos, pkgInfo) - } - - return pkgInfos, nil -} - -// gopathSrc modifies GOPATH elements from env to include the src directory. -func gopathSrc() (string, error) { - gopathAll := os.Getenv("GOPATH") - - if gopathAll == "" { - return "", fmt.Errorf("must be run from a gopath") - } - - var elements []string - for _, element := range strings.Split(gopathAll, ":") { // TODO(stevvooe): Make this work on windows. - elements = append(elements, filepath.Join(element, "src")) - } - - return strings.Join(elements, ":"), nil -} - -// gopathCurrent provides the top-level gopath for the current generation. -func gopathCurrent() (string, error) { - gopathAll := os.Getenv("GOPATH") - - if gopathAll == "" { - return "", fmt.Errorf("must be run from a gopath") - } - - return strings.Split(gopathAll, ":")[0], nil -} - -// closestVendorDir walks up from dir until it finds the vendor directory. -func closestVendorDir(dir string) (string, error) { - dir = filepath.Clean(dir) - for dir != "" && dir != string(filepath.Separator) { // TODO(stevvooe): May not work on windows - vendor := filepath.Join(dir, "vendor") - fi, err := os.Stat(vendor) - if err != nil { - if os.IsNotExist(err) { - // up we go! - dir = filepath.Dir(dir) - continue - } - return "", err - } - - if !fi.IsDir() { - // up we go! - dir = filepath.Dir(dir) - continue - } - - return vendor, nil - } - - return "", fmt.Errorf("no vendor dir found") -}