diff --git a/tools/goctl/env/check.go b/tools/goctl/env/check.go index c6e21d76..044e39e8 100644 --- a/tools/goctl/env/check.go +++ b/tools/goctl/env/check.go @@ -40,10 +40,10 @@ var bins = []bin{ func Check(ctx *cli.Context) error { install := ctx.Bool("install") force := ctx.Bool("force") - return check(install, force) + return Prepare(install, force) } -func check(install, force bool) error { +func Prepare(install, force bool) error { pending := true console.Info("[goctl-env]: preparing to check env") defer func() { @@ -56,8 +56,8 @@ func check(install, force bool) error { } else { console.Error(` [goctl-env]: check env finish, some dependencies is not found in PATH, you can execute -command 'goctl env check --install' or 'goctl env install' to install it, for details, -please see 'goctl env check --help' or 'goctl env install --help'`) +command 'goctl env check --install' to install it, for details, please execute command +'goctl env check --help'`) } }() for _, e := range bins { diff --git a/tools/goctl/goctl.go b/tools/goctl/goctl.go index c5cefa0c..a9bacf8e 100644 --- a/tools/goctl/goctl.go +++ b/tools/goctl/goctl.go @@ -542,22 +542,30 @@ var commands = []cli.Command{ Description: "for details, see https://go-zero.dev/cn/goctl-rpc.html", Action: rpc.ZRPC, Flags: []cli.Flag{ - cli.StringFlag{ + cli.StringSliceFlag{ Name: "go_out", Hidden: true, }, - cli.StringFlag{ + cli.StringSliceFlag{ Name: "go-grpc_out", Hidden: true, }, - cli.StringFlag{ + cli.StringSliceFlag{ Name: "go_opt", Hidden: true, }, - cli.StringFlag{ + cli.StringSliceFlag{ Name: "go-grpc_opt", Hidden: true, }, + cli.StringSliceFlag{ + Name: "plugin", + Hidden: true, + }, + cli.StringSliceFlag{ + Name: "proto_path,I", + Hidden: true, + }, cli.StringFlag{ Name: "zrpc_out", Usage: "the zrpc output directory", diff --git a/tools/goctl/pkg/golang/bin.go b/tools/goctl/pkg/golang/bin.go index c5278682..672f3b97 100644 --- a/tools/goctl/pkg/golang/bin.go +++ b/tools/goctl/pkg/golang/bin.go @@ -11,10 +11,10 @@ import ( // GoBin returns a path of GOBIN. func GoBin() string { def := build.Default - goroot := os.Getenv("GOROOT") + goroot := os.Getenv("GOPATH") bin := filepath.Join(goroot, "bin") if !pathx.FileExists(bin) { - gopath := os.Getenv("GOPATH") + gopath := os.Getenv("GOROOT") bin = filepath.Join(gopath, "bin") } if !pathx.FileExists(bin) { diff --git a/tools/goctl/pkg/protocgengo/protocgengo.go b/tools/goctl/pkg/protocgengo/protocgengo.go index 23420834..eaf1a67a 100644 --- a/tools/goctl/pkg/protocgengo/protocgengo.go +++ b/tools/goctl/pkg/protocgengo/protocgengo.go @@ -23,8 +23,11 @@ func Install(cacheDir string) (string, error) { } func Exists() bool { - _, err := env.LookUpProtocGenGo() - return err == nil + ver, err := Version() + if err != nil { + return false + } + return len(ver) > 0 } // Version is used to get the version of the protoc-gen-go plugin. For older versions, protoc-gen-go does not support diff --git a/tools/goctl/rpc/cli/zrpc.go b/tools/goctl/rpc/cli/zrpc.go index 19c10a46..361bf0d3 100644 --- a/tools/goctl/rpc/cli/zrpc.go +++ b/tools/goctl/rpc/cli/zrpc.go @@ -2,12 +2,10 @@ package cli import ( "errors" - "fmt" "os" "path/filepath" "strings" - "github.com/emicklei/proto" "github.com/urfave/cli" "github.com/zeromicro/go-zero/tools/goctl/rpc/generator" "github.com/zeromicro/go-zero/tools/goctl/util" @@ -15,7 +13,8 @@ import ( ) var ( - errInvalidGrpcOutput = errors.New("ZRPC: missing grpc output") + errInvalidGrpcOutput = errors.New("ZRPC: missing --go-grpc_out") + errInvalidGoOutput = errors.New("ZRPC: missing --go_out") errInvalidZrpcOutput = errors.New("ZRPC: missing zrpc output, please use --zrpc_out to specify the output") errInvalidInput = errors.New("ZRPC: missing source") errMultiInput = errors.New("ZRPC: only one source is expected") @@ -40,21 +39,29 @@ func ZRPC(c *cli.Context) error { if err != nil { return err } - src := filepath.Dir(source) - goPackage, protoPkg, err := getGoPackage(source) - if err != nil { - return err - } - grpcOut := c.String("go-grpc_out") - goOut := c.String("go_out") - goOpt := c.String("go_opt") - grpcOpt := c.String("go-grpc_opt") + grpcOutList := c.StringSlice("go-grpc_out") + goOutList := c.StringSlice("go_out") zrpcOut := c.String("zrpc_out") style := c.String("style") home := c.String("home") remote := c.String("remote") branch := c.String("branch") + if len(grpcOutList) == 0 { + return errInvalidGrpcOutput + } + if len(goOutList) == 0 { + return errInvalidGoOutput + } + goOut := goOutList[len(goOutList)-1] + grpcOut := grpcOutList[len(grpcOutList)-1] + if len(goOut) == 0 { + return errInvalidGrpcOutput + } + if len(zrpcOut) == 0 { + return errInvalidZrpcOutput + } + if len(remote) > 0 { repo, _ := util.CloneIntoGitHome(remote, branch) if len(repo) > 0 { @@ -65,38 +72,11 @@ func ZRPC(c *cli.Context) error { if len(home) > 0 { pathx.RegisterGoctlHome(home) } - if len(goOut) == 0 { - return errInvalidGrpcOutput - } - if len(zrpcOut) == 0 { - return errInvalidZrpcOutput - } if !filepath.IsAbs(zrpcOut) { zrpcOut = filepath.Join(pwd, zrpcOut) } - goOut = removePluginFlag(goOut) - goOut, err = parseOut(src, goOut, goOpt, goPackage) - if err != nil { - return err - } - isGooglePlugin := len(grpcOut) > 0 - // If grpcOut is not empty means that user generates grpc code by - // https://google.golang.org/protobuf/cmd/protoc-gen-go and - // https://google.golang.org/grpc/cmd/protoc-gen-go-grpc, - // for details please see https://grpc.io/docs/languages/go/quickstart/ - if isGooglePlugin { - grpcOut, err = parseOut(src, grpcOut, grpcOpt, goPackage) - if err != nil { - return err - } - } else { - // Else it means that user generates grpc code by - // https://github.com/golang/protobuf/tree/master/protoc-gen-go - grpcOut = goOut - } - goOut, err = filepath.Abs(goOut) if err != nil { return err @@ -110,23 +90,11 @@ func ZRPC(c *cli.Context) error { return err } - if isGooglePlugin && grpcOut != goOut { - return fmt.Errorf("the --go_out and --go-grpc_out must be the same") - } - - if goOut == zrpcOut || grpcOut == zrpcOut { - recommendName := goPackage - if len(recommendName) == 0 { - recommendName = protoPkg - } - return fmt.Errorf("the zrpc and grpc output can not be the same, it is recommended to output grpc to the %q", - filepath.Join(goOut, recommendName)) - } - var ctx generator.ZRpcContext ctx.Src = source - ctx.ProtoGenGoDir = goOut - ctx.ProtoGenGrpcDir = grpcOut + ctx.GoOutput = goOut + ctx.GrpcOutput = grpcOut + ctx.IsGooglePlugin = isGooglePlugin ctx.Output = zrpcOut ctx.ProtocCmd = strings.Join(protocArgs, " ") g, err := generator.NewDefaultRPCGenerator(style, generator.WithZRpcContext(&ctx)) @@ -137,52 +105,6 @@ func ZRPC(c *cli.Context) error { return g.Generate(source, zrpcOut, nil) } -// parseOut calculates the output place to grpc code, about to calculate logic for details -// please see https://developers.google.com/protocol-buffers/docs/reference/go-generated#invocation. -func parseOut(sourceDir, grpcOut, grpcOpt, goPackage string) (string, error) { - if !filepath.IsAbs(grpcOut) { - grpcOut = filepath.Join(sourceDir, grpcOut) - } - switch grpcOpt { - case "", optImport: - grpcOut = filepath.Join(grpcOut, goPackage) - case optSourceRelative: - grpcOut = filepath.Join(grpcOut) - default: - return "", fmt.Errorf("parseAndSetGrpcOut: unknown path type %q: want %q or %q", - grpcOpt, optImport, optSourceRelative) - } - - return grpcOut, nil -} - -func getGoPackage(source string) (string, string, error) { - r, err := os.Open(source) - if err != nil { - return "", "", err - } - defer func() { - _ = r.Close() - }() - - parser := proto.NewParser(r) - set, err := parser.Parse() - if err != nil { - return "", "", err - } - - var goPackage, protoPkg string - proto.Walk(set, proto.WithOption(func(option *proto.Option) { - if option.Name == "go_package" { - goPackage = option.Constant.Source - } - }), proto.WithPackage(func(p *proto.Package) { - protoPkg = p.Name - })) - - return goPackage, protoPkg, nil -} - func removeGoctlFlag(args []string) []string { var ret []string var step int diff --git a/tools/goctl/rpc/cli/zrpc_test.go b/tools/goctl/rpc/cli/zrpc_test.go index 54ce6ff5..051556ad 100644 --- a/tools/goctl/rpc/cli/zrpc_test.go +++ b/tools/goctl/rpc/cli/zrpc_test.go @@ -79,6 +79,10 @@ func Test_RemoveGoctlFlag(t *testing.T) { source: strings.Fields(`protoc foo.proto --go_opt=. --zrpc_out="bar" --style=goZero --home=bar`), expected: "protoc foo.proto --go_opt=.", }, + { + source: strings.Fields(`protoc --go_opt=. --go-grpc_out=. --zrpc_out=. foo.proto`), + expected: "protoc --go_opt=. --go-grpc_out=. foo.proto", + }, } for _, e := range testData { cmd := strings.Join(removeGoctlFlag(e.source), " ") diff --git a/tools/goctl/rpc/generator/defaultgenerator.go b/tools/goctl/rpc/generator/defaultgenerator.go index c9979369..8d20ddfa 100644 --- a/tools/goctl/rpc/generator/defaultgenerator.go +++ b/tools/goctl/rpc/generator/defaultgenerator.go @@ -1,8 +1,7 @@ package generator import ( - "os/exec" - + "github.com/zeromicro/go-zero/tools/goctl/env" "github.com/zeromicro/go-zero/tools/goctl/util/console" ) @@ -25,17 +24,5 @@ func NewDefaultGenerator() Generator { // Prepare provides environment detection generated by rpc service, // including go environment, protoc, whether protoc-gen-go is installed or not func (g *DefaultGenerator) Prepare() error { - _, err := exec.LookPath("go") - if err != nil { - return err - } - - _, err = exec.LookPath("protoc") - if err != nil { - return err - } - - _, err = exec.LookPath("protoc-gen-go") - - return err + return env.Prepare(true, true) } diff --git a/tools/goctl/rpc/generator/gen.go b/tools/goctl/rpc/generator/gen.go index acb05967..2d6a4683 100644 --- a/tools/goctl/rpc/generator/gen.go +++ b/tools/goctl/rpc/generator/gen.go @@ -24,6 +24,9 @@ type ZRpcContext struct { ProtocCmd string ProtoGenGrpcDir string ProtoGenGoDir string + IsGooglePlugin bool + GoOutput string + GrpcOutput string Output string } diff --git a/tools/goctl/rpc/generator/genpb.go b/tools/goctl/rpc/generator/genpb.go index 7c58ef11..8e09b538 100644 --- a/tools/goctl/rpc/generator/genpb.go +++ b/tools/goctl/rpc/generator/genpb.go @@ -3,6 +3,8 @@ package generator import ( "bytes" "errors" + "fmt" + "io/fs" "os" "path/filepath" "strings" @@ -19,7 +21,7 @@ const googleProtocGenGoErr = `--go_out: protoc-gen-go: plugins are not supported // but the commands and flags in protoc are not completely joined in goctl. At present, proto_path(-I) is introduced func (g *DefaultGenerator) GenPb(ctx DirContext, protoImportPath []string, proto parser.Proto, _ *conf.Config, c *ZRpcContext, goOptions ...string) error { if c != nil { - return g.genPbDirect(c) + return g.genPbDirect(ctx, c) } // deprecated: use genPbDirect instead. @@ -110,13 +112,74 @@ go get -u github.com/golang/protobuf/protoc-gen-go`) return nil } -func (g *DefaultGenerator) genPbDirect(c *ZRpcContext) error { - g.log.Debug(c.ProtocCmd) +func (g *DefaultGenerator) genPbDirect(ctx DirContext, c *ZRpcContext) error { + g.log.Debug("[command]: %s", c.ProtocCmd) pwd, err := os.Getwd() if err != nil { return err } _, err = execx.Run(c.ProtocCmd, pwd) - return err + if err != nil { + return err + } + return g.setPbDir(ctx, c) +} + +func (g *DefaultGenerator) setPbDir(ctx DirContext, c *ZRpcContext) error { + pbDir, err := findPbFile(c.GoOutput, false) + if err != nil { + return err + } + if len(pbDir) == 0 { + return fmt.Errorf("pg.go is not found under %q", c.GoOutput) + } + grpcDir, err := findPbFile(c.GrpcOutput, true) + if err != nil { + return err + } + if len(grpcDir) == 0 { + return fmt.Errorf("_grpc.pb.go is not found in %q", c.GrpcOutput) + } + if pbDir != grpcDir { + return fmt.Errorf("the pb.go and _grpc.pb.go must under the same dir: "+ + "\n pb.go: %s\n_grpc.pb.go: %s", pbDir, grpcDir) + } + if pbDir == c.Output { + return fmt.Errorf("the output of pb.go and _grpc.pb.go must not be the same "+ + "with --zrpc_out:\npb output: %s\nzrpc out: %s", pbDir, c.Output) + } + ctx.SetPbDir(pbDir, grpcDir) + return nil +} + +const ( + pbSuffix = "pb.go" + grpcSuffix = "_grpc.pb.go" +) + +func findPbFile(current string, grpc bool) (string, error) { + fileSystem := os.DirFS(current) + var ret string + err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error { + if d.IsDir() { + return nil + } + if strings.HasSuffix(path, pbSuffix) { + if grpc { + if strings.HasSuffix(path, grpcSuffix) { + ret = path + return os.ErrExist + } + } else if !strings.HasSuffix(path, grpcSuffix) { + ret = path + return os.ErrExist + } + } + return nil + }) + if err == os.ErrExist { + return filepath.Dir(filepath.Join(current, ret)), nil + } + return "", err } diff --git a/tools/goctl/rpc/generator/genpb_test.go b/tools/goctl/rpc/generator/genpb_test.go new file mode 100644 index 00000000..708853b4 --- /dev/null +++ b/tools/goctl/rpc/generator/genpb_test.go @@ -0,0 +1,197 @@ +package generator + +import ( + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/zeromicro/go-zero/tools/goctl/util/pathx" +) + +func Test_findPbFile(t *testing.T) { + dir := t.TempDir() + protoFile := filepath.Join(dir, "greet.proto") + err := ioutil.WriteFile(protoFile, []byte(` +syntax = "proto3"; + +package greet; +option go_package="./greet"; + +message Req{} +message Resp{} +service Greeter { + rpc greet(Req) returns (Resp); +} +`), 0666) + if err != nil { + t.Log(err) + return + } + t.Run("", func(t *testing.T) { + output := t.TempDir() + grpc := filepath.Join(output, "grpc") + err := pathx.MkdirIfNotExist(grpc) + if err != nil { + t.Log(err) + return + } + cmd := exec.Command("protoc", "-I="+filepath.Dir(protoFile), "--go_out="+output, "--go-grpc_out="+grpc, filepath.Base(protoFile)) + cmd.Dir = output + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Run() + if err != nil { + t.Log(err) + return + } + pbDir, err := findPbFile(output, false) + assert.Nil(t, err) + pbGo := filepath.Join(pbDir, "greet.pb.go") + assert.True(t, pathx.FileExists(pbGo)) + + grpcDir, err := findPbFile(output, true) + assert.Nil(t, err) + grpcGo := filepath.Join(grpcDir, "greet_grpc.pb.go") + assert.True(t, pathx.FileExists(grpcGo)) + }) + + t.Run("", func(t *testing.T) { + output := t.TempDir() + redirect := filepath.Join(output, "pb") + grpc := filepath.Join(output, "grpc") + err := pathx.MkdirIfNotExist(grpc) + if err != nil { + t.Log(err) + return + } + cmd := exec.Command("protoc", "-I="+filepath.Dir(protoFile), "--go_out="+output, + "--go-grpc_out="+grpc, filepath.Base(protoFile), "--go_opt=M"+filepath.Base(protoFile)+"="+redirect) + cmd.Dir = output + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Run() + if err != nil { + t.Log(err) + return + } + pbDir, err := findPbFile(output, false) + assert.Nil(t, err) + pbGo := filepath.Join(pbDir, "greet.pb.go") + assert.True(t, pathx.FileExists(pbGo)) + + grpcDir, err := findPbFile(output, true) + assert.Nil(t, err) + grpcGo := filepath.Join(grpcDir, "greet_grpc.pb.go") + assert.True(t, pathx.FileExists(grpcGo)) + }) + + t.Run("", func(t *testing.T) { + output := t.TempDir() + pbeRedirect := filepath.Join(output, "redirect") + grpc := filepath.Join(output, "grpc") + grpcRedirect := filepath.Join(grpc, "redirect") + err := pathx.MkdirIfNotExist(grpc) + if err != nil { + t.Log(err) + return + } + cmd := exec.Command("protoc", "-I="+filepath.Dir(protoFile), "--go_out="+output, + "--go-grpc_out="+grpc, filepath.Base(protoFile), "--go_opt=M"+filepath.Base(protoFile)+"="+pbeRedirect, + "--go-grpc_opt=M"+filepath.Base(protoFile)+"="+grpcRedirect) + cmd.Dir = output + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Run() + if err != nil { + t.Log(err) + return + } + pbDir, err := findPbFile(output, false) + assert.Nil(t, err) + pbGo := filepath.Join(pbDir, "greet.pb.go") + assert.True(t, pathx.FileExists(pbGo)) + + grpcDir, err := findPbFile(output, true) + assert.Nil(t, err) + grpcGo := filepath.Join(grpcDir, "greet_grpc.pb.go") + assert.True(t, pathx.FileExists(grpcGo)) + }) + + t.Run("", func(t *testing.T) { + output := t.TempDir() + pbeRedirect := filepath.Join(output, "redirect") + grpc := filepath.Join(output, "grpc") + grpcRedirect := filepath.Join(grpc, "redirect") + err := pathx.MkdirIfNotExist(grpc) + if err != nil { + t.Log(err) + return + } + cmd := exec.Command("protoc", "-I="+filepath.Dir(protoFile), "--go_out="+output, + "--go-grpc_out="+grpc, filepath.Base(protoFile), "--go_opt=M"+filepath.Base(protoFile)+"="+pbeRedirect, + "--go-grpc_opt=M"+filepath.Base(protoFile)+"="+grpcRedirect, "--go_opt=paths=import", "--go-grpc_opt=paths=source_relative") + cmd.Dir = output + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Run() + if err != nil { + t.Log(err) + return + } + pbDir, err := findPbFile(output, false) + assert.Nil(t, err) + pbGo := filepath.Join(pbDir, "greet.pb.go") + assert.True(t, pathx.FileExists(pbGo)) + + grpcDir, err := findPbFile(output, true) + assert.Nil(t, err) + grpcGo := filepath.Join(grpcDir, "greet_grpc.pb.go") + assert.True(t, pathx.FileExists(grpcGo)) + }) + + t.Run("", func(t *testing.T) { + output := t.TempDir() + pbeRedirect := filepath.Join(output, "redirect") + grpc := filepath.Join(output, "grpc") + grpcRedirect := filepath.Join(grpc, "redirect") + err := pathx.MkdirIfNotExist(grpc) + if err != nil { + t.Log(err) + return + } + err = pathx.MkdirIfNotExist(pbeRedirect) + if err != nil { + t.Log(err) + return + } + err = pathx.MkdirIfNotExist(grpcRedirect) + if err != nil { + t.Log(err) + return + } + cmd := exec.Command("protoc", "-I="+filepath.Dir(protoFile), "--go_out="+output, + "--go-grpc_out="+grpc, filepath.Base(protoFile), "--go_opt=M"+filepath.Base(protoFile)+"="+pbeRedirect, + "--go-grpc_opt=M"+filepath.Base(protoFile)+"="+grpcRedirect, "--go_opt=paths=import", "--go-grpc_opt=paths=source_relative", + "--go_out="+pbeRedirect, "--go-grpc_out="+grpcRedirect) + cmd.Dir = output + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Run() + if err != nil { + t.Log(err) + return + } + pbDir, err := findPbFile(output, false) + assert.Nil(t, err) + pbGo := filepath.Join(pbDir, "greet.pb.go") + assert.True(t, pathx.FileExists(pbGo)) + + grpcDir, err := findPbFile(output, true) + assert.Nil(t, err) + grpcGo := filepath.Join(grpcDir, "greet_grpc.pb.go") + assert.True(t, pathx.FileExists(grpcGo)) + }) +} diff --git a/tools/goctl/rpc/generator/mkdir.go b/tools/goctl/rpc/generator/mkdir.go index 12ffbfbc..b5ea23dd 100644 --- a/tools/goctl/rpc/generator/mkdir.go +++ b/tools/goctl/rpc/generator/mkdir.go @@ -38,6 +38,7 @@ type ( GetProtoGo() Dir GetMain() Dir GetServiceName() stringx.String + SetPbDir(pbDir, grpcDir string) } // Dir defines a directory @@ -50,6 +51,7 @@ type ( defaultDirContext struct { inner map[string]Dir serviceName stringx.String + ctx *ctx.ProjectContext } ) @@ -134,11 +136,26 @@ func mkdir(ctx *ctx.ProjectContext, proto parser.Proto, _ *conf.Config, c *ZRpcC } serviceName := strings.TrimSuffix(proto.Name, filepath.Ext(proto.Name)) return &defaultDirContext{ + ctx: ctx, inner: inner, serviceName: stringx.From(strings.ReplaceAll(serviceName, "-", "")), }, nil } +func (d *defaultDirContext) SetPbDir(pbDir, grpcDir string) { + d.inner[pb] = Dir{ + Filename: pbDir, + Package: filepath.ToSlash(filepath.Join(d.ctx.Path, strings.TrimPrefix(pbDir, d.ctx.Dir))), + Base: filepath.Base(pbDir), + } + + d.inner[protoGo] = Dir{ + Filename: grpcDir, + Package: filepath.ToSlash(filepath.Join(d.ctx.Path, strings.TrimPrefix(grpcDir, d.ctx.Dir))), + Base: filepath.Base(grpcDir), + } +} + func (d *defaultDirContext) GetCall() Dir { return d.inner[call] }