Fix #1585 #1547 (#1624)

tools/goctl/env/check.go

@@ -40,10 +40,10 @@ var bins = []bin{
 func Check(ctx *cli.Context) error {
 	install := ctx.Bool("install")
 	force := ctx.Bool("force")
-	return check(install, force)
+	return Prepare(install, force)
 }
 
-func check(install, force bool) error {
+func Prepare(install, force bool) error {
 	pending := true
 	console.Info("[goctl-env]: preparing to check env")
 	defer func() {
@@ -56,8 +56,8 @@ func check(install, force bool) error {
 		} else {
 			console.Error(`
 [goctl-env]: check env finish, some dependencies is not found in PATH, you can execute
-command 'goctl env check --install' or 'goctl env install' to install it, for details, 
-please see 'goctl env check --help' or 'goctl env install --help'`)
+command 'goctl env check --install' to install it, for details, please execute command 
+'goctl env check --help'`)
 		}
 	}()
 	for _, e := range bins {

tools/goctl/goctl.go

@@ -542,22 +542,30 @@ var commands = []cli.Command{
 				Description: "for details, see https://go-zero.dev/cn/goctl-rpc.html",
 				Action:      rpc.ZRPC,
 				Flags: []cli.Flag{
-					cli.StringFlag{
+					cli.StringSliceFlag{
 						Name:   "go_out",
 						Hidden: true,
 					},
-					cli.StringFlag{
+					cli.StringSliceFlag{
 						Name:   "go-grpc_out",
 						Hidden: true,
 					},
-					cli.StringFlag{
+					cli.StringSliceFlag{
 						Name:   "go_opt",
 						Hidden: true,
 					},
-					cli.StringFlag{
+					cli.StringSliceFlag{
 						Name:   "go-grpc_opt",
 						Hidden: true,
 					},
+					cli.StringSliceFlag{
+						Name:   "plugin",
+						Hidden: true,
+					},
+					cli.StringSliceFlag{
+						Name:   "proto_path,I",
+						Hidden: true,
+					},
 					cli.StringFlag{
 						Name:  "zrpc_out",
 						Usage: "the zrpc output directory",

tools/goctl/pkg/golang/bin.go

@@ -11,10 +11,10 @@ import (
 // GoBin returns a path of GOBIN.
 func GoBin() string {
 	def := build.Default
-	goroot := os.Getenv("GOROOT")
+	goroot := os.Getenv("GOPATH")
 	bin := filepath.Join(goroot, "bin")
 	if !pathx.FileExists(bin) {
-		gopath := os.Getenv("GOPATH")
+		gopath := os.Getenv("GOROOT")
 		bin = filepath.Join(gopath, "bin")
 	}
 	if !pathx.FileExists(bin) {

tools/goctl/pkg/protocgengo/protocgengo.go

@@ -23,8 +23,11 @@ func Install(cacheDir string) (string, error) {
 }
 
 func Exists() bool {
-	_, err := env.LookUpProtocGenGo()
-	return err == nil
+	ver, err := Version()
+	if err != nil {
+		return false
+	}
+	return len(ver) > 0
 }
 
 // Version is used to get the version of the protoc-gen-go plugin. For older versions, protoc-gen-go does not support

tools/goctl/rpc/cli/zrpc.go

@@ -2,12 +2,10 @@ package cli
 
 import (
 	"errors"
-	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
 
-	"github.com/emicklei/proto"
 	"github.com/urfave/cli"
 	"github.com/zeromicro/go-zero/tools/goctl/rpc/generator"
 	"github.com/zeromicro/go-zero/tools/goctl/util"
@@ -15,7 +13,8 @@ import (
 )
 
 var (
-	errInvalidGrpcOutput = errors.New("ZRPC: missing grpc output")
+	errInvalidGrpcOutput = errors.New("ZRPC: missing --go-grpc_out")
+	errInvalidGoOutput   = errors.New("ZRPC: missing --go_out")
 	errInvalidZrpcOutput = errors.New("ZRPC: missing zrpc output, please use --zrpc_out to specify the output")
 	errInvalidInput      = errors.New("ZRPC: missing source")
 	errMultiInput        = errors.New("ZRPC: only one source is expected")
@@ -40,21 +39,29 @@ func ZRPC(c *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	src := filepath.Dir(source)
-	goPackage, protoPkg, err := getGoPackage(source)
-	if err != nil {
-		return err
-	}
 
-	grpcOut := c.String("go-grpc_out")
-	goOut := c.String("go_out")
-	goOpt := c.String("go_opt")
-	grpcOpt := c.String("go-grpc_opt")
+	grpcOutList := c.StringSlice("go-grpc_out")
+	goOutList := c.StringSlice("go_out")
 	zrpcOut := c.String("zrpc_out")
 	style := c.String("style")
 	home := c.String("home")
 	remote := c.String("remote")
 	branch := c.String("branch")
+	if len(grpcOutList) == 0 {
+		return errInvalidGrpcOutput
+	}
+	if len(goOutList) == 0 {
+		return errInvalidGoOutput
+	}
+	goOut := goOutList[len(goOutList)-1]
+	grpcOut := grpcOutList[len(grpcOutList)-1]
+	if len(goOut) == 0 {
+		return errInvalidGrpcOutput
+	}
+	if len(zrpcOut) == 0 {
+		return errInvalidZrpcOutput
+	}
+
 	if len(remote) > 0 {
 		repo, _ := util.CloneIntoGitHome(remote, branch)
 		if len(repo) > 0 {
@@ -65,38 +72,11 @@ func ZRPC(c *cli.Context) error {
 	if len(home) > 0 {
 		pathx.RegisterGoctlHome(home)
 	}
-	if len(goOut) == 0 {
-		return errInvalidGrpcOutput
-	}
-	if len(zrpcOut) == 0 {
-		return errInvalidZrpcOutput
-	}
 	if !filepath.IsAbs(zrpcOut) {
 		zrpcOut = filepath.Join(pwd, zrpcOut)
 	}
 
-	goOut = removePluginFlag(goOut)
-	goOut, err = parseOut(src, goOut, goOpt, goPackage)
-	if err != nil {
-		return err
-	}
-
 	isGooglePlugin := len(grpcOut) > 0
-	// If grpcOut is not empty means that user generates grpc code by
-	// https://google.golang.org/protobuf/cmd/protoc-gen-go and
-	// https://google.golang.org/grpc/cmd/protoc-gen-go-grpc,
-	// for details please see https://grpc.io/docs/languages/go/quickstart/
-	if isGooglePlugin {
-		grpcOut, err = parseOut(src, grpcOut, grpcOpt, goPackage)
-		if err != nil {
-			return err
-		}
-	} else {
-		// Else it means that user generates grpc code by
-		// https://github.com/golang/protobuf/tree/master/protoc-gen-go
-		grpcOut = goOut
-	}
-
 	goOut, err = filepath.Abs(goOut)
 	if err != nil {
 		return err
@@ -110,23 +90,11 @@ func ZRPC(c *cli.Context) error {
 		return err
 	}
 
-	if isGooglePlugin && grpcOut != goOut {
-		return fmt.Errorf("the --go_out and --go-grpc_out must be the same")
-	}
-
-	if goOut == zrpcOut || grpcOut == zrpcOut {
-		recommendName := goPackage
-		if len(recommendName) == 0 {
-			recommendName = protoPkg
-		}
-		return fmt.Errorf("the zrpc and grpc output can not be the same, it is recommended to output grpc to the %q",
-			filepath.Join(goOut, recommendName))
-	}
-
 	var ctx generator.ZRpcContext
 	ctx.Src = source
-	ctx.ProtoGenGoDir = goOut
-	ctx.ProtoGenGrpcDir = grpcOut
+	ctx.GoOutput = goOut
+	ctx.GrpcOutput = grpcOut
+	ctx.IsGooglePlugin = isGooglePlugin
 	ctx.Output = zrpcOut
 	ctx.ProtocCmd = strings.Join(protocArgs, " ")
 	g, err := generator.NewDefaultRPCGenerator(style, generator.WithZRpcContext(&ctx))
@@ -137,52 +105,6 @@ func ZRPC(c *cli.Context) error {
 	return g.Generate(source, zrpcOut, nil)
 }
 
-// parseOut calculates the output place to grpc code, about to calculate logic for details
-// please see https://developers.google.com/protocol-buffers/docs/reference/go-generated#invocation.
-func parseOut(sourceDir, grpcOut, grpcOpt, goPackage string) (string, error) {
-	if !filepath.IsAbs(grpcOut) {
-		grpcOut = filepath.Join(sourceDir, grpcOut)
-	}
-	switch grpcOpt {
-	case "", optImport:
-		grpcOut = filepath.Join(grpcOut, goPackage)
-	case optSourceRelative:
-		grpcOut = filepath.Join(grpcOut)
-	default:
-		return "", fmt.Errorf("parseAndSetGrpcOut:  unknown path type %q: want %q or %q",
-			grpcOpt, optImport, optSourceRelative)
-	}
-
-	return grpcOut, nil
-}
-
-func getGoPackage(source string) (string, string, error) {
-	r, err := os.Open(source)
-	if err != nil {
-		return "", "", err
-	}
-	defer func() {
-		_ = r.Close()
-	}()
-
-	parser := proto.NewParser(r)
-	set, err := parser.Parse()
-	if err != nil {
-		return "", "", err
-	}
-
-	var goPackage, protoPkg string
-	proto.Walk(set, proto.WithOption(func(option *proto.Option) {
-		if option.Name == "go_package" {
-			goPackage = option.Constant.Source
-		}
-	}), proto.WithPackage(func(p *proto.Package) {
-		protoPkg = p.Name
-	}))
-
-	return goPackage, protoPkg, nil
-}
-
 func removeGoctlFlag(args []string) []string {
 	var ret []string
 	var step int

tools/goctl/rpc/cli/zrpc_test.go

@@ -79,6 +79,10 @@ func Test_RemoveGoctlFlag(t *testing.T) {
 			source:   strings.Fields(`protoc foo.proto --go_opt=. --zrpc_out="bar" --style=goZero  --home=bar`),
 			expected: "protoc foo.proto --go_opt=.",
 		},
+		{
+			source:   strings.Fields(`protoc --go_opt=. --go-grpc_out=. --zrpc_out=. foo.proto`),
+			expected: "protoc --go_opt=. --go-grpc_out=. foo.proto",
+		},
 	}
 	for _, e := range testData {
 		cmd := strings.Join(removeGoctlFlag(e.source), " ")

tools/goctl/rpc/generator/defaultgenerator.go

@@ -1,8 +1,7 @@
 package generator
 
 import (
-	"os/exec"
-
+	"github.com/zeromicro/go-zero/tools/goctl/env"
 	"github.com/zeromicro/go-zero/tools/goctl/util/console"
 )
 
@@ -25,17 +24,5 @@ func NewDefaultGenerator() Generator {
 // Prepare provides environment detection generated by rpc service,
 // including go environment, protoc, whether protoc-gen-go is installed or not
 func (g *DefaultGenerator) Prepare() error {
-	_, err := exec.LookPath("go")
-	if err != nil {
-		return err
-	}
-
-	_, err = exec.LookPath("protoc")
-	if err != nil {
-		return err
-	}
-
-	_, err = exec.LookPath("protoc-gen-go")
-
-	return err
+	return env.Prepare(true, true)
 }

tools/goctl/rpc/generator/gen.go

@@ -24,6 +24,9 @@ type ZRpcContext struct {
 	ProtocCmd       string
 	ProtoGenGrpcDir string
 	ProtoGenGoDir   string
+	IsGooglePlugin  bool
+	GoOutput        string
+	GrpcOutput      string
 	Output          string
 }
 

tools/goctl/rpc/generator/genpb.go

@@ -3,6 +3,8 @@ package generator
 import (
 	"bytes"
 	"errors"
+	"fmt"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
@@ -19,7 +21,7 @@ const googleProtocGenGoErr = `--go_out: protoc-gen-go: plugins are not supported
 // but the commands and flags in protoc are not completely joined in goctl. At present, proto_path(-I) is introduced
 func (g *DefaultGenerator) GenPb(ctx DirContext, protoImportPath []string, proto parser.Proto, _ *conf.Config, c *ZRpcContext, goOptions ...string) error {
 	if c != nil {
-		return g.genPbDirect(c)
+		return g.genPbDirect(ctx, c)
 	}
 
 	// deprecated: use genPbDirect instead.
@@ -110,13 +112,74 @@ go get -u github.com/golang/protobuf/protoc-gen-go`)
 	return nil
 }
 
-func (g *DefaultGenerator) genPbDirect(c *ZRpcContext) error {
-	g.log.Debug(c.ProtocCmd)
+func (g *DefaultGenerator) genPbDirect(ctx DirContext, c *ZRpcContext) error {
+	g.log.Debug("[command]: %s", c.ProtocCmd)
 	pwd, err := os.Getwd()
 	if err != nil {
 		return err
 	}
 
 	_, err = execx.Run(c.ProtocCmd, pwd)
+	if err != nil {
 		return err
+	}
+	return g.setPbDir(ctx, c)
+}
+
+func (g *DefaultGenerator) setPbDir(ctx DirContext, c *ZRpcContext) error {
+	pbDir, err := findPbFile(c.GoOutput, false)
+	if err != nil {
+		return err
+	}
+	if len(pbDir) == 0 {
+		return fmt.Errorf("pg.go is not found under %q", c.GoOutput)
+	}
+	grpcDir, err := findPbFile(c.GrpcOutput, true)
+	if err != nil {
+		return err
+	}
+	if len(grpcDir) == 0 {
+		return fmt.Errorf("_grpc.pb.go is not found in %q", c.GrpcOutput)
+	}
+	if pbDir != grpcDir {
+		return fmt.Errorf("the pb.go and _grpc.pb.go must under the same dir: "+
+			"\n pb.go: %s\n_grpc.pb.go: %s", pbDir, grpcDir)
+	}
+	if pbDir == c.Output {
+		return fmt.Errorf("the output of pb.go and _grpc.pb.go must not be the same "+
+			"with --zrpc_out:\npb output: %s\nzrpc out: %s", pbDir, c.Output)
+	}
+	ctx.SetPbDir(pbDir, grpcDir)
+	return nil
+}
+
+const (
+	pbSuffix   = "pb.go"
+	grpcSuffix = "_grpc.pb.go"
+)
+
+func findPbFile(current string, grpc bool) (string, error) {
+	fileSystem := os.DirFS(current)
+	var ret string
+	err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error {
+		if d.IsDir() {
+			return nil
+		}
+		if strings.HasSuffix(path, pbSuffix) {
+			if grpc {
+				if strings.HasSuffix(path, grpcSuffix) {
+					ret = path
+					return os.ErrExist
+				}
+			} else if !strings.HasSuffix(path, grpcSuffix) {
+				ret = path
+				return os.ErrExist
+			}
+		}
+		return nil
+	})
+	if err == os.ErrExist {
+		return filepath.Dir(filepath.Join(current, ret)), nil
+	}
+	return "", err
 }

tools/goctl/rpc/generator/genpb_test.go

@@ -0,0 +1,197 @@
+package generator
+
+import (
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/zeromicro/go-zero/tools/goctl/util/pathx"
+)
+
+func Test_findPbFile(t *testing.T) {
+	dir := t.TempDir()
+	protoFile := filepath.Join(dir, "greet.proto")
+	err := ioutil.WriteFile(protoFile, []byte(`
+syntax = "proto3";
+
+package greet;
+option go_package="./greet";
+
+message Req{}
+message Resp{}
+service Greeter {
+  rpc greet(Req) returns (Resp);
+}
+`), 0666)
+	if err != nil {
+		t.Log(err)
+		return
+	}
+	t.Run("", func(t *testing.T) {
+		output := t.TempDir()
+		grpc := filepath.Join(output, "grpc")
+		err := pathx.MkdirIfNotExist(grpc)
+		if err != nil {
+			t.Log(err)
+			return
+		}
+		cmd := exec.Command("protoc", "-I="+filepath.Dir(protoFile), "--go_out="+output, "--go-grpc_out="+grpc, filepath.Base(protoFile))
+		cmd.Dir = output
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		err = cmd.Run()
+		if err != nil {
+			t.Log(err)
+			return
+		}
+		pbDir, err := findPbFile(output, false)
+		assert.Nil(t, err)
+		pbGo := filepath.Join(pbDir, "greet.pb.go")
+		assert.True(t, pathx.FileExists(pbGo))
+
+		grpcDir, err := findPbFile(output, true)
+		assert.Nil(t, err)
+		grpcGo := filepath.Join(grpcDir, "greet_grpc.pb.go")
+		assert.True(t, pathx.FileExists(grpcGo))
+	})
+
+	t.Run("", func(t *testing.T) {
+		output := t.TempDir()
+		redirect := filepath.Join(output, "pb")
+		grpc := filepath.Join(output, "grpc")
+		err := pathx.MkdirIfNotExist(grpc)
+		if err != nil {
+			t.Log(err)
+			return
+		}
+		cmd := exec.Command("protoc", "-I="+filepath.Dir(protoFile), "--go_out="+output,
+			"--go-grpc_out="+grpc, filepath.Base(protoFile), "--go_opt=M"+filepath.Base(protoFile)+"="+redirect)
+		cmd.Dir = output
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		err = cmd.Run()
+		if err != nil {
+			t.Log(err)
+			return
+		}
+		pbDir, err := findPbFile(output, false)
+		assert.Nil(t, err)
+		pbGo := filepath.Join(pbDir, "greet.pb.go")
+		assert.True(t, pathx.FileExists(pbGo))
+
+		grpcDir, err := findPbFile(output, true)
+		assert.Nil(t, err)
+		grpcGo := filepath.Join(grpcDir, "greet_grpc.pb.go")
+		assert.True(t, pathx.FileExists(grpcGo))
+	})
+
+	t.Run("", func(t *testing.T) {
+		output := t.TempDir()
+		pbeRedirect := filepath.Join(output, "redirect")
+		grpc := filepath.Join(output, "grpc")
+		grpcRedirect := filepath.Join(grpc, "redirect")
+		err := pathx.MkdirIfNotExist(grpc)
+		if err != nil {
+			t.Log(err)
+			return
+		}
+		cmd := exec.Command("protoc", "-I="+filepath.Dir(protoFile), "--go_out="+output,
+			"--go-grpc_out="+grpc, filepath.Base(protoFile), "--go_opt=M"+filepath.Base(protoFile)+"="+pbeRedirect,
+			"--go-grpc_opt=M"+filepath.Base(protoFile)+"="+grpcRedirect)
+		cmd.Dir = output
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		err = cmd.Run()
+		if err != nil {
+			t.Log(err)
+			return
+		}
+		pbDir, err := findPbFile(output, false)
+		assert.Nil(t, err)
+		pbGo := filepath.Join(pbDir, "greet.pb.go")
+		assert.True(t, pathx.FileExists(pbGo))
+
+		grpcDir, err := findPbFile(output, true)
+		assert.Nil(t, err)
+		grpcGo := filepath.Join(grpcDir, "greet_grpc.pb.go")
+		assert.True(t, pathx.FileExists(grpcGo))
+	})
+
+	t.Run("", func(t *testing.T) {
+		output := t.TempDir()
+		pbeRedirect := filepath.Join(output, "redirect")
+		grpc := filepath.Join(output, "grpc")
+		grpcRedirect := filepath.Join(grpc, "redirect")
+		err := pathx.MkdirIfNotExist(grpc)
+		if err != nil {
+			t.Log(err)
+			return
+		}
+		cmd := exec.Command("protoc", "-I="+filepath.Dir(protoFile), "--go_out="+output,
+			"--go-grpc_out="+grpc, filepath.Base(protoFile), "--go_opt=M"+filepath.Base(protoFile)+"="+pbeRedirect,
+			"--go-grpc_opt=M"+filepath.Base(protoFile)+"="+grpcRedirect, "--go_opt=paths=import", "--go-grpc_opt=paths=source_relative")
+		cmd.Dir = output
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		err = cmd.Run()
+		if err != nil {
+			t.Log(err)
+			return
+		}
+		pbDir, err := findPbFile(output, false)
+		assert.Nil(t, err)
+		pbGo := filepath.Join(pbDir, "greet.pb.go")
+		assert.True(t, pathx.FileExists(pbGo))
+
+		grpcDir, err := findPbFile(output, true)
+		assert.Nil(t, err)
+		grpcGo := filepath.Join(grpcDir, "greet_grpc.pb.go")
+		assert.True(t, pathx.FileExists(grpcGo))
+	})
+
+	t.Run("", func(t *testing.T) {
+		output := t.TempDir()
+		pbeRedirect := filepath.Join(output, "redirect")
+		grpc := filepath.Join(output, "grpc")
+		grpcRedirect := filepath.Join(grpc, "redirect")
+		err := pathx.MkdirIfNotExist(grpc)
+		if err != nil {
+			t.Log(err)
+			return
+		}
+		err = pathx.MkdirIfNotExist(pbeRedirect)
+		if err != nil {
+			t.Log(err)
+			return
+		}
+		err = pathx.MkdirIfNotExist(grpcRedirect)
+		if err != nil {
+			t.Log(err)
+			return
+		}
+		cmd := exec.Command("protoc", "-I="+filepath.Dir(protoFile), "--go_out="+output,
+			"--go-grpc_out="+grpc, filepath.Base(protoFile), "--go_opt=M"+filepath.Base(protoFile)+"="+pbeRedirect,
+			"--go-grpc_opt=M"+filepath.Base(protoFile)+"="+grpcRedirect, "--go_opt=paths=import", "--go-grpc_opt=paths=source_relative",
+			"--go_out="+pbeRedirect, "--go-grpc_out="+grpcRedirect)
+		cmd.Dir = output
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		err = cmd.Run()
+		if err != nil {
+			t.Log(err)
+			return
+		}
+		pbDir, err := findPbFile(output, false)
+		assert.Nil(t, err)
+		pbGo := filepath.Join(pbDir, "greet.pb.go")
+		assert.True(t, pathx.FileExists(pbGo))
+
+		grpcDir, err := findPbFile(output, true)
+		assert.Nil(t, err)
+		grpcGo := filepath.Join(grpcDir, "greet_grpc.pb.go")
+		assert.True(t, pathx.FileExists(grpcGo))
+	})
+}

tools/goctl/rpc/generator/mkdir.go

@@ -38,6 +38,7 @@ type (
 		GetProtoGo() Dir
 		GetMain() Dir
 		GetServiceName() stringx.String
+		SetPbDir(pbDir, grpcDir string)
 	}
 
 	// Dir defines a directory
@@ -50,6 +51,7 @@ type (
 	defaultDirContext struct {
 		inner       map[string]Dir
 		serviceName stringx.String
+		ctx         *ctx.ProjectContext
 	}
 )
 
@@ -134,11 +136,26 @@ func mkdir(ctx *ctx.ProjectContext, proto parser.Proto, _ *conf.Config, c *ZRpcC
 	}
 	serviceName := strings.TrimSuffix(proto.Name, filepath.Ext(proto.Name))
 	return &defaultDirContext{
+		ctx:         ctx,
 		inner:       inner,
 		serviceName: stringx.From(strings.ReplaceAll(serviceName, "-", "")),
 	}, nil
 }
 
+func (d *defaultDirContext) SetPbDir(pbDir, grpcDir string) {
+	d.inner[pb] = Dir{
+		Filename: pbDir,
+		Package:  filepath.ToSlash(filepath.Join(d.ctx.Path, strings.TrimPrefix(pbDir, d.ctx.Dir))),
+		Base:     filepath.Base(pbDir),
+	}
+
+	d.inner[protoGo] = Dir{
+		Filename: grpcDir,
+		Package:  filepath.ToSlash(filepath.Join(d.ctx.Path, strings.TrimPrefix(grpcDir, d.ctx.Dir))),
+		Base:     filepath.Base(grpcDir),
+	}
+}
+
 func (d *defaultDirContext) GetCall() Dir {
 	return d.inner[call]
 }