From e08ba2fee80e4f9f12a2d433234d4e668bc77ddc Mon Sep 17 00:00:00 2001 From: kesonan Date: Sat, 2 Mar 2024 22:27:39 +0800 Subject: [PATCH] (goctl)fix parser issues (#3930) --- tools/goctl/pkg/parser/api/ast/ast.go | 13 ++++ tools/goctl/pkg/parser/api/parser/analyzer.go | 31 +++++++++- .../pkg/parser/api/parser/analyzer_test.go | 33 +++++++++- tools/goctl/pkg/parser/api/parser/filter.go | 14 ++--- tools/goctl/pkg/parser/api/parser/parser.go | 61 +++++++++++++++++-- .../pkg/parser/api/parser/parser_test.go | 7 +++ .../api/parser/testdata/atserver_test.api | 7 ++- .../api/parser/testdata/duplicate_type.api | 8 +++ .../parser/api/parser/testdata/example.api | 14 +++++ .../api/parser/testdata/example_base.api | 3 + .../api/parser/testdata/example_base1.api | 2 + .../api/parser/testdata/example_base2.api | 2 + tools/goctl/rpc/generator/gen_test.go | 8 +-- 13 files changed, 179 insertions(+), 24 deletions(-) create mode 100644 tools/goctl/pkg/parser/api/parser/testdata/duplicate_type.api create mode 100644 tools/goctl/pkg/parser/api/parser/testdata/example_base.api diff --git a/tools/goctl/pkg/parser/api/ast/ast.go b/tools/goctl/pkg/parser/api/ast/ast.go index 40b6bb4a..9b06e14a 100644 --- a/tools/goctl/pkg/parser/api/ast/ast.go +++ b/tools/goctl/pkg/parser/api/ast/ast.go @@ -84,6 +84,19 @@ func (t *TokenNode) SetLeadingCommentGroup(cg CommentGroup) { t.LeadingCommentGroup = cg } +// RawText returns the node's raw text. +func (t *TokenNode) RawText() string { + text := t.Token.Text + if strings.HasPrefix(text, "`") { + text = strings.TrimPrefix(text, "`") + text = strings.TrimSuffix(text, "`") + } else if strings.HasPrefix(text, `"`) { + text = strings.TrimPrefix(text, `"`) + text = strings.TrimSuffix(text, `"`) + } + return text +} + func (t *TokenNode) HasLeadingCommentGroup() bool { return t.LeadingCommentGroup.Valid() || t.leadingFlag } diff --git a/tools/goctl/pkg/parser/api/parser/analyzer.go b/tools/goctl/pkg/parser/api/parser/analyzer.go index a08974a9..35566b93 100644 --- a/tools/goctl/pkg/parser/api/parser/analyzer.go +++ b/tools/goctl/pkg/parser/api/parser/analyzer.go @@ -108,6 +108,8 @@ func (a *Analyzer) astTypeToSpec(in ast.DataType) (spec.Type, error) { } func (a *Analyzer) convert2Spec() error { + a.fillInfo() + if err := a.fillTypes(); err != nil { return err } @@ -128,7 +130,7 @@ func (a *Analyzer) convert2Spec() error { groups = append(groups, v) } sort.SliceStable(groups, func(i, j int) bool { - return groups[i].Annotation.Properties["group"] < groups[j].Annotation.Properties["group"] + return groups[i].Annotation.Properties[groupKeyText] < groups[j].Annotation.Properties[groupKeyText] }) a.spec.Service.Groups = groups @@ -150,7 +152,11 @@ func (a *Analyzer) convertKV(kv []*ast.KVExpr) map[string]string { var ret = map[string]string{} for _, v := range kv { key := strings.TrimSuffix(v.Key.Token.Text, ":") - ret[key] = v.Value.Token.Text + if key == summaryKeyText { + ret[key] = v.Value.RawText() + } else { + ret[key] = v.Value.Token.Text + } } return ret } @@ -270,6 +276,27 @@ func (a *Analyzer) fillService() error { return nil } +func (a *Analyzer) fillInfo() { + properties := make(map[string]string) + if a.api.info != nil { + for _, kv := range a.api.info.Values { + key := kv.Key.Token.Text + properties[strings.TrimSuffix(key, ":")] = kv.Value.RawText() + } + } + a.spec.Info.Properties = properties + infoKeyValue := make(map[string]string) + for key, value := range properties { + titleKey := strings.Title(strings.TrimSuffix(key, ":")) + infoKeyValue[titleKey] = value + } + a.spec.Info.Title = infoKeyValue[infoTitleKey] + a.spec.Info.Desc = infoKeyValue[infoDescKey] + a.spec.Info.Version = infoKeyValue[infoVersionKey] + a.spec.Info.Author = infoKeyValue[infoAuthorKey] + a.spec.Info.Email = infoKeyValue[infoEmailKey] +} + func (a *Analyzer) fillTypes() error { for _, item := range a.api.TypeStmt { switch v := (item).(type) { diff --git a/tools/goctl/pkg/parser/api/parser/analyzer_test.go b/tools/goctl/pkg/parser/api/parser/analyzer_test.go index 5e01d0f4..fa46ecf2 100644 --- a/tools/goctl/pkg/parser/api/parser/analyzer_test.go +++ b/tools/goctl/pkg/parser/api/parser/analyzer_test.go @@ -8,14 +8,40 @@ import ( "strings" "testing" + "github.com/zeromicro/go-zero/tools/goctl/api/spec" + "github.com/stretchr/testify/assert" "github.com/zeromicro/go-zero/tools/goctl/pkg/parser/api/assertx" ) func Test_Parse(t *testing.T) { t.Run("valid", func(t *testing.T) { - _, err := Parse("./testdata/example.api", nil) + apiSpec, err := Parse("./testdata/example.api", nil) assert.Nil(t, err) + ast := assert.New(t) + ast.Equal(spec.Info{ + Title: "type title here", + Desc: "type desc here", + Version: "type version here", + Author: "type author here", + Email: "type email here", + Properties: map[string]string{ + "title": "type title here", + "desc": "type desc here", + "version": "type version here", + "author": "type author here", + "email": "type email here", + }, + }, apiSpec.Info) + ast.True(func() bool { + for _, group := range apiSpec.Service.Groups { + value, ok := group.Annotation.Properties["summary"] + if ok { + return value == "test" + } + } + return false + }()) }) t.Run("invalid", func(t *testing.T) { data, err := os.ReadFile("./testdata/invalid.api") @@ -46,4 +72,9 @@ func Test_Parse(t *testing.T) { _, err := Parse("./testdata/link_import.api", nil) assert.Nil(t, err) }) + + t.Run("duplicate_types", func(t *testing.T) { + _, err := Parse("./testdata/duplicate_type.api", nil) + assertx.Error(t, err) + }) } diff --git a/tools/goctl/pkg/parser/api/parser/filter.go b/tools/goctl/pkg/parser/api/parser/filter.go index b660f271..88a66b2f 100644 --- a/tools/goctl/pkg/parser/api/parser/filter.go +++ b/tools/goctl/pkg/parser/api/parser/filter.go @@ -4,12 +4,12 @@ import ( "fmt" "github.com/zeromicro/go-zero/tools/goctl/pkg/parser/api/ast" - "github.com/zeromicro/go-zero/tools/goctl/pkg/parser/api/placeholder" + "github.com/zeromicro/go-zero/tools/goctl/pkg/parser/api/token" ) type filterBuilder struct { filename string - m map[string]placeholder.Type + m map[string]token.Position checkExprName string errorManager *errorManager } @@ -17,10 +17,10 @@ type filterBuilder struct { func (b *filterBuilder) check(nodes ...*ast.TokenNode) { for _, node := range nodes { fileNodeText := fmt.Sprintf("%s/%s", b.filename, node.Token.Text) - if _, ok := b.m[fileNodeText]; ok { + if pos, ok := b.m[fileNodeText]; ok && pos != node.Token.Position { b.errorManager.add(ast.DuplicateStmtError(node.Pos(), "duplicate "+b.checkExprName)) } else { - b.m[fileNodeText] = placeholder.PlaceHolder + b.m[fileNodeText] = node.Token.Position } } } @@ -28,10 +28,10 @@ func (b *filterBuilder) check(nodes ...*ast.TokenNode) { func (b *filterBuilder) checkNodeWithPrefix(prefix string, nodes ...*ast.TokenNode) { for _, node := range nodes { joinText := fmt.Sprintf("%s/%s", prefix, node.Token.Text) - if _, ok := b.m[joinText]; ok { + if pos, ok := b.m[joinText]; ok && pos != node.Token.Position { b.errorManager.add(ast.DuplicateStmtError(node.Pos(), "duplicate "+b.checkExprName)) } else { - b.m[joinText] = placeholder.PlaceHolder + b.m[joinText] = node.Token.Position } } } @@ -51,7 +51,7 @@ func newFilter() *filter { func (f *filter) addCheckItem(filename, checkExprName string) *filterBuilder { b := &filterBuilder{ filename: filename, - m: make(map[string]placeholder.Type), + m: make(map[string]token.Position), checkExprName: checkExprName, errorManager: newErrorManager(), } diff --git a/tools/goctl/pkg/parser/api/parser/parser.go b/tools/goctl/pkg/parser/api/parser/parser.go index 77bcd2db..92806d52 100644 --- a/tools/goctl/pkg/parser/api/parser/parser.go +++ b/tools/goctl/pkg/parser/api/parser/parser.go @@ -12,7 +12,17 @@ import ( "github.com/zeromicro/go-zero/tools/goctl/pkg/parser/api/token" ) -const idAPI = "api" +const ( + idAPI = "api" + summaryKeyExprText = "summary:" + summaryKeyText = "summary" + groupKeyText = "group" + infoTitleKey = "Title" + infoDescKey = "Desc" + infoVersionKey = "Version" + infoAuthorKey = "Author" + infoEmailKey = "Email" +) // Parser is the parser for api file. type Parser struct { @@ -1134,7 +1144,7 @@ func (p *Parser) parseAtServerKVExpression() *ast.KVExpr { var valueTok token.Token var leadingCommentGroup ast.CommentGroup - if p.notExpectPeekToken(token.QUO, token.DURATION, token.IDENT, token.INT) { + if p.notExpectPeekToken(token.QUO, token.DURATION, token.IDENT, token.INT, token.STRING) { return nil } @@ -1144,13 +1154,24 @@ func (p *Parser) parseAtServerKVExpression() *ast.KVExpr { } slashTok := p.curTok + var pathText = slashTok.Text if !p.advanceIfPeekTokenIs(token.IDENT) { return nil } + pathText += p.curTok.Text + if p.peekTokenIs(token.SUB) { // 解析 abc-efg 格式 + if !p.nextToken() { + return nil + } + pathText += p.curTok.Text + if !p.advanceIfPeekTokenIs(token.IDENT) { + return nil + } + pathText += p.curTok.Text + } - idTok := p.curTok valueTok = token.Token{ - Text: slashTok.Text + idTok.Text, + Text: pathText, Position: slashTok.Position, } leadingCommentGroup = p.curTokenNode().LeadingCommentGroup @@ -1170,6 +1191,22 @@ func (p *Parser) parseAtServerKVExpression() *ast.KVExpr { return nil } + valueTok = p.curTok + leadingCommentGroup = p.curTokenNode().LeadingCommentGroup + node := ast.NewTokenNode(valueTok) + node.SetLeadingCommentGroup(leadingCommentGroup) + expr.Value = node + return expr + } else if p.peekTokenIs(token.STRING) { + if expr.Key.Token.Text != summaryKeyExprText { + if p.notExpectPeekToken(token.QUO, token.DURATION, token.IDENT, token.INT) { + return nil + } + } + if !p.nextToken() { + return nil + } + valueTok = p.curTok leadingCommentGroup = p.curTokenNode().LeadingCommentGroup node := ast.NewTokenNode(valueTok) @@ -1221,13 +1258,25 @@ func (p *Parser) parseAtServerKVExpression() *ast.KVExpr { } slashTok := p.curTok + var pathText = valueTok.Text + pathText += slashTok.Text if !p.advanceIfPeekTokenIs(token.IDENT) { return nil } + pathText += p.curTok.Text + if p.peekTokenIs(token.SUB) { // 解析 abc-efg 格式 + if !p.nextToken() { + return nil + } + pathText += p.curTok.Text + if !p.advanceIfPeekTokenIs(token.IDENT) { + return nil + } + pathText += p.curTok.Text + } - idTok := p.curTok valueTok = token.Token{ - Text: valueTok.Text + slashTok.Text + idTok.Text, + Text: pathText, Position: valueTok.Position, } leadingCommentGroup = p.curTokenNode().LeadingCommentGroup diff --git a/tools/goctl/pkg/parser/api/parser/parser_test.go b/tools/goctl/pkg/parser/api/parser/parser_test.go index c37b03a7..e49eb2c6 100644 --- a/tools/goctl/pkg/parser/api/parser/parser_test.go +++ b/tools/goctl/pkg/parser/api/parser/parser_test.go @@ -299,6 +299,11 @@ func TestParser_Parse_atServerStmt(t *testing.T) { "timeout6:": "10ns", "timeout7:": "1h10m10s10ms10µs10ns", "maxBytes:": `1024`, + "prefix:": "/v1", + "prefix1:": "/v1/v2_test/v2-beta", + "prefix2:": "v1/v2_test/v2-beta", + "prefix3:": "v1/v2_", + "summary:": `"test"`, } p := New("foo.api", atServerTestAPI) @@ -349,6 +354,8 @@ func TestParser_Parse_atServerStmt(t *testing.T) { `@server(foo:/v1/v2`, `@server(foo: m1,`, `@server(foo: m1,)`, + `@server(foo: v1/v2-)`, + `@server(foo:"test")`, } for _, v := range testData { p := New("foo.api", v) diff --git a/tools/goctl/pkg/parser/api/parser/testdata/atserver_test.api b/tools/goctl/pkg/parser/api/parser/testdata/atserver_test.api index c321a031..14b2b4a9 100644 --- a/tools/goctl/pkg/parser/api/parser/testdata/atserver_test.api +++ b/tools/goctl/pkg/parser/api/parser/testdata/atserver_test.api @@ -13,4 +13,9 @@ timeout6: 10ns timeout7: 1h10m10s10ms10µs10ns maxBytes: 1024 -) \ No newline at end of file + prefix: /v1 + prefix1: /v1/v2_test/v2-beta + prefix2: v1/v2_test/v2-beta + prefix3: v1/v2_ + summary:"test" +) diff --git a/tools/goctl/pkg/parser/api/parser/testdata/duplicate_type.api b/tools/goctl/pkg/parser/api/parser/testdata/duplicate_type.api new file mode 100644 index 00000000..381931db --- /dev/null +++ b/tools/goctl/pkg/parser/api/parser/testdata/duplicate_type.api @@ -0,0 +1,8 @@ +syntax = "v1" + +type Example{ + A string +} +type Example{ + B string +} diff --git a/tools/goctl/pkg/parser/api/parser/testdata/example.api b/tools/goctl/pkg/parser/api/parser/testdata/example.api index 97ff1dd8..85cb5173 100644 --- a/tools/goctl/pkg/parser/api/parser/testdata/example.api +++ b/tools/goctl/pkg/parser/api/parser/testdata/example.api @@ -188,3 +188,17 @@ service example { post /example/nest2 (NestDemoReq2) returns (NestDemoResp2) } +@server ( + group: /g1/g2_test/g2_beta + prefix: /v1/v2_test/v2-beta + summary: "test" +) +service example { + @handler nestDemo1 + post /a/b_c/d-e/:f/123/g (NestDemoReq1) returns (NestDemoResp1) + + @handler nestDemo2 + post /example/nest2 (NestDemoReq2) returns (NestDemoResp2) +} + + diff --git a/tools/goctl/pkg/parser/api/parser/testdata/example_base.api b/tools/goctl/pkg/parser/api/parser/testdata/example_base.api new file mode 100644 index 00000000..24cfc7e5 --- /dev/null +++ b/tools/goctl/pkg/parser/api/parser/testdata/example_base.api @@ -0,0 +1,3 @@ +syntax = "v1" + +type Base{} diff --git a/tools/goctl/pkg/parser/api/parser/testdata/example_base1.api b/tools/goctl/pkg/parser/api/parser/testdata/example_base1.api index c0bd132b..4b8c71e4 100644 --- a/tools/goctl/pkg/parser/api/parser/testdata/example_base1.api +++ b/tools/goctl/pkg/parser/api/parser/testdata/example_base1.api @@ -1,5 +1,7 @@ syntax = "v1" +import "example_base.api" + info( title: "type title here" desc: "type desc here" diff --git a/tools/goctl/pkg/parser/api/parser/testdata/example_base2.api b/tools/goctl/pkg/parser/api/parser/testdata/example_base2.api index ecc07779..cdaf7940 100644 --- a/tools/goctl/pkg/parser/api/parser/testdata/example_base2.api +++ b/tools/goctl/pkg/parser/api/parser/testdata/example_base2.api @@ -1,5 +1,7 @@ syntax = "v1" +import "example_base.api" + info( title: "type title here" desc: "type desc here" diff --git a/tools/goctl/rpc/generator/gen_test.go b/tools/goctl/rpc/generator/gen_test.go index 093d89a0..1c87a1d7 100644 --- a/tools/goctl/rpc/generator/gen_test.go +++ b/tools/goctl/rpc/generator/gen_test.go @@ -5,7 +5,6 @@ import ( "go/build" "os" "path/filepath" - "strings" "testing" "github.com/stretchr/testify/assert" @@ -51,12 +50,7 @@ func TestRpcGenerate(t *testing.T) { err = g.Generate(ctx) assert.Nil(t, err) _, err = execx.Run("go test "+projectName, projectDir) - if err != nil { - assert.True(t, func() bool { - return strings.Contains(err.Error(), - "not in GOROOT") || strings.Contains(err.Error(), "cannot find package") - }()) - } + assert.Error(t, err) }) // case go mod