From c9494c8bc7017e49ffcbaa65a929b3cb2c71f08d Mon Sep 17 00:00:00 2001 From: Keson Date: Thu, 22 Oct 2020 18:33:09 +0800 Subject: [PATCH] model support globbing patterns (#153) * model support globbing patterns * optimize model * optimize model * format code --- tools/goctl/goctl.go | 4 +- tools/goctl/model/sql/README.MD | 56 +++++++++++---- tools/goctl/model/sql/command/command.go | 72 ++++++++++++++----- tools/goctl/model/sql/example/generator.sh | 8 ++- tools/goctl/model/sql/example/sql/user_1.sql | 15 ++++ tools/goctl/model/sql/gen/gen.go | 26 +++++-- tools/goctl/model/sql/gen/gen_test.go | 18 +++++ tools/goctl/model/sql/gen/new.go | 1 + tools/goctl/model/sql/gen/template.go | 2 + .../model/sql/model/informationschemamodel.go | 25 +++++++ tools/goctl/model/sql/template/errors.go | 2 +- tools/goctl/model/sql/template/model.go | 2 +- tools/goctl/model/sql/template/new.go | 4 +- tools/goctl/model/sql/util/match_test.go | 29 ++++++++ tools/goctl/model/sql/util/matcher.go | 38 ++++++++++ tools/goctl/model/sql/util/studeat.sql | 0 tools/goctl/model/sql/util/student.sql | 0 tools/goctl/model/sql/util/sub/sub.sql | 0 tools/goctl/model/sql/util/xx.sql | 0 tools/goctl/model/sql/util/xx.sql1 | 0 20 files changed, 258 insertions(+), 44 deletions(-) create mode 100644 tools/goctl/model/sql/example/sql/user_1.sql create mode 100644 tools/goctl/model/sql/gen/gen_test.go create mode 100644 tools/goctl/model/sql/model/informationschemamodel.go create mode 100644 tools/goctl/model/sql/util/match_test.go create mode 100644 tools/goctl/model/sql/util/matcher.go create mode 100644 tools/goctl/model/sql/util/studeat.sql create mode 100644 tools/goctl/model/sql/util/student.sql create mode 100644 tools/goctl/model/sql/util/sub/sub.sql create mode 100644 tools/goctl/model/sql/util/xx.sql create mode 100644 tools/goctl/model/sql/util/xx.sql1 diff --git a/tools/goctl/goctl.go b/tools/goctl/goctl.go index dff8487d..e8c6bceb 100644 --- a/tools/goctl/goctl.go +++ b/tools/goctl/goctl.go @@ -269,7 +269,7 @@ var ( Flags: []cli.Flag{ cli.StringFlag{ Name: "src, s", - Usage: "the file path of the ddl source file", + Usage: "the path or path globbing patterns of the ddl", }, cli.StringFlag{ Name: "dir, d", @@ -296,7 +296,7 @@ var ( }, cli.StringFlag{ Name: "table, t", - Usage: `source table,tables separated by commas,like "user,course`, + Usage: `the table or table globbing patterns in the database`, }, cli.BoolFlag{ Name: "cache, c", diff --git a/tools/goctl/model/sql/README.MD b/tools/goctl/model/sql/README.MD index d78f9d5f..5f6c2bb9 100644 --- a/tools/goctl/model/sql/README.MD +++ b/tools/goctl/model/sql/README.MD @@ -7,7 +7,7 @@ goctl model 为go-zero下的工具模块中的组件之一，目前支持识别m * 通过ddl生成 ```shell script - goctl model mysql ddl -src="./sql/user.sql" -dir="./sql/model" -c=true + goctl model mysql ddl -src="./*.sql" -dir="./sql/model" -c=true ``` 执行上述命令后即可快速生成CURD代码。 @@ -21,7 +21,7 @@ goctl model 为go-zero下的工具模块中的组件之一，目前支持识别m * 通过datasource生成 ```shell script - goctl model mysql datasource -url="user:password@tcp(127.0.0.1:3306)/database" -table="table1,table2" -dir="./model" + goctl model mysql datasource -url="user:password@tcp(127.0.0.1:3306)/database" -table="*" -dir="./model" ``` * 生成代码示例 @@ -205,14 +205,49 @@ OPTIONS: * ddl ```shell script - goctl model mysql -src={filename} -dir={dir} -cache=true + goctl model mysql -src={patterns} -dir={dir} -cache=true + ``` + + help + + ``` + NAME: + goctl model mysql ddl - generate mysql model from ddl + + USAGE: + goctl model mysql ddl [command options] [arguments...] + + OPTIONS: + --src value, -s value the path or path globbing patterns of the ddl + --dir value, -d value the target dir + --cache, -c generate code with cache [optional] + --idea for idea plugin [optional] ``` * datasource ```shell script - goctl model mysql datasource -url={datasource} -table={tables} -dir={dir} -cache=true + goctl model mysql datasource -url={datasource} -table={patterns} -dir={dir} -cache=true ``` + + help + + ``` + NAME: + goctl model mysql datasource - generate model from datasource + + USAGE: + goctl model mysql datasource [command options] [arguments...] + + OPTIONS: + --url value the data source of database,like "root:password@tcp(127.0.0.1:3306)/database + --table value, -t value the table or table globbing patterns in the database + --cache, -c generate code with cache [optional] + --dir value, -d value the target dir + --idea for idea plugin [optional] + ``` + + 示例用法请参考[用法](./example/generator.sh) 目前仅支持redis缓存，如果选择带缓存模式，即生成的`FindOne(ByXxx)`&`Delete`代码会生成带缓存逻辑的代码，目前仅支持单索引字段（除全文索引外），对于联合索引我们默认认为不需要带缓存，且不属于通用型代码，因此没有放在代码生成行列，如example中user表中的`id`、`name`、`mobile`字段均属于单字段索引。 @@ -221,26 +256,26 @@ OPTIONS: * ddl ```shell script - goctl model -src={filename} -dir={dir} + goctl model -src={patterns} -dir={dir} ``` * datasource ```shell script - goctl model mysql datasource -url={datasource} -table={tables} -dir={dir} + goctl model mysql datasource -url={datasource} -table={patterns} -dir={dir} ``` or * ddl ```shell script - goctl model -src={filename} -dir={dir} -cache=false + goctl model -src={patterns} -dir={dir} -cache=false ``` * datasource ```shell script - goctl model mysql datasource -url={datasource} -table={tables} -dir={dir} -cache=false + goctl model mysql datasource -url={datasource} -table={patterns} -dir={dir} -cache=false ``` 生成代码仅基本的CURD结构。 @@ -265,8 +300,3 @@ OPTIONS: 目前，我认为除了基本的CURD外，其他的代码均属于业务型代码，这个我觉得开发人员根据业务需要进行编写更好。 -## QA - -* goctl model除了命令行模式，支持插件模式吗？ - - 很快支持idea插件。 diff --git a/tools/goctl/model/sql/command/command.go b/tools/goctl/model/sql/command/command.go index ea54b445..087dbc59 100644 --- a/tools/goctl/model/sql/command/command.go +++ b/tools/goctl/model/sql/command/command.go @@ -1,15 +1,17 @@ package command import ( + "errors" "io/ioutil" "path/filepath" "strings" - "github.com/tal-tech/go-zero/core/collection" + "github.com/go-sql-driver/mysql" "github.com/tal-tech/go-zero/core/logx" "github.com/tal-tech/go-zero/core/stores/sqlx" "github.com/tal-tech/go-zero/tools/goctl/model/sql/gen" "github.com/tal-tech/go-zero/tools/goctl/model/sql/model" + "github.com/tal-tech/go-zero/tools/goctl/model/sql/util" "github.com/tal-tech/go-zero/tools/goctl/util/console" "github.com/urfave/cli" ) @@ -29,16 +31,25 @@ func MysqlDDL(ctx *cli.Context) error { cache := ctx.Bool(flagCache) idea := ctx.Bool(flagIdea) log := console.NewConsole(idea) - fileSrc, err := filepath.Abs(src) - if err != nil { - return err + src = strings.TrimSpace(src) + if len(src) == 0 { + return errors.New("expected path or path globbing patterns, but nothing found") } - data, err := ioutil.ReadFile(fileSrc) + + files, err := util.MatchFiles(src) if err != nil { return err } - source := string(data) - generator := gen.NewDefaultGenerator(source, dir, gen.WithConsoleOption(log)) + + var source []string + for _, file := range files { + data, err := ioutil.ReadFile(file) + if err != nil { + return err + } + source = append(source, string(data)) + } + generator := gen.NewDefaultGenerator(strings.Join(source, "\n"), dir, gen.WithConsoleOption(log)) err = generator.Start(cache) if err != nil { log.Error("%v", err) @@ -51,36 +62,63 @@ func MyDataSource(ctx *cli.Context) error { dir := strings.TrimSpace(ctx.String(flagDir)) cache := ctx.Bool(flagCache) idea := ctx.Bool(flagIdea) - table := strings.TrimSpace(ctx.String(flagTable)) + pattern := strings.TrimSpace(ctx.String(flagTable)) log := console.NewConsole(idea) if len(url) == 0 { - log.Error("%v", "expected data source of mysql, but is empty") + log.Error("%v", "expected data source of mysql, but nothing found") return nil } - if len(table) == 0 { - log.Error("%v", "expected table(s), but nothing found") + + if len(pattern) == 0 { + log.Error("%v", "expected table or table globbing patterns, but nothing found") return nil } + + cfg, err := mysql.ParseDSN(url) + if err != nil { + return err + } + logx.Disable() conn := sqlx.NewMysql(url) + databaseSource := strings.TrimSuffix(url, "/"+cfg.DBName) + "/information_schema" + db := sqlx.NewMysql(databaseSource) m := model.NewDDLModel(conn) - tables := collection.NewSet() - for _, item := range strings.Split(table, ",") { - item = strings.TrimSpace(item) - if len(item) == 0 { + im := model.NewInformationSchemaModel(db) + + tables, err := im.GetAllTables(cfg.DBName) + if err != nil { + return err + } + + var matchTables []string + for _, item := range tables { + match, err := filepath.Match(pattern, item) + if err != nil { + return err + } + + if !match { continue } - tables.AddStr(item) + + matchTables = append(matchTables, item) + } + if len(matchTables) == 0 { + return errors.New("no tables matched") } - ddl, err := m.ShowDDL(tables.KeysStr()...) + + ddl, err := m.ShowDDL(matchTables...) if err != nil { log.Error("%v", err) return nil } + generator := gen.NewDefaultGenerator(strings.Join(ddl, "\n"), dir, gen.WithConsoleOption(log)) err = generator.Start(cache) if err != nil { log.Error("%v", err) } + return nil } diff --git a/tools/goctl/model/sql/example/generator.sh b/tools/goctl/model/sql/example/generator.sh index 3d087e0a..71ead77b 100644 --- a/tools/goctl/model/sql/example/generator.sh +++ b/tools/goctl/model/sql/example/generator.sh @@ -1,7 +1,11 @@ #!/bin/bash # generate model with cache from ddl -goctl model mysql ddl -src="./sql/user.sql" -dir="./sql/model" -c +goctl model mysql ddl -src="./sql/*.sql" -dir="./sql/model/user" -c # generate model with cache from data source -#goctl model mysql datasource -url="user:password@tcp(127.0.0.1:3306)/database" -table="table1,table2" -dir="./model" \ No newline at end of file +#user=root +#password=password +#datasource=127.0.0.1:3306 +#database=test +#goctl model mysql datasource -url="${user}:${password}@tcp(${datasource})/${database}" -table="*" -dir ./model \ No newline at end of file diff --git a/tools/goctl/model/sql/example/sql/user_1.sql b/tools/goctl/model/sql/example/sql/user_1.sql new file mode 100644 index 00000000..0ecc666a --- /dev/null +++ b/tools/goctl/model/sql/example/sql/user_1.sql @@ -0,0 +1,15 @@ +-- 用户表 -- +CREATE TABLE `user1` ( + `id` bigint(10) NOT NULL AUTO_INCREMENT, + `name` varchar(255) COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT '用户名称', + `password` varchar(255) COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT '用户密码', + `mobile` varchar(255) COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT '手机号', + `gender` char(5) COLLATE utf8mb4_general_ci NOT NULL COMMENT '男｜女｜未公开', + `nickname` varchar(255) COLLATE utf8mb4_general_ci DEFAULT '' COMMENT '用户昵称', + `create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP, + `update_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `name_index` (`name`), + UNIQUE KEY `mobile_index` (`mobile`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; + diff --git a/tools/goctl/model/sql/gen/gen.go b/tools/goctl/model/sql/gen/gen.go index f351676c..dd0badb7 100644 --- a/tools/goctl/model/sql/gen/gen.go +++ b/tools/goctl/model/sql/gen/gen.go @@ -24,6 +24,7 @@ type ( source string dir string console.Console + pkg string } Option func(generator *defaultGenerator) ) @@ -59,6 +60,8 @@ func (g *defaultGenerator) Start(withCache bool) error { if err != nil { return err } + g.dir = dirAbs + g.pkg = filepath.Base(dirAbs) err = util.MkdirIfNotExist(dirAbs) if err != nil { return err @@ -82,12 +85,18 @@ func (g *defaultGenerator) Start(withCache bool) error { } // generate error file filename := filepath.Join(dirAbs, "vars.go") - if !util.FileExists(filename) { - err = ioutil.WriteFile(filename, []byte(template.Error), os.ModePerm) - if err != nil { - return err - } + text, err := util.LoadTemplate(category, modelTemplateFile, template.Error) + if err != nil { + return err } + + err = util.With("vars").Parse(text).SaveTo(map[string]interface{}{ + "pkg": g.pkg, + }, filename, false) + if err != nil { + return err + } + g.Success("Done.") return nil } @@ -119,8 +128,12 @@ type ( ) func (g *defaultGenerator) genModel(in parser.Table, withCache bool) (string, error) { + text, err := util.LoadTemplate(category, modelTemplateFile, template.Model) + if err != nil { + return "", err + } t := util.With("model"). - Parse(template.Model). + Parse(text). GoFmt(true) m, err := genCacheKeys(in) @@ -188,6 +201,7 @@ func (g *defaultGenerator) genModel(in parser.Table, withCache bool) (string, er } output, err := t.Execute(map[string]interface{}{ + "pkg": g.pkg, "imports": importsCode, "vars": varsCode, "types": typesCode, diff --git a/tools/goctl/model/sql/gen/gen_test.go b/tools/goctl/model/sql/gen/gen_test.go new file mode 100644 index 00000000..78bd1434 --- /dev/null +++ b/tools/goctl/model/sql/gen/gen_test.go @@ -0,0 +1,18 @@ +package gen + +import ( + "testing" + + "github.com/tal-tech/go-zero/core/logx" +) + +var ( + source = "-- 用户表 --\nCREATE TABLE `user` (\n `id` bigint(10) NOT NULL AUTO_INCREMENT,\n `name` varchar(255) COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT '用户名称',\n `password` varchar(255) COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT '用户密码',\n `mobile` varchar(255) COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT '手机号',\n `gender` char(5) COLLATE utf8mb4_general_ci NOT NULL COMMENT '男｜女｜未公开',\n `nickname` varchar(255) COLLATE utf8mb4_general_ci DEFAULT '' COMMENT '用户昵称',\n `create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP,\n `update_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,\n PRIMARY KEY (`id`),\n UNIQUE KEY `name_index` (`name`),\n UNIQUE KEY `mobile_index` (`mobile`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;\n\n" +) + +func TestNewDefaultGenerator(t *testing.T) { + _ = Clean() + g := NewDefaultGenerator(source, "./model/user") + err := g.Start(true) + logx.Must(err) +} diff --git a/tools/goctl/model/sql/gen/new.go b/tools/goctl/model/sql/gen/new.go index e3bcf639..6976ffc9 100644 --- a/tools/goctl/model/sql/gen/new.go +++ b/tools/goctl/model/sql/gen/new.go @@ -14,6 +14,7 @@ func genNew(table Table, withCache bool) (string, error) { output, err := util.With("new"). Parse(text). Execute(map[string]interface{}{ + "table": table.Name.Source(), "withCache": withCache, "upperStartCamelObject": table.Name.ToCamel(), }) diff --git a/tools/goctl/model/sql/gen/template.go b/tools/goctl/model/sql/gen/template.go index 31f2bcf4..e6421f1f 100644 --- a/tools/goctl/model/sql/gen/template.go +++ b/tools/goctl/model/sql/gen/template.go @@ -24,6 +24,7 @@ const ( typesTemplateFile = "types.tpl" updateTemplateFile = "update.tpl" varTemplateFile = "var.tpl" + errTemplateFile = "err.tpl" ) var templates = map[string]string{ @@ -41,6 +42,7 @@ var templates = map[string]string{ typesTemplateFile: template.Types, updateTemplateFile: template.Update, varTemplateFile: template.Vars, + errTemplateFile: template.Error, } func GenTemplates(_ *cli.Context) error { diff --git a/tools/goctl/model/sql/model/informationschemamodel.go b/tools/goctl/model/sql/model/informationschemamodel.go new file mode 100644 index 00000000..942f6137 --- /dev/null +++ b/tools/goctl/model/sql/model/informationschemamodel.go @@ -0,0 +1,25 @@ +package model + +import ( + "github.com/tal-tech/go-zero/core/stores/sqlx" +) + +type ( + InformationSchemaModel struct { + conn sqlx.SqlConn + } +) + +func NewInformationSchemaModel(conn sqlx.SqlConn) *InformationSchemaModel { + return &InformationSchemaModel{conn: conn} +} + +func (m *InformationSchemaModel) GetAllTables(database string) ([]string, error) { + query := `select TABLE_NAME from TABLES where TABLE_SCHEMA = ?` + var tables []string + err := m.conn.QueryRows(&tables, query, database) + if err != nil { + return nil, err + } + return tables, nil +} diff --git a/tools/goctl/model/sql/template/errors.go b/tools/goctl/model/sql/template/errors.go index f35e8942..eab19d9b 100644 --- a/tools/goctl/model/sql/template/errors.go +++ b/tools/goctl/model/sql/template/errors.go @@ -1,6 +1,6 @@ package template -var Error = `package model +var Error = `package {{.pkg}} import "github.com/tal-tech/go-zero/core/stores/sqlx" diff --git a/tools/goctl/model/sql/template/model.go b/tools/goctl/model/sql/template/model.go index cdc711c4..4fdb0e00 100644 --- a/tools/goctl/model/sql/template/model.go +++ b/tools/goctl/model/sql/template/model.go @@ -1,6 +1,6 @@ package template -var Model = `package model +var Model = `package {{.pkg}} {{.imports}} {{.vars}} {{.types}} diff --git a/tools/goctl/model/sql/template/new.go b/tools/goctl/model/sql/template/new.go index 2017259a..e42d3605 100644 --- a/tools/goctl/model/sql/template/new.go +++ b/tools/goctl/model/sql/template/new.go @@ -1,10 +1,10 @@ package template var New = ` -func New{{.upperStartCamelObject}}Model(conn sqlx.SqlConn,{{if .withCache}} c cache.CacheConf,{{end}} table string) *{{.upperStartCamelObject}}Model { +func New{{.upperStartCamelObject}}Model(conn sqlx.SqlConn{{if .withCache}}, c cache.CacheConf{{end}}) *{{.upperStartCamelObject}}Model { return &{{.upperStartCamelObject}}Model{ {{if .withCache}}CachedConn: sqlc.NewConn(conn, c){{else}}conn:conn{{end}}, - table: table, + table: "{{.table}}", } } ` diff --git a/tools/goctl/model/sql/util/match_test.go b/tools/goctl/model/sql/util/match_test.go new file mode 100644 index 00000000..b3e62ca2 --- /dev/null +++ b/tools/goctl/model/sql/util/match_test.go @@ -0,0 +1,29 @@ +package util + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMatchFiles(t *testing.T) { + dir, err := filepath.Abs("./") + assert.Nil(t, err) + + files, err := MatchFiles("./*.sql") + assert.Nil(t, err) + assert.Equal(t, []string{filepath.Join(dir, "studeat.sql"), filepath.Join(dir, "student.sql"), filepath.Join(dir, "xx.sql")}, files) + + files, err = MatchFiles("./??.sql") + assert.Nil(t, err) + assert.Equal(t, []string{filepath.Join(dir, "xx.sql")}, files) + + files, err = MatchFiles("./*.sq*") + assert.Nil(t, err) + assert.Equal(t, []string{filepath.Join(dir, "studeat.sql"), filepath.Join(dir, "student.sql"), filepath.Join(dir, "xx.sql"), filepath.Join(dir, "xx.sql1")}, files) + + files, err = MatchFiles("./student.sql") + assert.Nil(t, err) + assert.Equal(t, []string{filepath.Join(dir, "student.sql")}, files) +} diff --git a/tools/goctl/model/sql/util/matcher.go b/tools/goctl/model/sql/util/matcher.go new file mode 100644 index 00000000..fd123927 --- /dev/null +++ b/tools/goctl/model/sql/util/matcher.go @@ -0,0 +1,38 @@ +package util + +import ( + "io/ioutil" + "path/filepath" +) + +// expression: globbing patterns +func MatchFiles(in string) ([]string, error) { + dir, pattern := filepath.Split(in) + abs, err := filepath.Abs(dir) + if err != nil { + return nil, err + } + + files, err := ioutil.ReadDir(abs) + if err != nil { + return nil, err + } + var res []string + for _, file := range files { + if file.IsDir() { + continue + } + name := file.Name() + match, err := filepath.Match(pattern, name) + if err != nil { + return nil, err + } + + if !match { + continue + } + + res = append(res, filepath.Join(abs, name)) + } + return res, nil +} diff --git a/tools/goctl/model/sql/util/studeat.sql b/tools/goctl/model/sql/util/studeat.sql new file mode 100644 index 00000000..e69de29b diff --git a/tools/goctl/model/sql/util/student.sql b/tools/goctl/model/sql/util/student.sql new file mode 100644 index 00000000..e69de29b diff --git a/tools/goctl/model/sql/util/sub/sub.sql b/tools/goctl/model/sql/util/sub/sub.sql new file mode 100644 index 00000000..e69de29b diff --git a/tools/goctl/model/sql/util/xx.sql b/tools/goctl/model/sql/util/xx.sql new file mode 100644 index 00000000..e69de29b diff --git a/tools/goctl/model/sql/util/xx.sql1 b/tools/goctl/model/sql/util/xx.sql1 new file mode 100644 index 00000000..e69de29b