model support globbing patterns (#153)

* model support globbing patterns

* optimize model

* optimize model

* format code

tools/goctl/goctl.go

@@ -269,7 +269,7 @@ var (
 							Flags: []cli.Flag{
 								cli.StringFlag{
 									Name:  "src, s",
-									Usage: "the file path of the ddl source file",
+									Usage: "the path or path globbing patterns of the ddl",
 								},
 								cli.StringFlag{
 									Name:  "dir, d",
@@ -296,7 +296,7 @@ var (
 								},
 								cli.StringFlag{
 									Name:  "table, t",
-									Usage: `source table,tables separated by commas,like "user,course`,
+									Usage: `the table or table globbing patterns in the database`,
 								},
 								cli.BoolFlag{
 									Name:  "cache, c",

tools/goctl/model/sql/README.MD

@@ -7,7 +7,7 @@ goctl model 为go-zero下的工具模块中的组件之一，目前支持识别m
 * 通过ddl生成
 
     ```shell script
-    goctl model mysql ddl -src="./sql/user.sql" -dir="./sql/model" -c=true
+    goctl model mysql ddl -src="./*.sql" -dir="./sql/model" -c=true
     ```
 
     执行上述命令后即可快速生成CURD代码。
@@ -21,7 +21,7 @@ goctl model 为go-zero下的工具模块中的组件之一，目前支持识别m
 * 通过datasource生成
 
     ```shell script
-    goctl model mysql datasource -url="user:password@tcp(127.0.0.1:3306)/database" -table="table1,table2"  -dir="./model"
+    goctl model mysql datasource -url="user:password@tcp(127.0.0.1:3306)/database" -table="*"  -dir="./model"
     ```
 
 * 生成代码示例
@@ -205,15 +205,50 @@ OPTIONS:
   * ddl
 
 	```shell script
-	goctl model mysql -src={filename} -dir={dir} -cache=true
+	goctl model mysql -src={patterns} -dir={dir} -cache=true
+	```
+
+	help
+
+	```
+	NAME:
+		goctl model mysql ddl - generate mysql model from ddl
+
+		USAGE:
+		goctl model mysql ddl [command options] [arguments...]
+
+		OPTIONS:
+		--src value, -s value  the path or path globbing patterns of the ddl
+		--dir value, -d value  the target dir
+		--cache, -c            generate code with cache [optional]
+		--idea                 for idea plugin [optional]
 	```
 
   * datasource
 
 	```shell script
-	goctl model mysql datasource -url={datasource} -table={tables}  -dir={dir} -cache=true
+	goctl model mysql datasource -url={datasource} -table={patterns}  -dir={dir} -cache=true
 	```
 
+	help
+
+	```
+		NAME:
+		goctl model mysql datasource - generate model from datasource
+
+		USAGE:
+		goctl model mysql datasource [command options] [arguments...]
+
+		OPTIONS:
+		--url value              the data source of database,like "root:password@tcp(127.0.0.1:3306)/database
+		--table value, -t value  the table or table globbing patterns in the database
+		--cache, -c              generate code with cache [optional]
+		--dir value, -d value    the target dir
+		--idea                   for idea plugin [optional]
+	```
+
+	示例用法请参考[用法](./example/generator.sh)
+  
   目前仅支持redis缓存，如果选择带缓存模式，即生成的`FindOne(ByXxx)`&`Delete`代码会生成带缓存逻辑的代码，目前仅支持单索引字段（除全文索引外），对于联合索引我们默认认为不需要带缓存，且不属于通用型代码，因此没有放在代码生成行列，如example中user表中的`id`、`name`、`mobile`字段均属于单字段索引。
 
 * 不带缓存模式
@@ -221,26 +256,26 @@ OPTIONS:
   * ddl
   
       ```shell script
-        goctl model -src={filename} -dir={dir}
+        goctl model -src={patterns} -dir={dir}
       ```
 
   * datasource
   
       ```shell script
-        goctl model mysql datasource -url={datasource} -table={tables}  -dir={dir}
+        goctl model mysql datasource -url={datasource} -table={patterns}  -dir={dir}
       ```
 
   or
   * ddl
 
       ```shell script
-        goctl model -src={filename} -dir={dir} -cache=false
+        goctl model -src={patterns} -dir={dir} -cache=false
       ```
 
   * datasource
   
       ```shell script
-        goctl model mysql datasource -url={datasource} -table={tables}  -dir={dir} -cache=false
+        goctl model mysql datasource -url={datasource} -table={patterns}  -dir={dir} -cache=false
       ```
   
 生成代码仅基本的CURD结构。
@@ -265,8 +300,3 @@ OPTIONS:
   
   目前，我认为除了基本的CURD外，其他的代码均属于<i>业务型</i>代码，这个我觉得开发人员根据业务需要进行编写更好。
 
-## QA
-
-* goctl model除了命令行模式，支持插件模式吗？
-
-  很快支持idea插件。

tools/goctl/model/sql/command/command.go

@@ -1,15 +1,17 @@
 package command
 
 import (
+	"errors"
 	"io/ioutil"
 	"path/filepath"
 	"strings"
 
-	"github.com/tal-tech/go-zero/core/collection"
+	"github.com/go-sql-driver/mysql"
 	"github.com/tal-tech/go-zero/core/logx"
 	"github.com/tal-tech/go-zero/core/stores/sqlx"
 	"github.com/tal-tech/go-zero/tools/goctl/model/sql/gen"
 	"github.com/tal-tech/go-zero/tools/goctl/model/sql/model"
+	"github.com/tal-tech/go-zero/tools/goctl/model/sql/util"
 	"github.com/tal-tech/go-zero/tools/goctl/util/console"
 	"github.com/urfave/cli"
 )
@@ -29,16 +31,25 @@ func MysqlDDL(ctx *cli.Context) error {
 	cache := ctx.Bool(flagCache)
 	idea := ctx.Bool(flagIdea)
 	log := console.NewConsole(idea)
-	fileSrc, err := filepath.Abs(src)
+	src = strings.TrimSpace(src)
+	if len(src) == 0 {
+		return errors.New("expected path or path globbing patterns, but nothing found")
+	}
+
+	files, err := util.MatchFiles(src)
 	if err != nil {
 		return err
 	}
-	data, err := ioutil.ReadFile(fileSrc)
+
+	var source []string
+	for _, file := range files {
+		data, err := ioutil.ReadFile(file)
 		if err != nil {
 			return err
 		}
-	source := string(data)
+		source = append(source, string(data))
-	generator := gen.NewDefaultGenerator(source, dir, gen.WithConsoleOption(log))
+	}
+	generator := gen.NewDefaultGenerator(strings.Join(source, "\n"), dir, gen.WithConsoleOption(log))
 	err = generator.Start(cache)
 	if err != nil {
 		log.Error("%v", err)
@@ -51,36 +62,63 @@ func MyDataSource(ctx *cli.Context) error {
 	dir := strings.TrimSpace(ctx.String(flagDir))
 	cache := ctx.Bool(flagCache)
 	idea := ctx.Bool(flagIdea)
-	table := strings.TrimSpace(ctx.String(flagTable))
+	pattern := strings.TrimSpace(ctx.String(flagTable))
 	log := console.NewConsole(idea)
 	if len(url) == 0 {
-		log.Error("%v", "expected data source of mysql, but is empty")
+		log.Error("%v", "expected data source of mysql, but nothing found")
 		return nil
 	}
-	if len(table) == 0 {
+
-		log.Error("%v", "expected table(s), but nothing found")
+	if len(pattern) == 0 {
+		log.Error("%v", "expected table or table globbing patterns, but nothing found")
 		return nil
 	}
+
+	cfg, err := mysql.ParseDSN(url)
+	if err != nil {
+		return err
+	}
+
 	logx.Disable()
 	conn := sqlx.NewMysql(url)
+	databaseSource := strings.TrimSuffix(url, "/"+cfg.DBName) + "/information_schema"
+	db := sqlx.NewMysql(databaseSource)
 	m := model.NewDDLModel(conn)
-	tables := collection.NewSet()
+	im := model.NewInformationSchemaModel(db)
-	for _, item := range strings.Split(table, ",") {
+
-		item = strings.TrimSpace(item)
+	tables, err := im.GetAllTables(cfg.DBName)
-		if len(item) == 0 {
+	if err != nil {
+		return err
+	}
+
+	var matchTables []string
+	for _, item := range tables {
+		match, err := filepath.Match(pattern, item)
+		if err != nil {
+			return err
+		}
+
+		if !match {
 			continue
 		}
-		tables.AddStr(item)
+
+		matchTables = append(matchTables, item)
+	}
+	if len(matchTables) == 0 {
+		return errors.New("no tables matched")
 	}
-	ddl, err := m.ShowDDL(tables.KeysStr()...)
+
+	ddl, err := m.ShowDDL(matchTables...)
 	if err != nil {
 		log.Error("%v", err)
 		return nil
 	}
+
 	generator := gen.NewDefaultGenerator(strings.Join(ddl, "\n"), dir, gen.WithConsoleOption(log))
 	err = generator.Start(cache)
 	if err != nil {
 		log.Error("%v", err)
 	}
+
 	return nil
 }

tools/goctl/model/sql/example/generator.sh

@@ -1,7 +1,11 @@
 #!/bin/bash
 
 # generate model with cache from ddl
-goctl model mysql ddl -src="./sql/user.sql" -dir="./sql/model" -c
+goctl model mysql ddl -src="./sql/*.sql" -dir="./sql/model/user" -c
 
 # generate model with cache from data source
-#goctl model mysql datasource -url="user:password@tcp(127.0.0.1:3306)/database" -table="table1,table2"  -dir="./model"
+#user=root
+#password=password
+#datasource=127.0.0.1:3306
+#database=test
+#goctl model mysql datasource -url="${user}:${password}@tcp(${datasource})/${database}" -table="*" -dir ./model

tools/goctl/model/sql/example/sql/user_1.sql

@@ -0,0 +1,15 @@
+-- 用户表 --
+CREATE TABLE `user1` (
+  `id` bigint(10) NOT NULL AUTO_INCREMENT,
+  `name` varchar(255) COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT '用户名称',
+  `password` varchar(255) COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT '用户密码',
+  `mobile` varchar(255) COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT '手机号',
+  `gender` char(5) COLLATE utf8mb4_general_ci NOT NULL COMMENT '男｜女｜未公开',
+  `nickname` varchar(255) COLLATE utf8mb4_general_ci DEFAULT '' COMMENT '用户昵称',
+  `create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
+  `update_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `name_index` (`name`),
+  UNIQUE KEY `mobile_index` (`mobile`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
+

tools/goctl/model/sql/gen/gen.go

@@ -24,6 +24,7 @@ type (
 		source string
 		dir    string
 		console.Console
+		pkg string
 	}
 	Option func(generator *defaultGenerator)
 )
@@ -59,6 +60,8 @@ func (g *defaultGenerator) Start(withCache bool) error {
 	if err != nil {
 		return err
 	}
+	g.dir = dirAbs
+	g.pkg = filepath.Base(dirAbs)
 	err = util.MkdirIfNotExist(dirAbs)
 	if err != nil {
 		return err
@@ -82,12 +85,18 @@ func (g *defaultGenerator) Start(withCache bool) error {
 	}
 	// generate error file
 	filename := filepath.Join(dirAbs, "vars.go")
-	if !util.FileExists(filename) {
+	text, err := util.LoadTemplate(category, modelTemplateFile, template.Error)
-		err = ioutil.WriteFile(filename, []byte(template.Error), os.ModePerm)
 	if err != nil {
 		return err
 	}
+
+	err = util.With("vars").Parse(text).SaveTo(map[string]interface{}{
+		"pkg": g.pkg,
+	}, filename, false)
+	if err != nil {
+		return err
 	}
+
 	g.Success("Done.")
 	return nil
 }
@@ -119,8 +128,12 @@ type (
 )
 
 func (g *defaultGenerator) genModel(in parser.Table, withCache bool) (string, error) {
+	text, err := util.LoadTemplate(category, modelTemplateFile, template.Model)
+	if err != nil {
+		return "", err
+	}
 	t := util.With("model").
-		Parse(template.Model).
+		Parse(text).
 		GoFmt(true)
 
 	m, err := genCacheKeys(in)
@@ -188,6 +201,7 @@ func (g *defaultGenerator) genModel(in parser.Table, withCache bool) (string, er
 	}
 
 	output, err := t.Execute(map[string]interface{}{
+		"pkg":         g.pkg,
 		"imports":     importsCode,
 		"vars":        varsCode,
 		"types":       typesCode,

tools/goctl/model/sql/gen/gen_test.go

@@ -0,0 +1,18 @@
+package gen
+
+import (
+	"testing"
+
+	"github.com/tal-tech/go-zero/core/logx"
+)
+
+var (
+	source = "-- 用户表 --\nCREATE TABLE `user` (\n  `id` bigint(10) NOT NULL AUTO_INCREMENT,\n  `name` varchar(255) COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT '用户名称',\n  `password` varchar(255) COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT '用户密码',\n  `mobile` varchar(255) COLLATE utf8mb4_general_ci NOT NULL DEFAULT '' COMMENT '手机号',\n  `gender` char(5) COLLATE utf8mb4_general_ci NOT NULL COMMENT '男｜女｜未公开',\n  `nickname` varchar(255) COLLATE utf8mb4_general_ci DEFAULT '' COMMENT '用户昵称',\n  `create_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP,\n  `update_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,\n  PRIMARY KEY (`id`),\n  UNIQUE KEY `name_index` (`name`),\n  UNIQUE KEY `mobile_index` (`mobile`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;\n\n"
+)
+
+func TestNewDefaultGenerator(t *testing.T) {
+	_ = Clean()
+	g := NewDefaultGenerator(source, "./model/user")
+	err := g.Start(true)
+	logx.Must(err)
+}

tools/goctl/model/sql/gen/new.go

@@ -14,6 +14,7 @@ func genNew(table Table, withCache bool) (string, error) {
 	output, err := util.With("new").
 		Parse(text).
 		Execute(map[string]interface{}{
+			"table":                 table.Name.Source(),
 			"withCache":             withCache,
 			"upperStartCamelObject": table.Name.ToCamel(),
 		})

tools/goctl/model/sql/gen/template.go

@@ -24,6 +24,7 @@ const (
 	typesTemplateFile                     = "types.tpl"
 	updateTemplateFile                    = "update.tpl"
 	varTemplateFile                       = "var.tpl"
+	errTemplateFile                       = "err.tpl"
 )
 
 var templates = map[string]string{
@@ -41,6 +42,7 @@ var templates = map[string]string{
 	typesTemplateFile:                     template.Types,
 	updateTemplateFile:                    template.Update,
 	varTemplateFile:                       template.Vars,
+	errTemplateFile:                       template.Error,
 }
 
 func GenTemplates(_ *cli.Context) error {

tools/goctl/model/sql/model/informationschemamodel.go

@@ -0,0 +1,25 @@
+package model
+
+import (
+	"github.com/tal-tech/go-zero/core/stores/sqlx"
+)
+
+type (
+	InformationSchemaModel struct {
+		conn sqlx.SqlConn
+	}
+)
+
+func NewInformationSchemaModel(conn sqlx.SqlConn) *InformationSchemaModel {
+	return &InformationSchemaModel{conn: conn}
+}
+
+func (m *InformationSchemaModel) GetAllTables(database string) ([]string, error) {
+	query := `select TABLE_NAME from TABLES where TABLE_SCHEMA = ?`
+	var tables []string
+	err := m.conn.QueryRows(&tables, query, database)
+	if err != nil {
+		return nil, err
+	}
+	return tables, nil
+}

tools/goctl/model/sql/template/errors.go

@@ -1,6 +1,6 @@
 package template
 
-var Error = `package model
+var Error = `package {{.pkg}}
 
 import "github.com/tal-tech/go-zero/core/stores/sqlx"
 

tools/goctl/model/sql/template/model.go

@@ -1,6 +1,6 @@
 package template
 
-var Model = `package model
+var Model = `package {{.pkg}}
 {{.imports}}
 {{.vars}}
 {{.types}}

tools/goctl/model/sql/template/new.go

@@ -1,10 +1,10 @@
 package template
 
 var New = `
-func New{{.upperStartCamelObject}}Model(conn sqlx.SqlConn,{{if .withCache}} c cache.CacheConf,{{end}} table string) *{{.upperStartCamelObject}}Model {
+func New{{.upperStartCamelObject}}Model(conn sqlx.SqlConn{{if .withCache}}, c cache.CacheConf{{end}}) *{{.upperStartCamelObject}}Model {
 	return &{{.upperStartCamelObject}}Model{
 		{{if .withCache}}CachedConn: sqlc.NewConn(conn, c){{else}}conn:conn{{end}},
-		table:      table,
+		table:      "{{.table}}",
 	}
 }
 `

tools/goctl/model/sql/util/match_test.go

@@ -0,0 +1,29 @@
+package util
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMatchFiles(t *testing.T) {
+	dir, err := filepath.Abs("./")
+	assert.Nil(t, err)
+
+	files, err := MatchFiles("./*.sql")
+	assert.Nil(t, err)
+	assert.Equal(t, []string{filepath.Join(dir, "studeat.sql"), filepath.Join(dir, "student.sql"), filepath.Join(dir, "xx.sql")}, files)
+
+	files, err = MatchFiles("./??.sql")
+	assert.Nil(t, err)
+	assert.Equal(t, []string{filepath.Join(dir, "xx.sql")}, files)
+
+	files, err = MatchFiles("./*.sq*")
+	assert.Nil(t, err)
+	assert.Equal(t, []string{filepath.Join(dir, "studeat.sql"), filepath.Join(dir, "student.sql"), filepath.Join(dir, "xx.sql"), filepath.Join(dir, "xx.sql1")}, files)
+
+	files, err = MatchFiles("./student.sql")
+	assert.Nil(t, err)
+	assert.Equal(t, []string{filepath.Join(dir, "student.sql")}, files)
+}

tools/goctl/model/sql/util/matcher.go

@@ -0,0 +1,38 @@
+package util
+
+import (
+	"io/ioutil"
+	"path/filepath"
+)
+
+// expression: globbing patterns
+func MatchFiles(in string) ([]string, error) {
+	dir, pattern := filepath.Split(in)
+	abs, err := filepath.Abs(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	files, err := ioutil.ReadDir(abs)
+	if err != nil {
+		return nil, err
+	}
+	var res []string
+	for _, file := range files {
+		if file.IsDir() {
+			continue
+		}
+		name := file.Name()
+		match, err := filepath.Match(pattern, name)
+		if err != nil {
+			return nil, err
+		}
+
+		if !match {
+			continue
+		}
+
+		res = append(res, filepath.Join(abs, name))
+	}
+	return res, nil
+}