From 13477238a37452d2b162f4158362f627fd942928 Mon Sep 17 00:00:00 2001 From: Kevin Wan Date: Sat, 16 Jul 2022 14:11:34 +0800 Subject: [PATCH] feat: restful -> grpc gateway (#2155) * Revert "chore: remove unimplemented gateway (#2139)" This reverts commit d70e73ec66a37a173b7fe9a01193bf85ddb555e0. * feat: working gateway * feat: use mr to make it faster * feat: working gateway * chore: add comments * feat: support protoset besides reflection * feat: support zrpc client conf * docs: update readme * feat: support grpc-metadata- header to gateway- header conversion * chore: add docs --- gateway/config.go | 35 ++++++++++ gateway/headerbuilder.go | 29 +++++++++ gateway/headerbuilder_test.go | 21 ++++++ gateway/readme.md | 56 ++++++++++++++++ gateway/requestparser.go | 43 +++++++++++++ gateway/requestparser_test.go | 48 ++++++++++++++ gateway/server.go | 116 ++++++++++++++++++++++++++++++++++ go.mod | 3 + go.sum | 15 +++++ 9 files changed, 366 insertions(+) create mode 100644 gateway/config.go create mode 100644 gateway/headerbuilder.go create mode 100644 gateway/headerbuilder_test.go create mode 100644 gateway/readme.md create mode 100644 gateway/requestparser.go create mode 100644 gateway/requestparser_test.go create mode 100644 gateway/server.go diff --git a/gateway/config.go b/gateway/config.go new file mode 100644 index 00000000..aac835aa --- /dev/null +++ b/gateway/config.go @@ -0,0 +1,35 @@ +package gateway + +import ( + "time" + + "github.com/zeromicro/go-zero/rest" + "github.com/zeromicro/go-zero/zrpc" +) + +type ( + // GatewayConf is the configuration for gateway. + GatewayConf struct { + rest.RestConf + Upstreams []upstream + Timeout time.Duration `json:",default=5s"` + } + + // mapping is a mapping between a gateway route and a upstream rpc method. + mapping struct { + // Method is the HTTP method, like GET, POST, PUT, DELETE. + Method string + // Path is the HTTP path. + Path string + // Rpc is the gRPC rpc method, with format of package.service/method + Rpc string + } + // upstream is the configuration for upstream. + upstream struct { + // Grpc is the target of upstream. + Grpc zrpc.RpcClientConf + // ProtoSet is the file of proto set, like hello.pb + ProtoSet string `json:",optional"` + Mapping []mapping + } +) diff --git a/gateway/headerbuilder.go b/gateway/headerbuilder.go new file mode 100644 index 00000000..918f53d6 --- /dev/null +++ b/gateway/headerbuilder.go @@ -0,0 +1,29 @@ +package gateway + +import ( + "fmt" + "net/http" + "strings" +) + +const ( + metadataHeaderPrefix = "Grpc-Metadata-" + metadataPrefix = "gateway-" +) + +func buildHeaders(header http.Header) []string { + var headers []string + + for k, v := range header { + if !strings.HasPrefix(k, metadataHeaderPrefix) { + continue + } + + key := fmt.Sprintf("%s%s", metadataPrefix, strings.TrimPrefix(k, metadataHeaderPrefix)) + for _, vv := range v { + headers = append(headers, key+":"+vv) + } + } + + return headers +} diff --git a/gateway/headerbuilder_test.go b/gateway/headerbuilder_test.go new file mode 100644 index 00000000..32efd254 --- /dev/null +++ b/gateway/headerbuilder_test.go @@ -0,0 +1,21 @@ +package gateway + +import ( + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBuildHeadersNoValue(t *testing.T) { + req := httptest.NewRequest("GET", "/", nil) + req.Header.Add("a", "b") + assert.Nil(t, buildHeaders(req.Header)) +} + +func TestBuildHeadersWithValues(t *testing.T) { + req := httptest.NewRequest("GET", "/", nil) + req.Header.Add("grpc-metadata-a", "b") + req.Header.Add("grpc-metadata-b", "b") + assert.EqualValues(t, []string{"gateway-A:b", "gateway-B:b"}, buildHeaders(req.Header)) +} diff --git a/gateway/readme.md b/gateway/readme.md new file mode 100644 index 00000000..ea87b1bc --- /dev/null +++ b/gateway/readme.md @@ -0,0 +1,56 @@ +# Gateway + +## Usage + +- main.go + +```go +var configFile = flag.String("f", "config.yaml", "config file") + +func main() { + flag.Parse() + + var c gateway.GatewayConf + conf.MustLoad(*configFile, &c) + gw := gateway.MustNewServer(c) + defer gw.Stop() + gw.Start() +} +``` + +- config.yaml + +```yaml +Name: demo-gateway +Host: localhost +Port: 8888 +Upstreams: + - Grpc: + Etcd: + Hosts: + - localhost:2379 + Key: hello.rpc + # protoset mode + ProtoSet: hello.pb + Mapping: + - Method: get + Path: /pingHello/:ping + Rpc: hello.Hello/Ping + - Grpc: + Endpoints: + - localhost:8081 + # reflection mode, no ProtoSet settings + Mapping: + - Method: post + Path: /pingWorld + Rpc: world.World/Ping +``` + +## Generate ProtoSet files + +- example command + +```shell +protoc --descriptor_set_out=hello.pb hello.proto +``` + diff --git a/gateway/requestparser.go b/gateway/requestparser.go new file mode 100644 index 00000000..ed806319 --- /dev/null +++ b/gateway/requestparser.go @@ -0,0 +1,43 @@ +package gateway + +import ( + "bytes" + "encoding/json" + "net/http" + + "github.com/fullstorydev/grpcurl" + "github.com/golang/protobuf/jsonpb" + "github.com/zeromicro/go-zero/rest/pathvar" +) + +func newRequestParser(r *http.Request, resolver jsonpb.AnyResolver) (grpcurl.RequestParser, error) { + vars := pathvar.Vars(r) + if len(vars) == 0 { + return grpcurl.NewJSONRequestParser(r.Body, resolver), nil + } + + if r.ContentLength == 0 { + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(vars); err != nil { + return nil, err + } + + return grpcurl.NewJSONRequestParser(&buf, resolver), nil + } + + m := make(map[string]interface{}) + if err := json.NewDecoder(r.Body).Decode(&m); err != nil { + return nil, err + } + + for k, v := range vars { + m[k] = v + } + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(m); err != nil { + return nil, err + } + + return grpcurl.NewJSONRequestParser(&buf, resolver), nil +} diff --git a/gateway/requestparser_test.go b/gateway/requestparser_test.go new file mode 100644 index 00000000..3659b4d1 --- /dev/null +++ b/gateway/requestparser_test.go @@ -0,0 +1,48 @@ +package gateway + +import ( + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/zeromicro/go-zero/rest/pathvar" +) + +func TestNewRequestParserNoVar(t *testing.T) { + req := httptest.NewRequest("GET", "/", nil) + parser, err := newRequestParser(req, nil) + assert.Nil(t, err) + assert.NotNil(t, parser) +} + +func TestNewRequestParserWithVars(t *testing.T) { + req := httptest.NewRequest("GET", "/", nil) + req = pathvar.WithVars(req, map[string]string{"a": "b"}) + parser, err := newRequestParser(req, nil) + assert.Nil(t, err) + assert.NotNil(t, parser) +} + +func TestNewRequestParserNoVarWithBody(t *testing.T) { + req := httptest.NewRequest("GET", "/", strings.NewReader(`{"a": "b"}`)) + parser, err := newRequestParser(req, nil) + assert.Nil(t, err) + assert.NotNil(t, parser) +} + +func TestNewRequestParserWithVarsWithBody(t *testing.T) { + req := httptest.NewRequest("GET", "/", strings.NewReader(`{"a": "b"}`)) + req = pathvar.WithVars(req, map[string]string{"c": "d"}) + parser, err := newRequestParser(req, nil) + assert.Nil(t, err) + assert.NotNil(t, parser) +} + +func TestNewRequestParserWithVarsWithWrongBody(t *testing.T) { + req := httptest.NewRequest("GET", "/", strings.NewReader(`{"a": "b"`)) + req = pathvar.WithVars(req, map[string]string{"c": "d"}) + parser, err := newRequestParser(req, nil) + assert.NotNil(t, err) + assert.Nil(t, parser) +} diff --git a/gateway/server.go b/gateway/server.go new file mode 100644 index 00000000..02c26d00 --- /dev/null +++ b/gateway/server.go @@ -0,0 +1,116 @@ +package gateway + +import ( + "context" + "net/http" + "strings" + "time" + + "github.com/fullstorydev/grpcurl" + "github.com/golang/protobuf/jsonpb" + "github.com/jhump/protoreflect/grpcreflect" + "github.com/zeromicro/go-zero/core/logx" + "github.com/zeromicro/go-zero/core/mr" + "github.com/zeromicro/go-zero/rest" + "github.com/zeromicro/go-zero/rest/httpx" + "github.com/zeromicro/go-zero/zrpc" + "google.golang.org/grpc/reflection/grpc_reflection_v1alpha" +) + +// Server is a gateway server. +type Server struct { + svr *rest.Server + upstreams []upstream + timeout time.Duration +} + +// MustNewServer creates a new gateway server. +func MustNewServer(c GatewayConf) *Server { + return &Server{ + svr: rest.MustNewServer(c.RestConf), + upstreams: c.Upstreams, + timeout: c.Timeout, + } +} + +// Start starts the gateway server. +func (s *Server) Start() { + logx.Must(s.build()) + s.svr.Start() +} + +// Stop stops the gateway server. +func (s *Server) Stop() { + s.svr.Stop() +} + +func (s *Server) build() error { + return mr.MapReduceVoid(func(source chan<- interface{}) { + for _, up := range s.upstreams { + source <- up + } + }, func(item interface{}, writer mr.Writer, cancel func(error)) { + up := item.(upstream) + cli := zrpc.MustNewClient(up.Grpc) + source, err := s.createDescriptorSource(cli, up) + if err != nil { + cancel(err) + return + } + + resolver := grpcurl.AnyResolverFromDescriptorSource(source) + for _, m := range up.Mapping { + writer.Write(rest.Route{ + Method: strings.ToUpper(m.Method), + Path: m.Path, + Handler: s.buildHandler(source, resolver, cli, m), + }) + } + }, func(pipe <-chan interface{}, cancel func(error)) { + for item := range pipe { + route := item.(rest.Route) + s.svr.AddRoute(route) + } + }) +} + +func (s *Server) buildHandler(source grpcurl.DescriptorSource, resolver jsonpb.AnyResolver, + cli zrpc.Client, m mapping) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + handler := &grpcurl.DefaultEventHandler{ + Out: w, + Formatter: grpcurl.NewJSONFormatter(true, + grpcurl.AnyResolverFromDescriptorSource(source)), + } + parser, err := newRequestParser(r, resolver) + if err != nil { + httpx.Error(w, err) + return + } + + ctx, can := context.WithTimeout(r.Context(), s.timeout) + defer can() + if err := grpcurl.InvokeRPC(ctx, source, cli.Conn(), m.Rpc, buildHeaders(r.Header), + handler, parser.Next); err != nil { + httpx.Error(w, err) + } + } +} + +func (s *Server) createDescriptorSource(cli zrpc.Client, up upstream) (grpcurl.DescriptorSource, error) { + var source grpcurl.DescriptorSource + var err error + + if len(up.ProtoSet) > 0 { + source, err = grpcurl.DescriptorSourceFromProtoSets(up.ProtoSet) + if err != nil { + return nil, err + } + } else { + refCli := grpc_reflection_v1alpha.NewServerReflectionClient(cli.Conn()) + client := grpcreflect.NewClient(context.Background(), refCli) + source = grpcurl.DescriptorSourceFromServer(context.Background(), client) + } + + return source, nil +} diff --git a/go.mod b/go.mod index 3053a124..090d3f93 100644 --- a/go.mod +++ b/go.mod @@ -7,12 +7,15 @@ require ( github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/alicebob/miniredis/v2 v2.22.0 github.com/fatih/color v1.13.0 + github.com/fullstorydev/grpcurl v1.8.6 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 github.com/go-redis/redis/v8 v8.11.5 github.com/go-sql-driver/mysql v1.6.0 github.com/golang-jwt/jwt/v4 v4.4.2 github.com/golang/mock v1.6.0 + github.com/golang/protobuf v1.5.2 github.com/google/uuid v1.3.0 + github.com/jhump/protoreflect v1.12.0 github.com/justinas/alice v1.2.0 github.com/lib/pq v1.10.6 github.com/olekukonko/tablewriter v0.0.5 diff --git a/go.sum b/go.sum index 6f7000da..6f39ce1b 100644 --- a/go.sum +++ b/go.sum @@ -122,6 +122,8 @@ github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fullstorydev/grpcurl v1.8.6 h1:WylAwnPauJIofYSHqqMTC1eEfUIzqzevXyogBxnQquo= +github.com/fullstorydev/grpcurl v1.8.6/go.mod h1:WhP7fRQdhxz2TkL97u+TCb505sxfH78W1usyoB3tepw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= @@ -235,6 +237,7 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= +github.com/gordonklaus/ineffassign v0.0.0-20200309095847-7953dde2c7bf/go.mod h1:cuNKsD1zp2v6XfE/orVX2QE1LC+i254ceGcVeDT3pTU= github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= @@ -259,6 +262,13 @@ github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/U github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= +github.com/jhump/gopoet v0.0.0-20190322174617-17282ff210b3/go.mod h1:me9yfT6IJSlOL3FCfrg+L6yzUEZ+5jW6WHt4Sk+UPUI= +github.com/jhump/gopoet v0.1.0/go.mod h1:me9yfT6IJSlOL3FCfrg+L6yzUEZ+5jW6WHt4Sk+UPUI= +github.com/jhump/goprotoc v0.5.0/go.mod h1:VrbvcYrQOrTi3i0Vf+m+oqQWk9l72mjkJCYo7UvLHRQ= +github.com/jhump/protoreflect v1.10.3/go.mod h1:7GcYQDdMU/O/BBrl/cX6PNHpXh6cenjd8pneu5yW7Tg= +github.com/jhump/protoreflect v1.11.0/go.mod h1:U7aMIjN0NWq9swDP7xDdoMfRHb35uiuTd3Z9nFXJf5E= +github.com/jhump/protoreflect v1.12.0 h1:1NQ4FpWMgn3by/n1X0fbeKEUxP1wBt7+Oitpv01HR10= +github.com/jhump/protoreflect v1.12.0/go.mod h1:JytZfP5d0r8pVNLZvai7U/MCuTWITgrI4tTg7puQFKI= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -323,6 +333,7 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+ github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nishanths/predeclared v0.0.0-20200524104333-86fad755b4d3/go.mod h1:nt3d53pc1VYcphSCIaYAJtnPYnr3Zyn8fMq2wvPGPso= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -711,8 +722,10 @@ golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWc golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200717024301-6ddee64345a6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= @@ -799,6 +812,7 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.48.0 h1:rQOsyJ/8+ufEDJd/Gdsz7HG220Mh9HAhFHRGnIjda0w= google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= @@ -812,6 +826,7 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.25.1-0.20200805231151-a709e31e5d12/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=