From 3819f67cf4ce4e8ba9eb9c417af2397e0a3b1e95 Mon Sep 17 00:00:00 2001 From: bittoy Date: Mon, 16 Nov 2020 19:45:43 +0800 Subject: [PATCH] add redis geospatial (#209) * add redis geospatial * fix go test error --- core/limit/periodlimit_test.go | 2 +- core/limit/tokenlimit_test.go | 2 +- core/stores/cache/cachenode_test.go | 2 +- core/stores/kv/store_test.go | 2 +- core/stores/redis/redis.go | 107 ++++++++++++++++++++ core/stores/redis/redis_test.go | 34 ++++++- core/stores/redis/redistest/redistest.go | 2 +- core/stores/sqlc/cachedsql_test.go | 2 +- go.mod | 7 +- go.sum | 15 +-- rest/handler/contentsecurityhandler.go | 6 +- rest/handler/contentsecurityhandler_test.go | 10 +- 12 files changed, 161 insertions(+), 30 deletions(-) diff --git a/core/limit/periodlimit_test.go b/core/limit/periodlimit_test.go index 609f4ccc..6709ad7f 100644 --- a/core/limit/periodlimit_test.go +++ b/core/limit/periodlimit_test.go @@ -3,7 +3,7 @@ package limit import ( "testing" - "github.com/alicebob/miniredis" + "github.com/alicebob/miniredis/v2" "github.com/stretchr/testify/assert" "github.com/tal-tech/go-zero/core/stores/redis" "github.com/tal-tech/go-zero/core/stores/redis/redistest" diff --git a/core/limit/tokenlimit_test.go b/core/limit/tokenlimit_test.go index 4c9ce0f9..ea15742a 100644 --- a/core/limit/tokenlimit_test.go +++ b/core/limit/tokenlimit_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/alicebob/miniredis" + "github.com/alicebob/miniredis/v2" "github.com/stretchr/testify/assert" "github.com/tal-tech/go-zero/core/logx" "github.com/tal-tech/go-zero/core/stores/redis" diff --git a/core/stores/cache/cachenode_test.go b/core/stores/cache/cachenode_test.go index d8004602..ca8d4bb0 100644 --- a/core/stores/cache/cachenode_test.go +++ b/core/stores/cache/cachenode_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/alicebob/miniredis" + "github.com/alicebob/miniredis/v2" "github.com/stretchr/testify/assert" "github.com/tal-tech/go-zero/core/logx" "github.com/tal-tech/go-zero/core/mathx" diff --git a/core/stores/kv/store_test.go b/core/stores/kv/store_test.go index 59258c58..ba789c2d 100644 --- a/core/stores/kv/store_test.go +++ b/core/stores/kv/store_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/alicebob/miniredis" + "github.com/alicebob/miniredis/v2" "github.com/stretchr/testify/assert" "github.com/tal-tech/go-zero/core/hash" "github.com/tal-tech/go-zero/core/stores/cache" diff --git a/core/stores/redis/redis.go b/core/stores/redis/redis.go index 52219aa9..5dfb86db 100644 --- a/core/stores/redis/redis.go +++ b/core/stores/redis/redis.go @@ -42,6 +42,12 @@ type ( red.Cmdable } + // GeoLocation is used with GeoAdd to add geospatial location. + GeoLocation = red.GeoLocation + // GeoRadiusQuery is used with GeoRadius to query geospatial index. + GeoRadiusQuery = red.GeoRadiusQuery + GeoPos = red.GeoPos + Pipeliner = red.Pipeliner // Z represents sorted set member. @@ -173,6 +179,107 @@ func (s *Redis) Expireat(key string, expireTime int64) error { }, acceptable) } +func (s *Redis) GeoAdd(key string, geoLocation ...*GeoLocation) (val int64, err error) { + err = s.brk.DoWithAcceptable(func() error { + conn, err := getRedis(s) + if err != nil { + return err + } + + if v, err := conn.GeoAdd(key, geoLocation...).Result(); err != nil { + return err + } else { + val = v + return nil + } + }, acceptable) + return +} + +func (s *Redis) GeoDist(key string, member1, member2, unit string) (val float64, err error) { + err = s.brk.DoWithAcceptable(func() error { + conn, err := getRedis(s) + if err != nil { + return err + } + + if v, err := conn.GeoDist(key, member1, member2, unit).Result(); err != nil { + return err + } else { + val = v + return nil + } + }, acceptable) + return +} + +func (s *Redis) GeoHash(key string, members ...string) (val []string, err error) { + err = s.brk.DoWithAcceptable(func() error { + conn, err := getRedis(s) + if err != nil { + return err + } + + if v, err := conn.GeoHash(key, members...).Result(); err != nil { + return err + } else { + val = v + return nil + } + }, acceptable) + return +} + +func (s *Redis) GeoRadius(key string, longitude, latitude float64, query *GeoRadiusQuery) (val []GeoLocation, err error) { + err = s.brk.DoWithAcceptable(func() error { + conn, err := getRedis(s) + if err != nil { + return err + } + + if v, err := conn.GeoRadius(key, longitude, latitude, query).Result(); err != nil { + return err + } else { + val = v + return nil + } + }, acceptable) + return +} +func (s *Redis) GeoRadiusByMember(key, member string, query *GeoRadiusQuery) (val []GeoLocation, err error) { + err = s.brk.DoWithAcceptable(func() error { + conn, err := getRedis(s) + if err != nil { + return err + } + + if v, err := conn.GeoRadiusByMember(key, member, query).Result(); err != nil { + return err + } else { + val = v + return nil + } + }, acceptable) + return +} + +func (s *Redis) GeoPos(key string, members ...string) (val []*GeoPos, err error) { + err = s.brk.DoWithAcceptable(func() error { + conn, err := getRedis(s) + if err != nil { + return err + } + + if v, err := conn.GeoPos(key, members...).Result(); err != nil { + return err + } else { + val = v + return nil + } + }, acceptable) + return +} + func (s *Redis) Get(key string) (val string, err error) { err = s.brk.DoWithAcceptable(func() error { conn, err := getRedis(s) diff --git a/core/stores/redis/redis_test.go b/core/stores/redis/redis_test.go index 3ea5f212..be0a8416 100644 --- a/core/stores/redis/redis_test.go +++ b/core/stores/redis/redis_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/alicebob/miniredis" + "github.com/alicebob/miniredis/v2" red "github.com/go-redis/redis" "github.com/stretchr/testify/assert" ) @@ -816,6 +816,38 @@ func TestRedisBlpopEx(t *testing.T) { }) } +func TestRedisGeo(t *testing.T) { + runOnRedis(t, func(client *Redis) { + client.Ping() + var geoLocation = []*GeoLocation{{Longitude: 13.361389, Latitude: 38.115556, Name: "Palermo"}, {Longitude: 15.087269, Latitude: 37.502669, Name: "Catania"}} + v, err := client.GeoAdd("sicily", geoLocation...) + assert.Nil(t, err) + assert.Equal(t, int64(2), v) + v2, err := client.GeoDist("sicily", "Palermo", "Catania", "m") + assert.Nil(t, err) + assert.Equal(t, 166274, int(v2)) + // GeoHash not support + v3, err := client.GeoPos("sicily", "Palermo", "Catania") + assert.Nil(t, err) + assert.Equal(t, int64(v3[0].Longitude), int64(13)) + assert.Equal(t, int64(v3[0].Latitude), int64(38)) + assert.Equal(t, int64(v3[1].Longitude), int64(15)) + assert.Equal(t, int64(v3[1].Latitude), int64(37)) + v4, err := client.GeoRadius("sicily", 15, 37, &red.GeoRadiusQuery{WithDist: true, Unit: "km", Radius: 200}) + assert.Nil(t, err) + assert.Equal(t, int64(v4[0].Dist), int64(190)) + assert.Equal(t, int64(v4[1].Dist), int64(56)) + var geoLocation2 = []*GeoLocation{{Longitude: 13.583333, Latitude: 37.316667, Name: "Agrigento"}} + v5, err := client.GeoAdd("sicily", geoLocation2...) + assert.Nil(t, err) + assert.Equal(t, int64(1), v5) + v6, err := client.GeoRadiusByMember("sicily", "Agrigento", &red.GeoRadiusQuery{Unit: "km", Radius: 100}) + assert.Nil(t, err) + assert.Equal(t, v6[0].Name, "Agrigento") + assert.Equal(t, v6[1].Name, "Palermo") + }) +} + func runOnRedis(t *testing.T, fn func(client *Redis)) { s, err := miniredis.Run() assert.Nil(t, err) diff --git a/core/stores/redis/redistest/redistest.go b/core/stores/redis/redistest/redistest.go index ee6073c4..029761a0 100644 --- a/core/stores/redis/redistest/redistest.go +++ b/core/stores/redis/redistest/redistest.go @@ -3,7 +3,7 @@ package redistest import ( "time" - "github.com/alicebob/miniredis" + "github.com/alicebob/miniredis/v2" "github.com/tal-tech/go-zero/core/lang" "github.com/tal-tech/go-zero/core/stores/redis" ) diff --git a/core/stores/sqlc/cachedsql_test.go b/core/stores/sqlc/cachedsql_test.go index 3ce899fe..c4651801 100644 --- a/core/stores/sqlc/cachedsql_test.go +++ b/core/stores/sqlc/cachedsql_test.go @@ -14,7 +14,7 @@ import ( "testing" "time" - "github.com/alicebob/miniredis" + "github.com/alicebob/miniredis/v2" "github.com/stretchr/testify/assert" "github.com/tal-tech/go-zero/core/fx" "github.com/tal-tech/go-zero/core/logx" diff --git a/go.mod b/go.mod index 5d663c9d..ebd064ac 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,7 @@ go 1.13 require ( github.com/ClickHouse/clickhouse-go v1.4.3 github.com/DATA-DOG/go-sqlmock v1.4.1 - github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 // indirect - github.com/alicebob/miniredis v2.5.0+incompatible + github.com/alicebob/miniredis/v2 v2.14.1 github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/dchest/siphash v1.2.1 github.com/dgrijalva/jwt-go v3.2.0+incompatible @@ -21,7 +20,6 @@ require ( github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/mock v1.4.3 github.com/golang/protobuf v1.4.2 - github.com/gomodule/redigo v2.0.0+incompatible // indirect github.com/google/gops v0.3.7 github.com/google/uuid v1.1.1 github.com/gorilla/websocket v1.4.2 // indirect @@ -47,7 +45,6 @@ require ( github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 // indirect github.com/urfave/cli v1.22.5 github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 - github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb // indirect go.etcd.io/etcd v0.0.0-20200402134248-51bdeb39e698 go.uber.org/automaxprocs v1.3.0 golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect @@ -58,7 +55,7 @@ require ( golang.org/x/tools v0.0.0-20200410132612-ae9902aceb98 // indirect google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f // indirect google.golang.org/grpc v1.29.1 - google.golang.org/protobuf v1.25.0 + google.golang.org/protobuf v1.25.0 // indirect gopkg.in/cheggaaa/pb.v1 v1.0.28 gopkg.in/h2non/gock.v1 v1.0.15 gopkg.in/yaml.v2 v2.3.0 diff --git a/go.sum b/go.sum index 916a67be..4897f3bd 100644 --- a/go.sum +++ b/go.sum @@ -11,10 +11,10 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafo github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6 h1:45bxf7AZMwWcqkLzDAQugVEwedisr5nRJ1r+7LYnv0U= -github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= -github.com/alicebob/miniredis v2.5.0+incompatible h1:yBHoLpsyjupjz3NL3MhKMVkR41j82Yjf3KFv7ApYzUI= -github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= +github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk= +github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= +github.com/alicebob/miniredis/v2 v2.14.1 h1:GjlbSeoJ24bzdLRs13HoMEeaRZx9kg5nHoRW7QV/nCs= +github.com/alicebob/miniredis/v2 v2.14.1/go.mod h1:uS970Sw5Gs9/iK3yBg0l9Uj9s25wXxSpQUE9EaJ/Blg= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -114,8 +114,6 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= -github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -147,8 +145,6 @@ github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslC github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334 h1:VHgatEHNcBFEB7inlalqfNqw65aNkM1lGX2yt3NmbS8= -github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/iancoleman/strcase v0.1.2 h1:gnomlvw9tnV3ITTAxzKSgTF+8kFWcU/f+TgttpXGz1U= github.com/iancoleman/strcase v0.1.2/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -211,6 +207,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= @@ -281,8 +278,6 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1 github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 h1:lYIiVDtZnyTWlNwiAxLj0bbpTcx1BWCFhXjfsvmPdNc= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.4 h1:u7tSpNPPswAFymm8IehJhy4uJMlUuU/GmqSkvJ1InXA= -github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= diff --git a/rest/handler/contentsecurityhandler.go b/rest/handler/contentsecurityhandler.go index 32705ff2..ba1f6f83 100644 --- a/rest/handler/contentsecurityhandler.go +++ b/rest/handler/contentsecurityhandler.go @@ -26,11 +26,11 @@ func ContentSecurityHandler(decrypters map[string]codec.RsaDecrypter, tolerance case http.MethodDelete, http.MethodGet, http.MethodPost, http.MethodPut: header, err := security.ParseContentSecurity(decrypters, r) if err != nil { - logx.Infof("Signature parse failed, X-Content-Security: %s, error: %s", + logx.Errorf("Signature parse failed, X-Content-Security: %s, error: %s", r.Header.Get(contentSecurity), err.Error()) executeCallbacks(w, r, next, strict, httpx.CodeSignatureInvalidHeader, callbacks) } else if code := security.VerifySignature(r, header, tolerance); code != httpx.CodeSignaturePass { - logx.Infof("Signature verification failed, X-Content-Security: %s", + logx.Errorf("Signature verification failed, X-Content-Security: %s", r.Header.Get(contentSecurity)) executeCallbacks(w, r, next, strict, code, callbacks) } else if r.ContentLength > 0 && header.Encrypted() { @@ -54,7 +54,7 @@ func executeCallbacks(w http.ResponseWriter, r *http.Request, next http.Handler, func handleVerificationFailure(w http.ResponseWriter, r *http.Request, next http.Handler, strict bool, code int) { if strict { - w.WriteHeader(http.StatusUnauthorized) + w.WriteHeader(http.StatusForbidden) } else { next.ServeHTTP(w, r) } diff --git a/rest/handler/contentsecurityhandler_test.go b/rest/handler/contentsecurityhandler_test.go index 62903bd3..facf53cb 100644 --- a/rest/handler/contentsecurityhandler_test.go +++ b/rest/handler/contentsecurityhandler_test.go @@ -113,7 +113,7 @@ func TestContentSecurityHandler(t *testing.T) { strict: true, crypt: true, timestamp: time.Now().Add(timeDiff).Unix(), - statusCode: http.StatusUnauthorized, + statusCode: http.StatusForbidden, }, { method: http.MethodPost, @@ -122,7 +122,7 @@ func TestContentSecurityHandler(t *testing.T) { strict: true, crypt: true, timestamp: time.Now().Add(-timeDiff).Unix(), - statusCode: http.StatusUnauthorized, + statusCode: http.StatusForbidden, }, { method: http.MethodPost, @@ -148,7 +148,7 @@ func TestContentSecurityHandler(t *testing.T) { crypt: true, timestamp: time.Now().Add(-timeDiff).Unix(), fingerprint: "badone", - statusCode: http.StatusUnauthorized, + statusCode: http.StatusForbidden, }, { method: http.MethodPost, @@ -157,7 +157,7 @@ func TestContentSecurityHandler(t *testing.T) { strict: true, crypt: true, missHeader: true, - statusCode: http.StatusUnauthorized, + statusCode: http.StatusForbidden, }, { method: http.MethodHead, @@ -171,7 +171,7 @@ func TestContentSecurityHandler(t *testing.T) { strict: true, crypt: false, signature: "badone", - statusCode: http.StatusUnauthorized, + statusCode: http.StatusForbidden, }, }