From 17cd2399c849476abf72fb767b9dd78673f5c3c7 Mon Sep 17 00:00:00 2001 From: Janos Dobronszki Date: Wed, 19 May 2021 16:13:25 +0100 Subject: [PATCH] url: Distributed hit counter with in memory cache (#114) --- go.mod | 1 + url/handler/url.go | 35 +++++++++++++++++- url/proto/url.pb.go | 86 ++++++++++++++++++++++++++++----------------- url/proto/url.proto | 6 ++++ 4 files changed, 95 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index 7b70ab4..2166405 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/micro/dev v0.0.0-20201117163752-d3cfc9788dfa github.com/micro/micro/v3 v3.2.2-0.20210514120220-1ee39904d3dd github.com/miekg/dns v1.1.31 // indirect + github.com/patrickmn/go-cache v2.1.0+incompatible github.com/paulmach/go.geo v0.0.0-20180829195134-22b514266d33 github.com/pquerna/otp v1.3.0 github.com/stoewer/go-strcase v1.2.0 diff --git a/url/handler/url.go b/url/handler/url.go index 2a45ee1..fdd4078 100644 --- a/url/handler/url.go +++ b/url/handler/url.go @@ -7,7 +7,9 @@ import ( "time" "github.com/micro/micro/v3/service/config" + "github.com/micro/micro/v3/service/logger" "github.com/micro/micro/v3/service/model" + cache "github.com/patrickmn/go-cache" "github.com/teris-io/shortid" "github.com/micro/services/pkg/tenant" @@ -19,6 +21,7 @@ const hostPrefix = "https://m3o.one/u/" type Url struct { pairs model.Model ownerIndex model.Index + cache *cache.Cache hostPrefix string } @@ -44,6 +47,7 @@ func NewUrl() *Url { pairs: m, ownerIndex: ownerIndex, hostPrefix: hp, + cache: cache.New(cache.NoExpiration, cache.NoExpiration), } } @@ -85,6 +89,14 @@ func (e *Url) List(ctx context.Context, req *url.ListRequest, rsp *url.ListRespo return err } for _, v := range rsp.UrlPairs { + // get the counter and add it to db value to improve + // accuracy + // A thing to keep in mind is that in memory cache hits + // from other nodes wont be added to this. Still, hopefully good enough + count, ok := e.cache.Get(v.ShortURL) + if ok { + v.HitCount += count.(int64) + } v.ShortURL = e.hostPrefix + v.ShortURL } return nil @@ -92,10 +104,31 @@ func (e *Url) List(ctx context.Context, req *url.ListRequest, rsp *url.ListRespo func (e *Url) Proxy(ctx context.Context, req *url.ProxyRequest, rsp *url.ProxyResponse) error { var pair url.URLPair - err := e.pairs.Read(model.QueryEquals("shortURL", strings.Replace(req.ShortURL, e.hostPrefix, "", -1)), &pair) + id := strings.Replace(req.ShortURL, e.hostPrefix, "", -1) + err := e.pairs.Read(model.QueryEquals("shortURL", id), &pair) if err != nil { return err } + v, found := e.cache.Get(id) + // @todo there is an ABA problem with this solution + // when it comes to the hit counter + if !found { + e.cache.Set(id, int64(1), cache.NoExpiration) + } else { + // we null out the counter + e.cache.Set(id, 0, cache.NoExpiration) + if v.(int64)%10 == 0 { + go func() { + // We add instead of set in case the service runs in multiple + // instances + pair.HitCount += v.(int64) + int64(1) + err = e.pairs.Update(pair) + if err != nil { + logger.Error(err) + } + }() + } + } rsp.DestinationURL = pair.DestinationURL return nil diff --git a/url/proto/url.pb.go b/url/proto/url.pb.go index 039483d..423ad9c 100644 --- a/url/proto/url.pb.go +++ b/url/proto/url.pb.go @@ -124,6 +124,7 @@ type URLPair struct { ShortURL string `protobuf:"bytes,2,opt,name=shortURL,proto3" json:"shortURL,omitempty"` Owner string `protobuf:"bytes,3,opt,name=owner,proto3" json:"owner,omitempty"` Created int64 `protobuf:"varint,4,opt,name=created,proto3" json:"created,omitempty"` + HitCount int64 `protobuf:"varint,5,opt,name=hitCount,proto3" json:"hitCount,omitempty"` } func (x *URLPair) Reset() { @@ -186,12 +187,22 @@ func (x *URLPair) GetCreated() int64 { return 0 } +func (x *URLPair) GetHitCount() int64 { + if x != nil { + return x.HitCount + } + return 0 +} + // List shortened URLs. It has no input parameters, as it will take // the user ID from the token and list the user's (caller's) shortened URLs. type ListRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + + // filter by short URL, optional + ShortURL string `protobuf:"bytes,2,opt,name=shortURL,proto3" json:"shortURL,omitempty"` } func (x *ListRequest) Reset() { @@ -226,6 +237,13 @@ func (*ListRequest) Descriptor() ([]byte, []int) { return file_proto_url_proto_rawDescGZIP(), []int{3} } +func (x *ListRequest) GetShortURL() string { + if x != nil { + return x.ShortURL + } + return "" +} + type ListResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -383,38 +401,42 @@ var file_proto_url_proto_rawDesc = []byte{ 0x22, 0x2d, 0x0a, 0x0f, 0x53, 0x68, 0x6f, 0x72, 0x74, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x22, - 0x7d, 0x0a, 0x07, 0x55, 0x52, 0x4c, 0x50, 0x61, 0x69, 0x72, 0x12, 0x26, 0x0a, 0x0e, 0x64, 0x65, - 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0e, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, - 0x52, 0x4c, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x12, 0x14, - 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6f, - 0x77, 0x6e, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x22, 0x0d, - 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x38, 0x0a, - 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x28, 0x0a, - 0x08, 0x75, 0x72, 0x6c, 0x50, 0x61, 0x69, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x0c, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x55, 0x52, 0x4c, 0x50, 0x61, 0x69, 0x72, 0x52, 0x08, 0x75, - 0x72, 0x6c, 0x50, 0x61, 0x69, 0x72, 0x73, 0x22, 0x2a, 0x0a, 0x0c, 0x50, 0x72, 0x6f, 0x78, 0x79, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, - 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, - 0x55, 0x52, 0x4c, 0x22, 0x37, 0x0a, 0x0d, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x0e, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x64, 0x65, - 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x52, 0x4c, 0x32, 0x9e, 0x01, 0x0a, - 0x03, 0x55, 0x72, 0x6c, 0x12, 0x36, 0x0a, 0x07, 0x53, 0x68, 0x6f, 0x72, 0x74, 0x65, 0x6e, 0x12, - 0x13, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x53, 0x68, 0x6f, 0x72, 0x74, 0x65, 0x6e, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x53, 0x68, 0x6f, 0x72, 0x74, - 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2d, 0x0a, 0x04, - 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x4c, 0x69, 0x73, - 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x30, 0x0a, 0x05, 0x50, - 0x72, 0x6f, 0x78, 0x79, 0x12, 0x11, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x50, 0x72, - 0x6f, 0x78, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x29, 0x5a, - 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x69, 0x63, 0x72, - 0x6f, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x75, 0x72, 0x6c, 0x2f, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x3b, 0x75, 0x72, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x99, 0x01, 0x0a, 0x07, 0x55, 0x52, 0x4c, 0x50, 0x61, 0x69, 0x72, 0x12, 0x26, 0x0a, 0x0e, 0x64, + 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0e, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x55, 0x52, 0x4c, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x12, + 0x14, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12, + 0x1a, 0x0a, 0x08, 0x68, 0x69, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x08, 0x68, 0x69, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x29, 0x0a, 0x0b, 0x4c, + 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x68, + 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x68, + 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x22, 0x38, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x28, 0x0a, 0x08, 0x75, 0x72, 0x6c, 0x50, 0x61, 0x69, + 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x55, + 0x52, 0x4c, 0x50, 0x61, 0x69, 0x72, 0x52, 0x08, 0x75, 0x72, 0x6c, 0x50, 0x61, 0x69, 0x72, 0x73, + 0x22, 0x2a, 0x0a, 0x0c, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x22, 0x37, 0x0a, 0x0d, + 0x50, 0x72, 0x6f, 0x78, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, + 0x0e, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x52, 0x4c, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x55, 0x52, 0x4c, 0x32, 0x9e, 0x01, 0x0a, 0x03, 0x55, 0x72, 0x6c, 0x12, 0x36, 0x0a, + 0x07, 0x53, 0x68, 0x6f, 0x72, 0x74, 0x65, 0x6e, 0x12, 0x13, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x53, + 0x68, 0x6f, 0x72, 0x74, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, + 0x75, 0x72, 0x6c, 0x2e, 0x53, 0x68, 0x6f, 0x72, 0x74, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2d, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x2e, + 0x75, 0x72, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x11, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x30, 0x0a, 0x05, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x12, 0x11, 0x2e, + 0x75, 0x72, 0x6c, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x12, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x29, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x73, 0x2f, 0x75, 0x72, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3b, 0x75, 0x72, + 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/url/proto/url.proto b/url/proto/url.proto index b49988e..181cb9c 100644 --- a/url/proto/url.proto +++ b/url/proto/url.proto @@ -24,11 +24,17 @@ message URLPair { string shortURL = 2; string owner = 3; int64 created = 4; + // HitCount keeps track many times the short URL has been resolved. + // Hitcount only gets saved to disk (database) after every 10th hit, so + // its not intended to be 100% accurate, more like an almost correct estimate. + int64 hitCount = 5; } // List shortened URLs. It has no input parameters, as it will take // the user ID from the token and list the user's (caller's) shortened URLs. message ListRequest { + // filter by short URL, optional + string shortURL = 2; } message ListResponse {