totole/services
1
0
Fork 0
mirror of https://github.com/kevin-DL/services.git synced 2026-01-11 19:04:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

url: Distributed hit counter with in memory cache (#114)

Browse Source
This commit is contained in:
Janos Dobronszki
2021-05-19 16:13:25 +01:00
committed by GitHub
parent 26ad8bc08d
commit 17cd2399c8
4 changed files with 95 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@@ -15,6 +15,7 @@ require (
github.com/micro/dev v0.0.0-20201117163752-d3cfc9788dfa
github.com/micro/micro/v3 v3.2.2-0.20210514120220-1ee39904d3dd
github.com/miekg/dns v1.1.31 // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/paulmach/go.geo v0.0.0-20180829195134-22b514266d33
github.com/pquerna/otp v1.3.0
github.com/stoewer/go-strcase v1.2.0

35
url/handler/url.go
View File

@@ -7,7 +7,9 @@ import (
"time"
"github.com/micro/micro/v3/service/config"
"github.com/micro/micro/v3/service/logger"
"github.com/micro/micro/v3/service/model"
cache "github.com/patrickmn/go-cache"
"github.com/teris-io/shortid"
"github.com/micro/services/pkg/tenant"
@@ -19,6 +21,7 @@ const hostPrefix = "https://m3o.one/u/"
type Url struct {
pairs model.Model
ownerIndex model.Index
cache *cache.Cache
hostPrefix string
}
@@ -44,6 +47,7 @@ func NewUrl() *Url {
pairs: m,
ownerIndex: ownerIndex,
hostPrefix: hp,
cache: cache.New(cache.NoExpiration, cache.NoExpiration),
}
}
@@ -85,6 +89,14 @@ func (e *Url) List(ctx context.Context, req *url.ListRequest, rsp *url.ListRespo
return err
}
for _, v := range rsp.UrlPairs {
// get the counter and add it to db value to improve
// accuracy
// A thing to keep in mind is that in memory cache hits
// from other nodes wont be added to this. Still, hopefully good enough
count, ok := e.cache.Get(v.ShortURL)
if ok {
v.HitCount += count.(int64)
}
v.ShortURL = e.hostPrefix + v.ShortURL
}
return nil
@@ -92,10 +104,31 @@ func (e *Url) List(ctx context.Context, req *url.ListRequest, rsp *url.ListRespo
func (e *Url) Proxy(ctx context.Context, req *url.ProxyRequest, rsp *url.ProxyResponse) error {
var pair url.URLPair
err := e.pairs.Read(model.QueryEquals("shortURL", strings.Replace(req.ShortURL, e.hostPrefix, "", -1)), &pair)
id := strings.Replace(req.ShortURL, e.hostPrefix, "", -1)
err := e.pairs.Read(model.QueryEquals("shortURL", id), &pair)
if err != nil {
return err
}
v, found := e.cache.Get(id)
// @todo there is an ABA problem with this solution
// when it comes to the hit counter
if !found {
e.cache.Set(id, int64(1), cache.NoExpiration)
} else {
// we null out the counter
e.cache.Set(id, 0, cache.NoExpiration)
if v.(int64)%10 == 0 {
go func() {
// We add instead of set in case the service runs in multiple
// instances
pair.HitCount += v.(int64) + int64(1)
err = e.pairs.Update(pair)
if err != nil {
logger.Error(err)
}
}()
}
}
rsp.DestinationURL = pair.DestinationURL
return nil

86
url/proto/url.pb.go
View File

@@ -124,6 +124,7 @@ type URLPair struct {
ShortURL string `protobuf:"bytes,2,opt,name=shortURL,proto3" json:"shortURL,omitempty"`
Owner string `protobuf:"bytes,3,opt,name=owner,proto3" json:"owner,omitempty"`
Created int64 `protobuf:"varint,4,opt,name=created,proto3" json:"created,omitempty"`
HitCount int64 `protobuf:"varint,5,opt,name=hitCount,proto3" json:"hitCount,omitempty"`
}
func (x *URLPair) Reset() {
@@ -186,12 +187,22 @@ func (x *URLPair) GetCreated() int64 {
return 0
}
func (x *URLPair) GetHitCount() int64 {
if x != nil {
return x.HitCount
}
return 0
}
// List shortened URLs. It has no input parameters, as it will take
// the user ID from the token and list the user's (caller's) shortened URLs.
type ListRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// filter by short URL, optional
ShortURL string `protobuf:"bytes,2,opt,name=shortURL,proto3" json:"shortURL,omitempty"`
}
func (x *ListRequest) Reset() {
@@ -226,6 +237,13 @@ func (*ListRequest) Descriptor() ([]byte, []int) {
return file_proto_url_proto_rawDescGZIP(), []int{3}
}
func (x *ListRequest) GetShortURL() string {
if x != nil {
return x.ShortURL
}
return ""
}
type ListResponse struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -383,38 +401,42 @@ var file_proto_url_proto_rawDesc = []byte{
0x22, 0x2d, 0x0a, 0x0f, 0x53, 0x68, 0x6f, 0x72, 0x74, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x22,
0x7d, 0x0a, 0x07, 0x55, 0x52, 0x4c, 0x50, 0x61, 0x69, 0x72, 0x12, 0x26, 0x0a, 0x0e, 0x64, 0x65,
0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01,
0x28, 0x09, 0x52, 0x0e, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55,
0x52, 0x4c, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x18, 0x02,
0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x12, 0x14,
0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6f,
0x77, 0x6e, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x18,
0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x22, 0x0d,
0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x38, 0x0a,
0x0c, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x28, 0x0a,
0x08, 0x75, 0x72, 0x6c, 0x50, 0x61, 0x69, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32,
0x0c, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x55, 0x52, 0x4c, 0x50, 0x61, 0x69, 0x72, 0x52, 0x08, 0x75,
0x72, 0x6c, 0x50, 0x61, 0x69, 0x72, 0x73, 0x22, 0x2a, 0x0a, 0x0c, 0x50, 0x72, 0x6f, 0x78, 0x79,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74,
0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74,
0x55, 0x52, 0x4c, 0x22, 0x37, 0x0a, 0x0d, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x52, 0x65, 0x73, 0x70,
0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x0e, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74,
0x69, 0x6f, 0x6e, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x64, 0x65,
0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x52, 0x4c, 0x32, 0x9e, 0x01, 0x0a,
0x03, 0x55, 0x72, 0x6c, 0x12, 0x36, 0x0a, 0x07, 0x53, 0x68, 0x6f, 0x72, 0x74, 0x65, 0x6e, 0x12,
0x13, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x53, 0x68, 0x6f, 0x72, 0x74, 0x65, 0x6e, 0x52, 0x65, 0x71,
0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x53, 0x68, 0x6f, 0x72, 0x74,
0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2d, 0x0a, 0x04,
0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x4c, 0x69, 0x73,
0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x30, 0x0a, 0x05, 0x50,
0x72, 0x6f, 0x78, 0x79, 0x12, 0x11, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x50, 0x72,
0x6f, 0x78, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x29, 0x5a,
0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x69, 0x63, 0x72,
0x6f, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x75, 0x72, 0x6c, 0x2f, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x3b, 0x75, 0x72, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x99, 0x01, 0x0a, 0x07, 0x55, 0x52, 0x4c, 0x50, 0x61, 0x69, 0x72, 0x12, 0x26, 0x0a, 0x0e, 0x64,
0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20,
0x01, 0x28, 0x09, 0x52, 0x0e, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e,
0x55, 0x52, 0x4c, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x18,
0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x12,
0x14, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05,
0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64,
0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12,
0x1a, 0x0a, 0x08, 0x68, 0x69, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28,
0x03, 0x52, 0x08, 0x68, 0x69, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x29, 0x0a, 0x0b, 0x4c,
0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x68,
0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x68,
0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x22, 0x38, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65,
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x28, 0x0a, 0x08, 0x75, 0x72, 0x6c, 0x50, 0x61, 0x69,
0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x55,
0x52, 0x4c, 0x50, 0x61, 0x69, 0x72, 0x52, 0x08, 0x75, 0x72, 0x6c, 0x50, 0x61, 0x69, 0x72, 0x73,
0x22, 0x2a, 0x0a, 0x0c, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x12, 0x1a, 0x0a, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01,
0x28, 0x09, 0x52, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x22, 0x37, 0x0a, 0x0d,
0x50, 0x72, 0x6f, 0x78, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a,
0x0e, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x52, 0x4c, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69,
0x6f, 0x6e, 0x55, 0x52, 0x4c, 0x32, 0x9e, 0x01, 0x0a, 0x03, 0x55, 0x72, 0x6c, 0x12, 0x36, 0x0a,
0x07, 0x53, 0x68, 0x6f, 0x72, 0x74, 0x65, 0x6e, 0x12, 0x13, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x53,
0x68, 0x6f, 0x72, 0x74, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e,
0x75, 0x72, 0x6c, 0x2e, 0x53, 0x68, 0x6f, 0x72, 0x74, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2d, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x2e,
0x75, 0x72, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
0x11, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
0x73, 0x65, 0x22, 0x00, 0x12, 0x30, 0x0a, 0x05, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x12, 0x11, 0x2e,
0x75, 0x72, 0x6c, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x1a, 0x12, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x52, 0x65, 0x73, 0x70,
0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x29, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69,
0x63, 0x65, 0x73, 0x2f, 0x75, 0x72, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3b, 0x75, 0x72,
0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (

6
url/proto/url.proto
View File

@@ -24,11 +24,17 @@ message URLPair {
string shortURL = 2;
string owner = 3;
int64 created = 4;
// HitCount keeps track many times the short URL has been resolved.
// Hitcount only gets saved to disk (database) after every 10th hit, so
// its not intended to be 100% accurate, more like an almost correct estimate.
int64 hitCount = 5;
}
// List shortened URLs. It has no input parameters, as it will take
// the user ID from the token and list the user's (caller's) shortened URLs.
message ListRequest {
// filter by short URL, optional
string shortURL = 2;
}
message ListResponse {