url: Distributed hit counter with in memory cache (#114)

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/micro/dev v0.0.0-20201117163752-d3cfc9788dfa
 	github.com/micro/micro/v3 v3.2.2-0.20210514120220-1ee39904d3dd
 	github.com/miekg/dns v1.1.31 // indirect
+	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/paulmach/go.geo v0.0.0-20180829195134-22b514266d33
 	github.com/pquerna/otp v1.3.0
 	github.com/stoewer/go-strcase v1.2.0

url/handler/url.go

@@ -7,7 +7,9 @@ import (
 	"time"
 
 	"github.com/micro/micro/v3/service/config"
+	"github.com/micro/micro/v3/service/logger"
 	"github.com/micro/micro/v3/service/model"
+	cache "github.com/patrickmn/go-cache"
 	"github.com/teris-io/shortid"
 
 	"github.com/micro/services/pkg/tenant"
@@ -19,6 +21,7 @@ const hostPrefix = "https://m3o.one/u/"
 type Url struct {
 	pairs      model.Model
 	ownerIndex model.Index
+	cache      *cache.Cache
 	hostPrefix string
 }
 
@@ -44,6 +47,7 @@ func NewUrl() *Url {
 		pairs:      m,
 		ownerIndex: ownerIndex,
 		hostPrefix: hp,
+		cache:      cache.New(cache.NoExpiration, cache.NoExpiration),
 	}
 }
 
@@ -85,6 +89,14 @@ func (e *Url) List(ctx context.Context, req *url.ListRequest, rsp *url.ListRespo
 		return err
 	}
 	for _, v := range rsp.UrlPairs {
+		// get the counter and add it to db value to improve
+		// accuracy
+		// A thing to keep in mind is that in memory cache hits
+		// from other nodes wont be added to this. Still, hopefully good enough
+		count, ok := e.cache.Get(v.ShortURL)
+		if ok {
+			v.HitCount += count.(int64)
+		}
 		v.ShortURL = e.hostPrefix + v.ShortURL
 	}
 	return nil
@@ -92,10 +104,31 @@ func (e *Url) List(ctx context.Context, req *url.ListRequest, rsp *url.ListRespo
 
 func (e *Url) Proxy(ctx context.Context, req *url.ProxyRequest, rsp *url.ProxyResponse) error {
 	var pair url.URLPair
-	err := e.pairs.Read(model.QueryEquals("shortURL", strings.Replace(req.ShortURL, e.hostPrefix, "", -1)), &pair)
+	id := strings.Replace(req.ShortURL, e.hostPrefix, "", -1)
+	err := e.pairs.Read(model.QueryEquals("shortURL", id), &pair)
 	if err != nil {
 		return err
 	}
+	v, found := e.cache.Get(id)
+	// @todo there is an ABA problem with this solution
+	// when it comes to the hit counter
+	if !found {
+		e.cache.Set(id, int64(1), cache.NoExpiration)
+	} else {
+		// we null out the counter
+		e.cache.Set(id, 0, cache.NoExpiration)
+		if v.(int64)%10 == 0 {
+			go func() {
+				// We add instead of set in case the service runs in multiple
+				// instances
+				pair.HitCount += v.(int64) + int64(1)
+				err = e.pairs.Update(pair)
+				if err != nil {
+					logger.Error(err)
+				}
+			}()
+		}
+	}
 
 	rsp.DestinationURL = pair.DestinationURL
 	return nil

url/proto/url.pb.go

@@ -124,6 +124,7 @@ type URLPair struct {
 	ShortURL       string `protobuf:"bytes,2,opt,name=shortURL,proto3" json:"shortURL,omitempty"`
 	Owner          string `protobuf:"bytes,3,opt,name=owner,proto3" json:"owner,omitempty"`
 	Created        int64  `protobuf:"varint,4,opt,name=created,proto3" json:"created,omitempty"`
+	HitCount       int64  `protobuf:"varint,5,opt,name=hitCount,proto3" json:"hitCount,omitempty"`
 }
 
 func (x *URLPair) Reset() {
@@ -186,12 +187,22 @@ func (x *URLPair) GetCreated() int64 {
 	return 0
 }
 
+func (x *URLPair) GetHitCount() int64 {
+	if x != nil {
+		return x.HitCount
+	}
+	return 0
+}
+
 // List shortened URLs. It has no input parameters, as it will take
 // the user ID from the token and list the user's (caller's) shortened URLs.
 type ListRequest struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
+
+	// filter by short URL, optional
+	ShortURL string `protobuf:"bytes,2,opt,name=shortURL,proto3" json:"shortURL,omitempty"`
 }
 
 func (x *ListRequest) Reset() {
@@ -226,6 +237,13 @@ func (*ListRequest) Descriptor() ([]byte, []int) {
 	return file_proto_url_proto_rawDescGZIP(), []int{3}
 }
 
+func (x *ListRequest) GetShortURL() string {
+	if x != nil {
+		return x.ShortURL
+	}
+	return ""
+}
+
 type ListResponse struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -383,38 +401,42 @@ var file_proto_url_proto_rawDesc = []byte{
 	0x22, 0x2d, 0x0a, 0x0f, 0x53, 0x68, 0x6f, 0x72, 0x74, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f,
 	0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x18,
 	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x22,
-	0x7d, 0x0a, 0x07, 0x55, 0x52, 0x4c, 0x50, 0x61, 0x69, 0x72, 0x12, 0x26, 0x0a, 0x0e, 0x64, 0x65,
+	0x99, 0x01, 0x0a, 0x07, 0x55, 0x52, 0x4c, 0x50, 0x61, 0x69, 0x72, 0x12, 0x26, 0x0a, 0x0e, 0x64,
-	0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01,
+	0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20,
-	0x28, 0x09, 0x52, 0x0e, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55,
+	0x01, 0x28, 0x09, 0x52, 0x0e, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e,
-	0x52, 0x4c, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x18, 0x02,
+	0x55, 0x52, 0x4c, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x18,
-	0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x12, 0x14,
+	0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x12,
-	0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6f,
+	0x14, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05,
-	0x77, 0x6e, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x18,
+	0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64,
-	0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x22, 0x0d,
+	0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x12,
-	0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x38, 0x0a,
+	0x1a, 0x0a, 0x08, 0x68, 0x69, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28,
-	0x0c, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x28, 0x0a,
+	0x03, 0x52, 0x08, 0x68, 0x69, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x29, 0x0a, 0x0b, 0x4c,
-	0x08, 0x75, 0x72, 0x6c, 0x50, 0x61, 0x69, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32,
+	0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x68,
-	0x0c, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x55, 0x52, 0x4c, 0x50, 0x61, 0x69, 0x72, 0x52, 0x08, 0x75,
+	0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x68,
-	0x72, 0x6c, 0x50, 0x61, 0x69, 0x72, 0x73, 0x22, 0x2a, 0x0a, 0x0c, 0x50, 0x72, 0x6f, 0x78, 0x79,
+	0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x22, 0x38, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x28, 0x0a, 0x08, 0x75, 0x72, 0x6c, 0x50, 0x61, 0x69,
-	0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74,
+	0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x55,
-	0x55, 0x52, 0x4c, 0x22, 0x37, 0x0a, 0x0d, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x52, 0x65, 0x73, 0x70,
+	0x52, 0x4c, 0x50, 0x61, 0x69, 0x72, 0x52, 0x08, 0x75, 0x72, 0x6c, 0x50, 0x61, 0x69, 0x72, 0x73,
-	0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x0e, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74,
+	0x22, 0x2a, 0x0a, 0x0c, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
-	0x69, 0x6f, 0x6e, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x64, 0x65,
+	0x12, 0x1a, 0x0a, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x18, 0x01, 0x20, 0x01,
-	0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x52, 0x4c, 0x32, 0x9e, 0x01, 0x0a,
+	0x28, 0x09, 0x52, 0x08, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x55, 0x52, 0x4c, 0x22, 0x37, 0x0a, 0x0d,
-	0x03, 0x55, 0x72, 0x6c, 0x12, 0x36, 0x0a, 0x07, 0x53, 0x68, 0x6f, 0x72, 0x74, 0x65, 0x6e, 0x12,
+	0x50, 0x72, 0x6f, 0x78, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a,
-	0x13, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x53, 0x68, 0x6f, 0x72, 0x74, 0x65, 0x6e, 0x52, 0x65, 0x71,
+	0x0e, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x55, 0x52, 0x4c, 0x18,
-	0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x53, 0x68, 0x6f, 0x72, 0x74,
+	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69,
-	0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2d, 0x0a, 0x04,
+	0x6f, 0x6e, 0x55, 0x52, 0x4c, 0x32, 0x9e, 0x01, 0x0a, 0x03, 0x55, 0x72, 0x6c, 0x12, 0x36, 0x0a,
-	0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52,
+	0x07, 0x53, 0x68, 0x6f, 0x72, 0x74, 0x65, 0x6e, 0x12, 0x13, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x53,
-	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x4c, 0x69, 0x73,
+	0x68, 0x6f, 0x72, 0x74, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e,
-	0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x30, 0x0a, 0x05, 0x50,
+	0x75, 0x72, 0x6c, 0x2e, 0x53, 0x68, 0x6f, 0x72, 0x74, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f,
-	0x72, 0x6f, 0x78, 0x79, 0x12, 0x11, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79,
+	0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2d, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x2e,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x50, 0x72,
+	0x75, 0x72, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
-	0x6f, 0x78, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x29, 0x5a,
+	0x11, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
-	0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x69, 0x63, 0x72,
+	0x73, 0x65, 0x22, 0x00, 0x12, 0x30, 0x0a, 0x05, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x12, 0x11, 0x2e,
-	0x6f, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x75, 0x72, 0x6c, 0x2f, 0x70,
+	0x75, 0x72, 0x6c, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
-	0x72, 0x6f, 0x74, 0x6f, 0x3b, 0x75, 0x72, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x1a, 0x12, 0x2e, 0x75, 0x72, 0x6c, 0x2e, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x52, 0x65, 0x73, 0x70,
+	0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x29, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
+	0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x69, 0x63, 0x72, 0x6f, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69,
+	0x63, 0x65, 0x73, 0x2f, 0x75, 0x72, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3b, 0x75, 0x72,
+	0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

url/proto/url.proto

@@ -24,11 +24,17 @@ message URLPair {
 	string shortURL = 2;
 	string owner = 3;
 	int64 created = 4;
+	// HitCount keeps track many times the short URL has been resolved.
+	// Hitcount only gets saved to disk (database) after every 10th hit, so
+	// its not intended to be 100% accurate, more like an almost correct estimate.
+	int64 hitCount = 5;
 }
 
 // List shortened URLs. It has no input parameters, as it will take
 // the user ID from the token and list the user's (caller's) shortened URLs.
 message ListRequest {
+	// filter by short URL, optional
+	string shortURL = 2;
 }
 
 message ListResponse {