diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1d81abb..7c3a45f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -43,35 +43,17 @@ jobs: go get -u github.com/golang/protobuf/protoc-gen-go go install - - name: Install redoc cli - run: | - # https://github.com/actions/virtual-environments/issues/599 - sudo npm install -g redoc-cli - - name: Install openapi plugin working-directory: micro/cmd/protoc-gen-openapi run: | go install -# - name: Install hugo -# run: sudo snap install hugo --channel=extended - - name: Generate openapi spec and publish the api working-directory: services run: | go run cmd/publisher/main.go . env: MICRO_ADMIN_TOKEN: ${{ secrets.MICRO_ADMIN_TOKEN }} - -# - name: Deploy -# if: github.ref == 'refs/heads/master' -# uses: s0/git-publish-subdir-action@develop -# env: -# REPO: self -# BRANCH: gh-pages -# FOLDER: services/docs -# GITHUB_TOKEN: ${{ secrets.GH_PAT }} - - name: Generate package working-directory: services env: diff --git a/cmd/publisher/main.go b/cmd/publisher/main.go index 1224cce..c1b0a51 100644 --- a/cmd/publisher/main.go +++ b/cmd/publisher/main.go @@ -75,12 +75,12 @@ func main() { } fmt.Println("Processing folder", serviceDir) - makeProto := exec.Command("make", "docs") + makeProto := exec.Command("make", "api") makeProto.Dir = serviceDir fmt.Println(serviceDir) outp, err := makeProto.CombinedOutput() if err != nil { - fmt.Println("Failed to make docs", string(outp)) + fmt.Println("Failed to make api", string(outp)) os.Exit(1) } serviceName := f.Name() diff --git a/datastore/Makefile b/datastore/Makefile index 1875e30..422cee1 100644 --- a/datastore/Makefile +++ b/datastore/Makefile @@ -5,14 +5,14 @@ init: go get -u github.com/golang/protobuf/proto go get -u github.com/golang/protobuf/protoc-gen-go go get github.com/micro/micro/v3/cmd/protoc-gen-micro + go get github.com/micro/micro/v3/cmd/protoc-gen-openapi .PHONY: proto proto: protoc --openapi_out=. --proto_path=. --micro_out=. --go_out=:. proto/datastore.proto -.PHONY: docs -docs: - protoc --openapi_out=. --proto_path=. --micro_out=. --go_out=:. proto/datastore.proto - @redoc-cli bundle api-datastore.json +.PHONY: api +api: + protoc --openapi_out=. --proto_path=. proto/datastore.proto .PHONY: build build: diff --git a/files/Makefile b/files/Makefile index f868b5a..2559915 100644 --- a/files/Makefile +++ b/files/Makefile @@ -5,14 +5,14 @@ init: go get -u github.com/golang/protobuf/proto go get -u github.com/golang/protobuf/protoc-gen-go go get github.com/micro/micro/v3/cmd/protoc-gen-micro + go get github.com/micro/micro/v3/cmd/protoc-gen-openapi .PHONY: proto proto: protoc --proto_path=. --micro_out=. --go_out=:. proto/files.proto -.PHONY: docs -docs: - protoc --openapi_out=. --proto_path=. --micro_out=${MODIFY}:. --go_out=${MODIFY}:. proto/files.proto - @redoc-cli bundle api-files.json +.PHONY: api +api: + protoc --openapi_out=. --proto_path=. proto/files.proto .PHONY: build build: diff --git a/geocoding/Makefile b/geocoding/Makefile index 3b81c3b..1dd41a6 100644 --- a/geocoding/Makefile +++ b/geocoding/Makefile @@ -5,14 +5,14 @@ init: go get -u github.com/golang/protobuf/proto go get -u github.com/golang/protobuf/protoc-gen-go go get github.com/micro/micro/v3/cmd/protoc-gen-micro + go get github.com/micro/micro/v3/cmd/protoc-gen-openapi .PHONY: proto proto: protoc --openapi_out=. --proto_path=. --micro_out=. --go_out=:. proto/geocoding.proto -.PHONY: docs -docs: - protoc --openapi_out=. --proto_path=. --micro_out=. --go_out=:. proto/geocoding.proto - @redoc-cli bundle api-geocoding.json +.PHONY: api +api: + protoc --openapi_out=. --proto_path=. proto/geocoding.proto .PHONY: build build: diff --git a/go.mod b/go.mod index 8bfb475..7b70ab4 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/golang/protobuf v1.5.1 github.com/google/uuid v1.1.2 github.com/hailocab/go-geoindex v0.0.0-20160127134810-64631bfe9711 + github.com/hashicorp/golang-lru v0.5.3 github.com/lib/pq v1.9.0 // indirect github.com/micro/dev v0.0.0-20201117163752-d3cfc9788dfa github.com/micro/micro/v3 v3.2.2-0.20210514120220-1ee39904d3dd diff --git a/helloworld/Makefile b/helloworld/Makefile index 207b7d4..1923437 100644 --- a/helloworld/Makefile +++ b/helloworld/Makefile @@ -6,10 +6,9 @@ GOPATH:=$(shell go env GOPATH) proto: protoc --openapi_out=. --proto_path=${GOPATH}/src:. --micro_out=. --go_out=. proto/helloworld.proto -.PHONY: docs -docs: - protoc --openapi_out=. --proto_path=${GOPATH}/src:. --micro_out=. --go_out=. proto/helloworld.proto - @redoc-cli bundle api-helloworld.json +.PHONY: api +api: + protoc --openapi_out=. --proto_path=${GOPATH}/src:. proto/helloworld.proto .PHONY: build build: proto diff --git a/helloworld/proto/helloworld.proto b/helloworld/proto/helloworld.proto index 57ab33c..1ab0f13 100644 --- a/helloworld/proto/helloworld.proto +++ b/helloworld/proto/helloworld.proto @@ -1,6 +1,7 @@ syntax = "proto3"; package helloworld; + option go_package = "./proto;helloworld"; service Helloworld { diff --git a/image/Makefile b/image/Makefile index 8c0e197..5eca9ce 100644 --- a/image/Makefile +++ b/image/Makefile @@ -5,13 +5,14 @@ init: go get -u github.com/golang/protobuf/proto go get -u github.com/golang/protobuf/protoc-gen-go go get github.com/micro/micro/v3/cmd/protoc-gen-micro + go get github.com/micro/micro/v3/cmd/protoc-gen-openapi .PHONY: proto proto: protoc --openapi_out=. --proto_path=. --micro_out=. --go_out=. proto/image.proto -docs: - protoc --openapi_out=. --proto_path=. --micro_out=. --go_out=. proto/image.proto - @redoc-cli bundle api-image.json +.PHONE: api +api: + protoc --openapi_out=. --proto_path=. proto/image.proto .PHONY: build build: diff --git a/location/Makefile b/location/Makefile index 7fe0372..d636bea 100644 --- a/location/Makefile +++ b/location/Makefile @@ -5,14 +5,14 @@ init: go get -u github.com/golang/protobuf/proto go get -u github.com/golang/protobuf/protoc-gen-go go get github.com/micro/micro/v3/cmd/protoc-gen-micro + go get github.com/micro/micro/v3/cmd/protoc-gen-openapi .PHONY: proto proto: protoc --openapi_out=. --proto_path=. --micro_out=. --go_out=:. proto/location.proto -.PHONY: docs -docs: - protoc --openapi_out=. --proto_path=. --micro_out=. --go_out=:. proto/location.proto - @redoc-cli bundle api-location.json +.PHONY: api +api: + protoc --openapi_out=. --proto_path=. proto/location.proto .PHONY: build build: diff --git a/otp/Makefile b/otp/Makefile index 925b4b1..8e25500 100644 --- a/otp/Makefile +++ b/otp/Makefile @@ -5,13 +5,14 @@ init: go get -u github.com/golang/protobuf/proto go get -u github.com/golang/protobuf/protoc-gen-go go get github.com/micro/micro/v3/cmd/protoc-gen-micro + go get github.com/micro/micro/v3/cmd/protoc-gen-openapi .PHONY: proto proto: protoc --openapi_out=. --proto_path=. --micro_out=. --go_out=. proto/otp.proto -docs: - protoc --openapi_out=. --proto_path=. --micro_out=. --go_out=. proto/otp.proto - @redoc-cli bundle api-otp.json +.PHONY: api +api: + protoc --openapi_out=. --proto_path=. proto/otp.proto .PHONY: build build: diff --git a/otp/handler/otp.go b/otp/handler/otp.go index 19a9246..dc92073 100644 --- a/otp/handler/otp.go +++ b/otp/handler/otp.go @@ -38,7 +38,7 @@ func (e *Otp) Generate(ctx context.Context, req *pb.GenerateRequest, rsp *pb.Gen secret = key.Secret() - if err := cache.Context(ctx).Put(req.Id, secret, time.Now().Add(time.Minute*5)); err != nil { + if err := cache.Context(ctx).Set(req.Id, secret, time.Now().Add(time.Minute*5)); err != nil { logger.Error("Failed to store secret: %v", err) return errors.InternalServerError("otp.generate", "failed to generate code") } diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index b8934e8..e5e179e 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -4,9 +4,12 @@ package cache import ( "context" "encoding/json" + "errors" "fmt" + "sync" "time" + "github.com/hashicorp/golang-lru" "github.com/micro/micro/v3/service/store" "github.com/micro/services/pkg/tenant" ) @@ -15,21 +18,39 @@ type Cache interface { // Context returns a tenant scoped Cache Context(ctx context.Context) Cache Get(key string, val interface{}) error - Put(key string, val interface{}, expires time.Time) error + Set(key string, val interface{}, expires time.Time) error Delete(key string) error + Increment(key string, val int64) (int64, error) + Decrement(key string, val int64) (int64, error) } type cache struct { + sync.Mutex + LRU *lru.Cache Store store.Store Prefix string } +type item struct { + key string + val []byte + expires time.Time +} + var ( + DefaultCacheSize = 1000 + DefaultCache = New(nil) + + ErrNotFound = errors.New("not found") ) func New(st store.Store) Cache { - return &cache{Store: st} + l, _ := lru.New(DefaultCacheSize) + return &cache{ + LRU: l, + Store: st, + } } func (c *cache) Key(k string) string { @@ -51,6 +72,29 @@ func (c *cache) Context(ctx context.Context) Cache { } func (c *cache) Get(key string, val interface{}) error { + k := c.Key(key) + + // try the LRU + v, ok := c.LRU.Get(key) + if ok { + i := v.(*item) + + // check if the item expired + if !i.expires.IsZero() && i.expires.Sub(time.Now()).Seconds() < 0 { + // remove it + c.LRU.Remove(k) + return ErrNotFound + } + + // otherwise unmarshal and return it + if err := json.Unmarshal(i.val, val); err != nil { + return err + } + + return nil + } + + // otherwise check the store if c.Store == nil { c.Store = store.DefaultStore } @@ -60,7 +104,7 @@ func (c *cache) Get(key string, val interface{}) error { return err } if len(recs) == 0 { - return store.ErrNotFound + return ErrNotFound } if err := json.Unmarshal(recs[0].Value, val); err != nil { return err @@ -68,7 +112,7 @@ func (c *cache) Get(key string, val interface{}) error { return nil } -func (c *cache) Put(key string, val interface{}, expires time.Time) error { +func (c *cache) Set(key string, val interface{}, expires time.Time) error { if c.Store == nil { c.Store = store.DefaultStore } @@ -80,18 +124,60 @@ func (c *cache) Put(key string, val interface{}, expires time.Time) error { if expiry < time.Duration(0) { expiry = time.Duration(0) } - return c.Store.Write(&store.Record{ + rec := &store.Record{ Key: c.Key(key), Value: b, Expiry: expiry, - }) + } + if err := c.Store.Write(rec); err != nil { + return err + } + + // set in the lru + c.LRU.Add(rec.Key, &item{key: rec.Key, val: rec.Value, expires: expires}) + return nil } func (c *cache) Delete(key string) error { if c.Store == nil { c.Store = store.DefaultStore } - return c.Store.Delete(c.Key(key)) + + k := c.Key(key) + // remove from the lru + c.LRU.Remove(k) + // delete from the store + return c.Store.Delete(k) +} + +func (c *cache) Increment(key string, value int64) (int64, error) { + c.Lock() + defer c.Unlock() + + var val int64 + if err := c.Get(key, &val); err != nil { + return 0, err + } + val += value + if err := c.Set(key, val, time.Time{}); err != nil { + return val, err + } + return val, nil +} + +func (c *cache) Decrement(key string, value int64) (int64, error) { + c.Lock() + defer c.Unlock() + + var val int64 + if err := c.Get(key, &val); err != nil { + return 0, err + } + val -= value + if err := c.Set(key, val, time.Time{}); err != nil { + return val, err + } + return val, nil } func Context(ctx context.Context) Cache { @@ -102,10 +188,18 @@ func Get(key string, val interface{}) error { return DefaultCache.Get(key, val) } -func Put(key string, val interface{}, expires time.Time) error { - return DefaultCache.Put(key, val, expires) +func Set(key string, val interface{}, expires time.Time) error { + return DefaultCache.Set(key, val, expires) } func Delete(key string) error { return DefaultCache.Delete(key) } + +func Increment(key string, val int64) (int64, error) { + return DefaultCache.Increment(key, val) +} + +func Decrement(key string, val int64) (int64, error) { + return DefaultCache.Decrement(key, val) +} diff --git a/routing/Makefile b/routing/Makefile index d044869..a03adba 100644 --- a/routing/Makefile +++ b/routing/Makefile @@ -5,14 +5,14 @@ init: go get -u github.com/golang/protobuf/proto go get -u github.com/golang/protobuf/protoc-gen-go go get github.com/micro/micro/v3/cmd/protoc-gen-micro + go get github.com/micro/micro/v3/cmd/protoc-gen-openapi .PHONY: proto proto: - protoc --openapi_out=. --proto_path=. --micro_out=. --go_out=:. proto/routing.proto + protoc --proto_path=. --micro_out=. --go_out=:. proto/routing.proto -.PHONY: docs -docs: - protoc --openapi_out=. --proto_path=. --micro_out=. --go_out=:. proto/routing.proto - @redoc-cli bundle api-routing.json +.PHONY: api +api: + protoc --openapi_out=. --proto_path=. proto/routing.proto .PHONY: build build: diff --git a/routing/proto/routing.pb.go b/routing/proto/routing.pb.go index 23c1b35..0ccbbd0 100644 --- a/routing/proto/routing.pb.go +++ b/routing/proto/routing.pb.go @@ -25,7 +25,9 @@ type Point struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Latitude float64 `protobuf:"fixed64,1,opt,name=latitude,proto3" json:"latitude,omitempty"` + // Lat e.g 52.523219 + Latitude float64 `protobuf:"fixed64,1,opt,name=latitude,proto3" json:"latitude,omitempty"` + // Long e.g 13.428555 Longitude float64 `protobuf:"fixed64,2,opt,name=longitude,proto3" json:"longitude,omitempty"` } @@ -368,12 +370,15 @@ func (x *Direction) GetIntersections() []*Intersection { return nil } +// Turn by turn directions from a starting and endpoint including maneuvers and bearings type DirectionsRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Origin *Point `protobuf:"bytes,1,opt,name=origin,proto3" json:"origin,omitempty"` + // The staring point for the journey + Origin *Point `protobuf:"bytes,1,opt,name=origin,proto3" json:"origin,omitempty"` + // The destinationg of the journey Destination *Point `protobuf:"bytes,2,opt,name=destination,proto3" json:"destination,omitempty"` } @@ -498,12 +503,15 @@ func (x *DirectionsResponse) GetDuration() float64 { return 0 } +// Get the eta for a route from origin to destination. The eta is an estimated time based on car routes type EtaRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Origin *Point `protobuf:"bytes,1,opt,name=origin,proto3" json:"origin,omitempty"` + // The starting point for the eta calculation + Origin *Point `protobuf:"bytes,1,opt,name=origin,proto3" json:"origin,omitempty"` + // The end point for the eta calculation Destination *Point `protobuf:"bytes,2,opt,name=destination,proto3" json:"destination,omitempty"` // type of transport e.g car, foot, bicycle Type string `protobuf:"bytes,3,opt,name=type,proto3" json:"type,omitempty"` @@ -619,6 +627,7 @@ func (x *EtaResponse) GetDuration() float64 { return 0 } +// Retrieve a route as a simple list of gps points along with total distance and estimated duration type RouteRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache diff --git a/url/Makefile b/url/Makefile index c7557b8..1846ba9 100644 --- a/url/Makefile +++ b/url/Makefile @@ -5,6 +5,7 @@ init: go get -u github.com/golang/protobuf/proto go get -u github.com/golang/protobuf/protoc-gen-go go get github.com/micro/micro/v3/cmd/protoc-gen-micro + go get github.com/micro/micro/v3/cmd/protoc-gen-openapi .PHONY: proto proto: protoc --proto_path=. --micro_out=. --go_out=:. proto/url.proto @@ -13,10 +14,9 @@ proto: build: go build -o url *.go -.PHONY: docs -docs: - protoc --openapi_out=. --proto_path=. --micro_out=. --go_out=:. proto/url.proto - @redoc-cli bundle api-url.json +.PHONY: api +api: + protoc --openapi_out=. --proto_path=. proto/url.proto .PHONY: test test: diff --git a/users/Makefile b/users/Makefile index 02a419a..2c2d824 100644 --- a/users/Makefile +++ b/users/Makefile @@ -5,13 +5,14 @@ init: go get -u github.com/golang/protobuf/proto go get -u github.com/golang/protobuf/protoc-gen-go go get github.com/micro/micro/v3/cmd/protoc-gen-micro + go get github.com/micro/micro/v3/cmd/protoc-gen-openapi .PHONY: proto proto: protoc --proto_path=. --micro_out=. --go_out=:. proto/users.proto -docs: - protoc --openapi_out=. --proto_path=. --micro_out=. --go_out=. proto/users.proto - @redoc-cli bundle api-users.json +.PHONY: api +api: + protoc --openapi_out=. --proto_path=. proto/users.proto .PHONY: build build: