Multitenant streams api (#72)

2026-01-18 13:45:09 +00:00 · 2021-03-18 17:21:41 +00:00
parent 28ad626d91
commit 8dfe49f813
87 changed files with 1890 additions and 1064 deletions
--- a/places/Makefile
+++ b/places/Makefile
@@ -12,7 +12,7 @@ proto:
 .PHONY: docs
 docs:
 	protoc --openapi_out=. --proto_path=. --micro_out=. --go_out=:. proto/places.proto
-	@redoc-cli bundle api-places.json
+	@redoc-cli bundle api-protobuf.json

 .PHONY: build
 build:
--- a/places/handler/places.go
+++ b/places/handler/places.go
@@ -93,7 +93,7 @@ func (l *Places) Last(ctx context.Context, req *pb.LastRequest, rsp *pb.ListResp
 	}

 	// query the database
-	q := l.DB.Raw("SELECT DISTINCT ON (place_id) place_id, timestamp, latitude, longitude FROM places WHERE place_id IN (?) ORDER BY place_id, timestamp DESC", req.Ids)
+	q := l.DB.Raw("SELECT DISTINCT ON (place_id) place_id, timestamp, latitude, longitude FROM locations WHERE place_id IN (?) ORDER BY place_id, timestamp DESC", req.Ids)
 	var locs []*model.Location
 	if err := q.Find(&locs).Error; err != nil {
 		logger.Errorf("Error reading from the database: %v", err)
--- a/places/handler/places_test.go
+++ b/places/handler/places_test.go
@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"

+	"github.com/golang/protobuf/ptypes/timestamp"
 	"github.com/google/uuid"
 	geo "github.com/hailocab/go-geoindex"
 	"github.com/stretchr/testify/assert"
@@ -22,29 +23,29 @@ import (

 func testHandler(t *testing.T) pb.PlacesHandler {
 	// connect to the database
-	db, err := gorm.Open(postgres.Open("postgresql://postgres@localhost:5432/places?sslmode=disable"), &gorm.Config{})
+	addr := os.Getenv("POSTGRES_URL")
+	if len(addr) == 0 {
+		addr = "postgresql://postgres@localhost:5432/postgres?sslmode=disable"
+	}
+	db, err := gorm.Open(postgres.Open(addr), &gorm.Config{})
 	if err != nil {
 		t.Fatalf("Error connecting to database: %v", err)
 	}

+	// clean any data from a previous run
+	if err := db.Exec("DROP TABLE IF EXISTS locations CASCADE").Error; err != nil {
+		t.Fatalf("Error cleaning database: %v", err)
+	}
+
 	// migrate the database
 	if err := db.AutoMigrate(&model.Location{}); err != nil {
 		t.Fatalf("Error migrating database: %v", err)
 	}

-	// clean any data from a previous run
-	if err := db.Exec("TRUNCATE TABLE places CASCADE").Error; err != nil {
-		t.Fatalf("Error cleaning database: %v", err)
-	}
-
 	return &handler.Places{DB: db, Geoindex: geo.NewPointsIndex(geo.Km(0.1))}
 }

 func TestSave(t *testing.T) {
-	if v := os.Getenv("IN_TRAVIS_CI"); v == "yes" {
-		return
-	}
-
 	tt := []struct {
 		Name   string
 		Places []*pb.Location
@@ -116,9 +117,6 @@ func TestSave(t *testing.T) {
 }

 func TestLast(t *testing.T) {
-	if v := os.Getenv("IN_TRAVIS_CI"); v == "yes" {
-		return
-	}
 	h := testHandler(t)

 	t.Run("MissingIDs", func(t *testing.T) {
@@ -134,24 +132,25 @@ func TestLast(t *testing.T) {
 		assert.NoError(t, err)
 		assert.Empty(t, rsp.Places)
 	})
+	tn := time.Now()

 	// generate some example data to work with
 	loc1 := &pb.Location{
 		Latitude:  &wrapperspb.DoubleValue{Value: 51.5007},
 		Longitude: &wrapperspb.DoubleValue{Value: 0.1246},
-		Timestamp: timestamppb.New(time.Now()),
+		Timestamp: timestamppb.New(tn),
 		Id:        "a",
 	}
 	loc2 := &pb.Location{
 		Latitude:  &wrapperspb.DoubleValue{Value: 51.6007},
 		Longitude: &wrapperspb.DoubleValue{Value: 0.1546},
-		Timestamp: timestamppb.New(time.Now()),
+		Timestamp: timestamppb.New(tn.Add(1 * time.Microsecond)),
 		Id:        "b",
 	}
 	loc3 := &pb.Location{
 		Latitude:  &wrapperspb.DoubleValue{Value: 52.6007},
 		Longitude: &wrapperspb.DoubleValue{Value: 0.2546},
-		Timestamp: timestamppb.New(time.Now()),
+		Timestamp: timestamppb.New(tn.Add(2 * time.Microsecond)),
 		Id:        loc2.Id,
 	}
 	err := h.Save(context.TODO(), &pb.SaveRequest{
@@ -162,7 +161,7 @@ func TestLast(t *testing.T) {
 	t.Run("OneUser", func(t *testing.T) {
 		var rsp pb.ListResponse
 		err := h.Last(context.Background(), &pb.LastRequest{
-			Ids: []string{loc2.Id},
+			Ids: []string{loc3.Id},
 		}, &rsp)
 		assert.NoError(t, err)

@@ -172,8 +171,9 @@ func TestLast(t *testing.T) {
 		assert.Equal(t, loc3.Id, rsp.Places[0].Id)
 		assert.Equal(t, loc3.Latitude.Value, rsp.Places[0].Latitude.Value)
 		assert.Equal(t, loc3.Longitude.Value, rsp.Places[0].Longitude.Value)
-		assert.Equal(t, loc3.Timestamp.AsTime(), rsp.Places[0].Timestamp.AsTime())
+		assert.Equal(t, microSecondTime(loc3.Timestamp), microSecondTime(rsp.Places[0].Timestamp))
 	})
+
 	t.Run("ManyUser", func(t *testing.T) {
 		var rsp pb.ListResponse
 		err := h.Last(context.Background(), &pb.LastRequest{
@@ -193,19 +193,16 @@ func TestLast(t *testing.T) {
 		assert.Equal(t, loc1.Id, rsp.Places[1].Id)
 		assert.Equal(t, loc1.Latitude.Value, rsp.Places[1].Latitude.Value)
 		assert.Equal(t, loc1.Longitude.Value, rsp.Places[1].Longitude.Value)
-		assert.Equal(t, loc1.Timestamp.AsTime(), rsp.Places[1].Timestamp.AsTime())
+		assert.Equal(t, microSecondTime(loc1.Timestamp), microSecondTime(rsp.Places[1].Timestamp))

 		assert.Equal(t, loc3.Id, rsp.Places[0].Id)
 		assert.Equal(t, loc3.Latitude.Value, rsp.Places[0].Latitude.Value)
 		assert.Equal(t, loc3.Longitude.Value, rsp.Places[0].Longitude.Value)
-		assert.Equal(t, loc3.Timestamp.AsTime(), rsp.Places[0].Timestamp.AsTime())
+		assert.Equal(t, microSecondTime(loc3.Timestamp), microSecondTime(rsp.Places[0].Timestamp))
 	})
 }

 func TestNear(t *testing.T) {
-	if v := os.Getenv("IN_TRAVIS_CI"); v == "yes" {
-		return
-	}
 	lat := &wrapperspb.DoubleValue{Value: 51.510357}
 	lng := &wrapperspb.DoubleValue{Value: -0.116773}
 	rad := &wrapperspb.DoubleValue{Value: 2.0}
@@ -401,9 +398,6 @@ func TestNear(t *testing.T) {
 }

 func TestRead(t *testing.T) {
-	if v := os.Getenv("IN_TRAVIS_CI"); v == "yes" {
-		return
-	}
 	h := testHandler(t)

 	baseTime := time.Now().Add(time.Hour * -24)
@@ -482,12 +476,12 @@ func TestRead(t *testing.T) {
 		assert.Equal(t, loc2.Id, rsp.Places[0].Id)
 		assert.Equal(t, loc2.Latitude.Value, rsp.Places[0].Latitude.Value)
 		assert.Equal(t, loc2.Longitude.Value, rsp.Places[0].Longitude.Value)
-		assert.Equal(t, loc2.Timestamp.AsTime(), rsp.Places[0].Timestamp.AsTime())
+		assert.Equal(t, microSecondTime(loc2.Timestamp), microSecondTime(rsp.Places[0].Timestamp))

 		assert.Equal(t, loc3.Id, rsp.Places[1].Id)
 		assert.Equal(t, loc3.Latitude.Value, rsp.Places[1].Latitude.Value)
 		assert.Equal(t, loc3.Longitude.Value, rsp.Places[1].Longitude.Value)
-		assert.Equal(t, loc3.Timestamp.AsTime(), rsp.Places[1].Timestamp.AsTime())
+		assert.Equal(t, microSecondTime(loc3.Timestamp), microSecondTime(rsp.Places[1].Timestamp))
 	})

 	t.Run("OnePlaceIDReducedTime", func(t *testing.T) {
@@ -505,7 +499,7 @@ func TestRead(t *testing.T) {
 		assert.Equal(t, loc2.Id, rsp.Places[0].Id)
 		assert.Equal(t, loc2.Latitude.Value, rsp.Places[0].Latitude.Value)
 		assert.Equal(t, loc2.Longitude.Value, rsp.Places[0].Longitude.Value)
-		assert.Equal(t, loc2.Timestamp.AsTime(), rsp.Places[0].Timestamp.AsTime())
+		assert.Equal(t, microSecondTime(loc2.Timestamp), microSecondTime(rsp.Places[0].Timestamp))
 	})

 	t.Run("TwoPlaceIDs", func(t *testing.T) {
@@ -523,11 +517,17 @@ func TestRead(t *testing.T) {
 		assert.Equal(t, loc1.Id, rsp.Places[0].Id)
 		assert.Equal(t, loc1.Latitude.Value, rsp.Places[0].Latitude.Value)
 		assert.Equal(t, loc1.Longitude.Value, rsp.Places[0].Longitude.Value)
-		assert.Equal(t, loc1.Timestamp.AsTime(), rsp.Places[0].Timestamp.AsTime())
+		assert.Equal(t, microSecondTime(loc1.Timestamp), microSecondTime(rsp.Places[0].Timestamp))

 		assert.Equal(t, loc2.Id, rsp.Places[1].Id)
 		assert.Equal(t, loc2.Latitude.Value, rsp.Places[1].Latitude.Value)
 		assert.Equal(t, loc2.Longitude.Value, rsp.Places[1].Longitude.Value)
-		assert.Equal(t, loc2.Timestamp.AsTime(), rsp.Places[1].Timestamp.AsTime())
+		assert.Equal(t, microSecondTime(loc2.Timestamp), microSecondTime(rsp.Places[1].Timestamp))
 	})
 }
+
+// postgres has a resolution of 100microseconds so just test that it's accurate to the second
+func microSecondTime(t *timestamp.Timestamp) time.Time {
+	tt := t.AsTime()
+	return time.Unix(tt.Unix(), int64(tt.Nanosecond()-tt.Nanosecond()%1000))
+}
--- a/places/proto/places.pb.go
+++ b/places/proto/places.pb.go
@@ -1,17 +1,16 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.25.0
-// 	protoc        v3.6.1
+// 	protoc-gen-go v1.26.0
+// 	protoc        v3.15.5
 // source: proto/places.proto

 package places

 import (
-	proto "github.com/golang/protobuf/proto"
-	timestamp "github.com/golang/protobuf/ptypes/timestamp"
-	wrappers "github.com/golang/protobuf/ptypes/wrappers"
 	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
+	wrapperspb "google.golang.org/protobuf/types/known/wrapperspb"
 	reflect "reflect"
 	sync "sync"
 )
@@ -23,21 +22,17 @@ const (
 	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
 )

-// This is a compile-time assertion that a sufficiently up-to-date version
-// of the legacy proto package is being used.
-const _ = proto.ProtoPackageIsVersion4
-
 type Location struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields

-	Id        string                `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
-	Name      string                `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
-	Metadata  map[string]string     `protobuf:"bytes,3,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
-	Timestamp *timestamp.Timestamp  `protobuf:"bytes,4,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
-	Latitude  *wrappers.DoubleValue `protobuf:"bytes,5,opt,name=latitude,proto3" json:"latitude,omitempty"`
-	Longitude *wrappers.DoubleValue `protobuf:"bytes,6,opt,name=longitude,proto3" json:"longitude,omitempty"`
+	Id        string                  `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
+	Name      string                  `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
+	Metadata  map[string]string       `protobuf:"bytes,3,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	Timestamp *timestamppb.Timestamp  `protobuf:"bytes,4,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
+	Latitude  *wrapperspb.DoubleValue `protobuf:"bytes,5,opt,name=latitude,proto3" json:"latitude,omitempty"`
+	Longitude *wrapperspb.DoubleValue `protobuf:"bytes,6,opt,name=longitude,proto3" json:"longitude,omitempty"`
 }

 func (x *Location) Reset() {
@@ -93,21 +88,21 @@ func (x *Location) GetMetadata() map[string]string {
 	return nil
 }

-func (x *Location) GetTimestamp() *timestamp.Timestamp {
+func (x *Location) GetTimestamp() *timestamppb.Timestamp {
 	if x != nil {
 		return x.Timestamp
 	}
 	return nil
 }

-func (x *Location) GetLatitude() *wrappers.DoubleValue {
+func (x *Location) GetLatitude() *wrapperspb.DoubleValue {
 	if x != nil {
 		return x.Latitude
 	}
 	return nil
 }

-func (x *Location) GetLongitude() *wrappers.DoubleValue {
+func (x *Location) GetLongitude() *wrapperspb.DoubleValue {
 	if x != nil {
 		return x.Longitude
 	}
@@ -298,10 +293,10 @@ type NearRequest struct {
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields

-	Latitude  *wrappers.DoubleValue `protobuf:"bytes,1,opt,name=latitude,proto3" json:"latitude,omitempty"`
-	Longitude *wrappers.DoubleValue `protobuf:"bytes,2,opt,name=longitude,proto3" json:"longitude,omitempty"`
+	Latitude  *wrapperspb.DoubleValue `protobuf:"bytes,1,opt,name=latitude,proto3" json:"latitude,omitempty"`
+	Longitude *wrapperspb.DoubleValue `protobuf:"bytes,2,opt,name=longitude,proto3" json:"longitude,omitempty"`
 	// radius to search within, units km
-	Radius *wrappers.DoubleValue `protobuf:"bytes,3,opt,name=radius,proto3" json:"radius,omitempty"`
+	Radius *wrapperspb.DoubleValue `protobuf:"bytes,3,opt,name=radius,proto3" json:"radius,omitempty"`
 }

 func (x *NearRequest) Reset() {
@@ -336,21 +331,21 @@ func (*NearRequest) Descriptor() ([]byte, []int) {
 	return file_proto_places_proto_rawDescGZIP(), []int{5}
 }

-func (x *NearRequest) GetLatitude() *wrappers.DoubleValue {
+func (x *NearRequest) GetLatitude() *wrapperspb.DoubleValue {
 	if x != nil {
 		return x.Latitude
 	}
 	return nil
 }

-func (x *NearRequest) GetLongitude() *wrappers.DoubleValue {
+func (x *NearRequest) GetLongitude() *wrapperspb.DoubleValue {
 	if x != nil {
 		return x.Longitude
 	}
 	return nil
 }

-func (x *NearRequest) GetRadius() *wrappers.DoubleValue {
+func (x *NearRequest) GetRadius() *wrapperspb.DoubleValue {
 	if x != nil {
 		return x.Radius
 	}
@@ -362,9 +357,9 @@ type ReadRequest struct {
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields

-	Ids    []string             `protobuf:"bytes,1,rep,name=ids,proto3" json:"ids,omitempty"`
-	After  *timestamp.Timestamp `protobuf:"bytes,2,opt,name=after,proto3" json:"after,omitempty"`
-	Before *timestamp.Timestamp `protobuf:"bytes,3,opt,name=before,proto3" json:"before,omitempty"`
+	Ids    []string               `protobuf:"bytes,1,rep,name=ids,proto3" json:"ids,omitempty"`
+	After  *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=after,proto3" json:"after,omitempty"`
+	Before *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=before,proto3" json:"before,omitempty"`
 }

 func (x *ReadRequest) Reset() {
@@ -406,14 +401,14 @@ func (x *ReadRequest) GetIds() []string {
 	return nil
 }

-func (x *ReadRequest) GetAfter() *timestamp.Timestamp {
+func (x *ReadRequest) GetAfter() *timestamppb.Timestamp {
 	if x != nil {
 		return x.After
 	}
 	return nil
 }

-func (x *ReadRequest) GetBefore() *timestamp.Timestamp {
+func (x *ReadRequest) GetBefore() *timestamppb.Timestamp {
 	if x != nil {
 		return x.Before
 	}
@@ -494,8 +489,9 @@ var file_proto_places_proto_rawDesc = []byte{
 	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x04, 0x52, 0x65, 0x61,
 	0x64, 0x12, 0x13, 0x2e, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x73, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x52,
 	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x73, 0x2e,
-	0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x62, 0x06,
-	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x10,
+	0x5a, 0x0e, 0x2e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3b, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x73,
+	0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }

 var (
@@ -512,16 +508,16 @@ func file_proto_places_proto_rawDescGZIP() []byte {

 var file_proto_places_proto_msgTypes = make([]protoimpl.MessageInfo, 8)
 var file_proto_places_proto_goTypes = []interface{}{
-	(*Location)(nil),             // 0: places.Location
-	(*SaveRequest)(nil),          // 1: places.SaveRequest
-	(*SaveResponse)(nil),         // 2: places.SaveResponse
-	(*LastRequest)(nil),          // 3: places.LastRequest
-	(*ListResponse)(nil),         // 4: places.ListResponse
-	(*NearRequest)(nil),          // 5: places.NearRequest
-	(*ReadRequest)(nil),          // 6: places.ReadRequest
-	nil,                          // 7: places.Location.MetadataEntry
-	(*timestamp.Timestamp)(nil),  // 8: google.protobuf.Timestamp
-	(*wrappers.DoubleValue)(nil), // 9: google.protobuf.DoubleValue
+	(*Location)(nil),               // 0: places.Location
+	(*SaveRequest)(nil),            // 1: places.SaveRequest
+	(*SaveResponse)(nil),           // 2: places.SaveResponse
+	(*LastRequest)(nil),            // 3: places.LastRequest
+	(*ListResponse)(nil),           // 4: places.ListResponse
+	(*NearRequest)(nil),            // 5: places.NearRequest
+	(*ReadRequest)(nil),            // 6: places.ReadRequest
+	nil,                            // 7: places.Location.MetadataEntry
+	(*timestamppb.Timestamp)(nil),  // 8: google.protobuf.Timestamp
+	(*wrapperspb.DoubleValue)(nil), // 9: google.protobuf.DoubleValue
 }
 var file_proto_places_proto_depIdxs = []int32{
 	7,  // 0: places.Location.metadata:type_name -> places.Location.MetadataEntry
--- a/places/proto/places.pb.micro.go
+++ b/places/proto/places.pb.micro.go
@@ -6,8 +6,8 @@ package places
 import (
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	_ "github.com/golang/protobuf/ptypes/timestamp"
-	_ "github.com/golang/protobuf/ptypes/wrappers"
+	_ "google.golang.org/protobuf/types/known/timestamppb"
+	_ "google.golang.org/protobuf/types/known/wrapperspb"
 	math "math"
 )

--- a/places/proto/places.proto
+++ b/places/proto/places.proto
@@ -1,6 +1,7 @@
 syntax = "proto3";

 package places;
+option go_package = "./proto;places";

 import "google/protobuf/timestamp.proto";
 import "google/protobuf/wrappers.proto";