totole/services
1
0
Fork 0
mirror of https://github.com/kevin-DL/services.git synced 2026-01-14 12:04:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Space updates (#299)

Browse Source 
* updates for timestamps and storing meta for quick retrieval

* fix up head vis
This commit is contained in:
Dominic Wong
2021-12-09 16:25:34 +00:00
committed by GitHub
parent 395c190af3
commit dbdd3715a9
5 changed files with 239 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
space/handler/space.go
View File

@@ -14,6 +14,7 @@ import (
"github.com/micro/micro/v3/service/config"
"github.com/micro/micro/v3/service/errors"
log "github.com/micro/micro/v3/service/logger"
"github.com/micro/micro/v3/service/store"
"github.com/micro/services/pkg/tenant"
pb "github.com/micro/services/space/proto"
"github.com/minio/minio-go/v7/pkg/s3utils"
@@ -34,6 +35,8 @@ const (
visibilityPrivate = "private"
visibilityPublic = "public"
prefixByUser = "byUser"
)
type Space struct {
@@ -51,6 +54,12 @@ type conf struct {
BaseURL string `json:"base_url"`
}
type meta struct {
Visibility string
CreateTime string
ModifiedTime string
}
func NewSpace(srv *service.Service) *Space {
var c conf
val, err := config.Get("micro.space")
@@ -119,7 +128,7 @@ func (s Space) upsert(ctx context.Context, object []byte, name, visibility, meth
return "", errors.BadRequest(method, "Object already exists")
}
createTime := aws.String(fmt.Sprintf("%d", time.Now().Unix()))
createTime := aws.String(time.Now().Format(time.RFC3339Nano))
if exists {
createTime = hoo.Metadata[mdCreated]
}
@@ -146,8 +155,19 @@ func (s Space) upsert(ctx context.Context, object []byte, name, visibility, meth
return "", errors.InternalServerError(method, "Error creating object")
}
// TODO fix the url
return fmt.Sprintf("%s/%s", s.conf.BaseURL, objectName), nil
// store the metadata for easy retrieval for listing
if err := store.Write(store.NewRecord(
fmt.Sprintf("%s/%s", prefixByUser, objectName),
meta{Visibility: visibility, CreateTime: *createTime, ModifiedTime: time.Now().Format(time.RFC3339Nano)})); err != nil {
log.Errorf("Error writing object to store %s", err)
return "", errors.InternalServerError(method, "Error creating object")
}
retUrl := ""
if visibility == "public" {
retUrl = fmt.Sprintf("%s/%s", s.conf.BaseURL, objectName)
}
return retUrl, nil
}
@@ -174,6 +194,10 @@ func (s Space) Delete(ctx context.Context, request *pb.DeleteRequest, response *
log.Errorf("Error deleting object %s", err)
return errors.InternalServerError(method, "Error deleting object")
}
if err := store.Delete(fmt.Sprintf("%s/%s", prefixByUser, objectName)); err != nil {
log.Errorf("Error deleting store record %s", err)
return errors.InternalServerError(method, "Error deleting object")
}
return nil
}
@@ -192,12 +216,38 @@ func (s Space) List(ctx context.Context, request *pb.ListRequest, response *pb.L
log.Errorf("Error listing objects %s", err)
return errors.InternalServerError(method, "Error listing objects")
}
recs, err := store.Read(fmt.Sprintf("%s/%s", prefixByUser, objectName), store.ReadPrefix())
if err != nil {
log.Errorf("Error listing objects %s", err)
return errors.InternalServerError(method, "Error listing objects")
}
md := map[string]meta{}
for _, r := range recs {
var m meta
if err := json.Unmarshal(r.Value, &m); err != nil {
log.Errorf("Error unmarshaling meta %s", err)
return errors.InternalServerError(method, "Error listing objects")
}
md[strings.TrimPrefix(r.Key, prefixByUser+"/")] = m
}
response.Objects = []*pb.ListObject{}
for _, oi := range rsp.Contents {
m, ok := md[*oi.Key]
if !ok {
// hack for now
m = meta{}
}
url := ""
if m.Visibility == "public" {
url = fmt.Sprintf("%s/%s", s.conf.BaseURL, *oi.Key)
}
response.Objects = append(response.Objects, &pb.ListObject{
Name: strings.TrimPrefix(*oi.Key, tnt+"/"),
Modified: oi.LastModified.Unix(),
Url: fmt.Sprintf("%s/%s", s.conf.BaseURL, *oi.Key),
Name: strings.TrimPrefix(*oi.Key, tnt+"/"),
Modified: oi.LastModified.Format(time.RFC3339Nano),
Url: url,
Visibility: m.Visibility,
Created: m.CreateTime,
})
}
return nil
@@ -231,20 +281,31 @@ func (s Space) Head(ctx context.Context, request *pb.HeadRequest, response *pb.H
if md, ok := goo.Metadata[mdVisibility]; ok && len(*md) > 0 {
vis = *md
}
var created int64
var created string
if md, ok := goo.Metadata[mdCreated]; ok && len(*md) > 0 {
created, err = strconv.ParseInt(*md, 10, 64)
t, err := time.Parse(time.RFC3339Nano, *md)
if err != nil {
log.Errorf("Error %s", err)
// try as unix ts
createdI, err := strconv.ParseInt(*md, 10, 64)
if err != nil {
log.Errorf("Error %s", err)
} else {
t = time.Unix(createdI, 0)
}
}
created = t.Format(time.RFC3339Nano)
}
url := ""
if vis == "public" {
url = fmt.Sprintf("%s/%s", s.conf.BaseURL, objectName)
}
response.Object = &pb.HeadObject{
Name: request.Name,
Modified: goo.LastModified.Unix(),
Modified: goo.LastModified.Format(time.RFC3339Nano),
Created: created,
Visibility: vis,
Url: fmt.Sprintf("%s/%s", s.conf.BaseURL, objectName),
Url: url,
}
return nil

90
space/handler/space_test.go
View File

@@ -11,6 +11,8 @@ import (
"github.com/aws/aws-sdk-go/service/s3/s3iface"
"github.com/micro/micro/v3/service/auth"
"github.com/micro/micro/v3/service/errors"
"github.com/micro/micro/v3/service/store"
"github.com/micro/micro/v3/service/store/memory"
pb "github.com/micro/services/space/proto"
. "github.com/onsi/gomega"
@@ -96,7 +98,7 @@ func TestCreate(t *testing.T) {
{
name: "Simple case",
objName: "foo.jpg",
url: "https://my-space.ams3.example.com/micro/123/foo.jpg",
url: "",
head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) {
return nil, mockError{code: "NotFound"}
},
@@ -140,6 +142,7 @@ func TestCreate(t *testing.T) {
},
},
}
store.DefaultStore = memory.NewStore()
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
@@ -196,7 +199,7 @@ func TestUpdate(t *testing.T) {
{
name: "Does not exist",
objName: "foo.jpg",
url: "https://my-space.ams3.example.com/micro/123/foo.jpg",
url: "",
head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) {
return nil, mockError{code: "NotFound"}
},
@@ -250,7 +253,7 @@ func TestUpdate(t *testing.T) {
g.Expect(*input.Metadata[mdCreated]).To(Equal("1638541918"))
return &sthree.PutObjectOutput{}, nil
},
url: "https://my-space.ams3.example.com/micro/123/foo.jpg",
url: "",
},
{
name: "Already exists public",
@@ -277,6 +280,7 @@ func TestUpdate(t *testing.T) {
visibility: "public",
},
}
store.DefaultStore = memory.NewStore()
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
@@ -338,6 +342,7 @@ func TestDelete(t *testing.T) {
err: errors.BadRequest("space.Delete", "Missing name param"),
},
}
store.DefaultStore = memory.NewStore()
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
@@ -385,22 +390,38 @@ func TestDelete(t *testing.T) {
func TestList(t *testing.T) {
g := NewWithT(t)
tcs := []struct {
name string
prefix string
err error
list func(input *sthree.ListObjectsInput) (*sthree.ListObjectsInput, error)
name string
prefix string
err error
list func(input *sthree.ListObjectsInput) (*sthree.ListObjectsOutput, error)
visibility string
}{
{
name: "Simple case",
prefix: "foo",
prefix: "file",
},
{
name: "Empty prefix",
},
}
store.DefaultStore = memory.NewStore()
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
store.Write(
store.NewRecord(fmt.Sprintf("%s/micro/123/file.jpg", prefixByUser),
meta{
Visibility: "public",
CreateTime: "2009-11-10T23:00:00Z",
ModifiedTime: "2009-11-10T23:00:00Z",
}))
store.Write(
store.NewRecord(fmt.Sprintf("%s/micro/123/file2.jpg", prefixByUser),
meta{
Visibility: "private",
CreateTime: "2009-11-10T23:00:01Z",
ModifiedTime: "2009-11-10T23:00:01Z",
}))
handler := Space{
conf: conf{
AccessKey: "access",
@@ -415,7 +436,18 @@ func TestList(t *testing.T) {
list: func(input *sthree.ListObjectsInput) (*sthree.ListObjectsOutput, error) {
g.Expect(input.Bucket).To(Equal(aws.String("my-space")))
g.Expect(input.Prefix).To(Equal(aws.String("micro/123/" + tc.prefix)))
return &sthree.ListObjectsOutput{}, nil
return &sthree.ListObjectsOutput{
Contents: []*sthree.Object{
{
Key: aws.String("micro/123/file.jpg"),
LastModified: aws.Time(time.Unix(1257894000, 0)),
},
{
Key: aws.String("micro/123/file2.jpg"),
LastModified: aws.Time(time.Unix(1257894000, 0)),
},
},
}, nil
}},
}
ctx := context.Background()
@@ -435,6 +467,11 @@ func TestList(t *testing.T) {
g.Expect(err).To(Equal(tc.err))
} else {
g.Expect(err).To(BeNil())
g.Expect(rsp.Objects).To(HaveLen(2))
g.Expect(rsp.Objects[0].Name).To(Equal("file.jpg"))
g.Expect(rsp.Objects[0].Url).To(Equal("https://my-space.ams3.example.com/micro/123/file.jpg"))
g.Expect(rsp.Objects[1].Name).To(Equal("file2.jpg"))
g.Expect(rsp.Objects[1].Url).To(Equal(""))
}
})
@@ -449,8 +486,8 @@ func TestHead(t *testing.T) {
objectName string
url string
visibility string
modified int64
created int64
modified string
created string
err error
head func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error)
}{
@@ -459,21 +496,41 @@ func TestHead(t *testing.T) {
objectName: "foo.jpg",
visibility: "public",
url: "https://my-space.ams3.example.com/micro/123/foo.jpg",
created: 1638547905,
modified: 1638547906,
created: "2009-11-10T23:00:00Z",
modified: "2009-11-10T23:00:00Z",
head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) {
g.Expect(*input.Bucket).To(Equal("my-space"))
g.Expect(*input.Key).To(Equal("micro/123/foo.jpg"))
return &sthree.HeadObjectOutput{
LastModified: aws.Time(time.Unix(1638547906, 0)),
LastModified: aws.Time(time.Unix(1257894000, 0)),
Metadata: map[string]*string{
mdCreated: aws.String("1638547905"),
mdCreated: aws.String("1257894000"),
mdVisibility: aws.String(visibilityPublic),
},
}, nil
},
},
{
name: "Simple case private",
objectName: "foo.jpg",
visibility: "private",
url: "",
created: "2009-11-10T23:00:00Z",
modified: "2009-11-10T23:00:00Z",
head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) {
g.Expect(*input.Bucket).To(Equal("my-space"))
g.Expect(*input.Key).To(Equal("micro/123/foo.jpg"))
return &sthree.HeadObjectOutput{
LastModified: aws.Time(time.Unix(1257894000, 0)),
Metadata: map[string]*string{
mdCreated: aws.String("2009-11-10T23:00:00Z"),
mdVisibility: aws.String("private"),
},
}, nil
},
},
{
name: "Empty prefix",
err: errors.BadRequest("space.Head", "Missing name param"),
@@ -487,6 +544,7 @@ func TestHead(t *testing.T) {
},
},
}
store.DefaultStore = memory.NewStore()
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
@@ -522,7 +580,7 @@ func TestHead(t *testing.T) {
} else {
g.Expect(err).To(BeNil())
g.Expect(rsp.Object.Name).To(Equal(tc.objectName))
g.Expect(rsp.Object.Url).To(Equal("https://my-space.ams3.example.com/micro/123/" + tc.objectName))
g.Expect(rsp.Object.Url).To(Equal(tc.url))
g.Expect(rsp.Object.Visibility).To(Equal(tc.visibility))
g.Expect(rsp.Object.Created).To(Equal(tc.created))
g.Expect(rsp.Object.Modified).To(Equal(tc.modified))