totole/services
1
0
Fork 0
mirror of https://github.com/kevin-DL/services.git synced 2026-01-14 03:54:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Space.Upload (#313)

Browse Source
This commit is contained in:
Dominic Wong
2021-12-10 15:26:50 +00:00
committed by GitHub
parent 156042c6ea
commit f022c62a87
7 changed files with 355 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

184
space/handler/space.go
View File

@@ -6,7 +6,6 @@ import (
"encoding/json"
"fmt"
"io/ioutil"
"strconv"
"strings"
"time"
@@ -29,10 +28,8 @@ import (
)
const (
mdACL = "X-Amz-Acl"
mdACLPublic = "public-read"
mdCreated = "Micro-Created"
mdVisibility = "Micro-Visibility"
mdACL = "X-Amz-Acl"
mdACLPublic = "public-read"
visibilityPrivate = "private"
visibilityPublic = "public"
@@ -115,7 +112,7 @@ func (s Space) upsert(ctx context.Context, object []byte, name, visibility, meth
}
exists := false
hoo, err := s.client.HeadObject(&sthree.HeadObjectInput{
_, err := s.client.HeadObject(&sthree.HeadObjectInput{
Bucket: aws.String(s.conf.SpaceName),
Key: aws.String(objectName),
})
@@ -132,24 +129,30 @@ func (s Space) upsert(ctx context.Context, object []byte, name, visibility, meth
return "", errors.BadRequest(method, "Object already exists")
}
createTime := aws.String(time.Now().Format(time.RFC3339Nano))
if exists {
createTime = hoo.Metadata[mdCreated]
}
if len(visibility) == 0 {
visibility = visibilityPrivate
}
now := time.Now().Format(time.RFC3339Nano)
md := meta{
CreateTime: now,
ModifiedTime: now,
Visibility: visibility,
}
if exists {
m, err := s.objectMeta(objectName)
if err != nil {
log.Errorf("Error reading object meta %s", err)
return "", errors.BadRequest(method, "Error creating object")
}
md.CreateTime = m.CreateTime
}
putInput := &sthree.PutObjectInput{
Body: bytes.NewReader(object),
Key: aws.String(objectName),
Bucket: aws.String(s.conf.SpaceName),
Metadata: map[string]*string{
mdVisibility: aws.String(visibility),
mdCreated: createTime,
},
}
// TODO flesh out options - might want to do content-type for better serving of object
if visibility == visibilityPublic {
putInput.ACL = aws.String(mdACLPublic)
}
@@ -160,14 +163,12 @@ func (s Space) upsert(ctx context.Context, object []byte, name, visibility, meth
}
// store the metadata for easy retrieval for listing
if err := store.Write(store.NewRecord(
fmt.Sprintf("%s/%s", prefixByUser, objectName),
meta{Visibility: visibility, CreateTime: *createTime, ModifiedTime: time.Now().Format(time.RFC3339Nano)})); err != nil {
if err := store.Write(store.NewRecord(fmt.Sprintf("%s/%s", prefixByUser, objectName), md)); err != nil {
log.Errorf("Error writing object to store %s", err)
return "", errors.InternalServerError(method, "Error creating object")
}
retUrl := ""
if visibility == "public" {
if visibility == visibilityPublic {
retUrl = fmt.Sprintf("%s/%s", s.conf.BaseURL, objectName)
}
@@ -281,40 +282,39 @@ func (s Space) Head(ctx context.Context, request *pb.HeadRequest, response *pb.H
return errors.InternalServerError(method, "Error reading object")
}
vis := visibilityPrivate
if md, ok := goo.Metadata[mdVisibility]; ok && len(*md) > 0 {
vis = *md
}
var created string
if md, ok := goo.Metadata[mdCreated]; ok && len(*md) > 0 {
t, err := time.Parse(time.RFC3339Nano, *md)
if err != nil {
// try as unix ts
createdI, err := strconv.ParseInt(*md, 10, 64)
if err != nil {
log.Errorf("Error %s", err)
} else {
t = time.Unix(createdI, 0)
}
}
created = t.Format(time.RFC3339Nano)
md, err := s.objectMeta(objectName)
if err != nil {
log.Errorf("Error reading object meta %s", err)
return errors.InternalServerError(method, "Error reading object")
}
url := ""
if vis == "public" {
if md.Visibility == visibilityPublic {
url = fmt.Sprintf("%s/%s", s.conf.BaseURL, objectName)
}
response.Object = &pb.HeadObject{
Name: request.Name,
Modified: goo.LastModified.Format(time.RFC3339Nano),
Created: created,
Visibility: vis,
Created: md.CreateTime,
Visibility: md.Visibility,
Url: url,
}
return nil
}
func (s *Space) objectMeta(objName string) (*meta, error) {
recs, err := store.Read(fmt.Sprintf("%s/%s", prefixByUser, objName))
if err != nil {
return nil, err
}
var me meta
if err := json.Unmarshal(recs[0].Value, &me); err != nil {
return nil, err
}
return &me, nil
}
func (s *Space) Read(ctx context.Context, req *pb.ReadRequest, rsp *pb.ReadResponse) error {
method := "space.Read"
tnt, ok := tenant.FromContext(ctx)
@@ -330,55 +330,35 @@ func (s *Space) Read(ctx context.Context, req *pb.ReadRequest, rsp *pb.ReadRespo
objectName := fmt.Sprintf("%s/%s", tnt, name)
goo, err := s.client.HeadObject(&sthree.HeadObjectInput{
goo, err := s.client.GetObject(&sthree.GetObjectInput{
Bucket: aws.String(s.conf.SpaceName),
Key: aws.String(objectName),
})
if err != nil {
aerr, ok := err.(awserr.Error)
if ok && aerr.Code() == "NotFound" {
if ok && aerr.Code() == "NotSuchKey" {
return errors.BadRequest(method, "Object not found")
}
log.Errorf("Error s3 %s", err)
return errors.InternalServerError(method, "Error reading object")
}
_, gooreq := s.client.GetObjectRequest(&sthree.GetObjectInput{
Bucket: aws.String(s.conf.SpaceName),
Key: aws.String(objectName),
})
vis := visibilityPrivate
if md, ok := goo.Metadata[mdVisibility]; ok && len(*md) > 0 {
vis = *md
}
var created string
if md, ok := goo.Metadata[mdCreated]; ok && len(*md) > 0 {
t, err := time.Parse(time.RFC3339Nano, *md)
if err != nil {
// try as unix ts
createdI, err := strconv.ParseInt(*md, 10, 64)
if err != nil {
log.Errorf("Error %s", err)
} else {
t = time.Unix(createdI, 0)
}
}
created = t.Format(time.RFC3339Nano)
md, err := s.objectMeta(objectName)
if err != nil {
log.Errorf("Error reading meta %s", err)
return errors.InternalServerError(method, "Error reading object")
}
url := ""
if vis == "public" {
if md.Visibility == visibilityPublic {
url = fmt.Sprintf("%s/%s", s.conf.BaseURL, objectName)
}
if *gooreq.ContentLength > maxReadSize {
if *goo.ContentLength > maxReadSize {
return errors.BadRequest(method, "Exceeds max read size: %v bytes", maxReadSize)
}
b, err := ioutil.ReadAll(gooreq.Body)
b, err := ioutil.ReadAll(goo.Body)
if err != nil {
return errors.InternalServerError(method, "Failed to read data")
}
@@ -386,8 +366,8 @@ func (s *Space) Read(ctx context.Context, req *pb.ReadRequest, rsp *pb.ReadRespo
rsp.Object = &pb.Object{
Name: req.Name,
Modified: goo.LastModified.Format(time.RFC3339Nano),
Created: created,
Visibility: vis,
Created: md.CreateTime,
Visibility: md.Visibility,
Url: url,
Data: b,
}
@@ -461,3 +441,65 @@ func (s *Space) Download(ctx context.Context, req *api.Request, rsp *api.Respons
return nil
}
func (s Space) Upload(ctx context.Context, request *pb.UploadRequest, response *pb.UploadResponse) error {
method := "space.Upload"
tnt, ok := tenant.FromContext(ctx)
if !ok {
return errors.Unauthorized(method, "Unauthorized")
}
if len(request.Name) == 0 {
return errors.BadRequest(method, "Missing name param")
}
objectName := fmt.Sprintf("%s/%s", tnt, request.Name)
if err := s3utils.CheckValidObjectName(objectName); err != nil {
return errors.BadRequest(method, "Invalid name")
}
_, err := s.client.HeadObject(&sthree.HeadObjectInput{
Bucket: aws.String(s.conf.SpaceName),
Key: aws.String(objectName),
})
if err != nil {
aerr, ok := err.(awserr.Error)
if !ok || aerr.Code() != "NotFound" {
return errors.InternalServerError(method, "Error creating upload URL")
}
} else {
return errors.BadRequest(method, "Object already exists")
}
createTime := aws.String(time.Now().Format(time.RFC3339Nano))
if len(request.Visibility) == 0 {
request.Visibility = visibilityPrivate
}
putInput := &sthree.PutObjectInput{
Key: aws.String(objectName),
Bucket: aws.String(s.conf.SpaceName),
}
if request.Visibility == visibilityPublic {
putInput.ACL = aws.String(mdACLPublic)
}
req, _ := s.client.PutObjectRequest(putInput)
url, err := req.Presign(5 * time.Minute)
if err != nil {
return errors.InternalServerError(method, "Error creating upload URL")
}
response.Url = url
// store the metadata for easy retrieval for listing
if err := store.Write(store.NewRecord(
fmt.Sprintf("%s/%s", prefixByUser, objectName),
meta{
Visibility: request.Visibility,
CreateTime: *createTime,
ModifiedTime: time.Now().Format(time.RFC3339Nano),
})); err != nil {
log.Errorf("Error writing object to store %s", err)
return errors.InternalServerError(method, "Error creating upload URL")
}
return nil
}

54
space/handler/space_test.go
View File

@@ -105,8 +105,6 @@ func TestCreate(t *testing.T) {
put: func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) {
g.Expect(*input.Bucket).To(Equal("my-space"))
g.Expect(input.ACL).To(BeNil())
g.Expect(*input.Metadata[mdVisibility]).To(Equal(visibilityPrivate))
g.Expect(input.Metadata[mdCreated]).To(Not(BeNil()))
return &sthree.PutObjectOutput{}, nil
},
},
@@ -121,8 +119,6 @@ func TestCreate(t *testing.T) {
put: func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) {
g.Expect(*input.Bucket).To(Equal("my-space"))
g.Expect(*input.ACL).To(Equal(mdACLPublic))
g.Expect(*input.Metadata[mdVisibility]).To(Equal(visibilityPublic))
g.Expect(input.Metadata[mdCreated]).To(Not(BeNil()))
return &sthree.PutObjectOutput{}, nil
},
},
@@ -206,8 +202,6 @@ func TestUpdate(t *testing.T) {
put: func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) {
g.Expect(*input.Bucket).To(Equal("my-space"))
g.Expect(input.ACL).To(BeNil())
g.Expect(*input.Metadata[mdVisibility]).To(Equal(visibilityPrivate))
g.Expect(input.Metadata[mdCreated]).To(Not(BeNil()))
return &sthree.PutObjectOutput{}, nil
},
},
@@ -222,8 +216,6 @@ func TestUpdate(t *testing.T) {
put: func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) {
g.Expect(*input.Bucket).To(Equal("my-space"))
g.Expect(*input.ACL).To(Equal(mdACLPublic))
g.Expect(*input.Metadata[mdVisibility]).To(Equal(visibilityPublic))
g.Expect(input.Metadata[mdCreated]).To(Not(BeNil()))
return &sthree.PutObjectOutput{}, nil
},
},
@@ -238,19 +230,11 @@ func TestUpdate(t *testing.T) {
head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) {
g.Expect(*input.Bucket).To(Equal("my-space"))
g.Expect(*input.Key).To(Equal("micro/123/foo.jpg"))
return &sthree.HeadObjectOutput{
Metadata: map[string]*string{
mdCreated: aws.String("1638541918"),
mdVisibility: aws.String(visibilityPrivate),
},
}, nil
return &sthree.HeadObjectOutput{}, nil
},
put: func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) {
g.Expect(*input.Bucket).To(Equal("my-space"))
g.Expect(input.ACL).To(BeNil())
g.Expect(*input.Metadata[mdVisibility]).To(Equal(visibilityPrivate))
// created shouuld be copied from the previous
g.Expect(*input.Metadata[mdCreated]).To(Equal("1638541918"))
return &sthree.PutObjectOutput{}, nil
},
url: "",
@@ -261,19 +245,11 @@ func TestUpdate(t *testing.T) {
head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) {
g.Expect(*input.Bucket).To(Equal("my-space"))
g.Expect(*input.Key).To(Equal("micro/123/foo.jpg"))
return &sthree.HeadObjectOutput{
Metadata: map[string]*string{
mdCreated: aws.String("1638541918"),
mdVisibility: aws.String(visibilityPrivate),
},
}, nil
return &sthree.HeadObjectOutput{}, nil
},
put: func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) {
g.Expect(*input.Bucket).To(Equal("my-space"))
g.Expect(*input.ACL).To(Equal(mdACLPublic))
g.Expect(*input.Metadata[mdVisibility]).To(Equal(visibilityPublic))
// created shouuld be copied from the previous
g.Expect(*input.Metadata[mdCreated]).To(Equal("1638541918"))
return &sthree.PutObjectOutput{}, nil
},
url: "https://my-space.ams3.example.com/micro/123/foo.jpg",
@@ -404,10 +380,10 @@ func TestList(t *testing.T) {
name: "Empty prefix",
},
}
store.DefaultStore = memory.NewStore()
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
store.DefaultStore = memory.NewStore()
store.Write(
store.NewRecord(fmt.Sprintf("%s/micro/123/file.jpg", prefixByUser),
meta{
@@ -444,7 +420,7 @@ func TestList(t *testing.T) {
},
{
Key: aws.String("micro/123/file2.jpg"),
LastModified: aws.Time(time.Unix(1257894000, 0)),
LastModified: aws.Time(time.Unix(1257894001, 0)),
},
},
}, nil
@@ -469,9 +445,16 @@ func TestList(t *testing.T) {
g.Expect(err).To(BeNil())
g.Expect(rsp.Objects).To(HaveLen(2))
g.Expect(rsp.Objects[0].Name).To(Equal("file.jpg"))
g.Expect(rsp.Objects[0].Visibility).To(Equal("public"))
g.Expect(rsp.Objects[0].Created).To(Equal("2009-11-10T23:00:00Z"))
g.Expect(rsp.Objects[0].Modified).To(Equal("2009-11-10T23:00:00Z"))
g.Expect(rsp.Objects[0].Url).To(Equal("https://my-space.ams3.example.com/micro/123/file.jpg"))
g.Expect(rsp.Objects[1].Name).To(Equal("file2.jpg"))
g.Expect(rsp.Objects[1].Url).To(Equal(""))
g.Expect(rsp.Objects[1].Visibility).To(Equal("private"))
g.Expect(rsp.Objects[1].Created).To(Equal("2009-11-10T23:00:01Z"))
g.Expect(rsp.Objects[1].Modified).To(Equal("2009-11-10T23:00:01Z"))
}
})
@@ -504,10 +487,6 @@ func TestHead(t *testing.T) {
return &sthree.HeadObjectOutput{
LastModified: aws.Time(time.Unix(1257894000, 0)),
Metadata: map[string]*string{
mdCreated: aws.String("1257894000"),
mdVisibility: aws.String(visibilityPublic),
},
}, nil
},
},
@@ -524,10 +503,6 @@ func TestHead(t *testing.T) {
return &sthree.HeadObjectOutput{
LastModified: aws.Time(time.Unix(1257894000, 0)),
Metadata: map[string]*string{
mdCreated: aws.String("2009-11-10T23:00:00Z"),
mdVisibility: aws.String("private"),
},
}, nil
},
},
@@ -544,10 +519,15 @@ func TestHead(t *testing.T) {
},
},
}
store.DefaultStore = memory.NewStore()
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
store.DefaultStore = memory.NewStore()
store.Write(store.NewRecord(fmt.Sprintf("%s/micro/123/%s", prefixByUser, tc.objectName), meta{
Visibility: tc.visibility,
CreateTime: tc.created,
ModifiedTime: tc.modified,
}))
handler := Space{
conf: conf{
AccessKey: "access",