diff --git a/app/README.md b/app/README.md index 5fac41c..03aefff 100644 --- a/app/README.md +++ b/app/README.md @@ -1,7 +1,7 @@ -Global app deployment +Serverless app deployment # App Service -Deploy apps and services quickly and easily from a source url. +Deploy serverless apps and services quickly and easily from a source url. Run 10 apps for free. Pay to reserve more instances beyond it. diff --git a/app/handler/google.go b/app/handler/google.go index e47576f..1c041e0 100644 --- a/app/handler/google.go +++ b/app/handler/google.go @@ -19,6 +19,8 @@ import ( "github.com/micro/micro/v3/service/store" "github.com/micro/services/app/domain" pb "github.com/micro/services/app/proto" + pauth "github.com/micro/services/pkg/auth" + adminpb "github.com/micro/services/pkg/service/proto" "github.com/micro/services/pkg/tenant" "github.com/teris-io/shortid" ) @@ -555,7 +557,6 @@ func (e *GoogleApp) Delete(ctx context.Context, req *pb.DeleteRequest, rsp *pb.D if !ok { id = "micro" } - // read the app for the owner key := OwnerKey + id + "/" + req.Name recs, err := store.Read(key) @@ -575,11 +576,16 @@ func (e *GoogleApp) Delete(ctx context.Context, req *pb.DeleteRequest, rsp *pb.D return err } + return e.deleteApp(ctx, id, srv) +} + +func (e *GoogleApp) deleteApp(ctx context.Context, tenantID string, srv *pb.Service) error { + // check the status switch srv.Status { case domain.StatusUpdating, domain.StatusDeploying, domain.StatusDeleting: - log.Errorf("Won't delete: % is %s", req.Name, srv.Status) - return errors.BadRequest("app.delete", "% status: %s", req.Name, srv.Status) + log.Errorf("Won't delete: % is %s", srv.Name, srv.Status) + return errors.BadRequest("app.delete", "% status: %s", srv.Name, srv.Status) } // delete from the db @@ -587,7 +593,7 @@ func (e *GoogleApp) Delete(ctx context.Context, req *pb.DeleteRequest, rsp *pb.D // service key ServiceKey + srv.Id, // owner key - OwnerKey + id + "/" + req.Name, + OwnerKey + tenantID + "/" + srv.Name, } // set the delete status @@ -623,6 +629,35 @@ func (e *GoogleApp) Delete(ctx context.Context, req *pb.DeleteRequest, rsp *pb.D return nil } +func (e *GoogleApp) DeleteData(ctx context.Context, request *adminpb.DeleteDataRequest, response *adminpb.DeleteDataResponse) error { + method := "admin.DeleteData" + _, err := pauth.VerifyMicroAdmin(ctx, method) + if err != nil { + return err + } + + if len(request.TenantId) < 10 { // deliberate length check so we don't delete all the things + return errors.BadRequest(method, "Missing tenant ID") + } + + prefix := OwnerKey + request.TenantId + "/" + + recs, err := store.Read(prefix, store.ReadPrefix()) + if err != nil { + return err + } + + for _, rec := range recs { + var srv pb.Service + if err := rec.Decode(&srv); err != nil { + return err + } + e.deleteApp(ctx, request.TenantId, &srv) + } + log.Infof("Deleted %d functions for %s", len(recs), request.TenantId) + return nil +} + func (e *GoogleApp) List(ctx context.Context, req *pb.ListRequest, rsp *pb.ListResponse) error { log.Info("Received App.List request") diff --git a/app/main.go b/app/main.go index a689ba3..47cd66b 100644 --- a/app/main.go +++ b/app/main.go @@ -5,6 +5,7 @@ import ( "github.com/micro/micro/v3/service/logger" "github.com/micro/services/app/handler" pb "github.com/micro/services/app/proto" + admin "github.com/micro/services/pkg/service/proto" ) func main() { @@ -14,8 +15,10 @@ func main() { service.Version("latest"), ) + h := handler.New() // Register handler - pb.RegisterAppHandler(srv.Server(), handler.New()) + pb.RegisterAppHandler(srv.Server(), h) + admin.RegisterAdminHandler(srv.Server(), h) // Run service if err := srv.Run(); err != nil { diff --git a/cache/README.md b/cache/README.md index 4ef2033..6425c74 100644 --- a/cache/README.md +++ b/cache/README.md @@ -1,4 +1,4 @@ -Quick access key-value storage +Fast access key-value storage # Cache Service diff --git a/cache/handler/cache.go b/cache/handler/cache.go index 2f7f5d8..2ab6fc4 100644 --- a/cache/handler/cache.go +++ b/cache/handler/cache.go @@ -144,7 +144,7 @@ func (c *Cache) DeleteData(ctx context.Context, request *adminpb.DeleteDataReque return err } - if len(request.TenantId) == 0 { + if len(request.TenantId) < 10 { // deliberate length check so we don't delete all the things return errors.BadRequest(method, "Missing tenant ID") } diff --git a/contact/handler/contact.go b/contact/handler/contact.go index 2f2c7a5..d5c960c 100644 --- a/contact/handler/contact.go +++ b/contact/handler/contact.go @@ -137,7 +137,7 @@ func (c *contact) DeleteData(ctx context.Context, request *adminpb.DeleteDataReq return err } - if len(request.TenantId) == 0 { + if len(request.TenantId) < 10 { // deliberate length check so we don't delete all the things return errors.BadRequest(method, "Missing tenant ID") } diff --git a/db/README.md b/db/README.md index a10364f..6931d98 100644 --- a/db/README.md +++ b/db/README.md @@ -1,6 +1,6 @@ -Simple database service +Serverless postgres database # DB Service -The DB service is an easy to use database which provides persistent storage via a CRUD interface. It includes feature rich querying and JSON based formatted records for native use in Node.js and or any language. Powering the backend of your apps from anywhere in the world. +Our Database service is an easy to use serverless postgres database which provides persistent storage via a CRUD interface. It includes feature rich querying and JSON based formatted records for native use in Node.js and or any language. All data is stored securely on the M3O platform and can be accessed via [M3O Cloud](https://cloud.m3o.com). diff --git a/db/handler/db.go b/db/handler/db.go index d582286..301923e 100644 --- a/db/handler/db.go +++ b/db/handler/db.go @@ -451,7 +451,7 @@ func (e *Db) DeleteData(ctx context.Context, request *adminpb.DeleteDataRequest, return err } - if len(request.TenantId) == 0 { + if len(request.TenantId) < 10 { // deliberate length check so we don't delete all the things return errors.BadRequest(method, "Missing tenant ID") } diff --git a/db/publicapi.json b/db/publicapi.json index 5d90a68..d6a4077 100644 --- a/db/publicapi.json +++ b/db/publicapi.json @@ -2,5 +2,5 @@ "name": "db", "icon": "πŸ“¦", "category": "storage", - "display_name": "DB" + "display_name": "Database" } diff --git a/file/handler/files.go b/file/handler/files.go index e7c5792..857b326 100644 --- a/file/handler/files.go +++ b/file/handler/files.go @@ -10,6 +10,8 @@ import ( log "github.com/micro/micro/v3/service/logger" "github.com/micro/micro/v3/service/store" file "github.com/micro/services/file/proto" + pauth "github.com/micro/services/pkg/auth" + adminpb "github.com/micro/services/pkg/service/proto" "github.com/micro/services/pkg/tenant" ) @@ -150,3 +152,32 @@ func (e *File) List(ctx context.Context, req *file.ListRequest, rsp *file.ListRe return nil } + +func (e *File) DeleteData(ctx context.Context, request *adminpb.DeleteDataRequest, response *adminpb.DeleteDataResponse) error { + method := "admin.DeleteData" + _, err := pauth.VerifyMicroAdmin(ctx, method) + if err != nil { + return err + } + + if len(request.TenantId) < 10 { // deliberate length check so we don't delete all the things + return errors.BadRequest(method, "Missing tenant ID") + } + + path := filepath.Join("file", request.TenantId) + + // read all the files for the project + records, err := store.List(store.ListPrefix(path)) + if err != nil { + return err + } + + for _, file := range records { + if err := store.Delete(file); err != nil { + return err + } + } + log.Infof("Deleted %d records for %s", len(records), request.TenantId) + + return nil +} diff --git a/file/main.go b/file/main.go index b7d6144..f310a36 100644 --- a/file/main.go +++ b/file/main.go @@ -3,6 +3,7 @@ package main import ( "github.com/micro/services/file/handler" pb "github.com/micro/services/file/proto" + admin "github.com/micro/services/pkg/service/proto" "github.com/micro/services/pkg/tracing" "github.com/micro/micro/v3/service" @@ -16,8 +17,10 @@ func main() { service.Version("latest"), ) + h := handler.NewFile() // Register handler - pb.RegisterFileHandler(srv.Server(), handler.NewFile()) + pb.RegisterFileHandler(srv.Server(), h) + admin.RegisterAdminHandler(srv.Server(), h) traceCloser := tracing.SetupOpentracing("file") defer traceCloser.Close() diff --git a/function/README.md b/function/README.md index b962ed2..996030f 100644 --- a/function/README.md +++ b/function/README.md @@ -1,4 +1,4 @@ -Serverless compute as a service +Serverless functions # Function Service diff --git a/function/handler/google.go b/function/handler/google.go index befbc7f..e749c49 100644 --- a/function/handler/google.go +++ b/function/handler/google.go @@ -19,6 +19,8 @@ import ( "github.com/micro/micro/v3/service/runtime/source/git" "github.com/micro/micro/v3/service/store" function "github.com/micro/services/function/proto" + pauth "github.com/micro/services/pkg/auth" + adminpb "github.com/micro/services/pkg/service/proto" "github.com/micro/services/pkg/tenant" "github.com/teris-io/shortid" ) @@ -626,24 +628,26 @@ func (e *GoogleFunction) Delete(ctx context.Context, req *function.DeleteRequest } // async delete - go func() { - cmd := exec.Command("gcloud", "functions", "delete", "--quiet", "--project", e.project, "--region", fn.Region, fn.Id) - outp, err := cmd.CombinedOutput() - if err != nil && !strings.Contains(string(outp), "does not exist") { - log.Error(fmt.Errorf(string(outp))) - return - } - - // delete the owner key - store.Delete(key) - - // delete the global key - store.Delete(FunctionKey + fn.Id) - }() + go e.deleteFunction(fn, key) return nil } +func (e *GoogleFunction) deleteFunction(fn *function.Func, key string) { + cmd := exec.Command("gcloud", "functions", "delete", "--quiet", "--project", e.project, "--region", fn.Region, fn.Id) + outp, err := cmd.CombinedOutput() + if err != nil && !strings.Contains(string(outp), "does not exist") { + log.Error(fmt.Errorf(string(outp))) + return + } + + // delete the owner key + store.Delete(key) + + // delete the global key + store.Delete(FunctionKey + fn.Id) +} + func (e *GoogleFunction) List(ctx context.Context, req *function.ListRequest, rsp *function.ListResponse) error { log.Info("Received Function.List request") @@ -797,3 +801,32 @@ func (e *GoogleFunction) Regions(ctx context.Context, req *function.RegionsReque rsp.Regions = GoogleRegions return nil } + +func (e *GoogleFunction) DeleteData(ctx context.Context, request *adminpb.DeleteDataRequest, response *adminpb.DeleteDataResponse) error { + method := "admin.DeleteData" + _, err := pauth.VerifyMicroAdmin(ctx, method) + if err != nil { + return err + } + + if len(request.TenantId) < 10 { // deliberate length check so we don't delete all the things + return errors.BadRequest(method, "Missing tenant ID") + } + + prefix := OwnerKey + request.TenantId + "/" + + recs, err := store.Read(prefix, store.ReadPrefix()) + if err != nil { + return err + } + + for _, rec := range recs { + var fn function.Func + if err := rec.Decode(&fn); err != nil { + return err + } + e.deleteFunction(&fn, rec.Key) + } + log.Infof("Deleted %d functions for %s", len(recs), request.TenantId) + return nil +} diff --git a/function/main.go b/function/main.go index 504661e..0a0f2ec 100644 --- a/function/main.go +++ b/function/main.go @@ -5,6 +5,7 @@ import ( "github.com/micro/micro/v3/service/logger" "github.com/micro/services/function/handler" pb "github.com/micro/services/function/proto" + admin "github.com/micro/services/pkg/service/proto" ) func main() { @@ -14,8 +15,10 @@ func main() { service.Version("latest"), ) + h := handler.NewFunction() // Register handler - pb.RegisterFunctionHandler(srv.Server(), handler.NewFunction()) + pb.RegisterFunctionHandler(srv.Server(), h) + admin.RegisterAdminHandler(srv.Server(), h) // Run service if err := srv.Run(); err != nil { diff --git a/go.mod b/go.mod index 47574dc..fbfe184 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/m3o/goduckgo v0.0.0-20210630141545-c760fe67b945 github.com/mattheath/base62 v0.0.0-20150408093626-b80cdc656a7a // indirect github.com/mattheath/kala v0.0.0-20171219141654-d6276794bf0e - github.com/micro/micro/v3 v3.9.0 + github.com/micro/micro/v3 v3.9.1-0.20220203152611-b23544122058 github.com/miekg/dns v1.1.31 // indirect github.com/minio/minio-go/v7 v7.0.16 github.com/o1egl/govatar v0.3.0 @@ -52,13 +52,12 @@ require ( github.com/teamwork/test v0.0.0-20200108114543-02621bae84ad // indirect github.com/teamwork/utils v0.0.0-20211103135549-f7e7a68ba696 // indirect github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf - github.com/tidwall/gjson v1.12.1 + github.com/tidwall/pretty v1.2.0 // indirect github.com/tkuchiki/go-timezone v0.2.2 github.com/ttacon/builder v0.0.0-20170518171403-c099f663e1c2 // indirect github.com/ttacon/libphonenumber v1.2.1 // indirect go.mongodb.org/mongo-driver v1.7.2 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 - golang.org/x/net v0.0.0-20210614182718-04defd469f4e golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 golang.org/x/text v0.3.6 google.golang.org/api v0.59.0 diff --git a/go.sum b/go.sum index d74626d..338f1df 100644 --- a/go.sum +++ b/go.sum @@ -103,6 +103,7 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/caddyserver/certmagic v0.10.6 h1:sCya6FmfaN74oZE46kqfaFOVoROD/mF36rTQfjN7TZc= github.com/caddyserver/certmagic v0.10.6/go.mod h1:Y8jcUBctgk/IhpAzlHKfimZNyXCkfGgRTC0orl8gROQ= github.com/cdipaolo/goml v0.0.0-20190412180403-e1f51f713598 h1:j2XRGH5Y5uWtBYXGwmrjKeM/kfu/jh7ZcnrGvyN5Ttk= github.com/cdipaolo/goml v0.0.0-20190412180403-e1f51f713598/go.mod h1:sduMkaHcXDIWurl/Bd/z0rNEUHw5tr6LUA9IO8E9o0o= @@ -110,15 +111,18 @@ github.com/cdipaolo/sentiment v0.0.0-20200617002423-c697f64e7f10 h1:6dGQY3apkf7l github.com/cdipaolo/sentiment v0.0.0-20200617002423-c697f64e7f10/go.mod h1:JWoVf4GJxCxM3iCiZSVoXNMV+JFG49L+ou70KK3HTvQ= github.com/cenkalti/backoff v2.1.1+incompatible h1:tKJnvO2kl0zmb/jA5UKAt4VoEVw1qxKWjE/Bpp46npY= github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff/v4 v4.0.0 h1:6VeaLF9aI+MAUQ95106HwWzYZgJJpZ4stumjj6RFYAU= github.com/cenkalti/backoff/v4 v4.0.0/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2/go.mod h1:qhVI5MKwBGhdNU89ZRz2plgYutcJ5PCekLxXn56w6SY= +github.com/cloudflare/cloudflare-go v0.10.9 h1:d8KOgLpYiC+Xq3T4tuO+/goM+RZvuO+T4pojuv8giL8= github.com/cloudflare/cloudflare-go v0.10.9/go.mod h1:5TrsWH+3f4NV6WjtS5QFp+DifH81rph40gU374Sh0dQ= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -153,6 +157,7 @@ github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1 github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/dnaeon/go-vcr v0.0.0-20180814043457-aafff18a5cc2/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/dnsimple/dnsimple-go v0.30.0/go.mod h1:O5TJ0/U6r7AfT8niYNlmohpLbCSG+c71tQlGr9SeGrg= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= @@ -170,10 +175,12 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/evanphx/json-patch/v5 v5.0.0 h1:dKTrUeykyQwKb/kx7Z+4ukDs6l+4L41HqG1XHnhX7WE= github.com/evanphx/json-patch/v5 v5.0.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/exoscale/egoscale v0.18.1/go.mod h1:Z7OOdzzTOz1Q1PjQXumlz9Wn/CddH0zSYdCF3rnBKXE= +github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -318,8 +325,10 @@ github.com/gophercloud/gophercloud v0.3.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEo github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -441,6 +450,7 @@ github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4= github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= github.com/kolo/xmlrpc v0.0.0-20190717152603-07c4ee3fd181/go.mod h1:o03bZfuBwAXHetKXuInt4S7omeXUu62/A845kiycsSQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -488,13 +498,14 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ= github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= github.com/mattn/go-tty v0.0.0-20180219170247-931426f7535a/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/micro/micro/v3 v3.9.0 h1:5KsDuWQArTw2iQWv3B/o9EoMoFXSqkPxIcpkGqAcSVQ= -github.com/micro/micro/v3 v3.9.0/go.mod h1:fNx55Nv50LZMgd5Prytro1ZNwXphvVID78R6KJ+xLbQ= +github.com/micro/micro/v3 v3.9.1-0.20220203152611-b23544122058 h1:xsjuL35jhBpJk0PZGrya/gYm/uQzNWvqzFJdUpWD6lM= +github.com/micro/micro/v3 v3.9.1-0.20220203152611-b23544122058/go.mod h1:fNx55Nv50LZMgd5Prytro1ZNwXphvVID78R6KJ+xLbQ= github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo= @@ -529,6 +540,7 @@ github.com/o1egl/govatar v0.3.0 h1:hGDsiJJs6qgQ6Ea4JiaukRsUKTY2Ai4dgMEdsYvlUa0= github.com/o1egl/govatar v0.3.0/go.mod h1:YeDGDII+2Ji1RcBKvb1KqaPhk4PmuZyBq+rPYc6b+cQ= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.3/go.mod h1:YZeBtGzYYEsCHp2LST/u/0NDwGkRoBtmn1cIWCJiS6M= +github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -608,6 +620,7 @@ github.com/sendgrid/rest v2.6.4+incompatible h1:lq6gAQxLwVBf3mVyCCSHI6mgF+NfaJFJ github.com/sendgrid/rest v2.6.4+incompatible/go.mod h1:kXX7q3jZtJXK5c5qK83bSGMdV6tsOE70KbHoqJls4lE= github.com/sendgrid/sendgrid-go v3.10.0+incompatible h1:aSYyurHxEZSDy7kxhvZ4fH0inNkEEmRssZNbAmETR2c= github.com/sendgrid/sendgrid-go v3.10.0+incompatible/go.mod h1:QRQt+LX/NmgVEvmdRw0VT/QgUn499+iza2FnDca9fg8= +github.com/serenize/snaker v0.0.0-20171204205717-a683aaf2d516 h1:ofR1ZdrNSkiWcMsRrubK9tb2/SlZVWttAfqUjJi6QYc= github.com/serenize/snaker v0.0.0-20171204205717-a683aaf2d516/go.mod h1:Yow6lPLSAXx2ifx470yD/nUe22Dv5vBvxK/UK9UUTVs= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= @@ -650,10 +663,6 @@ github.com/teamwork/utils v0.0.0-20211103135549-f7e7a68ba696 h1:G6Hc6/KxUmHxZsCg github.com/teamwork/utils v0.0.0-20211103135549-f7e7a68ba696/go.mod h1:3Fn0qxFeRNpvsg/9T1+btOOOKkd1qG2nPYKKcOmNpcs= github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf h1:Z2X3Os7oRzpdJ75iPqWZc0HeJWFYNCvKsfpQwFpRNTA= github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf/go.mod h1:M8agBzgqHIhgj7wEn9/0hJUZcrvt9VY+Ln+S1I5Mha0= -github.com/tidwall/gjson v1.12.1 h1:ikuZsLdhr8Ws0IdROXUS1Gi4v9Z4pGqpX/CvJkxvfpo= -github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= -github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -688,6 +697,7 @@ github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= +github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= @@ -942,6 +952,7 @@ golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac h1:oN6lz7iLW/YC7un8pq+9bOLyXrprv2+DKfkJY+2LJJw= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1208,6 +1219,7 @@ gopkg.in/ns1/ns1-go.v2 v2.0.0-20190730140822-b51389932cbc/go.mod h1:VV+3haRsgDiV gopkg.in/resty.v1 v1.9.1/go.mod h1:vo52Hzryw9PnPHcJfPsBiFW62XhNx5OczbV9y+IMpgc= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.4.1 h1:H0TmLt7/KmzlrDOpa1F+zr0Tk90PbJYBfsVUmRLrf9Y= gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= diff --git a/id/publicapi.json b/id/publicapi.json index 9e70fd9..5838592 100644 --- a/id/publicapi.json +++ b/id/publicapi.json @@ -2,5 +2,5 @@ "name": "id", "icon": "πŸ†”", "category": "utility", - "display_name": "ID Generator" + "display_name": "IDgen" } diff --git a/image/handler/image.go b/image/handler/image.go index 70a4540..2d3aea0 100644 --- a/image/handler/image.go +++ b/image/handler/image.go @@ -21,6 +21,8 @@ import ( "github.com/micro/micro/v3/service/logger" "github.com/micro/micro/v3/service/store" img "github.com/micro/services/image/proto" + pauth "github.com/micro/services/pkg/auth" + adminpb "github.com/micro/services/pkg/service/proto" "github.com/micro/services/pkg/tenant" ) @@ -320,3 +322,32 @@ func (e *Image) Delete(ctx context.Context, request *img.DeleteRequest, response } return nil } + +func (e *Image) DeleteData(ctx context.Context, request *adminpb.DeleteDataRequest, response *adminpb.DeleteDataResponse) error { + method := "admin.DeleteData" + _, err := pauth.VerifyMicroAdmin(ctx, method) + if err != nil { + return err + } + + if len(request.TenantId) < 10 { // deliberate length check so we don't delete all the things + return merrors.BadRequest(method, "Missing tenant ID") + } + + path := fmt.Sprintf("%v/%v", pathPrefix, request.TenantId) + keys, err := store.DefaultBlobStore.List(store.BlobListPrefix(path)) + if err != nil { + return err + } + + for _, key := range keys { + err = store.DefaultBlobStore.Delete(key) + if err != nil { + return err + } + } + + logger.Infof("Deleted %d keys for %s", len(keys), request.TenantId) + + return nil +} diff --git a/image/main.go b/image/main.go index fb170dc..7bb4781 100644 --- a/image/main.go +++ b/image/main.go @@ -3,6 +3,7 @@ package main import ( "github.com/micro/services/image/handler" pb "github.com/micro/services/image/proto" + admin "github.com/micro/services/pkg/service/proto" "github.com/micro/services/pkg/tracing" "github.com/micro/micro/v3/service" @@ -16,8 +17,10 @@ func main() { service.Version("latest"), ) + h := handler.NewImage() // Register handler - pb.RegisterImageHandler(srv.Server(), handler.NewImage()) + pb.RegisterImageHandler(srv.Server(), h) + admin.RegisterAdminHandler(srv.Server(), h) traceCloser := tracing.SetupOpentracing("image") defer traceCloser.Close() diff --git a/ip/publicapi.json b/ip/publicapi.json index e05e450..52bb403 100644 --- a/ip/publicapi.json +++ b/ip/publicapi.json @@ -2,5 +2,5 @@ "name": "ip", "icon": "πŸ—ΊοΈ", "category": "utility", - "display_name": "IP to Geo" + "display_name": "IP2Geo" } diff --git a/location/domain/domain.go b/location/domain/domain.go index d54e5c4..e36eb30 100644 --- a/location/domain/domain.go +++ b/location/domain/domain.go @@ -111,7 +111,6 @@ func Search(ctx context.Context, typ string, entity *Entity, radius float64, num // get the index index := getIndex(ctx) - points := index.KNearest(entity, numEntities, geo.Meters(radius), func(p geo.Point) bool { e, ok := p.(*Entity) if !ok || e.Type != typ { @@ -132,3 +131,11 @@ func Search(ctx context.Context, typ string, entity *Entity, radius float64, num return entities } + +func DeleteIndex(tenantID string) error { + mtx.Lock() + defer mtx.Unlock() + delete(indexes, tenantID) + + return nil +} diff --git a/location/handler/handler.go b/location/handler/handler.go index fd216e7..7ef70b9 100644 --- a/location/handler/handler.go +++ b/location/handler/handler.go @@ -6,9 +6,12 @@ import ( "github.com/micro/micro/v3/service" "github.com/micro/micro/v3/service/errors" + "github.com/micro/micro/v3/service/logger" "github.com/micro/services/location/domain" loc "github.com/micro/services/location/proto" "github.com/micro/services/location/subscriber" + pauth "github.com/micro/services/pkg/auth" + adminpb "github.com/micro/services/pkg/service/proto" ) type Location struct{} @@ -70,3 +73,21 @@ func (l *Location) Search(ctx context.Context, req *loc.SearchRequest, rsp *loc. return nil } + +func (l *Location) DeleteData(ctx context.Context, request *adminpb.DeleteDataRequest, response *adminpb.DeleteDataResponse) error { + method := "admin.DeleteData" + _, err := pauth.VerifyMicroAdmin(ctx, method) + if err != nil { + return err + } + + if len(request.TenantId) < 10 { // deliberate length check so we don't delete all the things + return errors.BadRequest(method, "Missing tenant ID") + } + + if err := domain.DeleteIndex(request.TenantId); err != nil { + return err + } + logger.Infof("Deleted index for %s", request.TenantId) + return nil +} diff --git a/location/main.go b/location/main.go index 3e3f494..b074b12 100644 --- a/location/main.go +++ b/location/main.go @@ -6,6 +6,7 @@ import ( "github.com/micro/micro/v3/service" "github.com/micro/services/location/handler" pb "github.com/micro/services/location/proto" + admin "github.com/micro/services/pkg/service/proto" "github.com/micro/services/pkg/tracing" ) @@ -14,7 +15,9 @@ func main() { service.Name("location"), ) - pb.RegisterLocationHandler(location.Server(), new(handler.Location)) + h := new(handler.Location) + pb.RegisterLocationHandler(location.Server(), h) + admin.RegisterAdminHandler(location.Server(), h) // TODO reinstate me //service.Subscribe(subscriber.Topic, new(subscriber.Location)) diff --git a/mq/README.md b/mq/README.md index d8a4383..6eaa793 100644 --- a/mq/README.md +++ b/mq/README.md @@ -1,4 +1,4 @@ -Simple in-memory message broker +Ephemeral pubsub messaging # MQ Service diff --git a/notes/handler/notes.go b/notes/handler/notes.go index 7c82f17..214a383 100644 --- a/notes/handler/notes.go +++ b/notes/handler/notes.go @@ -9,9 +9,12 @@ import ( "github.com/google/uuid" "github.com/micro/micro/v3/service/client" "github.com/micro/micro/v3/service/errors" + "github.com/micro/micro/v3/service/logger" "github.com/micro/micro/v3/service/store" streamPb "github.com/micro/services/mq/proto" pb "github.com/micro/services/notes/proto" + pauth "github.com/micro/services/pkg/auth" + adminpb "github.com/micro/services/pkg/service/proto" "github.com/micro/services/pkg/tenant" "google.golang.org/protobuf/types/known/structpb" ) @@ -292,3 +295,29 @@ func (h *Notes) List(ctx context.Context, req *pb.ListRequest, rsp *pb.ListRespo return nil } + +func (h *Notes) DeleteData(ctx context.Context, request *adminpb.DeleteDataRequest, response *adminpb.DeleteDataResponse) error { + method := "admin.DeleteData" + _, err := pauth.VerifyMicroAdmin(ctx, method) + if err != nil { + return err + } + + if len(request.TenantId) < 10 { // deliberate length check so we don't delete all the things + return errors.BadRequest(method, "Missing tenant ID") + } + + keys, err := store.List(store.ListPrefix(request.TenantId)) + if err != nil { + return err + } + + for _, k := range keys { + if err := store.Delete(k); err != nil { + return err + } + } + + logger.Infof("Deleted %d keys for %s", len(keys), request.TenantId) + return nil +} diff --git a/notes/main.go b/notes/main.go index 969a878..db4532e 100644 --- a/notes/main.go +++ b/notes/main.go @@ -5,6 +5,7 @@ import ( log "github.com/micro/micro/v3/service/logger" "github.com/micro/services/notes/handler" pb "github.com/micro/services/notes/proto" + admin "github.com/micro/services/pkg/service/proto" ) func main() { @@ -17,8 +18,10 @@ func main() { // Initialise service srv.Init() + h := handler.New(srv.Client()) // Register Handler - pb.RegisterNotesHandler(srv.Server(), handler.New(srv.Client())) + pb.RegisterNotesHandler(srv.Server(), h) + admin.RegisterAdminHandler(srv.Server(), h) // Run service if err := srv.Run(); err != nil { diff --git a/otp/handler/otp.go b/otp/handler/otp.go index c57b037..900ce06 100644 --- a/otp/handler/otp.go +++ b/otp/handler/otp.go @@ -2,12 +2,16 @@ package handler import ( "context" + "strings" "time" "github.com/micro/micro/v3/service/errors" "github.com/micro/micro/v3/service/logger" pb "github.com/micro/services/otp/proto" + pauth "github.com/micro/services/pkg/auth" "github.com/micro/services/pkg/cache" + adminpb "github.com/micro/services/pkg/service/proto" + "github.com/micro/services/pkg/tenant" "github.com/pquerna/otp" "github.com/pquerna/otp/totp" @@ -121,3 +125,30 @@ func (e *Otp) Validate(ctx context.Context, req *pb.ValidateRequest, rsp *pb.Val return nil } + +func (e *Otp) DeleteData(ctx context.Context, request *adminpb.DeleteDataRequest, response *adminpb.DeleteDataResponse) error { + method := "admin.DeleteData" + _, err := pauth.VerifyMicroAdmin(ctx, method) + if err != nil { + return err + } + + if len(request.TenantId) < 10 { // deliberate length check so we don't delete all the things + return errors.BadRequest(method, "Missing tenant ID") + } + + split := strings.Split(request.TenantId, "/") + tctx := tenant.NewContext(split[1], split[0], split[1]) + keys, err := cache.Context(tctx).ListKeys() + if err != nil { + return err + } + + for _, k := range keys { + if err := cache.Context(tctx).Delete(k); err != nil { + return err + } + } + logger.Infof("Deleted %d keys for %s", len(keys), request.TenantId) + return nil +} diff --git a/otp/main.go b/otp/main.go index 748b051..80f3f76 100644 --- a/otp/main.go +++ b/otp/main.go @@ -3,6 +3,7 @@ package main import ( "github.com/micro/services/otp/handler" pb "github.com/micro/services/otp/proto" + admin "github.com/micro/services/pkg/service/proto" "github.com/micro/services/pkg/tracing" "github.com/micro/micro/v3/service" @@ -15,8 +16,10 @@ func main() { service.Name("otp"), ) + h := new(handler.Otp) // Register handler - pb.RegisterOtpHandler(srv.Server(), new(handler.Otp)) + pb.RegisterOtpHandler(srv.Server(), h) + admin.RegisterAdminHandler(srv.Server(), h) traceCloser := tracing.SetupOpentracing("otp") defer traceCloser.Close() diff --git a/qr/handler/qr.go b/qr/handler/qr.go index 26462bd..01224a6 100644 --- a/qr/handler/qr.go +++ b/qr/handler/qr.go @@ -12,6 +12,8 @@ import ( "github.com/micro/micro/v3/service/errors" log "github.com/micro/micro/v3/service/logger" "github.com/micro/micro/v3/service/store" + pauth "github.com/micro/services/pkg/auth" + adminpb "github.com/micro/services/pkg/service/proto" "github.com/micro/services/pkg/tenant" qr "github.com/micro/services/qr/proto" "github.com/skip2/go-qrcode" @@ -44,6 +46,10 @@ func New() *Qr { return &Qr{cdnPrefix: pref} } +func namespacePrefix(tenantID string) string { + return "micro/qr/" + tenantID +} + func (q *Qr) Generate(ctx context.Context, request *qr.GenerateRequest, response *qr.GenerateResponse) error { if len(request.Text) == 0 { return errors.BadRequest("qr.generate", "Missing parameter text") @@ -63,7 +69,7 @@ func (q *Qr) Generate(ctx context.Context, request *qr.GenerateRequest, response return errors.InternalServerError("qr.generate", "Error while generating QR code") } - nsPrefix := "micro/qr/" + ten + nsPrefix := namespacePrefix(ten) fileName := fmt.Sprintf("%s.png", uuid.New().String()) if err := store.DefaultBlobStore.Write( fileName, bytes.NewBuffer(qrc), @@ -90,3 +96,42 @@ func (q *Qr) Generate(ctx context.Context, request *qr.GenerateRequest, response response.Qr = fmt.Sprintf("%s/%s/%s", q.cdnPrefix, nsPrefix, rec.Filename) return nil } + +func (q *Qr) DeleteData(ctx context.Context, request *adminpb.DeleteDataRequest, response *adminpb.DeleteDataResponse) error { + method := "admin.DeleteData" + _, err := pauth.VerifyMicroAdmin(ctx, method) + if err != nil { + return err + } + + if len(request.TenantId) < 10 { // deliberate length check so we don't delete all the things + return errors.BadRequest(method, "Missing tenant ID") + } + ns := namespacePrefix(request.TenantId) + keys, err := store.DefaultBlobStore.List(store.BlobListNamespace(ns)) + if err != nil { + return err + } + + for _, key := range keys { + err = store.DefaultBlobStore.Delete(key, store.BlobNamespace(ns)) + if err != nil { + return err + } + } + log.Infof("Deleted %d objects from S3 for %s", len(keys), request.TenantId) + + keys, err = store.List(store.ListPrefix(fmt.Sprintf("%s/%s/", prefixByTenant, ns))) + if err != nil { + return err + } + for _, key := range keys { + if err := store.Delete(key); err != nil { + return err + } + } + + log.Infof("Deleted %d objects from store for %s", len(keys), request.TenantId) + + return nil +} diff --git a/qr/main.go b/qr/main.go index 9ea87ca..bc4ed73 100644 --- a/qr/main.go +++ b/qr/main.go @@ -1,6 +1,7 @@ package main import ( + admin "github.com/micro/services/pkg/service/proto" "github.com/micro/services/pkg/tracing" "github.com/micro/services/qr/handler" pb "github.com/micro/services/qr/proto" @@ -16,8 +17,10 @@ func main() { service.Version("latest"), ) + h := handler.New() // Register handler - pb.RegisterQrHandler(srv.Server(), handler.New()) + pb.RegisterQrHandler(srv.Server(), h) + admin.RegisterAdminHandler(srv.Server(), h) traceCloser := tracing.SetupOpentracing("qr") defer traceCloser.Close() diff --git a/rss/handler/crawl.go b/rss/handler/crawl.go index 1a7e632..588e0a9 100644 --- a/rss/handler/crawl.go +++ b/rss/handler/crawl.go @@ -51,6 +51,7 @@ func (e *crawl) FetchAll() { return } + currList := map[string]bool{} for _, v := range records { feed := pb.Feed{} if err := json.Unmarshal(v.Value, &feed); err != nil { @@ -62,6 +63,22 @@ func (e *crawl) FetchAll() { if err != nil { log.Errorf("Error saving post: %v", err) } + currList[feed.Url] = true + } + + // prune anything that has been deleted + rssSync.Lock() + defer rssSync.Unlock() + for url, _ := range rssFeeds { + if currList[url] { + continue + } + // this isn't in the current list. delete from store any entries + keys, _ := store.List(store.ListPrefix(generateEntryKey(url, ""))) + for _, k := range keys { + store.Delete(k) + } + delete(rssFeeds, url) } } diff --git a/rss/handler/rss.go b/rss/handler/rss.go index ed97237..1a9c387 100644 --- a/rss/handler/rss.go +++ b/rss/handler/rss.go @@ -5,10 +5,13 @@ import ( "encoding/json" "fmt" "hash/fnv" + "strings" "github.com/micro/micro/v3/service/errors" log "github.com/micro/micro/v3/service/logger" "github.com/micro/micro/v3/service/store" + pauth "github.com/micro/services/pkg/auth" + adminpb "github.com/micro/services/pkg/service/proto" "github.com/micro/services/pkg/tenant" pb "github.com/micro/services/rss/proto" @@ -175,3 +178,32 @@ func (e *Rss) Remove(ctx context.Context, req *pb.RemoveRequest, rsp *pb.RemoveR return e.store.Delete(generateFeedKey(ctx, req.Name)) } + +func (e *Rss) DeleteData(ctx context.Context, request *adminpb.DeleteDataRequest, response *adminpb.DeleteDataResponse) error { + method := "admin.DeleteData" + _, err := pauth.VerifyMicroAdmin(ctx, method) + if err != nil { + return err + } + + if len(request.TenantId) < 10 { // deliberate length check so we don't delete all the things + return errors.BadRequest(method, "Missing tenant ID") + } + split := strings.Split(request.TenantId, "/") + tctx := tenant.NewContext(split[1], split[0], split[1]) + + prefix := generateFeedKey(tctx, "") + records, err := e.store.Read(prefix, store.ReadPrefix()) + if err != nil { + return err + } + + for _, val := range records { + if err := e.store.Delete(val.Key); err != nil { + return err + } + } + log.Infof("Delete %d records for %s", len(records), request.TenantId) + return nil + +} diff --git a/rss/main.go b/rss/main.go index 2390872..12215a1 100644 --- a/rss/main.go +++ b/rss/main.go @@ -6,6 +6,7 @@ import ( "github.com/micro/micro/v3/service" "github.com/micro/micro/v3/service/logger" "github.com/micro/micro/v3/service/store" + admin "github.com/micro/services/pkg/service/proto" "github.com/micro/services/pkg/tracing" "github.com/micro/services/rss/handler" @@ -33,6 +34,7 @@ func main() { // Register handler pb.RegisterRssHandler(srv.Server(), rss) + admin.RegisterAdminHandler(srv.Server(), rss) traceCloser := tracing.SetupOpentracing("rss") defer traceCloser.Close() diff --git a/search/handler/search.go b/search/handler/search.go index e108ed2..38aa90b 100644 --- a/search/handler/search.go +++ b/search/handler/search.go @@ -16,6 +16,8 @@ import ( "github.com/micro/micro/v3/service/config" "github.com/micro/micro/v3/service/errors" log "github.com/micro/micro/v3/service/logger" + pauth "github.com/micro/services/pkg/auth" + adminpb "github.com/micro/services/pkg/service/proto" "github.com/micro/services/pkg/tenant" pb "github.com/micro/services/search/proto" open "github.com/opensearch-project/opensearch-go" @@ -55,6 +57,10 @@ type hit struct { Source map[string]interface{} `json:"_source"` } +type catIndicesEntry struct { + Index string `json:"index"` +} + func New(srv *service.Service) *Search { v, err := config.Get("micro.search") if err != nil { @@ -280,8 +286,12 @@ func (s *Search) DeleteIndex(ctx context.Context, request *pb.DeleteIndexRequest if len(request.Index) == 0 { return errors.BadRequest(method, "Missing index param") } + return s.deleteIndices(ctx, []string{indexName(tnt, request.Index)}, method) +} + +func (s *Search) deleteIndices(ctx context.Context, indices []string, method string) error { req := openapi.IndicesDeleteRequest{ - Index: []string{indexName(tnt, request.Index)}, + Index: indices, } rsp, err := req.Do(ctx, s.client) if err != nil { @@ -293,5 +303,51 @@ func (s *Search) DeleteIndex(ctx context.Context, request *pb.DeleteIndexRequest log.Errorf("Error deleting index %s", rsp.String()) return errors.InternalServerError(method, "Error deleting index") } + log.Infof("Deleted indices: %v", indices) + return nil + +} + +func (s *Search) DeleteData(ctx context.Context, request *adminpb.DeleteDataRequest, response *adminpb.DeleteDataResponse) error { + method := "admin.DeleteData" + _, err := pauth.VerifyMicroAdmin(ctx, method) + if err != nil { + return err + } + + if len(request.TenantId) < 10 { // deliberate length check, don't want to unwittingly delete all the things + return errors.BadRequest(method, "Missing tenant ID") + } + req := openapi.CatIndicesRequest{ + Format: "json", + } + rsp, err := req.Do(ctx, s.client) + if err != nil { + return err + } + defer rsp.Body.Close() + if rsp.IsError() { + return err + } + b, err := ioutil.ReadAll(rsp.Body) + if err != nil { + return err + } + + var entries []catIndicesEntry + if err := json.Unmarshal(b, &entries); err != nil { + return err + } + toDelete := []string{} + for _, entry := range entries { + if !strings.HasPrefix(entry.Index, indexName(request.TenantId, "")) { + continue + } + toDelete = append(toDelete, entry.Index) + + } + if len(toDelete) > 0 { + return s.deleteIndices(ctx, toDelete, method) + } return nil } diff --git a/search/main.go b/search/main.go index f2ce90f..a8d862c 100644 --- a/search/main.go +++ b/search/main.go @@ -3,6 +3,7 @@ package main import ( "github.com/micro/micro/v3/service" "github.com/micro/micro/v3/service/logger" + admin "github.com/micro/services/pkg/service/proto" "github.com/micro/services/search/handler" pb "github.com/micro/services/search/proto" ) @@ -14,8 +15,10 @@ func main() { service.Version("latest"), ) + h := handler.New(srv) // Register handler - pb.RegisterSearchHandler(srv.Server(), handler.New(srv)) + pb.RegisterSearchHandler(srv.Server(), h) + admin.RegisterAdminHandler(srv.Server(), h) // Run service if err := srv.Run(); err != nil { diff --git a/sms/README.md b/sms/README.md index 1628042..26af411 100644 --- a/sms/README.md +++ b/sms/README.md @@ -1,8 +1,8 @@ -Send an SMS message +Send SMS messages # SMS Service -Send an SMS message in seconds. No fuss, no muss. +Send SMS messages in seconds. Integrate into any product for verification codes, links, reminders and more. Powered by [twilio.com](https://twilio.com). diff --git a/space/handler/space.go b/space/handler/space.go index 6aeb8cb..aad78b0 100644 --- a/space/handler/space.go +++ b/space/handler/space.go @@ -15,6 +15,8 @@ import ( "github.com/micro/micro/v3/service/errors" log "github.com/micro/micro/v3/service/logger" "github.com/micro/micro/v3/service/store" + pauth "github.com/micro/services/pkg/auth" + adminpb "github.com/micro/services/pkg/service/proto" "github.com/micro/services/pkg/tenant" pb "github.com/micro/services/space/proto" "github.com/minio/minio-go/v7/pkg/s3utils" @@ -503,3 +505,56 @@ func (s Space) Upload(ctx context.Context, request *pb.UploadRequest, response * return nil } + +func (s Space) DeleteData(ctx context.Context, request *adminpb.DeleteDataRequest, response *adminpb.DeleteDataResponse) error { + method := "admin.DeleteData" + _, err := pauth.VerifyMicroAdmin(ctx, method) + if err != nil { + return err + } + + if len(request.TenantId) < 10 { // deliberate length check so we don't delete all the things + return errors.BadRequest(method, "Missing tenant ID") + } + + objectName := request.TenantId + rsp, err := s.client.ListObjects(&sthree.ListObjectsInput{ + Bucket: aws.String(s.conf.SpaceName), + Prefix: aws.String(objectName), + }) + if err != nil { + log.Errorf("Error listing objects %s", err) + return errors.InternalServerError(method, "Error listing objects") + } + + oIDs := []*sthree.ObjectIdentifier{} + for _, v := range rsp.Contents { + oIDs = append(oIDs, &sthree.ObjectIdentifier{Key: v.Key}) + } + + if _, err := s.client.DeleteObjects(&sthree.DeleteObjectsInput{ + Bucket: aws.String(s.conf.SpaceName), + Delete: &sthree.Delete{ + Objects: oIDs, + }, + }); err != nil { + return err + } + + log.Infof("Deleted %d objects from s3 for %s", len(oIDs), request.TenantId) + + keys, err := store.List(store.ListPrefix(fmt.Sprintf("%s/", prefixByUser))) + if err != nil { + log.Errorf("Error listing objects %s", err) + return errors.InternalServerError(method, "Error listing objects") + } + for _, k := range keys { + if err := store.Delete(k); err != nil { + return err + } + } + log.Infof("Deleted %d objects from store for %s", len(keys), request.TenantId) + + return nil + +} diff --git a/space/main.go b/space/main.go index 7ddd74d..8e1e2e5 100644 --- a/space/main.go +++ b/space/main.go @@ -4,6 +4,7 @@ import ( "github.com/micro/micro/v3/service" "github.com/micro/micro/v3/service/api" "github.com/micro/micro/v3/service/logger" + admin "github.com/micro/services/pkg/service/proto" "github.com/micro/services/space/handler" ) @@ -14,12 +15,12 @@ func main() { service.Version("latest"), ) + h := handler.NewSpace(srv) // Register handler - //pb.RegisterSpaceHandler(srv.Server(), handler.NewSpace(srv)) - + admin.RegisterAdminHandler(srv.Server(), h) srv.Server().Handle( srv.Server().NewHandler( - handler.NewSpace(srv), + h, api.WithEndpoint( &api.Endpoint{ Name: "Space.Download", diff --git a/url/handler/url.go b/url/handler/url.go index ed23a59..590b53e 100644 --- a/url/handler/url.go +++ b/url/handler/url.go @@ -7,7 +7,10 @@ import ( "github.com/micro/micro/v3/service/config" "github.com/micro/micro/v3/service/errors" + "github.com/micro/micro/v3/service/logger" "github.com/micro/micro/v3/service/store" + pauth "github.com/micro/services/pkg/auth" + adminpb "github.com/micro/services/pkg/service/proto" "github.com/micro/services/pkg/tenant" url "github.com/micro/services/url/proto" cache "github.com/patrickmn/go-cache" @@ -135,3 +138,35 @@ func (e *Url) Proxy(ctx context.Context, req *url.ProxyRequest, rsp *url.ProxyRe return nil } + +func (e *Url) DeleteData(ctx context.Context, request *adminpb.DeleteDataRequest, response *adminpb.DeleteDataResponse) error { + method := "admin.DeleteData" + _, err := pauth.VerifyMicroAdmin(ctx, method) + if err != nil { + return err + } + + if len(request.TenantId) < 10 { // deliberate length check so we don't delete all the things + return errors.BadRequest(method, "Missing tenant ID") + } + + prefix := "urlOwner/" + request.TenantId + "/" + + keys, err := store.List(store.ListPrefix(prefix)) + if err != nil { + return err + } + + for _, key := range keys { + id := strings.TrimPrefix(key, prefix) + if err := store.Delete("url/" + id); err != nil { + return err + } + if err := store.Delete(key); err != nil { + return err + } + } + logger.Infof("Deleted %d objects from S3 for %s", len(keys), request.TenantId) + + return nil +} diff --git a/url/main.go b/url/main.go index 42865a1..31e1421 100644 --- a/url/main.go +++ b/url/main.go @@ -1,6 +1,7 @@ package main import ( + admin "github.com/micro/services/pkg/service/proto" "github.com/micro/services/pkg/tracing" "github.com/micro/services/url/handler" pb "github.com/micro/services/url/proto" @@ -15,9 +16,10 @@ func main() { service.Name("url"), service.Version("latest"), ) - + h := handler.NewUrl() // Register handler - pb.RegisterUrlHandler(srv.Server(), handler.NewUrl()) + pb.RegisterUrlHandler(srv.Server(), h) + admin.RegisterAdminHandler(srv.Server(), h) traceCloser := tracing.SetupOpentracing("url") defer traceCloser.Close() diff --git a/url/publicapi.json b/url/publicapi.json index fa2094d..691116b 100644 --- a/url/publicapi.json +++ b/url/publicapi.json @@ -2,5 +2,5 @@ "name": "url", "icon": "πŸ”—", "category": "utility", - "display_name": "URL Shortener" + "display_name": "URLs" } diff --git a/user/README.md b/user/README.md index f5a129c..8c02f73 100644 --- a/user/README.md +++ b/user/README.md @@ -1,7 +1,7 @@ -User management and authentication +Authentication and user management # User Service The user service provides user account management and authentication. It includes the ability to -send verification and password reset emails. All data is stored in the DB service under the "users" -table. +send verification and password reset emails. All data is stored securely on the M3O platform +and can be accessed via [M3O Cloud](https://cloud.m3o.com). diff --git a/user/handler/handler.go b/user/handler/handler.go index 528383d..2e065c0 100644 --- a/user/handler/handler.go +++ b/user/handler/handler.go @@ -514,7 +514,7 @@ func (s *User) DeleteData(ctx context.Context, request *adminpb.DeleteDataReques return err } - if len(request.TenantId) == 0 { + if len(request.TenantId) < 10 { // deliberate length check so we don't delete all the things return errors.BadRequest("user.DeleteData", "Missing tenant ID") } return s.domain.DeleteTenantData(request.TenantId)