integrate spam check on email (#270)

This commit is contained in:
Dominic Wong
2021-11-11 23:30:10 +00:00
committed by GitHub
parent 2e50bd5dc3
commit bfac5997d1
8 changed files with 121 additions and 52 deletions

View File

@@ -8,12 +8,15 @@ import (
"io/ioutil"
"net/http"
"github.com/micro/micro/v3/service"
"github.com/micro/micro/v3/service/client"
"github.com/micro/micro/v3/service/config"
"github.com/micro/micro/v3/service/errors"
log "github.com/micro/micro/v3/service/logger"
"github.com/micro/micro/v3/service/store"
pb "github.com/micro/services/email/proto"
"github.com/micro/services/pkg/tenant"
spampb "github.com/micro/services/spam/proto"
)
const (
@@ -31,7 +34,7 @@ type sendgridConf struct {
EmailFrom string `json:"email_from"`
}
func NewEmailHandler() *Email {
func NewEmailHandler(svc *service.Service) *Email {
c := sendgridConf{}
val, err := config.Get("sendgridapi")
if err != nil {
@@ -46,11 +49,13 @@ func NewEmailHandler() *Email {
}
return &Email{
c,
spampb.NewSpamService("spam", svc.Client()),
}
}
type Email struct {
config sendgridConf
config sendgridConf
spamSvc spampb.SpamService
}
func (e *Email) Send(ctx context.Context, request *pb.SendRequest, response *pb.SendResponse) error {
@@ -67,6 +72,19 @@ func (e *Email) Send(ctx context.Context, request *pb.SendRequest, response *pb.
return errors.BadRequest("email.send.validation", "Missing email body")
}
spamReq := &spampb.ClassifyRequest{
TextBody: request.TextBody,
HtmlBody: request.HtmlBody,
To: request.To,
From: request.From,
Subject: request.Subject,
}
rsp, err := e.spamSvc.Classify(ctx, spamReq, client.WithAuthToken())
if err != nil || rsp.IsSpam {
log.Errorf("Error validating email %s %v", err, rsp)
return errors.InternalServerError("email.send", "Error validating email")
}
if err := e.sendEmail(ctx, request); err != nil {
log.Errorf("Error sending email: %v\n", err)
return errors.InternalServerError("email.sendemail", "Error sending email")

View File

@@ -16,7 +16,7 @@ func main() {
)
// Register handler
pb.RegisterEmailHandler(srv.Server(), handler.NewEmailHandler())
pb.RegisterEmailHandler(srv.Server(), handler.NewEmailHandler(srv))
traceCloser := tracing.SetupOpentracing("email")
defer traceCloser.Close()

2
go.mod
View File

@@ -60,6 +60,8 @@ require (
google.golang.org/grpc/examples v0.0.0-20211103202053-3b94303f3754 // indirect
google.golang.org/protobuf v1.27.1
googlemaps.github.io/maps v1.3.1
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
gopkg.in/yaml.v2 v2.3.0
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
gorm.io/datatypes v1.0.1

12
go.sum
View File

@@ -501,14 +501,6 @@ github.com/mattn/go-tty v0.0.0-20180219170247-931426f7535a/go.mod h1:XPvLUNfbS4f
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/micro/micro-go v0.0.0-20211101221015-79ab982f8163 h1:kNngAyoUre7ahqYWjlBVpT4GGDYM7r9BYUzpcOveaPs=
github.com/micro/micro-go v0.0.0-20211101221015-79ab982f8163/go.mod h1:o4fTExNn5LlnQRB/WiW3RChsohPwQTJ1AKdNCz2YEYA=
github.com/micro/micro/v3 v3.6.1-0.20211110104311-614fde05be0c h1:9+w31dXDHVUD11x1St5LiXiBQLNSUxgeH9GI9+sKv0M=
github.com/micro/micro/v3 v3.6.1-0.20211110104311-614fde05be0c/go.mod h1:NqYnFOGrnc0Apk912w49oX9qIk1YDJcCaO+y+CLaAXA=
github.com/micro/micro/v3 v3.7.1-0.20211111155628-5c83aaf82619 h1:SRfAA7YcpHh1P/F1HY8p7QprED5o6WOlovGZoZcuHxU=
github.com/micro/micro/v3 v3.7.1-0.20211111155628-5c83aaf82619/go.mod h1:NqYnFOGrnc0Apk912w49oX9qIk1YDJcCaO+y+CLaAXA=
github.com/micro/micro/v3 v3.7.1-0.20211111162246-1f3d81ada4a9 h1:1jn/GQhmv87dxGaoOOWou+LfJWc+McSFlg+8GQ78Xf0=
github.com/micro/micro/v3 v3.7.1-0.20211111162246-1f3d81ada4a9/go.mod h1:NqYnFOGrnc0Apk912w49oX9qIk1YDJcCaO+y+CLaAXA=
github.com/micro/micro/v3 v3.7.1-0.20211111164609-a157fe0c3185 h1:7CjYMqI5ACQAL276dJGoleoHeZRIkLmMM0a9rSio8Kk=
github.com/micro/micro/v3 v3.7.1-0.20211111164609-a157fe0c3185/go.mod h1:NqYnFOGrnc0Apk912w49oX9qIk1YDJcCaO+y+CLaAXA=
github.com/micro/micro/v3 v3.7.1-0.20211111170433-1ebb8328e280 h1:SOthdbPABgdxgsi9d7lXBDE8WzlTvj+2owHR51vW2AA=
github.com/micro/micro/v3 v3.7.1-0.20211111170433-1ebb8328e280/go.mod h1:NqYnFOGrnc0Apk912w49oX9qIk1YDJcCaO+y+CLaAXA=
github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
@@ -1183,6 +1175,8 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
googlemaps.github.io/maps v1.3.1 h1:VYFiLFgZyDVFYjPKLedOWxjmrwuaJFAc4EhqGNZfX40=
googlemaps.github.io/maps v1.3.1/go.mod h1:cCq0JKYAnnCRSdiaBi7Ex9CW15uxIAk7oPi8V/xEh6s=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -1191,6 +1185,8 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE=
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
gopkg.in/h2non/gock.v1 v1.0.15/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE=
gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=

View File

@@ -5,7 +5,7 @@
"run_check": true,
"request": {
"subject": "Welcome",
"email_body": "Hi there,\n\nWelcome to M3O.\n\nThanks\nM3O team",
"text_body": "Hi there,\n\nWelcome to M3O.\n\nThanks\nM3O team",
"from": "noreply@m3o.com",
"to": "hello@example.com"
},
@@ -18,6 +18,22 @@
"MISSING_MID, Missing Message-Id: header, 0.1"
]
}
},
{
"title": "Classify an email using the raw data",
"run_check": true,
"request": {
"email_body": "Subject: Welcome\r\nTo: hello@emaple.com\r\nFrom: noreply@m3o.com\r\n\r\nHi there,\n\nWelcome to M3O.\n\nThanks\nM3O team"
},
"response": {
"is_spam": false,
"score": 0.1,
"details": [
"NO_RELAYS, Informational: message was not relayed via SMTP, -0",
"NO_RECEIVED, Informational: message has no Received headers, -0",
"MISSING_MID, Missing Message-Id: header, 0.1"
]
}
}
]
}

View File

@@ -1,11 +1,9 @@
package handler
import (
"bufio"
"bytes"
"context"
"fmt"
"net/textproto"
"time"
"github.com/Teamwork/spamc"
@@ -13,6 +11,7 @@ import (
"github.com/micro/micro/v3/service/errors"
log "github.com/micro/micro/v3/service/logger"
spam "github.com/micro/services/spam/proto"
"gopkg.in/gomail.v2"
)
type conf struct {
@@ -40,26 +39,39 @@ func New() *Spam {
}
func (s *Spam) Classify(ctx context.Context, request *spam.ClassifyRequest, response *spam.ClassifyResponse) error {
if len(request.EmailBody) == 0 {
return errors.BadRequest("spam.Classify", "missing email_body")
if len(request.EmailBody) == 0 && len(request.TextBody) == 0 && len(request.HtmlBody) == 0 {
return errors.BadRequest("spam.Classify", "Missing one of email_body, html_body, text_body")
}
bf := bytes.Buffer{}
bf := bytes.NewBufferString("")
tp := textproto.NewWriter(bufio.NewWriter(bf))
if len(request.EmailBody) > 0 {
bf.WriteString(request.EmailBody)
} else {
m := gomail.NewMessage()
if len(request.To) > 0 {
m.SetHeader("To", request.To)
}
if len(request.From) > 0 {
m.SetHeader("From", request.From)
}
if len(request.Subject) > 0 {
m.SetHeader("Subject", request.Subject)
}
m.SetHeader("Date", time.Now().Format(time.RFC1123Z))
if len(request.TextBody) > 0 {
m.SetBody("text/plain", request.TextBody)
}
if len(request.HtmlBody) > 0 {
m.SetBody("text/html", request.HtmlBody)
}
if _, err := m.WriteTo(&bf); err != nil {
log.Errorf("Error classifying email %s", err)
return errors.InternalServerError("spam.Classify", "Error classifying email")
}
if len(request.To) > 0 {
tp.PrintfLine("To: %v", request.To)
}
if len(request.From) > 0 {
tp.PrintfLine("From: %v", request.From)
}
if len(request.Subject) > 0 {
tp.PrintfLine("Subject: %v", request.Subject)
}
tp.PrintfLine("Date: %s", time.Now().Format(time.RFC1123Z))
tp.PrintfLine("")
tp.PrintfLine("%v", request.EmailBody)
rc, err := s.client.Report(ctx, bf, spamc.Header{}.Set("Content-Length", fmt.Sprintf("%d", bf.Len())))
rc, err := s.client.Report(ctx, &bf, spamc.Header{}.Set("Content-Length", fmt.Sprintf("%d", bf.Len())))
if err != nil {
log.Errorf("Error checking spamd %s", err)
return errors.InternalServerError("spam.Classify", "Error classifying email")

View File

@@ -26,7 +26,7 @@ type ClassifyRequest struct {
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// The body of the email
// The raw body of the email including headers etc per RFC 822. Alternatively, use the other parameters to correctly format the message
EmailBody string `protobuf:"bytes,1,opt,name=email_body,json=emailBody,proto3" json:"email_body,omitempty"`
// The email address it is being sent to
To string `protobuf:"bytes,2,opt,name=to,proto3" json:"to,omitempty"`
@@ -34,6 +34,10 @@ type ClassifyRequest struct {
From string `protobuf:"bytes,3,opt,name=from,proto3" json:"from,omitempty"`
// The subject of the email
Subject string `protobuf:"bytes,4,opt,name=subject,proto3" json:"subject,omitempty"`
// the plain text version of the email body
TextBody string `protobuf:"bytes,5,opt,name=text_body,json=textBody,proto3" json:"text_body,omitempty"`
// the HTML version of the email body
HtmlBody string `protobuf:"bytes,6,opt,name=html_body,json=htmlBody,proto3" json:"html_body,omitempty"`
}
func (x *ClassifyRequest) Reset() {
@@ -96,6 +100,20 @@ func (x *ClassifyRequest) GetSubject() string {
return ""
}
func (x *ClassifyRequest) GetTextBody() string {
if x != nil {
return x.TextBody
}
return ""
}
func (x *ClassifyRequest) GetHtmlBody() string {
if x != nil {
return x.HtmlBody
}
return ""
}
type ClassifyResponse struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -166,26 +184,29 @@ var File_proto_spam_proto protoreflect.FileDescriptor
var file_proto_spam_proto_rawDesc = []byte{
0x0a, 0x10, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x70, 0x61, 0x6d, 0x2e, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x12, 0x04, 0x73, 0x70, 0x61, 0x6d, 0x22, 0x6e, 0x0a, 0x0f, 0x43, 0x6c, 0x61, 0x73,
0x73, 0x69, 0x66, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x65,
0x6d, 0x61, 0x69, 0x6c, 0x5f, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
0x09, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x0e, 0x0a, 0x02, 0x74, 0x6f,
0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x74, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x72,
0x6f, 0x6d, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x72, 0x6f, 0x6d, 0x12, 0x18,
0x0a, 0x07, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52,
0x07, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x22, 0x5b, 0x0a, 0x10, 0x43, 0x6c, 0x61, 0x73,
0x73, 0x69, 0x66, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x17, 0x0a, 0x07,
0x69, 0x73, 0x5f, 0x73, 0x70, 0x61, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x69,
0x73, 0x53, 0x70, 0x61, 0x6d, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x02,
0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x64,
0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x65,
0x74, 0x61, 0x69, 0x6c, 0x73, 0x32, 0x43, 0x0a, 0x04, 0x53, 0x70, 0x61, 0x6d, 0x12, 0x3b, 0x0a,
0x08, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x69, 0x66, 0x79, 0x12, 0x15, 0x2e, 0x73, 0x70, 0x61, 0x6d,
0x2e, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x69, 0x66, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x1a, 0x16, 0x2e, 0x73, 0x70, 0x61, 0x6d, 0x2e, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x69, 0x66, 0x79,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x0e, 0x5a, 0x0c, 0x2e, 0x2f,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3b, 0x73, 0x70, 0x61, 0x6d, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x33,
0x74, 0x6f, 0x12, 0x04, 0x73, 0x70, 0x61, 0x6d, 0x22, 0xa8, 0x01, 0x0a, 0x0f, 0x43, 0x6c, 0x61,
0x73, 0x73, 0x69, 0x66, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a,
0x65, 0x6d, 0x61, 0x69, 0x6c, 0x5f, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
0x52, 0x09, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x0e, 0x0a, 0x02, 0x74,
0x6f, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x74, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x66,
0x72, 0x6f, 0x6d, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x72, 0x6f, 0x6d, 0x12,
0x18, 0x0a, 0x07, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09,
0x52, 0x07, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x65, 0x78,
0x74, 0x5f, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x65,
0x78, 0x74, 0x42, 0x6f, 0x64, 0x79, 0x12, 0x1b, 0x0a, 0x09, 0x68, 0x74, 0x6d, 0x6c, 0x5f, 0x62,
0x6f, 0x64, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x68, 0x74, 0x6d, 0x6c, 0x42,
0x6f, 0x64, 0x79, 0x22, 0x5b, 0x0a, 0x10, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x69, 0x66, 0x79, 0x52,
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x73, 0x5f, 0x73, 0x70,
0x61, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x69, 0x73, 0x53, 0x70, 0x61, 0x6d,
0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52,
0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c,
0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73,
0x32, 0x43, 0x0a, 0x04, 0x53, 0x70, 0x61, 0x6d, 0x12, 0x3b, 0x0a, 0x08, 0x43, 0x6c, 0x61, 0x73,
0x73, 0x69, 0x66, 0x79, 0x12, 0x15, 0x2e, 0x73, 0x70, 0x61, 0x6d, 0x2e, 0x43, 0x6c, 0x61, 0x73,
0x73, 0x69, 0x66, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x73, 0x70,
0x61, 0x6d, 0x2e, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x69, 0x66, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x0e, 0x5a, 0x0c, 0x2e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x3b, 0x73, 0x70, 0x61, 0x6d, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (

View File

@@ -10,7 +10,7 @@ service Spam {
// Check whether an email is likely to be spam based on its attributes
message ClassifyRequest {
// The body of the email
// The raw body of the email including headers etc per RFC 822. Alternatively, use the other parameters to correctly format the message
string email_body = 1;
// The email address it is being sent to
string to = 2;
@@ -18,6 +18,10 @@ message ClassifyRequest {
string from = 3;
// The subject of the email
string subject = 4;
// the plain text version of the email body
string text_body = 5;
// the HTML version of the email body
string html_body = 6;
}
message ClassifyResponse {