diff --git a/email/handler/email_test.go b/email/handler/email_test.go index 5374f75..ee7605d 100644 --- a/email/handler/email_test.go +++ b/email/handler/email_test.go @@ -7,7 +7,6 @@ import ( ) func TestEmailValidation(t *testing.T) { - g := NewWithT(t) tcs := []struct { name string email string @@ -60,6 +59,7 @@ func TestEmailValidation(t *testing.T) { }, } for _, tc := range tcs { + g := NewWithT(t) t.Run(tc.name, func(t *testing.T) { g.Expect(validEmail(tc.email)).To(Equal(tc.valid)) }) diff --git a/go.mod b/go.mod index 99bda14..fa6cd3f 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/Teamwork/spamc v0.0.0-20200109085853-a4e0c5c3f7a0 github.com/asim/mq v0.1.0 github.com/aws/aws-sdk-go v1.42.17 + github.com/bitly/go-simplejson v0.5.0 github.com/cdipaolo/goml v0.0.0-20190412180403-e1f51f713598 // indirect github.com/cdipaolo/sentiment v0.0.0-20200617002423-c697f64e7f10 github.com/crufter/lexer v0.0.0-20120907053443-23fe8c7add01 @@ -36,6 +37,7 @@ require ( github.com/minio/minio-go/v7 v7.0.16 github.com/o1egl/govatar v0.3.0 github.com/onsi/gomega v1.10.5 + github.com/opensearch-project/opensearch-go v1.0.0 github.com/oschwald/geoip2-golang v1.5.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/paulmach/go.geo v0.0.0-20180829195134-22b514266d33 diff --git a/go.sum b/go.sum index 1cdcf66..c8188ef 100644 --- a/go.sum +++ b/go.sum @@ -540,6 +540,8 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= +github.com/opensearch-project/opensearch-go v1.0.0 h1:8Gh7B7Un5BxuxWAgmzleEF7lpOtC71pCgPp7lKr3ca8= +github.com/opensearch-project/opensearch-go v1.0.0/go.mod h1:FrUl/52DBegRYvK7ISF278AXmjDV647lyTnsLGBR7J4= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= diff --git a/search/README.md b/search/README.md index 04505a4..59c5751 100644 --- a/search/README.md +++ b/search/README.md @@ -1,7 +1,78 @@ -Indexing & full text search +Indexing and full text search # Search Service -Store and search for anything text based. The Search API provides -full indexing and text search. +Store and search JSON documents. The Search API provides full indexing and text search. +Powered by [OpenSearch](https://opensearch.org/). + +Search for a given word or phrase in a particular field of a document. Combine multiple with either `AND` or `OR` boolean operators to create complex queries. + +## Usage +Documents are inserted using the `/search/index` endpoint. Document fields are automatically indexed with no need to define which fields to index ahead of time. Documents are logically grouped in to `indexes` so you may have an index for customers and one for products. Once documents are inserted you are ready to search, simple as that. + +## Search query language + +The search API supports a simple query language to let you get to your data quickly without having to learn a complicated language. + +The most basic query looks like this + +```sql +key == 'value' +``` + +where you specify a key and a value to find. For example you might want to look for every customer with first name of John + +```sql +first_name == 'John' +``` + +String values support single or double quotes. + +Values can also be numbers + +```sql +age == 37 +``` + +or booleans + +```sql +verified == true +``` + +You can search on fields that are nested in the document using dot (`.`) as a separator + +```sql +address.city == 'London' +``` + +The API also supports wildcard `*` matching to enable scenarios like autocomplete. + +```sql +first_name == 'Joh*' +``` + +In addition to equality `==` the API support greater than or equals `>=` and less than or equals `<=` operators + +```sql +age >= 37 +age <= 37 +``` + +Simple queries can be combined with logical `and` + +```sql +first_name == "John" AND age <= 37 +``` + +or logical `or` +```sql +first_name == "John" OR first_name == "Jane" +``` + +If combining `and` and `or` operations you will need to use parentheses to explicitly define precedence + +```sql +(first_name == "John" OR first_name == "Jane") AND age <= 37 +``` diff --git a/search/examples.json b/search/examples.json index e957989..39bfcd4 100644 --- a/search/examples.json +++ b/search/examples.json @@ -1,14 +1,104 @@ { - "vote": [ + "index": [ { - "title": "Vote for the API", + "title": "Index a document", "run_check": false, "request": { - "message": "Launch it!" + "index": "customers", + "document": { + "id": "1234", + "contents": { + "name": "John Doe", + "age": 37, + "starsign": "Leo" + } + } }, "response": { - "message": "Thanks for the vote!" } } + ], + "search": [ + { + "title": "Search for a document", + "run_check": false, + "request": { + "index": "customers", + "query": "name == 'John'" + }, + "response": { + "documents": [ + { + "id": "1234", + "contents": { + "name": "John Doe", + "age": 37, + "starsign": "Leo" + } + } + ] + } + }, + { + "title": "Search on multiple fields (AND)", + "run_check": false, + "request": { + "index": "customers", + "query": "name == 'John' AND starsign == 'Leo'" + }, + "response": { + "documents": [ + { + "id": "1234", + "contents": { + "name": "John Doe", + "age": 37, + "starsign": "Leo" + } + } + ] + } + }, + { + "title": "Search on multiple fields (OR)", + "run_check": false, + "request": { + "index": "customers", + "query": "name == 'John' OR name == 'Jane'" + }, + "response": { + "documents": [ + { + "id": "1234", + "contents": { + "name": "John Doe", + "age": 37, + "starsign": "Leo" + } + } + ] + } + } + ], + "delete": [ + { + "title": "Delete a document", + "run_check": false, + "request": { + "id": "1234", + "index": "customers" + }, + "response": {} + } + ], + "deleteIndex": [ + { + "title": "Delete an index", + "run_check": false, + "request": { + "index": "customers" + }, + "response": {} + } ] } diff --git a/search/handler/lexer.go b/search/handler/lexer.go new file mode 100644 index 0000000..290bd74 --- /dev/null +++ b/search/handler/lexer.go @@ -0,0 +1,287 @@ +package handler + +import ( + "fmt" + "strings" + "unicode" + "unicode/utf8" +) + +type itemType int + +const ( + itemError itemType = iota + itemNumber + itemIdentifier + itemBoolean + itemBooleanOp + itemString + itemOperator + itemLeftParen + itemRightParen +) + +const ( + eof = -1 +) + +type item struct { + typ itemType + val string +} + +type stateFn func(*lexer) stateFn + +func (l *lexer) run() { + for state := lexStartStatement; state != nil; { + state = state(l) + } + close(l.items) +} + +type lexer struct { + name string //used only for error reports + input string // the string being scanned + start int // start position of this item + pos int // current position in the input + width int // width of last rune read + items chan item // channel of scanned items + +} + +func lex(name, input string) (*lexer, chan item) { + l := &lexer{ + name: name, + input: input, + items: make(chan item), + } + go l.run() + return l, l.items +} + +func (l *lexer) emit(t itemType) { + l.items <- item{t, l.input[l.start:l.pos]} + l.start = l.pos +} + +func (l *lexer) errorf(format string, args ...interface{}) stateFn { + l.items <- item{ + itemError, + fmt.Sprintf(format, args...), + } + return nil +} + +func (l *lexer) consumeSpace() { + l.acceptRun(" ") + l.ignore() +} + +// next returns the next rune in the input +func (l *lexer) next() (rune int32) { + if l.pos == len(l.input) { + l.width = 0 + return eof + } + rune, l.width = utf8.DecodeRuneInString(l.input[l.pos:]) + l.pos += l.width + return rune +} + +// ignore skips over the pending input before this point +func (l *lexer) ignore() { + l.start = l.pos +} + +// backup steps back one rune +// Can be called only once per call of next +func (l *lexer) backup() { + l.pos -= l.width +} + +// peek returns but does not consume +// the next rune in the input +func (l *lexer) peek() int32 { + rune := l.next() + l.backup() + return rune +} + +// accept consumes the next rune +// if it's from the valid set +func (l *lexer) accept(valid string) bool { + if strings.IndexRune(valid, l.next()) >= 0 { + return true + } + l.backup() + return false +} + +// acceptRune consumes a run of runes from the valid set +func (l *lexer) acceptRun(valid string) { + for strings.IndexRune(valid, l.next()) >= 0 { + + } + l.backup() +} + +func lexIdent(l *lexer) stateFn { + l.consumeSpace() + for { + r := l.next() + if r == eof { + return l.errorf("Unexpected end of input %q", l.input[l.start:]) + } + + if unicode.IsSpace(r) || strings.IndexRune("=><", r) >= 0 { + l.backup() + break + } + } + if l.pos > l.start { + l.emit(itemIdentifier) + } + return lexOperator(l) +} + +func lexValue(l *lexer) stateFn { + l.consumeSpace() + switch { + case l.accept(`"'`): + l.backup() + return lexString(l) + case strings.HasPrefix(l.input[l.start:], "true"), strings.HasPrefix(l.input[l.start:], "false"): + return lexBool(l) + default: + // try it as a number + return lexNumber(l) + } +} + +func lexString(l *lexer) stateFn { + // TODO support single and double quotes with escaping + if !l.accept(`"'`) { + return l.errorf("Unexpected value %v, expected a quote", l.peek()) + } + // ignore the quote + openQuote := l.input[l.start:l.pos] + l.ignore() + lastRead := "" + for { + r := l.next() + if r == eof { // should only happen in error case + return l.errorf("Unexpected value %v, incorrectly terminated value %s %v", l.input[l.start:], openQuote, lastRead) + } + if string(r) == openQuote && lastRead != `\` { + l.backup() + l.emit(itemString) + l.next() + l.ignore() // ignore the quote + return lexEndStatement(l) + } + lastRead = string(r) + + } +} + +const ( + operatorEquals = `==` + operatorGreater = `>=` + operatorLess = `<=` + parenLeft = `(` + parenRight = `)` +) + +func lexOperator(l *lexer) stateFn { + l.consumeSpace() + switch l.input[l.pos : l.pos+2] { + case operatorEquals, operatorGreater, operatorLess: + l.pos += 2 + l.emit(itemOperator) + return lexValue(l) + } + // look for identifier + return l.errorf("Unexpected operator %q", l.input[l.pos:l.pos+2]) +} + +func lexNumber(l *lexer) stateFn { + l.consumeSpace() + // optional leading sign + l.accept("+-") + digits := "0123456789" + if l.accept("0") && l.accept("xX") { + digits = "0123456789abcdefABCDEF" + } + l.acceptRun(digits) + if l.accept(".") { + l.acceptRun(digits) + } + if l.accept("eE") { + l.accept("+-") + l.acceptRun("0123456789") + } + if l.start == l.pos { + return l.errorf("Unexpected value %s", l.input[l.start:]) + } + l.emit(itemNumber) + return lexEndStatement(l) +} + +func lexStartStatement(l *lexer) stateFn { + l.consumeSpace() + if string(l.peek()) == parenLeft { + l.next() + l.emit(itemLeftParen) + return lexStartStatement(l) + } + return lexIdent(l) +} + +func lexEndStatement(l *lexer) stateFn { + // is this the end of the statement or can we find a boolean op + l.consumeSpace() + if string(l.peek()) == parenRight { + l.next() + l.emit(itemRightParen) + return lexEndStatement(l) + } + + for { + r := l.next() + if r == eof { + break + } + if unicode.IsSpace(r) { + l.backup() + break + } + } + if l.start == l.pos { + return nil + } + + if l.input[l.start:l.pos] == "and" || l.input[l.start:l.pos] == "AND" || l.input[l.start:l.pos] == "or" || l.input[l.start:l.pos] == "OR" { + l.emit(itemBooleanOp) + return lexStartStatement(l) + } + return l.errorf("Unexpected input %v", l.input[l.start:l.pos]) +} + +func lexBool(l *lexer) stateFn { + for { + r := l.next() + if r == eof { + break + } + if unicode.IsSpace(r) { + l.backup() + break + } + } + if l.input[l.start:l.pos] == "true" || l.input[l.start:l.pos] == "false" { + l.emit(itemBoolean) + return lexEndStatement(l) + } + + return l.errorf("Unexpected value %q, expecting a boolean", l.input[l.start:l.pos]) +} diff --git a/search/handler/lexer_test.go b/search/handler/lexer_test.go new file mode 100644 index 0000000..be5c5ad --- /dev/null +++ b/search/handler/lexer_test.go @@ -0,0 +1,499 @@ +package handler + +import ( + "fmt" + "testing" + + . "github.com/onsi/gomega" +) + +func TestLexer(t *testing.T) { + + tcs := []struct { + name string + input string + tokens []item + err error + }{ + { + name: "basic", + input: `foo == "bar"`, + tokens: []item{ + { + typ: itemIdentifier, + val: "foo", + }, + { + typ: itemOperator, + val: "==", + }, + { + typ: itemString, + val: `bar`, + }, + }, + }, + { + name: "basic", + input: `first_name == 'Dom'`, + tokens: []item{ + { + typ: itemIdentifier, + val: "first_name", + }, + { + typ: itemOperator, + val: "==", + }, + { + typ: itemString, + val: `Dom`, + }, + }, + }, + { + name: "basic compressed", + input: `foo=="bar"`, + tokens: []item{ + { + typ: itemIdentifier, + val: "foo", + }, + { + typ: itemOperator, + val: "==", + }, + { + typ: itemString, + val: `bar`, + }, + }, + }, + { + name: "basic bool", + input: `foo == true`, + tokens: []item{ + { + typ: itemIdentifier, + val: "foo", + }, + { + typ: itemOperator, + val: "==", + }, + { + typ: itemBoolean, + val: `true`, + }, + }, + }, + { + name: "basic bool false", + input: `foo == false`, + tokens: []item{ + { + typ: itemIdentifier, + val: "foo", + }, + { + typ: itemOperator, + val: "==", + }, + { + typ: itemBoolean, + val: `false`, + }, + }, + }, + { + name: "basic with spaces", + input: `foo == "hello there"`, + tokens: []item{ + { + typ: itemIdentifier, + val: "foo", + }, + { + typ: itemOperator, + val: "==", + }, + { + typ: itemString, + val: `hello there`, + }, + }, + }, + { + name: "basic number", + input: `foo == 123987`, + tokens: []item{ + { + typ: itemIdentifier, + val: "foo", + }, + { + typ: itemOperator, + val: "==", + }, + { + typ: itemNumber, + val: `123987`, + }, + }, + }, + { + name: "basic gt number", + input: `foo >= 123987`, + tokens: []item{ + { + typ: itemIdentifier, + val: "foo", + }, + { + typ: itemOperator, + val: ">=", + }, + { + typ: itemNumber, + val: `123987`, + }, + }, + }, + { + name: "basic lt number", + input: `foo <= 123987`, + tokens: []item{ + { + typ: itemIdentifier, + val: "foo", + }, + { + typ: itemOperator, + val: "<=", + }, + { + typ: itemNumber, + val: `123987`, + }, + }, + }, + { + name: "AND bool", + input: `foo == 'bar' AND baz == 'hello'`, + tokens: []item{ + { + typ: itemIdentifier, + val: "foo", + }, + { + typ: itemOperator, + val: "==", + }, + { + typ: itemString, + val: `bar`, + }, + { + typ: itemBooleanOp, + val: "AND", + }, + { + typ: itemIdentifier, + val: "baz", + }, + { + typ: itemOperator, + val: "==", + }, + { + typ: itemString, + val: `hello`, + }, + }, + }, + { + name: "and bool", + input: `foo == 'bar' and baz == 'hello'`, + tokens: []item{ + { + typ: itemIdentifier, + val: "foo", + }, + { + typ: itemOperator, + val: "==", + }, + { + typ: itemString, + val: `bar`, + }, + { + typ: itemBooleanOp, + val: "and", + }, + { + typ: itemIdentifier, + val: "baz", + }, + { + typ: itemOperator, + val: "==", + }, + { + typ: itemString, + val: `hello`, + }, + }, + }, + { + name: "OR bool", + input: `foo == 'bar' OR baz == 'hello'`, + tokens: []item{ + { + typ: itemIdentifier, + val: "foo", + }, + { + typ: itemOperator, + val: "==", + }, + { + typ: itemString, + val: `bar`, + }, + { + typ: itemBooleanOp, + val: "OR", + }, + { + typ: itemIdentifier, + val: "baz", + }, + { + typ: itemOperator, + val: "==", + }, + { + typ: itemString, + val: `hello`, + }, + }, + }, + { + name: "or bool", + input: `foo == 'bar' or baz == 'hello'`, + tokens: []item{ + { + typ: itemIdentifier, + val: "foo", + }, + { + typ: itemOperator, + val: "==", + }, + { + typ: itemString, + val: `bar`, + }, + { + typ: itemBooleanOp, + val: "or", + }, + { + typ: itemIdentifier, + val: "baz", + }, + { + typ: itemOperator, + val: "==", + }, + { + typ: itemString, + val: `hello`, + }, + }, + }, + { + name: "bad val", + input: `foo == bar`, + tokens: []item{ + { + typ: itemIdentifier, + val: "foo", + }, + { + typ: itemOperator, + val: "==", + }, + { + typ: itemError, + }, + }, + err: fmt.Errorf("blah"), + }, + { + name: "gibberish", + input: `123onddlqkjn oajsldkj`, + tokens: []item{ + { + typ: itemIdentifier, + val: "123onddlqkjn", + }, + { + typ: itemError, + }, + }, + err: fmt.Errorf("blah"), + }, + { + name: "gibberish", + input: `123onddlqkjn`, + tokens: []item{ + { + typ: itemError, + }, + }, + err: fmt.Errorf("blah"), + }, + { + name: "brackets", + input: `foo == 'bar' and (baz == 'hello' or customer.name == 'john doe')`, + tokens: []item{ + { + typ: itemIdentifier, + val: "foo", + }, + { + typ: itemOperator, + val: "==", + }, + { + typ: itemString, + val: `bar`, + }, + { + typ: itemBooleanOp, + val: "and", + }, + { + typ: itemLeftParen, + val: "(", + }, + { + typ: itemIdentifier, + val: "baz", + }, + { + typ: itemOperator, + val: "==", + }, + { + typ: itemString, + val: `hello`, + }, + { + typ: itemBooleanOp, + val: "or", + }, + { + typ: itemIdentifier, + val: "customer.name", + }, + { + typ: itemOperator, + val: "==", + }, + { + typ: itemString, + val: "john doe", + }, + { + typ: itemRightParen, + val: ")", + }, + }, + }, + { + name: "brackets", + input: `(foo == 'bar' and baz == 'hello') or customer.name == 'john doe'`, + tokens: []item{ + { + typ: itemLeftParen, + val: "(", + }, + { + typ: itemIdentifier, + val: "foo", + }, + { + typ: itemOperator, + val: "==", + }, + { + typ: itemString, + val: `bar`, + }, + { + typ: itemBooleanOp, + val: "and", + }, + { + typ: itemIdentifier, + val: "baz", + }, + { + typ: itemOperator, + val: "==", + }, + { + typ: itemString, + val: `hello`, + }, + { + typ: itemRightParen, + val: ")", + }, + { + typ: itemBooleanOp, + val: "or", + }, + { + typ: itemIdentifier, + val: "customer.name", + }, + { + typ: itemOperator, + val: "==", + }, + { + typ: itemString, + val: "john doe", + }, + }, + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + _, ch := lex(tc.name, tc.input) + erred := false + for _, tok := range tc.tokens { + it := <-ch + t.Logf("Got %v", it) + if it.typ == itemError { + g.Expect(tok.typ).To(Equal(itemError)) + erred = true + } else { + g.Expect(it).To(Equal(tok)) + } + + } + if tc.err != nil { + g.Expect(erred).To(BeTrue()) + } + }) + } + +} diff --git a/search/handler/parser.go b/search/handler/parser.go new file mode 100644 index 0000000..d7e7927 --- /dev/null +++ b/search/handler/parser.go @@ -0,0 +1,101 @@ +package handler + +import ( + "fmt" + "strings" + + "github.com/bitly/go-simplejson" +) + +func parseQueryString(query string) (*simplejson.Json, error) { + _, items := lex("foo", query) + res, err := parseQueryStringRec(items) + if err != nil { + return nil, err + } + js := simplejson.New() + js.Set("query", res) + return js, nil +} + +const ( + matchTypeRange = "range" + matchTypeMatch = "match" + matchTypeWildcard = "wildcard" +) + +func parseQueryStringRec(items chan item) (*simplejson.Json, error) { + retTerm := simplejson.New() + currFieldName := "" + currBool := "" + currMatchType := matchTypeMatch + currPathAddition := "" + terms := []*simplejson.Json{} +itemLoop: + for it := range items { + if it.typ == itemError { + return nil, fmt.Errorf(it.val) + } + + switch it.typ { + case itemIdentifier: + currFieldName = it.val + case itemString, itemBoolean, itemNumber: + currTerm := simplejson.New() + if strings.ContainsRune(it.val, '*') { + currMatchType = matchTypeWildcard + currPathAddition = "value" + } + path := []string{currMatchType, currFieldName} + if len(currPathAddition) > 0 { + path = append(path, currPathAddition) + } + currTerm.SetPath(path, it.val) + terms = append(terms, currTerm) + + // reset + currFieldName = "" + currMatchType = matchTypeMatch + + case itemBooleanOp: + thisBool := "must" + if strings.ToLower(it.val) == "or" { + thisBool = "should" + } + if len(currBool) == 0 { + currBool = thisBool + } else { + if thisBool != currBool { + // undefined behaviour order of precedence needs to be explicitly defined using parentheses + return nil, fmt.Errorf("error parsing query. Cannot mix boolean operators without explicitly defining precedence with parentheses") + } + } + case itemLeftParen: + currTerm, err := parseQueryStringRec(items) + if err != nil { + return nil, err + } + terms = append(terms, currTerm) + case itemRightParen: + break itemLoop + case itemOperator: + switch it.val { + case "==": + currMatchType = matchTypeMatch + currPathAddition = "" + case ">=": + currMatchType = matchTypeRange + currPathAddition = "gte" + case "<=": + currMatchType = matchTypeRange + currPathAddition = "lte" + } + + } + } + if len(currBool) == 0 { + currBool = "must" + } + retTerm.SetPath([]string{"bool", currBool}, terms) + return retTerm, nil +} diff --git a/search/handler/parser_test.go b/search/handler/parser_test.go new file mode 100644 index 0000000..de3177d --- /dev/null +++ b/search/handler/parser_test.go @@ -0,0 +1,130 @@ +package handler + +import ( + "fmt" + "testing" + + . "github.com/onsi/gomega" +) + +func TestParsing(t *testing.T) { + + tcs := []struct { + name string + input string + output string + err error + }{ + { + name: "basic", + input: `foo == "bar"`, + output: `{"query":{"bool":{"must":[{"match":{"foo":"bar"}}]}}}`, + }, + { + name: "basic", + input: `first_name == 'Dom'`, + output: `{"query":{"bool":{"must":[{"match":{"first_name":"Dom"}}]}}}`, + }, + { + name: "basic bool", + input: `foo == true`, + output: `{"query":{"bool":{"must":[{"match":{"foo":"true"}}]}}}`, + }, + { + name: "basic bool false", + input: `foo == false`, + output: `{"query":{"bool":{"must":[{"match":{"foo":"false"}}]}}}`, + }, + { + name: "basic with spaces", + input: `foo == "hello there"`, + output: `{"query":{"bool":{"must":[{"match":{"foo":"hello there"}}]}}}`, + }, + { + name: "basic number", + input: `foo == 123987`, + output: `{"query":{"bool":{"must":[{"match":{"foo":"123987"}}]}}}`, + }, + { + name: "basic gt number", + input: `foo >= 123987`, + output: `{"query":{"bool":{"must":[{"range":{"foo":{"gte":"123987"}}}]}}}`, + }, + { + name: "basic lt number", + input: `foo <= 123987`, + output: `{"query":{"bool":{"must":[{"range":{"foo":{"lte":"123987"}}}]}}}`, + }, + { + name: "AND bool", + input: `foo == 'bar' AND baz == 'hello'`, + output: `{"query":{"bool":{"must":[{"match":{"foo":"bar"}},{"match":{"baz":"hello"}}]}}}`, + }, + { + name: "and bool", + input: `foo == 'bar' and baz == 'hello'`, + output: `{"query":{"bool":{"must":[{"match":{"foo":"bar"}},{"match":{"baz":"hello"}}]}}}`, + }, + { + name: "OR bool", + input: `foo == 'bar' OR baz == 'hello'`, + output: `{"query":{"bool":{"should":[{"match":{"foo":"bar"}},{"match":{"baz":"hello"}}]}}}`, + }, + { + name: "or bool", + input: `foo == 'bar' or baz == 'hello'`, + output: `{"query":{"bool":{"should":[{"match":{"foo":"bar"}},{"match":{"baz":"hello"}}]}}}`, + }, + { + name: "bad val", + input: `foo == bar`, + err: fmt.Errorf("blah"), + }, + { + name: "gibberish", + input: `123onddlqkjn oajsldkj`, + err: fmt.Errorf("blah"), + }, + { + name: "gibberish", + input: `123onddlqkjn`, + err: fmt.Errorf("blah"), + }, + { + name: "brackets", + input: `foo == 'bar' and (baz == 'hello' or customer.name == 'john doe')`, + output: `{"query":{"bool":{"must":[{"match":{"foo":"bar"}},{"bool":{"should":[{"match":{"baz":"hello"}},{"match":{"customer.name":"john doe"}}]}}]}}}`, + }, + { + name: "gte", + input: `foo >= 6`, + output: `{"query":{"bool":{"must":[{"range":{"foo":{"gte":"6"}}}]}}}`, + }, + { + name: "lte", + input: `foo <= 6`, + output: `{"query":{"bool":{"must":[{"range":{"foo":{"lte":"6"}}}]}}}`, + }, + { + name: "wildcard", + input: `foo == "ba*"`, + output: `{"query":{"bool":{"must":[{"wildcard":{"foo":{"value":"ba*"}}}]}}}`, + }, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + js, err := parseQueryString(tc.input) + if tc.err != nil { + g.Expect(err).To(Not(BeNil())) + } else { + b, _ := js.MarshalJSON() + t.Logf("%+v", string(b)) + g.Expect(err).To(BeNil()) + g.Expect(string(b)).To(Equal(tc.output)) + } + + }) + } + +} diff --git a/search/handler/search.go b/search/handler/search.go index 38f0d45..273fbb3 100644 --- a/search/handler/search.go +++ b/search/handler/search.go @@ -1,48 +1,291 @@ package handler import ( + "bytes" "context" - "sync" - "time" + "crypto/tls" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "regexp" + "strings" - "github.com/micro/micro/v3/service/store" + "github.com/google/uuid" + "github.com/micro/micro/v3/service" + "github.com/micro/micro/v3/service/config" + "github.com/micro/micro/v3/service/errors" + log "github.com/micro/micro/v3/service/logger" "github.com/micro/services/pkg/tenant" pb "github.com/micro/services/search/proto" + open "github.com/opensearch-project/opensearch-go" + openapi "github.com/opensearch-project/opensearch-go/opensearchapi" + "google.golang.org/protobuf/types/known/structpb" ) -type Search struct{} - var ( - mtx sync.RWMutex - - voteKey = "votes/" + indexNameRegex = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9]$`) + shortIndexNameRegex = regexp.MustCompile(`[a-zA-Z0-9]`) ) -type Vote struct { - Id string `json:"id"` - Message string `json:"message"` - VotedAt time.Time `json:"voted_at"` +type Search struct { + conf conf + client *open.Client } -func (n *Search) Vote(ctx context.Context, req *pb.VoteRequest, rsp *pb.VoteResponse) error { - mtx.Lock() - defer mtx.Unlock() +type conf struct { + OpenAddr string `json:"open_addr"` + User string `json:"user"` + Pass string `json:"pass"` + Insecure bool `json:"insecure"` +} - id, ok := tenant.FromContext(ctx) - if !ok { - id = "micro" +type openSearchResponse struct { + Took int64 `json:"took"` + Hits hits `json:"hits"` +} + +type hits struct { + Total map[string]interface{} `json:"total"` + Hits []hit `json:"hits"` +} +type hit struct { + ID string `json:"_id"` + Score float64 `json:"_score"` + Source map[string]interface{} `json:"_source"` +} + +func New(srv *service.Service) *Search { + v, err := config.Get("micro.search") + if err != nil { + log.Fatalf("Failed to load config %s", err) + } + var c conf + if err := v.Scan(&c); err != nil { + log.Fatalf("Failed to load config %s", err) + } + if len(c.OpenAddr) == 0 || len(c.User) == 0 || len(c.Pass) == 0 { + log.Fatalf("Missing configuration") } - rec := store.NewRecord(voteKey+id, &Vote{ - Id: id, - Message: req.Message, - VotedAt: time.Now(), - }) + oc := open.Config{ + Addresses: []string{c.OpenAddr}, + Username: c.User, + Password: c.Pass, + } + if c.Insecure { + oc.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // For testing only. Use certificate for validation. + } + } - // we don't need to check the error - store.Write(rec) + client, err := open.NewClient(oc) + if err != nil { + log.Fatalf("Error configuring search client %s", err) + } + return &Search{ + conf: c, + client: client, + } +} - rsp.Message = "Thanks for the vote!" +func isValidIndexName(s string) bool { + if len(s) > 1 { + return indexNameRegex.MatchString(s) + } + return shortIndexNameRegex.MatchString(s) +} +func (s *Search) CreateIndex(ctx context.Context, request *pb.CreateIndexRequest, response *pb.CreateIndexResponse) error { + method := "search.CreateIndex" + + // TODO validate name https://opensearch.org/docs/latest/opensearch/rest-api/index-apis/create-index/#index-naming-restrictions + tnt, ok := tenant.FromContext(ctx) + if !ok { + return errors.Unauthorized(method, "Unauthorized") + } + if !isValidIndexName(request.Index) { + return errors.BadRequest(method, "Index name should contain only alphanumerics and hyphens") + } + req := openapi.CreateRequest{ + Index: indexName(tnt, request.Index), + Body: nil, // TODO populate with fields and their types + } + rsp, err := req.Do(ctx, s.client) + if err != nil { + log.Errorf("Error creating index %s", err) + return errors.InternalServerError(method, "Error creating index") + } + defer rsp.Body.Close() + if rsp.IsError() { + log.Errorf("Error creating index %s", rsp.String()) + return errors.InternalServerError(method, "Error creating index") + } + return nil +} + +func indexName(tnt, index string) string { + return fmt.Sprintf("%s-%s", strings.ReplaceAll(tnt, "/", "-"), index) +} + +func (s *Search) Index(ctx context.Context, request *pb.IndexRequest, response *pb.IndexResponse) error { + method := "search.Index" + // TODO validation + // TODO validate name https://opensearch.org/docs/latest/opensearch/rest-api/index-apis/create-index/#index-naming-restrictions + tnt, ok := tenant.FromContext(ctx) + if !ok { + return errors.Unauthorized(method, "Unauthorized") + } + if request.Document == nil { + return errors.BadRequest(method, "Missing document param") + } + if len(request.Document.Id) == 0 { + request.Document.Id = uuid.New().String() + } + if len(request.Index) == 0 { + return errors.BadRequest(method, "Missing index_name param") + } + if !isValidIndexName(request.Index) { + return errors.BadRequest(method, "Index name should contain only alphanumerics and hyphens") + } + if request.Document.Contents == nil { + return errors.BadRequest(method, "Missing document.contents param") + } + + b, err := request.Document.Contents.MarshalJSON() + if err != nil { + return errors.BadRequest(method, "Error processing document") + } + req := openapi.IndexRequest{ + Index: indexName(tnt, request.Index), + DocumentID: request.Document.Id, + Body: bytes.NewBuffer(b), + } + rsp, err := req.Do(ctx, s.client) + if err != nil { + log.Errorf("Error indexing doc %s", err) + return errors.InternalServerError(method, "Error indexing document") + } + defer rsp.Body.Close() + if rsp.IsError() { + log.Errorf("Error indexing doc %s", rsp.String()) + return errors.InternalServerError(method, "Error indexing document") + } + response.Id = req.DocumentID + return nil +} + +func (s *Search) Delete(ctx context.Context, request *pb.DeleteRequest, response *pb.DeleteResponse) error { + method := "search.Delete" + tnt, ok := tenant.FromContext(ctx) + if !ok { + return errors.Unauthorized(method, "Unauthorized") + } + req := openapi.DeleteRequest{ + Index: indexName(tnt, request.Index), + DocumentID: request.Id, + } + rsp, err := req.Do(ctx, s.client) + if err != nil { + log.Errorf("Error deleting doc %s", err) + return errors.InternalServerError(method, "Error deleting document") + } + defer rsp.Body.Close() + if rsp.IsError() { + log.Errorf("Error deleting doc %s", rsp.String()) + return errors.InternalServerError(method, "Error deleting document") + } + return nil +} + +func (s *Search) Search(ctx context.Context, request *pb.SearchRequest, response *pb.SearchResponse) error { + method := "search.Search" + if len(request.Index) == 0 { + return errors.BadRequest(method, "Missing index param") + } + + // Search models to support https://opensearch.org/docs/latest/opensearch/ux/ + // - Simple query + // - Autocomplete (prefix) queries + // - pagination + // - Sorting + // + tnt, ok := tenant.FromContext(ctx) + if !ok { + return errors.Unauthorized(method, "Unauthorized") + } + + // TODO fuzzy + if len(request.Query) == 0 { + return errors.BadRequest(method, "Missing query param") + } + + qs, err := parseQueryString(request.Query) + if err != nil { + log.Errorf("Error parsing string %s %s", request.Query, err) + return errors.BadRequest(method, "%s", err) + } + b, _ := qs.MarshalJSON() + req := openapi.SearchRequest{ + Index: []string{indexName(tnt, request.Index)}, + Body: bytes.NewBuffer(b), + } + rsp, err := req.Do(ctx, s.client) + if err != nil { + log.Errorf("Error searching index %s", err) + return errors.InternalServerError(method, "Error searching documents") + } + defer rsp.Body.Close() + if rsp.IsError() { + if rsp.StatusCode == 404 { // index not found + return errors.NotFound(method, "Index not found") + } + log.Errorf("Error searching index %s", rsp.String()) + return errors.InternalServerError(method, "Error searching documents") + } + b, err = ioutil.ReadAll(rsp.Body) + if err != nil { + log.Errorf("Error searching index %s", rsp.String()) + return errors.InternalServerError(method, "Error searching documents") + } + var os openSearchResponse + if err := json.Unmarshal(b, &os); err != nil { + log.Errorf("Error unmarshalling doc %s", err) + return errors.InternalServerError(method, "Error searching documents") + } + log.Infof("%s", string(b)) + for _, v := range os.Hits.Hits { + vs, err := structpb.NewStruct(v.Source) + if err != nil { + log.Errorf("Error unmarshalling doc %s", err) + return errors.InternalServerError(method, "Error searching documents") + } + response.Documents = append(response.Documents, &pb.Document{ + Id: v.ID, + Contents: vs, + }) + } + return nil +} + +func (s *Search) DeleteIndex(ctx context.Context, request *pb.DeleteIndexRequest, response *pb.DeleteIndexResponse) error { + method := "search.DeleteIndex" + tnt, ok := tenant.FromContext(ctx) + if !ok { + return errors.Unauthorized(method, "Unauthorized") + } + req := openapi.DeleteRequest{ + Index: indexName(tnt, request.Index), + } + rsp, err := req.Do(ctx, s.client) + if err != nil { + log.Errorf("Error deleting index %s", err) + return errors.InternalServerError(method, "Error deleting index") + } + defer rsp.Body.Close() + if rsp.IsError() { + log.Errorf("Error deleting index %s", rsp.String()) + return errors.InternalServerError(method, "Error deleting index") + } return nil } diff --git a/search/main.go b/search/main.go index 589127b..f2ce90f 100644 --- a/search/main.go +++ b/search/main.go @@ -15,7 +15,7 @@ func main() { ) // Register handler - pb.RegisterSearchHandler(srv.Server(), new(handler.Search)) + pb.RegisterSearchHandler(srv.Server(), handler.New(srv)) // Run service if err := srv.Run(); err != nil { diff --git a/search/proto/search.pb.go b/search/proto/search.pb.go index a563fbb..4baf871 100644 --- a/search/proto/search.pb.go +++ b/search/proto/search.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.15.6 +// protoc-gen-go v1.26.0 +// protoc v3.15.5 // source: proto/search.proto package search @@ -9,6 +9,7 @@ package search import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + structpb "google.golang.org/protobuf/types/known/structpb" reflect "reflect" sync "sync" ) @@ -20,18 +21,20 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// Vote to have the Search api launched faster! -type VoteRequest struct { +// Index a document i.e. insert a document to search for. +type IndexRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // optional message - Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` + // The document to index + Document *Document `protobuf:"bytes,1,opt,name=document,proto3" json:"document,omitempty"` + // The index this document belongs to + Index string `protobuf:"bytes,2,opt,name=index,proto3" json:"index,omitempty"` } -func (x *VoteRequest) Reset() { - *x = VoteRequest{} +func (x *IndexRequest) Reset() { + *x = IndexRequest{} if protoimpl.UnsafeEnabled { mi := &file_proto_search_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -39,13 +42,13 @@ func (x *VoteRequest) Reset() { } } -func (x *VoteRequest) String() string { +func (x *IndexRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*VoteRequest) ProtoMessage() {} +func (*IndexRequest) ProtoMessage() {} -func (x *VoteRequest) ProtoReflect() protoreflect.Message { +func (x *IndexRequest) ProtoReflect() protoreflect.Message { mi := &file_proto_search_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -57,29 +60,38 @@ func (x *VoteRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use VoteRequest.ProtoReflect.Descriptor instead. -func (*VoteRequest) Descriptor() ([]byte, []int) { +// Deprecated: Use IndexRequest.ProtoReflect.Descriptor instead. +func (*IndexRequest) Descriptor() ([]byte, []int) { return file_proto_search_proto_rawDescGZIP(), []int{0} } -func (x *VoteRequest) GetMessage() string { +func (x *IndexRequest) GetDocument() *Document { if x != nil { - return x.Message + return x.Document + } + return nil +} + +func (x *IndexRequest) GetIndex() string { + if x != nil { + return x.Index } return "" } -type VoteResponse struct { +type Document struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // response message - Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` + // The ID for this document. If blank, one will be generated + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // The JSON contents of the document + Contents *structpb.Struct `protobuf:"bytes,2,opt,name=contents,proto3" json:"contents,omitempty"` } -func (x *VoteResponse) Reset() { - *x = VoteResponse{} +func (x *Document) Reset() { + *x = Document{} if protoimpl.UnsafeEnabled { mi := &file_proto_search_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -87,13 +99,13 @@ func (x *VoteResponse) Reset() { } } -func (x *VoteResponse) String() string { +func (x *Document) String() string { return protoimpl.X.MessageStringOf(x) } -func (*VoteResponse) ProtoMessage() {} +func (*Document) ProtoMessage() {} -func (x *VoteResponse) ProtoReflect() protoreflect.Message { +func (x *Document) ProtoReflect() protoreflect.Message { mi := &file_proto_search_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -105,34 +117,577 @@ func (x *VoteResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use VoteResponse.ProtoReflect.Descriptor instead. -func (*VoteResponse) Descriptor() ([]byte, []int) { +// Deprecated: Use Document.ProtoReflect.Descriptor instead. +func (*Document) Descriptor() ([]byte, []int) { return file_proto_search_proto_rawDescGZIP(), []int{1} } -func (x *VoteResponse) GetMessage() string { +func (x *Document) GetId() string { if x != nil { - return x.Message + return x.Id } return "" } +func (x *Document) GetContents() *structpb.Struct { + if x != nil { + return x.Contents + } + return nil +} + +type IndexResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *IndexResponse) Reset() { + *x = IndexResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_proto_search_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *IndexResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IndexResponse) ProtoMessage() {} + +func (x *IndexResponse) ProtoReflect() protoreflect.Message { + mi := &file_proto_search_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IndexResponse.ProtoReflect.Descriptor instead. +func (*IndexResponse) Descriptor() ([]byte, []int) { + return file_proto_search_proto_rawDescGZIP(), []int{2} +} + +func (x *IndexResponse) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +// Delete a document given its ID +type DeleteRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The ID of the document to delete + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // The index the document belongs to + Index string `protobuf:"bytes,2,opt,name=index,proto3" json:"index,omitempty"` +} + +func (x *DeleteRequest) Reset() { + *x = DeleteRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_proto_search_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DeleteRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteRequest) ProtoMessage() {} + +func (x *DeleteRequest) ProtoReflect() protoreflect.Message { + mi := &file_proto_search_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteRequest.ProtoReflect.Descriptor instead. +func (*DeleteRequest) Descriptor() ([]byte, []int) { + return file_proto_search_proto_rawDescGZIP(), []int{3} +} + +func (x *DeleteRequest) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *DeleteRequest) GetIndex() string { + if x != nil { + return x.Index + } + return "" +} + +type DeleteResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *DeleteResponse) Reset() { + *x = DeleteResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_proto_search_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DeleteResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteResponse) ProtoMessage() {} + +func (x *DeleteResponse) ProtoReflect() protoreflect.Message { + mi := &file_proto_search_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteResponse.ProtoReflect.Descriptor instead. +func (*DeleteResponse) Descriptor() ([]byte, []int) { + return file_proto_search_proto_rawDescGZIP(), []int{4} +} + +// Search for documents in a given in index +type SearchRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The index the document belongs to + Index string `protobuf:"bytes,1,opt,name=index,proto3" json:"index,omitempty"` + // The query. See docs for query language examples + Query string `protobuf:"bytes,2,opt,name=query,proto3" json:"query,omitempty"` +} + +func (x *SearchRequest) Reset() { + *x = SearchRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_proto_search_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SearchRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SearchRequest) ProtoMessage() {} + +func (x *SearchRequest) ProtoReflect() protoreflect.Message { + mi := &file_proto_search_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SearchRequest.ProtoReflect.Descriptor instead. +func (*SearchRequest) Descriptor() ([]byte, []int) { + return file_proto_search_proto_rawDescGZIP(), []int{5} +} + +func (x *SearchRequest) GetIndex() string { + if x != nil { + return x.Index + } + return "" +} + +func (x *SearchRequest) GetQuery() string { + if x != nil { + return x.Query + } + return "" +} + +type SearchResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The matching documents + Documents []*Document `protobuf:"bytes,1,rep,name=documents,proto3" json:"documents,omitempty"` +} + +func (x *SearchResponse) Reset() { + *x = SearchResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_proto_search_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SearchResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SearchResponse) ProtoMessage() {} + +func (x *SearchResponse) ProtoReflect() protoreflect.Message { + mi := &file_proto_search_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SearchResponse.ProtoReflect.Descriptor instead. +func (*SearchResponse) Descriptor() ([]byte, []int) { + return file_proto_search_proto_rawDescGZIP(), []int{6} +} + +func (x *SearchResponse) GetDocuments() []*Document { + if x != nil { + return x.Documents + } + return nil +} + +// Create a search index by specifying which fields are to be queried +type CreateIndexRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // the name of the index + Index string `protobuf:"bytes,1,opt,name=index,proto3" json:"index,omitempty"` + Fields []*Field `protobuf:"bytes,2,rep,name=fields,proto3" json:"fields,omitempty"` +} + +func (x *CreateIndexRequest) Reset() { + *x = CreateIndexRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_proto_search_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CreateIndexRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateIndexRequest) ProtoMessage() {} + +func (x *CreateIndexRequest) ProtoReflect() protoreflect.Message { + mi := &file_proto_search_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateIndexRequest.ProtoReflect.Descriptor instead. +func (*CreateIndexRequest) Descriptor() ([]byte, []int) { + return file_proto_search_proto_rawDescGZIP(), []int{7} +} + +func (x *CreateIndexRequest) GetIndex() string { + if x != nil { + return x.Index + } + return "" +} + +func (x *CreateIndexRequest) GetFields() []*Field { + if x != nil { + return x.Fields + } + return nil +} + +type Field struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The name of the field. Use a `.` separator to define nested fields e.g. foo.bar + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // The type of the field - string, number + Type string `protobuf:"bytes,2,opt,name=type,proto3" json:"type,omitempty"` +} + +func (x *Field) Reset() { + *x = Field{} + if protoimpl.UnsafeEnabled { + mi := &file_proto_search_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Field) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Field) ProtoMessage() {} + +func (x *Field) ProtoReflect() protoreflect.Message { + mi := &file_proto_search_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Field.ProtoReflect.Descriptor instead. +func (*Field) Descriptor() ([]byte, []int) { + return file_proto_search_proto_rawDescGZIP(), []int{8} +} + +func (x *Field) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Field) GetType() string { + if x != nil { + return x.Type + } + return "" +} + +type CreateIndexResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *CreateIndexResponse) Reset() { + *x = CreateIndexResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_proto_search_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CreateIndexResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateIndexResponse) ProtoMessage() {} + +func (x *CreateIndexResponse) ProtoReflect() protoreflect.Message { + mi := &file_proto_search_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateIndexResponse.ProtoReflect.Descriptor instead. +func (*CreateIndexResponse) Descriptor() ([]byte, []int) { + return file_proto_search_proto_rawDescGZIP(), []int{9} +} + +// Delete an index. +type DeleteIndexRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The name of the index to delete + Index string `protobuf:"bytes,1,opt,name=index,proto3" json:"index,omitempty"` +} + +func (x *DeleteIndexRequest) Reset() { + *x = DeleteIndexRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_proto_search_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DeleteIndexRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteIndexRequest) ProtoMessage() {} + +func (x *DeleteIndexRequest) ProtoReflect() protoreflect.Message { + mi := &file_proto_search_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteIndexRequest.ProtoReflect.Descriptor instead. +func (*DeleteIndexRequest) Descriptor() ([]byte, []int) { + return file_proto_search_proto_rawDescGZIP(), []int{10} +} + +func (x *DeleteIndexRequest) GetIndex() string { + if x != nil { + return x.Index + } + return "" +} + +type DeleteIndexResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *DeleteIndexResponse) Reset() { + *x = DeleteIndexResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_proto_search_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DeleteIndexResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteIndexResponse) ProtoMessage() {} + +func (x *DeleteIndexResponse) ProtoReflect() protoreflect.Message { + mi := &file_proto_search_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteIndexResponse.ProtoReflect.Descriptor instead. +func (*DeleteIndexResponse) Descriptor() ([]byte, []int) { + return file_proto_search_proto_rawDescGZIP(), []int{11} +} + var File_proto_search_proto protoreflect.FileDescriptor var file_proto_search_proto_rawDesc = []byte{ 0x0a, 0x12, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x22, 0x27, 0x0a, 0x0b, - 0x56, 0x6f, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x6d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, - 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x28, 0x0a, 0x0c, 0x56, 0x6f, 0x74, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x32, - 0x3d, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x33, 0x0a, 0x04, 0x56, 0x6f, 0x74, - 0x65, 0x12, 0x13, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x56, 0x6f, 0x74, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, - 0x56, 0x6f, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x10, - 0x5a, 0x0e, 0x2e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3b, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x1a, 0x1c, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x73, 0x74, + 0x72, 0x75, 0x63, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x52, 0x0a, 0x0c, 0x49, 0x6e, + 0x64, 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2c, 0x0a, 0x08, 0x64, 0x6f, + 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x73, + 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x44, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x08, + 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x64, 0x65, + 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x22, 0x4f, + 0x0a, 0x08, 0x44, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x33, 0x0a, 0x08, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, + 0x74, 0x72, 0x75, 0x63, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x22, + 0x1f, 0x0a, 0x0d, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, + 0x22, 0x35, 0x0a, 0x0d, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, + 0x64, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x22, 0x10, 0x0a, 0x0e, 0x44, 0x65, 0x6c, 0x65, 0x74, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x3b, 0x0a, 0x0d, 0x53, 0x65, 0x61, + 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, + 0x64, 0x65, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78, + 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x22, 0x40, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x09, 0x64, 0x6f, 0x63, 0x75, + 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x73, 0x65, + 0x61, 0x72, 0x63, 0x68, 0x2e, 0x44, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x09, 0x64, + 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x51, 0x0a, 0x12, 0x43, 0x72, 0x65, 0x61, + 0x74, 0x65, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, + 0x0a, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, + 0x6e, 0x64, 0x65, 0x78, 0x12, 0x25, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x46, 0x69, + 0x65, 0x6c, 0x64, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x22, 0x2f, 0x0a, 0x05, 0x46, + 0x69, 0x65, 0x6c, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x15, 0x0a, 0x13, + 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x2a, 0x0a, 0x12, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x6e, 0x64, + 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x64, + 0x65, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x22, + 0x15, 0x0a, 0x13, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x80, 0x02, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, + 0x68, 0x12, 0x36, 0x0a, 0x05, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x14, 0x2e, 0x73, 0x65, 0x61, + 0x72, 0x63, 0x68, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x15, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x12, 0x15, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x73, 0x65, 0x61, + 0x72, 0x63, 0x68, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x15, + 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x53, + 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, + 0x48, 0x0a, 0x0b, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x1a, + 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x6e, + 0x64, 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x73, 0x65, 0x61, + 0x72, 0x63, 0x68, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x10, 0x5a, 0x0e, 0x2e, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x3b, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( @@ -147,19 +702,40 @@ func file_proto_search_proto_rawDescGZIP() []byte { return file_proto_search_proto_rawDescData } -var file_proto_search_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_proto_search_proto_msgTypes = make([]protoimpl.MessageInfo, 12) var file_proto_search_proto_goTypes = []interface{}{ - (*VoteRequest)(nil), // 0: search.VoteRequest - (*VoteResponse)(nil), // 1: search.VoteResponse + (*IndexRequest)(nil), // 0: search.IndexRequest + (*Document)(nil), // 1: search.Document + (*IndexResponse)(nil), // 2: search.IndexResponse + (*DeleteRequest)(nil), // 3: search.DeleteRequest + (*DeleteResponse)(nil), // 4: search.DeleteResponse + (*SearchRequest)(nil), // 5: search.SearchRequest + (*SearchResponse)(nil), // 6: search.SearchResponse + (*CreateIndexRequest)(nil), // 7: search.CreateIndexRequest + (*Field)(nil), // 8: search.Field + (*CreateIndexResponse)(nil), // 9: search.CreateIndexResponse + (*DeleteIndexRequest)(nil), // 10: search.DeleteIndexRequest + (*DeleteIndexResponse)(nil), // 11: search.DeleteIndexResponse + (*structpb.Struct)(nil), // 12: google.protobuf.Struct } var file_proto_search_proto_depIdxs = []int32{ - 0, // 0: search.Search.Vote:input_type -> search.VoteRequest - 1, // 1: search.Search.Vote:output_type -> search.VoteResponse - 1, // [1:2] is the sub-list for method output_type - 0, // [0:1] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name + 1, // 0: search.IndexRequest.document:type_name -> search.Document + 12, // 1: search.Document.contents:type_name -> google.protobuf.Struct + 1, // 2: search.SearchResponse.documents:type_name -> search.Document + 8, // 3: search.CreateIndexRequest.fields:type_name -> search.Field + 0, // 4: search.Search.Index:input_type -> search.IndexRequest + 3, // 5: search.Search.Delete:input_type -> search.DeleteRequest + 5, // 6: search.Search.Search:input_type -> search.SearchRequest + 10, // 7: search.Search.DeleteIndex:input_type -> search.DeleteIndexRequest + 2, // 8: search.Search.Index:output_type -> search.IndexResponse + 4, // 9: search.Search.Delete:output_type -> search.DeleteResponse + 6, // 10: search.Search.Search:output_type -> search.SearchResponse + 11, // 11: search.Search.DeleteIndex:output_type -> search.DeleteIndexResponse + 8, // [8:12] is the sub-list for method output_type + 4, // [4:8] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name } func init() { file_proto_search_proto_init() } @@ -169,7 +745,7 @@ func file_proto_search_proto_init() { } if !protoimpl.UnsafeEnabled { file_proto_search_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VoteRequest); i { + switch v := v.(*IndexRequest); i { case 0: return &v.state case 1: @@ -181,7 +757,127 @@ func file_proto_search_proto_init() { } } file_proto_search_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*VoteResponse); i { + switch v := v.(*Document); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_proto_search_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*IndexResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_proto_search_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DeleteRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_proto_search_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DeleteResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_proto_search_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SearchRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_proto_search_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SearchResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_proto_search_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CreateIndexRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_proto_search_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Field); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_proto_search_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CreateIndexResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_proto_search_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DeleteIndexRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_proto_search_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DeleteIndexResponse); i { case 0: return &v.state case 1: @@ -199,7 +895,7 @@ func file_proto_search_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_proto_search_proto_rawDesc, NumEnums: 0, - NumMessages: 2, + NumMessages: 12, NumExtensions: 0, NumServices: 1, }, diff --git a/search/proto/search.pb.micro.go b/search/proto/search.pb.micro.go index 735ea1a..a0560ea 100644 --- a/search/proto/search.pb.micro.go +++ b/search/proto/search.pb.micro.go @@ -6,6 +6,7 @@ package search import ( fmt "fmt" proto "github.com/golang/protobuf/proto" + _ "google.golang.org/protobuf/types/known/structpb" math "math" ) @@ -42,7 +43,10 @@ func NewSearchEndpoints() []*api.Endpoint { // Client API for Search service type SearchService interface { - Vote(ctx context.Context, in *VoteRequest, opts ...client.CallOption) (*VoteResponse, error) + Index(ctx context.Context, in *IndexRequest, opts ...client.CallOption) (*IndexResponse, error) + Delete(ctx context.Context, in *DeleteRequest, opts ...client.CallOption) (*DeleteResponse, error) + Search(ctx context.Context, in *SearchRequest, opts ...client.CallOption) (*SearchResponse, error) + DeleteIndex(ctx context.Context, in *DeleteIndexRequest, opts ...client.CallOption) (*DeleteIndexResponse, error) } type searchService struct { @@ -57,9 +61,39 @@ func NewSearchService(name string, c client.Client) SearchService { } } -func (c *searchService) Vote(ctx context.Context, in *VoteRequest, opts ...client.CallOption) (*VoteResponse, error) { - req := c.c.NewRequest(c.name, "Search.Vote", in) - out := new(VoteResponse) +func (c *searchService) Index(ctx context.Context, in *IndexRequest, opts ...client.CallOption) (*IndexResponse, error) { + req := c.c.NewRequest(c.name, "Search.Index", in) + out := new(IndexResponse) + err := c.c.Call(ctx, req, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *searchService) Delete(ctx context.Context, in *DeleteRequest, opts ...client.CallOption) (*DeleteResponse, error) { + req := c.c.NewRequest(c.name, "Search.Delete", in) + out := new(DeleteResponse) + err := c.c.Call(ctx, req, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *searchService) Search(ctx context.Context, in *SearchRequest, opts ...client.CallOption) (*SearchResponse, error) { + req := c.c.NewRequest(c.name, "Search.Search", in) + out := new(SearchResponse) + err := c.c.Call(ctx, req, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *searchService) DeleteIndex(ctx context.Context, in *DeleteIndexRequest, opts ...client.CallOption) (*DeleteIndexResponse, error) { + req := c.c.NewRequest(c.name, "Search.DeleteIndex", in) + out := new(DeleteIndexResponse) err := c.c.Call(ctx, req, out, opts...) if err != nil { return nil, err @@ -70,12 +104,18 @@ func (c *searchService) Vote(ctx context.Context, in *VoteRequest, opts ...clien // Server API for Search service type SearchHandler interface { - Vote(context.Context, *VoteRequest, *VoteResponse) error + Index(context.Context, *IndexRequest, *IndexResponse) error + Delete(context.Context, *DeleteRequest, *DeleteResponse) error + Search(context.Context, *SearchRequest, *SearchResponse) error + DeleteIndex(context.Context, *DeleteIndexRequest, *DeleteIndexResponse) error } func RegisterSearchHandler(s server.Server, hdlr SearchHandler, opts ...server.HandlerOption) error { type search interface { - Vote(ctx context.Context, in *VoteRequest, out *VoteResponse) error + Index(ctx context.Context, in *IndexRequest, out *IndexResponse) error + Delete(ctx context.Context, in *DeleteRequest, out *DeleteResponse) error + Search(ctx context.Context, in *SearchRequest, out *SearchResponse) error + DeleteIndex(ctx context.Context, in *DeleteIndexRequest, out *DeleteIndexResponse) error } type Search struct { search @@ -88,6 +128,18 @@ type searchHandler struct { SearchHandler } -func (h *searchHandler) Vote(ctx context.Context, in *VoteRequest, out *VoteResponse) error { - return h.SearchHandler.Vote(ctx, in, out) +func (h *searchHandler) Index(ctx context.Context, in *IndexRequest, out *IndexResponse) error { + return h.SearchHandler.Index(ctx, in, out) +} + +func (h *searchHandler) Delete(ctx context.Context, in *DeleteRequest, out *DeleteResponse) error { + return h.SearchHandler.Delete(ctx, in, out) +} + +func (h *searchHandler) Search(ctx context.Context, in *SearchRequest, out *SearchResponse) error { + return h.SearchHandler.Search(ctx, in, out) +} + +func (h *searchHandler) DeleteIndex(ctx context.Context, in *DeleteIndexRequest, out *DeleteIndexResponse) error { + return h.SearchHandler.DeleteIndex(ctx, in, out) } diff --git a/search/proto/search.proto b/search/proto/search.proto index 2231e7d..6450fb3 100644 --- a/search/proto/search.proto +++ b/search/proto/search.proto @@ -1,20 +1,87 @@ syntax = "proto3"; +import "google/protobuf/struct.proto"; + package search; option go_package = "./proto;search"; service Search { - rpc Vote(VoteRequest) returns (VoteResponse) {} + // TODO reinstate when we have a reason for more fine grained control of index creation, for now just rely on lazy creation on first index call + // rpc CreateIndex(CreateIndexRequest) returns (CreateIndexResponse) {} + + rpc Index(IndexRequest) returns (IndexResponse) {} + rpc Delete(DeleteRequest) returns (DeleteResponse) {} + rpc Search(SearchRequest) returns (SearchResponse) {} + rpc DeleteIndex(DeleteIndexRequest) returns (DeleteIndexResponse) {} + } -// Vote to have the Search api launched faster! -message VoteRequest { - // optional message - string message = 1; +// Index a document i.e. insert a document to search for. +message IndexRequest { + // The document to index + Document document = 1; + // The index this document belongs to + string index = 2; + } -message VoteResponse { - // response message - string message = 2; +message Document { + // The ID for this document. If blank, one will be generated + string id = 1; + // The JSON contents of the document + google.protobuf.Struct contents = 2; } + +message IndexResponse { + string id = 1; +} + +// Delete a document given its ID +message DeleteRequest { + // The ID of the document to delete + string id = 1; + // The index the document belongs to + string index = 2; +} + +message DeleteResponse {} + +// Search for documents in a given in index +message SearchRequest { + // The index the document belongs to + string index = 1; + + // The query. See docs for query language examples + string query = 2; +} + +message SearchResponse { + // The matching documents + repeated Document documents = 1; + +} + +// Create a search index by specifying which fields are to be queried +message CreateIndexRequest { + // the name of the index + string index = 1; + repeated Field fields = 2; +} + +message Field { + // The name of the field. Use a `.` separator to define nested fields e.g. foo.bar + string name = 1; + // The type of the field - string, number + string type = 2; +} + +message CreateIndexResponse {} + +// Delete an index. +message DeleteIndexRequest { + // The name of the index to delete + string index = 1; +} + +message DeleteIndexResponse {} diff --git a/search/publicapi.json b/search/publicapi.json index 4d62c1c..f88a43c 100644 --- a/search/publicapi.json +++ b/search/publicapi.json @@ -1,6 +1,6 @@ { "name": "search", "icon": "🔎", - "category": "coming soon", - "display_name": "Search (Coming Soon)" + "category": "search", + "display_name": "Search" } diff --git a/space/handler/space_test.go b/space/handler/space_test.go index fb65430..0244243 100644 --- a/space/handler/space_test.go +++ b/space/handler/space_test.go @@ -85,24 +85,23 @@ func (m mockError) OrigErr() error { } func TestCreate(t *testing.T) { - g := NewWithT(t) tcs := []struct { name string objName string visibility string err error url string - head func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) - put func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) + head func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error) + put func(input *sthree.PutObjectInput, g *WithT) (*sthree.PutObjectOutput, error) }{ { name: "Simple case", objName: "foo.jpg", url: "", - head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) { + head: func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error) { return nil, mockError{code: "NotFound"} }, - put: func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) { + put: func(input *sthree.PutObjectInput, g *WithT) (*sthree.PutObjectOutput, error) { g.Expect(*input.Bucket).To(Equal("my-space")) g.Expect(input.ACL).To(BeNil()) return &sthree.PutObjectOutput{}, nil @@ -113,10 +112,10 @@ func TestCreate(t *testing.T) { objName: "bar/baz/foo.jpg", visibility: "public", url: "https://my-space.ams3.example.com/micro/123/bar/baz/foo.jpg", - head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) { + head: func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error) { return nil, mockError{code: "NotFound"} }, - put: func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) { + put: func(input *sthree.PutObjectInput, g *WithT) (*sthree.PutObjectOutput, error) { g.Expect(*input.Bucket).To(Equal("my-space")) g.Expect(*input.ACL).To(Equal(mdACLPublic)) return &sthree.PutObjectOutput{}, nil @@ -131,7 +130,7 @@ func TestCreate(t *testing.T) { name: "Already exists", objName: "foo.jpg", err: errors.BadRequest("space.Create", "Object already exists"), - head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) { + head: func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error) { g.Expect(*input.Bucket).To(Equal("my-space")) g.Expect(*input.Key).To(Equal("micro/123/foo.jpg")) return &sthree.HeadObjectOutput{}, nil @@ -142,6 +141,7 @@ func TestCreate(t *testing.T) { for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) handler := Space{ conf: conf{ AccessKey: "access", @@ -152,7 +152,13 @@ func TestCreate(t *testing.T) { Region: "ams3", BaseURL: "https://my-space.ams3.example.com", }, - client: &mockS3Client{head: tc.head, put: tc.put}, + client: &mockS3Client{ + head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) { + return tc.head(input, g) + }, + put: func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) { + return tc.put(input, g) + }}, } ctx := context.Background() ctx = auth.ContextWithAccount(ctx, &auth.Account{ @@ -182,24 +188,23 @@ func TestCreate(t *testing.T) { } func TestUpdate(t *testing.T) { - g := NewWithT(t) tcs := []struct { name string objName string visibility string err error url string - head func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) - put func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) + head func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error) + put func(input *sthree.PutObjectInput, g *WithT) (*sthree.PutObjectOutput, error) }{ { name: "Does not exist", objName: "foo.jpg", url: "", - head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) { + head: func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error) { return nil, mockError{code: "NotFound"} }, - put: func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) { + put: func(input *sthree.PutObjectInput, g *WithT) (*sthree.PutObjectOutput, error) { g.Expect(*input.Bucket).To(Equal("my-space")) g.Expect(input.ACL).To(BeNil()) return &sthree.PutObjectOutput{}, nil @@ -210,10 +215,10 @@ func TestUpdate(t *testing.T) { objName: "bar/baz/foo.jpg", visibility: "public", url: "https://my-space.ams3.example.com/micro/123/bar/baz/foo.jpg", - head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) { + head: func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error) { return nil, mockError{code: "NotFound"} }, - put: func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) { + put: func(input *sthree.PutObjectInput, g *WithT) (*sthree.PutObjectOutput, error) { g.Expect(*input.Bucket).To(Equal("my-space")) g.Expect(*input.ACL).To(Equal(mdACLPublic)) return &sthree.PutObjectOutput{}, nil @@ -227,12 +232,12 @@ func TestUpdate(t *testing.T) { { name: "Already exists", objName: "foo.jpg", - head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) { + head: func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error) { g.Expect(*input.Bucket).To(Equal("my-space")) g.Expect(*input.Key).To(Equal("micro/123/foo.jpg")) return &sthree.HeadObjectOutput{}, nil }, - put: func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) { + put: func(input *sthree.PutObjectInput, g *WithT) (*sthree.PutObjectOutput, error) { g.Expect(*input.Bucket).To(Equal("my-space")) g.Expect(input.ACL).To(BeNil()) return &sthree.PutObjectOutput{}, nil @@ -242,12 +247,12 @@ func TestUpdate(t *testing.T) { { name: "Already exists public", objName: "foo.jpg", - head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) { + head: func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error) { g.Expect(*input.Bucket).To(Equal("my-space")) g.Expect(*input.Key).To(Equal("micro/123/foo.jpg")) return &sthree.HeadObjectOutput{}, nil }, - put: func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) { + put: func(input *sthree.PutObjectInput, g *WithT) (*sthree.PutObjectOutput, error) { g.Expect(*input.Bucket).To(Equal("my-space")) g.Expect(*input.ACL).To(Equal(mdACLPublic)) return &sthree.PutObjectOutput{}, nil @@ -260,6 +265,7 @@ func TestUpdate(t *testing.T) { for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) handler := Space{ conf: conf{ AccessKey: "access", @@ -270,7 +276,13 @@ func TestUpdate(t *testing.T) { Region: "ams3", BaseURL: "https://my-space.ams3.example.com", }, - client: &mockS3Client{head: tc.head, put: tc.put}, + client: &mockS3Client{ + head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) { + return tc.head(input, g) + }, + put: func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) { + return tc.put(input, g) + }}, } ctx := context.Background() ctx = auth.ContextWithAccount(ctx, &auth.Account{ @@ -301,7 +313,6 @@ func TestUpdate(t *testing.T) { } func TestDelete(t *testing.T) { - g := NewWithT(t) tcs := []struct { name string objName string @@ -322,6 +333,7 @@ func TestDelete(t *testing.T) { for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) handler := Space{ conf: conf{ AccessKey: "access", @@ -364,7 +376,6 @@ func TestDelete(t *testing.T) { } func TestList(t *testing.T) { - g := NewWithT(t) tcs := []struct { name string prefix string @@ -383,6 +394,7 @@ func TestList(t *testing.T) { for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) store.DefaultStore = memory.NewStore() store.Write( store.NewRecord(fmt.Sprintf("%s/micro/123/file.jpg", prefixByUser), @@ -463,7 +475,6 @@ func TestList(t *testing.T) { } func TestHead(t *testing.T) { - g := NewWithT(t) tcs := []struct { name string objectName string @@ -472,7 +483,7 @@ func TestHead(t *testing.T) { modified string created string err error - head func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) + head func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error) }{ { name: "Simple case", @@ -481,7 +492,7 @@ func TestHead(t *testing.T) { url: "https://my-space.ams3.example.com/micro/123/foo.jpg", created: "2009-11-10T23:00:00Z", modified: "2009-11-10T23:00:00Z", - head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) { + head: func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error) { g.Expect(*input.Bucket).To(Equal("my-space")) g.Expect(*input.Key).To(Equal("micro/123/foo.jpg")) @@ -497,7 +508,7 @@ func TestHead(t *testing.T) { url: "", created: "2009-11-10T23:00:00Z", modified: "2009-11-10T23:00:00Z", - head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) { + head: func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error) { g.Expect(*input.Bucket).To(Equal("my-space")) g.Expect(*input.Key).To(Equal("micro/123/foo.jpg")) @@ -514,7 +525,7 @@ func TestHead(t *testing.T) { name: "Not found", objectName: "foo.jpg", err: errors.BadRequest("space.Head", "Object not found"), - head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) { + head: func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error) { return nil, mockError{code: "NotFound"} }, }, @@ -522,6 +533,7 @@ func TestHead(t *testing.T) { for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) store.DefaultStore = memory.NewStore() store.Write(store.NewRecord(fmt.Sprintf("%s/micro/123/%s", prefixByUser, tc.objectName), meta{ Visibility: tc.visibility, @@ -539,7 +551,9 @@ func TestHead(t *testing.T) { BaseURL: "https://my-space.ams3.example.com", }, client: &mockS3Client{ - head: tc.head, + head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) { + return tc.head(input, g) + }, }, } ctx := context.Background()