mirror of
https://github.com/kevin-DL/services.git
synced 2026-01-19 22:15:24 +00:00
Search API (#350)
This commit is contained in:
Dominic Wong
287
search/handler/lexer.go
Normal file
287
search/handler/lexer.go
Normal file
@@ -0,0 +1,287 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
type itemType int
|
||||
|
||||
const (
|
||||
itemError itemType = iota
|
||||
itemNumber
|
||||
itemIdentifier
|
||||
itemBoolean
|
||||
itemBooleanOp
|
||||
itemString
|
||||
itemOperator
|
||||
itemLeftParen
|
||||
itemRightParen
|
||||
)
|
||||
|
||||
const (
|
||||
eof = -1
|
||||
)
|
||||
|
||||
type item struct {
|
||||
typ itemType
|
||||
val string
|
||||
}
|
||||
|
||||
type stateFn func(*lexer) stateFn
|
||||
|
||||
func (l *lexer) run() {
|
||||
for state := lexStartStatement; state != nil; {
|
||||
state = state(l)
|
||||
}
|
||||
close(l.items)
|
||||
}
|
||||
|
||||
type lexer struct {
|
||||
name string //used only for error reports
|
||||
input string // the string being scanned
|
||||
start int // start position of this item
|
||||
pos int // current position in the input
|
||||
width int // width of last rune read
|
||||
items chan item // channel of scanned items
|
||||
|
||||
}
|
||||
|
||||
func lex(name, input string) (*lexer, chan item) {
|
||||
l := &lexer{
|
||||
name: name,
|
||||
input: input,
|
||||
items: make(chan item),
|
||||
}
|
||||
go l.run()
|
||||
return l, l.items
|
||||
}
|
||||
|
||||
func (l *lexer) emit(t itemType) {
|
||||
l.items <- item{t, l.input[l.start:l.pos]}
|
||||
l.start = l.pos
|
||||
}
|
||||
|
||||
func (l *lexer) errorf(format string, args ...interface{}) stateFn {
|
||||
l.items <- item{
|
||||
itemError,
|
||||
fmt.Sprintf(format, args...),
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *lexer) consumeSpace() {
|
||||
l.acceptRun(" ")
|
||||
l.ignore()
|
||||
}
|
||||
|
||||
// next returns the next rune in the input
|
||||
func (l *lexer) next() (rune int32) {
|
||||
if l.pos == len(l.input) {
|
||||
l.width = 0
|
||||
return eof
|
||||
}
|
||||
rune, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
|
||||
l.pos += l.width
|
||||
return rune
|
||||
}
|
||||
|
||||
// ignore skips over the pending input before this point
|
||||
func (l *lexer) ignore() {
|
||||
l.start = l.pos
|
||||
}
|
||||
|
||||
// backup steps back one rune
|
||||
// Can be called only once per call of next
|
||||
func (l *lexer) backup() {
|
||||
l.pos -= l.width
|
||||
}
|
||||
|
||||
// peek returns but does not consume
|
||||
// the next rune in the input
|
||||
func (l *lexer) peek() int32 {
|
||||
rune := l.next()
|
||||
l.backup()
|
||||
return rune
|
||||
}
|
||||
|
||||
// accept consumes the next rune
|
||||
// if it's from the valid set
|
||||
func (l *lexer) accept(valid string) bool {
|
||||
if strings.IndexRune(valid, l.next()) >= 0 {
|
||||
return true
|
||||
}
|
||||
l.backup()
|
||||
return false
|
||||
}
|
||||
|
||||
// acceptRune consumes a run of runes from the valid set
|
||||
func (l *lexer) acceptRun(valid string) {
|
||||
for strings.IndexRune(valid, l.next()) >= 0 {
|
||||
|
||||
}
|
||||
l.backup()
|
||||
}
|
||||
|
||||
func lexIdent(l *lexer) stateFn {
|
||||
l.consumeSpace()
|
||||
for {
|
||||
r := l.next()
|
||||
if r == eof {
|
||||
return l.errorf("Unexpected end of input %q", l.input[l.start:])
|
||||
}
|
||||
|
||||
if unicode.IsSpace(r) || strings.IndexRune("=><", r) >= 0 {
|
||||
l.backup()
|
||||
break
|
||||
}
|
||||
}
|
||||
if l.pos > l.start {
|
||||
l.emit(itemIdentifier)
|
||||
}
|
||||
return lexOperator(l)
|
||||
}
|
||||
|
||||
func lexValue(l *lexer) stateFn {
|
||||
l.consumeSpace()
|
||||
switch {
|
||||
case l.accept(`"'`):
|
||||
l.backup()
|
||||
return lexString(l)
|
||||
case strings.HasPrefix(l.input[l.start:], "true"), strings.HasPrefix(l.input[l.start:], "false"):
|
||||
return lexBool(l)
|
||||
default:
|
||||
// try it as a number
|
||||
return lexNumber(l)
|
||||
}
|
||||
}
|
||||
|
||||
func lexString(l *lexer) stateFn {
|
||||
// TODO support single and double quotes with escaping
|
||||
if !l.accept(`"'`) {
|
||||
return l.errorf("Unexpected value %v, expected a quote", l.peek())
|
||||
}
|
||||
// ignore the quote
|
||||
openQuote := l.input[l.start:l.pos]
|
||||
l.ignore()
|
||||
lastRead := ""
|
||||
for {
|
||||
r := l.next()
|
||||
if r == eof { // should only happen in error case
|
||||
return l.errorf("Unexpected value %v, incorrectly terminated value %s %v", l.input[l.start:], openQuote, lastRead)
|
||||
}
|
||||
if string(r) == openQuote && lastRead != `\` {
|
||||
l.backup()
|
||||
l.emit(itemString)
|
||||
l.next()
|
||||
l.ignore() // ignore the quote
|
||||
return lexEndStatement(l)
|
||||
}
|
||||
lastRead = string(r)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
operatorEquals = `==`
|
||||
operatorGreater = `>=`
|
||||
operatorLess = `<=`
|
||||
parenLeft = `(`
|
||||
parenRight = `)`
|
||||
)
|
||||
|
||||
func lexOperator(l *lexer) stateFn {
|
||||
l.consumeSpace()
|
||||
switch l.input[l.pos : l.pos+2] {
|
||||
case operatorEquals, operatorGreater, operatorLess:
|
||||
l.pos += 2
|
||||
l.emit(itemOperator)
|
||||
return lexValue(l)
|
||||
}
|
||||
// look for identifier
|
||||
return l.errorf("Unexpected operator %q", l.input[l.pos:l.pos+2])
|
||||
}
|
||||
|
||||
func lexNumber(l *lexer) stateFn {
|
||||
l.consumeSpace()
|
||||
// optional leading sign
|
||||
l.accept("+-")
|
||||
digits := "0123456789"
|
||||
if l.accept("0") && l.accept("xX") {
|
||||
digits = "0123456789abcdefABCDEF"
|
||||
}
|
||||
l.acceptRun(digits)
|
||||
if l.accept(".") {
|
||||
l.acceptRun(digits)
|
||||
}
|
||||
if l.accept("eE") {
|
||||
l.accept("+-")
|
||||
l.acceptRun("0123456789")
|
||||
}
|
||||
if l.start == l.pos {
|
||||
return l.errorf("Unexpected value %s", l.input[l.start:])
|
||||
}
|
||||
l.emit(itemNumber)
|
||||
return lexEndStatement(l)
|
||||
}
|
||||
|
||||
func lexStartStatement(l *lexer) stateFn {
|
||||
l.consumeSpace()
|
||||
if string(l.peek()) == parenLeft {
|
||||
l.next()
|
||||
l.emit(itemLeftParen)
|
||||
return lexStartStatement(l)
|
||||
}
|
||||
return lexIdent(l)
|
||||
}
|
||||
|
||||
func lexEndStatement(l *lexer) stateFn {
|
||||
// is this the end of the statement or can we find a boolean op
|
||||
l.consumeSpace()
|
||||
if string(l.peek()) == parenRight {
|
||||
l.next()
|
||||
l.emit(itemRightParen)
|
||||
return lexEndStatement(l)
|
||||
}
|
||||
|
||||
for {
|
||||
r := l.next()
|
||||
if r == eof {
|
||||
break
|
||||
}
|
||||
if unicode.IsSpace(r) {
|
||||
l.backup()
|
||||
break
|
||||
}
|
||||
}
|
||||
if l.start == l.pos {
|
||||
return nil
|
||||
}
|
||||
|
||||
if l.input[l.start:l.pos] == "and" || l.input[l.start:l.pos] == "AND" || l.input[l.start:l.pos] == "or" || l.input[l.start:l.pos] == "OR" {
|
||||
l.emit(itemBooleanOp)
|
||||
return lexStartStatement(l)
|
||||
}
|
||||
return l.errorf("Unexpected input %v", l.input[l.start:l.pos])
|
||||
}
|
||||
|
||||
func lexBool(l *lexer) stateFn {
|
||||
for {
|
||||
r := l.next()
|
||||
if r == eof {
|
||||
break
|
||||
}
|
||||
if unicode.IsSpace(r) {
|
||||
l.backup()
|
||||
break
|
||||
}
|
||||
}
|
||||
if l.input[l.start:l.pos] == "true" || l.input[l.start:l.pos] == "false" {
|
||||
l.emit(itemBoolean)
|
||||
return lexEndStatement(l)
|
||||
}
|
||||
|
||||
return l.errorf("Unexpected value %q, expecting a boolean", l.input[l.start:l.pos])
|
||||
}
|
||||
499
search/handler/lexer_test.go
Normal file
499
search/handler/lexer_test.go
Normal file
@@ -0,0 +1,499 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestLexer(t *testing.T) {
|
||||
|
||||
tcs := []struct {
|
||||
name string
|
||||
input string
|
||||
tokens []item
|
||||
err error
|
||||
}{
|
||||
{
|
||||
name: "basic",
|
||||
input: `foo == "bar"`,
|
||||
tokens: []item{
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "foo",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: "==",
|
||||
},
|
||||
{
|
||||
typ: itemString,
|
||||
val: `bar`,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "basic",
|
||||
input: `first_name == 'Dom'`,
|
||||
tokens: []item{
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "first_name",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: "==",
|
||||
},
|
||||
{
|
||||
typ: itemString,
|
||||
val: `Dom`,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "basic compressed",
|
||||
input: `foo=="bar"`,
|
||||
tokens: []item{
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "foo",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: "==",
|
||||
},
|
||||
{
|
||||
typ: itemString,
|
||||
val: `bar`,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "basic bool",
|
||||
input: `foo == true`,
|
||||
tokens: []item{
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "foo",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: "==",
|
||||
},
|
||||
{
|
||||
typ: itemBoolean,
|
||||
val: `true`,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "basic bool false",
|
||||
input: `foo == false`,
|
||||
tokens: []item{
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "foo",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: "==",
|
||||
},
|
||||
{
|
||||
typ: itemBoolean,
|
||||
val: `false`,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "basic with spaces",
|
||||
input: `foo == "hello there"`,
|
||||
tokens: []item{
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "foo",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: "==",
|
||||
},
|
||||
{
|
||||
typ: itemString,
|
||||
val: `hello there`,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "basic number",
|
||||
input: `foo == 123987`,
|
||||
tokens: []item{
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "foo",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: "==",
|
||||
},
|
||||
{
|
||||
typ: itemNumber,
|
||||
val: `123987`,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "basic gt number",
|
||||
input: `foo >= 123987`,
|
||||
tokens: []item{
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "foo",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: ">=",
|
||||
},
|
||||
{
|
||||
typ: itemNumber,
|
||||
val: `123987`,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "basic lt number",
|
||||
input: `foo <= 123987`,
|
||||
tokens: []item{
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "foo",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: "<=",
|
||||
},
|
||||
{
|
||||
typ: itemNumber,
|
||||
val: `123987`,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "AND bool",
|
||||
input: `foo == 'bar' AND baz == 'hello'`,
|
||||
tokens: []item{
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "foo",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: "==",
|
||||
},
|
||||
{
|
||||
typ: itemString,
|
||||
val: `bar`,
|
||||
},
|
||||
{
|
||||
typ: itemBooleanOp,
|
||||
val: "AND",
|
||||
},
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "baz",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: "==",
|
||||
},
|
||||
{
|
||||
typ: itemString,
|
||||
val: `hello`,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "and bool",
|
||||
input: `foo == 'bar' and baz == 'hello'`,
|
||||
tokens: []item{
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "foo",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: "==",
|
||||
},
|
||||
{
|
||||
typ: itemString,
|
||||
val: `bar`,
|
||||
},
|
||||
{
|
||||
typ: itemBooleanOp,
|
||||
val: "and",
|
||||
},
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "baz",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: "==",
|
||||
},
|
||||
{
|
||||
typ: itemString,
|
||||
val: `hello`,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "OR bool",
|
||||
input: `foo == 'bar' OR baz == 'hello'`,
|
||||
tokens: []item{
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "foo",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: "==",
|
||||
},
|
||||
{
|
||||
typ: itemString,
|
||||
val: `bar`,
|
||||
},
|
||||
{
|
||||
typ: itemBooleanOp,
|
||||
val: "OR",
|
||||
},
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "baz",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: "==",
|
||||
},
|
||||
{
|
||||
typ: itemString,
|
||||
val: `hello`,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "or bool",
|
||||
input: `foo == 'bar' or baz == 'hello'`,
|
||||
tokens: []item{
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "foo",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: "==",
|
||||
},
|
||||
{
|
||||
typ: itemString,
|
||||
val: `bar`,
|
||||
},
|
||||
{
|
||||
typ: itemBooleanOp,
|
||||
val: "or",
|
||||
},
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "baz",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: "==",
|
||||
},
|
||||
{
|
||||
typ: itemString,
|
||||
val: `hello`,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "bad val",
|
||||
input: `foo == bar`,
|
||||
tokens: []item{
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "foo",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: "==",
|
||||
},
|
||||
{
|
||||
typ: itemError,
|
||||
},
|
||||
},
|
||||
err: fmt.Errorf("blah"),
|
||||
},
|
||||
{
|
||||
name: "gibberish",
|
||||
input: `123onddlqkjn oajsldkj`,
|
||||
tokens: []item{
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "123onddlqkjn",
|
||||
},
|
||||
{
|
||||
typ: itemError,
|
||||
},
|
||||
},
|
||||
err: fmt.Errorf("blah"),
|
||||
},
|
||||
{
|
||||
name: "gibberish",
|
||||
input: `123onddlqkjn`,
|
||||
tokens: []item{
|
||||
{
|
||||
typ: itemError,
|
||||
},
|
||||
},
|
||||
err: fmt.Errorf("blah"),
|
||||
},
|
||||
{
|
||||
name: "brackets",
|
||||
input: `foo == 'bar' and (baz == 'hello' or customer.name == 'john doe')`,
|
||||
tokens: []item{
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "foo",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: "==",
|
||||
},
|
||||
{
|
||||
typ: itemString,
|
||||
val: `bar`,
|
||||
},
|
||||
{
|
||||
typ: itemBooleanOp,
|
||||
val: "and",
|
||||
},
|
||||
{
|
||||
typ: itemLeftParen,
|
||||
val: "(",
|
||||
},
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "baz",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: "==",
|
||||
},
|
||||
{
|
||||
typ: itemString,
|
||||
val: `hello`,
|
||||
},
|
||||
{
|
||||
typ: itemBooleanOp,
|
||||
val: "or",
|
||||
},
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "customer.name",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: "==",
|
||||
},
|
||||
{
|
||||
typ: itemString,
|
||||
val: "john doe",
|
||||
},
|
||||
{
|
||||
typ: itemRightParen,
|
||||
val: ")",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "brackets",
|
||||
input: `(foo == 'bar' and baz == 'hello') or customer.name == 'john doe'`,
|
||||
tokens: []item{
|
||||
{
|
||||
typ: itemLeftParen,
|
||||
val: "(",
|
||||
},
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "foo",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: "==",
|
||||
},
|
||||
{
|
||||
typ: itemString,
|
||||
val: `bar`,
|
||||
},
|
||||
{
|
||||
typ: itemBooleanOp,
|
||||
val: "and",
|
||||
},
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "baz",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: "==",
|
||||
},
|
||||
{
|
||||
typ: itemString,
|
||||
val: `hello`,
|
||||
},
|
||||
{
|
||||
typ: itemRightParen,
|
||||
val: ")",
|
||||
},
|
||||
{
|
||||
typ: itemBooleanOp,
|
||||
val: "or",
|
||||
},
|
||||
{
|
||||
typ: itemIdentifier,
|
||||
val: "customer.name",
|
||||
},
|
||||
{
|
||||
typ: itemOperator,
|
||||
val: "==",
|
||||
},
|
||||
{
|
||||
typ: itemString,
|
||||
val: "john doe",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tcs {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
|
||||
_, ch := lex(tc.name, tc.input)
|
||||
erred := false
|
||||
for _, tok := range tc.tokens {
|
||||
it := <-ch
|
||||
t.Logf("Got %v", it)
|
||||
if it.typ == itemError {
|
||||
g.Expect(tok.typ).To(Equal(itemError))
|
||||
erred = true
|
||||
} else {
|
||||
g.Expect(it).To(Equal(tok))
|
||||
}
|
||||
|
||||
}
|
||||
if tc.err != nil {
|
||||
g.Expect(erred).To(BeTrue())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
101
search/handler/parser.go
Normal file
101
search/handler/parser.go
Normal file
@@ -0,0 +1,101 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/bitly/go-simplejson"
|
||||
)
|
||||
|
||||
func parseQueryString(query string) (*simplejson.Json, error) {
|
||||
_, items := lex("foo", query)
|
||||
res, err := parseQueryStringRec(items)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js := simplejson.New()
|
||||
js.Set("query", res)
|
||||
return js, nil
|
||||
}
|
||||
|
||||
const (
|
||||
matchTypeRange = "range"
|
||||
matchTypeMatch = "match"
|
||||
matchTypeWildcard = "wildcard"
|
||||
)
|
||||
|
||||
func parseQueryStringRec(items chan item) (*simplejson.Json, error) {
|
||||
retTerm := simplejson.New()
|
||||
currFieldName := ""
|
||||
currBool := ""
|
||||
currMatchType := matchTypeMatch
|
||||
currPathAddition := ""
|
||||
terms := []*simplejson.Json{}
|
||||
itemLoop:
|
||||
for it := range items {
|
||||
if it.typ == itemError {
|
||||
return nil, fmt.Errorf(it.val)
|
||||
}
|
||||
|
||||
switch it.typ {
|
||||
case itemIdentifier:
|
||||
currFieldName = it.val
|
||||
case itemString, itemBoolean, itemNumber:
|
||||
currTerm := simplejson.New()
|
||||
if strings.ContainsRune(it.val, '*') {
|
||||
currMatchType = matchTypeWildcard
|
||||
currPathAddition = "value"
|
||||
}
|
||||
path := []string{currMatchType, currFieldName}
|
||||
if len(currPathAddition) > 0 {
|
||||
path = append(path, currPathAddition)
|
||||
}
|
||||
currTerm.SetPath(path, it.val)
|
||||
terms = append(terms, currTerm)
|
||||
|
||||
// reset
|
||||
currFieldName = ""
|
||||
currMatchType = matchTypeMatch
|
||||
|
||||
case itemBooleanOp:
|
||||
thisBool := "must"
|
||||
if strings.ToLower(it.val) == "or" {
|
||||
thisBool = "should"
|
||||
}
|
||||
if len(currBool) == 0 {
|
||||
currBool = thisBool
|
||||
} else {
|
||||
if thisBool != currBool {
|
||||
// undefined behaviour order of precedence needs to be explicitly defined using parentheses
|
||||
return nil, fmt.Errorf("error parsing query. Cannot mix boolean operators without explicitly defining precedence with parentheses")
|
||||
}
|
||||
}
|
||||
case itemLeftParen:
|
||||
currTerm, err := parseQueryStringRec(items)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
terms = append(terms, currTerm)
|
||||
case itemRightParen:
|
||||
break itemLoop
|
||||
case itemOperator:
|
||||
switch it.val {
|
||||
case "==":
|
||||
currMatchType = matchTypeMatch
|
||||
currPathAddition = ""
|
||||
case ">=":
|
||||
currMatchType = matchTypeRange
|
||||
currPathAddition = "gte"
|
||||
case "<=":
|
||||
currMatchType = matchTypeRange
|
||||
currPathAddition = "lte"
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
if len(currBool) == 0 {
|
||||
currBool = "must"
|
||||
}
|
||||
retTerm.SetPath([]string{"bool", currBool}, terms)
|
||||
return retTerm, nil
|
||||
}
|
||||
130
search/handler/parser_test.go
Normal file
130
search/handler/parser_test.go
Normal file
@@ -0,0 +1,130 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestParsing(t *testing.T) {
|
||||
|
||||
tcs := []struct {
|
||||
name string
|
||||
input string
|
||||
output string
|
||||
err error
|
||||
}{
|
||||
{
|
||||
name: "basic",
|
||||
input: `foo == "bar"`,
|
||||
output: `{"query":{"bool":{"must":[{"match":{"foo":"bar"}}]}}}`,
|
||||
},
|
||||
{
|
||||
name: "basic",
|
||||
input: `first_name == 'Dom'`,
|
||||
output: `{"query":{"bool":{"must":[{"match":{"first_name":"Dom"}}]}}}`,
|
||||
},
|
||||
{
|
||||
name: "basic bool",
|
||||
input: `foo == true`,
|
||||
output: `{"query":{"bool":{"must":[{"match":{"foo":"true"}}]}}}`,
|
||||
},
|
||||
{
|
||||
name: "basic bool false",
|
||||
input: `foo == false`,
|
||||
output: `{"query":{"bool":{"must":[{"match":{"foo":"false"}}]}}}`,
|
||||
},
|
||||
{
|
||||
name: "basic with spaces",
|
||||
input: `foo == "hello there"`,
|
||||
output: `{"query":{"bool":{"must":[{"match":{"foo":"hello there"}}]}}}`,
|
||||
},
|
||||
{
|
||||
name: "basic number",
|
||||
input: `foo == 123987`,
|
||||
output: `{"query":{"bool":{"must":[{"match":{"foo":"123987"}}]}}}`,
|
||||
},
|
||||
{
|
||||
name: "basic gt number",
|
||||
input: `foo >= 123987`,
|
||||
output: `{"query":{"bool":{"must":[{"range":{"foo":{"gte":"123987"}}}]}}}`,
|
||||
},
|
||||
{
|
||||
name: "basic lt number",
|
||||
input: `foo <= 123987`,
|
||||
output: `{"query":{"bool":{"must":[{"range":{"foo":{"lte":"123987"}}}]}}}`,
|
||||
},
|
||||
{
|
||||
name: "AND bool",
|
||||
input: `foo == 'bar' AND baz == 'hello'`,
|
||||
output: `{"query":{"bool":{"must":[{"match":{"foo":"bar"}},{"match":{"baz":"hello"}}]}}}`,
|
||||
},
|
||||
{
|
||||
name: "and bool",
|
||||
input: `foo == 'bar' and baz == 'hello'`,
|
||||
output: `{"query":{"bool":{"must":[{"match":{"foo":"bar"}},{"match":{"baz":"hello"}}]}}}`,
|
||||
},
|
||||
{
|
||||
name: "OR bool",
|
||||
input: `foo == 'bar' OR baz == 'hello'`,
|
||||
output: `{"query":{"bool":{"should":[{"match":{"foo":"bar"}},{"match":{"baz":"hello"}}]}}}`,
|
||||
},
|
||||
{
|
||||
name: "or bool",
|
||||
input: `foo == 'bar' or baz == 'hello'`,
|
||||
output: `{"query":{"bool":{"should":[{"match":{"foo":"bar"}},{"match":{"baz":"hello"}}]}}}`,
|
||||
},
|
||||
{
|
||||
name: "bad val",
|
||||
input: `foo == bar`,
|
||||
err: fmt.Errorf("blah"),
|
||||
},
|
||||
{
|
||||
name: "gibberish",
|
||||
input: `123onddlqkjn oajsldkj`,
|
||||
err: fmt.Errorf("blah"),
|
||||
},
|
||||
{
|
||||
name: "gibberish",
|
||||
input: `123onddlqkjn`,
|
||||
err: fmt.Errorf("blah"),
|
||||
},
|
||||
{
|
||||
name: "brackets",
|
||||
input: `foo == 'bar' and (baz == 'hello' or customer.name == 'john doe')`,
|
||||
output: `{"query":{"bool":{"must":[{"match":{"foo":"bar"}},{"bool":{"should":[{"match":{"baz":"hello"}},{"match":{"customer.name":"john doe"}}]}}]}}}`,
|
||||
},
|
||||
{
|
||||
name: "gte",
|
||||
input: `foo >= 6`,
|
||||
output: `{"query":{"bool":{"must":[{"range":{"foo":{"gte":"6"}}}]}}}`,
|
||||
},
|
||||
{
|
||||
name: "lte",
|
||||
input: `foo <= 6`,
|
||||
output: `{"query":{"bool":{"must":[{"range":{"foo":{"lte":"6"}}}]}}}`,
|
||||
},
|
||||
{
|
||||
name: "wildcard",
|
||||
input: `foo == "ba*"`,
|
||||
output: `{"query":{"bool":{"must":[{"wildcard":{"foo":{"value":"ba*"}}}]}}}`,
|
||||
},
|
||||
}
|
||||
for _, tc := range tcs {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
js, err := parseQueryString(tc.input)
|
||||
if tc.err != nil {
|
||||
g.Expect(err).To(Not(BeNil()))
|
||||
} else {
|
||||
b, _ := js.MarshalJSON()
|
||||
t.Logf("%+v", string(b))
|
||||
g.Expect(err).To(BeNil())
|
||||
g.Expect(string(b)).To(Equal(tc.output))
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,48 +1,291 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/micro/micro/v3/service/store"
|
||||
"github.com/google/uuid"
|
||||
"github.com/micro/micro/v3/service"
|
||||
"github.com/micro/micro/v3/service/config"
|
||||
"github.com/micro/micro/v3/service/errors"
|
||||
log "github.com/micro/micro/v3/service/logger"
|
||||
"github.com/micro/services/pkg/tenant"
|
||||
pb "github.com/micro/services/search/proto"
|
||||
open "github.com/opensearch-project/opensearch-go"
|
||||
openapi "github.com/opensearch-project/opensearch-go/opensearchapi"
|
||||
"google.golang.org/protobuf/types/known/structpb"
|
||||
)
|
||||
|
||||
type Search struct{}
|
||||
|
||||
var (
|
||||
mtx sync.RWMutex
|
||||
|
||||
voteKey = "votes/"
|
||||
indexNameRegex = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9]$`)
|
||||
shortIndexNameRegex = regexp.MustCompile(`[a-zA-Z0-9]`)
|
||||
)
|
||||
|
||||
type Vote struct {
|
||||
Id string `json:"id"`
|
||||
Message string `json:"message"`
|
||||
VotedAt time.Time `json:"voted_at"`
|
||||
type Search struct {
|
||||
conf conf
|
||||
client *open.Client
|
||||
}
|
||||
|
||||
func (n *Search) Vote(ctx context.Context, req *pb.VoteRequest, rsp *pb.VoteResponse) error {
|
||||
mtx.Lock()
|
||||
defer mtx.Unlock()
|
||||
type conf struct {
|
||||
OpenAddr string `json:"open_addr"`
|
||||
User string `json:"user"`
|
||||
Pass string `json:"pass"`
|
||||
Insecure bool `json:"insecure"`
|
||||
}
|
||||
|
||||
id, ok := tenant.FromContext(ctx)
|
||||
if !ok {
|
||||
id = "micro"
|
||||
type openSearchResponse struct {
|
||||
Took int64 `json:"took"`
|
||||
Hits hits `json:"hits"`
|
||||
}
|
||||
|
||||
type hits struct {
|
||||
Total map[string]interface{} `json:"total"`
|
||||
Hits []hit `json:"hits"`
|
||||
}
|
||||
type hit struct {
|
||||
ID string `json:"_id"`
|
||||
Score float64 `json:"_score"`
|
||||
Source map[string]interface{} `json:"_source"`
|
||||
}
|
||||
|
||||
func New(srv *service.Service) *Search {
|
||||
v, err := config.Get("micro.search")
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to load config %s", err)
|
||||
}
|
||||
var c conf
|
||||
if err := v.Scan(&c); err != nil {
|
||||
log.Fatalf("Failed to load config %s", err)
|
||||
}
|
||||
if len(c.OpenAddr) == 0 || len(c.User) == 0 || len(c.Pass) == 0 {
|
||||
log.Fatalf("Missing configuration")
|
||||
}
|
||||
|
||||
rec := store.NewRecord(voteKey+id, &Vote{
|
||||
Id: id,
|
||||
Message: req.Message,
|
||||
VotedAt: time.Now(),
|
||||
})
|
||||
oc := open.Config{
|
||||
Addresses: []string{c.OpenAddr},
|
||||
Username: c.User,
|
||||
Password: c.Pass,
|
||||
}
|
||||
if c.Insecure {
|
||||
oc.Transport = &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // For testing only. Use certificate for validation.
|
||||
}
|
||||
}
|
||||
|
||||
// we don't need to check the error
|
||||
store.Write(rec)
|
||||
client, err := open.NewClient(oc)
|
||||
if err != nil {
|
||||
log.Fatalf("Error configuring search client %s", err)
|
||||
}
|
||||
return &Search{
|
||||
conf: c,
|
||||
client: client,
|
||||
}
|
||||
}
|
||||
|
||||
rsp.Message = "Thanks for the vote!"
|
||||
func isValidIndexName(s string) bool {
|
||||
if len(s) > 1 {
|
||||
return indexNameRegex.MatchString(s)
|
||||
}
|
||||
return shortIndexNameRegex.MatchString(s)
|
||||
}
|
||||
|
||||
func (s *Search) CreateIndex(ctx context.Context, request *pb.CreateIndexRequest, response *pb.CreateIndexResponse) error {
|
||||
method := "search.CreateIndex"
|
||||
|
||||
// TODO validate name https://opensearch.org/docs/latest/opensearch/rest-api/index-apis/create-index/#index-naming-restrictions
|
||||
tnt, ok := tenant.FromContext(ctx)
|
||||
if !ok {
|
||||
return errors.Unauthorized(method, "Unauthorized")
|
||||
}
|
||||
if !isValidIndexName(request.Index) {
|
||||
return errors.BadRequest(method, "Index name should contain only alphanumerics and hyphens")
|
||||
}
|
||||
req := openapi.CreateRequest{
|
||||
Index: indexName(tnt, request.Index),
|
||||
Body: nil, // TODO populate with fields and their types
|
||||
}
|
||||
rsp, err := req.Do(ctx, s.client)
|
||||
if err != nil {
|
||||
log.Errorf("Error creating index %s", err)
|
||||
return errors.InternalServerError(method, "Error creating index")
|
||||
}
|
||||
defer rsp.Body.Close()
|
||||
if rsp.IsError() {
|
||||
log.Errorf("Error creating index %s", rsp.String())
|
||||
return errors.InternalServerError(method, "Error creating index")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func indexName(tnt, index string) string {
|
||||
return fmt.Sprintf("%s-%s", strings.ReplaceAll(tnt, "/", "-"), index)
|
||||
}
|
||||
|
||||
func (s *Search) Index(ctx context.Context, request *pb.IndexRequest, response *pb.IndexResponse) error {
|
||||
method := "search.Index"
|
||||
// TODO validation
|
||||
// TODO validate name https://opensearch.org/docs/latest/opensearch/rest-api/index-apis/create-index/#index-naming-restrictions
|
||||
tnt, ok := tenant.FromContext(ctx)
|
||||
if !ok {
|
||||
return errors.Unauthorized(method, "Unauthorized")
|
||||
}
|
||||
if request.Document == nil {
|
||||
return errors.BadRequest(method, "Missing document param")
|
||||
}
|
||||
if len(request.Document.Id) == 0 {
|
||||
request.Document.Id = uuid.New().String()
|
||||
}
|
||||
if len(request.Index) == 0 {
|
||||
return errors.BadRequest(method, "Missing index_name param")
|
||||
}
|
||||
if !isValidIndexName(request.Index) {
|
||||
return errors.BadRequest(method, "Index name should contain only alphanumerics and hyphens")
|
||||
}
|
||||
if request.Document.Contents == nil {
|
||||
return errors.BadRequest(method, "Missing document.contents param")
|
||||
}
|
||||
|
||||
b, err := request.Document.Contents.MarshalJSON()
|
||||
if err != nil {
|
||||
return errors.BadRequest(method, "Error processing document")
|
||||
}
|
||||
req := openapi.IndexRequest{
|
||||
Index: indexName(tnt, request.Index),
|
||||
DocumentID: request.Document.Id,
|
||||
Body: bytes.NewBuffer(b),
|
||||
}
|
||||
rsp, err := req.Do(ctx, s.client)
|
||||
if err != nil {
|
||||
log.Errorf("Error indexing doc %s", err)
|
||||
return errors.InternalServerError(method, "Error indexing document")
|
||||
}
|
||||
defer rsp.Body.Close()
|
||||
if rsp.IsError() {
|
||||
log.Errorf("Error indexing doc %s", rsp.String())
|
||||
return errors.InternalServerError(method, "Error indexing document")
|
||||
}
|
||||
response.Id = req.DocumentID
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Search) Delete(ctx context.Context, request *pb.DeleteRequest, response *pb.DeleteResponse) error {
|
||||
method := "search.Delete"
|
||||
tnt, ok := tenant.FromContext(ctx)
|
||||
if !ok {
|
||||
return errors.Unauthorized(method, "Unauthorized")
|
||||
}
|
||||
req := openapi.DeleteRequest{
|
||||
Index: indexName(tnt, request.Index),
|
||||
DocumentID: request.Id,
|
||||
}
|
||||
rsp, err := req.Do(ctx, s.client)
|
||||
if err != nil {
|
||||
log.Errorf("Error deleting doc %s", err)
|
||||
return errors.InternalServerError(method, "Error deleting document")
|
||||
}
|
||||
defer rsp.Body.Close()
|
||||
if rsp.IsError() {
|
||||
log.Errorf("Error deleting doc %s", rsp.String())
|
||||
return errors.InternalServerError(method, "Error deleting document")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Search) Search(ctx context.Context, request *pb.SearchRequest, response *pb.SearchResponse) error {
|
||||
method := "search.Search"
|
||||
if len(request.Index) == 0 {
|
||||
return errors.BadRequest(method, "Missing index param")
|
||||
}
|
||||
|
||||
// Search models to support https://opensearch.org/docs/latest/opensearch/ux/
|
||||
// - Simple query
|
||||
// - Autocomplete (prefix) queries
|
||||
// - pagination
|
||||
// - Sorting
|
||||
//
|
||||
tnt, ok := tenant.FromContext(ctx)
|
||||
if !ok {
|
||||
return errors.Unauthorized(method, "Unauthorized")
|
||||
}
|
||||
|
||||
// TODO fuzzy
|
||||
if len(request.Query) == 0 {
|
||||
return errors.BadRequest(method, "Missing query param")
|
||||
}
|
||||
|
||||
qs, err := parseQueryString(request.Query)
|
||||
if err != nil {
|
||||
log.Errorf("Error parsing string %s %s", request.Query, err)
|
||||
return errors.BadRequest(method, "%s", err)
|
||||
}
|
||||
b, _ := qs.MarshalJSON()
|
||||
req := openapi.SearchRequest{
|
||||
Index: []string{indexName(tnt, request.Index)},
|
||||
Body: bytes.NewBuffer(b),
|
||||
}
|
||||
rsp, err := req.Do(ctx, s.client)
|
||||
if err != nil {
|
||||
log.Errorf("Error searching index %s", err)
|
||||
return errors.InternalServerError(method, "Error searching documents")
|
||||
}
|
||||
defer rsp.Body.Close()
|
||||
if rsp.IsError() {
|
||||
if rsp.StatusCode == 404 { // index not found
|
||||
return errors.NotFound(method, "Index not found")
|
||||
}
|
||||
log.Errorf("Error searching index %s", rsp.String())
|
||||
return errors.InternalServerError(method, "Error searching documents")
|
||||
}
|
||||
b, err = ioutil.ReadAll(rsp.Body)
|
||||
if err != nil {
|
||||
log.Errorf("Error searching index %s", rsp.String())
|
||||
return errors.InternalServerError(method, "Error searching documents")
|
||||
}
|
||||
var os openSearchResponse
|
||||
if err := json.Unmarshal(b, &os); err != nil {
|
||||
log.Errorf("Error unmarshalling doc %s", err)
|
||||
return errors.InternalServerError(method, "Error searching documents")
|
||||
}
|
||||
log.Infof("%s", string(b))
|
||||
for _, v := range os.Hits.Hits {
|
||||
vs, err := structpb.NewStruct(v.Source)
|
||||
if err != nil {
|
||||
log.Errorf("Error unmarshalling doc %s", err)
|
||||
return errors.InternalServerError(method, "Error searching documents")
|
||||
}
|
||||
response.Documents = append(response.Documents, &pb.Document{
|
||||
Id: v.ID,
|
||||
Contents: vs,
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Search) DeleteIndex(ctx context.Context, request *pb.DeleteIndexRequest, response *pb.DeleteIndexResponse) error {
|
||||
method := "search.DeleteIndex"
|
||||
tnt, ok := tenant.FromContext(ctx)
|
||||
if !ok {
|
||||
return errors.Unauthorized(method, "Unauthorized")
|
||||
}
|
||||
req := openapi.DeleteRequest{
|
||||
Index: indexName(tnt, request.Index),
|
||||
}
|
||||
rsp, err := req.Do(ctx, s.client)
|
||||
if err != nil {
|
||||
log.Errorf("Error deleting index %s", err)
|
||||
return errors.InternalServerError(method, "Error deleting index")
|
||||
}
|
||||
defer rsp.Body.Close()
|
||||
if rsp.IsError() {
|
||||
log.Errorf("Error deleting index %s", rsp.String())
|
||||
return errors.InternalServerError(method, "Error deleting index")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user