Search API (#350)

email/handler/email_test.go

@@ -7,7 +7,6 @@ import (
 )
 
 func TestEmailValidation(t *testing.T) {
-	g := NewWithT(t)
 	tcs := []struct {
 		name  string
 		email string
@@ -60,6 +59,7 @@ func TestEmailValidation(t *testing.T) {
 		},
 	}
 	for _, tc := range tcs {
+		g := NewWithT(t)
 		t.Run(tc.name, func(t *testing.T) {
 			g.Expect(validEmail(tc.email)).To(Equal(tc.valid))
 		})

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/Teamwork/spamc v0.0.0-20200109085853-a4e0c5c3f7a0
 	github.com/asim/mq v0.1.0
 	github.com/aws/aws-sdk-go v1.42.17
+	github.com/bitly/go-simplejson v0.5.0
 	github.com/cdipaolo/goml v0.0.0-20190412180403-e1f51f713598 // indirect
 	github.com/cdipaolo/sentiment v0.0.0-20200617002423-c697f64e7f10
 	github.com/crufter/lexer v0.0.0-20120907053443-23fe8c7add01
@@ -36,6 +37,7 @@ require (
 	github.com/minio/minio-go/v7 v7.0.16
 	github.com/o1egl/govatar v0.3.0
 	github.com/onsi/gomega v1.10.5
+	github.com/opensearch-project/opensearch-go v1.0.0
 	github.com/oschwald/geoip2-golang v1.5.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/paulmach/go.geo v0.0.0-20180829195134-22b514266d33

go.sum

@@ -540,6 +540,8 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ=
 github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48=
+github.com/opensearch-project/opensearch-go v1.0.0 h1:8Gh7B7Un5BxuxWAgmzleEF7lpOtC71pCgPp7lKr3ca8=
+github.com/opensearch-project/opensearch-go v1.0.0/go.mod h1:FrUl/52DBegRYvK7ISF278AXmjDV647lyTnsLGBR7J4=
 github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=

search/README.md

@@ -1,7 +1,78 @@
-Indexing & full text search
+Indexing and full text search
 
 # Search Service
 
-Store and search for anything text based. The Search API provides 
+Store and search JSON documents. The Search API provides full indexing and text search.
-full indexing and text search.
 
+Powered by [OpenSearch](https://opensearch.org/).
+
+Search for a given word or phrase in a particular field of a document. Combine multiple with either `AND` or `OR` boolean operators to create complex queries.
+
+## Usage
+Documents are inserted using the `/search/index` endpoint. Document fields are automatically indexed with no need to define which fields to index ahead of time. Documents are logically grouped in to `indexes` so you may have an index for customers and one for products. Once documents are inserted you are ready to search, simple as that.
+
+## Search query language
+
+The search API supports a simple query language to let you get to your data quickly without having to learn a complicated language. 
+
+The most basic query looks like this
+
+```sql
+key == 'value'
+```
+
+where you specify a key and a value to find. For example you might want to look for every customer with first name of John
+
+```sql
+first_name == 'John'
+```
+
+String values support single or double quotes. 
+
+Values can also be numbers 
+
+```sql
+age == 37
+```
+
+or booleans
+
+```sql
+verified == true
+```
+
+You can search on fields that are nested in the document using dot (`.`) as a separator 
+
+```sql
+address.city == 'London'
+```
+
+The API also supports wildcard `*` matching to enable scenarios like autocomplete.
+
+```sql
+first_name == 'Joh*'
+```
+
+In addition to equality `==` the API support greater than or equals `>=` and less than or equals `<=` operators
+
+```sql
+age >= 37
+age <= 37
+```
+
+Simple queries can be combined with logical `and` 
+
+```sql
+first_name == "John" AND age <= 37
+```
+
+or logical `or`
+```sql
+first_name == "John" OR first_name == "Jane"
+```
+
+If combining `and` and `or` operations you will need to use parentheses to explicitly define precedence
+
+```sql
+(first_name == "John" OR first_name == "Jane") AND age <= 37 
+```

search/examples.json

@@ -1,14 +1,104 @@
 {
-  "vote": [
+  "index": [
     {
-      "title": "Vote for the API",
+      "title": "Index a document",
       "run_check": false,
       "request": {
-        "message": "Launch it!"
+        "index": "customers",
+        "document": {
+          "id": "1234",
+          "contents": {
+            "name": "John Doe",
+            "age": 37,
+            "starsign": "Leo"
+          }
+        }
       },
       "response": {
-	"message": "Thanks for the vote!"
       }
     }
+  ],
+  "search": [
+    {
+      "title": "Search for a document",
+      "run_check": false,
+      "request": {
+        "index": "customers",
+        "query": "name == 'John'"
+      },
+      "response": {
+        "documents": [
+          {
+            "id": "1234",
+            "contents": {
+              "name": "John Doe",
+              "age": 37,
+              "starsign": "Leo"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "title": "Search on multiple fields (AND)",
+      "run_check": false,
+      "request": {
+        "index": "customers",
+        "query": "name == 'John' AND starsign == 'Leo'"
+      },
+      "response": {
+        "documents": [
+          {
+            "id": "1234",
+            "contents": {
+              "name": "John Doe",
+              "age": 37,
+              "starsign": "Leo"
+            }
+          }
+        ]
+      }
+    },
+    {
+      "title": "Search on multiple fields (OR)",
+      "run_check": false,
+      "request": {
+        "index": "customers",
+        "query": "name == 'John' OR name == 'Jane'"
+      },
+      "response": {
+        "documents": [
+          {
+            "id": "1234",
+            "contents": {
+              "name": "John Doe",
+              "age": 37,
+              "starsign": "Leo"
+            }
+          }
+        ]
+      }
+    }
+  ],
+  "delete": [
+    {
+      "title": "Delete a document",
+      "run_check": false,
+      "request": {
+        "id": "1234",
+        "index": "customers"
+      },
+      "response": {}
+    }
+  ],
+  "deleteIndex": [
+    {
+      "title": "Delete an index",
+      "run_check": false,
+      "request": {
+        "index": "customers"
+      },
+      "response": {}
+    }
   ]
 }

search/handler/lexer.go

@@ -0,0 +1,287 @@
+package handler
+
+import (
+	"fmt"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+type itemType int
+
+const (
+	itemError itemType = iota
+	itemNumber
+	itemIdentifier
+	itemBoolean
+	itemBooleanOp
+	itemString
+	itemOperator
+	itemLeftParen
+	itemRightParen
+)
+
+const (
+	eof = -1
+)
+
+type item struct {
+	typ itemType
+	val string
+}
+
+type stateFn func(*lexer) stateFn
+
+func (l *lexer) run() {
+	for state := lexStartStatement; state != nil; {
+		state = state(l)
+	}
+	close(l.items)
+}
+
+type lexer struct {
+	name  string    //used only for error reports
+	input string    // the string being scanned
+	start int       // start position of this item
+	pos   int       // current position in the input
+	width int       // width of last rune read
+	items chan item // channel of scanned items
+
+}
+
+func lex(name, input string) (*lexer, chan item) {
+	l := &lexer{
+		name:  name,
+		input: input,
+		items: make(chan item),
+	}
+	go l.run()
+	return l, l.items
+}
+
+func (l *lexer) emit(t itemType) {
+	l.items <- item{t, l.input[l.start:l.pos]}
+	l.start = l.pos
+}
+
+func (l *lexer) errorf(format string, args ...interface{}) stateFn {
+	l.items <- item{
+		itemError,
+		fmt.Sprintf(format, args...),
+	}
+	return nil
+}
+
+func (l *lexer) consumeSpace() {
+	l.acceptRun(" ")
+	l.ignore()
+}
+
+// next returns the next rune in the input
+func (l *lexer) next() (rune int32) {
+	if l.pos == len(l.input) {
+		l.width = 0
+		return eof
+	}
+	rune, l.width = utf8.DecodeRuneInString(l.input[l.pos:])
+	l.pos += l.width
+	return rune
+}
+
+// ignore skips over the pending input before this point
+func (l *lexer) ignore() {
+	l.start = l.pos
+}
+
+// backup steps back one rune
+// Can be called only once per call of next
+func (l *lexer) backup() {
+	l.pos -= l.width
+}
+
+// peek returns but does not consume
+// the next rune in the input
+func (l *lexer) peek() int32 {
+	rune := l.next()
+	l.backup()
+	return rune
+}
+
+// accept consumes the next rune
+// if it's from the valid set
+func (l *lexer) accept(valid string) bool {
+	if strings.IndexRune(valid, l.next()) >= 0 {
+		return true
+	}
+	l.backup()
+	return false
+}
+
+// acceptRune consumes a run of runes from the valid set
+func (l *lexer) acceptRun(valid string) {
+	for strings.IndexRune(valid, l.next()) >= 0 {
+
+	}
+	l.backup()
+}
+
+func lexIdent(l *lexer) stateFn {
+	l.consumeSpace()
+	for {
+		r := l.next()
+		if r == eof {
+			return l.errorf("Unexpected end of input %q", l.input[l.start:])
+		}
+
+		if unicode.IsSpace(r) || strings.IndexRune("=><", r) >= 0 {
+			l.backup()
+			break
+		}
+	}
+	if l.pos > l.start {
+		l.emit(itemIdentifier)
+	}
+	return lexOperator(l)
+}
+
+func lexValue(l *lexer) stateFn {
+	l.consumeSpace()
+	switch {
+	case l.accept(`"'`):
+		l.backup()
+		return lexString(l)
+	case strings.HasPrefix(l.input[l.start:], "true"), strings.HasPrefix(l.input[l.start:], "false"):
+		return lexBool(l)
+	default:
+		// try it as a number
+		return lexNumber(l)
+	}
+}
+
+func lexString(l *lexer) stateFn {
+	// TODO support single and double quotes with escaping
+	if !l.accept(`"'`) {
+		return l.errorf("Unexpected value %v, expected a quote", l.peek())
+	}
+	// ignore the quote
+	openQuote := l.input[l.start:l.pos]
+	l.ignore()
+	lastRead := ""
+	for {
+		r := l.next()
+		if r == eof { // should only happen in error case
+			return l.errorf("Unexpected value %v, incorrectly terminated value %s %v", l.input[l.start:], openQuote, lastRead)
+		}
+		if string(r) == openQuote && lastRead != `\` {
+			l.backup()
+			l.emit(itemString)
+			l.next()
+			l.ignore() // ignore the quote
+			return lexEndStatement(l)
+		}
+		lastRead = string(r)
+
+	}
+}
+
+const (
+	operatorEquals  = `==`
+	operatorGreater = `>=`
+	operatorLess    = `<=`
+	parenLeft       = `(`
+	parenRight      = `)`
+)
+
+func lexOperator(l *lexer) stateFn {
+	l.consumeSpace()
+	switch l.input[l.pos : l.pos+2] {
+	case operatorEquals, operatorGreater, operatorLess:
+		l.pos += 2
+		l.emit(itemOperator)
+		return lexValue(l)
+	}
+	// look for identifier
+	return l.errorf("Unexpected operator %q", l.input[l.pos:l.pos+2])
+}
+
+func lexNumber(l *lexer) stateFn {
+	l.consumeSpace()
+	// optional leading sign
+	l.accept("+-")
+	digits := "0123456789"
+	if l.accept("0") && l.accept("xX") {
+		digits = "0123456789abcdefABCDEF"
+	}
+	l.acceptRun(digits)
+	if l.accept(".") {
+		l.acceptRun(digits)
+	}
+	if l.accept("eE") {
+		l.accept("+-")
+		l.acceptRun("0123456789")
+	}
+	if l.start == l.pos {
+		return l.errorf("Unexpected value %s", l.input[l.start:])
+	}
+	l.emit(itemNumber)
+	return lexEndStatement(l)
+}
+
+func lexStartStatement(l *lexer) stateFn {
+	l.consumeSpace()
+	if string(l.peek()) == parenLeft {
+		l.next()
+		l.emit(itemLeftParen)
+		return lexStartStatement(l)
+	}
+	return lexIdent(l)
+}
+
+func lexEndStatement(l *lexer) stateFn {
+	// is this the end of the statement or can we find a boolean op
+	l.consumeSpace()
+	if string(l.peek()) == parenRight {
+		l.next()
+		l.emit(itemRightParen)
+		return lexEndStatement(l)
+	}
+
+	for {
+		r := l.next()
+		if r == eof {
+			break
+		}
+		if unicode.IsSpace(r) {
+			l.backup()
+			break
+		}
+	}
+	if l.start == l.pos {
+		return nil
+	}
+
+	if l.input[l.start:l.pos] == "and" || l.input[l.start:l.pos] == "AND" || l.input[l.start:l.pos] == "or" || l.input[l.start:l.pos] == "OR" {
+		l.emit(itemBooleanOp)
+		return lexStartStatement(l)
+	}
+	return l.errorf("Unexpected input %v", l.input[l.start:l.pos])
+}
+
+func lexBool(l *lexer) stateFn {
+	for {
+		r := l.next()
+		if r == eof {
+			break
+		}
+		if unicode.IsSpace(r) {
+			l.backup()
+			break
+		}
+	}
+	if l.input[l.start:l.pos] == "true" || l.input[l.start:l.pos] == "false" {
+		l.emit(itemBoolean)
+		return lexEndStatement(l)
+	}
+
+	return l.errorf("Unexpected value %q, expecting a boolean", l.input[l.start:l.pos])
+}

search/handler/lexer_test.go

@@ -0,0 +1,499 @@
+package handler
+
+import (
+	"fmt"
+	"testing"
+
+	. "github.com/onsi/gomega"
+)
+
+func TestLexer(t *testing.T) {
+
+	tcs := []struct {
+		name   string
+		input  string
+		tokens []item
+		err    error
+	}{
+		{
+			name:  "basic",
+			input: `foo == "bar"`,
+			tokens: []item{
+				{
+					typ: itemIdentifier,
+					val: "foo",
+				},
+				{
+					typ: itemOperator,
+					val: "==",
+				},
+				{
+					typ: itemString,
+					val: `bar`,
+				},
+			},
+		},
+		{
+			name:  "basic",
+			input: `first_name == 'Dom'`,
+			tokens: []item{
+				{
+					typ: itemIdentifier,
+					val: "first_name",
+				},
+				{
+					typ: itemOperator,
+					val: "==",
+				},
+				{
+					typ: itemString,
+					val: `Dom`,
+				},
+			},
+		},
+		{
+			name:  "basic compressed",
+			input: `foo=="bar"`,
+			tokens: []item{
+				{
+					typ: itemIdentifier,
+					val: "foo",
+				},
+				{
+					typ: itemOperator,
+					val: "==",
+				},
+				{
+					typ: itemString,
+					val: `bar`,
+				},
+			},
+		},
+		{
+			name:  "basic bool",
+			input: `foo == true`,
+			tokens: []item{
+				{
+					typ: itemIdentifier,
+					val: "foo",
+				},
+				{
+					typ: itemOperator,
+					val: "==",
+				},
+				{
+					typ: itemBoolean,
+					val: `true`,
+				},
+			},
+		},
+		{
+			name:  "basic bool false",
+			input: `foo == false`,
+			tokens: []item{
+				{
+					typ: itemIdentifier,
+					val: "foo",
+				},
+				{
+					typ: itemOperator,
+					val: "==",
+				},
+				{
+					typ: itemBoolean,
+					val: `false`,
+				},
+			},
+		},
+		{
+			name:  "basic with spaces",
+			input: `foo == "hello there"`,
+			tokens: []item{
+				{
+					typ: itemIdentifier,
+					val: "foo",
+				},
+				{
+					typ: itemOperator,
+					val: "==",
+				},
+				{
+					typ: itemString,
+					val: `hello there`,
+				},
+			},
+		},
+		{
+			name:  "basic number",
+			input: `foo == 123987`,
+			tokens: []item{
+				{
+					typ: itemIdentifier,
+					val: "foo",
+				},
+				{
+					typ: itemOperator,
+					val: "==",
+				},
+				{
+					typ: itemNumber,
+					val: `123987`,
+				},
+			},
+		},
+		{
+			name:  "basic gt number",
+			input: `foo >= 123987`,
+			tokens: []item{
+				{
+					typ: itemIdentifier,
+					val: "foo",
+				},
+				{
+					typ: itemOperator,
+					val: ">=",
+				},
+				{
+					typ: itemNumber,
+					val: `123987`,
+				},
+			},
+		},
+		{
+			name:  "basic lt number",
+			input: `foo <= 123987`,
+			tokens: []item{
+				{
+					typ: itemIdentifier,
+					val: "foo",
+				},
+				{
+					typ: itemOperator,
+					val: "<=",
+				},
+				{
+					typ: itemNumber,
+					val: `123987`,
+				},
+			},
+		},
+		{
+			name:  "AND bool",
+			input: `foo == 'bar' AND baz == 'hello'`,
+			tokens: []item{
+				{
+					typ: itemIdentifier,
+					val: "foo",
+				},
+				{
+					typ: itemOperator,
+					val: "==",
+				},
+				{
+					typ: itemString,
+					val: `bar`,
+				},
+				{
+					typ: itemBooleanOp,
+					val: "AND",
+				},
+				{
+					typ: itemIdentifier,
+					val: "baz",
+				},
+				{
+					typ: itemOperator,
+					val: "==",
+				},
+				{
+					typ: itemString,
+					val: `hello`,
+				},
+			},
+		},
+		{
+			name:  "and bool",
+			input: `foo == 'bar' and baz == 'hello'`,
+			tokens: []item{
+				{
+					typ: itemIdentifier,
+					val: "foo",
+				},
+				{
+					typ: itemOperator,
+					val: "==",
+				},
+				{
+					typ: itemString,
+					val: `bar`,
+				},
+				{
+					typ: itemBooleanOp,
+					val: "and",
+				},
+				{
+					typ: itemIdentifier,
+					val: "baz",
+				},
+				{
+					typ: itemOperator,
+					val: "==",
+				},
+				{
+					typ: itemString,
+					val: `hello`,
+				},
+			},
+		},
+		{
+			name:  "OR bool",
+			input: `foo == 'bar' OR baz == 'hello'`,
+			tokens: []item{
+				{
+					typ: itemIdentifier,
+					val: "foo",
+				},
+				{
+					typ: itemOperator,
+					val: "==",
+				},
+				{
+					typ: itemString,
+					val: `bar`,
+				},
+				{
+					typ: itemBooleanOp,
+					val: "OR",
+				},
+				{
+					typ: itemIdentifier,
+					val: "baz",
+				},
+				{
+					typ: itemOperator,
+					val: "==",
+				},
+				{
+					typ: itemString,
+					val: `hello`,
+				},
+			},
+		},
+		{
+			name:  "or bool",
+			input: `foo == 'bar' or baz == 'hello'`,
+			tokens: []item{
+				{
+					typ: itemIdentifier,
+					val: "foo",
+				},
+				{
+					typ: itemOperator,
+					val: "==",
+				},
+				{
+					typ: itemString,
+					val: `bar`,
+				},
+				{
+					typ: itemBooleanOp,
+					val: "or",
+				},
+				{
+					typ: itemIdentifier,
+					val: "baz",
+				},
+				{
+					typ: itemOperator,
+					val: "==",
+				},
+				{
+					typ: itemString,
+					val: `hello`,
+				},
+			},
+		},
+		{
+			name:  "bad val",
+			input: `foo == bar`,
+			tokens: []item{
+				{
+					typ: itemIdentifier,
+					val: "foo",
+				},
+				{
+					typ: itemOperator,
+					val: "==",
+				},
+				{
+					typ: itemError,
+				},
+			},
+			err: fmt.Errorf("blah"),
+		},
+		{
+			name:  "gibberish",
+			input: `123onddlqkjn oajsldkj`,
+			tokens: []item{
+				{
+					typ: itemIdentifier,
+					val: "123onddlqkjn",
+				},
+				{
+					typ: itemError,
+				},
+			},
+			err: fmt.Errorf("blah"),
+		},
+		{
+			name:  "gibberish",
+			input: `123onddlqkjn`,
+			tokens: []item{
+				{
+					typ: itemError,
+				},
+			},
+			err: fmt.Errorf("blah"),
+		},
+		{
+			name:  "brackets",
+			input: `foo == 'bar' and (baz == 'hello' or customer.name == 'john doe')`,
+			tokens: []item{
+				{
+					typ: itemIdentifier,
+					val: "foo",
+				},
+				{
+					typ: itemOperator,
+					val: "==",
+				},
+				{
+					typ: itemString,
+					val: `bar`,
+				},
+				{
+					typ: itemBooleanOp,
+					val: "and",
+				},
+				{
+					typ: itemLeftParen,
+					val: "(",
+				},
+				{
+					typ: itemIdentifier,
+					val: "baz",
+				},
+				{
+					typ: itemOperator,
+					val: "==",
+				},
+				{
+					typ: itemString,
+					val: `hello`,
+				},
+				{
+					typ: itemBooleanOp,
+					val: "or",
+				},
+				{
+					typ: itemIdentifier,
+					val: "customer.name",
+				},
+				{
+					typ: itemOperator,
+					val: "==",
+				},
+				{
+					typ: itemString,
+					val: "john doe",
+				},
+				{
+					typ: itemRightParen,
+					val: ")",
+				},
+			},
+		},
+		{
+			name:  "brackets",
+			input: `(foo == 'bar' and baz == 'hello') or customer.name == 'john doe'`,
+			tokens: []item{
+				{
+					typ: itemLeftParen,
+					val: "(",
+				},
+				{
+					typ: itemIdentifier,
+					val: "foo",
+				},
+				{
+					typ: itemOperator,
+					val: "==",
+				},
+				{
+					typ: itemString,
+					val: `bar`,
+				},
+				{
+					typ: itemBooleanOp,
+					val: "and",
+				},
+				{
+					typ: itemIdentifier,
+					val: "baz",
+				},
+				{
+					typ: itemOperator,
+					val: "==",
+				},
+				{
+					typ: itemString,
+					val: `hello`,
+				},
+				{
+					typ: itemRightParen,
+					val: ")",
+				},
+				{
+					typ: itemBooleanOp,
+					val: "or",
+				},
+				{
+					typ: itemIdentifier,
+					val: "customer.name",
+				},
+				{
+					typ: itemOperator,
+					val: "==",
+				},
+				{
+					typ: itemString,
+					val: "john doe",
+				},
+			},
+		},
+	}
+
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			g := NewWithT(t)
+
+			_, ch := lex(tc.name, tc.input)
+			erred := false
+			for _, tok := range tc.tokens {
+				it := <-ch
+				t.Logf("Got %v", it)
+				if it.typ == itemError {
+					g.Expect(tok.typ).To(Equal(itemError))
+					erred = true
+				} else {
+					g.Expect(it).To(Equal(tok))
+				}
+
+			}
+			if tc.err != nil {
+				g.Expect(erred).To(BeTrue())
+			}
+		})
+	}
+
+}

search/handler/parser.go

@@ -0,0 +1,101 @@
+package handler
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/bitly/go-simplejson"
+)
+
+func parseQueryString(query string) (*simplejson.Json, error) {
+	_, items := lex("foo", query)
+	res, err := parseQueryStringRec(items)
+	if err != nil {
+		return nil, err
+	}
+	js := simplejson.New()
+	js.Set("query", res)
+	return js, nil
+}
+
+const (
+	matchTypeRange    = "range"
+	matchTypeMatch    = "match"
+	matchTypeWildcard = "wildcard"
+)
+
+func parseQueryStringRec(items chan item) (*simplejson.Json, error) {
+	retTerm := simplejson.New()
+	currFieldName := ""
+	currBool := ""
+	currMatchType := matchTypeMatch
+	currPathAddition := ""
+	terms := []*simplejson.Json{}
+itemLoop:
+	for it := range items {
+		if it.typ == itemError {
+			return nil, fmt.Errorf(it.val)
+		}
+
+		switch it.typ {
+		case itemIdentifier:
+			currFieldName = it.val
+		case itemString, itemBoolean, itemNumber:
+			currTerm := simplejson.New()
+			if strings.ContainsRune(it.val, '*') {
+				currMatchType = matchTypeWildcard
+				currPathAddition = "value"
+			}
+			path := []string{currMatchType, currFieldName}
+			if len(currPathAddition) > 0 {
+				path = append(path, currPathAddition)
+			}
+			currTerm.SetPath(path, it.val)
+			terms = append(terms, currTerm)
+
+			// reset
+			currFieldName = ""
+			currMatchType = matchTypeMatch
+
+		case itemBooleanOp:
+			thisBool := "must"
+			if strings.ToLower(it.val) == "or" {
+				thisBool = "should"
+			}
+			if len(currBool) == 0 {
+				currBool = thisBool
+			} else {
+				if thisBool != currBool {
+					// undefined behaviour order of precedence needs to be explicitly defined using parentheses
+					return nil, fmt.Errorf("error parsing query. Cannot mix boolean operators without explicitly defining precedence with parentheses")
+				}
+			}
+		case itemLeftParen:
+			currTerm, err := parseQueryStringRec(items)
+			if err != nil {
+				return nil, err
+			}
+			terms = append(terms, currTerm)
+		case itemRightParen:
+			break itemLoop
+		case itemOperator:
+			switch it.val {
+			case "==":
+				currMatchType = matchTypeMatch
+				currPathAddition = ""
+			case ">=":
+				currMatchType = matchTypeRange
+				currPathAddition = "gte"
+			case "<=":
+				currMatchType = matchTypeRange
+				currPathAddition = "lte"
+			}
+
+		}
+	}
+	if len(currBool) == 0 {
+		currBool = "must"
+	}
+	retTerm.SetPath([]string{"bool", currBool}, terms)
+	return retTerm, nil
+}

search/handler/parser_test.go

@@ -0,0 +1,130 @@
+package handler
+
+import (
+	"fmt"
+	"testing"
+
+	. "github.com/onsi/gomega"
+)
+
+func TestParsing(t *testing.T) {
+
+	tcs := []struct {
+		name   string
+		input  string
+		output string
+		err    error
+	}{
+		{
+			name:   "basic",
+			input:  `foo == "bar"`,
+			output: `{"query":{"bool":{"must":[{"match":{"foo":"bar"}}]}}}`,
+		},
+		{
+			name:   "basic",
+			input:  `first_name == 'Dom'`,
+			output: `{"query":{"bool":{"must":[{"match":{"first_name":"Dom"}}]}}}`,
+		},
+		{
+			name:   "basic bool",
+			input:  `foo == true`,
+			output: `{"query":{"bool":{"must":[{"match":{"foo":"true"}}]}}}`,
+		},
+		{
+			name:   "basic bool false",
+			input:  `foo == false`,
+			output: `{"query":{"bool":{"must":[{"match":{"foo":"false"}}]}}}`,
+		},
+		{
+			name:   "basic with spaces",
+			input:  `foo == "hello there"`,
+			output: `{"query":{"bool":{"must":[{"match":{"foo":"hello there"}}]}}}`,
+		},
+		{
+			name:   "basic number",
+			input:  `foo == 123987`,
+			output: `{"query":{"bool":{"must":[{"match":{"foo":"123987"}}]}}}`,
+		},
+		{
+			name:   "basic gt number",
+			input:  `foo >= 123987`,
+			output: `{"query":{"bool":{"must":[{"range":{"foo":{"gte":"123987"}}}]}}}`,
+		},
+		{
+			name:   "basic lt number",
+			input:  `foo <= 123987`,
+			output: `{"query":{"bool":{"must":[{"range":{"foo":{"lte":"123987"}}}]}}}`,
+		},
+		{
+			name:   "AND bool",
+			input:  `foo == 'bar' AND baz == 'hello'`,
+			output: `{"query":{"bool":{"must":[{"match":{"foo":"bar"}},{"match":{"baz":"hello"}}]}}}`,
+		},
+		{
+			name:   "and bool",
+			input:  `foo == 'bar' and baz == 'hello'`,
+			output: `{"query":{"bool":{"must":[{"match":{"foo":"bar"}},{"match":{"baz":"hello"}}]}}}`,
+		},
+		{
+			name:   "OR bool",
+			input:  `foo == 'bar' OR baz == 'hello'`,
+			output: `{"query":{"bool":{"should":[{"match":{"foo":"bar"}},{"match":{"baz":"hello"}}]}}}`,
+		},
+		{
+			name:   "or bool",
+			input:  `foo == 'bar' or baz == 'hello'`,
+			output: `{"query":{"bool":{"should":[{"match":{"foo":"bar"}},{"match":{"baz":"hello"}}]}}}`,
+		},
+		{
+			name:  "bad val",
+			input: `foo == bar`,
+			err:   fmt.Errorf("blah"),
+		},
+		{
+			name:  "gibberish",
+			input: `123onddlqkjn oajsldkj`,
+			err:   fmt.Errorf("blah"),
+		},
+		{
+			name:  "gibberish",
+			input: `123onddlqkjn`,
+			err:   fmt.Errorf("blah"),
+		},
+		{
+			name:   "brackets",
+			input:  `foo == 'bar' and (baz == 'hello' or customer.name == 'john doe')`,
+			output: `{"query":{"bool":{"must":[{"match":{"foo":"bar"}},{"bool":{"should":[{"match":{"baz":"hello"}},{"match":{"customer.name":"john doe"}}]}}]}}}`,
+		},
+		{
+			name:   "gte",
+			input:  `foo >= 6`,
+			output: `{"query":{"bool":{"must":[{"range":{"foo":{"gte":"6"}}}]}}}`,
+		},
+		{
+			name:   "lte",
+			input:  `foo <= 6`,
+			output: `{"query":{"bool":{"must":[{"range":{"foo":{"lte":"6"}}}]}}}`,
+		},
+		{
+			name:   "wildcard",
+			input:  `foo == "ba*"`,
+			output: `{"query":{"bool":{"must":[{"wildcard":{"foo":{"value":"ba*"}}}]}}}`,
+		},
+	}
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			g := NewWithT(t)
+			js, err := parseQueryString(tc.input)
+			if tc.err != nil {
+				g.Expect(err).To(Not(BeNil()))
+			} else {
+				b, _ := js.MarshalJSON()
+				t.Logf("%+v", string(b))
+				g.Expect(err).To(BeNil())
+				g.Expect(string(b)).To(Equal(tc.output))
+			}
+
+		})
+	}
+
+}

search/handler/search.go

@@ -1,48 +1,291 @@
 package handler
 
 import (
+	"bytes"
 	"context"
-	"sync"
+	"crypto/tls"
-	"time"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"regexp"
+	"strings"
 
-	"github.com/micro/micro/v3/service/store"
+	"github.com/google/uuid"
+	"github.com/micro/micro/v3/service"
+	"github.com/micro/micro/v3/service/config"
+	"github.com/micro/micro/v3/service/errors"
+	log "github.com/micro/micro/v3/service/logger"
 	"github.com/micro/services/pkg/tenant"
 	pb "github.com/micro/services/search/proto"
+	open "github.com/opensearch-project/opensearch-go"
+	openapi "github.com/opensearch-project/opensearch-go/opensearchapi"
+	"google.golang.org/protobuf/types/known/structpb"
 )
 
-type Search struct{}
-
 var (
-	mtx sync.RWMutex
+	indexNameRegex      = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9]$`)
-
+	shortIndexNameRegex = regexp.MustCompile(`[a-zA-Z0-9]`)
-	voteKey = "votes/"
 )
 
-type Vote struct {
+type Search struct {
-	Id      string    `json:"id"`
+	conf   conf
-	Message string    `json:"message"`
+	client *open.Client
-	VotedAt time.Time `json:"voted_at"`
 }
 
-func (n *Search) Vote(ctx context.Context, req *pb.VoteRequest, rsp *pb.VoteResponse) error {
+type conf struct {
-	mtx.Lock()
+	OpenAddr string `json:"open_addr"`
-	defer mtx.Unlock()
+	User     string `json:"user"`
+	Pass     string `json:"pass"`
+	Insecure bool   `json:"insecure"`
+}
 
-	id, ok := tenant.FromContext(ctx)
+type openSearchResponse struct {
-	if !ok {
+	Took int64 `json:"took"`
-		id = "micro"
+	Hits hits  `json:"hits"`
+}
+
+type hits struct {
+	Total map[string]interface{} `json:"total"`
+	Hits  []hit                  `json:"hits"`
+}
+type hit struct {
+	ID     string                 `json:"_id"`
+	Score  float64                `json:"_score"`
+	Source map[string]interface{} `json:"_source"`
+}
+
+func New(srv *service.Service) *Search {
+	v, err := config.Get("micro.search")
+	if err != nil {
+		log.Fatalf("Failed to load config %s", err)
+	}
+	var c conf
+	if err := v.Scan(&c); err != nil {
+		log.Fatalf("Failed to load config %s", err)
+	}
+	if len(c.OpenAddr) == 0 || len(c.User) == 0 || len(c.Pass) == 0 {
+		log.Fatalf("Missing configuration")
 	}
 
-	rec := store.NewRecord(voteKey+id, &Vote{
+	oc := open.Config{
-		Id:      id,
+		Addresses: []string{c.OpenAddr},
-		Message: req.Message,
+		Username:  c.User,
-		VotedAt: time.Now(),
+		Password:  c.Pass,
-	})
+	}
+	if c.Insecure {
+		oc.Transport = &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // For testing only. Use certificate for validation.
+		}
+	}
 
-	// we don't need to check the error
+	client, err := open.NewClient(oc)
-	store.Write(rec)
+	if err != nil {
+		log.Fatalf("Error configuring search client %s", err)
+	}
+	return &Search{
+		conf:   c,
+		client: client,
+	}
+}
 
-	rsp.Message = "Thanks for the vote!"
+func isValidIndexName(s string) bool {
+	if len(s) > 1 {
+		return indexNameRegex.MatchString(s)
+	}
+	return shortIndexNameRegex.MatchString(s)
+}
 
+func (s *Search) CreateIndex(ctx context.Context, request *pb.CreateIndexRequest, response *pb.CreateIndexResponse) error {
+	method := "search.CreateIndex"
+
+	// TODO validate name https://opensearch.org/docs/latest/opensearch/rest-api/index-apis/create-index/#index-naming-restrictions
+	tnt, ok := tenant.FromContext(ctx)
+	if !ok {
+		return errors.Unauthorized(method, "Unauthorized")
+	}
+	if !isValidIndexName(request.Index) {
+		return errors.BadRequest(method, "Index name should contain only alphanumerics and hyphens")
+	}
+	req := openapi.CreateRequest{
+		Index: indexName(tnt, request.Index),
+		Body:  nil, // TODO populate with fields and their types
+	}
+	rsp, err := req.Do(ctx, s.client)
+	if err != nil {
+		log.Errorf("Error creating index %s", err)
+		return errors.InternalServerError(method, "Error creating index")
+	}
+	defer rsp.Body.Close()
+	if rsp.IsError() {
+		log.Errorf("Error creating index %s", rsp.String())
+		return errors.InternalServerError(method, "Error creating index")
+	}
+	return nil
+}
+
+func indexName(tnt, index string) string {
+	return fmt.Sprintf("%s-%s", strings.ReplaceAll(tnt, "/", "-"), index)
+}
+
+func (s *Search) Index(ctx context.Context, request *pb.IndexRequest, response *pb.IndexResponse) error {
+	method := "search.Index"
+	// TODO validation
+	// TODO validate name https://opensearch.org/docs/latest/opensearch/rest-api/index-apis/create-index/#index-naming-restrictions
+	tnt, ok := tenant.FromContext(ctx)
+	if !ok {
+		return errors.Unauthorized(method, "Unauthorized")
+	}
+	if request.Document == nil {
+		return errors.BadRequest(method, "Missing document param")
+	}
+	if len(request.Document.Id) == 0 {
+		request.Document.Id = uuid.New().String()
+	}
+	if len(request.Index) == 0 {
+		return errors.BadRequest(method, "Missing index_name param")
+	}
+	if !isValidIndexName(request.Index) {
+		return errors.BadRequest(method, "Index name should contain only alphanumerics and hyphens")
+	}
+	if request.Document.Contents == nil {
+		return errors.BadRequest(method, "Missing document.contents param")
+	}
+
+	b, err := request.Document.Contents.MarshalJSON()
+	if err != nil {
+		return errors.BadRequest(method, "Error processing document")
+	}
+	req := openapi.IndexRequest{
+		Index:      indexName(tnt, request.Index),
+		DocumentID: request.Document.Id,
+		Body:       bytes.NewBuffer(b),
+	}
+	rsp, err := req.Do(ctx, s.client)
+	if err != nil {
+		log.Errorf("Error indexing doc %s", err)
+		return errors.InternalServerError(method, "Error indexing document")
+	}
+	defer rsp.Body.Close()
+	if rsp.IsError() {
+		log.Errorf("Error indexing doc %s", rsp.String())
+		return errors.InternalServerError(method, "Error indexing document")
+	}
+	response.Id = req.DocumentID
+	return nil
+}
+
+func (s *Search) Delete(ctx context.Context, request *pb.DeleteRequest, response *pb.DeleteResponse) error {
+	method := "search.Delete"
+	tnt, ok := tenant.FromContext(ctx)
+	if !ok {
+		return errors.Unauthorized(method, "Unauthorized")
+	}
+	req := openapi.DeleteRequest{
+		Index:      indexName(tnt, request.Index),
+		DocumentID: request.Id,
+	}
+	rsp, err := req.Do(ctx, s.client)
+	if err != nil {
+		log.Errorf("Error deleting doc %s", err)
+		return errors.InternalServerError(method, "Error deleting document")
+	}
+	defer rsp.Body.Close()
+	if rsp.IsError() {
+		log.Errorf("Error deleting doc %s", rsp.String())
+		return errors.InternalServerError(method, "Error deleting document")
+	}
+	return nil
+}
+
+func (s *Search) Search(ctx context.Context, request *pb.SearchRequest, response *pb.SearchResponse) error {
+	method := "search.Search"
+	if len(request.Index) == 0 {
+		return errors.BadRequest(method, "Missing index param")
+	}
+
+	// Search models to support https://opensearch.org/docs/latest/opensearch/ux/
+	// - Simple query
+	// - Autocomplete (prefix) queries
+	// - pagination
+	// - Sorting
+	//
+	tnt, ok := tenant.FromContext(ctx)
+	if !ok {
+		return errors.Unauthorized(method, "Unauthorized")
+	}
+
+	// TODO fuzzy
+	if len(request.Query) == 0 {
+		return errors.BadRequest(method, "Missing query param")
+	}
+
+	qs, err := parseQueryString(request.Query)
+	if err != nil {
+		log.Errorf("Error parsing string %s %s", request.Query, err)
+		return errors.BadRequest(method, "%s", err)
+	}
+	b, _ := qs.MarshalJSON()
+	req := openapi.SearchRequest{
+		Index: []string{indexName(tnt, request.Index)},
+		Body:  bytes.NewBuffer(b),
+	}
+	rsp, err := req.Do(ctx, s.client)
+	if err != nil {
+		log.Errorf("Error searching index %s", err)
+		return errors.InternalServerError(method, "Error searching documents")
+	}
+	defer rsp.Body.Close()
+	if rsp.IsError() {
+		if rsp.StatusCode == 404 { // index not found
+			return errors.NotFound(method, "Index not found")
+		}
+		log.Errorf("Error searching index %s", rsp.String())
+		return errors.InternalServerError(method, "Error searching documents")
+	}
+	b, err = ioutil.ReadAll(rsp.Body)
+	if err != nil {
+		log.Errorf("Error searching index %s", rsp.String())
+		return errors.InternalServerError(method, "Error searching documents")
+	}
+	var os openSearchResponse
+	if err := json.Unmarshal(b, &os); err != nil {
+		log.Errorf("Error unmarshalling doc %s", err)
+		return errors.InternalServerError(method, "Error searching documents")
+	}
+	log.Infof("%s", string(b))
+	for _, v := range os.Hits.Hits {
+		vs, err := structpb.NewStruct(v.Source)
+		if err != nil {
+			log.Errorf("Error unmarshalling doc %s", err)
+			return errors.InternalServerError(method, "Error searching documents")
+		}
+		response.Documents = append(response.Documents, &pb.Document{
+			Id:       v.ID,
+			Contents: vs,
+		})
+	}
+	return nil
+}
+
+func (s *Search) DeleteIndex(ctx context.Context, request *pb.DeleteIndexRequest, response *pb.DeleteIndexResponse) error {
+	method := "search.DeleteIndex"
+	tnt, ok := tenant.FromContext(ctx)
+	if !ok {
+		return errors.Unauthorized(method, "Unauthorized")
+	}
+	req := openapi.DeleteRequest{
+		Index: indexName(tnt, request.Index),
+	}
+	rsp, err := req.Do(ctx, s.client)
+	if err != nil {
+		log.Errorf("Error deleting index %s", err)
+		return errors.InternalServerError(method, "Error deleting index")
+	}
+	defer rsp.Body.Close()
+	if rsp.IsError() {
+		log.Errorf("Error deleting index %s", rsp.String())
+		return errors.InternalServerError(method, "Error deleting index")
+	}
 	return nil
 }

search/main.go

@@ -15,7 +15,7 @@ func main() {
 	)
 
 	// Register handler
-	pb.RegisterSearchHandler(srv.Server(), new(handler.Search))
+	pb.RegisterSearchHandler(srv.Server(), handler.New(srv))
 
 	// Run service
 	if err := srv.Run(); err != nil {

search/proto/search.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
-// 	protoc-gen-go v1.27.1
+// 	protoc-gen-go v1.26.0
-// 	protoc        v3.15.6
+// 	protoc        v3.15.5
 // source: proto/search.proto
 
 package search
@@ -9,6 +9,7 @@ package search
 import (
 	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	structpb "google.golang.org/protobuf/types/known/structpb"
 	reflect "reflect"
 	sync "sync"
 )
@@ -20,18 +21,20 @@ const (
 	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
 )
 
-// Vote to have the Search api launched faster!
+// Index a document i.e. insert a document to search for.
-type VoteRequest struct {
+type IndexRequest struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	// optional message
+	// The document to index
-	Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"`
+	Document *Document `protobuf:"bytes,1,opt,name=document,proto3" json:"document,omitempty"`
+	// The index this document belongs to
+	Index string `protobuf:"bytes,2,opt,name=index,proto3" json:"index,omitempty"`
 }
 
-func (x *VoteRequest) Reset() {
+func (x *IndexRequest) Reset() {
-	*x = VoteRequest{}
+	*x = IndexRequest{}
 	if protoimpl.UnsafeEnabled {
 		mi := &file_proto_search_proto_msgTypes[0]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -39,13 +42,13 @@ func (x *VoteRequest) Reset() {
 	}
 }
 
-func (x *VoteRequest) String() string {
+func (x *IndexRequest) String() string {
 	return protoimpl.X.MessageStringOf(x)
 }
 
-func (*VoteRequest) ProtoMessage() {}
+func (*IndexRequest) ProtoMessage() {}
 
-func (x *VoteRequest) ProtoReflect() protoreflect.Message {
+func (x *IndexRequest) ProtoReflect() protoreflect.Message {
 	mi := &file_proto_search_proto_msgTypes[0]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -57,29 +60,38 @@ func (x *VoteRequest) ProtoReflect() protoreflect.Message {
 	return mi.MessageOf(x)
 }
 
-// Deprecated: Use VoteRequest.ProtoReflect.Descriptor instead.
+// Deprecated: Use IndexRequest.ProtoReflect.Descriptor instead.
-func (*VoteRequest) Descriptor() ([]byte, []int) {
+func (*IndexRequest) Descriptor() ([]byte, []int) {
 	return file_proto_search_proto_rawDescGZIP(), []int{0}
 }
 
-func (x *VoteRequest) GetMessage() string {
+func (x *IndexRequest) GetDocument() *Document {
 	if x != nil {
-		return x.Message
+		return x.Document
+	}
+	return nil
+}
+
+func (x *IndexRequest) GetIndex() string {
+	if x != nil {
+		return x.Index
 	}
 	return ""
 }
 
-type VoteResponse struct {
+type Document struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	// response message
+	// The ID for this document. If blank, one will be generated
-	Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
+	Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
+	// The JSON contents of the document
+	Contents *structpb.Struct `protobuf:"bytes,2,opt,name=contents,proto3" json:"contents,omitempty"`
 }
 
-func (x *VoteResponse) Reset() {
+func (x *Document) Reset() {
-	*x = VoteResponse{}
+	*x = Document{}
 	if protoimpl.UnsafeEnabled {
 		mi := &file_proto_search_proto_msgTypes[1]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -87,13 +99,13 @@ func (x *VoteResponse) Reset() {
 	}
 }
 
-func (x *VoteResponse) String() string {
+func (x *Document) String() string {
 	return protoimpl.X.MessageStringOf(x)
 }
 
-func (*VoteResponse) ProtoMessage() {}
+func (*Document) ProtoMessage() {}
 
-func (x *VoteResponse) ProtoReflect() protoreflect.Message {
+func (x *Document) ProtoReflect() protoreflect.Message {
 	mi := &file_proto_search_proto_msgTypes[1]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -105,34 +117,577 @@ func (x *VoteResponse) ProtoReflect() protoreflect.Message {
 	return mi.MessageOf(x)
 }
 
-// Deprecated: Use VoteResponse.ProtoReflect.Descriptor instead.
+// Deprecated: Use Document.ProtoReflect.Descriptor instead.
-func (*VoteResponse) Descriptor() ([]byte, []int) {
+func (*Document) Descriptor() ([]byte, []int) {
 	return file_proto_search_proto_rawDescGZIP(), []int{1}
 }
 
-func (x *VoteResponse) GetMessage() string {
+func (x *Document) GetId() string {
 	if x != nil {
-		return x.Message
+		return x.Id
 	}
 	return ""
 }
 
+func (x *Document) GetContents() *structpb.Struct {
+	if x != nil {
+		return x.Contents
+	}
+	return nil
+}
+
+type IndexResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
+}
+
+func (x *IndexResponse) Reset() {
+	*x = IndexResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_proto_search_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *IndexResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*IndexResponse) ProtoMessage() {}
+
+func (x *IndexResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_proto_search_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use IndexResponse.ProtoReflect.Descriptor instead.
+func (*IndexResponse) Descriptor() ([]byte, []int) {
+	return file_proto_search_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *IndexResponse) GetId() string {
+	if x != nil {
+		return x.Id
+	}
+	return ""
+}
+
+// Delete a document given its ID
+type DeleteRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The ID of the document to delete
+	Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
+	// The index the document belongs to
+	Index string `protobuf:"bytes,2,opt,name=index,proto3" json:"index,omitempty"`
+}
+
+func (x *DeleteRequest) Reset() {
+	*x = DeleteRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_proto_search_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *DeleteRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*DeleteRequest) ProtoMessage() {}
+
+func (x *DeleteRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_proto_search_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use DeleteRequest.ProtoReflect.Descriptor instead.
+func (*DeleteRequest) Descriptor() ([]byte, []int) {
+	return file_proto_search_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *DeleteRequest) GetId() string {
+	if x != nil {
+		return x.Id
+	}
+	return ""
+}
+
+func (x *DeleteRequest) GetIndex() string {
+	if x != nil {
+		return x.Index
+	}
+	return ""
+}
+
+type DeleteResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+}
+
+func (x *DeleteResponse) Reset() {
+	*x = DeleteResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_proto_search_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *DeleteResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*DeleteResponse) ProtoMessage() {}
+
+func (x *DeleteResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_proto_search_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use DeleteResponse.ProtoReflect.Descriptor instead.
+func (*DeleteResponse) Descriptor() ([]byte, []int) {
+	return file_proto_search_proto_rawDescGZIP(), []int{4}
+}
+
+// Search for documents in a given in index
+type SearchRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The index the document belongs to
+	Index string `protobuf:"bytes,1,opt,name=index,proto3" json:"index,omitempty"`
+	// The query. See docs for query language examples
+	Query string `protobuf:"bytes,2,opt,name=query,proto3" json:"query,omitempty"`
+}
+
+func (x *SearchRequest) Reset() {
+	*x = SearchRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_proto_search_proto_msgTypes[5]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *SearchRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*SearchRequest) ProtoMessage() {}
+
+func (x *SearchRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_proto_search_proto_msgTypes[5]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use SearchRequest.ProtoReflect.Descriptor instead.
+func (*SearchRequest) Descriptor() ([]byte, []int) {
+	return file_proto_search_proto_rawDescGZIP(), []int{5}
+}
+
+func (x *SearchRequest) GetIndex() string {
+	if x != nil {
+		return x.Index
+	}
+	return ""
+}
+
+func (x *SearchRequest) GetQuery() string {
+	if x != nil {
+		return x.Query
+	}
+	return ""
+}
+
+type SearchResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The matching documents
+	Documents []*Document `protobuf:"bytes,1,rep,name=documents,proto3" json:"documents,omitempty"`
+}
+
+func (x *SearchResponse) Reset() {
+	*x = SearchResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_proto_search_proto_msgTypes[6]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *SearchResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*SearchResponse) ProtoMessage() {}
+
+func (x *SearchResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_proto_search_proto_msgTypes[6]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use SearchResponse.ProtoReflect.Descriptor instead.
+func (*SearchResponse) Descriptor() ([]byte, []int) {
+	return file_proto_search_proto_rawDescGZIP(), []int{6}
+}
+
+func (x *SearchResponse) GetDocuments() []*Document {
+	if x != nil {
+		return x.Documents
+	}
+	return nil
+}
+
+// Create a search index by specifying which fields are to be queried
+type CreateIndexRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// the name of the index
+	Index  string   `protobuf:"bytes,1,opt,name=index,proto3" json:"index,omitempty"`
+	Fields []*Field `protobuf:"bytes,2,rep,name=fields,proto3" json:"fields,omitempty"`
+}
+
+func (x *CreateIndexRequest) Reset() {
+	*x = CreateIndexRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_proto_search_proto_msgTypes[7]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *CreateIndexRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*CreateIndexRequest) ProtoMessage() {}
+
+func (x *CreateIndexRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_proto_search_proto_msgTypes[7]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use CreateIndexRequest.ProtoReflect.Descriptor instead.
+func (*CreateIndexRequest) Descriptor() ([]byte, []int) {
+	return file_proto_search_proto_rawDescGZIP(), []int{7}
+}
+
+func (x *CreateIndexRequest) GetIndex() string {
+	if x != nil {
+		return x.Index
+	}
+	return ""
+}
+
+func (x *CreateIndexRequest) GetFields() []*Field {
+	if x != nil {
+		return x.Fields
+	}
+	return nil
+}
+
+type Field struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The name of the field. Use a `.` separator to define nested fields e.g. foo.bar
+	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	// The type of the field - string, number
+	Type string `protobuf:"bytes,2,opt,name=type,proto3" json:"type,omitempty"`
+}
+
+func (x *Field) Reset() {
+	*x = Field{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_proto_search_proto_msgTypes[8]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Field) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Field) ProtoMessage() {}
+
+func (x *Field) ProtoReflect() protoreflect.Message {
+	mi := &file_proto_search_proto_msgTypes[8]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Field.ProtoReflect.Descriptor instead.
+func (*Field) Descriptor() ([]byte, []int) {
+	return file_proto_search_proto_rawDescGZIP(), []int{8}
+}
+
+func (x *Field) GetName() string {
+	if x != nil {
+		return x.Name
+	}
+	return ""
+}
+
+func (x *Field) GetType() string {
+	if x != nil {
+		return x.Type
+	}
+	return ""
+}
+
+type CreateIndexResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+}
+
+func (x *CreateIndexResponse) Reset() {
+	*x = CreateIndexResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_proto_search_proto_msgTypes[9]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *CreateIndexResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*CreateIndexResponse) ProtoMessage() {}
+
+func (x *CreateIndexResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_proto_search_proto_msgTypes[9]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use CreateIndexResponse.ProtoReflect.Descriptor instead.
+func (*CreateIndexResponse) Descriptor() ([]byte, []int) {
+	return file_proto_search_proto_rawDescGZIP(), []int{9}
+}
+
+// Delete an index.
+type DeleteIndexRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The name of the index to delete
+	Index string `protobuf:"bytes,1,opt,name=index,proto3" json:"index,omitempty"`
+}
+
+func (x *DeleteIndexRequest) Reset() {
+	*x = DeleteIndexRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_proto_search_proto_msgTypes[10]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *DeleteIndexRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*DeleteIndexRequest) ProtoMessage() {}
+
+func (x *DeleteIndexRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_proto_search_proto_msgTypes[10]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use DeleteIndexRequest.ProtoReflect.Descriptor instead.
+func (*DeleteIndexRequest) Descriptor() ([]byte, []int) {
+	return file_proto_search_proto_rawDescGZIP(), []int{10}
+}
+
+func (x *DeleteIndexRequest) GetIndex() string {
+	if x != nil {
+		return x.Index
+	}
+	return ""
+}
+
+type DeleteIndexResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+}
+
+func (x *DeleteIndexResponse) Reset() {
+	*x = DeleteIndexResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_proto_search_proto_msgTypes[11]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *DeleteIndexResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*DeleteIndexResponse) ProtoMessage() {}
+
+func (x *DeleteIndexResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_proto_search_proto_msgTypes[11]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use DeleteIndexResponse.ProtoReflect.Descriptor instead.
+func (*DeleteIndexResponse) Descriptor() ([]byte, []int) {
+	return file_proto_search_proto_rawDescGZIP(), []int{11}
+}
+
 var File_proto_search_proto protoreflect.FileDescriptor
 
 var file_proto_search_proto_rawDesc = []byte{
 	0x0a, 0x12, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x70,
-	0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x22, 0x27, 0x0a, 0x0b,
+	0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x1a, 0x1c, 0x67, 0x6f,
-	0x56, 0x6f, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x6d,
+	0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x73, 0x74,
-	0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65,
+	0x72, 0x75, 0x63, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x52, 0x0a, 0x0c, 0x49, 0x6e,
-	0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x28, 0x0a, 0x0c, 0x56, 0x6f, 0x74, 0x65, 0x52, 0x65, 0x73,
+	0x64, 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2c, 0x0a, 0x08, 0x64, 0x6f,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65,
+	0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x73,
-	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x32,
+	0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x44, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x08,
-	0x3d, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x33, 0x0a, 0x04, 0x56, 0x6f, 0x74,
+	0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x64, 0x65,
-	0x65, 0x12, 0x13, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x56, 0x6f, 0x74, 0x65, 0x52,
+	0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x22, 0x4f,
-	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e,
+	0x0a, 0x08, 0x44, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64,
-	0x56, 0x6f, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x10,
+	0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x33, 0x0a, 0x08, 0x63, 0x6f,
-	0x5a, 0x0e, 0x2e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3b, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68,
+	0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67,
-	0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53,
+	0x74, 0x72, 0x75, 0x63, 0x74, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x22,
+	0x1f, 0x0a, 0x0d, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64,
+	0x22, 0x35, 0x0a, 0x0d, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69,
+	0x64, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x22, 0x10, 0x0a, 0x0e, 0x44, 0x65, 0x6c, 0x65, 0x74,
+	0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x3b, 0x0a, 0x0d, 0x53, 0x65, 0x61,
+	0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e,
+	0x64, 0x65, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78,
+	0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x22, 0x40, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x09, 0x64, 0x6f, 0x63, 0x75,
+	0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x73, 0x65,
+	0x61, 0x72, 0x63, 0x68, 0x2e, 0x44, 0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x09, 0x64,
+	0x6f, 0x63, 0x75, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x51, 0x0a, 0x12, 0x43, 0x72, 0x65, 0x61,
+	0x74, 0x65, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14,
+	0x0a, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69,
+	0x6e, 0x64, 0x65, 0x78, 0x12, 0x25, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02,
+	0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x46, 0x69,
+	0x65, 0x6c, 0x64, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x22, 0x2f, 0x0a, 0x05, 0x46,
+	0x69, 0x65, 0x6c, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01,
+	0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x15, 0x0a, 0x13,
+	0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x6e, 0x73, 0x65, 0x22, 0x2a, 0x0a, 0x12, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x6e, 0x64,
+	0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x64,
+	0x65, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x22,
+	0x15, 0x0a, 0x13, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x80, 0x02, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63,
+	0x68, 0x12, 0x36, 0x0a, 0x05, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x14, 0x2e, 0x73, 0x65, 0x61,
+	0x72, 0x63, 0x68, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x1a, 0x15, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x06, 0x44, 0x65, 0x6c,
+	0x65, 0x74, 0x65, 0x12, 0x15, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x44, 0x65, 0x6c,
+	0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x73, 0x65, 0x61,
+	0x72, 0x63, 0x68, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
+	0x73, 0x65, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x15,
+	0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x53,
+	0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12,
+	0x48, 0x0a, 0x0b, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x1a,
+	0x2e, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x6e,
+	0x64, 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x73, 0x65, 0x61,
+	0x72, 0x63, 0x68, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52,
+	0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x10, 0x5a, 0x0e, 0x2e, 0x2f, 0x70,
+	0x72, 0x6f, 0x74, 0x6f, 0x3b, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x62, 0x06, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x33,
 }
 
 var (
@@ -147,19 +702,40 @@ func file_proto_search_proto_rawDescGZIP() []byte {
 	return file_proto_search_proto_rawDescData
 }
 
-var file_proto_search_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
+var file_proto_search_proto_msgTypes = make([]protoimpl.MessageInfo, 12)
 var file_proto_search_proto_goTypes = []interface{}{
-	(*VoteRequest)(nil),  // 0: search.VoteRequest
+	(*IndexRequest)(nil),        // 0: search.IndexRequest
-	(*VoteResponse)(nil), // 1: search.VoteResponse
+	(*Document)(nil),            // 1: search.Document
+	(*IndexResponse)(nil),       // 2: search.IndexResponse
+	(*DeleteRequest)(nil),       // 3: search.DeleteRequest
+	(*DeleteResponse)(nil),      // 4: search.DeleteResponse
+	(*SearchRequest)(nil),       // 5: search.SearchRequest
+	(*SearchResponse)(nil),      // 6: search.SearchResponse
+	(*CreateIndexRequest)(nil),  // 7: search.CreateIndexRequest
+	(*Field)(nil),               // 8: search.Field
+	(*CreateIndexResponse)(nil), // 9: search.CreateIndexResponse
+	(*DeleteIndexRequest)(nil),  // 10: search.DeleteIndexRequest
+	(*DeleteIndexResponse)(nil), // 11: search.DeleteIndexResponse
+	(*structpb.Struct)(nil),     // 12: google.protobuf.Struct
 }
 var file_proto_search_proto_depIdxs = []int32{
-	0, // 0: search.Search.Vote:input_type -> search.VoteRequest
+	1,  // 0: search.IndexRequest.document:type_name -> search.Document
-	1, // 1: search.Search.Vote:output_type -> search.VoteResponse
+	12, // 1: search.Document.contents:type_name -> google.protobuf.Struct
-	1, // [1:2] is the sub-list for method output_type
+	1,  // 2: search.SearchResponse.documents:type_name -> search.Document
-	0, // [0:1] is the sub-list for method input_type
+	8,  // 3: search.CreateIndexRequest.fields:type_name -> search.Field
-	0, // [0:0] is the sub-list for extension type_name
+	0,  // 4: search.Search.Index:input_type -> search.IndexRequest
-	0, // [0:0] is the sub-list for extension extendee
+	3,  // 5: search.Search.Delete:input_type -> search.DeleteRequest
-	0, // [0:0] is the sub-list for field type_name
+	5,  // 6: search.Search.Search:input_type -> search.SearchRequest
+	10, // 7: search.Search.DeleteIndex:input_type -> search.DeleteIndexRequest
+	2,  // 8: search.Search.Index:output_type -> search.IndexResponse
+	4,  // 9: search.Search.Delete:output_type -> search.DeleteResponse
+	6,  // 10: search.Search.Search:output_type -> search.SearchResponse
+	11, // 11: search.Search.DeleteIndex:output_type -> search.DeleteIndexResponse
+	8,  // [8:12] is the sub-list for method output_type
+	4,  // [4:8] is the sub-list for method input_type
+	4,  // [4:4] is the sub-list for extension type_name
+	4,  // [4:4] is the sub-list for extension extendee
+	0,  // [0:4] is the sub-list for field type_name
 }
 
 func init() { file_proto_search_proto_init() }
@@ -169,7 +745,7 @@ func file_proto_search_proto_init() {
 	}
 	if !protoimpl.UnsafeEnabled {
 		file_proto_search_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*VoteRequest); i {
+			switch v := v.(*IndexRequest); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -181,7 +757,127 @@ func file_proto_search_proto_init() {
 			}
 		}
 		file_proto_search_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*VoteResponse); i {
+			switch v := v.(*Document); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_proto_search_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*IndexResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_proto_search_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*DeleteRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_proto_search_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*DeleteResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_proto_search_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SearchRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_proto_search_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*SearchResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_proto_search_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*CreateIndexRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_proto_search_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Field); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_proto_search_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*CreateIndexResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_proto_search_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*DeleteIndexRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_proto_search_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*DeleteIndexResponse); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -199,7 +895,7 @@ func file_proto_search_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_proto_search_proto_rawDesc,
 			NumEnums:      0,
-			NumMessages:   2,
+			NumMessages:   12,
 			NumExtensions: 0,
 			NumServices:   1,
 		},

search/proto/search.pb.micro.go

@@ -6,6 +6,7 @@ package search
 import (
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
+	_ "google.golang.org/protobuf/types/known/structpb"
 	math "math"
 )
 
@@ -42,7 +43,10 @@ func NewSearchEndpoints() []*api.Endpoint {
 // Client API for Search service
 
 type SearchService interface {
-	Vote(ctx context.Context, in *VoteRequest, opts ...client.CallOption) (*VoteResponse, error)
+	Index(ctx context.Context, in *IndexRequest, opts ...client.CallOption) (*IndexResponse, error)
+	Delete(ctx context.Context, in *DeleteRequest, opts ...client.CallOption) (*DeleteResponse, error)
+	Search(ctx context.Context, in *SearchRequest, opts ...client.CallOption) (*SearchResponse, error)
+	DeleteIndex(ctx context.Context, in *DeleteIndexRequest, opts ...client.CallOption) (*DeleteIndexResponse, error)
 }
 
 type searchService struct {
@@ -57,9 +61,39 @@ func NewSearchService(name string, c client.Client) SearchService {
 	}
 }
 
-func (c *searchService) Vote(ctx context.Context, in *VoteRequest, opts ...client.CallOption) (*VoteResponse, error) {
+func (c *searchService) Index(ctx context.Context, in *IndexRequest, opts ...client.CallOption) (*IndexResponse, error) {
-	req := c.c.NewRequest(c.name, "Search.Vote", in)
+	req := c.c.NewRequest(c.name, "Search.Index", in)
-	out := new(VoteResponse)
+	out := new(IndexResponse)
+	err := c.c.Call(ctx, req, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *searchService) Delete(ctx context.Context, in *DeleteRequest, opts ...client.CallOption) (*DeleteResponse, error) {
+	req := c.c.NewRequest(c.name, "Search.Delete", in)
+	out := new(DeleteResponse)
+	err := c.c.Call(ctx, req, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *searchService) Search(ctx context.Context, in *SearchRequest, opts ...client.CallOption) (*SearchResponse, error) {
+	req := c.c.NewRequest(c.name, "Search.Search", in)
+	out := new(SearchResponse)
+	err := c.c.Call(ctx, req, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *searchService) DeleteIndex(ctx context.Context, in *DeleteIndexRequest, opts ...client.CallOption) (*DeleteIndexResponse, error) {
+	req := c.c.NewRequest(c.name, "Search.DeleteIndex", in)
+	out := new(DeleteIndexResponse)
 	err := c.c.Call(ctx, req, out, opts...)
 	if err != nil {
 		return nil, err
@@ -70,12 +104,18 @@ func (c *searchService) Vote(ctx context.Context, in *VoteRequest, opts ...clien
 // Server API for Search service
 
 type SearchHandler interface {
-	Vote(context.Context, *VoteRequest, *VoteResponse) error
+	Index(context.Context, *IndexRequest, *IndexResponse) error
+	Delete(context.Context, *DeleteRequest, *DeleteResponse) error
+	Search(context.Context, *SearchRequest, *SearchResponse) error
+	DeleteIndex(context.Context, *DeleteIndexRequest, *DeleteIndexResponse) error
 }
 
 func RegisterSearchHandler(s server.Server, hdlr SearchHandler, opts ...server.HandlerOption) error {
 	type search interface {
-		Vote(ctx context.Context, in *VoteRequest, out *VoteResponse) error
+		Index(ctx context.Context, in *IndexRequest, out *IndexResponse) error
+		Delete(ctx context.Context, in *DeleteRequest, out *DeleteResponse) error
+		Search(ctx context.Context, in *SearchRequest, out *SearchResponse) error
+		DeleteIndex(ctx context.Context, in *DeleteIndexRequest, out *DeleteIndexResponse) error
 	}
 	type Search struct {
 		search
@@ -88,6 +128,18 @@ type searchHandler struct {
 	SearchHandler
 }
 
-func (h *searchHandler) Vote(ctx context.Context, in *VoteRequest, out *VoteResponse) error {
+func (h *searchHandler) Index(ctx context.Context, in *IndexRequest, out *IndexResponse) error {
-	return h.SearchHandler.Vote(ctx, in, out)
+	return h.SearchHandler.Index(ctx, in, out)
+}
+
+func (h *searchHandler) Delete(ctx context.Context, in *DeleteRequest, out *DeleteResponse) error {
+	return h.SearchHandler.Delete(ctx, in, out)
+}
+
+func (h *searchHandler) Search(ctx context.Context, in *SearchRequest, out *SearchResponse) error {
+	return h.SearchHandler.Search(ctx, in, out)
+}
+
+func (h *searchHandler) DeleteIndex(ctx context.Context, in *DeleteIndexRequest, out *DeleteIndexResponse) error {
+	return h.SearchHandler.DeleteIndex(ctx, in, out)
 }

search/proto/search.proto

@@ -1,20 +1,87 @@
 syntax = "proto3";
 
+import "google/protobuf/struct.proto";
+
 package search;
 
 option go_package = "./proto;search";
 
 service Search {
-	rpc Vote(VoteRequest) returns (VoteResponse) {}
+	// TODO reinstate when we have a reason for more fine grained control of index creation, for now just rely on lazy creation on first index call
+	//	rpc CreateIndex(CreateIndexRequest) returns (CreateIndexResponse) {}
+
+	rpc Index(IndexRequest) returns (IndexResponse) {}
+	rpc Delete(DeleteRequest) returns (DeleteResponse) {}
+	rpc Search(SearchRequest) returns (SearchResponse) {}
+	rpc DeleteIndex(DeleteIndexRequest) returns (DeleteIndexResponse) {}
+
 }
 
-// Vote to have the Search api launched faster!
+// Index a document i.e. insert a document to search for.
-message VoteRequest {
+message IndexRequest {
-	// optional message
+	// The document to index
-	string message = 1;
+	Document document = 1;
+	// The index this document belongs to
+	string index = 2;
+
 }
 
-message VoteResponse {
+message Document {
-	// response message
+	// The ID for this document. If blank, one will be generated
-	string message = 2;
+	string id = 1;
+	// The JSON contents of the document
+	google.protobuf.Struct contents = 2;
 }
+
+message IndexResponse {
+	string id = 1;
+}
+
+// Delete a document given its ID
+message DeleteRequest {
+	// The ID of the document to delete
+	string id = 1;
+	// The index the document belongs to
+	string index = 2;
+}
+
+message DeleteResponse {}
+
+// Search for documents in a given in index
+message SearchRequest {
+	// The index the document belongs to
+	string index = 1;
+
+	// The query. See docs for query language examples
+	string query = 2;
+}
+
+message SearchResponse {
+	// The matching documents
+	repeated Document documents = 1;
+
+}
+
+// Create a search index by specifying which fields are to be queried
+message CreateIndexRequest {
+	// the name of the index
+	string index = 1;
+	repeated Field fields = 2;
+}
+
+message Field {
+	// The name of the field. Use a `.` separator to define nested fields e.g. foo.bar
+	string name = 1;
+	// The type of the field - string, number
+	string type = 2;
+}
+
+message CreateIndexResponse {}
+
+// Delete an index.
+message DeleteIndexRequest {
+	// The name of the index to delete
+	string index = 1;
+}
+
+message DeleteIndexResponse {}

search/publicapi.json

@@ -1,6 +1,6 @@
 {
   "name": "search",
   "icon": "🔎",
-  "category": "coming soon",
+  "category": "search",
-  "display_name": "Search (Coming Soon)"
+  "display_name": "Search"
 }

space/handler/space_test.go

@@ -85,24 +85,23 @@ func (m mockError) OrigErr() error {
 }
 
 func TestCreate(t *testing.T) {
-	g := NewWithT(t)
 	tcs := []struct {
 		name       string
 		objName    string
 		visibility string
 		err        error
 		url        string
-		head       func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error)
+		head       func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error)
-		put        func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error)
+		put        func(input *sthree.PutObjectInput, g *WithT) (*sthree.PutObjectOutput, error)
 	}{
 		{
 			name:    "Simple case",
 			objName: "foo.jpg",
 			url:     "",
-			head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) {
+			head: func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error) {
 				return nil, mockError{code: "NotFound"}
 			},
-			put: func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) {
+			put: func(input *sthree.PutObjectInput, g *WithT) (*sthree.PutObjectOutput, error) {
 				g.Expect(*input.Bucket).To(Equal("my-space"))
 				g.Expect(input.ACL).To(BeNil())
 				return &sthree.PutObjectOutput{}, nil
@@ -113,10 +112,10 @@ func TestCreate(t *testing.T) {
 			objName:    "bar/baz/foo.jpg",
 			visibility: "public",
 			url:        "https://my-space.ams3.example.com/micro/123/bar/baz/foo.jpg",
-			head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) {
+			head: func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error) {
 				return nil, mockError{code: "NotFound"}
 			},
-			put: func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) {
+			put: func(input *sthree.PutObjectInput, g *WithT) (*sthree.PutObjectOutput, error) {
 				g.Expect(*input.Bucket).To(Equal("my-space"))
 				g.Expect(*input.ACL).To(Equal(mdACLPublic))
 				return &sthree.PutObjectOutput{}, nil
@@ -131,7 +130,7 @@ func TestCreate(t *testing.T) {
 			name:    "Already exists",
 			objName: "foo.jpg",
 			err:     errors.BadRequest("space.Create", "Object already exists"),
-			head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) {
+			head: func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error) {
 				g.Expect(*input.Bucket).To(Equal("my-space"))
 				g.Expect(*input.Key).To(Equal("micro/123/foo.jpg"))
 				return &sthree.HeadObjectOutput{}, nil
@@ -142,6 +141,7 @@ func TestCreate(t *testing.T) {
 
 	for _, tc := range tcs {
 		t.Run(tc.name, func(t *testing.T) {
+			g := NewWithT(t)
 			handler := Space{
 				conf: conf{
 					AccessKey: "access",
@@ -152,7 +152,13 @@ func TestCreate(t *testing.T) {
 					Region:    "ams3",
 					BaseURL:   "https://my-space.ams3.example.com",
 				},
-				client: &mockS3Client{head: tc.head, put: tc.put},
+				client: &mockS3Client{
+					head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) {
+						return tc.head(input, g)
+					},
+					put: func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) {
+						return tc.put(input, g)
+					}},
 			}
 			ctx := context.Background()
 			ctx = auth.ContextWithAccount(ctx, &auth.Account{
@@ -182,24 +188,23 @@ func TestCreate(t *testing.T) {
 }
 
 func TestUpdate(t *testing.T) {
-	g := NewWithT(t)
 	tcs := []struct {
 		name       string
 		objName    string
 		visibility string
 		err        error
 		url        string
-		head       func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error)
+		head       func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error)
-		put        func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error)
+		put        func(input *sthree.PutObjectInput, g *WithT) (*sthree.PutObjectOutput, error)
 	}{
 		{
 			name:    "Does not exist",
 			objName: "foo.jpg",
 			url:     "",
-			head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) {
+			head: func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error) {
 				return nil, mockError{code: "NotFound"}
 			},
-			put: func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) {
+			put: func(input *sthree.PutObjectInput, g *WithT) (*sthree.PutObjectOutput, error) {
 				g.Expect(*input.Bucket).To(Equal("my-space"))
 				g.Expect(input.ACL).To(BeNil())
 				return &sthree.PutObjectOutput{}, nil
@@ -210,10 +215,10 @@ func TestUpdate(t *testing.T) {
 			objName:    "bar/baz/foo.jpg",
 			visibility: "public",
 			url:        "https://my-space.ams3.example.com/micro/123/bar/baz/foo.jpg",
-			head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) {
+			head: func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error) {
 				return nil, mockError{code: "NotFound"}
 			},
-			put: func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) {
+			put: func(input *sthree.PutObjectInput, g *WithT) (*sthree.PutObjectOutput, error) {
 				g.Expect(*input.Bucket).To(Equal("my-space"))
 				g.Expect(*input.ACL).To(Equal(mdACLPublic))
 				return &sthree.PutObjectOutput{}, nil
@@ -227,12 +232,12 @@ func TestUpdate(t *testing.T) {
 		{
 			name:    "Already exists",
 			objName: "foo.jpg",
-			head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) {
+			head: func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error) {
 				g.Expect(*input.Bucket).To(Equal("my-space"))
 				g.Expect(*input.Key).To(Equal("micro/123/foo.jpg"))
 				return &sthree.HeadObjectOutput{}, nil
 			},
-			put: func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) {
+			put: func(input *sthree.PutObjectInput, g *WithT) (*sthree.PutObjectOutput, error) {
 				g.Expect(*input.Bucket).To(Equal("my-space"))
 				g.Expect(input.ACL).To(BeNil())
 				return &sthree.PutObjectOutput{}, nil
@@ -242,12 +247,12 @@ func TestUpdate(t *testing.T) {
 		{
 			name:    "Already exists public",
 			objName: "foo.jpg",
-			head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) {
+			head: func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error) {
 				g.Expect(*input.Bucket).To(Equal("my-space"))
 				g.Expect(*input.Key).To(Equal("micro/123/foo.jpg"))
 				return &sthree.HeadObjectOutput{}, nil
 			},
-			put: func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) {
+			put: func(input *sthree.PutObjectInput, g *WithT) (*sthree.PutObjectOutput, error) {
 				g.Expect(*input.Bucket).To(Equal("my-space"))
 				g.Expect(*input.ACL).To(Equal(mdACLPublic))
 				return &sthree.PutObjectOutput{}, nil
@@ -260,6 +265,7 @@ func TestUpdate(t *testing.T) {
 
 	for _, tc := range tcs {
 		t.Run(tc.name, func(t *testing.T) {
+			g := NewWithT(t)
 			handler := Space{
 				conf: conf{
 					AccessKey: "access",
@@ -270,7 +276,13 @@ func TestUpdate(t *testing.T) {
 					Region:    "ams3",
 					BaseURL:   "https://my-space.ams3.example.com",
 				},
-				client: &mockS3Client{head: tc.head, put: tc.put},
+				client: &mockS3Client{
+					head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) {
+						return tc.head(input, g)
+					},
+					put: func(input *sthree.PutObjectInput) (*sthree.PutObjectOutput, error) {
+						return tc.put(input, g)
+					}},
 			}
 			ctx := context.Background()
 			ctx = auth.ContextWithAccount(ctx, &auth.Account{
@@ -301,7 +313,6 @@ func TestUpdate(t *testing.T) {
 }
 
 func TestDelete(t *testing.T) {
-	g := NewWithT(t)
 	tcs := []struct {
 		name    string
 		objName string
@@ -322,6 +333,7 @@ func TestDelete(t *testing.T) {
 
 	for _, tc := range tcs {
 		t.Run(tc.name, func(t *testing.T) {
+			g := NewWithT(t)
 			handler := Space{
 				conf: conf{
 					AccessKey: "access",
@@ -364,7 +376,6 @@ func TestDelete(t *testing.T) {
 }
 
 func TestList(t *testing.T) {
-	g := NewWithT(t)
 	tcs := []struct {
 		name       string
 		prefix     string
@@ -383,6 +394,7 @@ func TestList(t *testing.T) {
 
 	for _, tc := range tcs {
 		t.Run(tc.name, func(t *testing.T) {
+			g := NewWithT(t)
 			store.DefaultStore = memory.NewStore()
 			store.Write(
 				store.NewRecord(fmt.Sprintf("%s/micro/123/file.jpg", prefixByUser),
@@ -463,7 +475,6 @@ func TestList(t *testing.T) {
 }
 
 func TestHead(t *testing.T) {
-	g := NewWithT(t)
 	tcs := []struct {
 		name       string
 		objectName string
@@ -472,7 +483,7 @@ func TestHead(t *testing.T) {
 		modified   string
 		created    string
 		err        error
-		head       func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error)
+		head       func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error)
 	}{
 		{
 			name:       "Simple case",
@@ -481,7 +492,7 @@ func TestHead(t *testing.T) {
 			url:        "https://my-space.ams3.example.com/micro/123/foo.jpg",
 			created:    "2009-11-10T23:00:00Z",
 			modified:   "2009-11-10T23:00:00Z",
-			head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) {
+			head: func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error) {
 				g.Expect(*input.Bucket).To(Equal("my-space"))
 				g.Expect(*input.Key).To(Equal("micro/123/foo.jpg"))
 
@@ -497,7 +508,7 @@ func TestHead(t *testing.T) {
 			url:        "",
 			created:    "2009-11-10T23:00:00Z",
 			modified:   "2009-11-10T23:00:00Z",
-			head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) {
+			head: func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error) {
 				g.Expect(*input.Bucket).To(Equal("my-space"))
 				g.Expect(*input.Key).To(Equal("micro/123/foo.jpg"))
 
@@ -514,7 +525,7 @@ func TestHead(t *testing.T) {
 			name:       "Not found",
 			objectName: "foo.jpg",
 			err:        errors.BadRequest("space.Head", "Object not found"),
-			head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) {
+			head: func(input *sthree.HeadObjectInput, g *WithT) (*sthree.HeadObjectOutput, error) {
 				return nil, mockError{code: "NotFound"}
 			},
 		},
@@ -522,6 +533,7 @@ func TestHead(t *testing.T) {
 
 	for _, tc := range tcs {
 		t.Run(tc.name, func(t *testing.T) {
+			g := NewWithT(t)
 			store.DefaultStore = memory.NewStore()
 			store.Write(store.NewRecord(fmt.Sprintf("%s/micro/123/%s", prefixByUser, tc.objectName), meta{
 				Visibility:   tc.visibility,
@@ -539,7 +551,9 @@ func TestHead(t *testing.T) {
 					BaseURL:   "https://my-space.ams3.example.com",
 				},
 				client: &mockS3Client{
-					head: tc.head,
+					head: func(input *sthree.HeadObjectInput) (*sthree.HeadObjectOutput, error) {
+						return tc.head(input, g)
+					},
 				},
 			}
 			ctx := context.Background()