Upgrade blevesearch dependency to v2.0.1 (#14346)

* Upgrade blevesearch dependency to v2.0.1

* Update rupture to v1.0.0

* Fix test

vendor/github.com/blevesearch/bleve/v2/search/highlight/format/html/html.go

@@ -0,0 +1,91 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package html
+
+import (
+	"html"
+
+	"github.com/blevesearch/bleve/v2/registry"
+	"github.com/blevesearch/bleve/v2/search/highlight"
+)
+
+const Name = "html"
+
+const defaultHTMLHighlightBefore = "<mark>"
+const defaultHTMLHighlightAfter = "</mark>"
+
+type FragmentFormatter struct {
+	before string
+	after  string
+}
+
+func NewFragmentFormatter(before, after string) *FragmentFormatter {
+	return &FragmentFormatter{
+		before: before,
+		after:  after,
+	}
+}
+
+func (a *FragmentFormatter) Format(f *highlight.Fragment, orderedTermLocations highlight.TermLocations) string {
+	rv := ""
+	curr := f.Start
+	for _, termLocation := range orderedTermLocations {
+		if termLocation == nil {
+			continue
+		}
+		// make sure the array positions match
+		if !termLocation.ArrayPositions.Equals(f.ArrayPositions) {
+			continue
+		}
+		if termLocation.Start < curr {
+			continue
+		}
+		if termLocation.End > f.End {
+			break
+		}
+		// add the stuff before this location
+		rv += html.EscapeString(string(f.Orig[curr:termLocation.Start]))
+		// start the <mark> tag
+		rv += a.before
+		// add the term itself
+		rv += html.EscapeString(string(f.Orig[termLocation.Start:termLocation.End]))
+		// end the <mark> tag
+		rv += a.after
+		// update current
+		curr = termLocation.End
+	}
+	// add any remaining text after the last token
+	rv += html.EscapeString(string(f.Orig[curr:f.End]))
+
+	return rv
+}
+
+func Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.FragmentFormatter, error) {
+	before := defaultHTMLHighlightBefore
+	beforeVal, ok := config["before"].(string)
+	if ok {
+		before = beforeVal
+	}
+	after := defaultHTMLHighlightAfter
+	afterVal, ok := config["after"].(string)
+	if ok {
+		after = afterVal
+	}
+	return NewFragmentFormatter(before, after), nil
+}
+
+func init() {
+	registry.RegisterFragmentFormatter(Name, Constructor)
+}

vendor/github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple/simple.go

@@ -0,0 +1,147 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package simple
+
+import (
+	"unicode/utf8"
+
+	"github.com/blevesearch/bleve/v2/registry"
+	"github.com/blevesearch/bleve/v2/search/highlight"
+)
+
+const Name = "simple"
+
+const defaultFragmentSize = 200
+
+type Fragmenter struct {
+	fragmentSize int
+}
+
+func NewFragmenter(fragmentSize int) *Fragmenter {
+	return &Fragmenter{
+		fragmentSize: fragmentSize,
+	}
+}
+
+func (s *Fragmenter) Fragment(orig []byte, ot highlight.TermLocations) []*highlight.Fragment {
+	var rv []*highlight.Fragment
+	maxbegin := 0
+OUTER:
+	for currTermIndex, termLocation := range ot {
+		// start with this
+		// it should be the highest scoring fragment with this term first
+		start := termLocation.Start
+		end := start
+		used := 0
+		for end < len(orig) && used < s.fragmentSize {
+			r, size := utf8.DecodeRune(orig[end:])
+			if r == utf8.RuneError {
+				continue OUTER // bail
+			}
+			end += size
+			used++
+		}
+
+		// if we still have more characters available to us
+		// push back towards beginning
+		// without cross maxbegin
+		for start > 0 && used < s.fragmentSize {
+			if start > len(orig) {
+				// bail if out of bounds, possibly due to token replacement
+				// e.g with a regexp replacement
+				continue OUTER
+			}
+			r, size := utf8.DecodeLastRune(orig[0:start])
+			if r == utf8.RuneError {
+				continue OUTER // bail
+			}
+			if start-size >= maxbegin {
+				start -= size
+				used++
+			} else {
+				break
+			}
+		}
+
+		// however, we'd rather have the tokens centered more in the frag
+		// lets try to do that as best we can, without affecting the score
+		// find the end of the last term in this fragment
+		minend := end
+		for _, innerTermLocation := range ot[currTermIndex:] {
+			if innerTermLocation.End > end {
+				break
+			}
+			minend = innerTermLocation.End
+		}
+
+		// find the smaller of the two rooms to move
+		roomToMove := utf8.RuneCount(orig[minend:end])
+		roomToMoveStart := 0
+		if start >= maxbegin {
+			roomToMoveStart = utf8.RuneCount(orig[maxbegin:start])
+		}
+		if roomToMoveStart < roomToMove {
+			roomToMove = roomToMoveStart
+		}
+
+		offset := roomToMove / 2
+
+		for offset > 0 {
+			r, size := utf8.DecodeLastRune(orig[0:start])
+			if r == utf8.RuneError {
+				continue OUTER // bail
+			}
+			start -= size
+
+			r, size = utf8.DecodeLastRune(orig[0:end])
+			if r == utf8.RuneError {
+				continue OUTER // bail
+			}
+			end -= size
+			offset--
+		}
+
+		rv = append(rv, &highlight.Fragment{Orig: orig, Start: start - offset, End: end - offset})
+		// set maxbegin to the end of the current term location
+		// so that next one won't back up to include it
+		maxbegin = termLocation.End
+
+	}
+	if len(ot) == 0 {
+		// if there were no terms to highlight
+		// produce a single fragment from the beginning
+		start := 0
+		end := start + s.fragmentSize
+		if end > len(orig) {
+			end = len(orig)
+		}
+		rv = append(rv, &highlight.Fragment{Orig: orig, Start: start, End: end})
+	}
+
+	return rv
+}
+
+func Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.Fragmenter, error) {
+	size := defaultFragmentSize
+	sizeVal, ok := config["size"].(float64)
+	if ok {
+		size = int(sizeVal)
+	}
+	return NewFragmenter(size), nil
+}
+
+func init() {
+	registry.RegisterFragmenter(Name, Constructor)
+}

vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter.go

@@ -0,0 +1,64 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package highlight
+
+import (
+	"github.com/blevesearch/bleve/v2/search"
+	index "github.com/blevesearch/bleve_index_api"
+)
+
+type Fragment struct {
+	Orig           []byte
+	ArrayPositions []uint64
+	Start          int
+	End            int
+	Score          float64
+	Index          int // used by heap
+}
+
+func (f *Fragment) Overlaps(other *Fragment) bool {
+	if other.Start >= f.Start && other.Start < f.End {
+		return true
+	} else if f.Start >= other.Start && f.Start < other.End {
+		return true
+	}
+	return false
+}
+
+type Fragmenter interface {
+	Fragment([]byte, TermLocations) []*Fragment
+}
+
+type FragmentFormatter interface {
+	Format(f *Fragment, orderedTermLocations TermLocations) string
+}
+
+type FragmentScorer interface {
+	Score(f *Fragment) float64
+}
+
+type Highlighter interface {
+	Fragmenter() Fragmenter
+	SetFragmenter(Fragmenter)
+
+	FragmentFormatter() FragmentFormatter
+	SetFragmentFormatter(FragmentFormatter)
+
+	Separator() string
+	SetSeparator(string)
+
+	BestFragmentInField(*search.DocumentMatch, index.Document, string) string
+	BestFragmentsInField(*search.DocumentMatch, index.Document, string, int) []string
+}

vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter/html/html.go

@@ -0,0 +1,50 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package html
+
+import (
+	"fmt"
+
+	"github.com/blevesearch/bleve/v2/registry"
+	"github.com/blevesearch/bleve/v2/search/highlight"
+	htmlFormatter "github.com/blevesearch/bleve/v2/search/highlight/format/html"
+	simpleFragmenter "github.com/blevesearch/bleve/v2/search/highlight/fragmenter/simple"
+	simpleHighlighter "github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple"
+)
+
+const Name = "html"
+
+func Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.Highlighter, error) {
+
+	fragmenter, err := cache.FragmenterNamed(simpleFragmenter.Name)
+	if err != nil {
+		return nil, fmt.Errorf("error building fragmenter: %v", err)
+	}
+
+	formatter, err := cache.FragmentFormatterNamed(htmlFormatter.Name)
+	if err != nil {
+		return nil, fmt.Errorf("error building fragment formatter: %v", err)
+	}
+
+	return simpleHighlighter.NewHighlighter(
+			fragmenter,
+			formatter,
+			simpleHighlighter.DefaultSeparator),
+		nil
+}
+
+func init() {
+	registry.RegisterHighlighter(Name, Constructor)
+}

vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple/fragment_scorer_simple.go

@@ -0,0 +1,49 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package simple
+
+import (
+	"github.com/blevesearch/bleve/v2/search"
+	"github.com/blevesearch/bleve/v2/search/highlight"
+)
+
+// FragmentScorer will score fragments by how many
+// unique terms occur in the fragment with no regard for
+// any boost values used in the original query
+type FragmentScorer struct {
+	tlm search.TermLocationMap
+}
+
+func NewFragmentScorer(tlm search.TermLocationMap) *FragmentScorer {
+	return &FragmentScorer{
+		tlm: tlm,
+	}
+}
+
+func (s *FragmentScorer) Score(f *highlight.Fragment) {
+	score := 0.0
+OUTER:
+	for _, locations := range s.tlm {
+		for _, location := range locations {
+			if location.ArrayPositions.Equals(f.ArrayPositions) && int(location.Start) >= f.Start && int(location.End) <= f.End {
+				score += 1.0
+				// once we find a term in the fragment
+				// don't care about additional matches
+				continue OUTER
+			}
+		}
+	}
+	f.Score = score
+}

vendor/github.com/blevesearch/bleve/v2/search/highlight/highlighter/simple/highlighter_simple.go

@@ -0,0 +1,221 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package simple
+
+import (
+	"container/heap"
+	"fmt"
+	index "github.com/blevesearch/bleve_index_api"
+
+	"github.com/blevesearch/bleve/v2/registry"
+	"github.com/blevesearch/bleve/v2/search"
+	"github.com/blevesearch/bleve/v2/search/highlight"
+)
+
+const Name = "simple"
+const DefaultSeparator = "…"
+
+type Highlighter struct {
+	fragmenter highlight.Fragmenter
+	formatter  highlight.FragmentFormatter
+	sep        string
+}
+
+func NewHighlighter(fragmenter highlight.Fragmenter, formatter highlight.FragmentFormatter, separator string) *Highlighter {
+	return &Highlighter{
+		fragmenter: fragmenter,
+		formatter:  formatter,
+		sep:        separator,
+	}
+}
+
+func (s *Highlighter) Fragmenter() highlight.Fragmenter {
+	return s.fragmenter
+}
+
+func (s *Highlighter) SetFragmenter(f highlight.Fragmenter) {
+	s.fragmenter = f
+}
+
+func (s *Highlighter) FragmentFormatter() highlight.FragmentFormatter {
+	return s.formatter
+}
+
+func (s *Highlighter) SetFragmentFormatter(f highlight.FragmentFormatter) {
+	s.formatter = f
+}
+
+func (s *Highlighter) Separator() string {
+	return s.sep
+}
+
+func (s *Highlighter) SetSeparator(sep string) {
+	s.sep = sep
+}
+
+func (s *Highlighter) BestFragmentInField(dm *search.DocumentMatch, doc index.Document, field string) string {
+	fragments := s.BestFragmentsInField(dm, doc, field, 1)
+	if len(fragments) > 0 {
+		return fragments[0]
+	}
+	return ""
+}
+
+func (s *Highlighter) BestFragmentsInField(dm *search.DocumentMatch, doc index.Document, field string, num int) []string {
+	tlm := dm.Locations[field]
+	orderedTermLocations := highlight.OrderTermLocations(tlm)
+	scorer := NewFragmentScorer(tlm)
+
+	// score the fragments and put them into a priority queue ordered by score
+	fq := make(FragmentQueue, 0)
+	heap.Init(&fq)
+	doc.VisitFields(func(f index.Field) {
+		if f.Name() == field {
+			_, ok := f.(index.TextField)
+			if ok {
+				termLocationsSameArrayPosition := make(highlight.TermLocations, 0)
+				for _, otl := range orderedTermLocations {
+					if otl.ArrayPositions.Equals(f.ArrayPositions()) {
+						termLocationsSameArrayPosition = append(termLocationsSameArrayPosition, otl)
+					}
+				}
+
+				fieldData := f.Value()
+				fragments := s.fragmenter.Fragment(fieldData, termLocationsSameArrayPosition)
+				for _, fragment := range fragments {
+					fragment.ArrayPositions = f.ArrayPositions()
+					scorer.Score(fragment)
+					heap.Push(&fq, fragment)
+				}
+			}
+		}
+	})
+
+	// now find the N best non-overlapping fragments
+	var bestFragments []*highlight.Fragment
+	if len(fq) > 0 {
+		candidate := heap.Pop(&fq)
+	OUTER:
+		for candidate != nil && len(bestFragments) < num {
+			// see if this overlaps with any of the best already identified
+			if len(bestFragments) > 0 {
+				for _, frag := range bestFragments {
+					if candidate.(*highlight.Fragment).Overlaps(frag) {
+						if len(fq) < 1 {
+							break OUTER
+						}
+						candidate = heap.Pop(&fq)
+						continue OUTER
+					}
+				}
+				bestFragments = append(bestFragments, candidate.(*highlight.Fragment))
+			} else {
+				bestFragments = append(bestFragments, candidate.(*highlight.Fragment))
+			}
+
+			if len(fq) < 1 {
+				break
+			}
+			candidate = heap.Pop(&fq)
+		}
+	}
+
+	// now that we have the best fragments, we can format them
+	orderedTermLocations.MergeOverlapping()
+	formattedFragments := make([]string, len(bestFragments))
+	for i, fragment := range bestFragments {
+		formattedFragments[i] = ""
+		if fragment.Start != 0 {
+			formattedFragments[i] += s.sep
+		}
+		formattedFragments[i] += s.formatter.Format(fragment, orderedTermLocations)
+		if fragment.End != len(fragment.Orig) {
+			formattedFragments[i] += s.sep
+		}
+	}
+
+	if dm.Fragments == nil {
+		dm.Fragments = make(search.FieldFragmentMap, 0)
+	}
+	if len(formattedFragments) > 0 {
+		dm.Fragments[field] = formattedFragments
+	}
+
+	return formattedFragments
+}
+
+// FragmentQueue implements heap.Interface and holds Items.
+type FragmentQueue []*highlight.Fragment
+
+func (fq FragmentQueue) Len() int { return len(fq) }
+
+func (fq FragmentQueue) Less(i, j int) bool {
+	// We want Pop to give us the highest, not lowest, priority so we use greater-than here.
+	return fq[i].Score > fq[j].Score
+}
+
+func (fq FragmentQueue) Swap(i, j int) {
+	fq[i], fq[j] = fq[j], fq[i]
+	fq[i].Index = i
+	fq[j].Index = j
+}
+
+func (fq *FragmentQueue) Push(x interface{}) {
+	n := len(*fq)
+	item := x.(*highlight.Fragment)
+	item.Index = n
+	*fq = append(*fq, item)
+}
+
+func (fq *FragmentQueue) Pop() interface{} {
+	old := *fq
+	n := len(old)
+	item := old[n-1]
+	item.Index = -1 // for safety
+	*fq = old[0 : n-1]
+	return item
+}
+
+func Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.Highlighter, error) {
+	separator := DefaultSeparator
+	separatorVal, ok := config["separator"].(string)
+	if ok {
+		separator = separatorVal
+	}
+
+	fragmenterName, ok := config["fragmenter"].(string)
+	if !ok {
+		return nil, fmt.Errorf("must specify fragmenter")
+	}
+	fragmenter, err := cache.FragmenterNamed(fragmenterName)
+	if err != nil {
+		return nil, fmt.Errorf("error building fragmenter: %v", err)
+	}
+
+	formatterName, ok := config["formatter"].(string)
+	if !ok {
+		return nil, fmt.Errorf("must specify formatter")
+	}
+	formatter, err := cache.FragmentFormatterNamed(formatterName)
+	if err != nil {
+		return nil, fmt.Errorf("error building fragment formatter: %v", err)
+	}
+
+	return NewHighlighter(fragmenter, formatter, separator), nil
+}
+
+func init() {
+	registry.RegisterHighlighter(Name, Constructor)
+}

vendor/github.com/blevesearch/bleve/v2/search/highlight/term_locations.go

@@ -0,0 +1,105 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package highlight
+
+import (
+	"reflect"
+	"sort"
+
+	"github.com/blevesearch/bleve/v2/search"
+)
+
+type TermLocation struct {
+	Term           string
+	ArrayPositions search.ArrayPositions
+	Pos            int
+	Start          int
+	End            int
+}
+
+func (tl *TermLocation) Overlaps(other *TermLocation) bool {
+	if reflect.DeepEqual(tl.ArrayPositions, other.ArrayPositions) {
+		if other.Start >= tl.Start && other.Start < tl.End {
+			return true
+		} else if tl.Start >= other.Start && tl.Start < other.End {
+			return true
+		}
+	}
+	return false
+}
+
+type TermLocations []*TermLocation
+
+func (t TermLocations) Len() int      { return len(t) }
+func (t TermLocations) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
+func (t TermLocations) Less(i, j int) bool {
+
+	shortestArrayPositions := len(t[i].ArrayPositions)
+	if len(t[j].ArrayPositions) < shortestArrayPositions {
+		shortestArrayPositions = len(t[j].ArrayPositions)
+	}
+
+	// compare all the common array positions
+	for api := 0; api < shortestArrayPositions; api++ {
+		if t[i].ArrayPositions[api] < t[j].ArrayPositions[api] {
+			return true
+		}
+		if t[i].ArrayPositions[api] > t[j].ArrayPositions[api] {
+			return false
+		}
+	}
+	// all the common array positions are the same
+	if len(t[i].ArrayPositions) < len(t[j].ArrayPositions) {
+		return true // j array positions, longer so greater
+	} else if len(t[i].ArrayPositions) > len(t[j].ArrayPositions) {
+		return false // j array positions, shorter so less
+	}
+
+	// array positions the same, compare starts
+	return t[i].Start < t[j].Start
+}
+
+func (t TermLocations) MergeOverlapping() {
+	var lastTl *TermLocation
+	for i, tl := range t {
+		if lastTl == nil && tl != nil {
+			lastTl = tl
+		} else if lastTl != nil && tl != nil {
+			if lastTl.Overlaps(tl) {
+				// ok merge this with previous
+				lastTl.End = tl.End
+				t[i] = nil
+			}
+		}
+	}
+}
+
+func OrderTermLocations(tlm search.TermLocationMap) TermLocations {
+	rv := make(TermLocations, 0)
+	for term, locations := range tlm {
+		for _, location := range locations {
+			tl := TermLocation{
+				Term:           term,
+				ArrayPositions: location.ArrayPositions,
+				Pos:            int(location.Pos),
+				Start:          int(location.Start),
+				End:            int(location.End),
+			}
+			rv = append(rv, &tl)
+		}
+	}
+	sort.Sort(rv)
+	return rv
+}