Improve issue search (#2387)

* Improve issue indexer

* Fix new issue sqlite bug

* Different test indexer paths for each db

* Add integration indexer paths to make clean

vendor/github.com/blevesearch/bleve/search/searcher/search_phrase.go

@@ -15,6 +15,7 @@
 package searcher
 
 import (
+	"fmt"
 	"math"
 
 	"github.com/blevesearch/bleve/index"
@@ -27,11 +28,72 @@ type PhraseSearcher struct {
 	queryNorm    float64
 	currMust     *search.DocumentMatch
 	slop         int
-	terms        []string
+	terms        [][]string
 	initialized  bool
 }
 
-func NewPhraseSearcher(indexReader index.IndexReader, mustSearcher *ConjunctionSearcher, terms []string) (*PhraseSearcher, error) {
+func NewPhraseSearcher(indexReader index.IndexReader, terms []string, field string, options search.SearcherOptions) (*PhraseSearcher, error) {
+	// turn flat terms []string into [][]string
+	mterms := make([][]string, len(terms))
+	for i, term := range terms {
+		mterms[i] = []string{term}
+	}
+	return NewMultiPhraseSearcher(indexReader, mterms, field, options)
+}
+
+func NewMultiPhraseSearcher(indexReader index.IndexReader, terms [][]string, field string, options search.SearcherOptions) (*PhraseSearcher, error) {
+	options.IncludeTermVectors = true
+	var termPositionSearchers []search.Searcher
+	for _, termPos := range terms {
+		if len(termPos) == 1 && termPos[0] != "" {
+			// single term
+			ts, err := NewTermSearcher(indexReader, termPos[0], field, 1.0, options)
+			if err != nil {
+				// close any searchers already opened
+				for _, ts := range termPositionSearchers {
+					_ = ts.Close()
+				}
+				return nil, fmt.Errorf("phrase searcher error building term searcher: %v", err)
+			}
+			termPositionSearchers = append(termPositionSearchers, ts)
+		} else if len(termPos) > 1 {
+			// multiple terms
+			var termSearchers []search.Searcher
+			for _, term := range termPos {
+				if term == "" {
+					continue
+				}
+				ts, err := NewTermSearcher(indexReader, term, field, 1.0, options)
+				if err != nil {
+					// close any searchers already opened
+					for _, ts := range termPositionSearchers {
+						_ = ts.Close()
+					}
+					return nil, fmt.Errorf("phrase searcher error building term searcher: %v", err)
+				}
+				termSearchers = append(termSearchers, ts)
+			}
+			disjunction, err := NewDisjunctionSearcher(indexReader, termSearchers, 1, options)
+			if err != nil {
+				// close any searchers already opened
+				for _, ts := range termPositionSearchers {
+					_ = ts.Close()
+				}
+				return nil, fmt.Errorf("phrase searcher error building term position disjunction searcher: %v", err)
+			}
+			termPositionSearchers = append(termPositionSearchers, disjunction)
+		}
+	}
+
+	mustSearcher, err := NewConjunctionSearcher(indexReader, termPositionSearchers, options)
+	if err != nil {
+		// close any searchers already opened
+		for _, ts := range termPositionSearchers {
+			_ = ts.Close()
+		}
+		return nil, fmt.Errorf("phrase searcher error building conjunction searcher: %v", err)
+	}
+
 	// build our searcher
 	rv := PhraseSearcher{
 		indexReader:  indexReader,
@@ -96,68 +158,150 @@ func (s *PhraseSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch,
 		}
 	}
 
-	var rv *search.DocumentMatch
 	for s.currMust != nil {
-		rvftlm := make(search.FieldTermLocationMap, 0)
-		freq := 0
-		firstTerm := s.terms[0]
-		for field, termLocMap := range s.currMust.Locations {
-			rvtlm := make(search.TermLocationMap, 0)
-			locations, ok := termLocMap[firstTerm]
-			if ok {
-			OUTER:
-				for _, location := range locations {
-					crvtlm := make(search.TermLocationMap, 0)
-				INNER:
-					for i := 0; i < len(s.terms); i++ {
-						nextTerm := s.terms[i]
-						if nextTerm != "" {
-							// look through all these term locations
-							// to try and find the correct offsets
-							nextLocations, ok := termLocMap[nextTerm]
-							if ok {
-								for _, nextLocation := range nextLocations {
-									if nextLocation.Pos == location.Pos+float64(i) && nextLocation.SameArrayElement(location) {
-										// found a location match for this term
-										crvtlm.AddLocation(nextTerm, nextLocation)
-										continue INNER
-									}
-								}
-								// if we got here we didn't find a location match for this term
-								continue OUTER
-							} else {
-								continue OUTER
-							}
-						}
-					}
-					// if we got here all the terms matched
-					freq++
-					search.MergeTermLocationMaps(rvtlm, crvtlm)
-					rvftlm[field] = rvtlm
-				}
-			}
-		}
-
-		if freq > 0 {
-			// return match
-			rv = s.currMust
-			rv.Locations = rvftlm
-			err := s.advanceNextMust(ctx)
-			if err != nil {
-				return nil, err
-			}
-			return rv, nil
-		}
+		// check this match against phrase constraints
+		rv := s.checkCurrMustMatch(ctx)
 
+		// prepare for next iteration (either loop or subsequent call to Next())
 		err := s.advanceNextMust(ctx)
 		if err != nil {
 			return nil, err
 		}
+
+		// if match satisfied phrase constraints return it as a hit
+		if rv != nil {
+			return rv, nil
+		}
 	}
 
 	return nil, nil
 }
 
+// checkCurrMustMatch is soley concerned with determining if the DocumentMatch
+// pointed to by s.currMust (which satisifies the pre-condition searcher)
+// also satisfies the phase constraints.  if so, it returns a DocumentMatch
+// for this document, otherwise nil
+func (s *PhraseSearcher) checkCurrMustMatch(ctx *search.SearchContext) *search.DocumentMatch {
+	rvftlm := make(search.FieldTermLocationMap, 0)
+	freq := 0
+	// typically we would expect there to only actually be results in
+	// one field, but we allow for this to not be the case
+	// but, we note that phrase constraints can only be satisfied within
+	// a single field, so we can check them each independently
+	for field, tlm := range s.currMust.Locations {
+
+		f, rvtlm := s.checkCurrMustMatchField(ctx, tlm)
+		if f > 0 {
+			freq += f
+			rvftlm[field] = rvtlm
+		}
+	}
+
+	if freq > 0 {
+		// return match
+		rv := s.currMust
+		rv.Locations = rvftlm
+		return rv
+	}
+
+	return nil
+}
+
+// checkCurrMustMatchField is soley concerned with determining if one particular
+// field within the currMust DocumentMatch Locations satisfies the phase
+// constraints (possibly more than once).  if so, the number of times it was
+// satisfied, and these locations are returned.  otherwise 0 and either
+// a nil or empty TermLocationMap
+func (s *PhraseSearcher) checkCurrMustMatchField(ctx *search.SearchContext, tlm search.TermLocationMap) (int, search.TermLocationMap) {
+	paths := findPhrasePaths(0, nil, s.terms, tlm, nil, 0)
+	rv := make(search.TermLocationMap, len(s.terms))
+	for _, p := range paths {
+		p.MergeInto(rv)
+	}
+	return len(paths), rv
+}
+
+type phrasePart struct {
+	term string
+	loc  *search.Location
+}
+
+type phrasePath []*phrasePart
+
+func (p phrasePath) MergeInto(in search.TermLocationMap) {
+	for _, pp := range p {
+		in[pp.term] = append(in[pp.term], pp.loc)
+	}
+}
+
+// findPhrasePaths is a function to identify phase matches from a set of known
+// term locations.  the implementation is recursive, so care must be taken
+// with arguments and return values.
+//
+// prev - the previous location, nil on first invocation
+// phraseTerms - slice containing the phrase terms themselves
+//               may contain empty string as placeholder (don't care)
+// tlm - the Term Location Map containing all relevant term locations
+// offset - the offset from the previous that this next term must match
+// p - the current path being explored (appended to in recursive calls)
+//     this is the primary state being built during the traversal
+//
+// returns slice of paths, or nil if invocation did not find any successul paths
+func findPhrasePaths(prevPos uint64, ap search.ArrayPositions, phraseTerms [][]string, tlm search.TermLocationMap, p phrasePath, remainingSlop int) []phrasePath {
+
+	// no more terms
+	if len(phraseTerms) < 1 {
+		return []phrasePath{p}
+	}
+
+	car := phraseTerms[0]
+	cdr := phraseTerms[1:]
+
+	// empty term is treated as match (continue)
+	if len(car) == 0 || (len(car) == 1 && car[0] == "") {
+		nextPos := prevPos + 1
+		if prevPos == 0 {
+			// if prevPos was 0, don't set it to 1 (as thats not a real abs pos)
+			nextPos = 0 // don't advance nextPos if prevPos was 0
+		}
+		return findPhrasePaths(nextPos, ap, cdr, tlm, p, remainingSlop)
+	}
+
+	var rv []phrasePath
+	// locations for this term
+	for _, carTerm := range car {
+		locations := tlm[carTerm]
+		for _, loc := range locations {
+			if prevPos != 0 && !loc.ArrayPositions.Equals(ap) {
+				// if the array positions are wrong, can't match, try next location
+				continue
+			}
+
+			// compute distance from previous phrase term
+			dist := 0
+			if prevPos != 0 {
+				dist = editDistance(prevPos+1, loc.Pos)
+			}
+
+			// if enough slop reamining, continue recursively
+			if prevPos == 0 || (remainingSlop-dist) >= 0 {
+				// this location works, add it to the path (but not for empty term)
+				px := append(p, &phrasePart{term: carTerm, loc: loc})
+				rv = append(rv, findPhrasePaths(loc.Pos, loc.ArrayPositions, cdr, tlm, px, remainingSlop-dist)...)
+			}
+		}
+	}
+	return rv
+}
+
+func editDistance(p1, p2 uint64) int {
+	dist := int(p1 - p2)
+	if dist < 0 {
+		return -dist
+	}
+	return dist
+}
+
 func (s *PhraseSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
 	if !s.initialized {
 		err := s.initSearchers(ctx)