Improve issue search (#2387)

* Improve issue indexer

* Fix new issue sqlite bug

* Different test indexer paths for each db

* Add integration indexer paths to make clean

models/fixtures/issue.yml

@@ -5,7 +5,7 @@
   poster_id: 1
   assignee_id: 1
   name: issue1
-  content: content1
+  content: content for the first issue
   is_closed: false
   is_pull: false
   num_comments: 2
@@ -18,7 +18,7 @@
   index: 2
   poster_id: 1
   name: issue2
-  content: content2
+  content: content for the second issue
   milestone_id: 1
   is_closed: false
   is_pull: true
@@ -32,7 +32,7 @@
   index: 3
   poster_id: 1
   name: issue3
-  content: content4
+  content: content for the third issue
   is_closed: false
   is_pull: true
   created_unix: 946684820
@@ -44,7 +44,7 @@
   index: 1
   poster_id: 2
   name: issue4
-  content: content4
+  content: content for the fourth issue
   is_closed: true
   is_pull: false
 
@@ -54,7 +54,7 @@
   index: 4
   poster_id: 2
   name: issue5
-  content: content5
+  content: content for the fifth issue
   is_closed: true
   is_pull: false
 -

models/issue.go

@@ -155,6 +155,17 @@ func (issue *Issue) loadPullRequest(e Engine) (err error) {
 	return nil
 }
 
+func (issue *Issue) loadComments(e Engine) (err error) {
+	if issue.Comments != nil {
+		return nil
+	}
+	issue.Comments, err = findComments(e, FindCommentsOptions{
+		IssueID: issue.ID,
+		Type:    CommentTypeUnknown,
+	})
+	return err
+}
+
 func (issue *Issue) loadAttributes(e Engine) (err error) {
 	if err = issue.loadRepo(e); err != nil {
 		return
@@ -191,14 +202,8 @@ func (issue *Issue) loadAttributes(e Engine) (err error) {
 		}
 	}
 
-	if issue.Comments == nil {
-		issue.Comments, err = findComments(e, FindCommentsOptions{
-			IssueID: issue.ID,
-			Type:    CommentTypeUnknown,
-		})
-		if err != nil {
-			return fmt.Errorf("getCommentsByIssueID [%d]: %v", issue.ID, err)
-		}
+	if err = issue.loadComments(e); err != nil {
+		return
 	}
 
 	return nil
@@ -577,7 +582,7 @@ func updateIssueCols(e Engine, issue *Issue, cols ...string) error {
 	if _, err := e.Id(issue.ID).Cols(cols...).Update(issue); err != nil {
 		return err
 	}
-	UpdateIssueIndexer(issue)
+	UpdateIssueIndexer(issue.ID)
 	return nil
 }
 
@@ -907,8 +912,6 @@ func newIssue(e *xorm.Session, doer *User, opts NewIssueOptions) (err error) {
 		return err
 	}
 
-	UpdateIssueIndexer(opts.Issue)
-
 	if len(opts.Attachments) > 0 {
 		attachments, err := getAttachmentsByUUIDs(e, opts.Attachments)
 		if err != nil {
@@ -947,6 +950,8 @@ func NewIssue(repo *Repository, issue *Issue, labelIDs []int64, uuids []string)
 		return fmt.Errorf("Commit: %v", err)
 	}
 
+	UpdateIssueIndexer(issue.ID)
+
 	if err = NotifyWatchers(&Action{
 		ActUserID: issue.Poster.ID,
 		ActUser:   issue.Poster,
@@ -1448,7 +1453,7 @@ func updateIssue(e Engine, issue *Issue) error {
 	if err != nil {
 		return err
 	}
-	UpdateIssueIndexer(issue)
+	UpdateIssueIndexer(issue.ID)
 	return nil
 }
 

models/issue_comment.go

@@ -520,7 +520,14 @@ func CreateComment(opts *CreateCommentOptions) (comment *Comment, err error) {
 		return nil, err
 	}
 
-	return comment, sess.Commit()
+	if err = sess.Commit(); err != nil {
+		return nil, err
+	}
+
+	if opts.Type == CommentTypeComment {
+		UpdateIssueIndexer(opts.Issue.ID)
+	}
+	return comment, nil
 }
 
 // CreateIssueComment creates a plain issue comment.
@@ -645,8 +652,12 @@ func GetCommentsByRepoIDSince(repoID, since int64) ([]*Comment, error) {
 
 // UpdateComment updates information of comment.
 func UpdateComment(c *Comment) error {
-	_, err := x.Id(c.ID).AllCols().Update(c)
-	return err
+	if _, err := x.Id(c.ID).AllCols().Update(c); err != nil {
+		return err
+	} else if c.Type == CommentTypeComment {
+		UpdateIssueIndexer(c.IssueID)
+	}
+	return nil
 }
 
 // DeleteComment deletes the comment
@@ -672,5 +683,10 @@ func DeleteComment(comment *Comment) error {
 		return err
 	}
 
-	return sess.Commit()
+	if err := sess.Commit(); err != nil {
+		return err
+	} else if comment.Type == CommentTypeComment {
+		UpdateIssueIndexer(comment.IssueID)
+	}
+	return nil
 }

models/issue_indexer.go

@@ -6,112 +6,21 @@ package models
 
 import (
 	"fmt"
-	"os"
-	"strconv"
-	"strings"
 
+	"code.gitea.io/gitea/modules/indexer"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
-
-	"github.com/blevesearch/bleve"
-	"github.com/blevesearch/bleve/analysis/analyzer/simple"
-	"github.com/blevesearch/bleve/search/query"
 )
 
-// issueIndexerUpdateQueue queue of issues that need to be updated in the issues
-// indexer
-var issueIndexerUpdateQueue chan *Issue
-
-// issueIndexer (thread-safe) index for searching issues
-var issueIndexer bleve.Index
-
-// issueIndexerData data stored in the issue indexer
-type issueIndexerData struct {
-	ID     int64
-	RepoID int64
-
-	Title   string
-	Content string
-}
-
-// numericQuery an numeric-equality query for the given value and field
-func numericQuery(value int64, field string) *query.NumericRangeQuery {
-	f := float64(value)
-	tru := true
-	q := bleve.NewNumericRangeInclusiveQuery(&f, &f, &tru, &tru)
-	q.SetField(field)
-	return q
-}
-
-// SearchIssuesByKeyword searches for issues by given conditions.
-// Returns the matching issue IDs
-func SearchIssuesByKeyword(repoID int64, keyword string) ([]int64, error) {
-	terms := strings.Fields(strings.ToLower(keyword))
-	indexerQuery := bleve.NewConjunctionQuery(
-		numericQuery(repoID, "RepoID"),
-		bleve.NewDisjunctionQuery(
-			bleve.NewPhraseQuery(terms, "Title"),
-			bleve.NewPhraseQuery(terms, "Content"),
-		))
-	search := bleve.NewSearchRequestOptions(indexerQuery, 2147483647, 0, false)
-	search.Fields = []string{"ID"}
-
-	result, err := issueIndexer.Search(search)
-	if err != nil {
-		return nil, err
-	}
-
-	issueIDs := make([]int64, len(result.Hits))
-	for i, hit := range result.Hits {
-		issueIDs[i] = int64(hit.Fields["ID"].(float64))
-	}
-	return issueIDs, nil
-}
+// issueIndexerUpdateQueue queue of issue ids to be updated
+var issueIndexerUpdateQueue chan int64
 
 // InitIssueIndexer initialize issue indexer
 func InitIssueIndexer() {
-	_, err := os.Stat(setting.Indexer.IssuePath)
-	if err != nil {
-		if os.IsNotExist(err) {
-			if err = createIssueIndexer(); err != nil {
-				log.Fatal(4, "CreateIssuesIndexer: %v", err)
-			}
-			if err = populateIssueIndexer(); err != nil {
-				log.Fatal(4, "PopulateIssuesIndex: %v", err)
-			}
-		} else {
-			log.Fatal(4, "InitIssuesIndexer: %v", err)
-		}
-	} else {
-		issueIndexer, err = bleve.Open(setting.Indexer.IssuePath)
-		if err != nil {
-			log.Fatal(4, "InitIssuesIndexer, open index: %v", err)
-		}
-	}
-	issueIndexerUpdateQueue = make(chan *Issue, setting.Indexer.UpdateQueueLength)
+	indexer.InitIssueIndexer(populateIssueIndexer)
+	issueIndexerUpdateQueue = make(chan int64, setting.Indexer.UpdateQueueLength)
 	go processIssueIndexerUpdateQueue()
-	// TODO close issueIndexer when Gitea closes
-}
-
-// createIssueIndexer create an issue indexer if one does not already exist
-func createIssueIndexer() error {
-	mapping := bleve.NewIndexMapping()
-	docMapping := bleve.NewDocumentMapping()
-
-	docMapping.AddFieldMappingsAt("ID", bleve.NewNumericFieldMapping())
-	docMapping.AddFieldMappingsAt("RepoID", bleve.NewNumericFieldMapping())
-
-	textFieldMapping := bleve.NewTextFieldMapping()
-	textFieldMapping.Analyzer = simple.Name
-	docMapping.AddFieldMappingsAt("Title", textFieldMapping)
-	docMapping.AddFieldMappingsAt("Content", textFieldMapping)
-
-	mapping.AddDocumentMapping("issues", docMapping)
-
-	var err error
-	issueIndexer, err = bleve.New(setting.Indexer.IssuePath, mapping)
-	return err
 }
 
 // populateIssueIndexer populate the issue indexer with issue data
@@ -127,57 +36,64 @@ func populateIssueIndexer() error {
 		if len(repos) == 0 {
 			return nil
 		}
-		batch := issueIndexer.NewBatch()
 		for _, repo := range repos {
 			issues, err := Issues(&IssuesOptions{
 				RepoID:   repo.ID,
 				IsClosed: util.OptionalBoolNone,
 				IsPull:   util.OptionalBoolNone,
 			})
-			if err != nil {
-				return fmt.Errorf("Issues: %v", err)
+			updates := make([]indexer.IssueIndexerUpdate, len(issues))
+			for i, issue := range issues {
+				updates[i] = issue.update()
 			}
-			for _, issue := range issues {
-				err = batch.Index(issue.indexUID(), issue.issueData())
-				if err != nil {
-					return fmt.Errorf("batch.Index: %v", err)
-				}
+			if err = indexer.BatchUpdateIssues(updates...); err != nil {
+				return fmt.Errorf("BatchUpdate: %v", err)
 			}
 		}
-		if err = issueIndexer.Batch(batch); err != nil {
-			return fmt.Errorf("index.Batch: %v", err)
-		}
 	}
 }
 
 func processIssueIndexerUpdateQueue() {
 	for {
 		select {
-		case issue := <-issueIndexerUpdateQueue:
-			if err := issueIndexer.Index(issue.indexUID(), issue.issueData()); err != nil {
+		case issueID := <-issueIndexerUpdateQueue:
+			issue, err := GetIssueByID(issueID)
+			if err != nil {
+				log.Error(4, "issuesIndexer.Index: %v", err)
+				continue
+			}
+			if err = indexer.UpdateIssue(issue.update()); err != nil {
 				log.Error(4, "issuesIndexer.Index: %v", err)
 			}
 		}
 	}
 }
 
-// indexUID a unique identifier for an issue used in full-text indices
-func (issue *Issue) indexUID() string {
-	return strconv.FormatInt(issue.ID, 36)
-}
-
-func (issue *Issue) issueData() *issueIndexerData {
-	return &issueIndexerData{
-		ID:      issue.ID,
-		RepoID:  issue.RepoID,
-		Title:   issue.Title,
-		Content: issue.Content,
+func (issue *Issue) update() indexer.IssueIndexerUpdate {
+	comments := make([]string, 0, 5)
+	for _, comment := range issue.Comments {
+		if comment.Type == CommentTypeComment {
+			comments = append(comments, comment.Content)
+		}
+	}
+	return indexer.IssueIndexerUpdate{
+		IssueID: issue.ID,
+		Data: &indexer.IssueIndexerData{
+			RepoID:   issue.RepoID,
+			Title:    issue.Title,
+			Content:  issue.Content,
+			Comments: comments,
+		},
 	}
 }
 
 // UpdateIssueIndexer add/update an issue to the issue indexer
-func UpdateIssueIndexer(issue *Issue) {
-	go func() {
-		issueIndexerUpdateQueue <- issue
-	}()
+func UpdateIssueIndexer(issueID int64) {
+	select {
+	case issueIndexerUpdateQueue <- issueID:
+	default:
+		go func() {
+			issueIndexerUpdateQueue <- issueID
+		}()
+	}
 }

models/pull.go

@@ -640,6 +640,8 @@ func NewPullRequest(repo *Repository, pull *Issue, labelIDs []int64, uuids []str
 		return fmt.Errorf("Commit: %v", err)
 	}
 
+	UpdateIssueIndexer(pull.ID)
+
 	if err = NotifyWatchers(&Action{
 		ActUserID: pull.Poster.ID,
 		ActUser:   pull.Poster,