Search bar for issues/pulls (#530)

models/issue_indexer.go

@@ -0,0 +1,183 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package models
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/util"
+	"github.com/blevesearch/bleve"
+	"github.com/blevesearch/bleve/analysis/analyzer/simple"
+	"github.com/blevesearch/bleve/search/query"
+)
+
+// issueIndexerUpdateQueue queue of issues that need to be updated in the issues
+// indexer
+var issueIndexerUpdateQueue chan *Issue
+
+// issueIndexer (thread-safe) index for searching issues
+var issueIndexer bleve.Index
+
+// issueIndexerData data stored in the issue indexer
+type issueIndexerData struct {
+	ID     int64
+	RepoID int64
+
+	Title   string
+	Content string
+}
+
+// numericQuery an numeric-equality query for the given value and field
+func numericQuery(value int64, field string) *query.NumericRangeQuery {
+	f := float64(value)
+	tru := true
+	q := bleve.NewNumericRangeInclusiveQuery(&f, &f, &tru, &tru)
+	q.SetField(field)
+	return q
+}
+
+// SearchIssuesByKeyword searches for issues by given conditions.
+// Returns the matching issue IDs
+func SearchIssuesByKeyword(repoID int64, keyword string) ([]int64, error) {
+	fields := strings.Fields(strings.ToLower(keyword))
+	indexerQuery := bleve.NewConjunctionQuery(
+		numericQuery(repoID, "RepoID"),
+		bleve.NewDisjunctionQuery(
+			bleve.NewPhraseQuery(fields, "Title"),
+			bleve.NewPhraseQuery(fields, "Content"),
+		))
+	search := bleve.NewSearchRequestOptions(indexerQuery, 2147483647, 0, false)
+	search.Fields = []string{"ID"}
+
+	result, err := issueIndexer.Search(search)
+	if err != nil {
+		return nil, err
+	}
+
+	issueIDs := make([]int64, len(result.Hits))
+	for i, hit := range result.Hits {
+		issueIDs[i] = int64(hit.Fields["ID"].(float64))
+	}
+	return issueIDs, nil
+}
+
+// InitIssueIndexer initialize issue indexer
+func InitIssueIndexer() {
+	_, err := os.Stat(setting.Indexer.IssuePath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			if err = createIssueIndexer(); err != nil {
+				log.Fatal(4, "CreateIssuesIndexer: %v", err)
+			}
+			if err = populateIssueIndexer(); err != nil {
+				log.Fatal(4, "PopulateIssuesIndex: %v", err)
+			}
+		} else {
+			log.Fatal(4, "InitIssuesIndexer: %v", err)
+		}
+	} else {
+		issueIndexer, err = bleve.Open(setting.Indexer.IssuePath)
+		if err != nil {
+			log.Fatal(4, "InitIssuesIndexer, open index: %v", err)
+		}
+	}
+	issueIndexerUpdateQueue = make(chan *Issue, setting.Indexer.UpdateQueueLength)
+	go processIssueIndexerUpdateQueue()
+	// TODO close issueIndexer when Gitea closes
+}
+
+// createIssueIndexer create an issue indexer if one does not already exist
+func createIssueIndexer() error {
+	mapping := bleve.NewIndexMapping()
+	docMapping := bleve.NewDocumentMapping()
+
+	docMapping.AddFieldMappingsAt("ID", bleve.NewNumericFieldMapping())
+	docMapping.AddFieldMappingsAt("RepoID", bleve.NewNumericFieldMapping())
+
+	textFieldMapping := bleve.NewTextFieldMapping()
+	textFieldMapping.Analyzer = simple.Name
+	docMapping.AddFieldMappingsAt("Title", textFieldMapping)
+	docMapping.AddFieldMappingsAt("Content", textFieldMapping)
+
+	mapping.AddDocumentMapping("issues", docMapping)
+
+	var err error
+	issueIndexer, err = bleve.New(setting.Indexer.IssuePath, mapping)
+	return err
+}
+
+// populateIssueIndexer populate the issue indexer with issue data
+func populateIssueIndexer() error {
+	for page := 1; ; page++ {
+		repos, err := Repositories(&SearchRepoOptions{
+			Page:     page,
+			PageSize: 10,
+		})
+		if err != nil {
+			return fmt.Errorf("Repositories: %v", err)
+		}
+		if len(repos) == 0 {
+			return nil
+		}
+		batch := issueIndexer.NewBatch()
+		for _, repo := range repos {
+			issues, err := Issues(&IssuesOptions{
+				RepoID:   repo.ID,
+				IsClosed: util.OptionalBoolNone,
+				IsPull:   util.OptionalBoolNone,
+				Page:     -1, // do not page
+			})
+			if err != nil {
+				return fmt.Errorf("Issues: %v", err)
+			}
+			for _, issue := range issues {
+				err = batch.Index(issue.indexUID(), issue.issueData())
+				if err != nil {
+					return fmt.Errorf("batch.Index: %v", err)
+				}
+			}
+		}
+		if err = issueIndexer.Batch(batch); err != nil {
+			return fmt.Errorf("index.Batch: %v", err)
+		}
+	}
+}
+
+func processIssueIndexerUpdateQueue() {
+	for {
+		select {
+		case issue := <-issueIndexerUpdateQueue:
+			if err := issueIndexer.Index(issue.indexUID(), issue.issueData()); err != nil {
+				log.Error(4, "issuesIndexer.Index: %v", err)
+			}
+		}
+	}
+}
+
+// indexUID a unique identifier for an issue used in full-text indices
+func (issue *Issue) indexUID() string {
+	return strconv.FormatInt(issue.ID, 36)
+}
+
+func (issue *Issue) issueData() *issueIndexerData {
+	return &issueIndexerData{
+		ID:      issue.ID,
+		RepoID:  issue.RepoID,
+		Title:   issue.Title,
+		Content: issue.Content,
+	}
+}
+
+// UpdateIssueIndexer add/update an issue to the issue indexer
+func UpdateIssueIndexer(issue *Issue) {
+	go func() {
+		issueIndexerUpdateQueue <- issue
+	}()
+}