tdsds.com/gitea
1
1
Fork 0
mirror of https://github.com/go-gitea/gitea synced 2025-08-02 07:38:35 +00:00
Code Releases Activity

Refactor indexer (#25174)

Browse Source 
Refactor `modules/indexer` to make it more maintainable. And it can be
easier to support more features. I'm trying to solve some of issue
searching, this is a precursor to making functional changes.

Current supported engines and the index versions:

| engines | issues | code |
| - | - | - |
| db | Just a wrapper for database queries, doesn't need version | - |
| bleve | The version of index is **2** | The version of index is **6**
|
| elasticsearch | The old index has no version, will be treated as
version **0** in this PR | The version of index is **1** |
| meilisearch | The old index has no version, will be treated as version
**0** in this PR | - |


## Changes

### Split

Splited it into mutiple packages

```text
indexer
├── internal
│   ├── bleve
│   ├── db
│   ├── elasticsearch
│   └── meilisearch
├── code
│   ├── bleve
│   ├── elasticsearch
│   └── internal
└── issues
    ├── bleve
    ├── db
    ├── elasticsearch
    ├── internal
    └── meilisearch
```

- `indexer/interanal`: Internal shared package for indexer.
- `indexer/interanal/[engine]`: Internal shared package for each engine
(bleve/db/elasticsearch/meilisearch).
- `indexer/code`: Implementations for code indexer.
- `indexer/code/internal`: Internal shared package for code indexer.
- `indexer/code/[engine]`: Implementation via each engine for code
indexer.
- `indexer/issues`: Implementations for issues indexer.

### Deduplication

- Combine `Init/Ping/Close` for code indexer and issues indexer.
- ~Combine `issues.indexerHolder` and `code.wrappedIndexer` to
`internal.IndexHolder`.~ Remove it, use dummy indexer instead when the
indexer is not ready.
- Duplicate two copies of creating ES clients.
- Duplicate two copies of `indexerID()`.


### Enhancement

- [x] Support index version for elasticsearch issues indexer, the old
index without version will be treated as version 0.
- [x] Fix spell of `elastic_search/ElasticSearch`, it should be
`Elasticsearch`.
- [x] Improve versioning of ES index. We don't need `Aliases`:
- Gitea does't need aliases for "Zero Downtime" because it never delete
old indexes.
- The old code of issues indexer uses the orignal name to create issue
index, so it's tricky to convert it to an alias.
- [x] Support index version for meilisearch issues indexer, the old
index without version will be treated as version 0.
- [x] Do "ping" only when `Ping` has been called, don't ping
periodically and cache the status.
- [x] Support the context parameter whenever possible.
- [x] Fix outdated example config.
- [x] Give up the requeue logic of issues indexer: When indexing fails,
call Ping to check if it was caused by the engine being unavailable, and
only requeue the task if the engine is unavailable.
- It is fragile and tricky, could cause data losing (It did happen when
I was doing some tests for this PR). And it works for ES only.
- Just always requeue the failed task, if it caused by bad data, it's a
bug of Gitea which should be fixed.

---------

Co-authored-by: Giteabot <teabot@gitea.io>
This commit is contained in:
Jason Song
2023-06-23 20:37:56 +08:00
committed by GitHub
parent b0215c40cd
commit 375fd15fbf
43 changed files with 1374 additions and 1426 deletions
Download Patch File Download Diff File Expand all files Collapse all files

155
modules/indexer/issues/bleve.go → modules/indexer/issues/bleve/bleve.go
View File

@@ -1,17 +1,14 @@
// Copyright 2018 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package issues
package bleve
import (
"context"
"fmt"
"os"
"strconv"
gitea_bleve "code.gitea.io/gitea/modules/indexer/bleve"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/util"
indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
inner_bleve "code.gitea.io/gitea/modules/indexer/internal/bleve"
"code.gitea.io/gitea/modules/indexer/issues/internal"
"github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
@@ -19,10 +16,8 @@ import (
"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
"github.com/blevesearch/bleve/v2/analysis/token/unicodenorm"
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
"github.com/blevesearch/bleve/v2/index/upsidedown"
"github.com/blevesearch/bleve/v2/mapping"
"github.com/blevesearch/bleve/v2/search/query"
"github.com/ethantkoenig/rupture"
)
const (
@@ -31,20 +26,6 @@ const (
issueIndexerLatestVersion = 2
)
// indexerID a bleve-compatible unique identifier for an integer id
func indexerID(id int64) string {
return strconv.FormatInt(id, 36)
}
// idOfIndexerID the integer id associated with an indexer id
func idOfIndexerID(indexerID string) (int64, error) {
id, err := strconv.ParseInt(indexerID, 36, 64)
if err != nil {
return 0, fmt.Errorf("Unexpected indexer ID %s: %w", indexerID, err)
}
return id, nil
}
// numericEqualityQuery a numeric equality query for the given value and field
func numericEqualityQuery(value int64, field string) *query.NumericRangeQuery {
f := float64(value)
@@ -72,49 +53,16 @@ func addUnicodeNormalizeTokenFilter(m *mapping.IndexMappingImpl) error {
const maxBatchSize = 16
// openIndexer open the index at the specified path, checking for metadata
// updates and bleve version updates. If index needs to be created (or
// re-created), returns (nil, nil)
func openIndexer(path string, latestVersion int) (bleve.Index, error) {
_, err := os.Stat(path)
if err != nil && os.IsNotExist(err) {
return nil, nil
} else if err != nil {
return nil, err
}
metadata, err := rupture.ReadIndexMetadata(path)
if err != nil {
return nil, err
}
if metadata.Version < latestVersion {
// the indexer is using a previous version, so we should delete it and
// re-populate
return nil, util.RemoveAll(path)
}
index, err := bleve.Open(path)
if err != nil && err == upsidedown.IncompatibleVersion {
// the indexer was built with a previous version of bleve, so we should
// delete it and re-populate
return nil, util.RemoveAll(path)
} else if err != nil {
return nil, err
}
return index, nil
}
// BleveIndexerData an update to the issue indexer
type BleveIndexerData IndexerData
// IndexerData an update to the issue indexer
type IndexerData internal.IndexerData
// Type returns the document type, for bleve's mapping.Classifier interface.
func (i *BleveIndexerData) Type() string {
func (i *IndexerData) Type() string {
return issueIndexerDocType
}
// createIssueIndexer create an issue indexer if one does not already exist
func createIssueIndexer(path string, latestVersion int) (bleve.Index, error) {
// generateIssueIndexMapping generates the bleve index mapping for issues
func generateIssueIndexMapping() (mapping.IndexMapping, error) {
mapping := bleve.NewIndexMapping()
docMapping := bleve.NewDocumentMapping()
@@ -144,68 +92,31 @@ func createIssueIndexer(path string, latestVersion int) (bleve.Index, error) {
mapping.AddDocumentMapping(issueIndexerDocType, docMapping)
mapping.AddDocumentMapping("_all", bleve.NewDocumentDisabledMapping())
index, err := bleve.New(path, mapping)
if err != nil {
return nil, err
}
if err = rupture.WriteIndexMetadata(path, &rupture.IndexMetadata{
Version: latestVersion,
}); err != nil {
return nil, err
}
return index, nil
return mapping, nil
}
var _ Indexer = &BleveIndexer{}
var _ internal.Indexer = &Indexer{}
// BleveIndexer implements Indexer interface
type BleveIndexer struct {
indexDir string
indexer bleve.Index
// Indexer implements Indexer interface
type Indexer struct {
inner *inner_bleve.Indexer
indexer_internal.Indexer // do not composite inner_bleve.Indexer directly to avoid exposing too much
}
// NewBleveIndexer creates a new bleve local indexer
func NewBleveIndexer(indexDir string) *BleveIndexer {
return &BleveIndexer{
indexDir: indexDir,
}
}
// Init will initialize the indexer
func (b *BleveIndexer) Init() (bool, error) {
var err error
b.indexer, err = openIndexer(b.indexDir, issueIndexerLatestVersion)
if err != nil {
return false, err
}
if b.indexer != nil {
return true, nil
}
b.indexer, err = createIssueIndexer(b.indexDir, issueIndexerLatestVersion)
return false, err
}
// Ping does nothing
func (b *BleveIndexer) Ping() bool {
return true
}
// Close will close the bleve indexer
func (b *BleveIndexer) Close() {
if b.indexer != nil {
if err := b.indexer.Close(); err != nil {
log.Error("Error whilst closing indexer: %v", err)
}
// NewIndexer creates a new bleve local indexer
func NewIndexer(indexDir string) *Indexer {
inner := inner_bleve.NewIndexer(indexDir, issueIndexerLatestVersion, generateIssueIndexMapping)
return &Indexer{
Indexer: inner,
inner: inner,
}
}
// Index will save the index data
func (b *BleveIndexer) Index(issues []*IndexerData) error {
batch := gitea_bleve.NewFlushingBatch(b.indexer, maxBatchSize)
func (b *Indexer) Index(_ context.Context, issues []*internal.IndexerData) error {
batch := inner_bleve.NewFlushingBatch(b.inner.Indexer, maxBatchSize)
for _, issue := range issues {
if err := batch.Index(indexerID(issue.ID), struct {
if err := batch.Index(indexer_internal.Base36(issue.ID), struct {
RepoID int64
Title string
Content string
@@ -223,10 +134,10 @@ func (b *BleveIndexer) Index(issues []*IndexerData) error {
}
// Delete deletes indexes by ids
func (b *BleveIndexer) Delete(ids ...int64) error {
batch := gitea_bleve.NewFlushingBatch(b.indexer, maxBatchSize)
func (b *Indexer) Delete(_ context.Context, ids ...int64) error {
batch := inner_bleve.NewFlushingBatch(b.inner.Indexer, maxBatchSize)
for _, id := range ids {
if err := batch.Delete(indexerID(id)); err != nil {
if err := batch.Delete(indexer_internal.Base36(id)); err != nil {
return err
}
}
@@ -235,7 +146,7 @@ func (b *BleveIndexer) Delete(ids ...int64) error {
// Search searches for issues by given conditions.
// Returns the matching issue IDs
func (b *BleveIndexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*SearchResult, error) {
func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*internal.SearchResult, error) {
var repoQueriesP []*query.NumericRangeQuery
for _, repoID := range repoIDs {
repoQueriesP = append(repoQueriesP, numericEqualityQuery(repoID, "RepoID"))
@@ -255,20 +166,20 @@ func (b *BleveIndexer) Search(ctx context.Context, keyword string, repoIDs []int
search := bleve.NewSearchRequestOptions(indexerQuery, limit, start, false)
search.SortBy([]string{"-_score"})
result, err := b.indexer.SearchInContext(ctx, search)
result, err := b.inner.Indexer.SearchInContext(ctx, search)
if err != nil {
return nil, err
}
ret := SearchResult{
Hits: make([]Match, 0, len(result.Hits)),
ret := internal.SearchResult{
Hits: make([]internal.Match, 0, len(result.Hits)),
}
for _, hit := range result.Hits {
id, err := idOfIndexerID(hit.ID)
id, err := indexer_internal.ParseBase36(hit.ID)
if err != nil {
return nil, err
}
ret.Hits = append(ret.Hits, Match{
ret.Hits = append(ret.Hits, internal.Match{
ID: id,
})
}

10
modules/indexer/issues/bleve_test.go → modules/indexer/issues/bleve/bleve_test.go
View File

@@ -1,26 +1,28 @@
// Copyright 2018 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package issues
package bleve
import (
"context"
"testing"
"code.gitea.io/gitea/modules/indexer/issues/internal"
"github.com/stretchr/testify/assert"
)
func TestBleveIndexAndSearch(t *testing.T) {
dir := t.TempDir()
indexer := NewBleveIndexer(dir)
indexer := NewIndexer(dir)
defer indexer.Close()
if _, err := indexer.Init(); err != nil {
if _, err := indexer.Init(context.Background()); err != nil {
assert.Fail(t, "Unable to initialize bleve indexer: %v", err)
return
}
err := indexer.Index([]*IndexerData{
err := indexer.Index(context.Background(), []*internal.IndexerData{
{
ID: 1,
RepoID: 2,

56
modules/indexer/issues/db.go
View File

@@ -1,56 +0,0 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package issues
import (
"context"
"code.gitea.io/gitea/models/db"
issues_model "code.gitea.io/gitea/models/issues"
)
// DBIndexer implements Indexer interface to use database's like search
type DBIndexer struct{}
// Init dummy function
func (i *DBIndexer) Init() (bool, error) {
return false, nil
}
// Ping checks if database is available
func (i *DBIndexer) Ping() bool {
return db.GetEngine(db.DefaultContext).Ping() != nil
}
// Index dummy function
func (i *DBIndexer) Index(issue []*IndexerData) error {
return nil
}
// Delete dummy function
func (i *DBIndexer) Delete(ids ...int64) error {
return nil
}
// Close dummy function
func (i *DBIndexer) Close() {
}
// Search dummy function
func (i *DBIndexer) Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*SearchResult, error) {
total, ids, err := issues_model.SearchIssueIDsByKeyword(ctx, kw, repoIDs, limit, start)
if err != nil {
return nil, err
}
result := SearchResult{
Total: total,
Hits: make([]Match, 0, limit),
}
for _, id := range ids {
result.Hits = append(result.Hits, Match{
ID: id,
})
}
return &result, nil
}

54
modules/indexer/issues/db/db.go Normal file
View File

@@ -0,0 +1,54 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package db
import (
"context"
issues_model "code.gitea.io/gitea/models/issues"
indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
inner_db "code.gitea.io/gitea/modules/indexer/internal/db"
"code.gitea.io/gitea/modules/indexer/issues/internal"
)
var _ internal.Indexer = &Indexer{}
// Indexer implements Indexer interface to use database's like search
type Indexer struct {
indexer_internal.Indexer
}
func NewIndexer() *Indexer {
return &Indexer{
Indexer: &inner_db.Indexer{},
}
}
// Index dummy function
func (i *Indexer) Index(_ context.Context, _ []*internal.IndexerData) error {
return nil
}
// Delete dummy function
func (i *Indexer) Delete(_ context.Context, _ ...int64) error {
return nil
}
// Search searches for issues
func (i *Indexer) Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*internal.SearchResult, error) {
total, ids, err := issues_model.SearchIssueIDsByKeyword(ctx, kw, repoIDs, limit, start)
if err != nil {
return nil, err
}
result := internal.SearchResult{
Total: total,
Hits: make([]internal.Match, 0, limit),
}
for _, id := range ids {
result.Hits = append(result.Hits, internal.Match{
ID: id,
})
}
return &result, nil
}

287
modules/indexer/issues/elastic_search.go
View File

@@ -1,287 +0,0 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package issues
import (
"context"
"errors"
"fmt"
"net"
"strconv"
"sync"
"time"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
"github.com/olivere/elastic/v7"
)
var _ Indexer = &ElasticSearchIndexer{}
// ElasticSearchIndexer implements Indexer interface
type ElasticSearchIndexer struct {
client *elastic.Client
indexerName string
available bool
stopTimer chan struct{}
lock sync.RWMutex
}
// NewElasticSearchIndexer creates a new elasticsearch indexer
func NewElasticSearchIndexer(url, indexerName string) (*ElasticSearchIndexer, error) {
opts := []elastic.ClientOptionFunc{
elastic.SetURL(url),
elastic.SetSniff(false),
elastic.SetHealthcheckInterval(10 * time.Second),
elastic.SetGzip(false),
}
logger := log.GetLogger(log.DEFAULT)
opts = append(opts, elastic.SetTraceLog(&log.PrintfLogger{Logf: logger.Trace}))
opts = append(opts, elastic.SetInfoLog(&log.PrintfLogger{Logf: logger.Info}))
opts = append(opts, elastic.SetErrorLog(&log.PrintfLogger{Logf: logger.Error}))
client, err := elastic.NewClient(opts...)
if err != nil {
return nil, err
}
indexer := &ElasticSearchIndexer{
client: client,
indexerName: indexerName,
available: true,
stopTimer: make(chan struct{}),
}
ticker := time.NewTicker(10 * time.Second)
go func() {
for {
select {
case <-ticker.C:
indexer.checkAvailability()
case <-indexer.stopTimer:
ticker.Stop()
return
}
}
}()
return indexer, nil
}
const (
defaultMapping = `{
"mappings": {
"properties": {
"id": {
"type": "integer",
"index": true
},
"repo_id": {
"type": "integer",
"index": true
},
"title": {
"type": "text",
"index": true
},
"content": {
"type": "text",
"index": true
},
"comments": {
"type" : "text",
"index": true
}
}
}
}`
)
// Init will initialize the indexer
func (b *ElasticSearchIndexer) Init() (bool, error) {
ctx := graceful.GetManager().HammerContext()
exists, err := b.client.IndexExists(b.indexerName).Do(ctx)
if err != nil {
return false, b.checkError(err)
}
if !exists {
mapping := defaultMapping
createIndex, err := b.client.CreateIndex(b.indexerName).BodyString(mapping).Do(ctx)
if err != nil {
return false, b.checkError(err)
}
if !createIndex.Acknowledged {
return false, errors.New("init failed")
}
return false, nil
}
return true, nil
}
// Ping checks if elastic is available
func (b *ElasticSearchIndexer) Ping() bool {
b.lock.RLock()
defer b.lock.RUnlock()
return b.available
}
// Index will save the index data
func (b *ElasticSearchIndexer) Index(issues []*IndexerData) error {
if len(issues) == 0 {
return nil
} else if len(issues) == 1 {
issue := issues[0]
_, err := b.client.Index().
Index(b.indexerName).
Id(fmt.Sprintf("%d", issue.ID)).
BodyJson(map[string]interface{}{
"id": issue.ID,
"repo_id": issue.RepoID,
"title": issue.Title,
"content": issue.Content,
"comments": issue.Comments,
}).
Do(graceful.GetManager().HammerContext())
return b.checkError(err)
}
reqs := make([]elastic.BulkableRequest, 0)
for _, issue := range issues {
reqs = append(reqs,
elastic.NewBulkIndexRequest().
Index(b.indexerName).
Id(fmt.Sprintf("%d", issue.ID)).
Doc(map[string]interface{}{
"id": issue.ID,
"repo_id": issue.RepoID,
"title": issue.Title,
"content": issue.Content,
"comments": issue.Comments,
}),
)
}
_, err := b.client.Bulk().
Index(b.indexerName).
Add(reqs...).
Do(graceful.GetManager().HammerContext())
return b.checkError(err)
}
// Delete deletes indexes by ids
func (b *ElasticSearchIndexer) Delete(ids ...int64) error {
if len(ids) == 0 {
return nil
} else if len(ids) == 1 {
_, err := b.client.Delete().
Index(b.indexerName).
Id(fmt.Sprintf("%d", ids[0])).
Do(graceful.GetManager().HammerContext())
return b.checkError(err)
}
reqs := make([]elastic.BulkableRequest, 0)
for _, id := range ids {
reqs = append(reqs,
elastic.NewBulkDeleteRequest().
Index(b.indexerName).
Id(fmt.Sprintf("%d", id)),
)
}
_, err := b.client.Bulk().
Index(b.indexerName).
Add(reqs...).
Do(graceful.GetManager().HammerContext())
return b.checkError(err)
}
// Search searches for issues by given conditions.
// Returns the matching issue IDs
func (b *ElasticSearchIndexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*SearchResult, error) {
kwQuery := elastic.NewMultiMatchQuery(keyword, "title", "content", "comments")
query := elastic.NewBoolQuery()
query = query.Must(kwQuery)
if len(repoIDs) > 0 {
repoStrs := make([]interface{}, 0, len(repoIDs))
for _, repoID := range repoIDs {
repoStrs = append(repoStrs, repoID)
}
repoQuery := elastic.NewTermsQuery("repo_id", repoStrs...)
query = query.Must(repoQuery)
}
searchResult, err := b.client.Search().
Index(b.indexerName).
Query(query).
Sort("_score", false).
From(start).Size(limit).
Do(ctx)
if err != nil {
return nil, b.checkError(err)
}
hits := make([]Match, 0, limit)
for _, hit := range searchResult.Hits.Hits {
id, _ := strconv.ParseInt(hit.Id, 10, 64)
hits = append(hits, Match{
ID: id,
})
}
return &SearchResult{
Total: searchResult.TotalHits(),
Hits: hits,
}, nil
}
// Close implements indexer
func (b *ElasticSearchIndexer) Close() {
select {
case <-b.stopTimer:
default:
close(b.stopTimer)
}
}
func (b *ElasticSearchIndexer) checkError(err error) error {
var opErr *net.OpError
if !(elastic.IsConnErr(err) || (errors.As(err, &opErr) && (opErr.Op == "dial" || opErr.Op == "read"))) {
return err
}
b.setAvailability(false)
return err
}
func (b *ElasticSearchIndexer) checkAvailability() {
if b.Ping() {
return
}
// Request cluster state to check if elastic is available again
_, err := b.client.ClusterState().Do(graceful.GetManager().ShutdownContext())
if err != nil {
b.setAvailability(false)
return
}
b.setAvailability(true)
}
func (b *ElasticSearchIndexer) setAvailability(available bool) {
b.lock.Lock()
defer b.lock.Unlock()
if b.available == available {
return
}
b.available = available
}

177
modules/indexer/issues/elasticsearch/elasticsearch.go Normal file
View File

@@ -0,0 +1,177 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package elasticsearch
import (
"context"
"fmt"
"strconv"
"code.gitea.io/gitea/modules/graceful"
indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
inner_elasticsearch "code.gitea.io/gitea/modules/indexer/internal/elasticsearch"
"code.gitea.io/gitea/modules/indexer/issues/internal"
"github.com/olivere/elastic/v7"
)
const (
issueIndexerLatestVersion = 0
)
var _ internal.Indexer = &Indexer{}
// Indexer implements Indexer interface
type Indexer struct {
inner *inner_elasticsearch.Indexer
indexer_internal.Indexer // do not composite inner_elasticsearch.Indexer directly to avoid exposing too much
}
// NewIndexer creates a new elasticsearch indexer
func NewIndexer(url, indexerName string) *Indexer {
inner := inner_elasticsearch.NewIndexer(url, indexerName, issueIndexerLatestVersion, defaultMapping)
indexer := &Indexer{
inner: inner,
Indexer: inner,
}
return indexer
}
const (
defaultMapping = `{
"mappings": {
"properties": {
"id": {
"type": "integer",
"index": true
},
"repo_id": {
"type": "integer",
"index": true
},
"title": {
"type": "text",
"index": true
},
"content": {
"type": "text",
"index": true
},
"comments": {
"type" : "text",
"index": true
}
}
}
}`
)
// Index will save the index data
func (b *Indexer) Index(ctx context.Context, issues []*internal.IndexerData) error {
if len(issues) == 0 {
return nil
} else if len(issues) == 1 {
issue := issues[0]
_, err := b.inner.Client.Index().
Index(b.inner.VersionedIndexName()).
Id(fmt.Sprintf("%d", issue.ID)).
BodyJson(map[string]interface{}{
"id": issue.ID,
"repo_id": issue.RepoID,
"title": issue.Title,
"content": issue.Content,
"comments": issue.Comments,
}).
Do(ctx)
return err
}
reqs := make([]elastic.BulkableRequest, 0)
for _, issue := range issues {
reqs = append(reqs,
elastic.NewBulkIndexRequest().
Index(b.inner.VersionedIndexName()).
Id(fmt.Sprintf("%d", issue.ID)).
Doc(map[string]interface{}{
"id": issue.ID,
"repo_id": issue.RepoID,
"title": issue.Title,
"content": issue.Content,
"comments": issue.Comments,
}),
)
}
_, err := b.inner.Client.Bulk().
Index(b.inner.VersionedIndexName()).
Add(reqs...).
Do(graceful.GetManager().HammerContext())
return err
}
// Delete deletes indexes by ids
func (b *Indexer) Delete(ctx context.Context, ids ...int64) error {
if len(ids) == 0 {
return nil
} else if len(ids) == 1 {
_, err := b.inner.Client.Delete().
Index(b.inner.VersionedIndexName()).
Id(fmt.Sprintf("%d", ids[0])).
Do(ctx)
return err
}
reqs := make([]elastic.BulkableRequest, 0)
for _, id := range ids {
reqs = append(reqs,
elastic.NewBulkDeleteRequest().
Index(b.inner.VersionedIndexName()).
Id(fmt.Sprintf("%d", id)),
)
}
_, err := b.inner.Client.Bulk().
Index(b.inner.VersionedIndexName()).
Add(reqs...).
Do(graceful.GetManager().HammerContext())
return err
}
// Search searches for issues by given conditions.
// Returns the matching issue IDs
func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*internal.SearchResult, error) {
kwQuery := elastic.NewMultiMatchQuery(keyword, "title", "content", "comments")
query := elastic.NewBoolQuery()
query = query.Must(kwQuery)
if len(repoIDs) > 0 {
repoStrs := make([]interface{}, 0, len(repoIDs))
for _, repoID := range repoIDs {
repoStrs = append(repoStrs, repoID)
}
repoQuery := elastic.NewTermsQuery("repo_id", repoStrs...)
query = query.Must(repoQuery)
}
searchResult, err := b.inner.Client.Search().
Index(b.inner.VersionedIndexName()).
Query(query).
Sort("_score", false).
From(start).Size(limit).
Do(ctx)
if err != nil {
return nil, err
}
hits := make([]internal.Match, 0, limit)
for _, hit := range searchResult.Hits.Hits {
id, _ := strconv.ParseInt(hit.Id, 10, 64)
hits = append(hits, internal.Match{
ID: id,
})
}
return &internal.SearchResult{
Total: searchResult.TotalHits(),
Hits: hits,
}, nil
}

199
modules/indexer/issues/indexer.go
View File

@@ -5,16 +5,20 @@ package issues
import (
"context"
"fmt"
"os"
"runtime/pprof"
"sync"
"sync/atomic"
"time"
"code.gitea.io/gitea/models/db"
db_model "code.gitea.io/gitea/models/db"
issues_model "code.gitea.io/gitea/models/issues"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/indexer/issues/bleve"
"code.gitea.io/gitea/modules/indexer/issues/db"
"code.gitea.io/gitea/modules/indexer/issues/elasticsearch"
"code.gitea.io/gitea/modules/indexer/issues/internal"
"code.gitea.io/gitea/modules/indexer/issues/meilisearch"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/process"
"code.gitea.io/gitea/modules/queue"
@@ -22,81 +26,22 @@ import (
"code.gitea.io/gitea/modules/util"
)
// IndexerData data stored in the issue indexer
type IndexerData struct {
ID int64 `json:"id"`
RepoID int64 `json:"repo_id"`
Title string `json:"title"`
Content string `json:"content"`
Comments []string `json:"comments"`
IsDelete bool `json:"is_delete"`
IDs []int64 `json:"ids"`
}
// Match represents on search result
type Match struct {
ID int64 `json:"id"`
Score float64 `json:"score"`
}
// SearchResult represents search results
type SearchResult struct {
Total int64
Hits []Match
}
// Indexer defines an interface to indexer issues contents
type Indexer interface {
Init() (bool, error)
Ping() bool
Index(issue []*IndexerData) error
Delete(ids ...int64) error
Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*SearchResult, error)
Close()
}
type indexerHolder struct {
indexer Indexer
mutex sync.RWMutex
cond *sync.Cond
cancelled bool
}
func newIndexerHolder() *indexerHolder {
h := &indexerHolder{}
h.cond = sync.NewCond(h.mutex.RLocker())
return h
}
func (h *indexerHolder) cancel() {
h.mutex.Lock()
defer h.mutex.Unlock()
h.cancelled = true
h.cond.Broadcast()
}
func (h *indexerHolder) set(indexer Indexer) {
h.mutex.Lock()
defer h.mutex.Unlock()
h.indexer = indexer
h.cond.Broadcast()
}
func (h *indexerHolder) get() Indexer {
h.mutex.RLock()
defer h.mutex.RUnlock()
if h.indexer == nil && !h.cancelled {
h.cond.Wait()
}
return h.indexer
}
var (
// issueIndexerQueue queue of issue ids to be updated
issueIndexerQueue *queue.WorkerPoolQueue[*IndexerData]
holder = newIndexerHolder()
issueIndexerQueue *queue.WorkerPoolQueue[*internal.IndexerData]
// globalIndexer is the global indexer, it cannot be nil.
// When the real indexer is not ready, it will be a dummy indexer which will return error to explain it's not ready.
// So it's always safe use it as *globalIndexer.Load() and call its methods.
globalIndexer atomic.Pointer[internal.Indexer]
dummyIndexer *internal.Indexer
)
func init() {
i := internal.NewDummyIndexer()
dummyIndexer = &i
globalIndexer.Store(dummyIndexer)
}
// InitIssueIndexer initialize issue indexer, syncReindex is true then reindex until
// all issue index done.
func InitIssueIndexer(syncReindex bool) {
@@ -107,33 +52,23 @@ func InitIssueIndexer(syncReindex bool) {
// Create the Queue
switch setting.Indexer.IssueType {
case "bleve", "elasticsearch", "meilisearch":
handler := func(items ...*IndexerData) (unhandled []*IndexerData) {
indexer := holder.get()
if indexer == nil {
log.Warn("Issue indexer handler: indexer is not ready, retry later.")
return items
}
toIndex := make([]*IndexerData, 0, len(items))
handler := func(items ...*internal.IndexerData) (unhandled []*internal.IndexerData) {
indexer := *globalIndexer.Load()
toIndex := make([]*internal.IndexerData, 0, len(items))
for _, indexerData := range items {
log.Trace("IndexerData Process: %d %v %t", indexerData.ID, indexerData.IDs, indexerData.IsDelete)
if indexerData.IsDelete {
if err := indexer.Delete(indexerData.IDs...); err != nil {
if err := indexer.Delete(ctx, indexerData.IDs...); err != nil {
log.Error("Issue indexer handler: failed to from index: %v Error: %v", indexerData.IDs, err)
if !indexer.Ping() {
log.Error("Issue indexer handler: indexer is unavailable when deleting")
unhandled = append(unhandled, indexerData)
}
unhandled = append(unhandled, indexerData)
}
continue
}
toIndex = append(toIndex, indexerData)
}
if err := indexer.Index(toIndex); err != nil {
if err := indexer.Index(ctx, toIndex); err != nil {
log.Error("Error whilst indexing: %v Error: %v", toIndex, err)
if !indexer.Ping() {
log.Error("Issue indexer handler: indexer is unavailable when indexing")
unhandled = append(unhandled, toIndex...)
}
unhandled = append(unhandled, toIndex...)
}
return unhandled
}
@@ -144,7 +79,7 @@ func InitIssueIndexer(syncReindex bool) {
log.Fatal("Unable to create issue indexer queue")
}
default:
issueIndexerQueue = queue.CreateSimpleQueue[*IndexerData](ctx, "issue_indexer", nil)
issueIndexerQueue = queue.CreateSimpleQueue[*internal.IndexerData](ctx, "issue_indexer", nil)
}
graceful.GetManager().RunAtTerminate(finished)
@@ -154,7 +89,11 @@ func InitIssueIndexer(syncReindex bool) {
pprof.SetGoroutineLabels(ctx)
start := time.Now()
log.Info("PID %d: Initializing Issue Indexer: %s", os.Getpid(), setting.Indexer.IssueType)
var populate bool
var (
issueIndexer internal.Indexer
existed bool
err error
)
switch setting.Indexer.IssueType {
case "bleve":
defer func() {
@@ -162,62 +101,45 @@ func InitIssueIndexer(syncReindex bool) {
log.Error("PANIC whilst initializing issue indexer: %v\nStacktrace: %s", err, log.Stack(2))
log.Error("The indexer files are likely corrupted and may need to be deleted")
log.Error("You can completely remove the %q directory to make Gitea recreate the indexes", setting.Indexer.IssuePath)
holder.cancel()
globalIndexer.Store(dummyIndexer)
log.Fatal("PID: %d Unable to initialize the Bleve Issue Indexer at path: %s Error: %v", os.Getpid(), setting.Indexer.IssuePath, err)
}
}()
issueIndexer := NewBleveIndexer(setting.Indexer.IssuePath)
exist, err := issueIndexer.Init()
issueIndexer = bleve.NewIndexer(setting.Indexer.IssuePath)
existed, err = issueIndexer.Init(ctx)
if err != nil {
holder.cancel()
log.Fatal("Unable to initialize Bleve Issue Indexer at path: %s Error: %v", setting.Indexer.IssuePath, err)
}
populate = !exist
holder.set(issueIndexer)
graceful.GetManager().RunAtTerminate(func() {
log.Debug("Closing issue indexer")
issueIndexer := holder.get()
if issueIndexer != nil {
issueIndexer.Close()
}
log.Info("PID: %d Issue Indexer closed", os.Getpid())
})
log.Debug("Created Bleve Indexer")
case "elasticsearch":
issueIndexer, err := NewElasticSearchIndexer(setting.Indexer.IssueConnStr, setting.Indexer.IssueIndexerName)
if err != nil {
log.Fatal("Unable to initialize Elastic Search Issue Indexer at connection: %s Error: %v", setting.Indexer.IssueConnStr, err)
}
exist, err := issueIndexer.Init()
issueIndexer = elasticsearch.NewIndexer(setting.Indexer.IssueConnStr, setting.Indexer.IssueIndexerName)
existed, err = issueIndexer.Init(ctx)
if err != nil {
log.Fatal("Unable to issueIndexer.Init with connection %s Error: %v", setting.Indexer.IssueConnStr, err)
}
populate = !exist
holder.set(issueIndexer)
case "db":
issueIndexer := &DBIndexer{}
holder.set(issueIndexer)
issueIndexer = db.NewIndexer()
case "meilisearch":
issueIndexer, err := NewMeilisearchIndexer(setting.Indexer.IssueConnStr, setting.Indexer.IssueConnAuth, setting.Indexer.IssueIndexerName)
if err != nil {
log.Fatal("Unable to initialize Meilisearch Issue Indexer at connection: %s Error: %v", setting.Indexer.IssueConnStr, err)
}
exist, err := issueIndexer.Init()
issueIndexer = meilisearch.NewIndexer(setting.Indexer.IssueConnStr, setting.Indexer.IssueConnAuth, setting.Indexer.IssueIndexerName)
existed, err = issueIndexer.Init(ctx)
if err != nil {
log.Fatal("Unable to issueIndexer.Init with connection %s Error: %v", setting.Indexer.IssueConnStr, err)
}
populate = !exist
holder.set(issueIndexer)
default:
holder.cancel()
log.Fatal("Unknown issue indexer type: %s", setting.Indexer.IssueType)
}
globalIndexer.Store(&issueIndexer)
graceful.GetManager().RunAtTerminate(func() {
log.Debug("Closing issue indexer")
(*globalIndexer.Load()).Close()
log.Info("PID: %d Issue Indexer closed", os.Getpid())
})
// Start processing the queue
go graceful.GetManager().RunWithCancel(issueIndexerQueue)
// Populate the index
if populate {
if !existed {
if syncReindex {
graceful.GetManager().RunWithShutdownContext(populateIssueIndexer)
} else {
@@ -266,8 +188,8 @@ func populateIssueIndexer(ctx context.Context) {
default:
}
repos, _, err := repo_model.SearchRepositoryByName(ctx, &repo_model.SearchRepoOptions{
ListOptions: db.ListOptions{Page: page, PageSize: repo_model.RepositoryListDefaultPageSize},
OrderBy: db.SearchOrderByID,
ListOptions: db_model.ListOptions{Page: page, PageSize: repo_model.RepositoryListDefaultPageSize},
OrderBy: db_model.SearchOrderByID,
Private: true,
Collaborate: util.OptionalBoolFalse,
})
@@ -320,7 +242,7 @@ func UpdateIssueIndexer(issue *issues_model.Issue) {
comments = append(comments, comment.Content)
}
}
indexerData := &IndexerData{
indexerData := &internal.IndexerData{
ID: issue.ID,
RepoID: issue.RepoID,
Title: issue.Title,
@@ -345,7 +267,7 @@ func DeleteRepoIssueIndexer(ctx context.Context, repo *repo_model.Repository) {
if len(ids) == 0 {
return
}
indexerData := &IndexerData{
indexerData := &internal.IndexerData{
IDs: ids,
IsDelete: true,
}
@@ -358,12 +280,7 @@ func DeleteRepoIssueIndexer(ctx context.Context, repo *repo_model.Repository) {
// WARNNING: You have to ensure user have permission to visit repoIDs' issues
func SearchIssuesByKeyword(ctx context.Context, repoIDs []int64, keyword string) ([]int64, error) {
var issueIDs []int64
indexer := holder.get()
if indexer == nil {
log.Error("SearchIssuesByKeyword(): unable to get indexer!")
return nil, fmt.Errorf("unable to get issue indexer")
}
indexer := *globalIndexer.Load()
res, err := indexer.Search(ctx, keyword, repoIDs, 50, 0)
if err != nil {
return nil, err
@@ -375,12 +292,6 @@ func SearchIssuesByKeyword(ctx context.Context, repoIDs []int64, keyword string)
}
// IsAvailable checks if issue indexer is available
func IsAvailable() bool {
indexer := holder.get()
if indexer == nil {
log.Error("IsAvailable(): unable to get indexer!")
return false
}
return indexer.Ping()
func IsAvailable(ctx context.Context) bool {
return (*globalIndexer.Load()).Ping(ctx) == nil
}

4
modules/indexer/issues/indexer_test.go
View File

@@ -11,6 +11,7 @@ import (
"time"
"code.gitea.io/gitea/models/unittest"
"code.gitea.io/gitea/modules/indexer/issues/bleve"
"code.gitea.io/gitea/modules/setting"
_ "code.gitea.io/gitea/models"
@@ -42,8 +43,7 @@ func TestBleveSearchIssues(t *testing.T) {
setting.LoadQueueSettings()
InitIssueIndexer(true)
defer func() {
indexer := holder.get()
if bleveIndexer, ok := indexer.(*BleveIndexer); ok {
if bleveIndexer, ok := (*globalIndexer.Load()).(*bleve.Indexer); ok {
bleveIndexer.Close()
}
}()

42
modules/indexer/issues/internal/indexer.go Normal file
View File

@@ -0,0 +1,42 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package internal
import (
"context"
"fmt"
"code.gitea.io/gitea/modules/indexer/internal"
)
// Indexer defines an interface to indexer issues contents
type Indexer interface {
internal.Indexer
Index(ctx context.Context, issue []*IndexerData) error
Delete(ctx context.Context, ids ...int64) error
Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*SearchResult, error)
}
// NewDummyIndexer returns a dummy indexer
func NewDummyIndexer() Indexer {
return &dummyIndexer{
Indexer: internal.NewDummyIndexer(),
}
}
type dummyIndexer struct {
internal.Indexer
}
func (d *dummyIndexer) Index(ctx context.Context, issue []*IndexerData) error {
return fmt.Errorf("indexer is not ready")
}
func (d *dummyIndexer) Delete(ctx context.Context, ids ...int64) error {
return fmt.Errorf("indexer is not ready")
}
func (d *dummyIndexer) Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*SearchResult, error) {
return nil, fmt.Errorf("indexer is not ready")
}

27
modules/indexer/issues/internal/model.go Normal file
View File

@@ -0,0 +1,27 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package internal
// IndexerData data stored in the issue indexer
type IndexerData struct {
ID int64 `json:"id"`
RepoID int64 `json:"repo_id"`
Title string `json:"title"`
Content string `json:"content"`
Comments []string `json:"comments"`
IsDelete bool `json:"is_delete"`
IDs []int64 `json:"ids"`
}
// Match represents on search result
type Match struct {
ID int64 `json:"id"`
Score float64 `json:"score"`
}
// SearchResult represents search results
type SearchResult struct {
Total int64
Hits []Match
}

173
modules/indexer/issues/meilisearch.go
View File

@@ -1,173 +0,0 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package issues
import (
"context"
"strconv"
"strings"
"sync"
"time"
"github.com/meilisearch/meilisearch-go"
)
var _ Indexer = &MeilisearchIndexer{}
// MeilisearchIndexer implements Indexer interface
type MeilisearchIndexer struct {
client *meilisearch.Client
indexerName string
available bool
stopTimer chan struct{}
lock sync.RWMutex
}
// MeilisearchIndexer creates a new meilisearch indexer
func NewMeilisearchIndexer(url, apiKey, indexerName string) (*MeilisearchIndexer, error) {
client := meilisearch.NewClient(meilisearch.ClientConfig{
Host: url,
APIKey: apiKey,
})
indexer := &MeilisearchIndexer{
client: client,
indexerName: indexerName,
available: true,
stopTimer: make(chan struct{}),
}
ticker := time.NewTicker(10 * time.Second)
go func() {
for {
select {
case <-ticker.C:
indexer.checkAvailability()
case <-indexer.stopTimer:
ticker.Stop()
return
}
}
}()
return indexer, nil
}
// Init will initialize the indexer
func (b *MeilisearchIndexer) Init() (bool, error) {
_, err := b.client.GetIndex(b.indexerName)
if err == nil {
return true, nil
}
_, err = b.client.CreateIndex(&meilisearch.IndexConfig{
Uid: b.indexerName,
PrimaryKey: "id",
})
if err != nil {
return false, b.checkError(err)
}
_, err = b.client.Index(b.indexerName).UpdateFilterableAttributes(&[]string{"repo_id"})
return false, b.checkError(err)
}
// Ping checks if meilisearch is available
func (b *MeilisearchIndexer) Ping() bool {
b.lock.RLock()
defer b.lock.RUnlock()
return b.available
}
// Index will save the index data
func (b *MeilisearchIndexer) Index(issues []*IndexerData) error {
if len(issues) == 0 {
return nil
}
for _, issue := range issues {
_, err := b.client.Index(b.indexerName).AddDocuments(issue)
if err != nil {
return b.checkError(err)
}
}
// TODO: bulk send index data
return nil
}
// Delete deletes indexes by ids
func (b *MeilisearchIndexer) Delete(ids ...int64) error {
if len(ids) == 0 {
return nil
}
for _, id := range ids {
_, err := b.client.Index(b.indexerName).DeleteDocument(strconv.FormatInt(id, 10))
if err != nil {
return b.checkError(err)
}
}
// TODO: bulk send deletes
return nil
}
// Search searches for issues by given conditions.
// Returns the matching issue IDs
func (b *MeilisearchIndexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*SearchResult, error) {
repoFilters := make([]string, 0, len(repoIDs))
for _, repoID := range repoIDs {
repoFilters = append(repoFilters, "repo_id = "+strconv.FormatInt(repoID, 10))
}
filter := strings.Join(repoFilters, " OR ")
searchRes, err := b.client.Index(b.indexerName).Search(keyword, &meilisearch.SearchRequest{
Filter: filter,
Limit: int64(limit),
Offset: int64(start),
})
if err != nil {
return nil, b.checkError(err)
}
hits := make([]Match, 0, len(searchRes.Hits))
for _, hit := range searchRes.Hits {
hits = append(hits, Match{
ID: int64(hit.(map[string]interface{})["id"].(float64)),
})
}
return &SearchResult{
Total: searchRes.TotalHits,
Hits: hits,
}, nil
}
// Close implements indexer
func (b *MeilisearchIndexer) Close() {
select {
case <-b.stopTimer:
default:
close(b.stopTimer)
}
}
func (b *MeilisearchIndexer) checkError(err error) error {
return err
}
func (b *MeilisearchIndexer) checkAvailability() {
_, err := b.client.Health()
if err != nil {
b.setAvailability(false)
return
}
b.setAvailability(true)
}
func (b *MeilisearchIndexer) setAvailability(available bool) {
b.lock.Lock()
defer b.lock.Unlock()
if b.available == available {
return
}
b.available = available
}

98
modules/indexer/issues/meilisearch/meilisearch.go Normal file
View File

@@ -0,0 +1,98 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package meilisearch
import (
"context"
"strconv"
"strings"
indexer_internal "code.gitea.io/gitea/modules/indexer/internal"
inner_meilisearch "code.gitea.io/gitea/modules/indexer/internal/meilisearch"
"code.gitea.io/gitea/modules/indexer/issues/internal"
"github.com/meilisearch/meilisearch-go"
)
const (
issueIndexerLatestVersion = 0
)
var _ internal.Indexer = &Indexer{}
// Indexer implements Indexer interface
type Indexer struct {
inner *inner_meilisearch.Indexer
indexer_internal.Indexer // do not composite inner_meilisearch.Indexer directly to avoid exposing too much
}
// NewIndexer creates a new meilisearch indexer
func NewIndexer(url, apiKey, indexerName string) *Indexer {
inner := inner_meilisearch.NewIndexer(url, apiKey, indexerName, issueIndexerLatestVersion)
indexer := &Indexer{
inner: inner,
Indexer: inner,
}
return indexer
}
// Index will save the index data
func (b *Indexer) Index(_ context.Context, issues []*internal.IndexerData) error {
if len(issues) == 0 {
return nil
}
for _, issue := range issues {
_, err := b.inner.Client.Index(b.inner.VersionedIndexName()).AddDocuments(issue)
if err != nil {
return err
}
}
// TODO: bulk send index data
return nil
}
// Delete deletes indexes by ids
func (b *Indexer) Delete(_ context.Context, ids ...int64) error {
if len(ids) == 0 {
return nil
}
for _, id := range ids {
_, err := b.inner.Client.Index(b.inner.VersionedIndexName()).DeleteDocument(strconv.FormatInt(id, 10))
if err != nil {
return err
}
}
// TODO: bulk send deletes
return nil
}
// Search searches for issues by given conditions.
// Returns the matching issue IDs
func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*internal.SearchResult, error) {
repoFilters := make([]string, 0, len(repoIDs))
for _, repoID := range repoIDs {
repoFilters = append(repoFilters, "repo_id = "+strconv.FormatInt(repoID, 10))
}
filter := strings.Join(repoFilters, " OR ")
searchRes, err := b.inner.Client.Index(b.inner.VersionedIndexName()).Search(keyword, &meilisearch.SearchRequest{
Filter: filter,
Limit: int64(limit),
Offset: int64(start),
})
if err != nil {
return nil, err
}
hits := make([]internal.Match, 0, len(searchRes.Hits))
for _, hit := range searchRes.Hits {
hits = append(hits, internal.Match{
ID: int64(hit.(map[string]interface{})["id"].(float64)),
})
}
return &internal.SearchResult{
Total: searchRes.TotalHits,
Hits: hits,
}, nil
}