// Copyright 2016 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. package code import ( "context" "os" "strconv" "strings" "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" ) // SearchResult result of performing a search in a repo type SearchResult struct { RepoID int64 StartIndex int EndIndex int Filename string Content string CommitID string UpdatedUnix timeutil.TimeStamp Language string Color string } // SearchResultLanguages result of top languages count in search results type SearchResultLanguages struct { Language string Color string Count int } // Indexer defines an interface to indexer issues contents type Indexer interface { Index(repo *models.Repository, sha string, changes *repoChanges) error Delete(repoID int64) error Search(repoIDs []int64, language, keyword string, page, pageSize int) (int64, []*SearchResult, []*SearchResultLanguages, error) Close() } func filenameIndexerID(repoID int64, filename string) string { return indexerID(repoID) + "_" + filename } func parseIndexerID(indexerID string) (int64, string) { index := strings.IndexByte(indexerID, '_') if index == -1 { log.Error("Unexpected ID in repo indexer: %s", indexerID) } repoID, _ := strconv.ParseInt(indexerID[:index], 10, 64) return repoID, indexerID[index+1:] } func filenameOfIndexerID(indexerID string) string { index := strings.IndexByte(indexerID, '_') if index == -1 { log.Error("Unexpected ID in repo indexer: %s", indexerID) } return indexerID[index+1:] } // Init initialize the repo indexer func Init() { if !setting.Indexer.RepoIndexerEnabled { indexer.Close() return } initQueue(setting.Indexer.UpdateQueueLength) ctx, cancel := context.WithCancel(context.Background()) graceful.GetManager().RunAtTerminate(ctx, func() { log.Debug("Closing repository indexer") indexer.Close() log.Info("PID: %d Repository Indexer closed", os.Getpid()) }) waitChannel := make(chan time.Duration) go func() { start := time.Now() var ( rIndexer Indexer populate bool err error ) switch setting.Indexer.RepoType { case "bleve": log.Info("PID: %d Initializing Repository Indexer at: %s", os.Getpid(), setting.Indexer.RepoPath) defer func() { if err := recover(); err != nil { log.Error("PANIC whilst initializing repository indexer: %v\nStacktrace: %s", err, log.Stack(2)) log.Error("The indexer files are likely corrupted and may need to be deleted") log.Error("You can completely remove the \"%s\" directory to make Gitea recreate the indexes", setting.Indexer.RepoPath) } }() rIndexer, populate, err = NewBleveIndexer(setting.Indexer.RepoPath) if err != nil { if rIndexer != nil { rIndexer.Close() } cancel() indexer.Close() close(waitChannel) log.Fatal("PID: %d Unable to initialize the bleve Repository Indexer at path: %s Error: %v", os.Getpid(), setting.Indexer.RepoPath, err) } case "elasticsearch": log.Info("PID: %d Initializing Repository Indexer at: %s", os.Getpid(), setting.Indexer.RepoConnStr) defer func() { if err := recover(); err != nil { log.Error("PANIC whilst initializing repository indexer: %v\nStacktrace: %s", err, log.Stack(2)) log.Error("The indexer files are likely corrupted and may need to be deleted") log.Error("You can completely remove the \"%s\" index to make Gitea recreate the indexes", setting.Indexer.RepoConnStr) } }() rIndexer, populate, err = NewElasticSearchIndexer(setting.Indexer.RepoConnStr, setting.Indexer.RepoIndexerName) if err != nil { if rIndexer != nil { rIndexer.Close() } cancel() indexer.Close() close(waitChannel) log.Fatal("PID: %d Unable to initialize the elasticsearch Repository Indexer connstr: %s Error: %v", os.Getpid(), setting.Indexer.RepoConnStr, err) } default: log.Fatal("PID: %d Unknown Indexer type: %s", os.Getpid(), setting.Indexer.RepoType) } indexer.set(rIndexer) go processRepoIndexerOperationQueue(indexer) if populate { go populateRepoIndexer() } select { case waitChannel <- time.Since(start): case <-graceful.GetManager().IsShutdown(): } close(waitChannel) }() if setting.Indexer.StartupTimeout > 0 { go func() { timeout := setting.Indexer.StartupTimeout if graceful.GetManager().IsChild() && setting.GracefulHammerTime > 0 { timeout += setting.GracefulHammerTime } select { case <-graceful.GetManager().IsShutdown(): log.Warn("Shutdown before Repository Indexer completed initialization") cancel() indexer.Close() case duration, ok := <-waitChannel: if !ok { log.Warn("Repository Indexer Initialization failed") cancel() indexer.Close() return } log.Info("Repository Indexer Initialization took %v", duration) case <-time.After(timeout): cancel() indexer.Close() log.Fatal("Repository Indexer Initialization Timed-Out after: %v", timeout) } }() } }