Global code search support (#3664)

* add global code search on explore

* fix bug when no anyone public repos

* change the icon

* fix typo and add UnitTypeCode check for login non-admin user

* fix ui description when no match

models/repo.go

@@ -1945,6 +1945,12 @@ func GetRepositoryByID(id int64) (*Repository, error) {
 	return getRepositoryByID(x, id)
 }
 
+// GetRepositoriesMapByIDs returns the repositories by given id slice.
+func GetRepositoriesMapByIDs(ids []int64) (map[int64]*Repository, error) {
+	var repos = make(map[int64]*Repository, len(ids))
+	return repos, x.In("id", ids).Find(&repos)
+}
+
 // GetUserRepositories returns a list of repositories of given user.
 func GetUserRepositories(userID int64, private bool, page, pageSize int, orderBy string) ([]*Repository, error) {
 	if len(orderBy) == 0 {

models/repo_list.go

@@ -249,3 +249,28 @@ func SearchRepositoryByName(opts *SearchRepoOptions) (RepositoryList, int64, err
 
 	return repos, count, nil
 }
+
+// FindUserAccessibleRepoIDs find all accessible repositories' ID by user's id
+func FindUserAccessibleRepoIDs(userID int64) ([]int64, error) {
+	var accessCond builder.Cond = builder.Eq{"is_private": false}
+
+	if userID > 0 {
+		accessCond = accessCond.Or(
+			builder.Eq{"owner_id": userID},
+			builder.And(
+				builder.Expr("id IN (SELECT repo_id FROM `access` WHERE access.user_id = ?)", userID),
+				builder.Neq{"owner_id": userID},
+			),
+		)
+	}
+
+	repoIDs := make([]int64, 0, 10)
+	if err := x.
+		Table("repository").
+		Cols("id").
+		Where(accessCond).
+		Find(&repoIDs); err != nil {
+		return nil, fmt.Errorf("FindUserAccesibleRepoIDs: %v", err)
+	}
+	return repoIDs, nil
+}

modules/indexer/repo.go

@@ -16,6 +16,7 @@ import (
 	"github.com/blevesearch/bleve/analysis/token/lowercase"
 	"github.com/blevesearch/bleve/analysis/token/unique"
 	"github.com/blevesearch/bleve/analysis/tokenizer/unicode"
+	"github.com/blevesearch/bleve/search/query"
 	"github.com/ethantkoenig/rupture"
 )
 
@@ -158,6 +159,7 @@ func DeleteRepoFromIndexer(repoID int64) error {
 
 // RepoSearchResult result of performing a search in a repo
 type RepoSearchResult struct {
+	RepoID     int64
 	StartIndex int
 	EndIndex   int
 	Filename   string
@@ -166,17 +168,29 @@ type RepoSearchResult struct {
 
 // SearchRepoByKeyword searches for files in the specified repo.
 // Returns the matching file-paths
-func SearchRepoByKeyword(repoID int64, keyword string, page, pageSize int) (int64, []*RepoSearchResult, error) {
+func SearchRepoByKeyword(repoIDs []int64, keyword string, page, pageSize int) (int64, []*RepoSearchResult, error) {
 	phraseQuery := bleve.NewMatchPhraseQuery(keyword)
 	phraseQuery.FieldVal = "Content"
 	phraseQuery.Analyzer = repoIndexerAnalyzer
-	indexerQuery := bleve.NewConjunctionQuery(
+
-		numericEqualityQuery(repoID, "RepoID"),
+	var indexerQuery query.Query
+	if len(repoIDs) > 0 {
+		var repoQueries = make([]query.Query, 0, len(repoIDs))
+		for _, repoID := range repoIDs {
+			repoQueries = append(repoQueries, numericEqualityQuery(repoID, "RepoID"))
+		}
+
+		indexerQuery = bleve.NewConjunctionQuery(
+			bleve.NewDisjunctionQuery(repoQueries...),
 			phraseQuery,
 		)
+	} else {
+		indexerQuery = phraseQuery
+	}
+
 	from := (page - 1) * pageSize
 	searchRequest := bleve.NewSearchRequestOptions(indexerQuery, pageSize, from, false)
-	searchRequest.Fields = []string{"Content"}
+	searchRequest.Fields = []string{"Content", "RepoID"}
 	searchRequest.IncludeLocations = true
 
 	result, err := repoIndexer.Search(searchRequest)
@@ -199,6 +213,7 @@ func SearchRepoByKeyword(repoID int64, keyword string, page, pageSize int) (int6
 			}
 		}
 		searchResults[i] = &RepoSearchResult{
+			RepoID:     int64(hit.Fields["RepoID"].(float64)),
 			StartIndex: startIndex,
 			EndIndex:   endIndex,
 			Filename:   filenameOfIndexerID(hit.ID),

modules/search/search.go

@@ -17,6 +17,7 @@ import (
 
 // Result a search result to display
 type Result struct {
+	RepoID         int64
 	Filename       string
 	HighlightClass string
 	LineNumbers    []int
@@ -98,6 +99,7 @@ func searchResult(result *indexer.RepoSearchResult, startIndex, endIndex int) (*
 		index += len(line)
 	}
 	return &Result{
+		RepoID:         result.RepoID,
 		Filename:       result.Filename,
 		HighlightClass: highlight.FileNameToHighlightClass(result.Filename),
 		LineNumbers:    lineNumbers,
@@ -106,12 +108,12 @@ func searchResult(result *indexer.RepoSearchResult, startIndex, endIndex int) (*
 }
 
 // PerformSearch perform a search on a repository
-func PerformSearch(repoID int64, keyword string, page, pageSize int) (int, []*Result, error) {
+func PerformSearch(repoIDs []int64, keyword string, page, pageSize int) (int, []*Result, error) {
 	if len(keyword) == 0 {
 		return 0, nil, nil
 	}
 
-	total, results, err := indexer.SearchRepoByKeyword(repoID, keyword, page, pageSize)
+	total, results, err := indexer.SearchRepoByKeyword(repoIDs, keyword, page, pageSize)
 	if err != nil {
 		return 0, nil, err
 	}

options/locale/locale_en-US.ini

@@ -169,9 +169,12 @@ repos = Repositories
 users = Users
 organizations = Organizations
 search = Search
+code = Code
 repo_no_results = No matching repositories have been found.
 user_no_results = No matching users have been found.
 org_no_results = No matching organizations have been found.
+code_no_results = No matching codes have been found.
+code_search_results = Search results for "%s"
 
 [auth]
 create_new_account = Create Account

routers/home.go

@@ -11,6 +11,7 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/search"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
 	"code.gitea.io/gitea/routers/user"
@@ -27,6 +28,8 @@ const (
 	tplExploreUsers base.TplName = "explore/users"
 	// tplExploreOrganizations explore organizations page template
 	tplExploreOrganizations base.TplName = "explore/organizations"
+	// tplExploreCode explore code page template
+	tplExploreCode base.TplName = "explore/code"
 )
 
 // Home render home page
@@ -49,6 +52,7 @@ func Home(ctx *context.Context) {
 	}
 
 	ctx.Data["PageIsHome"] = true
+	ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
 	ctx.HTML(200, tplHome)
 }
 
@@ -124,6 +128,7 @@ func RenderRepoSearch(ctx *context.Context, opts *RepoSearchOptions) {
 	ctx.Data["Total"] = count
 	ctx.Data["Page"] = paginater.New(int(count), opts.PageSize, page, 5)
 	ctx.Data["Repos"] = repos
+	ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
 
 	ctx.HTML(200, opts.TplName)
 }
@@ -133,6 +138,7 @@ func ExploreRepos(ctx *context.Context) {
 	ctx.Data["Title"] = ctx.Tr("explore")
 	ctx.Data["PageIsExplore"] = true
 	ctx.Data["PageIsExploreRepositories"] = true
+	ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
 
 	var ownerID int64
 	if ctx.User != nil && !ctx.User.IsAdmin {
@@ -194,6 +200,7 @@ func RenderUserSearch(ctx *context.Context, opts *models.SearchUserOptions, tplN
 	ctx.Data["Page"] = paginater.New(int(count), opts.PageSize, opts.Page, 5)
 	ctx.Data["Users"] = users
 	ctx.Data["ShowUserEmail"] = setting.UI.ShowUserEmail
+	ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
 
 	ctx.HTML(200, tplName)
 }
@@ -203,6 +210,7 @@ func ExploreUsers(ctx *context.Context) {
 	ctx.Data["Title"] = ctx.Tr("explore")
 	ctx.Data["PageIsExplore"] = true
 	ctx.Data["PageIsExploreUsers"] = true
+	ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
 
 	RenderUserSearch(ctx, &models.SearchUserOptions{
 		Type:     models.UserTypeIndividual,
@@ -216,6 +224,7 @@ func ExploreOrganizations(ctx *context.Context) {
 	ctx.Data["Title"] = ctx.Tr("explore")
 	ctx.Data["PageIsExplore"] = true
 	ctx.Data["PageIsExploreOrganizations"] = true
+	ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
 
 	RenderUserSearch(ctx, &models.SearchUserOptions{
 		Type:     models.UserTypeOrganization,
@@ -223,6 +232,113 @@ func ExploreOrganizations(ctx *context.Context) {
 	}, tplExploreOrganizations)
 }
 
+// ExploreCode render explore code page
+func ExploreCode(ctx *context.Context) {
+	if !setting.Indexer.RepoIndexerEnabled {
+		ctx.Redirect("/explore", 302)
+		return
+	}
+
+	ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
+	ctx.Data["Title"] = ctx.Tr("explore")
+	ctx.Data["PageIsExplore"] = true
+	ctx.Data["PageIsExploreCode"] = true
+
+	keyword := strings.TrimSpace(ctx.Query("q"))
+	page := ctx.QueryInt("page")
+	if page <= 0 {
+		page = 1
+	}
+
+	var (
+		repoIDs []int64
+		err     error
+		isAdmin bool
+		userID  int64
+	)
+	if ctx.User != nil {
+		userID = ctx.User.ID
+		isAdmin = ctx.User.IsAdmin
+	}
+
+	// guest user or non-admin user
+	if ctx.User == nil || !isAdmin {
+		repoIDs, err = models.FindUserAccessibleRepoIDs(userID)
+		if err != nil {
+			ctx.ServerError("SearchResults", err)
+			return
+		}
+	}
+
+	var (
+		total         int
+		searchResults []*search.Result
+	)
+
+	// if non-admin login user, we need check UnitTypeCode at first
+	if ctx.User != nil && len(repoIDs) > 0 {
+		repoMaps, err := models.GetRepositoriesMapByIDs(repoIDs)
+		if err != nil {
+			ctx.ServerError("SearchResults", err)
+			return
+		}
+
+		var rightRepoMap = make(map[int64]*models.Repository, len(repoMaps))
+		repoIDs = make([]int64, 0, len(repoMaps))
+		for id, repo := range repoMaps {
+			if repo.CheckUnitUser(userID, isAdmin, models.UnitTypeCode) {
+				rightRepoMap[id] = repo
+				repoIDs = append(repoIDs, id)
+			}
+		}
+
+		ctx.Data["RepoMaps"] = rightRepoMap
+
+		total, searchResults, err = search.PerformSearch(repoIDs, keyword, page, setting.UI.RepoSearchPagingNum)
+		if err != nil {
+			ctx.ServerError("SearchResults", err)
+			return
+		}
+		// if non-login user or isAdmin, no need to check UnitTypeCode
+	} else if (ctx.User == nil && len(repoIDs) > 0) || isAdmin {
+		total, searchResults, err = search.PerformSearch(repoIDs, keyword, page, setting.UI.RepoSearchPagingNum)
+		if err != nil {
+			ctx.ServerError("SearchResults", err)
+			return
+		}
+
+		var loadRepoIDs = make([]int64, 0, len(searchResults))
+		for _, result := range searchResults {
+			var find bool
+			for _, id := range loadRepoIDs {
+				if id == result.RepoID {
+					find = true
+					break
+				}
+			}
+			if !find {
+				loadRepoIDs = append(loadRepoIDs, result.RepoID)
+			}
+		}
+
+		repoMaps, err := models.GetRepositoriesMapByIDs(loadRepoIDs)
+		if err != nil {
+			ctx.ServerError("SearchResults", err)
+			return
+		}
+
+		ctx.Data["RepoMaps"] = repoMaps
+	}
+
+	ctx.Data["Keyword"] = keyword
+	pager := paginater.New(total, setting.UI.RepoSearchPagingNum, page, 5)
+	ctx.Data["Page"] = pager
+	ctx.Data["SearchResults"] = searchResults
+	ctx.Data["RequireHighlightJS"] = true
+	ctx.Data["PageIsViewCode"] = true
+	ctx.HTML(200, tplExploreCode)
+}
+
 // NotFound render 404 page
 func NotFound(ctx *context.Context) {
 	ctx.Data["Title"] = "Page Not Found"

routers/repo/search.go

@@ -29,7 +29,8 @@ func Search(ctx *context.Context) {
 	if page <= 0 {
 		page = 1
 	}
-	total, searchResults, err := search.PerformSearch(ctx.Repo.Repository.ID, keyword, page, setting.UI.RepoSearchPagingNum)
+	total, searchResults, err := search.PerformSearch([]int64{ctx.Repo.Repository.ID},
+		keyword, page, setting.UI.RepoSearchPagingNum)
 	if err != nil {
 		ctx.ServerError("SearchResults", err)
 		return

routers/routes/routes.go

@@ -170,6 +170,7 @@ func RegisterRoutes(m *macaron.Macaron) {
 		m.Get("/repos", routers.ExploreRepos)
 		m.Get("/users", routers.ExploreUsers)
 		m.Get("/organizations", routers.ExploreOrganizations)
+		m.Get("/code", routers.ExploreCode)
 	}, ignSignIn)
 	m.Combo("/install", routers.InstallInit).Get(routers.Install).
 		Post(bindIgnErr(auth.InstallForm{}), routers.InstallPost)

templates/explore/code.tmpl

@@ -0,0 +1,55 @@
+{{template "base/head" .}}
+<div class="explore users">
+	{{template "explore/navbar" .}}
+	<div class="ui container">
+		<form class="ui form" style="max-width: 100%">
+            <div class="ui fluid action input">
+                <input name="q" value="{{.Keyword}}" placeholder="{{.i18n.Tr "explore.search"}}..." autofocus>
+                <input type="hidden" name="tab" value="{{$.TabName}}">
+                <button class="ui blue button">{{.i18n.Tr "explore.search"}}</button>
+            </div>
+        </form>
+        <div class="ui divider"></div>
+
+		<div class="ui user list">
+			{{if .SearchResults}}
+                <h3>
+                    {{.i18n.Tr "explore.code_search_results" (.Keyword|Escape) | Str2html }}
+                </h3>
+                <div class="repository search">
+                    {{range $result := .SearchResults}}
+                        {{$repo := (index $.RepoMaps .RepoID)}}
+                        <div class="diff-file-box diff-box file-content non-diff-file-content repo-search-result">
+                            <h4 class="ui top attached normal header">
+                                <span class="file"><a rel="nofollow" href="{{EscapePound $repo.HTMLURL}}">{{$repo.FullName}}</a> - {{.Filename}}</span>
+                                <a class="ui basic grey tiny button" rel="nofollow" href="{{EscapePound $repo.HTMLURL}}/src/branch/{{$repo.DefaultBranch}}/{{EscapePound .Filename}}">{{$.i18n.Tr "repo.diff.view_file"}}</a>
+                            </h4>
+                            <div class="ui attached table segment">
+                                <div class="file-body file-code code-view">
+                                    <table>
+                                        <tbody>
+                                            <tr>
+                                                <td class="lines-num">
+                                                    {{range .LineNumbers}}
+                                                        <a href="{{EscapePound $repo.HTMLURL}}/src/branch/{{$repo.DefaultBranch}}/{{EscapePound $result.Filename}}#L{{.}}"><span>{{.}}</span></a>
+                                                    {{end}}
+                                                </td>
+                                                <td class="lines-code"><pre><code class="{{.HighlightClass}}"><ol class="linenums">{{.FormattedLines}}</ol></code></pre></td>
+                                            </tr>
+                                        </tbody>
+                                    </table>
+                                </div>
+                            </div>
+                        </div>
+                    {{end}}
+                </div>
+			{{else}}
+				<div>{{$.i18n.Tr "explore.code_no_results"}}</div>
+			{{end}}
+		</div>
+
+		{{template "base/paginate" .}}
+	</div>
+</div>
+{{template "base/footer" .}}
+

templates/explore/navbar.tmpl

@@ -8,4 +8,9 @@
 	<a class="{{if .PageIsExploreOrganizations}}active{{end}} item" href="{{AppSubUrl}}/explore/organizations">
 		<span class="octicon octicon-organization"></span> {{.i18n.Tr "explore.organizations"}}
 	</a>
+	{{if .IsRepoIndexerEnabled}}
+	<a class="{{if .PageIsExploreCode}}active{{end}} item" href="{{AppSubUrl}}/explore/code">
+		<span class="octicon octicon-code"></span> {{.i18n.Tr "explore.code"}}
+	</a>
+	{{end}}
 </div>