Restrict repository indexing by file extension

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -177,6 +177,8 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 
 - `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size).
 - `REPO_INDEXER_PATH`: **indexers/repos.bleve**: Index file used for code search.
+- `REPO_INDEXER_EXTENSIONS`: **empty**: A comma separated list of file extensions to include/exclude from the index; a \`.' matches files with no extension. An empty list means include all files, disregarding `REPO_EXTENSIONS_LIST_EXCLUDE`.
+- `REPO_EXTENSIONS_LIST_EXCLUDE`: **false**: If true, `REPO_INDEXER_EXTENSIONS` are the file extensions to exclude rather than include in the index.
 - `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request.
 - `MAX_FILE_SIZE`: **1048576**: Maximum size in bytes of files to be indexed.
 

models/repo_indexer.go

@@ -232,6 +232,17 @@ func addDelete(filename string, repo *Repository, batch rupture.FlushingBatch) e
 }
 
 func isIndexable(entry *git.TreeEntry) bool {
+	if setting.Indexer.FileExtensions != nil {
+		var ext string
+		parts := strings.Split(entry.Name(), ".")
+		cnt := len(parts)
+		if cnt > 1 {
+			ext = strings.ToLower(parts[cnt-1])
+		}
+		if setting.Indexer.FileExtensions[ext] == setting.Indexer.ExcludeExtensions {
+			return false
+		}
+	}
 	return entry.IsRegular() || entry.IsExecutable()
 }
 

modules/setting/indexer.go

@@ -7,6 +7,7 @@ package setting
 import (
 	"path"
 	"path/filepath"
+	"strings"
 )
 
 // enumerates all the indexer queue types
@@ -29,6 +30,8 @@ var (
 		IssueQueueDir         string
 		IssueQueueConnStr     string
 		IssueQueueBatchNumber int
+		FileExtensions        map[string]bool
+		ExcludeExtensions     bool
 	}{
 		IssueType:             "bleve",
 		IssuePath:             "indexers/issues.bleve",
@@ -51,6 +54,9 @@ func newIndexerService() {
 	if !filepath.IsAbs(Indexer.RepoPath) {
 		Indexer.RepoPath = path.Join(AppWorkPath, Indexer.RepoPath)
 	}
+	Indexer.FileExtensions = extensionsFromString(sec.Key("REPO_INDEXER_EXTENSIONS").MustString(""))
+	Indexer.ExcludeExtensions = sec.Key("REPO_EXTENSIONS_LIST_EXCLUDE").MustBool(false)
+
 	Indexer.UpdateQueueLength = sec.Key("UPDATE_BUFFER_LEN").MustInt(20)
 	Indexer.MaxIndexerFileSize = sec.Key("MAX_FILE_SIZE").MustInt64(1024 * 1024)
 	Indexer.IssueQueueType = sec.Key("ISSUE_INDEXER_QUEUE_TYPE").MustString(LevelQueueType)
@@ -58,3 +64,19 @@ func newIndexerService() {
 	Indexer.IssueQueueConnStr = sec.Key("ISSUE_INDEXER_QUEUE_CONN_STR").MustString(path.Join(AppDataPath, ""))
 	Indexer.IssueQueueBatchNumber = sec.Key("ISSUE_INDEXER_QUEUE_BATCH_NUMBER").MustInt(20)
 }
+
+func extensionsFromString(from string) map[string]bool {
+	extmap := make(map[string]bool)
+	for _, ext := range strings.Split(strings.ToLower(from), ",") {
+		ext = strings.TrimSpace(ext)
+		if ext == "." {
+			extmap[""] = true
+		} else if ext != "" && !strings.Contains(ext, ".") {
+			extmap[ext] = true
+		}
+	}
+	if len(extmap) == 0 {
+		return nil
+	}
+	return extmap
+}