Refactor git attributes (#29356)

modules/git/attribute.go

@@ -0,0 +1,35 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package git
+
+import (
+	"code.gitea.io/gitea/modules/optional"
+)
+
+const (
+	AttributeLinguistVendored      = "linguist-vendored"
+	AttributeLinguistGenerated     = "linguist-generated"
+	AttributeLinguistDocumentation = "linguist-documentation"
+	AttributeLinguistDetectable    = "linguist-detectable"
+	AttributeLinguistLanguage      = "linguist-language"
+	AttributeGitlabLanguage        = "gitlab-language"
+)
+
+// true if "set"/"true", false if "unset"/"false", none otherwise
+func AttributeToBool(attr map[string]string, name string) optional.Option[bool] {
+	switch attr[name] {
+	case "set", "true":
+		return optional.Some(true)
+	case "unset", "false":
+		return optional.Some(false)
+	}
+	return optional.None[bool]()
+}
+
+func AttributeToString(attr map[string]string, name string) optional.Option[string] {
+	if value, has := attr[name]; has && value != "unspecified" {
+		return optional.Some(value)
+	}
+	return optional.None[string]()
+}

modules/git/repo_attribute.go

@@ -11,7 +11,6 @@ import (
 	"os"
 
 	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/optional"
 )
 
 // CheckAttributeOpts represents the possible options to CheckAttribute
@@ -292,10 +291,17 @@ func (repo *Repository) CheckAttributeReader(commitID string) (*CheckAttributeRe
 	}
 
 	checker := &CheckAttributeReader{
-		Attributes: []string{"linguist-vendored", "linguist-generated", "linguist-language", "gitlab-language", "linguist-documentation", "linguist-detectable"},
+		Attributes: []string{
-		Repo:       repo,
+			AttributeLinguistVendored,
-		IndexFile:  indexFilename,
+			AttributeLinguistGenerated,
-		WorkTree:   worktree,
+			AttributeLinguistDocumentation,
+			AttributeLinguistDetectable,
+			AttributeLinguistLanguage,
+			AttributeGitlabLanguage,
+		},
+		Repo:      repo,
+		IndexFile: indexFilename,
+		WorkTree:  worktree,
 	}
 	ctx, cancel := context.WithCancel(repo.Ctx)
 	if err := checker.Init(ctx); err != nil {
@@ -317,23 +323,3 @@ func (repo *Repository) CheckAttributeReader(commitID string) (*CheckAttributeRe
 
 	return checker, deferable
 }
-
-// true if "set"/"true", false if "unset"/"false", none otherwise
-func attributeToBool(attr map[string]string, name string) optional.Option[bool] {
-	if value, has := attr[name]; has && value != "unspecified" {
-		switch value {
-		case "set", "true":
-			return optional.Some(true)
-		case "unset", "false":
-			return optional.Some(false)
-		}
-	}
-	return optional.None[bool]()
-}
-
-func attributeToString(attr map[string]string, name string) optional.Option[string] {
-	if value, has := attr[name]; has && value != "unspecified" {
-		return optional.Some(value)
-	}
-	return optional.None[string]()
-}

modules/git/repo_attribute_test.go

@@ -24,7 +24,7 @@ func Test_nulSeparatedAttributeWriter_ReadAttribute(t *testing.T) {
 	select {
 	case attr := <-wr.ReadAttribute():
 		assert.Equal(t, ".gitignore\"\n", attr.Filename)
-		assert.Equal(t, "linguist-vendored", attr.Attribute)
+		assert.Equal(t, AttributeLinguistVendored, attr.Attribute)
 		assert.Equal(t, "unspecified", attr.Value)
 	case <-time.After(100 * time.Millisecond):
 		assert.FailNow(t, "took too long to read an attribute from the list")
@@ -38,7 +38,7 @@ func Test_nulSeparatedAttributeWriter_ReadAttribute(t *testing.T) {
 	select {
 	case attr := <-wr.ReadAttribute():
 		assert.Equal(t, ".gitignore\"\n", attr.Filename)
-		assert.Equal(t, "linguist-vendored", attr.Attribute)
+		assert.Equal(t, AttributeLinguistVendored, attr.Attribute)
 		assert.Equal(t, "unspecified", attr.Value)
 	case <-time.After(100 * time.Millisecond):
 		assert.FailNow(t, "took too long to read an attribute from the list")
@@ -77,21 +77,21 @@ func Test_nulSeparatedAttributeWriter_ReadAttribute(t *testing.T) {
 	assert.NoError(t, err)
 	assert.EqualValues(t, attributeTriple{
 		Filename:  "shouldbe.vendor",
-		Attribute: "linguist-vendored",
+		Attribute: AttributeLinguistVendored,
 		Value:     "set",
 	}, attr)
 	attr = <-wr.ReadAttribute()
 	assert.NoError(t, err)
 	assert.EqualValues(t, attributeTriple{
 		Filename:  "shouldbe.vendor",
-		Attribute: "linguist-generated",
+		Attribute: AttributeLinguistGenerated,
 		Value:     "unspecified",
 	}, attr)
 	attr = <-wr.ReadAttribute()
 	assert.NoError(t, err)
 	assert.EqualValues(t, attributeTriple{
 		Filename:  "shouldbe.vendor",
-		Attribute: "linguist-language",
+		Attribute: AttributeLinguistLanguage,
 		Value:     "unspecified",
 	}, attr)
 }

modules/git/repo_language_stats.go

@@ -6,6 +6,8 @@ package git
 import (
 	"strings"
 	"unicode"
+
+	"code.gitea.io/gitea/modules/optional"
 )
 
 const (
@@ -46,3 +48,20 @@ func mergeLanguageStats(stats map[string]int64) map[string]int64 {
 	}
 	return res
 }
+
+func TryReadLanguageAttribute(attrs map[string]string) optional.Option[string] {
+	language := AttributeToString(attrs, AttributeLinguistLanguage)
+	if language.Value() == "" {
+		language = AttributeToString(attrs, AttributeGitlabLanguage)
+		if language.Has() {
+			raw := language.Value()
+			// gitlab-language may have additional parameters after the language
+			// ignore them and just use the main language
+			// https://docs.gitlab.com/ee/user/project/highlighting.html#override-syntax-highlighting-for-a-file-type
+			if idx := strings.IndexByte(raw, '?'); idx >= 0 {
+				language = optional.Some(raw[:idx])
+			}
+		}
+	}
+	return language
+}

modules/git/repo_language_stats_gogit.go

@@ -8,7 +8,6 @@ package git
 import (
 	"bytes"
 	"io"
-	"strings"
 
 	"code.gitea.io/gitea/modules/analyze"
 	"code.gitea.io/gitea/modules/optional"
@@ -66,36 +65,27 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 		if checker != nil {
 			attrs, err := checker.CheckPath(f.Name)
 			if err == nil {
-				isVendored = attributeToBool(attrs, "linguist-vendored")
+				isVendored = AttributeToBool(attrs, AttributeLinguistVendored)
 				if isVendored.ValueOrDefault(false) {
 					return nil
 				}
 
-				isGenerated = attributeToBool(attrs, "linguist-generated")
+				isGenerated = AttributeToBool(attrs, AttributeLinguistGenerated)
 				if isGenerated.ValueOrDefault(false) {
 					return nil
 				}
 
-				isDocumentation = attributeToBool(attrs, "linguist-documentation")
+				isDocumentation = AttributeToBool(attrs, AttributeLinguistDocumentation)
 				if isDocumentation.ValueOrDefault(false) {
 					return nil
 				}
 
-				isDetectable = attributeToBool(attrs, "linguist-detectable")
+				isDetectable = AttributeToBool(attrs, AttributeLinguistDetectable)
 				if !isDetectable.ValueOrDefault(true) {
 					return nil
 				}
 
-				hasLanguage := attributeToString(attrs, "linguist-language")
+				hasLanguage := TryReadLanguageAttribute(attrs)
-				if hasLanguage.Value() == "" {
-					hasLanguage = attributeToString(attrs, "gitlab-language")
-					if hasLanguage.Has() {
-						language := hasLanguage.Value()
-						if idx := strings.IndexByte(language, '?'); idx >= 0 {
-							hasLanguage = optional.Some(language[:idx])
-						}
-					}
-				}
 				if hasLanguage.Value() != "" {
 					language := hasLanguage.Value()
 

modules/git/repo_language_stats_nogogit.go

@@ -8,7 +8,6 @@ package git
 import (
 	"bytes"
 	"io"
-	"strings"
 
 	"code.gitea.io/gitea/modules/analyze"
 	"code.gitea.io/gitea/modules/log"
@@ -97,36 +96,27 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 		if checker != nil {
 			attrs, err := checker.CheckPath(f.Name())
 			if err == nil {
-				isVendored = attributeToBool(attrs, "linguist-vendored")
+				isVendored = AttributeToBool(attrs, AttributeLinguistVendored)
 				if isVendored.ValueOrDefault(false) {
 					continue
 				}
 
-				isGenerated = attributeToBool(attrs, "linguist-generated")
+				isGenerated = AttributeToBool(attrs, AttributeLinguistGenerated)
 				if isGenerated.ValueOrDefault(false) {
 					continue
 				}
 
-				isDocumentation = attributeToBool(attrs, "linguist-documentation")
+				isDocumentation = AttributeToBool(attrs, AttributeLinguistDocumentation)
 				if isDocumentation.ValueOrDefault(false) {
 					continue
 				}
 
-				isDetectable = attributeToBool(attrs, "linguist-detectable")
+				isDetectable = AttributeToBool(attrs, AttributeLinguistDetectable)
 				if !isDetectable.ValueOrDefault(true) {
 					continue
 				}
 
-				hasLanguage := attributeToString(attrs, "linguist-language")
+				hasLanguage := TryReadLanguageAttribute(attrs)
-				if hasLanguage.Value() == "" {
-					hasLanguage = attributeToString(attrs, "gitlab-language")
-					if hasLanguage.Has() {
-						language := hasLanguage.Value()
-						if idx := strings.IndexByte(language, '?'); idx >= 0 {
-							hasLanguage = optional.Some(language[:idx])
-						}
-					}
-				}
 				if hasLanguage.Value() != "" {
 					language := hasLanguage.Value()
 

routers/web/repo/view.go

@@ -635,11 +635,8 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry) {
 			defer deferable()
 			attrs, err := checker.CheckPath(ctx.Repo.TreePath)
 			if err == nil {
-				vendored, has := attrs["linguist-vendored"]
+				ctx.Data["IsVendored"] = git.AttributeToBool(attrs, git.AttributeLinguistVendored).Value()
-				ctx.Data["IsVendored"] = has && (vendored == "set" || vendored == "true")
+				ctx.Data["IsGenerated"] = git.AttributeToBool(attrs, git.AttributeLinguistGenerated).Value()
-
-				generated, has := attrs["linguist-generated"]
-				ctx.Data["IsGenerated"] = has && (generated == "set" || generated == "true")
 			}
 		}
 	}

services/gitdiff/gitdiff.go

@@ -29,6 +29,7 @@ import (
 	"code.gitea.io/gitea/modules/highlight"
 	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/optional"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/translation"
 
@@ -1181,41 +1182,30 @@ func GetDiff(ctx context.Context, gitRepo *git.Repository, opts *DiffOptions, fi
 
 	for _, diffFile := range diff.Files {
 
-		gotVendor := false
+		isVendored := optional.None[bool]()
-		gotGenerated := false
+		isGenerated := optional.None[bool]()
 		if checker != nil {
 			attrs, err := checker.CheckPath(diffFile.Name)
 			if err == nil {
-				if vendored, has := attrs["linguist-vendored"]; has {
+				isVendored = git.AttributeToBool(attrs, git.AttributeLinguistVendored)
-					if vendored == "set" || vendored == "true" {
+				isGenerated = git.AttributeToBool(attrs, git.AttributeLinguistGenerated)
-						diffFile.IsVendored = true
+
-						gotVendor = true
+				language := git.TryReadLanguageAttribute(attrs)
-					} else {
+				if language.Has() {
-						gotVendor = vendored == "false"
+					diffFile.Language = language.Value()
-					}
-				}
-				if generated, has := attrs["linguist-generated"]; has {
-					if generated == "set" || generated == "true" {
-						diffFile.IsGenerated = true
-						gotGenerated = true
-					} else {
-						gotGenerated = generated == "false"
-					}
-				}
-				if language, has := attrs["linguist-language"]; has && language != "unspecified" && language != "" {
-					diffFile.Language = language
-				} else if language, has := attrs["gitlab-language"]; has && language != "unspecified" && language != "" {
-					diffFile.Language = language
 				}
 			}
 		}
 
-		if !gotVendor {
+		if !isVendored.Has() {
-			diffFile.IsVendored = analyze.IsVendor(diffFile.Name)
+			isVendored = optional.Some(analyze.IsVendor(diffFile.Name))
 		}
-		if !gotGenerated {
+		diffFile.IsVendored = isVendored.Value()
-			diffFile.IsGenerated = analyze.IsGenerated(diffFile.Name)
+
+		if !isGenerated.Has() {
+			isGenerated = optional.Some(analyze.IsGenerated(diffFile.Name))
 		}
+		diffFile.IsGenerated = isGenerated.Value()
 
 		tailSection := diffFile.GetTailSection(gitRepo, opts.BeforeCommitID, opts.AfterCommitID)
 		if tailSection != nil {

services/repository/files/content.go

@@ -282,7 +282,7 @@ func TryGetContentLanguage(gitRepo *git.Repository, commitID, treePath string) (
 
 	filename2attribute2info, err := gitRepo.CheckAttribute(git.CheckAttributeOpts{
 		CachedOnly: true,
-		Attributes: []string{"linguist-language", "gitlab-language"},
+		Attributes: []string{git.AttributeLinguistLanguage, git.AttributeGitlabLanguage},
 		Filenames:  []string{treePath},
 		IndexFile:  indexFilename,
 		WorkTree:   worktree,
@@ -291,13 +291,7 @@ func TryGetContentLanguage(gitRepo *git.Repository, commitID, treePath string) (
 		return "", err
 	}
 
-	language := filename2attribute2info[treePath]["linguist-language"]
+	language := git.TryReadLanguageAttribute(filename2attribute2info[treePath])
-	if language == "" || language == "unspecified" {
-		language = filename2attribute2info[treePath]["gitlab-language"]
-	}
-	if language == "unspecified" {
-		language = ""
-	}
 
-	return language, nil
+	return language.Value(), nil
 }