1
1
mirror of https://github.com/go-gitea/gitea synced 2024-12-24 17:44:27 +00:00

Add option to disable ambiguous unicode characters detection (#28454) (#28499)

Backport #28454 (the only conflict is caused by some comments)

* Close #24483
* Close #28123
* Close #23682
* Close #23149
This commit is contained in:
wxiaoguang 2023-12-18 12:20:37 +08:00 committed by GitHub
parent 8ee1ed877b
commit a3f403f438
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 112 additions and 148 deletions

View File

@ -1210,6 +1210,9 @@ LEVEL = Info
;; Max size of files to be displayed (default is 8MiB) ;; Max size of files to be displayed (default is 8MiB)
;MAX_DISPLAY_FILE_SIZE = 8388608 ;MAX_DISPLAY_FILE_SIZE = 8388608
;; ;;
;; Detect ambiguous unicode characters in file contents and show warnings on the UI
;AMBIGUOUS_UNICODE_DETECTION = true
;;
;; Whether the email of the user should be shown in the Explore Users page ;; Whether the email of the user should be shown in the Explore Users page
;SHOW_USER_EMAIL = true ;SHOW_USER_EMAIL = true
;; ;;

View File

@ -220,6 +220,7 @@ The following configuration set `Content-Type: application/vnd.android.package-a
- `THEMES`: **auto,gitea,arc-green**: All available themes. Allow users select personalized themes. - `THEMES`: **auto,gitea,arc-green**: All available themes. Allow users select personalized themes.
regardless of the value of `DEFAULT_THEME`. regardless of the value of `DEFAULT_THEME`.
- `MAX_DISPLAY_FILE_SIZE`: **8388608**: Max size of files to be displayed (default is 8MiB) - `MAX_DISPLAY_FILE_SIZE`: **8388608**: Max size of files to be displayed (default is 8MiB)
- `AMBIGUOUS_UNICODE_DETECTION`: **true**: Detect ambiguous unicode characters in file contents and show warnings on the UI
- `REACTIONS`: All available reactions users can choose on issues/prs and comments - `REACTIONS`: All available reactions users can choose on issues/prs and comments
Values can be emoji alias (:smile:) or a unicode emoji. Values can be emoji alias (:smile:) or a unicode emoji.
For custom reactions, add a tightly cropped square image to public/assets/img/emoji/reaction_name.png For custom reactions, add a tightly cropped square image to public/assets/img/emoji/reaction_name.png

View File

@ -8,11 +8,12 @@
package charset package charset
import ( import (
"bufio" "html/template"
"io" "io"
"strings" "strings"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/translation" "code.gitea.io/gitea/modules/translation"
) )
@ -20,20 +21,18 @@ import (
const RuneNBSP = 0xa0 const RuneNBSP = 0xa0
// EscapeControlHTML escapes the unicode control sequences in a provided html document // EscapeControlHTML escapes the unicode control sequences in a provided html document
func EscapeControlHTML(text string, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, output string) { func EscapeControlHTML(html template.HTML, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, output template.HTML) {
sb := &strings.Builder{} sb := &strings.Builder{}
outputStream := &HTMLStreamerWriter{Writer: sb} escaped, _ = EscapeControlReader(strings.NewReader(string(html)), sb, locale, allowed...) // err has been handled in EscapeControlReader
streamer := NewEscapeStreamer(locale, outputStream, allowed...).(*escapeStreamer) return escaped, template.HTML(sb.String())
if err := StreamHTML(strings.NewReader(text), streamer); err != nil {
streamer.escaped.HasError = true
log.Error("Error whilst escaping: %v", err)
}
return streamer.escaped, sb.String()
} }
// EscapeControlReaders escapes the unicode control sequences in a provided reader of HTML content and writer in a locale and returns the findings as an EscapeStatus and the escaped []byte // EscapeControlReader escapes the unicode control sequences in a provided reader of HTML content and writer in a locale and returns the findings as an EscapeStatus
func EscapeControlReader(reader io.Reader, writer io.Writer, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, err error) { func EscapeControlReader(reader io.Reader, writer io.Writer, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, err error) {
if !setting.UI.AmbiguousUnicodeDetection {
_, err = io.Copy(writer, reader)
return &EscapeStatus{}, err
}
outputStream := &HTMLStreamerWriter{Writer: writer} outputStream := &HTMLStreamerWriter{Writer: writer}
streamer := NewEscapeStreamer(locale, outputStream, allowed...).(*escapeStreamer) streamer := NewEscapeStreamer(locale, outputStream, allowed...).(*escapeStreamer)
@ -43,41 +42,3 @@ func EscapeControlReader(reader io.Reader, writer io.Writer, locale translation.
} }
return streamer.escaped, err return streamer.escaped, err
} }
// EscapeControlStringReader escapes the unicode control sequences in a provided reader of string content and writer in a locale and returns the findings as an EscapeStatus and the escaped []byte. HTML line breaks are not inserted after every newline by this method.
func EscapeControlStringReader(reader io.Reader, writer io.Writer, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, err error) {
bufRd := bufio.NewReader(reader)
outputStream := &HTMLStreamerWriter{Writer: writer}
streamer := NewEscapeStreamer(locale, outputStream, allowed...).(*escapeStreamer)
for {
line, rdErr := bufRd.ReadString('\n')
if len(line) > 0 {
if err := streamer.Text(line); err != nil {
streamer.escaped.HasError = true
log.Error("Error whilst escaping: %v", err)
return streamer.escaped, err
}
}
if rdErr != nil {
if rdErr != io.EOF {
err = rdErr
}
break
}
}
return streamer.escaped, err
}
// EscapeControlString escapes the unicode control sequences in a provided string and returns the findings as an EscapeStatus and the escaped string
func EscapeControlString(text string, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, output string) {
sb := &strings.Builder{}
outputStream := &HTMLStreamerWriter{Writer: sb}
streamer := NewEscapeStreamer(locale, outputStream, allowed...).(*escapeStreamer)
if err := streamer.Text(text); err != nil {
streamer.escaped.HasError = true
log.Error("Error whilst escaping: %v", err)
}
return streamer.escaped, sb.String()
}

View File

@ -64,7 +64,7 @@ func (e *escapeStreamer) Text(data string) error {
until, next = nextIdxs[0]+pos, nextIdxs[1]+pos until, next = nextIdxs[0]+pos, nextIdxs[1]+pos
} }
// from pos until until we know that the runes are not \r\t\n or even ' ' // from pos until we know that the runes are not \r\t\n or even ' '
runes := make([]rune, 0, next-until) runes := make([]rune, 0, next-until)
positions := make([]int, 0, next-until+1) positions := make([]int, 0, next-until+1)

View File

@ -4,11 +4,14 @@
package charset package charset
import ( import (
"reflect"
"strings" "strings"
"testing" "testing"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/test"
"code.gitea.io/gitea/modules/translation" "code.gitea.io/gitea/modules/translation"
"github.com/stretchr/testify/assert"
) )
type escapeControlTest struct { type escapeControlTest struct {
@ -132,22 +135,8 @@ then resh (ר), and finally heh (ה) (which should appear leftmost).`,
}, },
} }
func TestEscapeControlString(t *testing.T) {
for _, tt := range escapeControlTests {
t.Run(tt.name, func(t *testing.T) {
status, result := EscapeControlString(tt.text, &translation.MockLocale{})
if !reflect.DeepEqual(*status, tt.status) {
t.Errorf("EscapeControlString() status = %v, wanted= %v", status, tt.status)
}
if result != tt.result {
t.Errorf("EscapeControlString()\nresult= %v,\nwanted= %v", result, tt.result)
}
})
}
}
func TestEscapeControlReader(t *testing.T) { func TestEscapeControlReader(t *testing.T) {
// lets add some control characters to the tests // add some control characters to the tests
tests := make([]escapeControlTest, 0, len(escapeControlTests)*3) tests := make([]escapeControlTest, 0, len(escapeControlTests)*3)
copy(tests, escapeControlTests) copy(tests, escapeControlTests)
@ -169,29 +158,20 @@ func TestEscapeControlReader(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
input := strings.NewReader(tt.text)
output := &strings.Builder{} output := &strings.Builder{}
status, err := EscapeControlReader(input, output, &translation.MockLocale{}) status, err := EscapeControlReader(strings.NewReader(tt.text), output, &translation.MockLocale{})
result := output.String() assert.NoError(t, err)
if err != nil { assert.Equal(t, tt.status, *status)
t.Errorf("EscapeControlReader(): err = %v", err) assert.Equal(t, tt.result, output.String())
}
if !reflect.DeepEqual(*status, tt.status) {
t.Errorf("EscapeControlReader() status = %v, wanted= %v", status, tt.status)
}
if result != tt.result {
t.Errorf("EscapeControlReader()\nresult= %v,\nwanted= %v", result, tt.result)
}
}) })
} }
} }
func TestEscapeControlReader_panic(t *testing.T) { func TestSettingAmbiguousUnicodeDetection(t *testing.T) {
bs := make([]byte, 0, 20479) defer test.MockVariableValue(&setting.UI.AmbiguousUnicodeDetection, true)()
bs = append(bs, 'A') _, out := EscapeControlHTML("a test", &translation.MockLocale{})
for i := 0; i < 6826; i++ { assert.EqualValues(t, `a<span class="escaped-code-point" data-escaped="[U+00A0]"><span class="char"> </span></span>test`, out)
bs = append(bs, []byte("—")...) setting.UI.AmbiguousUnicodeDetection = false
} _, out = EscapeControlHTML("a test", &translation.MockLocale{})
_, _ = EscapeControlString(string(bs), &translation.MockLocale{}) assert.EqualValues(t, `a test`, out)
} }

View File

@ -14,7 +14,6 @@ import (
"os/exec" "os/exec"
"strings" "strings"
"time" "time"
"unsafe"
"code.gitea.io/gitea/modules/git/internal" //nolint:depguard // only this file can use the internal type CmdArg, other files and packages should use AddXxx functions "code.gitea.io/gitea/modules/git/internal" //nolint:depguard // only this file can use the internal type CmdArg, other files and packages should use AddXxx functions
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
@ -389,15 +388,11 @@ func (r *runStdError) IsExitCode(code int) bool {
return false return false
} }
func bytesToString(b []byte) string {
return *(*string)(unsafe.Pointer(&b)) // that's what Golang's strings.Builder.String() does (go/src/strings/builder.go)
}
// RunStdString runs the command with options and returns stdout/stderr as string. and store stderr to returned error (err combined with stderr). // RunStdString runs the command with options and returns stdout/stderr as string. and store stderr to returned error (err combined with stderr).
func (c *Command) RunStdString(opts *RunOpts) (stdout, stderr string, runErr RunStdError) { func (c *Command) RunStdString(opts *RunOpts) (stdout, stderr string, runErr RunStdError) {
stdoutBytes, stderrBytes, err := c.RunStdBytes(opts) stdoutBytes, stderrBytes, err := c.RunStdBytes(opts)
stdout = bytesToString(stdoutBytes) stdout = util.UnsafeBytesToString(stdoutBytes)
stderr = bytesToString(stderrBytes) stderr = util.UnsafeBytesToString(stderrBytes)
if err != nil { if err != nil {
return stdout, stderr, &runStdError{err: err, stderr: stderr} return stdout, stderr, &runStdError{err: err, stderr: stderr}
} }
@ -432,7 +427,7 @@ func (c *Command) RunStdBytes(opts *RunOpts) (stdout, stderr []byte, runErr RunS
err := c.Run(newOpts) err := c.Run(newOpts)
stderr = stderrBuf.Bytes() stderr = stderrBuf.Bytes()
if err != nil { if err != nil {
return nil, stderr, &runStdError{err: err, stderr: bytesToString(stderr)} return nil, stderr, &runStdError{err: err, stderr: util.UnsafeBytesToString(stderr)}
} }
// even if there is no err, there could still be some stderr output // even if there is no err, there could still be some stderr output
return stdoutBuf.Bytes(), stderr, nil return stdoutBuf.Bytes(), stderr, nil

View File

@ -9,6 +9,7 @@ import (
"bytes" "bytes"
"fmt" "fmt"
gohtml "html" gohtml "html"
"html/template"
"io" "io"
"path/filepath" "path/filepath"
"strings" "strings"
@ -55,7 +56,7 @@ func NewContext() {
} }
// Code returns a HTML version of code string with chroma syntax highlighting classes and the matched lexer name // Code returns a HTML version of code string with chroma syntax highlighting classes and the matched lexer name
func Code(fileName, language, code string) (string, string) { func Code(fileName, language, code string) (output template.HTML, lexerName string) {
NewContext() NewContext()
// diff view newline will be passed as empty, change to literal '\n' so it can be copied // diff view newline will be passed as empty, change to literal '\n' so it can be copied
@ -65,7 +66,7 @@ func Code(fileName, language, code string) (string, string) {
} }
if len(code) > sizeLimit { if len(code) > sizeLimit {
return code, "" return template.HTML(template.HTMLEscapeString(code)), ""
} }
var lexer chroma.Lexer var lexer chroma.Lexer
@ -102,13 +103,11 @@ func Code(fileName, language, code string) (string, string) {
cache.Add(fileName, lexer) cache.Add(fileName, lexer)
} }
lexerName := formatLexerName(lexer.Config().Name) return CodeFromLexer(lexer, code), formatLexerName(lexer.Config().Name)
return CodeFromLexer(lexer, code), lexerName
} }
// CodeFromLexer returns a HTML version of code string with chroma syntax highlighting classes // CodeFromLexer returns a HTML version of code string with chroma syntax highlighting classes
func CodeFromLexer(lexer chroma.Lexer, code string) string { func CodeFromLexer(lexer chroma.Lexer, code string) template.HTML {
formatter := html.New(html.WithClasses(true), formatter := html.New(html.WithClasses(true),
html.WithLineNumbers(false), html.WithLineNumbers(false),
html.PreventSurroundingPre(true), html.PreventSurroundingPre(true),
@ -120,23 +119,23 @@ func CodeFromLexer(lexer chroma.Lexer, code string) string {
iterator, err := lexer.Tokenise(nil, code) iterator, err := lexer.Tokenise(nil, code)
if err != nil { if err != nil {
log.Error("Can't tokenize code: %v", err) log.Error("Can't tokenize code: %v", err)
return code return template.HTML(template.HTMLEscapeString(code))
} }
// style not used for live site but need to pass something // style not used for live site but need to pass something
err = formatter.Format(htmlw, githubStyles, iterator) err = formatter.Format(htmlw, githubStyles, iterator)
if err != nil { if err != nil {
log.Error("Can't format code: %v", err) log.Error("Can't format code: %v", err)
return code return template.HTML(template.HTMLEscapeString(code))
} }
_ = htmlw.Flush() _ = htmlw.Flush()
// Chroma will add newlines for certain lexers in order to highlight them properly // Chroma will add newlines for certain lexers in order to highlight them properly
// Once highlighted, strip them here, so they don't cause copy/paste trouble in HTML output // Once highlighted, strip them here, so they don't cause copy/paste trouble in HTML output
return strings.TrimSuffix(htmlbuf.String(), "\n") return template.HTML(strings.TrimSuffix(htmlbuf.String(), "\n"))
} }
// File returns a slice of chroma syntax highlighted HTML lines of code and the matched lexer name // File returns a slice of chroma syntax highlighted HTML lines of code and the matched lexer name
func File(fileName, language string, code []byte) ([]string, string, error) { func File(fileName, language string, code []byte) ([]template.HTML, string, error) {
NewContext() NewContext()
if len(code) > sizeLimit { if len(code) > sizeLimit {
@ -183,14 +182,14 @@ func File(fileName, language string, code []byte) ([]string, string, error) {
tokensLines := chroma.SplitTokensIntoLines(iterator.Tokens()) tokensLines := chroma.SplitTokensIntoLines(iterator.Tokens())
htmlBuf := &bytes.Buffer{} htmlBuf := &bytes.Buffer{}
lines := make([]string, 0, len(tokensLines)) lines := make([]template.HTML, 0, len(tokensLines))
for _, tokens := range tokensLines { for _, tokens := range tokensLines {
iterator = chroma.Literator(tokens...) iterator = chroma.Literator(tokens...)
err = formatter.Format(htmlBuf, githubStyles, iterator) err = formatter.Format(htmlBuf, githubStyles, iterator)
if err != nil { if err != nil {
return nil, "", fmt.Errorf("can't format code: %w", err) return nil, "", fmt.Errorf("can't format code: %w", err)
} }
lines = append(lines, htmlBuf.String()) lines = append(lines, template.HTML(htmlBuf.String()))
htmlBuf.Reset() htmlBuf.Reset()
} }
@ -198,9 +197,9 @@ func File(fileName, language string, code []byte) ([]string, string, error) {
} }
// PlainText returns non-highlighted HTML for code // PlainText returns non-highlighted HTML for code
func PlainText(code []byte) []string { func PlainText(code []byte) []template.HTML {
r := bufio.NewReader(bytes.NewReader(code)) r := bufio.NewReader(bytes.NewReader(code))
m := make([]string, 0, bytes.Count(code, []byte{'\n'})+1) m := make([]template.HTML, 0, bytes.Count(code, []byte{'\n'})+1)
for { for {
content, err := r.ReadString('\n') content, err := r.ReadString('\n')
if err != nil && err != io.EOF { if err != nil && err != io.EOF {
@ -210,7 +209,7 @@ func PlainText(code []byte) []string {
if content == "" && err == io.EOF { if content == "" && err == io.EOF {
break break
} }
s := gohtml.EscapeString(content) s := template.HTML(gohtml.EscapeString(content))
m = append(m, s) m = append(m, s)
} }
return m return m

View File

@ -4,21 +4,36 @@
package highlight package highlight
import ( import (
"html/template"
"strings" "strings"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
func lines(s string) []string { func lines(s string) (out []template.HTML) {
return strings.Split(strings.ReplaceAll(strings.TrimSpace(s), `\n`, "\n"), "\n") // "" => [], "a" => ["a"], "a\n" => ["a\n"], "a\nb" => ["a\n", "b"] (each line always includes EOL "\n" if it exists)
out = make([]template.HTML, 0)
s = strings.ReplaceAll(strings.ReplaceAll(strings.TrimSpace(s), "\n", ""), `\n`, "\n")
for {
if p := strings.IndexByte(s, '\n'); p != -1 {
out = append(out, template.HTML(s[:p+1]))
s = s[p+1:]
} else {
break
}
}
if s != "" {
out = append(out, template.HTML(s))
}
return out
} }
func TestFile(t *testing.T) { func TestFile(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
code string code string
want []string want []template.HTML
lexerName string lexerName string
}{ }{
{ {
@ -99,10 +114,7 @@ c=2
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
out, lexerName, err := File(tt.name, "", []byte(tt.code)) out, lexerName, err := File(tt.name, "", []byte(tt.code))
assert.NoError(t, err) assert.NoError(t, err)
expected := strings.Join(tt.want, "\n") assert.EqualValues(t, tt.want, out)
actual := strings.Join(out, "\n")
assert.Equal(t, strings.Count(actual, "<span"), strings.Count(actual, "</span>"))
assert.EqualValues(t, expected, actual)
assert.Equal(t, tt.lexerName, lexerName) assert.Equal(t, tt.lexerName, lexerName)
}) })
} }
@ -112,7 +124,7 @@ func TestPlainText(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
code string code string
want []string want []template.HTML
}{ }{
{ {
name: "empty.py", name: "empty.py",
@ -165,9 +177,7 @@ c=2`),
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
out := PlainText([]byte(tt.code)) out := PlainText([]byte(tt.code))
expected := strings.Join(tt.want, "\n") assert.EqualValues(t, tt.want, out)
actual := strings.Join(out, "\n")
assert.EqualValues(t, expected, actual)
}) })
} }
} }

View File

@ -6,6 +6,7 @@ package code
import ( import (
"bytes" "bytes"
"context" "context"
"html/template"
"strings" "strings"
"code.gitea.io/gitea/modules/highlight" "code.gitea.io/gitea/modules/highlight"
@ -22,7 +23,7 @@ type Result struct {
Language string Language string
Color string Color string
LineNumbers []int LineNumbers []int
FormattedLines string FormattedLines template.HTML
} }
type SearchResultLanguages = internal.SearchResultLanguages type SearchResultLanguages = internal.SearchResultLanguages

View File

@ -87,7 +87,7 @@ func Render(ctx *markup.RenderContext, input io.Reader, output io.Writer) error
} }
lexer = chroma.Coalesce(lexer) lexer = chroma.Coalesce(lexer)
if _, err := w.WriteString(highlight.CodeFromLexer(lexer, source)); err != nil { if _, err := w.WriteString(string(highlight.CodeFromLexer(lexer, source))); err != nil {
return "" return ""
} }
} }

View File

@ -34,6 +34,8 @@ var UI = struct {
SearchRepoDescription bool SearchRepoDescription bool
OnlyShowRelevantRepos bool OnlyShowRelevantRepos bool
AmbiguousUnicodeDetection bool
Notification struct { Notification struct {
MinTimeout time.Duration MinTimeout time.Duration
TimeoutStep time.Duration TimeoutStep time.Duration
@ -81,6 +83,9 @@ var UI = struct {
Reactions: []string{`+1`, `-1`, `laugh`, `hooray`, `confused`, `heart`, `rocket`, `eyes`}, Reactions: []string{`+1`, `-1`, `laugh`, `hooray`, `confused`, `heart`, `rocket`, `eyes`},
CustomEmojis: []string{`git`, `gitea`, `codeberg`, `gitlab`, `github`, `gogs`}, CustomEmojis: []string{`git`, `gitea`, `codeberg`, `gitlab`, `github`, `gogs`},
CustomEmojisMap: map[string]string{"git": ":git:", "gitea": ":gitea:", "codeberg": ":codeberg:", "gitlab": ":gitlab:", "github": ":github:", "gogs": ":gogs:"}, CustomEmojisMap: map[string]string{"git": ":git:", "gitea": ":gitea:", "codeberg": ":codeberg:", "gitlab": ":gitlab:", "github": ":github:", "gogs": ":gogs:"},
AmbiguousUnicodeDetection: true,
Notification: struct { Notification: struct {
MinTimeout time.Duration MinTimeout time.Duration
TimeoutStep time.Duration TimeoutStep time.Duration

View File

@ -3,7 +3,7 @@
package util package util
import "github.com/yuin/goldmark/util" import "unsafe"
func isSnakeCaseUpper(c byte) bool { func isSnakeCaseUpper(c byte) bool {
return 'A' <= c && c <= 'Z' return 'A' <= c && c <= 'Z'
@ -83,5 +83,15 @@ func ToSnakeCase(input string) string {
} }
} }
} }
return util.BytesToReadOnlyString(res) return UnsafeBytesToString(res)
}
// UnsafeBytesToString uses Go's unsafe package to convert a byte slice to a string.
// TODO: replace all "goldmark/util.BytesToReadOnlyString" with this official approach
func UnsafeBytesToString(b []byte) string {
return unsafe.String(unsafe.SliceData(b), len(b))
}
func UnsafeStringToBytes(s string) []byte {
return unsafe.Slice(unsafe.StringData(s), len(s))
} }

View File

@ -310,8 +310,7 @@ func renderBlame(ctx *context.Context, blameParts []git.BlamePart, commitNames m
lexerName = lexerNameForLine lexerName = lexerNameForLine
} }
br.EscapeStatus, line = charset.EscapeControlHTML(line, ctx.Locale) br.EscapeStatus, br.Code = charset.EscapeControlHTML(line, ctx.Locale)
br.Code = gotemplate.HTML(line)
rows = append(rows, br) rows = append(rows, br)
escapeStatus = escapeStatus.Or(br.EscapeStatus) escapeStatus = escapeStatus.Or(br.EscapeStatus)
} }

View File

@ -9,6 +9,7 @@ import (
gocontext "context" gocontext "context"
"encoding/base64" "encoding/base64"
"fmt" "fmt"
"html/template"
"image" "image"
"io" "io"
"net/http" "net/http"
@ -317,19 +318,18 @@ func renderReadmeFile(ctx *context.Context, subfolder string, readmeFile *git.Tr
}, rd) }, rd)
if err != nil { if err != nil {
log.Error("Render failed for %s in %-v: %v Falling back to rendering source", readmeFile.Name(), ctx.Repo.Repository, err) log.Error("Render failed for %s in %-v: %v Falling back to rendering source", readmeFile.Name(), ctx.Repo.Repository, err)
buf := &bytes.Buffer{} delete(ctx.Data, "IsMarkup")
ctx.Data["EscapeStatus"], _ = charset.EscapeControlStringReader(rd, buf, ctx.Locale)
ctx.Data["FileContent"] = buf.String()
}
} else {
ctx.Data["IsPlainText"] = true
buf := &bytes.Buffer{}
ctx.Data["EscapeStatus"], err = charset.EscapeControlStringReader(rd, buf, ctx.Locale)
if err != nil {
log.Error("Read failed: %v", err)
} }
}
ctx.Data["FileContent"] = buf.String() if ctx.Data["IsMarkup"] != true {
ctx.Data["IsPlainText"] = true
content, err := io.ReadAll(rd)
if err != nil {
log.Error("Read readme content failed: %v", err)
}
contentEscaped := template.HTMLEscapeString(util.UnsafeBytesToString(content))
ctx.Data["EscapeStatus"], ctx.Data["FileContent"] = charset.EscapeControlHTML(template.HTML(contentEscaped), ctx.Locale)
} }
} }
@ -611,7 +611,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
} }
} }
func markupRender(ctx *context.Context, renderCtx *markup.RenderContext, input io.Reader) (escaped *charset.EscapeStatus, output string, err error) { func markupRender(ctx *context.Context, renderCtx *markup.RenderContext, input io.Reader) (escaped *charset.EscapeStatus, output template.HTML, err error) {
markupRd, markupWr := io.Pipe() markupRd, markupWr := io.Pipe()
defer markupWr.Close() defer markupWr.Close()
done := make(chan struct{}) done := make(chan struct{})
@ -619,7 +619,7 @@ func markupRender(ctx *context.Context, renderCtx *markup.RenderContext, input i
sb := &strings.Builder{} sb := &strings.Builder{}
// We allow NBSP here this is rendered // We allow NBSP here this is rendered
escaped, _ = charset.EscapeControlReader(markupRd, sb, ctx.Locale, charset.RuneNBSP) escaped, _ = charset.EscapeControlReader(markupRd, sb, ctx.Locale, charset.RuneNBSP)
output = sb.String() output = template.HTML(sb.String())
close(done) close(done)
}() }()
err = markup.Render(renderCtx, input, markupWr) err = markup.Render(renderCtx, input, markupWr)

View File

@ -285,15 +285,15 @@ type DiffInline struct {
// DiffInlineWithUnicodeEscape makes a DiffInline with hidden unicode characters escaped // DiffInlineWithUnicodeEscape makes a DiffInline with hidden unicode characters escaped
func DiffInlineWithUnicodeEscape(s template.HTML, locale translation.Locale) DiffInline { func DiffInlineWithUnicodeEscape(s template.HTML, locale translation.Locale) DiffInline {
status, content := charset.EscapeControlHTML(string(s), locale) status, content := charset.EscapeControlHTML(s, locale)
return DiffInline{EscapeStatus: status, Content: template.HTML(content)} return DiffInline{EscapeStatus: status, Content: content}
} }
// DiffInlineWithHighlightCode makes a DiffInline with code highlight and hidden unicode characters escaped // DiffInlineWithHighlightCode makes a DiffInline with code highlight and hidden unicode characters escaped
func DiffInlineWithHighlightCode(fileName, language, code string, locale translation.Locale) DiffInline { func DiffInlineWithHighlightCode(fileName, language, code string, locale translation.Locale) DiffInline {
highlighted, _ := highlight.Code(fileName, language, code) highlighted, _ := highlight.Code(fileName, language, code)
status, content := charset.EscapeControlHTML(highlighted, locale) status, content := charset.EscapeControlHTML(highlighted, locale)
return DiffInline{EscapeStatus: status, Content: template.HTML(content)} return DiffInline{EscapeStatus: status, Content: content}
} }
// GetComputedInlineDiffFor computes inline diff for the given line. // GetComputedInlineDiffFor computes inline diff for the given line.

View File

@ -93,10 +93,10 @@ func (hcd *highlightCodeDiff) diffWithHighlight(filename, language, codeA, codeB
highlightCodeA, _ := highlight.Code(filename, language, codeA) highlightCodeA, _ := highlight.Code(filename, language, codeA)
highlightCodeB, _ := highlight.Code(filename, language, codeB) highlightCodeB, _ := highlight.Code(filename, language, codeB)
highlightCodeA = hcd.convertToPlaceholders(highlightCodeA) convertedCodeA := hcd.convertToPlaceholders(string(highlightCodeA))
highlightCodeB = hcd.convertToPlaceholders(highlightCodeB) convertedCodeB := hcd.convertToPlaceholders(string(highlightCodeB))
diffs := diffMatchPatch.DiffMain(highlightCodeA, highlightCodeB, true) diffs := diffMatchPatch.DiffMain(convertedCodeA, convertedCodeB, true)
diffs = diffMatchPatch.DiffCleanupEfficiency(diffs) diffs = diffMatchPatch.DiffCleanupEfficiency(diffs)
for i := range diffs { for i := range diffs {

View File

@ -69,9 +69,9 @@
{{end}} {{end}}
<div class="file-view{{if .IsMarkup}} markup {{.MarkupType}}{{else if .IsPlainText}} plain-text{{else if .IsTextSource}} code-view{{end}}"> <div class="file-view{{if .IsMarkup}} markup {{.MarkupType}}{{else if .IsPlainText}} plain-text{{else if .IsTextSource}} code-view{{end}}">
{{if .IsMarkup}} {{if .IsMarkup}}
{{if .FileContent}}{{.FileContent | Safe}}{{end}} {{if .FileContent}}{{.FileContent}}{{end}}
{{else if .IsPlainText}} {{else if .IsPlainText}}
<pre>{{if .FileContent}}{{.FileContent | Safe}}{{end}}</pre> <pre>{{if .FileContent}}{{.FileContent}}{{end}}</pre>
{{else if not .IsTextSource}} {{else if not .IsTextSource}}
<div class="view-raw"> <div class="view-raw">
{{if .IsImageFile}} {{if .IsImageFile}}
@ -109,7 +109,7 @@
{{if $.EscapeStatus.Escaped}} {{if $.EscapeStatus.Escaped}}
<td class="lines-escape">{{if (index $.LineEscapeStatus $idx).Escaped}}<button class="toggle-escape-button btn interact-bg" title="{{if (index $.LineEscapeStatus $idx).HasInvisible}}{{ctx.Locale.Tr "repo.invisible_runes_line"}} {{end}}{{if (index $.LineEscapeStatus $idx).HasAmbiguous}}{{ctx.Locale.Tr "repo.ambiguous_runes_line"}}{{end}}"></button>{{end}}</td> <td class="lines-escape">{{if (index $.LineEscapeStatus $idx).Escaped}}<button class="toggle-escape-button btn interact-bg" title="{{if (index $.LineEscapeStatus $idx).HasInvisible}}{{ctx.Locale.Tr "repo.invisible_runes_line"}} {{end}}{{if (index $.LineEscapeStatus $idx).HasAmbiguous}}{{ctx.Locale.Tr "repo.ambiguous_runes_line"}}{{end}}"></button>{{end}}</td>
{{end}} {{end}}
<td rel="L{{$line}}" class="lines-code chroma"><code class="code-inner">{{$code | Safe}}</code></td> <td rel="L{{$line}}" class="lines-code chroma"><code class="code-inner">{{$code}}</code></td>
</tr> </tr>
{{end}} {{end}}
</tbody> </tbody>