From 176962c03e3d82805e87e452cc2af047e5b3d9fc Mon Sep 17 00:00:00 2001 From: Kerwin Bryant Date: Mon, 30 Jun 2025 16:12:25 +0800 Subject: [PATCH] Add support for 3D/CAD file formats preview (#34794) Fix #34775 --------- Co-authored-by: wxiaoguang --- modules/git/blob.go | 21 +- modules/markup/console/console.go | 38 +- modules/markup/console/console_test.go | 32 +- modules/markup/renderer.go | 12 +- modules/typesniffer/typesniffer.go | 63 ++-- modules/typesniffer/typesniffer_test.go | 16 +- package-lock.json | 51 +++ package.json | 1 + routers/web/repo/editor.go | 4 +- routers/web/repo/setting/lfs.go | 4 +- routers/web/repo/view.go | 53 +-- routers/web/repo/view_file.go | 350 +++++++++---------- routers/web/repo/view_home.go | 2 +- routers/web/repo/view_readme.go | 11 +- templates/repo/blame.tmpl | 2 + templates/repo/editor/common_breadcrumb.tmpl | 2 +- templates/repo/settings/lfs_file.tmpl | 2 - templates/repo/view_file.tmpl | 88 ++--- tests/integration/lfs_view_test.go | 9 +- web_src/css/modules/animations.css | 3 +- web_src/css/repo.css | 40 --- web_src/css/repo/file-view.css | 30 ++ web_src/js/features/copycontent.ts | 14 +- web_src/js/features/file-view.ts | 76 ++++ web_src/js/index.ts | 5 +- web_src/js/render/pdf.ts | 17 - web_src/js/render/plugin.ts | 10 + web_src/js/render/plugins/3d-viewer.ts | 60 ++++ web_src/js/render/plugins/pdf-viewer.ts | 20 ++ 29 files changed, 627 insertions(+), 409 deletions(-) create mode 100644 web_src/js/features/file-view.ts delete mode 100644 web_src/js/render/pdf.ts create mode 100644 web_src/js/render/plugin.ts create mode 100644 web_src/js/render/plugins/3d-viewer.ts create mode 100644 web_src/js/render/plugins/pdf-viewer.ts diff --git a/modules/git/blob.go b/modules/git/blob.go index ab9deec8d1..40d8f44e79 100644 --- a/modules/git/blob.go +++ b/modules/git/blob.go @@ -22,17 +22,22 @@ func (b *Blob) Name() string { return b.name } -// GetBlobContent Gets the limited content of the blob as raw text -func (b *Blob) GetBlobContent(limit int64) (string, error) { +// GetBlobBytes Gets the limited content of the blob +func (b *Blob) GetBlobBytes(limit int64) ([]byte, error) { if limit <= 0 { - return "", nil + return nil, nil } dataRc, err := b.DataAsync() if err != nil { - return "", err + return nil, err } defer dataRc.Close() - buf, err := util.ReadWithLimit(dataRc, int(limit)) + return util.ReadWithLimit(dataRc, int(limit)) +} + +// GetBlobContent Gets the limited content of the blob as raw text +func (b *Blob) GetBlobContent(limit int64) (string, error) { + buf, err := b.GetBlobBytes(limit) return string(buf), err } @@ -99,11 +104,9 @@ loop: // GuessContentType guesses the content type of the blob. func (b *Blob) GuessContentType() (typesniffer.SniffedType, error) { - r, err := b.DataAsync() + buf, err := b.GetBlobBytes(typesniffer.SniffContentSize) if err != nil { return typesniffer.SniffedType{}, err } - defer r.Close() - - return typesniffer.DetectContentTypeFromReader(r) + return typesniffer.DetectContentType(buf), nil } diff --git a/modules/markup/console/console.go b/modules/markup/console/console.go index 06f3acfa68..492579b0a5 100644 --- a/modules/markup/console/console.go +++ b/modules/markup/console/console.go @@ -6,13 +6,14 @@ package console import ( "bytes" "io" - "path" + "unicode/utf8" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/typesniffer" + "code.gitea.io/gitea/modules/util" trend "github.com/buildkite/terminal-to-html/v3" - "github.com/go-enry/go-enry/v2" ) func init() { @@ -22,6 +23,8 @@ func init() { // Renderer implements markup.Renderer type Renderer struct{} +var _ markup.RendererContentDetector = (*Renderer)(nil) + // Name implements markup.Renderer func (Renderer) Name() string { return "console" @@ -40,15 +43,36 @@ func (Renderer) SanitizerRules() []setting.MarkupSanitizerRule { } // CanRender implements markup.RendererContentDetector -func (Renderer) CanRender(filename string, input io.Reader) bool { - buf, err := io.ReadAll(input) - if err != nil { +func (Renderer) CanRender(filename string, sniffedType typesniffer.SniffedType, prefetchBuf []byte) bool { + if !sniffedType.IsTextPlain() { return false } - if enry.GetLanguage(path.Base(filename), buf) != enry.OtherLanguage { + + s := util.UnsafeBytesToString(prefetchBuf) + rs := []rune(s) + cnt := 0 + firstErrPos := -1 + isCtrlSep := func(p int) bool { + return p < len(rs) && (rs[p] == ';' || rs[p] == 'm') + } + for i, c := range rs { + if c == 0 { + return false + } + if c == '\x1b' { + match := i+1 < len(rs) && rs[i+1] == '[' + if match && (isCtrlSep(i+2) || isCtrlSep(i+3) || isCtrlSep(i+4) || isCtrlSep(i+5)) { + cnt++ + } + } + if c == utf8.RuneError && firstErrPos == -1 { + firstErrPos = i + } + } + if firstErrPos != -1 && firstErrPos != len(rs)-1 { return false } - return bytes.ContainsRune(buf, '\x1b') + return cnt >= 2 // only render it as console output if there are at least two escape sequences } // Render renders terminal colors to HTML with all specific handling stuff. diff --git a/modules/markup/console/console_test.go b/modules/markup/console/console_test.go index 539f965ea1..d1192bebc2 100644 --- a/modules/markup/console/console_test.go +++ b/modules/markup/console/console_test.go @@ -8,23 +8,39 @@ import ( "testing" "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/typesniffer" "github.com/stretchr/testify/assert" ) func TestRenderConsole(t *testing.T) { - var render Renderer - kases := map[string]string{ - "\x1b[37m\x1b[40mnpm\x1b[0m \x1b[0m\x1b[32minfo\x1b[0m \x1b[0m\x1b[35mit worked if it ends with\x1b[0m ok": "npm info it worked if it ends with ok", + cases := []struct { + input string + expected string + }{ + {"\x1b[37m\x1b[40mnpm\x1b[0m \x1b[0m\x1b[32minfo\x1b[0m \x1b[0m\x1b[35mit worked if it ends with\x1b[0m ok", `npm info it worked if it ends with ok`}, + {"\x1b[1;2m \x1b[123m 啊", ` 啊`}, + {"\x1b[1;2m \x1b[123m \xef", ` �`}, + {"\x1b[1;2m \x1b[123m \xef \xef", ``}, + {"\x1b[12", ``}, + {"\x1b[1", ``}, + {"\x1b[FOO\x1b[", ``}, + {"\x1b[mFOO\x1b[m", `FOO`}, } - for k, v := range kases { + var render Renderer + for i, c := range cases { var buf strings.Builder - canRender := render.CanRender("test", strings.NewReader(k)) - assert.True(t, canRender) + st := typesniffer.DetectContentType([]byte(c.input)) + canRender := render.CanRender("test", st, []byte(c.input)) + if c.expected == "" { + assert.False(t, canRender, "case %d: expected not to render", i) + continue + } - err := render.Render(markup.NewRenderContext(t.Context()), strings.NewReader(k), &buf) + assert.True(t, canRender) + err := render.Render(markup.NewRenderContext(t.Context()), strings.NewReader(c.input), &buf) assert.NoError(t, err) - assert.Equal(t, v, buf.String()) + assert.Equal(t, c.expected, buf.String()) } } diff --git a/modules/markup/renderer.go b/modules/markup/renderer.go index 35f90eb46c..b6e9c348b7 100644 --- a/modules/markup/renderer.go +++ b/modules/markup/renderer.go @@ -4,12 +4,12 @@ package markup import ( - "bytes" "io" "path" "strings" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/typesniffer" ) // Renderer defines an interface for rendering markup file to HTML @@ -37,7 +37,7 @@ type ExternalRenderer interface { // RendererContentDetector detects if the content can be rendered // by specified renderer type RendererContentDetector interface { - CanRender(filename string, input io.Reader) bool + CanRender(filename string, sniffedType typesniffer.SniffedType, prefetchBuf []byte) bool } var ( @@ -60,13 +60,9 @@ func GetRendererByFileName(filename string) Renderer { } // DetectRendererType detects the markup type of the content -func DetectRendererType(filename string, input io.Reader) string { - buf, err := io.ReadAll(input) - if err != nil { - return "" - } +func DetectRendererType(filename string, sniffedType typesniffer.SniffedType, prefetchBuf []byte) string { for _, renderer := range renderers { - if detector, ok := renderer.(RendererContentDetector); ok && detector.CanRender(filename, bytes.NewReader(buf)) { + if detector, ok := renderer.(RendererContentDetector); ok && detector.CanRender(filename, sniffedType, prefetchBuf) { return renderer.Name() } } diff --git a/modules/typesniffer/typesniffer.go b/modules/typesniffer/typesniffer.go index 8cb3d278ce..2e8d9c4a1e 100644 --- a/modules/typesniffer/typesniffer.go +++ b/modules/typesniffer/typesniffer.go @@ -6,18 +6,14 @@ package typesniffer import ( "bytes" "encoding/binary" - "fmt" - "io" "net/http" "regexp" "slices" "strings" - - "code.gitea.io/gitea/modules/util" + "sync" ) -// Use at most this many bytes to determine Content Type. -const sniffLen = 1024 +const SniffContentSize = 1024 const ( MimeTypeImageSvg = "image/svg+xml" @@ -26,22 +22,30 @@ const ( MimeTypeApplicationOctetStream = "application/octet-stream" ) -var ( - svgComment = regexp.MustCompile(`(?s)`) - svgTagRegex = regexp.MustCompile(`(?si)\A\s*(?:(|>))\s*)*\s*(?:(|>))\s*)*`) + ret.svgTagRegex = regexp.MustCompile(`(?si)\A\s*(?:(|>))\s*)*\s*(?:(|>))\s*)* sniffLen { - data = data[:sniffLen] + if len(data) > SniffContentSize { + data = data[:SniffContentSize] } + vars := globalVars() // SVG is unsupported by http.DetectContentType, https://github.com/golang/go/issues/15888 detectByHTML := strings.Contains(ct, "text/plain") || strings.Contains(ct, "text/html") detectByXML := strings.Contains(ct, "text/xml") if detectByHTML || detectByXML { - dataProcessed := svgComment.ReplaceAll(data, nil) + dataProcessed := vars.svgComment.ReplaceAll(data, nil) dataProcessed = bytes.TrimSpace(dataProcessed) - if detectByHTML && svgTagRegex.Match(dataProcessed) || - detectByXML && svgTagInXMLRegex.Match(dataProcessed) { + if detectByHTML && vars.svgTagRegex.Match(dataProcessed) || + detectByXML && vars.svgTagInXMLRegex.Match(dataProcessed) { ct = MimeTypeImageSvg } } if strings.HasPrefix(ct, "audio/") && bytes.HasPrefix(data, []byte("ID3")) { // The MP3 detection is quite inaccurate, any content with "ID3" prefix will result in "audio/mpeg". - // So remove the "ID3" prefix and detect again, if result is text, then it must be text content. + // So remove the "ID3" prefix and detect again, then if the result is "text", it must be text content. // This works especially because audio files contain many unprintable/invalid characters like `0x00` ct2 := http.DetectContentType(data[3:]) if strings.HasPrefix(ct2, "text/") { @@ -155,15 +160,3 @@ func DetectContentType(data []byte) SniffedType { } return SniffedType{ct} } - -// DetectContentTypeFromReader guesses the content type contained in the reader. -func DetectContentTypeFromReader(r io.Reader) (SniffedType, error) { - buf := make([]byte, sniffLen) - n, err := util.ReadAtMost(r, buf) - if err != nil { - return SniffedType{}, fmt.Errorf("DetectContentTypeFromReader io error: %w", err) - } - buf = buf[:n] - - return DetectContentType(buf), nil -} diff --git a/modules/typesniffer/typesniffer_test.go b/modules/typesniffer/typesniffer_test.go index 3e5db3308b..a0c824b912 100644 --- a/modules/typesniffer/typesniffer_test.go +++ b/modules/typesniffer/typesniffer_test.go @@ -4,7 +4,6 @@ package typesniffer import ( - "bytes" "encoding/base64" "encoding/hex" "strings" @@ -17,7 +16,7 @@ func TestDetectContentTypeLongerThanSniffLen(t *testing.T) { // Pre-condition: Shorter than sniffLen detects SVG. assert.Equal(t, "image/svg+xml", DetectContentType([]byte(``)).contentType) // Longer than sniffLen detects something else. - assert.NotEqual(t, "image/svg+xml", DetectContentType([]byte(``)).contentType) + assert.NotEqual(t, "image/svg+xml", DetectContentType([]byte(``)).contentType) } func TestIsTextFile(t *testing.T) { @@ -116,22 +115,13 @@ func TestIsAudio(t *testing.T) { assert.True(t, DetectContentType([]byte("ID3Toy\n====\t* hi 🌞, ..."+"🌛"[0:2])).IsText()) // test ID3 tag with incomplete UTF8 char } -func TestDetectContentTypeFromReader(t *testing.T) { - mp3, _ := base64.StdEncoding.DecodeString("SUQzBAAAAAABAFRYWFgAAAASAAADbWFqb3JfYnJhbmQAbXA0MgBUWFhYAAAAEQAAA21pbm9yX3Zl") - st, err := DetectContentTypeFromReader(bytes.NewReader(mp3)) - assert.NoError(t, err) - assert.True(t, st.IsAudio()) -} - func TestDetectContentTypeOgg(t *testing.T) { oggAudio, _ := hex.DecodeString("4f67675300020000000000000000352f0000000000007dc39163011e01766f72626973000000000244ac0000000000000071020000000000b8014f6767530000") - st, err := DetectContentTypeFromReader(bytes.NewReader(oggAudio)) - assert.NoError(t, err) + st := DetectContentType(oggAudio) assert.True(t, st.IsAudio()) oggVideo, _ := hex.DecodeString("4f676753000200000000000000007d9747ef000000009b59daf3012a807468656f7261030201001e00110001e000010e00020000001e00000001000001000001") - st, err = DetectContentTypeFromReader(bytes.NewReader(oggVideo)) - assert.NoError(t, err) + st = DetectContentType(oggVideo) assert.True(t, st.IsVideo()) } diff --git a/package-lock.json b/package-lock.json index 6356a04365..132efb8635 100644 --- a/package-lock.json +++ b/package-lock.json @@ -40,6 +40,7 @@ "minimatch": "10.0.2", "monaco-editor": "0.52.2", "monaco-editor-webpack-plugin": "7.1.0", + "online-3d-viewer": "0.16.0", "pdfobject": "2.3.1", "perfect-debounce": "1.0.0", "postcss": "8.5.5", @@ -2026,6 +2027,16 @@ "vue": "^3.2.29" } }, + "node_modules/@simonwep/pickr": { + "version": "1.9.0", + "resolved": "https://registry.npmmirror.com/@simonwep/pickr/-/pickr-1.9.0.tgz", + "integrity": "sha512-oEYvv15PyfZzjoAzvXYt3UyNGwzsrpFxLaZKzkOSd0WYBVwLd19iJerePDONxC1iF6+DpcswPdLIM2KzCJuYFg==", + "license": "MIT", + "dependencies": { + "core-js": "3.32.2", + "nanopop": "2.3.0" + } + }, "node_modules/@stoplight/better-ajv-errors": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/@stoplight/better-ajv-errors/-/better-ajv-errors-1.0.3.tgz", @@ -5337,6 +5348,17 @@ "integrity": "sha512-1NB+BKqhtNipMsov4xI/NnhCKp9XG9NamYp5PVm9klAT0fsrNPjaFICsCFhNhwZJKNh7zB/3q8qXz0E9oaMNtQ==", "license": "MIT" }, + "node_modules/core-js": { + "version": "3.32.2", + "resolved": "https://registry.npmmirror.com/core-js/-/core-js-3.32.2.tgz", + "integrity": "sha512-pxXSw1mYZPDGvTQqEc5vgIb83jGQKFGYWY76z4a7weZXUolw3G+OvpZqSRcfYOoOVUQJYEPsWeQK8pKEnUtWxQ==", + "hasInstallScript": true, + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/core-js" + } + }, "node_modules/core-js-compat": { "version": "3.43.0", "resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.43.0.tgz", @@ -7721,6 +7743,12 @@ } } }, + "node_modules/fflate": { + "version": "0.8.2", + "resolved": "https://registry.npmmirror.com/fflate/-/fflate-0.8.2.tgz", + "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==", + "license": "MIT" + }, "node_modules/file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -10285,6 +10313,12 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/nanopop": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/nanopop/-/nanopop-2.3.0.tgz", + "integrity": "sha512-fzN+T2K7/Ah25XU02MJkPZ5q4Tj5FpjmIYq4rvoHX4yb16HzFdCO6JxFFn5Y/oBhQ8no8fUZavnyIv9/+xkBBw==", + "license": "MIT" + }, "node_modules/napi-postinstall": { "version": "0.2.4", "resolved": "https://registry.npmjs.org/napi-postinstall/-/napi-postinstall-0.2.4.tgz", @@ -10525,6 +10559,17 @@ "wrappy": "1" } }, + "node_modules/online-3d-viewer": { + "version": "0.16.0", + "resolved": "https://registry.npmmirror.com/online-3d-viewer/-/online-3d-viewer-0.16.0.tgz", + "integrity": "sha512-Mcmo41TM3K+svlMDRH8ySKSY2e8s7Sssdb5U9LV3gkFKVWGGuS304Vk5gqxopAJbE72DpsC67Ve3YNtcAuROwQ==", + "license": "MIT", + "dependencies": { + "@simonwep/pickr": "1.9.0", + "fflate": "0.8.2", + "three": "0.176.0" + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -13193,6 +13238,12 @@ "node": ">=0.8" } }, + "node_modules/three": { + "version": "0.176.0", + "resolved": "https://registry.npmmirror.com/three/-/three-0.176.0.tgz", + "integrity": "sha512-PWRKYWQo23ojf9oZSlRGH8K09q7nRSWx6LY/HF/UUrMdYgN9i1e2OwJYHoQjwc6HF/4lvvYLC5YC1X8UJL2ZpA==", + "license": "MIT" + }, "node_modules/throttle-debounce": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/throttle-debounce/-/throttle-debounce-5.0.2.tgz", diff --git a/package.json b/package.json index 5595e55fa5..c8a48bb5d9 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "minimatch": "10.0.2", "monaco-editor": "0.52.2", "monaco-editor-webpack-plugin": "7.1.0", + "online-3d-viewer": "0.16.0", "pdfobject": "2.3.1", "perfect-debounce": "1.0.0", "postcss": "8.5.5", diff --git a/routers/web/repo/editor.go b/routers/web/repo/editor.go index 9aee3d6a86..2a5ac10282 100644 --- a/routers/web/repo/editor.go +++ b/routers/web/repo/editor.go @@ -244,7 +244,7 @@ func editFileOpenExisting(ctx *context.Context) (prefetch []byte, dataRc io.Read return nil, nil, nil } - if fInfo.isLFSFile { + if fInfo.isLFSFile() { lfsLock, err := git_model.GetTreePathLock(ctx, ctx.Repo.Repository.ID, ctx.Repo.TreePath) if err != nil { _ = dataRc.Close() @@ -298,7 +298,7 @@ func EditFile(ctx *context.Context) { ctx.Data["FileSize"] = fInfo.fileSize // Only some file types are editable online as text. - if fInfo.isLFSFile { + if fInfo.isLFSFile() { ctx.Data["NotEditableReason"] = ctx.Tr("repo.editor.cannot_edit_lfs_files") } else if !fInfo.st.IsRepresentableAsText() { ctx.Data["NotEditableReason"] = ctx.Tr("repo.editor.cannot_edit_non_text_files") diff --git a/routers/web/repo/setting/lfs.go b/routers/web/repo/setting/lfs.go index bbbb99dc89..af6708e841 100644 --- a/routers/web/repo/setting/lfs.go +++ b/routers/web/repo/setting/lfs.go @@ -267,8 +267,10 @@ func LFSFileGet(ctx *context.Context) { buf = buf[:n] st := typesniffer.DetectContentType(buf) + // FIXME: there is no IsPlainText set, but template uses it ctx.Data["IsTextFile"] = st.IsText() ctx.Data["FileSize"] = meta.Size + // FIXME: the last field is the URL-base64-encoded filename, it should not be "direct" ctx.Data["RawFileLink"] = fmt.Sprintf("%s%s/%s.git/info/lfs/objects/%s/%s", setting.AppURL, url.PathEscape(ctx.Repo.Repository.OwnerName), url.PathEscape(ctx.Repo.Repository.Name), url.PathEscape(meta.Oid), "direct") switch { case st.IsRepresentableAsText(): @@ -309,8 +311,6 @@ func LFSFileGet(ctx *context.Context) { } ctx.Data["LineNums"] = gotemplate.HTML(output.String()) - case st.IsPDF(): - ctx.Data["IsPDFFile"] = true case st.IsVideo(): ctx.Data["IsVideoFile"] = true case st.IsAudio(): diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go index f0d90f9533..d9ff90568d 100644 --- a/routers/web/repo/view.go +++ b/routers/web/repo/view.go @@ -59,60 +59,63 @@ const ( ) type fileInfo struct { - isTextFile bool - isLFSFile bool - fileSize int64 - lfsMeta *lfs.Pointer - st typesniffer.SniffedType + fileSize int64 + lfsMeta *lfs.Pointer + st typesniffer.SniffedType } -func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) ([]byte, io.ReadCloser, *fileInfo, error) { - dataRc, err := blob.DataAsync() +func (fi *fileInfo) isLFSFile() bool { + return fi.lfsMeta != nil && fi.lfsMeta.Oid != "" +} + +func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) (buf []byte, dataRc io.ReadCloser, fi *fileInfo, err error) { + dataRc, err = blob.DataAsync() if err != nil { return nil, nil, nil, err } - buf := make([]byte, 1024) + const prefetchSize = lfs.MetaFileMaxSize + + buf = make([]byte, prefetchSize) n, _ := util.ReadAtMost(dataRc, buf) buf = buf[:n] - st := typesniffer.DetectContentType(buf) - isTextFile := st.IsText() + fi = &fileInfo{fileSize: blob.Size(), st: typesniffer.DetectContentType(buf)} // FIXME: what happens when README file is an image? - if !isTextFile || !setting.LFS.StartServer { - return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil + if !fi.st.IsText() || !setting.LFS.StartServer { + return buf, dataRc, fi, nil } pointer, _ := lfs.ReadPointerFromBuffer(buf) - if !pointer.IsValid() { // fallback to plain file - return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil + if !pointer.IsValid() { // fallback to a plain file + return buf, dataRc, fi, nil } meta, err := git_model.GetLFSMetaObjectByOid(ctx, repoID, pointer.Oid) - if err != nil { // fallback to plain file + if err != nil { // fallback to a plain file log.Warn("Unable to access LFS pointer %s in repo %d: %v", pointer.Oid, repoID, err) - return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil + return buf, dataRc, fi, nil } - dataRc.Close() - + // close the old dataRc and open the real LFS target + _ = dataRc.Close() dataRc, err = lfs.ReadMetaObject(pointer) if err != nil { return nil, nil, nil, err } - buf = make([]byte, 1024) + buf = make([]byte, prefetchSize) n, err = util.ReadAtMost(dataRc, buf) if err != nil { - dataRc.Close() - return nil, nil, nil, err + _ = dataRc.Close() + return nil, nil, fi, err } buf = buf[:n] - - st = typesniffer.DetectContentType(buf) - - return buf, dataRc, &fileInfo{st.IsText(), true, meta.Size, &meta.Pointer, st}, nil + fi.st = typesniffer.DetectContentType(buf) + fi.fileSize = blob.Size() + fi.lfsMeta = &meta.Pointer + return buf, dataRc, fi, nil } func loadLatestCommitData(ctx *context.Context, latestCommit *git.Commit) bool { diff --git a/routers/web/repo/view_file.go b/routers/web/repo/view_file.go index 5606a8e6ec..2d5bddd939 100644 --- a/routers/web/repo/view_file.go +++ b/routers/web/repo/view_file.go @@ -23,6 +23,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/typesniffer" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/context" issue_service "code.gitea.io/gitea/services/issue" @@ -40,7 +41,128 @@ func prepareLatestCommitInfo(ctx *context.Context) bool { return loadLatestCommitData(ctx, commit) } -func prepareToRenderFile(ctx *context.Context, entry *git.TreeEntry) { +func prepareFileViewLfsAttrs(ctx *context.Context) (*attribute.Attributes, bool) { + attrsMap, err := attribute.CheckAttributes(ctx, ctx.Repo.GitRepo, ctx.Repo.CommitID, attribute.CheckAttributeOpts{ + Filenames: []string{ctx.Repo.TreePath}, + Attributes: []string{attribute.LinguistGenerated, attribute.LinguistVendored, attribute.LinguistLanguage, attribute.GitlabLanguage}, + }) + if err != nil { + ctx.ServerError("attribute.CheckAttributes", err) + return nil, false + } + attrs := attrsMap[ctx.Repo.TreePath] + if attrs == nil { + // this case shouldn't happen, just in case. + setting.PanicInDevOrTesting("no attributes found for %s", ctx.Repo.TreePath) + attrs = attribute.NewAttributes() + } + ctx.Data["IsVendored"], ctx.Data["IsGenerated"] = attrs.GetVendored().Value(), attrs.GetGenerated().Value() + return attrs, true +} + +func handleFileViewRenderMarkup(ctx *context.Context, filename string, sniffedType typesniffer.SniffedType, prefetchBuf []byte, utf8Reader io.Reader) bool { + markupType := markup.DetectMarkupTypeByFileName(filename) + if markupType == "" { + markupType = markup.DetectRendererType(filename, sniffedType, prefetchBuf) + } + if markupType == "" { + return false + } + + ctx.Data["HasSourceRenderedToggle"] = true + + if ctx.FormString("display") == "source" { + return false + } + + ctx.Data["MarkupType"] = markupType + metas := ctx.Repo.Repository.ComposeRepoFileMetas(ctx) + metas["RefTypeNameSubURL"] = ctx.Repo.RefTypeNameSubURL() + rctx := renderhelper.NewRenderContextRepoFile(ctx, ctx.Repo.Repository, renderhelper.RepoFileOptions{ + CurrentRefPath: ctx.Repo.RefTypeNameSubURL(), + CurrentTreePath: path.Dir(ctx.Repo.TreePath), + }). + WithMarkupType(markupType). + WithRelativePath(ctx.Repo.TreePath). + WithMetas(metas) + + var err error + ctx.Data["EscapeStatus"], ctx.Data["FileContent"], err = markupRender(ctx, rctx, utf8Reader) + if err != nil { + ctx.ServerError("Render", err) + return true + } + // to prevent iframe from loading third-party url + ctx.Resp.Header().Add("Content-Security-Policy", "frame-src 'self'") + return true +} + +func handleFileViewRenderSource(ctx *context.Context, filename string, attrs *attribute.Attributes, fInfo *fileInfo, utf8Reader io.Reader) bool { + if ctx.FormString("display") == "rendered" || !fInfo.st.IsRepresentableAsText() { + return false + } + + if !fInfo.st.IsText() { + if ctx.FormString("display") == "" { + // not text but representable as text, e.g. SVG + // since there is no "display" is specified, let other renders to handle + return false + } + ctx.Data["HasSourceRenderedToggle"] = true + } + + buf, _ := io.ReadAll(utf8Reader) + // The Open Group Base Specification: https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap03.html + // empty: 0 lines; "a": 1 incomplete-line; "a\n": 1 line; "a\nb": 1 line, 1 incomplete-line; + // Gitea uses the definition (like most modern editors): + // empty: 0 lines; "a": 1 line; "a\n": 2 lines; "a\nb": 2 lines; + // When rendering, the last empty line is not rendered in UI, while the line-number is still counted, to tell users that the file contains a trailing EOL. + // To make the UI more consistent, it could use an icon mark to indicate that there is no trailing EOL, and show line-number as the rendered lines. + // This NumLines is only used for the display on the UI: "xxx lines" + if len(buf) == 0 { + ctx.Data["NumLines"] = 0 + } else { + ctx.Data["NumLines"] = bytes.Count(buf, []byte{'\n'}) + 1 + } + + language := attrs.GetLanguage().Value() + fileContent, lexerName, err := highlight.File(filename, language, buf) + ctx.Data["LexerName"] = lexerName + if err != nil { + log.Error("highlight.File failed, fallback to plain text: %v", err) + fileContent = highlight.PlainText(buf) + } + status := &charset.EscapeStatus{} + statuses := make([]*charset.EscapeStatus, len(fileContent)) + for i, line := range fileContent { + statuses[i], fileContent[i] = charset.EscapeControlHTML(line, ctx.Locale) + status = status.Or(statuses[i]) + } + ctx.Data["EscapeStatus"] = status + ctx.Data["FileContent"] = fileContent + ctx.Data["LineEscapeStatus"] = statuses + return true +} + +func handleFileViewRenderImage(ctx *context.Context, fInfo *fileInfo, prefetchBuf []byte) bool { + if !fInfo.st.IsImage() { + return false + } + if fInfo.st.IsSvgImage() && !setting.UI.SVG.Enabled { + return false + } + if fInfo.st.IsSvgImage() { + ctx.Data["HasSourceRenderedToggle"] = true + } else { + img, _, err := image.DecodeConfig(bytes.NewReader(prefetchBuf)) + if err == nil { // ignore the error for the formats that are not supported by image.DecodeConfig + ctx.Data["ImageSize"] = fmt.Sprintf("%dx%dpx", img.Width, img.Height) + } + } + return true +} + +func prepareFileView(ctx *context.Context, entry *git.TreeEntry) { ctx.Data["IsViewFile"] = true ctx.Data["HideRepoInfo"] = true @@ -86,11 +208,8 @@ func prepareToRenderFile(ctx *context.Context, entry *git.TreeEntry) { } } - isDisplayingSource := ctx.FormString("display") == "source" - isDisplayingRendered := !isDisplayingSource - // Don't call any other repository functions depends on git.Repository until the dataRc closed to - // avoid create unnecessary temporary cat file. + // avoid creating an unnecessary temporary cat file. buf, dataRc, fInfo, err := getFileReader(ctx, ctx.Repo.Repository.ID, blob) if err != nil { ctx.ServerError("getFileReader", err) @@ -98,207 +217,62 @@ func prepareToRenderFile(ctx *context.Context, entry *git.TreeEntry) { } defer dataRc.Close() - if fInfo.isLFSFile { + if fInfo.isLFSFile() { ctx.Data["RawFileLink"] = ctx.Repo.RepoLink + "/media/" + ctx.Repo.RefTypeNameSubURL() + "/" + util.PathEscapeSegments(ctx.Repo.TreePath) } - isRepresentableAsText := fInfo.st.IsRepresentableAsText() - if !isRepresentableAsText { - // If we can't show plain text, always try to render. - isDisplayingSource = false - isDisplayingRendered = true + if !prepareFileViewEditorButtons(ctx) { + return } - ctx.Data["IsLFSFile"] = fInfo.isLFSFile + + ctx.Data["IsLFSFile"] = fInfo.isLFSFile() ctx.Data["FileSize"] = fInfo.fileSize - ctx.Data["IsTextFile"] = fInfo.isTextFile - ctx.Data["IsRepresentableAsText"] = isRepresentableAsText - ctx.Data["IsDisplayingSource"] = isDisplayingSource - ctx.Data["IsDisplayingRendered"] = isDisplayingRendered + ctx.Data["IsRepresentableAsText"] = fInfo.st.IsRepresentableAsText() ctx.Data["IsExecutable"] = entry.IsExecutable() + ctx.Data["CanCopyContent"] = fInfo.st.IsRepresentableAsText() || fInfo.st.IsImage() - isTextSource := fInfo.isTextFile || isDisplayingSource - ctx.Data["IsTextSource"] = isTextSource - if isTextSource { - ctx.Data["CanCopyContent"] = true - } - - // Check LFS Lock - lfsLock, err := git_model.GetTreePathLock(ctx, ctx.Repo.Repository.ID, ctx.Repo.TreePath) - ctx.Data["LFSLock"] = lfsLock - if err != nil { - ctx.ServerError("GetTreePathLock", err) + attrs, ok := prepareFileViewLfsAttrs(ctx) + if !ok { return } - if lfsLock != nil { - u, err := user_model.GetUserByID(ctx, lfsLock.OwnerID) - if err != nil { - ctx.ServerError("GetTreePathLock", err) - return - } - ctx.Data["LFSLockOwner"] = u.Name - ctx.Data["LFSLockOwnerHomeLink"] = u.HomeLink() - ctx.Data["LFSLockHint"] = ctx.Tr("repo.editor.this_file_locked") - } - // read all needed attributes which will be used later - // there should be no performance different between reading 2 or 4 here - attrsMap, err := attribute.CheckAttributes(ctx, ctx.Repo.GitRepo, ctx.Repo.CommitID, attribute.CheckAttributeOpts{ - Filenames: []string{ctx.Repo.TreePath}, - Attributes: []string{attribute.LinguistGenerated, attribute.LinguistVendored, attribute.LinguistLanguage, attribute.GitlabLanguage}, - }) - if err != nil { - ctx.ServerError("attribute.CheckAttributes", err) - return - } - attrs := attrsMap[ctx.Repo.TreePath] - if attrs == nil { - // this case shouldn't happen, just in case. - setting.PanicInDevOrTesting("no attributes found for %s", ctx.Repo.TreePath) - attrs = attribute.NewAttributes() - } + // TODO: in the future maybe we need more accurate flags, for example: + // * IsRepresentableAsText: some files are text, some are not + // * IsRenderableXxx: some files are rendered by backend "markup" engine, some are rendered by frontend (pdf, 3d) + // * DefaultViewMode: when there is no "display" query parameter, which view mode should be used by default, source or rendered + utf8Reader := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc), charset.ConvertOpts{}) switch { - case isRepresentableAsText: - if fInfo.fileSize >= setting.UI.MaxDisplayFileSize { - ctx.Data["IsFileTooLarge"] = true - break - } - - if fInfo.st.IsSvgImage() { - ctx.Data["IsImageFile"] = true - ctx.Data["CanCopyContent"] = true - ctx.Data["HasSourceRenderedToggle"] = true - } - - rd := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc), charset.ConvertOpts{}) - - shouldRenderSource := ctx.FormString("display") == "source" - readmeExist := util.IsReadmeFileName(blob.Name()) - ctx.Data["ReadmeExist"] = readmeExist - - markupType := markup.DetectMarkupTypeByFileName(blob.Name()) - if markupType == "" { - markupType = markup.DetectRendererType(blob.Name(), bytes.NewReader(buf)) - } - if markupType != "" { - ctx.Data["HasSourceRenderedToggle"] = true - } - if markupType != "" && !shouldRenderSource { - ctx.Data["IsMarkup"] = true - ctx.Data["MarkupType"] = markupType - metas := ctx.Repo.Repository.ComposeRepoFileMetas(ctx) - metas["RefTypeNameSubURL"] = ctx.Repo.RefTypeNameSubURL() - rctx := renderhelper.NewRenderContextRepoFile(ctx, ctx.Repo.Repository, renderhelper.RepoFileOptions{ - CurrentRefPath: ctx.Repo.RefTypeNameSubURL(), - CurrentTreePath: path.Dir(ctx.Repo.TreePath), - }). - WithMarkupType(markupType). - WithRelativePath(ctx.Repo.TreePath). - WithMetas(metas) - - ctx.Data["EscapeStatus"], ctx.Data["FileContent"], err = markupRender(ctx, rctx, rd) - if err != nil { - ctx.ServerError("Render", err) - return - } - // to prevent iframe load third-party url - ctx.Resp.Header().Add("Content-Security-Policy", "frame-src 'self'") - } else { - buf, _ := io.ReadAll(rd) - - // The Open Group Base Specification: https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap03.html - // empty: 0 lines; "a": 1 incomplete-line; "a\n": 1 line; "a\nb": 1 line, 1 incomplete-line; - // Gitea uses the definition (like most modern editors): - // empty: 0 lines; "a": 1 line; "a\n": 2 lines; "a\nb": 2 lines; - // When rendering, the last empty line is not rendered in UI, while the line-number is still counted, to tell users that the file contains a trailing EOL. - // To make the UI more consistent, it could use an icon mark to indicate that there is no trailing EOL, and show line-number as the rendered lines. - // This NumLines is only used for the display on the UI: "xxx lines" - if len(buf) == 0 { - ctx.Data["NumLines"] = 0 - } else { - ctx.Data["NumLines"] = bytes.Count(buf, []byte{'\n'}) + 1 - } - - language := attrs.GetLanguage().Value() - fileContent, lexerName, err := highlight.File(blob.Name(), language, buf) - ctx.Data["LexerName"] = lexerName - if err != nil { - log.Error("highlight.File failed, fallback to plain text: %v", err) - fileContent = highlight.PlainText(buf) - } - status := &charset.EscapeStatus{} - statuses := make([]*charset.EscapeStatus, len(fileContent)) - for i, line := range fileContent { - statuses[i], fileContent[i] = charset.EscapeControlHTML(line, ctx.Locale) - status = status.Or(statuses[i]) - } - ctx.Data["EscapeStatus"] = status - ctx.Data["FileContent"] = fileContent - ctx.Data["LineEscapeStatus"] = statuses - } - - case fInfo.st.IsPDF(): - ctx.Data["IsPDFFile"] = true + case fInfo.fileSize >= setting.UI.MaxDisplayFileSize: + ctx.Data["IsFileTooLarge"] = true + case handleFileViewRenderMarkup(ctx, entry.Name(), fInfo.st, buf, utf8Reader): + // it also sets ctx.Data["FileContent"] and more + ctx.Data["IsMarkup"] = true + case handleFileViewRenderSource(ctx, entry.Name(), attrs, fInfo, utf8Reader): + // it also sets ctx.Data["FileContent"] and more + ctx.Data["IsDisplayingSource"] = true + case handleFileViewRenderImage(ctx, fInfo, buf): + ctx.Data["IsImageFile"] = true case fInfo.st.IsVideo(): ctx.Data["IsVideoFile"] = true case fInfo.st.IsAudio(): ctx.Data["IsAudioFile"] = true - case fInfo.st.IsImage() && (setting.UI.SVG.Enabled || !fInfo.st.IsSvgImage()): - ctx.Data["IsImageFile"] = true - ctx.Data["CanCopyContent"] = true default: - if fInfo.fileSize >= setting.UI.MaxDisplayFileSize { - ctx.Data["IsFileTooLarge"] = true - break - } - - // TODO: this logic duplicates with "isRepresentableAsText=true", it is not the same as "LFSFileGet" in "lfs.go" - // It is used by "external renders", markupRender will execute external programs to get rendered content. - if markupType := markup.DetectMarkupTypeByFileName(blob.Name()); markupType != "" { - rd := io.MultiReader(bytes.NewReader(buf), dataRc) - ctx.Data["IsMarkup"] = true - ctx.Data["MarkupType"] = markupType - - rctx := renderhelper.NewRenderContextRepoFile(ctx, ctx.Repo.Repository, renderhelper.RepoFileOptions{ - CurrentRefPath: ctx.Repo.RefTypeNameSubURL(), - CurrentTreePath: path.Dir(ctx.Repo.TreePath), - }). - WithMarkupType(markupType). - WithRelativePath(ctx.Repo.TreePath) - - ctx.Data["EscapeStatus"], ctx.Data["FileContent"], err = markupRender(ctx, rctx, rd) - if err != nil { - ctx.ServerError("Render", err) - return - } - } + // unable to render anything, show the "view raw" or let frontend handle it } - - ctx.Data["IsVendored"], ctx.Data["IsGenerated"] = attrs.GetVendored().Value(), attrs.GetGenerated().Value() - - if fInfo.st.IsImage() && !fInfo.st.IsSvgImage() { - img, _, err := image.DecodeConfig(bytes.NewReader(buf)) - if err == nil { - // There are Image formats go can't decode - // Instead of throwing an error in that case, we show the size only when we can decode - ctx.Data["ImageSize"] = fmt.Sprintf("%dx%dpx", img.Width, img.Height) - } - } - - prepareToRenderButtons(ctx, lfsLock) } -func prepareToRenderButtons(ctx *context.Context, lfsLock *git_model.LFSLock) { +func prepareFileViewEditorButtons(ctx *context.Context) bool { // archived or mirror repository, the buttons should not be shown if !ctx.Repo.Repository.CanEnableEditor() { - return + return true } // The buttons should not be shown if it's not a branch if !ctx.Repo.RefFullName.IsBranch() { ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.must_be_on_a_branch") ctx.Data["DeleteFileTooltip"] = ctx.Tr("repo.editor.must_be_on_a_branch") - return + return true } if !ctx.Repo.CanWriteToBranch(ctx, ctx.Doer, ctx.Repo.BranchName) { @@ -306,7 +280,24 @@ func prepareToRenderButtons(ctx *context.Context, lfsLock *git_model.LFSLock) { ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.fork_before_edit") ctx.Data["CanDeleteFile"] = true ctx.Data["DeleteFileTooltip"] = ctx.Tr("repo.editor.must_have_write_access") - return + return true + } + + lfsLock, err := git_model.GetTreePathLock(ctx, ctx.Repo.Repository.ID, ctx.Repo.TreePath) + ctx.Data["LFSLock"] = lfsLock + if err != nil { + ctx.ServerError("GetTreePathLock", err) + return false + } + if lfsLock != nil { + u, err := user_model.GetUserByID(ctx, lfsLock.OwnerID) + if err != nil { + ctx.ServerError("GetTreePathLock", err) + return false + } + ctx.Data["LFSLockOwner"] = u.Name + ctx.Data["LFSLockOwnerHomeLink"] = u.HomeLink() + ctx.Data["LFSLockHint"] = ctx.Tr("repo.editor.this_file_locked") } // it's a lfs file and the user is not the owner of the lock @@ -315,4 +306,5 @@ func prepareToRenderButtons(ctx *context.Context, lfsLock *git_model.LFSLock) { ctx.Data["EditFileTooltip"] = util.Iif(isLFSLocked, ctx.Tr("repo.editor.this_file_locked"), ctx.Tr("repo.editor.edit_this_file")) ctx.Data["CanDeleteFile"] = !isLFSLocked ctx.Data["DeleteFileTooltip"] = util.Iif(isLFSLocked, ctx.Tr("repo.editor.this_file_locked"), ctx.Tr("repo.editor.delete_this_file")) + return true } diff --git a/routers/web/repo/view_home.go b/routers/web/repo/view_home.go index 48fa47d738..8ed9179290 100644 --- a/routers/web/repo/view_home.go +++ b/routers/web/repo/view_home.go @@ -339,7 +339,7 @@ func prepareToRenderDirOrFile(entry *git.TreeEntry) func(ctx *context.Context) { if entry.IsDir() { prepareToRenderDirectory(ctx) } else { - prepareToRenderFile(ctx, entry) + prepareFileView(ctx, entry) } } } diff --git a/routers/web/repo/view_readme.go b/routers/web/repo/view_readme.go index 4ce22d79db..a34de06e8e 100644 --- a/routers/web/repo/view_readme.go +++ b/routers/web/repo/view_readme.go @@ -161,24 +161,23 @@ func prepareToRenderReadmeFile(ctx *context.Context, subfolder string, readmeFil } defer dataRc.Close() - ctx.Data["FileIsText"] = fInfo.isTextFile + ctx.Data["FileIsText"] = fInfo.st.IsText() ctx.Data["FileTreePath"] = path.Join(ctx.Repo.TreePath, subfolder, readmeFile.Name()) ctx.Data["FileSize"] = fInfo.fileSize - ctx.Data["IsLFSFile"] = fInfo.isLFSFile + ctx.Data["IsLFSFile"] = fInfo.isLFSFile() - if fInfo.isLFSFile { + if fInfo.isLFSFile() { filenameBase64 := base64.RawURLEncoding.EncodeToString([]byte(readmeFile.Name())) ctx.Data["RawFileLink"] = fmt.Sprintf("%s.git/info/lfs/objects/%s/%s", ctx.Repo.Repository.Link(), url.PathEscape(fInfo.lfsMeta.Oid), url.PathEscape(filenameBase64)) } - if !fInfo.isTextFile { + if !fInfo.st.IsText() { return } if fInfo.fileSize >= setting.UI.MaxDisplayFileSize { // Pretend that this is a normal text file to display 'This file is too large to be shown' ctx.Data["IsFileTooLarge"] = true - ctx.Data["IsTextFile"] = true return } @@ -212,7 +211,7 @@ func prepareToRenderReadmeFile(ctx *context.Context, subfolder string, readmeFil ctx.Data["EscapeStatus"], ctx.Data["FileContent"] = charset.EscapeControlHTML(template.HTML(contentEscaped), ctx.Locale) } - if !fInfo.isLFSFile && ctx.Repo.Repository.CanEnableEditor() { + if !fInfo.isLFSFile() && ctx.Repo.Repository.CanEnableEditor() { ctx.Data["CanEditReadmeFile"] = true } } diff --git a/templates/repo/blame.tmpl b/templates/repo/blame.tmpl index 9596fe837a..c4d9f0741f 100644 --- a/templates/repo/blame.tmpl +++ b/templates/repo/blame.tmpl @@ -82,6 +82,8 @@ {{end}}{{/* end if .IsFileTooLarge */}}
+ {{/*FIXME: the "HasSourceRenderedToggle" is never set on blame page, it should mean "whether the file is renderable". + If the file is renderable, then it must has the "display=source" parameter to make sure the file view page shows the source code, then line number works. */}} {{if $.Permission.CanRead ctx.Consts.RepoUnitTypeIssues}} {{ctx.Locale.Tr "repo.issues.context.reference_issue"}} {{end}} diff --git a/templates/repo/editor/common_breadcrumb.tmpl b/templates/repo/editor/common_breadcrumb.tmpl index df36f00504..8cfbe09d3e 100644 --- a/templates/repo/editor/common_breadcrumb.tmpl +++ b/templates/repo/editor/common_breadcrumb.tmpl @@ -5,7 +5,7 @@ {{range $i, $v := .TreeNames}}
/
{{if eq $i $l}} - + {{svg "octicon-info"}} {{else}} {{$v}} diff --git a/templates/repo/settings/lfs_file.tmpl b/templates/repo/settings/lfs_file.tmpl index 1a8014e218..cd1b168401 100644 --- a/templates/repo/settings/lfs_file.tmpl +++ b/templates/repo/settings/lfs_file.tmpl @@ -30,8 +30,6 @@ - {{else if .IsPDFFile}} -
{{else}} {{ctx.Locale.Tr "repo.file_view_raw"}} {{end}} diff --git a/templates/repo/view_file.tmpl b/templates/repo/view_file.tmpl index b49818c6b7..1486d7181d 100644 --- a/templates/repo/view_file.tmpl +++ b/templates/repo/view_file.tmpl @@ -1,4 +1,6 @@ -
+
+ {{- if .FileError}}
{{.FileError}}
@@ -32,13 +34,14 @@ {{template "repo/file_info" .}} {{end}}
-
- {{if .HasSourceRenderedToggle}} -
- {{svg "octicon-code" 15}} - {{svg "octicon-file" 15}} -
- {{end}} +
+ {{/* this componment is also controlled by frontend plugin renders */}} +
+ {{if .IsRepresentableAsText}} + {{svg "octicon-code" 15}} + {{end}} + {{svg "octicon-file" 15}} +
{{if not .ReadmeInList}}
{{ctx.Locale.Tr "repo.file_raw"}} @@ -55,7 +58,10 @@ {{end}}
{{svg "octicon-download"}} - {{svg "octicon-copy"}} + {{svg "octicon-copy"}} {{if .EnableFeed}} {{svg "octicon-rss"}} @@ -82,20 +88,36 @@ {{end}}
+
- {{if not (or .IsMarkup .IsRenderedHTML)}} - {{template "repo/unicode_escape_prompt" dict "EscapeStatus" .EscapeStatus "root" $}} + {{if not .IsMarkup}} + {{template "repo/unicode_escape_prompt" dict "EscapeStatus" .EscapeStatus}} {{end}} -
+
{{if .IsFileTooLarge}} {{template "shared/filetoolarge" dict "RawFileLink" .RawFileLink}} {{else if not .FileSize}} {{template "shared/fileisempty"}} {{else if .IsMarkup}} - {{if .FileContent}}{{.FileContent}}{{end}} + {{.FileContent}} {{else if .IsPlainText}} 
{{if .FileContent}}{{.FileContent}}{{end}}
- {{else if not .IsTextSource}} + {{else if .FileContent}} + + + {{range $idx, $code := .FileContent}} + {{$line := Eval $idx "+" 1}} + + + {{if $.EscapeStatus.Escaped}} + + {{end}} + + + {{end}} + +
{{if (index $.LineEscapeStatus $idx).Escaped}}{{end}}{{$code}}
+ {{else}}
{{if .IsImageFile}} {{$.RawFileLink}} @@ -107,35 +129,23 @@ - {{else if .IsPDFFile}} -
{{else}} - {{ctx.Locale.Tr "repo.file_view_raw"}} +
+
+ {{ctx.Locale.Tr "repo.file_view_raw"}} +
+
{{end}}
- {{else if .FileSize}} - - - {{range $idx, $code := .FileContent}} - {{$line := Eval $idx "+" 1}} - - - {{if $.EscapeStatus.Escaped}} - - {{end}} - - - {{end}} - -
{{if (index $.LineEscapeStatus $idx).Escaped}}{{end}}{{$code}}
-
- {{if $.Permission.CanRead ctx.Consts.RepoUnitTypeIssues}} - {{ctx.Locale.Tr "repo.issues.context.reference_issue"}} - {{end}} - {{ctx.Locale.Tr "repo.view_git_blame"}} - {{ctx.Locale.Tr "repo.file_copy_permalink"}} -
{{end}}
+ +
+ {{if $.Permission.CanRead ctx.Consts.RepoUnitTypeIssues}} + {{ctx.Locale.Tr "repo.issues.context.reference_issue"}} + {{end}} + {{ctx.Locale.Tr "repo.view_git_blame"}} + {{ctx.Locale.Tr "repo.file_copy_permalink"}} +
diff --git a/tests/integration/lfs_view_test.go b/tests/integration/lfs_view_test.go index 64ffebaa78..c26ece22be 100644 --- a/tests/integration/lfs_view_test.go +++ b/tests/integration/lfs_view_test.go @@ -68,14 +68,15 @@ func TestLFSRender(t *testing.T) { req := NewRequest(t, "GET", "/user2/lfs/src/branch/master/crypt.bin") resp := session.MakeRequest(t, req, http.StatusOK) - doc := NewHTMLParser(t, resp.Body).doc + doc := NewHTMLParser(t, resp.Body) fileInfo := doc.Find("div.file-info-entry").First().Text() assert.Contains(t, fileInfo, "LFS") - rawLink, exists := doc.Find("div.file-view > div.view-raw > a").Attr("href") - assert.True(t, exists, "Download link should render instead of content because this is a binary file") - assert.Equal(t, "/user2/lfs/media/branch/master/crypt.bin", rawLink, "The download link should use the proper /media link because it's in LFS") + // find new file view container + fileViewContainer := doc.Find("[data-global-init=initRepoFileView]") + assert.Equal(t, "/user2/lfs/media/branch/master/crypt.bin", fileViewContainer.AttrOr("data-raw-file-link", "")) + AssertHTMLElement(t, doc, ".view-raw > .file-view-render-container > .file-view-raw-prompt", 1) }) // check that a directory with a README file shows its text diff --git a/web_src/css/modules/animations.css b/web_src/css/modules/animations.css index 8edf31ddbd..deaaf83680 100644 --- a/web_src/css/modules/animations.css +++ b/web_src/css/modules/animations.css @@ -52,8 +52,7 @@ form.single-button-form.is-loading .button { } .markup pre.is-loading, -.editor-loading.is-loading, -.pdf-content.is-loading { +.editor-loading.is-loading { height: var(--height-loading); } diff --git a/web_src/css/repo.css b/web_src/css/repo.css index 41fec58f94..a72709c382 100644 --- a/web_src/css/repo.css +++ b/web_src/css/repo.css @@ -183,42 +183,6 @@ td .commit-summary { cursor: default; } -.view-raw { - display: flex; - justify-content: center; - align-items: center; -} - -.view-raw > * { - max-width: 100%; -} - -.view-raw audio, -.view-raw video, -.view-raw img { - margin: 1rem 0; - border-radius: 0; - object-fit: contain; -} - -.view-raw img[src$=".svg" i] { - max-height: 600px !important; - max-width: 600px !important; -} - -.pdf-content { - width: 100%; - height: 600px; - border: none !important; - display: flex; - align-items: center; - justify-content: center; -} - -.pdf-content .pdf-fallback-button { - margin: 50px auto; -} - .repository.file.list .non-diff-file-content .plain-text { padding: 1em 2em; } @@ -241,10 +205,6 @@ td .commit-summary { padding: 0 !important; } -.non-diff-file-content .pdfobject { - border-radius: 0 0 var(--border-radius) var(--border-radius); -} - .repo-editor-header { width: 100%; } diff --git a/web_src/css/repo/file-view.css b/web_src/css/repo/file-view.css index 54af5f4602..907f136afe 100644 --- a/web_src/css/repo/file-view.css +++ b/web_src/css/repo/file-view.css @@ -60,3 +60,33 @@ .file-view.code-view .ui.button.code-line-button:hover { background: var(--color-secondary); } + +.view-raw { + display: flex; + justify-content: center; +} + +.view-raw > * { + max-width: 100%; +} + +.view-raw audio, +.view-raw video, +.view-raw img { + margin: 1rem; + border-radius: 0; + object-fit: contain; +} + +.view-raw img[src$=".svg" i] { + max-height: 600px !important; + max-width: 600px !important; +} + +.file-view-render-container { + width: 100%; +} + +.file-view-render-container :last-child { + border-radius: 0 0 var(--border-radius) var(--border-radius); /* to match the "ui segment" bottom radius */ +} diff --git a/web_src/js/features/copycontent.ts b/web_src/js/features/copycontent.ts index d58f6c8246..0fec2a6235 100644 --- a/web_src/js/features/copycontent.ts +++ b/web_src/js/features/copycontent.ts @@ -9,17 +9,17 @@ const {i18n} = window.config; export function initCopyContent() { registerGlobalEventFunc('click', 'onCopyContentButtonClick', async (btn: HTMLElement) => { if (btn.classList.contains('disabled') || btn.classList.contains('is-loading')) return; - let content; - let isRasterImage = false; - const link = btn.getAttribute('data-link'); + const rawFileLink = btn.getAttribute('data-raw-file-link'); - // when data-link is present, we perform a fetch. this is either because - // the text to copy is not in the DOM, or it is an image which should be + let content, isRasterImage = false; + + // when "data-raw-link" is present, we perform a fetch. this is either because + // the text to copy is not in the DOM, or it is an image that should be // fetched to copy in full resolution - if (link) { + if (rawFileLink) { btn.classList.add('is-loading', 'loading-icon-2px'); try { - const res = await GET(link, {credentials: 'include', redirect: 'follow'}); + const res = await GET(rawFileLink, {credentials: 'include', redirect: 'follow'}); const contentType = res.headers.get('content-type'); if (contentType.startsWith('image/') && !contentType.startsWith('image/svg')) { diff --git a/web_src/js/features/file-view.ts b/web_src/js/features/file-view.ts new file mode 100644 index 0000000000..867f946297 --- /dev/null +++ b/web_src/js/features/file-view.ts @@ -0,0 +1,76 @@ +import type {FileRenderPlugin} from '../render/plugin.ts'; +import {newRenderPlugin3DViewer} from '../render/plugins/3d-viewer.ts'; +import {newRenderPluginPdfViewer} from '../render/plugins/pdf-viewer.ts'; +import {registerGlobalInitFunc} from '../modules/observer.ts'; +import {createElementFromHTML, showElem, toggleClass} from '../utils/dom.ts'; +import {htmlEscape} from 'escape-goat'; +import {basename} from '../utils.ts'; + +const plugins: FileRenderPlugin[] = []; + +function initPluginsOnce(): void { + if (plugins.length) return; + plugins.push(newRenderPlugin3DViewer(), newRenderPluginPdfViewer()); +} + +function findFileRenderPlugin(filename: string, mimeType: string): FileRenderPlugin | null { + return plugins.find((plugin) => plugin.canHandle(filename, mimeType)) || null; +} + +function showRenderRawFileButton(elFileView: HTMLElement, renderContainer: HTMLElement | null): void { + const toggleButtons = elFileView.querySelector('.file-view-toggle-buttons'); + showElem(toggleButtons); + const displayingRendered = Boolean(renderContainer); + toggleClass(toggleButtons.querySelectorAll('.file-view-toggle-source'), 'active', !displayingRendered); // it may not exist + toggleClass(toggleButtons.querySelector('.file-view-toggle-rendered'), 'active', displayingRendered); + // TODO: if there is only one button, hide it? +} + +async function renderRawFileToContainer(container: HTMLElement, rawFileLink: string, mimeType: string) { + const elViewRawPrompt = container.querySelector('.file-view-raw-prompt'); + if (!rawFileLink || !elViewRawPrompt) throw new Error('unexpected file view container'); + + let rendered = false, errorMsg = ''; + try { + const plugin = findFileRenderPlugin(basename(rawFileLink), mimeType); + if (plugin) { + container.classList.add('is-loading'); + container.setAttribute('data-render-name', plugin.name); // not used yet + await plugin.render(container, rawFileLink); + rendered = true; + } + } catch (e) { + errorMsg = `${e}`; + } finally { + container.classList.remove('is-loading'); + } + + if (rendered) { + elViewRawPrompt.remove(); + return; + } + + // remove all children from the container, and only show the raw file link + container.replaceChildren(elViewRawPrompt); + + if (errorMsg) { + const elErrorMessage = createElementFromHTML(htmlEscape`
${errorMsg}
`); + elViewRawPrompt.insertAdjacentElement('afterbegin', elErrorMessage); + } +} + +export function initRepoFileView(): void { + registerGlobalInitFunc('initRepoFileView', async (elFileView: HTMLElement) => { + initPluginsOnce(); + const rawFileLink = elFileView.getAttribute('data-raw-file-link'); + const mimeType = elFileView.getAttribute('data-mime-type') || ''; // not used yet + // TODO: we should also provide the prefetched file head bytes to let the plugin decide whether to render or not + const plugin = findFileRenderPlugin(basename(rawFileLink), mimeType); + if (!plugin) return; + + const renderContainer = elFileView.querySelector('.file-view-render-container'); + showRenderRawFileButton(elFileView, renderContainer); + // maybe in the future multiple plugins can render the same file, so we should not assume only one plugin will render it + if (renderContainer) await renderRawFileToContainer(renderContainer, rawFileLink, mimeType); + }); +} diff --git a/web_src/js/index.ts b/web_src/js/index.ts index 7e84773bc1..347aad2709 100644 --- a/web_src/js/index.ts +++ b/web_src/js/index.ts @@ -19,7 +19,7 @@ import {initRepoIssueContentHistory} from './features/repo-issue-content.ts'; import {initStopwatch} from './features/stopwatch.ts'; import {initFindFileInRepo} from './features/repo-findfile.ts'; import {initMarkupContent} from './markup/content.ts'; -import {initPdfViewer} from './render/pdf.ts'; +import {initRepoFileView} from './features/file-view.ts'; import {initUserAuthOauth2, initUserCheckAppUrl} from './features/user-auth.ts'; import {initRepoPullRequestAllowMaintainerEdit, initRepoPullRequestReview, initRepoIssueSidebarDependency, initRepoIssueFilterItemLabel} from './features/repo-issue.ts'; import {initRepoEllipsisButton, initCommitStatuses} from './features/repo-commit.ts'; @@ -159,10 +159,11 @@ onDomReady(() => { initUserAuthWebAuthnRegister, initUserSettings, initRepoDiffView, - initPdfViewer, initColorPickers, initOAuth2SettingsDisableCheckbox, + + initRepoFileView, ]); // it must be the last one, then the "querySelectorAll" only needs to be executed once for global init functions. diff --git a/web_src/js/render/pdf.ts b/web_src/js/render/pdf.ts deleted file mode 100644 index 283b4ed85c..0000000000 --- a/web_src/js/render/pdf.ts +++ /dev/null @@ -1,17 +0,0 @@ -import {htmlEscape} from 'escape-goat'; -import {registerGlobalInitFunc} from '../modules/observer.ts'; - -export async function initPdfViewer() { - registerGlobalInitFunc('initPdfViewer', async (el: HTMLInputElement) => { - const pdfobject = await import(/* webpackChunkName: "pdfobject" */'pdfobject'); - - const src = el.getAttribute('data-src'); - const fallbackText = el.getAttribute('data-fallback-button-text'); - pdfobject.embed(src, el, { - fallbackLink: htmlEscape` - ${fallbackText} - `, - }); - el.classList.remove('is-loading'); - }); -} diff --git a/web_src/js/render/plugin.ts b/web_src/js/render/plugin.ts new file mode 100644 index 0000000000..a8dd0a7c05 --- /dev/null +++ b/web_src/js/render/plugin.ts @@ -0,0 +1,10 @@ +export type FileRenderPlugin = { + // unique plugin name + name: string; + + // test if plugin can handle a specified file + canHandle: (filename: string, mimeType: string) => boolean; + + // render file content + render: (container: HTMLElement, fileUrl: string, options?: any) => Promise; +} diff --git a/web_src/js/render/plugins/3d-viewer.ts b/web_src/js/render/plugins/3d-viewer.ts new file mode 100644 index 0000000000..2a0929359d --- /dev/null +++ b/web_src/js/render/plugins/3d-viewer.ts @@ -0,0 +1,60 @@ +import type {FileRenderPlugin} from '../plugin.ts'; +import {extname} from '../../utils.ts'; + +// support common 3D model file formats, use online-3d-viewer library for rendering + +// eslint-disable-next-line multiline-comment-style +/* a simple text STL file example: +solid SimpleTriangle + facet normal 0 0 1 + outer loop + vertex 0 0 0 + vertex 1 0 0 + vertex 0 1 0 + endloop + endfacet +endsolid SimpleTriangle +*/ + +export function newRenderPlugin3DViewer(): FileRenderPlugin { + // Some extensions are text-based formats: + // .3mf .amf .brep: XML + // .fbx: XML or BINARY + // .dae .gltf: JSON + // .ifc, .igs, .iges, .stp, .step are: TEXT + // .stl .ply: TEXT or BINARY + // .obj .off .wrl: TEXT + // So we need to be able to render when the file is recognized as plaintext file by backend. + // + // It needs more logic to make it overall right (render a text 3D model automatically): + // we need to distinguish the ambiguous filename extensions. + // For example: "*.obj, *.off, *.step" might be or not be a 3D model file. + // So when it is a text file, we can't assume that "we only render it by 3D plugin", + // otherwise the end users would be impossible to view its real content when the file is not a 3D model. + const SUPPORTED_EXTENSIONS = [ + '.3dm', '.3ds', '.3mf', '.amf', '.bim', '.brep', + '.dae', '.fbx', '.fcstd', '.glb', '.gltf', + '.ifc', '.igs', '.iges', '.stp', '.step', + '.stl', '.obj', '.off', '.ply', '.wrl', + ]; + + return { + name: '3d-model-viewer', + + canHandle(filename: string, _mimeType: string): boolean { + const ext = extname(filename).toLowerCase(); + return SUPPORTED_EXTENSIONS.includes(ext); + }, + + async render(container: HTMLElement, fileUrl: string): Promise { + // TODO: height and/or max-height? + const OV = await import(/* webpackChunkName: "online-3d-viewer" */'online-3d-viewer'); + const viewer = new OV.EmbeddedViewer(container, { + backgroundColor: new OV.RGBAColor(59, 68, 76, 0), + defaultColor: new OV.RGBColor(65, 131, 196), + edgeSettings: new OV.EdgeSettings(false, new OV.RGBColor(0, 0, 0), 1), + }); + viewer.LoadModelFromUrlList([fileUrl]); + }, + }; +} diff --git a/web_src/js/render/plugins/pdf-viewer.ts b/web_src/js/render/plugins/pdf-viewer.ts new file mode 100644 index 0000000000..40623be055 --- /dev/null +++ b/web_src/js/render/plugins/pdf-viewer.ts @@ -0,0 +1,20 @@ +import type {FileRenderPlugin} from '../plugin.ts'; + +export function newRenderPluginPdfViewer(): FileRenderPlugin { + return { + name: 'pdf-viewer', + + canHandle(filename: string, _mimeType: string): boolean { + return filename.toLowerCase().endsWith('.pdf'); + }, + + async render(container: HTMLElement, fileUrl: string): Promise { + const PDFObject = await import(/* webpackChunkName: "pdfobject" */'pdfobject'); + // TODO: the PDFObject library does not support dynamic height adjustment, + container.style.height = `${window.innerHeight - 100}px`; + if (!PDFObject.default.embed(fileUrl, container)) { + throw new Error('Unable to render the PDF file'); + } + }, + }; +}