Markdown rendering overhaul (#186)

* Markdown rendering overhaul

Cleaned up and squashed commits into single one.

Signed-off-by: Andrew Boyarshin <boyarshinand@gmail.com>

* Fix markdown API, add markdown module and API tests, improve code coverage

Signed-off-by: Andrew Boyarshin <boyarshinand@gmail.com>

routers/api/v1/misc/markdown.go

@@ -9,6 +9,7 @@ import (
 
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/markdown"
+	"code.gitea.io/gitea/modules/setting"
 )
 
 // Markdown render markdown document to HTML
@@ -26,9 +27,9 @@ func Markdown(ctx *context.APIContext, form api.MarkdownOption) {
 
 	switch form.Mode {
 	case "gfm":
-		ctx.Write(markdown.Render([]byte(form.Text), form.Context, nil))
+		ctx.Write(markdown.Render([]byte(form.Text), markdown.URLJoin(setting.AppURL, form.Context), nil))
 	default:
-		ctx.Write(markdown.RenderRaw([]byte(form.Text), ""))
+		ctx.Write(markdown.RenderRaw([]byte(form.Text), "", false))
 	}
 }
 
@@ -40,5 +41,5 @@ func MarkdownRaw(ctx *context.APIContext) {
 		ctx.Error(422, "", err)
 		return
 	}
-	ctx.Write(markdown.RenderRaw(body, ""))
+	ctx.Write(markdown.RenderRaw(body, "", false))
 }

routers/api/v1/misc/markdown_test.go

@@ -0,0 +1,184 @@
+package misc
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	macaron "gopkg.in/macaron.v1"
+
+	"net/url"
+
+	"io/ioutil"
+	"strings"
+
+	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/markdown"
+	"code.gitea.io/gitea/modules/setting"
+	api "code.gitea.io/sdk/gitea"
+	"github.com/go-macaron/inject"
+	"github.com/stretchr/testify/assert"
+)
+
+const AppURL = "http://localhost:3000/"
+const Repo = "gogits/gogs"
+const AppSubURL = AppURL + Repo + "/"
+
+func createContext(req *http.Request) (*macaron.Context, *httptest.ResponseRecorder) {
+	resp := httptest.NewRecorder()
+	c := &macaron.Context{
+		Injector: inject.New(),
+		Req:      macaron.Request{req},
+		Resp:     macaron.NewResponseWriter(resp),
+		Render:   &macaron.DummyRender{resp},
+		Data:     make(map[string]interface{}),
+	}
+	c.Map(c)
+	c.Map(req)
+	return c, resp
+}
+
+func wrap(ctx *macaron.Context) *context.APIContext {
+	return &context.APIContext{
+		Context: &context.Context{
+			Context: ctx,
+		},
+	}
+}
+
+func TestAPI_RenderGFM(t *testing.T) {
+	setting.AppURL = AppURL
+
+	options := api.MarkdownOption{
+		Mode:    "gfm",
+		Text:    "",
+		Context: Repo,
+	}
+	requrl, _ := url.Parse(markdown.URLJoin(AppURL, "api", "v1", "markdown"))
+	req := &http.Request{
+		Method: "POST",
+		URL:    requrl,
+	}
+	m, resp := createContext(req)
+	ctx := wrap(m)
+
+	testCases := []string{
+		// dear imgui wiki markdown extract: special wiki syntax
+		`Wiki! Enjoy :)
+- [[Links, Language bindings, Engine bindings|Links]]
+- [[Tips]]
+- Bezier widget (by @r-lyeh) https://github.com/ocornut/imgui/issues/786`,
+		// rendered
+		`<p>Wiki! Enjoy :)</p>
+
+<ul>
+<li><a href="` + AppSubURL + `wiki/Links" rel="nofollow">Links, Language bindings, Engine bindings</a></li>
+<li><a href="` + AppSubURL + `wiki/Tips" rel="nofollow">Tips</a></li>
+<li>Bezier widget (by <a href="` + AppURL + `r-lyeh" rel="nofollow">@r-lyeh</a>)<a href="` + AppSubURL + `issues/786" rel="nofollow">#786</a></li>
+</ul>
+`,
+		// wine-staging wiki home extract: special wiki syntax, images
+		`## What is Wine Staging?
+**Wine Staging** on website [wine-staging.com](http://wine-staging.com).
+
+## Quick Links
+Here are some links to the most important topics. You can find the full list of pages at the sidebar.
+
+[[Configuration]]
+[[images/icon-bug.png]]
+`,
+		// rendered
+		`<h2>What is Wine Staging?</h2>
+
+<p><strong>Wine Staging</strong> on website <a href="http://wine-staging.com" rel="nofollow">wine-staging.com</a>.</p>
+
+<h2>Quick Links</h2>
+
+<p>Here are some links to the most important topics. You can find the full list of pages at the sidebar.</p>
+
+<p><a href="` + AppSubURL + `wiki/Configuration" rel="nofollow">Configuration</a>
+<a href="` + AppSubURL + `wiki/raw/images%2Ficon-bug.png" rel="nofollow"><img src="` + AppSubURL + `wiki/raw/images%2Ficon-bug.png" alt="images/icon-bug.png" title="icon-bug.png"/></a></p>
+`,
+		// Guard wiki sidebar: special syntax
+		`[[Guardfile-DSL / Configuring-Guard|Guardfile-DSL---Configuring-Guard]]`,
+		// rendered
+		`<p><a href="` + AppSubURL + `wiki/Guardfile-DSL---Configuring-Guard" rel="nofollow">Guardfile-DSL / Configuring-Guard</a></p>
+`,
+		// special syntax
+		`[[Name|Link]]`,
+		// rendered
+		`<p><a href="` + AppSubURL + `wiki/Link" rel="nofollow">Name</a></p>
+`,
+		// empty
+		``,
+		// rendered
+		``,
+	}
+
+	for i := 0; i < len(testCases); i += 2 {
+		options.Text = testCases[i]
+		Markdown(ctx, options)
+		assert.Equal(t, testCases[i+1], resp.Body.String())
+		resp.Body.Reset()
+	}
+}
+
+var simpleCases = []string{
+	// Guard wiki sidebar: special syntax
+	`[[Guardfile-DSL / Configuring-Guard|Guardfile-DSL---Configuring-Guard]]`,
+	// rendered
+	`<p>[[Guardfile-DSL / Configuring-Guard|Guardfile-DSL---Configuring-Guard]]</p>
+`,
+	// special syntax
+	`[[Name|Link]]`,
+	// rendered
+	`<p>[[Name|Link]]</p>
+`,
+	// empty
+	``,
+	// rendered
+	``,
+}
+
+func TestAPI_RenderSimple(t *testing.T) {
+	setting.AppURL = AppURL
+
+	options := api.MarkdownOption{
+		Mode:    "markdown",
+		Text:    "",
+		Context: Repo,
+	}
+	requrl, _ := url.Parse(markdown.URLJoin(AppURL, "api", "v1", "markdown"))
+	req := &http.Request{
+		Method: "POST",
+		URL:    requrl,
+	}
+	m, resp := createContext(req)
+	ctx := wrap(m)
+
+	for i := 0; i < len(simpleCases); i += 2 {
+		options.Text = simpleCases[i]
+		Markdown(ctx, options)
+		assert.Equal(t, simpleCases[i+1], resp.Body.String())
+		resp.Body.Reset()
+	}
+}
+
+func TestAPI_RenderRaw(t *testing.T) {
+	setting.AppURL = AppURL
+
+	requrl, _ := url.Parse(markdown.URLJoin(AppURL, "api", "v1", "markdown"))
+	req := &http.Request{
+		Method: "POST",
+		URL:    requrl,
+	}
+	m, resp := createContext(req)
+	ctx := wrap(m)
+
+	for i := 0; i < len(simpleCases); i += 2 {
+		ctx.Req.Request.Body = ioutil.NopCloser(strings.NewReader(simpleCases[i]))
+		MarkdownRaw(ctx)
+		assert.Equal(t, simpleCases[i+1], resp.Body.String())
+		resp.Body.Reset()
+	}
+}

routers/repo/wiki.go

@@ -5,7 +5,10 @@
 package repo
 
 import (
+	"fmt"
 	"io/ioutil"
+	"net/url"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -47,16 +50,145 @@ type PageMeta struct {
 	Updated time.Time
 }
 
-func renderWikiPage(ctx *context.Context, isViewPage bool) (*git.Repository, string) {
+func urlEncoded(str string) string {
+	u, err := url.Parse(str)
+	if err != nil {
+		return str
+	}
+	return u.String()
+}
+func urlDecoded(str string) string {
+	res, err := url.QueryUnescape(str)
+	if err != nil {
+		return str
+	}
+	return res
+}
+
+// commitTreeBlobEntry processes found file and checks if it matches search target
+func commitTreeBlobEntry(entry *git.TreeEntry, path string, targets []string, textOnly bool) *git.TreeEntry {
+	name := entry.Name()
+	ext := filepath.Ext(name)
+	if !textOnly || markdown.IsMarkdownFile(name) || ext == ".textile" {
+		for _, target := range targets {
+			if matchName(path, target) || matchName(urlEncoded(path), target) || matchName(urlDecoded(path), target) {
+				return entry
+			}
+			pathNoExt := strings.TrimSuffix(path, ext)
+			if matchName(pathNoExt, target) || matchName(urlEncoded(pathNoExt), target) || matchName(urlDecoded(pathNoExt), target) {
+				return entry
+			}
+		}
+	}
+	return nil
+}
+
+// commitTreeDirEntry is a recursive file tree traversal function
+func commitTreeDirEntry(repo *git.Repository, commit *git.Commit, entries []*git.TreeEntry, prevPath string, targets []string, textOnly bool) (*git.TreeEntry, error) {
+	for i := range entries {
+		entry := entries[i]
+		var path string
+		if len(prevPath) == 0 {
+			path = entry.Name()
+		} else {
+			path = prevPath + "/" + entry.Name()
+		}
+		if entry.Type == git.ObjectBlob {
+			// File
+			if res := commitTreeBlobEntry(entry, path, targets, textOnly); res != nil {
+				return res, nil
+			}
+		} else if entry.IsDir() {
+			// Directory
+			// Get our tree entry, handling all possible errors
+			var err error
+			var tree *git.Tree
+			if tree, err = repo.GetTree(entry.ID.String()); tree == nil || err != nil {
+				if err == nil {
+					err = fmt.Errorf("repo.GetTree(%s) => nil", entry.ID.String())
+				}
+				return nil, err
+			}
+			// Found us, get children entries
+			var ls git.Entries
+			if ls, err = tree.ListEntries(); err != nil {
+				return nil, err
+			}
+			// Call itself recursively to find needed entry
+			var te *git.TreeEntry
+			if te, err = commitTreeDirEntry(repo, commit, ls, path, targets, textOnly); err != nil {
+				return nil, err
+			}
+			if te != nil {
+				return te, nil
+			}
+		}
+	}
+	return nil, nil
+}
+
+// commitTreeEntry is a first step of commitTreeDirEntry, which should be never called directly
+func commitTreeEntry(repo *git.Repository, commit *git.Commit, targets []string, textOnly bool) (*git.TreeEntry, error) {
+	entries, err := commit.ListEntries()
+	if err != nil {
+		return nil, err
+	}
+	return commitTreeDirEntry(repo, commit, entries, "", targets, textOnly)
+}
+
+// findFile finds the best match for given filename in repo file tree
+func findFile(repo *git.Repository, commit *git.Commit, target string, textOnly bool) (*git.TreeEntry, error) {
+	targets := []string{target, urlEncoded(target), urlDecoded(target)}
+	var entry *git.TreeEntry
+	var err error
+	if entry, err = commitTreeEntry(repo, commit, targets, textOnly); err != nil {
+		return nil, err
+	}
+	return entry, nil
+}
+
+// matchName matches generic name representation of the file with required one
+func matchName(target, name string) bool {
+	if len(target) != len(name) {
+		return false
+	}
+	name = strings.ToLower(name)
+	target = strings.ToLower(target)
+	if name == target {
+		return true
+	}
+	target = strings.Replace(target, " ", "?", -1)
+	target = strings.Replace(target, "-", "?", -1)
+	for i := range name {
+		ch := name[i]
+		reqCh := target[i]
+		if ch != reqCh {
+			if string(reqCh) != "?" {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+func findWikiRepoCommit(ctx *context.Context) (*git.Repository, *git.Commit, error) {
 	wikiRepo, err := git.OpenRepository(ctx.Repo.Repository.WikiPath())
 	if err != nil {
-		ctx.Handle(500, "OpenRepository", err)
-		return nil, ""
+		// ctx.Handle(500, "OpenRepository", err)
+		return nil, nil, err
 	}
 	commit, err := wikiRepo.GetBranchCommit("master")
 	if err != nil {
 		ctx.Handle(500, "GetBranchCommit", err)
-		return nil, ""
+		return wikiRepo, nil, err
+	}
+	return wikiRepo, commit, nil
+}
+
+func renderWikiPage(ctx *context.Context, isViewPage bool) (*git.Repository, *git.TreeEntry) {
+	wikiRepo, commit, err := findWikiRepoCommit(ctx)
+	if err != nil {
+		return nil, nil
 	}
 
 	// Get page list.
@@ -64,16 +196,23 @@ func renderWikiPage(ctx *context.Context, isViewPage bool) (*git.Repository, str
 		entries, err := commit.ListEntries()
 		if err != nil {
 			ctx.Handle(500, "ListEntries", err)
-			return nil, ""
+			return nil, nil
 		}
-		pages := make([]PageMeta, 0, len(entries))
+		pages := []PageMeta{}
 		for i := range entries {
-			if entries[i].Type == git.ObjectBlob && strings.HasSuffix(entries[i].Name(), ".md") {
-				name := strings.TrimSuffix(entries[i].Name(), ".md")
-				pages = append(pages, PageMeta{
-					Name: name,
-					URL:  models.ToWikiPageURL(name),
-				})
+			if entries[i].Type == git.ObjectBlob {
+				name := entries[i].Name()
+				ext := filepath.Ext(name)
+				if markdown.IsMarkdownFile(name) || ext == ".textile" {
+					name = strings.TrimSuffix(name, ext)
+					if name == "" || name == "_Sidebar" || name == "_Footer" || name == "_Header" {
+						continue
+					}
+					pages = append(pages, PageMeta{
+						Name: strings.Replace(name, "-", " ", -1),
+						URL:  models.ToWikiPageURL(name),
+					})
+				}
 			}
 		}
 		ctx.Data["Pages"] = pages
@@ -91,35 +230,71 @@ func renderWikiPage(ctx *context.Context, isViewPage bool) (*git.Repository, str
 	ctx.Data["title"] = pageName
 	ctx.Data["RequireHighlightJS"] = true
 
-	blob, err := commit.GetBlobByPath(pageURL + ".md")
-	if err != nil {
-		if git.IsErrNotExist(err) {
-			ctx.Redirect(ctx.Repo.RepoLink + "/wiki/_pages")
-		} else {
-			ctx.Handle(500, "GetBlobByPath", err)
-		}
-		return nil, ""
+	var entry *git.TreeEntry
+	if entry, err = findFile(wikiRepo, commit, pageName, true); err != nil {
+		ctx.Handle(500, "findFile", err)
+		return nil, nil
 	}
+	if entry == nil {
+		ctx.Redirect(ctx.Repo.RepoLink + "/wiki/_pages")
+		return nil, nil
+	}
+	blob := entry.Blob()
 	r, err := blob.Data()
 	if err != nil {
 		ctx.Handle(500, "Data", err)
-		return nil, ""
+		return nil, nil
 	}
 	data, err := ioutil.ReadAll(r)
 	if err != nil {
 		ctx.Handle(500, "ReadAll", err)
-		return nil, ""
+		return nil, nil
+	}
+	sidebarPresent := false
+	sidebarContent := []byte{}
+	sentry, err := findFile(wikiRepo, commit, "_Sidebar", true)
+	if err == nil && sentry != nil {
+		r, err = sentry.Blob().Data()
+		if err == nil {
+			dataSB, err := ioutil.ReadAll(r)
+			if err == nil {
+				sidebarPresent = true
+				sidebarContent = dataSB
+			}
+		}
+	}
+	footerPresent := false
+	footerContent := []byte{}
+	sentry, err = findFile(wikiRepo, commit, "_Footer", true)
+	if err == nil && sentry != nil {
+		r, err = sentry.Blob().Data()
+		if err == nil {
+			dataSB, err := ioutil.ReadAll(r)
+			if err == nil {
+				footerPresent = true
+				footerContent = dataSB
+			}
+		}
 	}
 	if isViewPage {
-		ctx.Data["content"] = string(markdown.Render(data, ctx.Repo.RepoLink, ctx.Repo.Repository.ComposeMetas()))
+		metas := ctx.Repo.Repository.ComposeMetas()
+		ctx.Data["content"] = markdown.RenderWiki(data, ctx.Repo.RepoLink, metas)
+		ctx.Data["sidebarPresent"] = sidebarPresent
+		ctx.Data["sidebarContent"] = markdown.RenderWiki(sidebarContent, ctx.Repo.RepoLink, metas)
+		ctx.Data["footerPresent"] = footerPresent
+		ctx.Data["footerContent"] = markdown.RenderWiki(footerContent, ctx.Repo.RepoLink, metas)
 	} else {
 		ctx.Data["content"] = string(data)
+		ctx.Data["sidebarPresent"] = false
+		ctx.Data["sidebarContent"] = ""
+		ctx.Data["footerPresent"] = false
+		ctx.Data["footerContent"] = ""
 	}
 
-	return wikiRepo, pageURL
+	return wikiRepo, entry
 }
 
-// Wiki render wiki page
+// Wiki renders single wiki page
 func Wiki(ctx *context.Context) {
 	ctx.Data["PageIsWiki"] = true
 
@@ -129,13 +304,18 @@ func Wiki(ctx *context.Context) {
 		return
 	}
 
-	wikiRepo, pagePath := renderWikiPage(ctx, true)
+	wikiRepo, entry := renderWikiPage(ctx, true)
 	if ctx.Written() {
 		return
 	}
 
+	ename := entry.Name()
+	if !markdown.IsMarkdownFile(ename) {
+		ext := strings.ToUpper(filepath.Ext(ename))
+		ctx.Data["FormatWarning"] = fmt.Sprintf("%s rendering is not supported at the moment. Rendered as Markdown.", ext)
+	}
 	// Get last change information.
-	lastCommit, err := wikiRepo.GetCommitByPath(pagePath + ".md")
+	lastCommit, err := wikiRepo.GetCommitByPath(ename)
 	if err != nil {
 		ctx.Handle(500, "GetCommitByPath", err)
 		return
@@ -155,14 +335,8 @@ func WikiPages(ctx *context.Context) {
 		return
 	}
 
-	wikiRepo, err := git.OpenRepository(ctx.Repo.Repository.WikiPath())
+	wikiRepo, commit, err := findWikiRepoCommit(ctx)
 	if err != nil {
-		ctx.Handle(500, "OpenRepository", err)
-		return
-	}
-	commit, err := wikiRepo.GetBranchCommit("master")
-	if err != nil {
-		ctx.Handle(500, "GetBranchCommit", err)
 		return
 	}
 
@@ -173,18 +347,25 @@ func WikiPages(ctx *context.Context) {
 	}
 	pages := make([]PageMeta, 0, len(entries))
 	for i := range entries {
-		if entries[i].Type == git.ObjectBlob && strings.HasSuffix(entries[i].Name(), ".md") {
+		if entries[i].Type == git.ObjectBlob {
 			c, err := wikiRepo.GetCommitByPath(entries[i].Name())
 			if err != nil {
 				ctx.Handle(500, "GetCommit", err)
 				return
 			}
-			name := strings.TrimSuffix(entries[i].Name(), ".md")
-			pages = append(pages, PageMeta{
-				Name:    name,
-				URL:     models.ToWikiPageURL(name),
-				Updated: c.Author.When,
-			})
+			name := entries[i].Name()
+			ext := filepath.Ext(name)
+			if markdown.IsMarkdownFile(name) || ext == ".textile" {
+				name = strings.TrimSuffix(name, ext)
+				if name == "" {
+					continue
+				}
+				pages = append(pages, PageMeta{
+					Name:    name,
+					URL:     models.ToWikiPageURL(name),
+					Updated: c.Author.When,
+				})
+			}
 		}
 	}
 	ctx.Data["Pages"] = pages
@@ -192,6 +373,44 @@ func WikiPages(ctx *context.Context) {
 	ctx.HTML(200, tplWikiPages)
 }
 
+// WikiRaw outputs raw blob requested by user (image for example)
+func WikiRaw(ctx *context.Context) {
+	wikiRepo, commit, err := findWikiRepoCommit(ctx)
+	if err != nil {
+		if wikiRepo != nil {
+			return
+		}
+	}
+	uri := ctx.Params("*")
+	var entry *git.TreeEntry
+	if commit != nil {
+		entry, err = findFile(wikiRepo, commit, uri, false)
+	}
+	if err != nil || entry == nil {
+		if entry == nil || commit == nil {
+			defBranch := ctx.Repo.Repository.DefaultBranch
+			if commit, err = ctx.Repo.GitRepo.GetBranchCommit(defBranch); commit == nil || err != nil {
+				ctx.Handle(500, "GetBranchCommit", err)
+				return
+			}
+			if entry, err = findFile(ctx.Repo.GitRepo, commit, uri, false); err != nil {
+				ctx.Handle(500, "findFile", err)
+				return
+			}
+			if entry == nil {
+				ctx.Handle(404, "findFile", nil)
+				return
+			}
+		} else {
+			ctx.Handle(500, "findFile", err)
+			return
+		}
+	}
+	if err = ServeBlob(ctx, entry.Blob()); err != nil {
+		ctx.Handle(500, "ServeBlob", err)
+	}
+}
+
 // NewWiki render wiki create page
 func NewWiki(ctx *context.Context) {
 	ctx.Data["Title"] = ctx.Tr("repo.wiki.new_page")