mirror of
https://github.com/go-gitea/gitea
synced 2024-12-23 00:54:28 +00:00
Add sitemap support (#18407)
This commit is contained in:
parent
97bfabc745
commit
95383b7a16
@ -1097,6 +1097,9 @@ PATH =
|
|||||||
;; Number of items that are displayed in home feed
|
;; Number of items that are displayed in home feed
|
||||||
;FEED_PAGING_NUM = 20
|
;FEED_PAGING_NUM = 20
|
||||||
;;
|
;;
|
||||||
|
;; Number of items that are displayed in a single subsitemap
|
||||||
|
;SITEMAP_PAGING_NUM = 20
|
||||||
|
;;
|
||||||
;; Number of maximum commits displayed in commit graph.
|
;; Number of maximum commits displayed in commit graph.
|
||||||
;GRAPH_MAX_COMMIT_NUM = 100
|
;GRAPH_MAX_COMMIT_NUM = 100
|
||||||
;;
|
;;
|
||||||
|
@ -174,6 +174,7 @@ The following configuration set `Content-Type: application/vnd.android.package-a
|
|||||||
- `MEMBERS_PAGING_NUM`: **20**: Number of members that are shown in organization members.
|
- `MEMBERS_PAGING_NUM`: **20**: Number of members that are shown in organization members.
|
||||||
- `FEED_MAX_COMMIT_NUM`: **5**: Number of maximum commits shown in one activity feed.
|
- `FEED_MAX_COMMIT_NUM`: **5**: Number of maximum commits shown in one activity feed.
|
||||||
- `FEED_PAGING_NUM`: **20**: Number of items that are displayed in home feed.
|
- `FEED_PAGING_NUM`: **20**: Number of items that are displayed in home feed.
|
||||||
|
- `SITEMAP_PAGING_NUM`: **20**: Number of items that are displayed in a single subsitemap.
|
||||||
- `GRAPH_MAX_COMMIT_NUM`: **100**: Number of maximum commits shown in the commit graph.
|
- `GRAPH_MAX_COMMIT_NUM`: **100**: Number of maximum commits shown in the commit graph.
|
||||||
- `CODE_COMMENT_LINES`: **4**: Number of line of codes shown for a code comment.
|
- `CODE_COMMENT_LINES`: **4**: Number of line of codes shown for a code comment.
|
||||||
- `DEFAULT_THEME`: **auto**: \[auto, gitea, arc-green\]: Set the default theme for the Gitea install.
|
- `DEFAULT_THEME`: **auto**: \[auto, gitea, arc-green\]: Set the default theme for the Gitea install.
|
||||||
|
@ -207,6 +207,7 @@ var (
|
|||||||
// UI settings
|
// UI settings
|
||||||
UI = struct {
|
UI = struct {
|
||||||
ExplorePagingNum int
|
ExplorePagingNum int
|
||||||
|
SitemapPagingNum int
|
||||||
IssuePagingNum int
|
IssuePagingNum int
|
||||||
RepoSearchPagingNum int
|
RepoSearchPagingNum int
|
||||||
MembersPagingNum int
|
MembersPagingNum int
|
||||||
@ -260,6 +261,7 @@ var (
|
|||||||
} `ini:"ui.meta"`
|
} `ini:"ui.meta"`
|
||||||
}{
|
}{
|
||||||
ExplorePagingNum: 20,
|
ExplorePagingNum: 20,
|
||||||
|
SitemapPagingNum: 20,
|
||||||
IssuePagingNum: 10,
|
IssuePagingNum: 10,
|
||||||
RepoSearchPagingNum: 10,
|
RepoSearchPagingNum: 10,
|
||||||
MembersPagingNum: 20,
|
MembersPagingNum: 20,
|
||||||
|
69
modules/sitemap/sitemap.go
Normal file
69
modules/sitemap/sitemap.go
Normal file
@ -0,0 +1,69 @@
|
|||||||
|
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package sitemap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/xml"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// sitemapFileLimit contains the maximum size of a sitemap file
|
||||||
|
const sitemapFileLimit = 50 * 1024 * 1024
|
||||||
|
|
||||||
|
// Url represents a single sitemap entry
|
||||||
|
type URL struct {
|
||||||
|
URL string `xml:"loc"`
|
||||||
|
LastMod *time.Time `xml:"lastmod,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// SitemapUrl represents a sitemap
|
||||||
|
type Sitemap struct {
|
||||||
|
XMLName xml.Name
|
||||||
|
Namespace string `xml:"xmlns,attr"`
|
||||||
|
|
||||||
|
URLs []URL `xml:"url"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewSitemap creates a sitemap
|
||||||
|
func NewSitemap() *Sitemap {
|
||||||
|
return &Sitemap{
|
||||||
|
XMLName: xml.Name{Local: "urlset"},
|
||||||
|
Namespace: "http://www.sitemaps.org/schemas/sitemap/0.9",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewSitemap creates a sitemap index.
|
||||||
|
func NewSitemapIndex() *Sitemap {
|
||||||
|
return &Sitemap{
|
||||||
|
XMLName: xml.Name{Local: "sitemapindex"},
|
||||||
|
Namespace: "http://www.sitemaps.org/schemas/sitemap/0.9",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add adds a URL to the sitemap
|
||||||
|
func (s *Sitemap) Add(u URL) {
|
||||||
|
s.URLs = append(s.URLs, u)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write writes the sitemap to a response
|
||||||
|
func (s *Sitemap) WriteTo(w io.Writer) (int64, error) {
|
||||||
|
if len(s.URLs) > 50000 {
|
||||||
|
return 0, fmt.Errorf("The sitemap contains too many URLs: %d", len(s.URLs))
|
||||||
|
}
|
||||||
|
buf := bytes.NewBufferString(xml.Header)
|
||||||
|
if err := xml.NewEncoder(buf).Encode(s); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if err := buf.WriteByte('\n'); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if buf.Len() > sitemapFileLimit {
|
||||||
|
return 0, fmt.Errorf("The sitemap is too big: %d", buf.Len())
|
||||||
|
}
|
||||||
|
return buf.WriteTo(w)
|
||||||
|
}
|
77
modules/sitemap/sitemap_test.go
Normal file
77
modules/sitemap/sitemap_test.go
Normal file
@ -0,0 +1,77 @@
|
|||||||
|
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package sitemap
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/xml"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestOk(t *testing.T) {
|
||||||
|
testReal := func(s *Sitemap, name string, urls []URL, expected string) {
|
||||||
|
for _, url := range urls {
|
||||||
|
s.Add(url)
|
||||||
|
}
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
_, err := s.WriteTo(buf)
|
||||||
|
assert.NoError(t, nil, err)
|
||||||
|
assert.Equal(t, xml.Header+"<"+name+" xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">"+expected+"</"+name+">\n", buf.String())
|
||||||
|
}
|
||||||
|
test := func(urls []URL, expected string) {
|
||||||
|
testReal(NewSitemap(), "urlset", urls, expected)
|
||||||
|
testReal(NewSitemapIndex(), "sitemapindex", urls, expected)
|
||||||
|
}
|
||||||
|
|
||||||
|
ts := time.Unix(1651322008, 0).UTC()
|
||||||
|
|
||||||
|
test(
|
||||||
|
[]URL{},
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
test(
|
||||||
|
[]URL{
|
||||||
|
{URL: "https://gitea.io/test1", LastMod: &ts},
|
||||||
|
},
|
||||||
|
"<url><loc>https://gitea.io/test1</loc><lastmod>2022-04-30T12:33:28Z</lastmod></url>",
|
||||||
|
)
|
||||||
|
test(
|
||||||
|
[]URL{
|
||||||
|
{URL: "https://gitea.io/test2", LastMod: nil},
|
||||||
|
},
|
||||||
|
"<url><loc>https://gitea.io/test2</loc></url>",
|
||||||
|
)
|
||||||
|
test(
|
||||||
|
[]URL{
|
||||||
|
{URL: "https://gitea.io/test1", LastMod: &ts},
|
||||||
|
{URL: "https://gitea.io/test2", LastMod: nil},
|
||||||
|
},
|
||||||
|
"<url><loc>https://gitea.io/test1</loc><lastmod>2022-04-30T12:33:28Z</lastmod></url>"+
|
||||||
|
"<url><loc>https://gitea.io/test2</loc></url>",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTooManyURLs(t *testing.T) {
|
||||||
|
s := NewSitemap()
|
||||||
|
for i := 0; i < 50001; i++ {
|
||||||
|
s.Add(URL{URL: fmt.Sprintf("https://gitea.io/test%d", i)})
|
||||||
|
}
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
_, err := s.WriteTo(buf)
|
||||||
|
assert.EqualError(t, err, "The sitemap contains too many URLs: 50001")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSitemapTooBig(t *testing.T) {
|
||||||
|
s := NewSitemap()
|
||||||
|
s.Add(URL{URL: strings.Repeat("b", sitemapFileLimit)})
|
||||||
|
buf := &bytes.Buffer{}
|
||||||
|
_, err := s.WriteTo(buf)
|
||||||
|
assert.EqualError(t, err, "The sitemap is too big: 52428931")
|
||||||
|
}
|
@ -11,7 +11,9 @@ import (
|
|||||||
repo_model "code.gitea.io/gitea/models/repo"
|
repo_model "code.gitea.io/gitea/models/repo"
|
||||||
"code.gitea.io/gitea/modules/base"
|
"code.gitea.io/gitea/modules/base"
|
||||||
"code.gitea.io/gitea/modules/context"
|
"code.gitea.io/gitea/modules/context"
|
||||||
|
"code.gitea.io/gitea/modules/log"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
"code.gitea.io/gitea/modules/sitemap"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -30,11 +32,21 @@ type RepoSearchOptions struct {
|
|||||||
|
|
||||||
// RenderRepoSearch render repositories search page
|
// RenderRepoSearch render repositories search page
|
||||||
func RenderRepoSearch(ctx *context.Context, opts *RepoSearchOptions) {
|
func RenderRepoSearch(ctx *context.Context, opts *RepoSearchOptions) {
|
||||||
page := ctx.FormInt("page")
|
// Sitemap index for sitemap paths
|
||||||
|
page := int(ctx.ParamsInt64("idx"))
|
||||||
|
isSitemap := ctx.Params("idx") != ""
|
||||||
|
if page <= 1 {
|
||||||
|
page = ctx.FormInt("page")
|
||||||
|
}
|
||||||
|
|
||||||
if page <= 0 {
|
if page <= 0 {
|
||||||
page = 1
|
page = 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if isSitemap {
|
||||||
|
opts.PageSize = setting.UI.SitemapPagingNum
|
||||||
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
repos []*repo_model.Repository
|
repos []*repo_model.Repository
|
||||||
count int64
|
count int64
|
||||||
@ -100,6 +112,18 @@ func RenderRepoSearch(ctx *context.Context, opts *RepoSearchOptions) {
|
|||||||
ctx.ServerError("SearchRepository", err)
|
ctx.ServerError("SearchRepository", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if isSitemap {
|
||||||
|
m := sitemap.NewSitemap()
|
||||||
|
for _, item := range repos {
|
||||||
|
m.Add(sitemap.URL{URL: item.HTMLURL(), LastMod: item.UpdatedUnix.AsTimePtr()})
|
||||||
|
}
|
||||||
|
ctx.Resp.Header().Set("Content-Type", "text/xml")
|
||||||
|
if _, err := m.WriteTo(ctx.Resp); err != nil {
|
||||||
|
log.Error("Failed writing sitemap: %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
ctx.Data["Keyword"] = keyword
|
ctx.Data["Keyword"] = keyword
|
||||||
ctx.Data["Total"] = count
|
ctx.Data["Total"] = count
|
||||||
ctx.Data["Repos"] = repos
|
ctx.Data["Repos"] = repos
|
||||||
|
@ -12,7 +12,9 @@ import (
|
|||||||
user_model "code.gitea.io/gitea/models/user"
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
"code.gitea.io/gitea/modules/base"
|
"code.gitea.io/gitea/modules/base"
|
||||||
"code.gitea.io/gitea/modules/context"
|
"code.gitea.io/gitea/modules/context"
|
||||||
|
"code.gitea.io/gitea/modules/log"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
"code.gitea.io/gitea/modules/sitemap"
|
||||||
"code.gitea.io/gitea/modules/structs"
|
"code.gitea.io/gitea/modules/structs"
|
||||||
"code.gitea.io/gitea/modules/util"
|
"code.gitea.io/gitea/modules/util"
|
||||||
)
|
)
|
||||||
@ -33,11 +35,20 @@ func isKeywordValid(keyword string) bool {
|
|||||||
|
|
||||||
// RenderUserSearch render user search page
|
// RenderUserSearch render user search page
|
||||||
func RenderUserSearch(ctx *context.Context, opts *user_model.SearchUserOptions, tplName base.TplName) {
|
func RenderUserSearch(ctx *context.Context, opts *user_model.SearchUserOptions, tplName base.TplName) {
|
||||||
opts.Page = ctx.FormInt("page")
|
// Sitemap index for sitemap paths
|
||||||
|
opts.Page = int(ctx.ParamsInt64("idx"))
|
||||||
|
isSitemap := ctx.Params("idx") != ""
|
||||||
|
if opts.Page <= 1 {
|
||||||
|
opts.Page = ctx.FormInt("page")
|
||||||
|
}
|
||||||
if opts.Page <= 1 {
|
if opts.Page <= 1 {
|
||||||
opts.Page = 1
|
opts.Page = 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if isSitemap {
|
||||||
|
opts.PageSize = setting.UI.SitemapPagingNum
|
||||||
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
users []*user_model.User
|
users []*user_model.User
|
||||||
count int64
|
count int64
|
||||||
@ -73,6 +84,18 @@ func RenderUserSearch(ctx *context.Context, opts *user_model.SearchUserOptions,
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if isSitemap {
|
||||||
|
m := sitemap.NewSitemap()
|
||||||
|
for _, item := range users {
|
||||||
|
m.Add(sitemap.URL{URL: item.HTMLURL(), LastMod: item.UpdatedUnix.AsTimePtr()})
|
||||||
|
}
|
||||||
|
ctx.Resp.Header().Set("Content-Type", "text/xml")
|
||||||
|
if _, err := m.WriteTo(ctx.Resp); err != nil {
|
||||||
|
log.Error("Failed writing sitemap: %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
ctx.Data["Keyword"] = opts.Keyword
|
ctx.Data["Keyword"] = opts.Keyword
|
||||||
ctx.Data["Total"] = count
|
ctx.Data["Total"] = count
|
||||||
ctx.Data["Users"] = users
|
ctx.Data["Users"] = users
|
||||||
|
@ -7,11 +7,18 @@ package web
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
|
repo_model "code.gitea.io/gitea/models/repo"
|
||||||
|
user_model "code.gitea.io/gitea/models/user"
|
||||||
"code.gitea.io/gitea/modules/base"
|
"code.gitea.io/gitea/modules/base"
|
||||||
"code.gitea.io/gitea/modules/context"
|
"code.gitea.io/gitea/modules/context"
|
||||||
"code.gitea.io/gitea/modules/log"
|
"code.gitea.io/gitea/modules/log"
|
||||||
"code.gitea.io/gitea/modules/setting"
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
"code.gitea.io/gitea/modules/sitemap"
|
||||||
|
"code.gitea.io/gitea/modules/structs"
|
||||||
|
"code.gitea.io/gitea/modules/util"
|
||||||
"code.gitea.io/gitea/modules/web/middleware"
|
"code.gitea.io/gitea/modules/web/middleware"
|
||||||
"code.gitea.io/gitea/routers/web/auth"
|
"code.gitea.io/gitea/routers/web/auth"
|
||||||
"code.gitea.io/gitea/routers/web/user"
|
"code.gitea.io/gitea/routers/web/user"
|
||||||
@ -59,6 +66,52 @@ func Home(ctx *context.Context) {
|
|||||||
ctx.HTML(http.StatusOK, tplHome)
|
ctx.HTML(http.StatusOK, tplHome)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// HomeSitemap renders the main sitemap
|
||||||
|
func HomeSitemap(ctx *context.Context) {
|
||||||
|
m := sitemap.NewSitemapIndex()
|
||||||
|
if !setting.Service.Explore.DisableUsersPage {
|
||||||
|
_, cnt, err := user_model.SearchUsers(&user_model.SearchUserOptions{
|
||||||
|
Type: user_model.UserTypeIndividual,
|
||||||
|
ListOptions: db.ListOptions{PageSize: 1},
|
||||||
|
IsActive: util.OptionalBoolTrue,
|
||||||
|
Visible: []structs.VisibleType{structs.VisibleTypePublic},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
ctx.ServerError("SearchUsers", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
count := int(cnt)
|
||||||
|
idx := 1
|
||||||
|
for i := 0; i < count; i += setting.UI.SitemapPagingNum {
|
||||||
|
m.Add(sitemap.URL{URL: setting.AppURL + "explore/users/sitemap-" + strconv.Itoa(idx) + ".xml"})
|
||||||
|
idx++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_, cnt, err := repo_model.SearchRepository(&repo_model.SearchRepoOptions{
|
||||||
|
ListOptions: db.ListOptions{
|
||||||
|
PageSize: 1,
|
||||||
|
},
|
||||||
|
Actor: ctx.Doer,
|
||||||
|
AllPublic: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
ctx.ServerError("SearchRepository", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
count := int(cnt)
|
||||||
|
idx := 1
|
||||||
|
for i := 0; i < count; i += setting.UI.SitemapPagingNum {
|
||||||
|
m.Add(sitemap.URL{URL: setting.AppURL + "explore/repos/sitemap-" + strconv.Itoa(idx) + ".xml"})
|
||||||
|
idx++
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.Resp.Header().Set("Content-Type", "text/xml")
|
||||||
|
if _, err := m.WriteTo(ctx.Resp); err != nil {
|
||||||
|
log.Error("Failed writing sitemap: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// NotFound render 404 page
|
// NotFound render 404 page
|
||||||
func NotFound(ctx *context.Context) {
|
func NotFound(ctx *context.Context) {
|
||||||
ctx.Data["Title"] = "Page Not Found"
|
ctx.Data["Title"] = "Page Not Found"
|
||||||
|
@ -294,6 +294,7 @@ func RegisterRoutes(m *web.Route) {
|
|||||||
// Routers.
|
// Routers.
|
||||||
// for health check
|
// for health check
|
||||||
m.Get("/", Home)
|
m.Get("/", Home)
|
||||||
|
m.Get("/sitemap.xml", ignExploreSignIn, HomeSitemap)
|
||||||
m.Group("/.well-known", func() {
|
m.Group("/.well-known", func() {
|
||||||
m.Get("/openid-configuration", auth.OIDCWellKnown)
|
m.Get("/openid-configuration", auth.OIDCWellKnown)
|
||||||
m.Group("", func() {
|
m.Group("", func() {
|
||||||
@ -310,7 +311,9 @@ func RegisterRoutes(m *web.Route) {
|
|||||||
ctx.Redirect(setting.AppSubURL + "/explore/repos")
|
ctx.Redirect(setting.AppSubURL + "/explore/repos")
|
||||||
})
|
})
|
||||||
m.Get("/repos", explore.Repos)
|
m.Get("/repos", explore.Repos)
|
||||||
|
m.Get("/repos/sitemap-{idx}.xml", explore.Repos)
|
||||||
m.Get("/users", explore.Users)
|
m.Get("/users", explore.Users)
|
||||||
|
m.Get("/users/sitemap-{idx}.xml", explore.Users)
|
||||||
m.Get("/organizations", explore.Organizations)
|
m.Get("/organizations", explore.Organizations)
|
||||||
m.Get("/code", explore.Code)
|
m.Get("/code", explore.Code)
|
||||||
m.Get("/topics/search", explore.TopicSearch)
|
m.Get("/topics/search", explore.TopicSearch)
|
||||||
|
Loading…
Reference in New Issue
Block a user