From eca05b09aa269dda1309ee77ac750e29e71c3fd3 Mon Sep 17 00:00:00 2001 From: Lauris BH Date: Thu, 26 Oct 2017 04:37:33 +0300 Subject: [PATCH] Add commit count caching (#2774) * Add commit count caching * Small refactoring * Add different key prefix for refs and commits * Add configuratuion option to allow to change caching time or disable it --- conf/app.ini | 3 ++ models/repo.go | 11 ++++++ models/update.go | 13 +++++-- modules/cache/cache.go | 72 ++++++++++++++++++++++++++++++++++++++ modules/context/repo.go | 21 +++++++++-- modules/setting/setting.go | 38 +++++++++++++------- routers/admin/admin.go | 6 ++-- routers/init.go | 3 ++ routers/repo/commit.go | 4 +-- routers/routes/routes.go | 10 +++--- 10 files changed, 153 insertions(+), 28 deletions(-) create mode 100644 modules/cache/cache.go diff --git a/conf/app.ini b/conf/app.ini index 0ce8aae52e..3a850106c1 100644 --- a/conf/app.ini +++ b/conf/app.ini @@ -339,6 +339,9 @@ INTERVAL = 60 ; redis: network=tcp,addr=:6379,password=macaron,db=0,pool_size=100,idle_timeout=180 ; memcache: `127.0.0.1:11211` HOST = +; Time to keep items in cache if not used, default is 16 hours. +; Setting it to 0 disables caching +ITEM_TTL = 16h [session] ; Either "memory", "file", or "redis", default is "memory" diff --git a/models/repo.go b/models/repo.go index 1b1be62f87..eca71568ee 100644 --- a/models/repo.go +++ b/models/repo.go @@ -258,6 +258,17 @@ func (repo *Repository) APIFormat(mode AccessMode) *api.Repository { return repo.innerAPIFormat(mode, false) } +// GetCommitsCountCacheKey returns cache key used for commits count caching. +func (repo *Repository) GetCommitsCountCacheKey(contextName string, isRef bool) string { + var prefix string + if isRef { + prefix = "ref" + } else { + prefix = "commit" + } + return fmt.Sprintf("commits-count-%d-%s-%s", repo.ID, prefix, contextName) +} + func (repo *Repository) innerAPIFormat(mode AccessMode, isParent bool) *api.Repository { var parent *api.Repository diff --git a/models/update.go b/models/update.go index 62d13ce209..82369bf636 100644 --- a/models/update.go +++ b/models/update.go @@ -11,7 +11,7 @@ import ( "strings" "code.gitea.io/git" - + "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/log" ) @@ -205,19 +205,26 @@ func pushUpdate(opts PushUpdateOptions) (repo *Repository, err error) { var commits = &PushCommits{} if strings.HasPrefix(opts.RefFullName, git.TagPrefix) { // If is tag reference + tagName := opts.RefFullName[len(git.TagPrefix):] if isDelRef { - err = pushUpdateDeleteTag(repo, gitRepo, opts.RefFullName[len(git.TagPrefix):]) + err = pushUpdateDeleteTag(repo, gitRepo, tagName) if err != nil { return nil, fmt.Errorf("pushUpdateDeleteTag: %v", err) } } else { - err = pushUpdateAddTag(repo, gitRepo, opts.RefFullName[len(git.TagPrefix):]) + // Clear cache for tag commit count + cache.Remove(repo.GetCommitsCountCacheKey(tagName, true)) + err = pushUpdateAddTag(repo, gitRepo, tagName) if err != nil { return nil, fmt.Errorf("pushUpdateAddTag: %v", err) } } } else if !isDelRef { // If is branch reference + + // Clear cache for branch commit count + cache.Remove(repo.GetCommitsCountCacheKey(opts.RefFullName[len(git.BranchPrefix):], true)) + newCommit, err := gitRepo.GetCommit(opts.NewCommitID) if err != nil { return nil, fmt.Errorf("gitRepo.GetCommit: %v", err) diff --git a/modules/cache/cache.go b/modules/cache/cache.go new file mode 100644 index 0000000000..0a73ae8ae9 --- /dev/null +++ b/modules/cache/cache.go @@ -0,0 +1,72 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package cache + +import ( + "code.gitea.io/gitea/modules/setting" + + mc "github.com/go-macaron/cache" +) + +var conn mc.Cache + +// NewContext start cache service +func NewContext() error { + if setting.CacheService == nil || conn != nil { + return nil + } + + var err error + conn, err = mc.NewCacher(setting.CacheService.Adapter, mc.Options{ + Adapter: setting.CacheService.Adapter, + AdapterConfig: setting.CacheService.Conn, + Interval: setting.CacheService.Interval, + }) + return err +} + +// GetInt returns key value from cache with callback when no key exists in cache +func GetInt(key string, getFunc func() (int, error)) (int, error) { + if conn == nil || setting.CacheService.TTL == 0 { + return getFunc() + } + if !conn.IsExist(key) { + var ( + value int + err error + ) + if value, err = getFunc(); err != nil { + return value, err + } + conn.Put(key, value, int64(setting.CacheService.TTL.Seconds())) + } + return conn.Get(key).(int), nil +} + +// GetInt64 returns key value from cache with callback when no key exists in cache +func GetInt64(key string, getFunc func() (int64, error)) (int64, error) { + if conn == nil || setting.CacheService.TTL == 0 { + return getFunc() + } + if !conn.IsExist(key) { + var ( + value int64 + err error + ) + if value, err = getFunc(); err != nil { + return value, err + } + conn.Put(key, value, int64(setting.CacheService.TTL.Seconds())) + } + return conn.Get(key).(int64), nil +} + +// Remove key from cache +func Remove(key string) { + if conn == nil { + return + } + conn.Delete(key) +} diff --git a/modules/context/repo.go b/modules/context/repo.go index c33396c0f9..3aaf1ce64a 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -13,7 +13,9 @@ import ( "code.gitea.io/git" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/setting" + "github.com/Unknwon/com" "gopkg.in/editorconfig/editorconfig-core-go.v1" "gopkg.in/macaron.v1" @@ -100,6 +102,21 @@ func (r *Repository) CanUseTimetracker(issue *models.Issue, user *models.User) b r.IsWriter() || issue.IsPoster(user.ID) || issue.AssigneeID == user.ID) } +// GetCommitsCount returns cached commit count for current view +func (r *Repository) GetCommitsCount() (int64, error) { + var contextName string + if r.IsViewBranch { + contextName = r.BranchName + } else if r.IsViewTag { + contextName = r.TagName + } else { + contextName = r.CommitID + } + return cache.GetInt64(r.Repository.GetCommitsCountCacheKey(contextName, r.IsViewBranch || r.IsViewTag), func() (int64, error) { + return r.Commit.CommitsCount() + }) +} + // GetEditorconfig returns the .editorconfig definition if found in the // HEAD of the default repo branch. func (r *Repository) GetEditorconfig() (*editorconfig.Editorconfig, error) { @@ -535,9 +552,9 @@ func RepoRef() macaron.Handler { ctx.Data["IsViewCommit"] = ctx.Repo.IsViewCommit ctx.Data["CanCreateBranch"] = ctx.Repo.CanCreateBranch() - ctx.Repo.CommitsCount, err = ctx.Repo.Commit.CommitsCount() + ctx.Repo.CommitsCount, err = ctx.Repo.GetCommitsCount() if err != nil { - ctx.Handle(500, "CommitsCount", err) + ctx.Handle(500, "GetCommitsCount", err) return } ctx.Data["CommitsCount"] = ctx.Repo.CommitsCount diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 9787e09280..6c89381f3b 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -325,11 +325,6 @@ var ( // Time settings TimeFormat string - // Cache settings - CacheAdapter string - CacheInterval int - CacheConn string - // Session settings SessionConfig session.Options CSRFCookieName = "_csrf" @@ -1295,16 +1290,33 @@ func NewXORMLogService(disableConsole bool) { } } +// Cache represents cache settings +type Cache struct { + Adapter string + Interval int + Conn string + TTL time.Duration +} + +var ( + // CacheService the global cache + CacheService *Cache +) + func newCacheService() { - CacheAdapter = Cfg.Section("cache").Key("ADAPTER").In("memory", []string{"memory", "redis", "memcache"}) - switch CacheAdapter { - case "memory": - CacheInterval = Cfg.Section("cache").Key("INTERVAL").MustInt(60) - case "redis", "memcache": - CacheConn = strings.Trim(Cfg.Section("cache").Key("HOST").String(), "\" ") - default: - log.Fatal(4, "Unknown cache adapter: %s", CacheAdapter) + sec := Cfg.Section("cache") + CacheService = &Cache{ + Adapter: sec.Key("ADAPTER").In("memory", []string{"memory", "redis", "memcache"}), } + switch CacheService.Adapter { + case "memory": + CacheService.Interval = sec.Key("INTERVAL").MustInt(60) + case "redis", "memcache": + CacheService.Conn = strings.Trim(sec.Key("HOST").String(), "\" ") + default: + log.Fatal(4, "Unknown cache adapter: %s", CacheService.Adapter) + } + CacheService.TTL = sec.Key("ITEM_TTL").MustDuration(16 * time.Hour) log.Info("Cache Service Enabled") } diff --git a/routers/admin/admin.go b/routers/admin/admin.go index 94b88a05c3..39a8f718ca 100644 --- a/routers/admin/admin.go +++ b/routers/admin/admin.go @@ -224,9 +224,9 @@ func Config(ctx *context.Context) { ctx.Data["Mailer"] = setting.MailService } - ctx.Data["CacheAdapter"] = setting.CacheAdapter - ctx.Data["CacheInterval"] = setting.CacheInterval - ctx.Data["CacheConn"] = setting.CacheConn + ctx.Data["CacheAdapter"] = setting.CacheService.Adapter + ctx.Data["CacheInterval"] = setting.CacheService.Interval + ctx.Data["CacheConn"] = setting.CacheService.Conn ctx.Data["SessionConfig"] = setting.SessionConfig diff --git a/routers/init.go b/routers/init.go index 006f285266..8cfbe39ee5 100644 --- a/routers/init.go +++ b/routers/init.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/git" "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/migrations" + "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/cron" "code.gitea.io/gitea/modules/highlight" "code.gitea.io/gitea/modules/log" @@ -18,6 +19,7 @@ import ( "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/ssh" + macaron "gopkg.in/macaron.v1" ) @@ -37,6 +39,7 @@ func checkRunMode() { func NewServices() { setting.NewServices() mailer.NewContext() + cache.NewContext() } // GlobalInit is for global configuration reload-able. diff --git a/routers/repo/commit.go b/routers/repo/commit.go index 21a0d9dd9f..637a50543a 100644 --- a/routers/repo/commit.go +++ b/routers/repo/commit.go @@ -55,7 +55,7 @@ func Commits(ctx *context.Context) { } ctx.Data["PageIsViewCode"] = true - commitsCount, err := ctx.Repo.Commit.CommitsCount() + commitsCount, err := ctx.Repo.GetCommitsCount() if err != nil { ctx.Handle(500, "GetCommitsCount", err) return @@ -91,7 +91,7 @@ func Graph(ctx *context.Context) { ctx.Data["PageIsCommits"] = true ctx.Data["PageIsViewCode"] = true - commitsCount, err := ctx.Repo.Commit.CommitsCount() + commitsCount, err := ctx.Repo.GetCommitsCount() if err != nil { ctx.Handle(500, "GetCommitsCount", err) return diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 227b4fff9c..703afbb4a7 100644 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -99,9 +99,9 @@ func NewMacaron() *macaron.Macaron { Redirect: true, })) m.Use(cache.Cacher(cache.Options{ - Adapter: setting.CacheAdapter, - AdapterConfig: setting.CacheConn, - Interval: setting.CacheInterval, + Adapter: setting.CacheService.Adapter, + AdapterConfig: setting.CacheService.Conn, + Interval: setting.CacheService.Interval, })) m.Use(captcha.Captchaer(captcha.Options{ SubURL: setting.AppSubURL, @@ -576,9 +576,9 @@ func RegisterRoutes(m *macaron.Macaron) { ctx.Handle(500, "GetBranchCommit", err) return } - ctx.Repo.CommitsCount, err = ctx.Repo.Commit.CommitsCount() + ctx.Repo.CommitsCount, err = ctx.Repo.GetCommitsCount() if err != nil { - ctx.Handle(500, "CommitsCount", err) + ctx.Handle(500, "GetCommitsCount", err) return } ctx.Data["CommitsCount"] = ctx.Repo.CommitsCount