1
1
mirror of https://github.com/go-gitea/gitea synced 2024-11-05 01:34:25 +00:00
gitea/services/cron/tasks.go
Giteabot 6f4d5c0b8c
Recover from panic in cron task (#28409) (#28425)
Backport #28409 by @earl-warren

- Currently there's code to recover gracefully from panics that happen
within the execution of cron tasks. However this recover code wasn't
being run, because `RunWithShutdownContext` also contains code to
recover from any panic and then gracefully shutdown Forgejo. Because
`RunWithShutdownContext` registers that code as last, that would get run
first which in this case is not behavior that we want.
- Move the recover code to inside the function, so that is run first
before `RunWithShutdownContext`'s recover code (which is now a noop).

Fixes: https://codeberg.org/forgejo/forgejo/issues/1910

Co-authored-by: Earl Warren <109468362+earl-warren@users.noreply.github.com>
Co-authored-by: Gusted <postmaster@gusted.xyz>
2023-12-12 03:28:56 +00:00

231 lines
6.1 KiB
Go

// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cron
import (
"context"
"fmt"
"reflect"
"strings"
"sync"
"time"
"code.gitea.io/gitea/models/db"
system_model "code.gitea.io/gitea/models/system"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/process"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/translation"
)
var (
lock = sync.Mutex{}
started = false
tasks = []*Task{}
tasksMap = map[string]*Task{}
)
// Task represents a Cron task
type Task struct {
lock sync.Mutex
Name string
config Config
fun func(context.Context, *user_model.User, Config) error
Status string
LastMessage string
LastDoer string
ExecTimes int64
// This stores the time of the last manual run of this task.
LastRun time.Time
}
// DoRunAtStart returns if this task should run at the start
func (t *Task) DoRunAtStart() bool {
return t.config.DoRunAtStart()
}
// IsEnabled returns if this task is enabled as cron task
func (t *Task) IsEnabled() bool {
return t.config.IsEnabled()
}
// GetConfig will return a copy of the task's config
func (t *Task) GetConfig() Config {
if reflect.TypeOf(t.config).Kind() == reflect.Ptr {
// Pointer:
return reflect.New(reflect.ValueOf(t.config).Elem().Type()).Interface().(Config)
}
// Not pointer:
return reflect.New(reflect.TypeOf(t.config)).Elem().Interface().(Config)
}
// Run will run the task incrementing the cron counter with no user defined
func (t *Task) Run() {
t.RunWithUser(&user_model.User{
ID: -1,
Name: "(Cron)",
LowerName: "(cron)",
}, t.config)
}
// RunWithUser will run the task incrementing the cron counter at the time with User
func (t *Task) RunWithUser(doer *user_model.User, config Config) {
if !taskStatusTable.StartIfNotRunning(t.Name) {
return
}
t.lock.Lock()
if config == nil {
config = t.config
}
t.ExecTimes++
t.lock.Unlock()
defer func() {
taskStatusTable.Stop(t.Name)
}()
graceful.GetManager().RunWithShutdownContext(func(baseCtx context.Context) {
defer func() {
if err := recover(); err != nil {
// Recover a panic within the execution of the task.
combinedErr := fmt.Errorf("%s\n%s", err, log.Stack(2))
log.Error("PANIC whilst running task: %s Value: %v", t.Name, combinedErr)
}
}()
// Store the time of this run, before the function is executed, so it
// matches the behavior of what the cron library does.
t.lock.Lock()
t.LastRun = time.Now()
t.lock.Unlock()
pm := process.GetManager()
doerName := ""
if doer != nil && doer.ID != -1 {
doerName = doer.Name
}
ctx, _, finished := pm.AddContext(baseCtx, config.FormatMessage(translation.NewLocale("en-US"), t.Name, "process", doerName))
defer finished()
if err := t.fun(ctx, doer, config); err != nil {
var message string
var status string
if db.IsErrCancelled(err) {
status = "cancelled"
message = err.(db.ErrCancelled).Message
} else {
status = "error"
message = err.Error()
}
t.lock.Lock()
t.LastMessage = message
t.Status = status
t.LastDoer = doerName
t.lock.Unlock()
if err := system_model.CreateNotice(ctx, system_model.NoticeTask, config.FormatMessage(translation.NewLocale("en-US"), t.Name, "cancelled", doerName, message)); err != nil {
log.Error("CreateNotice: %v", err)
}
return
}
t.lock.Lock()
t.Status = "finished"
t.LastMessage = ""
t.LastDoer = doerName
t.lock.Unlock()
if config.DoNoticeOnSuccess() {
if err := system_model.CreateNotice(ctx, system_model.NoticeTask, config.FormatMessage(translation.NewLocale("en-US"), t.Name, "finished", doerName)); err != nil {
log.Error("CreateNotice: %v", err)
}
}
})
}
// GetTask gets the named task
func GetTask(name string) *Task {
lock.Lock()
defer lock.Unlock()
log.Info("Getting %s in %v", name, tasksMap[name])
return tasksMap[name]
}
// RegisterTask allows a task to be registered with the cron service
func RegisterTask(name string, config Config, fun func(context.Context, *user_model.User, Config) error) error {
log.Debug("Registering task: %s", name)
i18nKey := "admin.dashboard." + name
if value := translation.NewLocale("en-US").Tr(i18nKey); value == i18nKey {
return fmt.Errorf("translation is missing for task %q, please add translation for %q", name, i18nKey)
}
_, err := setting.GetCronSettings(name, config)
if err != nil {
log.Error("Unable to register cron task with name: %s Error: %v", name, err)
return err
}
task := &Task{
Name: name,
config: config,
fun: fun,
}
lock.Lock()
locked := true
defer func() {
if locked {
lock.Unlock()
}
}()
if _, has := tasksMap[task.Name]; has {
log.Error("A task with this name: %s has already been registered", name)
return fmt.Errorf("duplicate task with name: %s", task.Name)
}
if config.IsEnabled() {
// We cannot use the entry return as there is no way to lock it
if err := addTaskToScheduler(task); err != nil {
return err
}
}
tasks = append(tasks, task)
tasksMap[task.Name] = task
if started && config.IsEnabled() && config.DoRunAtStart() {
lock.Unlock()
locked = false
task.Run()
}
return nil
}
// RegisterTaskFatal will register a task but if there is an error log.Fatal
func RegisterTaskFatal(name string, config Config, fun func(context.Context, *user_model.User, Config) error) {
if err := RegisterTask(name, config, fun); err != nil {
log.Fatal("Unable to register cron task %s Error: %v", name, err)
}
}
func addTaskToScheduler(task *Task) error {
tags := []string{task.Name, task.config.GetSchedule()} // name and schedule can't be get from job, so we add them as tag
if scheduleHasSeconds(task.config.GetSchedule()) {
scheduler = scheduler.CronWithSeconds(task.config.GetSchedule())
} else {
scheduler = scheduler.Cron(task.config.GetSchedule())
}
if _, err := scheduler.Tag(tags...).Do(task.Run); err != nil {
log.Error("Unable to register cron task with name: %s Error: %v", task.Name, err)
return err
}
return nil
}
func scheduleHasSeconds(schedule string) bool {
return len(strings.Fields(schedule)) >= 6
}