From 4569339a4b8ead2b9bb110f4b7fc393ef6b3b4ba Mon Sep 17 00:00:00 2001 From: zeripath Date: Wed, 2 Dec 2020 04:56:04 +0000 Subject: [PATCH] Refactor doctor (#12264) * Refactor Logger Refactor Logger to make a logger interface and make it possible to wrap loggers for specific purposes. * Refactor Doctor Move the gitea doctor functions into its own module. Use a logger for its messages instead of returning a results string[] Signed-off-by: Andrew Thornton * Update modules/doctor/misc.go Co-authored-by: 6543 <6543@obermui.de> * Update modules/doctor/misc.go Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: techknowlogick --- cmd/doctor.go | 558 ++----------------------------- modules/doctor/authorizedkeys.go | 95 ++++++ modules/doctor/dbconsistency.go | 127 +++++++ modules/doctor/dbversion.go | 42 +++ modules/doctor/doctor.go | 105 ++++++ modules/doctor/mergebase.go | 110 ++++++ modules/doctor/misc.go | 146 ++++++++ modules/doctor/paths.go | 126 +++++++ 8 files changed, 787 insertions(+), 522 deletions(-) create mode 100644 modules/doctor/authorizedkeys.go create mode 100644 modules/doctor/dbconsistency.go create mode 100644 modules/doctor/dbversion.go create mode 100644 modules/doctor/doctor.go create mode 100644 modules/doctor/mergebase.go create mode 100644 modules/doctor/misc.go create mode 100644 modules/doctor/paths.go diff --git a/cmd/doctor.go b/cmd/doctor.go index 2ca2bb5e70..0152aebe39 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -5,27 +5,19 @@ package cmd import ( - "bufio" - "bytes" "context" "fmt" - "io/ioutil" golog "log" "os" - "os/exec" - "path/filepath" "strings" "text/tabwriter" "code.gitea.io/gitea/models" "code.gitea.io/gitea/models/migrations" - "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/doctor" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/options" - "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" - "xorm.io/builder" + "xorm.io/xorm" "github.com/urfave/cli" @@ -62,6 +54,10 @@ var CmdDoctor = cli.Command{ Name: "log-file", Usage: `Name of the log file (default: "doctor.log"). Set to "-" to output to stdout, set to "" to disable`, }, + cli.BoolFlag{ + Name: "color, H", + Usage: "Use color for outputted information", + }, }, Subcommands: []cli.Command{ cmdRecreateTable, @@ -86,78 +82,6 @@ You should back-up your database before doing this and ensure that your database Action: runRecreateTable, } -type check struct { - title string - name string - isDefault bool - f func(ctx *cli.Context) ([]string, error) - abortIfFailed bool - skipDatabaseInit bool -} - -// checklist represents list for all checks -var checklist = []check{ - { - // NOTE: this check should be the first in the list - title: "Check paths and basic configuration", - name: "paths", - isDefault: true, - f: runDoctorPathInfo, - abortIfFailed: true, - skipDatabaseInit: true, - }, - { - title: "Check Database Version", - name: "check-db-version", - isDefault: true, - f: runDoctorCheckDBVersion, - abortIfFailed: false, - }, - { - title: "Check consistency of database", - name: "check-db-consistency", - isDefault: false, - f: runDoctorCheckDBConsistency, - }, - { - title: "Check if OpenSSH authorized_keys file is up-to-date", - name: "authorized_keys", - isDefault: true, - f: runDoctorAuthorizedKeys, - }, - { - title: "Check if SCRIPT_TYPE is available", - name: "script-type", - isDefault: false, - f: runDoctorScriptType, - }, - { - title: "Check if hook files are up-to-date and executable", - name: "hooks", - isDefault: false, - f: runDoctorHooks, - }, - { - title: "Recalculate merge bases", - name: "recalculate_merge_bases", - isDefault: false, - f: runDoctorPRMergeBase, - }, - { - title: "Recalculate Stars number for all user", - name: "recalculate_stars_number", - isDefault: false, - f: runDoctorUserStarNum, - }, - { - title: "Enable push options", - name: "enable-push-options", - isDefault: false, - f: runDoctorEnablePushOptions, - }, - // more checks please append here -} - func runRecreateTable(ctx *cli.Context) error { // Redirect the default golog to here golog.SetFlags(0) @@ -211,10 +135,15 @@ func runDoctor(ctx *cli.Context) error { logFile = "doctor.log" } + colorize := log.CanColorStdout + if ctx.IsSet("color") { + colorize = ctx.Bool("color") + } + if len(logFile) == 0 { - log.NewLogger(1000, "doctor", "console", `{"level":"NONE","stacktracelevel":"NONE","colorize":"%t"}`) + log.NewLogger(1000, "doctor", "console", fmt.Sprintf(`{"level":"NONE","stacktracelevel":"NONE","colorize":%t}`, colorize)) } else if logFile == "-" { - log.NewLogger(1000, "doctor", "console", `{"level":"trace","stacktracelevel":"NONE"}`) + log.NewLogger(1000, "doctor", "console", fmt.Sprintf(`{"level":"trace","stacktracelevel":"NONE","colorize":%t}`, colorize)) } else { log.NewLogger(1000, "doctor", "file", fmt.Sprintf(`{"filename":%q,"level":"trace","stacktracelevel":"NONE"}`, logFile)) } @@ -225,24 +154,24 @@ func runDoctor(ctx *cli.Context) error { golog.SetOutput(log.NewLoggerAsWriter("INFO", log.GetLogger(log.DEFAULT))) if ctx.IsSet("list") { - w := tabwriter.NewWriter(os.Stdout, 0, 8, 0, '\t', 0) + w := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', 0) _, _ = w.Write([]byte("Default\tName\tTitle\n")) - for _, check := range checklist { - if check.isDefault { + for _, check := range doctor.Checks { + if check.IsDefault { _, _ = w.Write([]byte{'*'}) } _, _ = w.Write([]byte{'\t'}) - _, _ = w.Write([]byte(check.name)) + _, _ = w.Write([]byte(check.Name)) _, _ = w.Write([]byte{'\t'}) - _, _ = w.Write([]byte(check.title)) + _, _ = w.Write([]byte(check.Title)) _, _ = w.Write([]byte{'\n'}) } return w.Flush() } - var checks []check + var checks []*doctor.Check if ctx.Bool("all") { - checks = checklist + checks = doctor.Checks } else if ctx.IsSet("run") { addDefault := ctx.Bool("default") names := ctx.StringSlice("run") @@ -250,452 +179,37 @@ func runDoctor(ctx *cli.Context) error { names[i] = strings.ToLower(strings.TrimSpace(name)) } - for _, check := range checklist { - if addDefault && check.isDefault { + for _, check := range doctor.Checks { + if addDefault && check.IsDefault { checks = append(checks, check) continue } for _, name := range names { - if name == check.name { + if name == check.Name { checks = append(checks, check) break } } } } else { - for _, check := range checklist { - if check.isDefault { + for _, check := range doctor.Checks { + if check.IsDefault { checks = append(checks, check) } } } - dbIsInit := false - - for i, check := range checks { - if !dbIsInit && !check.skipDatabaseInit { - // Only open database after the most basic configuration check - setting.EnableXORMLog = false - if err := initDBDisableConsole(true); err != nil { - fmt.Println(err) - fmt.Println("Check if you are using the right config file. You can use a --config directive to specify one.") - return nil - } - dbIsInit = true - } - fmt.Println("[", i+1, "]", check.title) - messages, err := check.f(ctx) - for _, message := range messages { - fmt.Println("-", message) - } - if err != nil { - fmt.Println("Error:", err) - if check.abortIfFailed { - return nil - } - } else { - fmt.Println("OK.") - } - fmt.Println() - } - return nil -} - -func runDoctorPathInfo(ctx *cli.Context) ([]string, error) { - - res := make([]string, 0, 10) - - if fi, err := os.Stat(setting.CustomConf); err != nil || !fi.Mode().IsRegular() { - res = append(res, fmt.Sprintf("Failed to find configuration file at '%s'.", setting.CustomConf)) - res = append(res, fmt.Sprintf("If you've never ran Gitea yet, this is normal and '%s' will be created for you on first run.", setting.CustomConf)) - res = append(res, "Otherwise check that you are running this command from the correct path and/or provide a `--config` parameter.") - return res, fmt.Errorf("can't proceed without a configuration file") - } - - setting.NewContext() - - fail := false - - check := func(name, path string, is_dir, required, is_write bool) { - res = append(res, fmt.Sprintf("%-25s '%s'", name+":", path)) - fi, err := os.Stat(path) - if err != nil { - if os.IsNotExist(err) && ctx.Bool("fix") && is_dir { - if err := os.MkdirAll(path, 0777); err != nil { - res = append(res, fmt.Sprintf(" ERROR: %v", err)) - fail = true - return - } - fi, err = os.Stat(path) - } - } - if err != nil { - if required { - res = append(res, fmt.Sprintf(" ERROR: %v", err)) - fail = true - return - } - res = append(res, fmt.Sprintf(" NOTICE: not accessible (%v)", err)) - return - } - - if is_dir && !fi.IsDir() { - res = append(res, " ERROR: not a directory") - fail = true - return - } else if !is_dir && !fi.Mode().IsRegular() { - res = append(res, " ERROR: not a regular file") - fail = true - } else if is_write { - if err := runDoctorWritableDir(path); err != nil { - res = append(res, fmt.Sprintf(" ERROR: not writable: %v", err)) - fail = true - } - } - } - - // Note print paths inside quotes to make any leading/trailing spaces evident - check("Configuration File Path", setting.CustomConf, false, true, false) - check("Repository Root Path", setting.RepoRootPath, true, true, true) - check("Data Root Path", setting.AppDataPath, true, true, true) - check("Custom File Root Path", setting.CustomPath, true, false, false) - check("Work directory", setting.AppWorkPath, true, true, false) - check("Log Root Path", setting.LogRootPath, true, true, true) - - if options.IsDynamic() { - // Do not check/report on StaticRootPath if data is embedded in Gitea (-tags bindata) - check("Static File Root Path", setting.StaticRootPath, true, true, false) - } - - if fail { - return res, fmt.Errorf("please check your configuration file and try again") - } - - return res, nil -} - -func runDoctorWritableDir(path string) error { - // There's no platform-independent way of checking if a directory is writable - // https://stackoverflow.com/questions/20026320/how-to-tell-if-folder-exists-and-is-writable - - tmpFile, err := ioutil.TempFile(path, "doctors-order") - if err != nil { + // Now we can set up our own logger to return information about what the doctor is doing + if err := log.NewNamedLogger("doctorouter", + 1000, + "console", + "console", + fmt.Sprintf(`{"level":"INFO","stacktracelevel":"NONE","colorize":%t,"flags":-1}`, colorize)); err != nil { + fmt.Println(err) return err } - if err := util.Remove(tmpFile.Name()); err != nil { - fmt.Printf("Warning: can't remove temporary file: '%s'\n", tmpFile.Name()) - } - tmpFile.Close() - return nil -} - -const tplCommentPrefix = `# gitea public key` - -func runDoctorAuthorizedKeys(ctx *cli.Context) ([]string, error) { - if setting.SSH.StartBuiltinServer || !setting.SSH.CreateAuthorizedKeysFile { - return nil, nil - } - - fPath := filepath.Join(setting.SSH.RootPath, "authorized_keys") - f, err := os.Open(fPath) - if err != nil { - if ctx.Bool("fix") { - return []string{fmt.Sprintf("Error whilst opening authorized_keys: %v. Attempting regeneration", err)}, models.RewriteAllPublicKeys() - } - return nil, err - } - defer f.Close() - - linesInAuthorizedKeys := map[string]bool{} - - scanner := bufio.NewScanner(f) - for scanner.Scan() { - line := scanner.Text() - if strings.HasPrefix(line, tplCommentPrefix) { - continue - } - linesInAuthorizedKeys[line] = true - } - f.Close() - - // now we regenerate and check if there are any lines missing - regenerated := &bytes.Buffer{} - if err := models.RegeneratePublicKeys(regenerated); err != nil { - return nil, err - } - scanner = bufio.NewScanner(regenerated) - for scanner.Scan() { - line := scanner.Text() - if strings.HasPrefix(line, tplCommentPrefix) { - continue - } - if ok := linesInAuthorizedKeys[line]; ok { - continue - } - if ctx.Bool("fix") { - return []string{"authorized_keys is out of date, attempting regeneration"}, models.RewriteAllPublicKeys() - } - return nil, fmt.Errorf(`authorized_keys is out of date and should be regenerated with "gitea admin regenerate keys" or "gitea doctor --run authorized_keys --fix"`) - } - return nil, nil -} - -func runDoctorCheckDBVersion(ctx *cli.Context) ([]string, error) { - if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil { - if ctx.Bool("fix") { - return []string{fmt.Sprintf("WARN: Got Error %v during ensure up to date", err), "Attempting to migrate to the latest DB version to fix this."}, models.NewEngine(context.Background(), migrations.Migrate) - } - return nil, err - } - return nil, nil -} - -func iterateRepositories(each func(*models.Repository) ([]string, error)) ([]string, error) { - results := []string{} - err := models.Iterate( - models.DefaultDBContext(), - new(models.Repository), - builder.Gt{"id": 0}, - func(idx int, bean interface{}) error { - res, err := each(bean.(*models.Repository)) - results = append(results, res...) - return err - }, - ) - return results, err -} - -func iteratePRs(repo *models.Repository, each func(*models.Repository, *models.PullRequest) ([]string, error)) ([]string, error) { - results := []string{} - err := models.Iterate( - models.DefaultDBContext(), - new(models.PullRequest), - builder.Eq{"base_repo_id": repo.ID}, - func(idx int, bean interface{}) error { - res, err := each(repo, bean.(*models.PullRequest)) - results = append(results, res...) - return err - }, - ) - return results, err -} - -func runDoctorHooks(ctx *cli.Context) ([]string, error) { - // Need to iterate across all of the repositories - return iterateRepositories(func(repo *models.Repository) ([]string, error) { - results, err := repository.CheckDelegateHooks(repo.RepoPath()) - if err != nil { - return nil, err - } - if len(results) > 0 && ctx.Bool("fix") { - return []string{fmt.Sprintf("regenerated hooks for %s", repo.FullName())}, repository.CreateDelegateHooks(repo.RepoPath()) - } - - return results, nil - }) -} - -func runDoctorPRMergeBase(ctx *cli.Context) ([]string, error) { - numRepos := 0 - numPRs := 0 - numPRsUpdated := 0 - results, err := iterateRepositories(func(repo *models.Repository) ([]string, error) { - numRepos++ - return iteratePRs(repo, func(repo *models.Repository, pr *models.PullRequest) ([]string, error) { - numPRs++ - results := []string{} - pr.BaseRepo = repo - repoPath := repo.RepoPath() - - oldMergeBase := pr.MergeBase - - if !pr.HasMerged { - var err error - pr.MergeBase, err = git.NewCommand("merge-base", "--", pr.BaseBranch, pr.GetGitRefName()).RunInDir(repoPath) - if err != nil { - var err2 error - pr.MergeBase, err2 = git.NewCommand("rev-parse", git.BranchPrefix+pr.BaseBranch).RunInDir(repoPath) - if err2 != nil { - results = append(results, fmt.Sprintf("WARN: Unable to get merge base for PR ID %d, #%d onto %s in %s/%s", pr.ID, pr.Index, pr.BaseBranch, pr.BaseRepo.OwnerName, pr.BaseRepo.Name)) - log.Error("Unable to get merge base for PR ID %d, Index %d in %s/%s. Error: %v & %v", pr.ID, pr.Index, pr.BaseRepo.OwnerName, pr.BaseRepo.Name, err, err2) - return results, nil - } - } - } else { - parentsString, err := git.NewCommand("rev-list", "--parents", "-n", "1", pr.MergedCommitID).RunInDir(repoPath) - if err != nil { - results = append(results, fmt.Sprintf("WARN: Unable to get parents for merged PR ID %d, #%d onto %s in %s/%s", pr.ID, pr.Index, pr.BaseBranch, pr.BaseRepo.OwnerName, pr.BaseRepo.Name)) - log.Error("Unable to get parents for merged PR ID %d, Index %d in %s/%s. Error: %v", pr.ID, pr.Index, pr.BaseRepo.OwnerName, pr.BaseRepo.Name, err) - return results, nil - } - parents := strings.Split(strings.TrimSpace(parentsString), " ") - if len(parents) < 2 { - return results, nil - } - - args := append([]string{"merge-base", "--"}, parents[1:]...) - args = append(args, pr.GetGitRefName()) - - pr.MergeBase, err = git.NewCommand(args...).RunInDir(repoPath) - if err != nil { - results = append(results, fmt.Sprintf("WARN: Unable to get merge base for merged PR ID %d, #%d onto %s in %s/%s", pr.ID, pr.Index, pr.BaseBranch, pr.BaseRepo.OwnerName, pr.BaseRepo.Name)) - log.Error("Unable to get merge base for merged PR ID %d, Index %d in %s/%s. Error: %v", pr.ID, pr.Index, pr.BaseRepo.OwnerName, pr.BaseRepo.Name, err) - return results, nil - } - } - pr.MergeBase = strings.TrimSpace(pr.MergeBase) - if pr.MergeBase != oldMergeBase { - if ctx.Bool("fix") { - if err := pr.UpdateCols("merge_base"); err != nil { - return results, err - } - } else { - results = append(results, fmt.Sprintf("#%d onto %s in %s/%s: MergeBase should be %s but is %s", pr.Index, pr.BaseBranch, pr.BaseRepo.OwnerName, pr.BaseRepo.Name, oldMergeBase, pr.MergeBase)) - } - numPRsUpdated++ - } - return results, nil - }) - }) - - if ctx.Bool("fix") { - results = append(results, fmt.Sprintf("%d PR mergebases updated of %d PRs total in %d repos", numPRsUpdated, numPRs, numRepos)) - } else { - if numPRsUpdated > 0 && err == nil { - return results, fmt.Errorf("%d PRs with incorrect mergebases of %d PRs total in %d repos", numPRsUpdated, numPRs, numRepos) - } - results = append(results, fmt.Sprintf("%d PRs with incorrect mergebases of %d PRs total in %d repos", numPRsUpdated, numPRs, numRepos)) - } - - return results, err -} - -func runDoctorUserStarNum(ctx *cli.Context) ([]string, error) { - return nil, models.DoctorUserStarNum() -} - -func runDoctorScriptType(ctx *cli.Context) ([]string, error) { - path, err := exec.LookPath(setting.ScriptType) - if err != nil { - return []string{fmt.Sprintf("ScriptType %s is not on the current PATH", setting.ScriptType)}, err - } - return []string{fmt.Sprintf("ScriptType %s is on the current PATH at %s", setting.ScriptType, path)}, nil -} - -func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) { - var results []string - - // make sure DB version is uptodate - if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil { - return nil, fmt.Errorf("model version on the database does not match the current Gitea version. Model consistency will not be checked until the database is upgraded") - } - - //find labels without existing repo or org - count, err := models.CountOrphanedLabels() - if err != nil { - return nil, err - } - if count > 0 { - if ctx.Bool("fix") { - if err = models.DeleteOrphanedLabels(); err != nil { - return nil, err - } - results = append(results, fmt.Sprintf("%d labels without existing repository/organisation deleted", count)) - } else { - results = append(results, fmt.Sprintf("%d labels without existing repository/organisation", count)) - } - } - - //find issues without existing repository - count, err = models.CountOrphanedIssues() - if err != nil { - return nil, err - } - if count > 0 { - if ctx.Bool("fix") { - if err = models.DeleteOrphanedIssues(); err != nil { - return nil, err - } - results = append(results, fmt.Sprintf("%d issues without existing repository deleted", count)) - } else { - results = append(results, fmt.Sprintf("%d issues without existing repository", count)) - } - } - - //find pulls without existing issues - count, err = models.CountOrphanedObjects("pull_request", "issue", "pull_request.issue_id=issue.id") - if err != nil { - return nil, err - } - if count > 0 { - if ctx.Bool("fix") { - if err = models.DeleteOrphanedObjects("pull_request", "issue", "pull_request.issue_id=issue.id"); err != nil { - return nil, err - } - results = append(results, fmt.Sprintf("%d pull requests without existing issue deleted", count)) - } else { - results = append(results, fmt.Sprintf("%d pull requests without existing issue", count)) - } - } - - //find tracked times without existing issues/pulls - count, err = models.CountOrphanedObjects("tracked_time", "issue", "tracked_time.issue_id=issue.id") - if err != nil { - return nil, err - } - if count > 0 { - if ctx.Bool("fix") { - if err = models.DeleteOrphanedObjects("tracked_time", "issue", "tracked_time.issue_id=issue.id"); err != nil { - return nil, err - } - results = append(results, fmt.Sprintf("%d tracked times without existing issue deleted", count)) - } else { - results = append(results, fmt.Sprintf("%d tracked times without existing issue", count)) - } - } - - count, err = models.CountNullArchivedRepository() - if err != nil { - return nil, err - } - if count > 0 { - if ctx.Bool("fix") { - updatedCount, err := models.FixNullArchivedRepository() - if err != nil { - return nil, err - } - results = append(results, fmt.Sprintf("%d repositories with null is_archived updated", updatedCount)) - } else { - results = append(results, fmt.Sprintf("%d repositories with null is_archived", count)) - } - } - - //ToDo: function to recalc all counters - - return results, nil -} - -func runDoctorEnablePushOptions(ctx *cli.Context) ([]string, error) { - numRepos := 0 - _, err := iterateRepositories(func(repo *models.Repository) ([]string, error) { - numRepos++ - r, err := git.OpenRepository(repo.RepoPath()) - if err != nil { - return nil, err - } - defer r.Close() - - if ctx.Bool("fix") { - _, err := git.NewCommand("config", "receive.advertisePushOptions", "true").RunInDir(r.Path) - return nil, err - } - - return nil, nil - }) - - var prefix string - if !ctx.Bool("fix") { - prefix = "DRY RUN: " - } - return []string{fmt.Sprintf("%sEnabled push options for %d repositories.", prefix, numRepos)}, err + + logger := log.GetLogger("doctorouter") + defer logger.Close() + return doctor.RunChecks(logger, ctx.Bool("fix"), checks) } diff --git a/modules/doctor/authorizedkeys.go b/modules/doctor/authorizedkeys.go new file mode 100644 index 0000000000..d9bc70bb77 --- /dev/null +++ b/modules/doctor/authorizedkeys.go @@ -0,0 +1,95 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package doctor + +import ( + "bufio" + "bytes" + "fmt" + "os" + "path/filepath" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" +) + +const tplCommentPrefix = `# gitea public key` + +func checkAuthorizedKeys(logger log.Logger, autofix bool) error { + if setting.SSH.StartBuiltinServer || !setting.SSH.CreateAuthorizedKeysFile { + return nil + } + + fPath := filepath.Join(setting.SSH.RootPath, "authorized_keys") + f, err := os.Open(fPath) + if err != nil { + if !autofix { + logger.Critical("Unable to open authorized_keys file. ERROR: %v", err) + return fmt.Errorf("Unable to open authorized_keys file. ERROR: %v", err) + } + logger.Warn("Unable to open authorized_keys. (ERROR: %v). Attempting to rewrite...", err) + if err = models.RewriteAllPublicKeys(); err != nil { + logger.Critical("Unable to rewrite authorized_keys file. ERROR: %v", err) + return fmt.Errorf("Unable to rewrite authorized_keys file. ERROR: %v", err) + } + } + defer f.Close() + + linesInAuthorizedKeys := map[string]bool{} + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, tplCommentPrefix) { + continue + } + linesInAuthorizedKeys[line] = true + } + f.Close() + + // now we regenerate and check if there are any lines missing + regenerated := &bytes.Buffer{} + if err := models.RegeneratePublicKeys(regenerated); err != nil { + logger.Critical("Unable to regenerate authorized_keys file. ERROR: %v", err) + return fmt.Errorf("Unable to regenerate authorized_keys file. ERROR: %v", err) + } + scanner = bufio.NewScanner(regenerated) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, tplCommentPrefix) { + continue + } + if ok := linesInAuthorizedKeys[line]; ok { + continue + } + if !autofix { + logger.Critical( + "authorized_keys file %q is out of date.\nRegenerate it with:\n\t\"%s\"\nor\n\t\"%s\"", + fPath, + "gitea admin regenerate keys", + "gitea doctor --run authorized_keys --fix") + return fmt.Errorf(`authorized_keys is out of date and should be regenerated with "gitea admin regenerate keys" or "gitea doctor --run authorized_keys --fix"`) + } + logger.Warn("authorized_keys is out of date. Attempting rewrite...") + err = models.RewriteAllPublicKeys() + if err != nil { + logger.Critical("Unable to rewrite authorized_keys file. ERROR: %v", err) + return fmt.Errorf("Unable to rewrite authorized_keys file. ERROR: %v", err) + } + } + return nil +} + +func init() { + Register(&Check{ + Title: "Check if OpenSSH authorized_keys file is up-to-date", + Name: "authorized-keys", + IsDefault: true, + Run: checkAuthorizedKeys, + Priority: 4, + }) +} diff --git a/modules/doctor/dbconsistency.go b/modules/doctor/dbconsistency.go new file mode 100644 index 0000000000..f09aaa6d1b --- /dev/null +++ b/modules/doctor/dbconsistency.go @@ -0,0 +1,127 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package doctor + +import ( + "context" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/migrations" + "code.gitea.io/gitea/modules/log" +) + +func checkDBConsistency(logger log.Logger, autofix bool) error { + // make sure DB version is uptodate + if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil { + logger.Critical("Model version on the database does not match the current Gitea version. Model consistency will not be checked until the database is upgraded") + return err + } + + // find labels without existing repo or org + count, err := models.CountOrphanedLabels() + if err != nil { + logger.Critical("Error: %v whilst counting orphaned labels") + return err + } + + if count > 0 { + if autofix { + if err = models.DeleteOrphanedLabels(); err != nil { + logger.Critical("Error: %v whilst deleting orphaned labels") + return err + } + logger.Info("%d labels without existing repository/organisation deleted", count) + } else { + logger.Warn("%d labels without existing repository/organisation", count) + } + } + + // find issues without existing repository + count, err = models.CountOrphanedIssues() + if err != nil { + logger.Critical("Error: %v whilst counting orphaned issues") + return err + } + if count > 0 { + if autofix { + if err = models.DeleteOrphanedIssues(); err != nil { + logger.Critical("Error: %v whilst deleting orphaned issues") + return err + } + logger.Info("%d issues without existing repository deleted", count) + } else { + logger.Warn("%d issues without existing repository", count) + } + } + + // find pulls without existing issues + count, err = models.CountOrphanedObjects("pull_request", "issue", "pull_request.issue_id=issue.id") + if err != nil { + logger.Critical("Error: %v whilst counting orphaned objects") + return err + } + if count > 0 { + if autofix { + if err = models.DeleteOrphanedObjects("pull_request", "issue", "pull_request.issue_id=issue.id"); err != nil { + logger.Critical("Error: %v whilst deleting orphaned objects") + return err + } + logger.Info("%d pull requests without existing issue deleted", count) + } else { + logger.Warn("%d pull requests without existing issue", count) + } + } + + // find tracked times without existing issues/pulls + count, err = models.CountOrphanedObjects("tracked_time", "issue", "tracked_time.issue_id=issue.id") + if err != nil { + logger.Critical("Error: %v whilst counting orphaned objects") + return err + } + if count > 0 { + if autofix { + if err = models.DeleteOrphanedObjects("tracked_time", "issue", "tracked_time.issue_id=issue.id"); err != nil { + logger.Critical("Error: %v whilst deleting orphaned objects") + return err + } + logger.Info("%d tracked times without existing issue deleted", count) + } else { + logger.Warn("%d tracked times without existing issue", count) + } + } + + // find null archived repositories + count, err = models.CountNullArchivedRepository() + if err != nil { + logger.Critical("Error: %v whilst counting null archived repositories") + return err + } + if count > 0 { + if autofix { + updatedCount, err := models.FixNullArchivedRepository() + if err != nil { + logger.Critical("Error: %v whilst fixing null archived repositories") + return err + } + logger.Info("%d repositories with null is_archived updated", updatedCount) + } else { + logger.Warn("%d repositories with null is_archived", count) + } + } + + // TODO: function to recalc all counters + + return nil +} + +func init() { + Register(&Check{ + Title: "Check consistency of database", + Name: "check-db-consistency", + IsDefault: false, + Run: checkDBConsistency, + Priority: 3, + }) +} diff --git a/modules/doctor/dbversion.go b/modules/doctor/dbversion.go new file mode 100644 index 0000000000..f82cf72093 --- /dev/null +++ b/modules/doctor/dbversion.go @@ -0,0 +1,42 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package doctor + +import ( + "context" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/models/migrations" + "code.gitea.io/gitea/modules/log" +) + +func checkDBVersion(logger log.Logger, autofix bool) error { + if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil { + if !autofix { + logger.Critical("Error: %v during ensure up to date", err) + return err + } + logger.Warn("Got Error: %v during ensure up to date", err) + logger.Warn("Attempting to migrate to the latest DB version to fix this.") + + err = models.NewEngine(context.Background(), migrations.Migrate) + if err != nil { + logger.Critical("Error: %v during migration") + } + return err + } + return nil +} + +func init() { + Register(&Check{ + Title: "Check Database Version", + Name: "check-db-version", + IsDefault: true, + Run: checkDBVersion, + AbortIfFailed: false, + Priority: 2, + }) +} diff --git a/modules/doctor/doctor.go b/modules/doctor/doctor.go new file mode 100644 index 0000000000..8c0d467304 --- /dev/null +++ b/modules/doctor/doctor.go @@ -0,0 +1,105 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package doctor + +import ( + "fmt" + "sort" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" +) + +// Check represents a Doctor check +type Check struct { + Title string + Name string + IsDefault bool + Run func(logger log.Logger, autofix bool) error + AbortIfFailed bool + SkipDatabaseInitialization bool + Priority int +} + +type wrappedLevelLogger struct { + log.LevelLogger +} + +func (w *wrappedLevelLogger) Log(skip int, level log.Level, format string, v ...interface{}) error { + return w.LevelLogger.Log( + skip+1, + level, + " - %s "+format, + append( + []interface{}{ + log.NewColoredValueBytes( + fmt.Sprintf("[%s]", strings.ToUpper(level.String()[0:1])), + level.Color()), + }, v...)...) +} + +func initDBDisableConsole(disableConsole bool) error { + setting.NewContext() + setting.InitDBConfig() + + setting.NewXORMLogService(disableConsole) + if err := models.SetEngine(); err != nil { + return fmt.Errorf("models.SetEngine: %v", err) + } + return nil +} + +// Checks is the list of available commands +var Checks []*Check + +// RunChecks runs the doctor checks for the provided list +func RunChecks(logger log.Logger, autofix bool, checks []*Check) error { + wrappedLogger := log.LevelLoggerLogger{ + LevelLogger: &wrappedLevelLogger{logger}, + } + + dbIsInit := false + for i, check := range checks { + if !dbIsInit && !check.SkipDatabaseInitialization { + // Only open database after the most basic configuration check + setting.EnableXORMLog = false + if err := initDBDisableConsole(true); err != nil { + logger.Error("Error whilst initializing the database: %v", err) + logger.Error("Check if you are using the right config file. You can use a --config directive to specify one.") + return nil + } + dbIsInit = true + } + logger.Info("[%d] %s", log.NewColoredIDValue(i+1), check.Title) + logger.Flush() + if err := check.Run(&wrappedLogger, autofix); err != nil { + if check.AbortIfFailed { + logger.Critical("FAIL") + return err + } + logger.Error("ERROR") + } else { + logger.Info("OK") + logger.Flush() + } + } + return nil +} + +// Register registers a command with the list +func Register(command *Check) { + Checks = append(Checks, command) + sort.SliceStable(Checks, func(i, j int) bool { + if Checks[i].Priority == Checks[j].Priority { + return Checks[i].Name < Checks[j].Name + } + if Checks[i].Priority == 0 { + return false + } + return Checks[i].Priority < Checks[j].Priority + }) +} diff --git a/modules/doctor/mergebase.go b/modules/doctor/mergebase.go new file mode 100644 index 0000000000..7e9a065b77 --- /dev/null +++ b/modules/doctor/mergebase.go @@ -0,0 +1,110 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package doctor + +import ( + "fmt" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + "xorm.io/builder" +) + +func iteratePRs(repo *models.Repository, each func(*models.Repository, *models.PullRequest) error) error { + return models.Iterate( + models.DefaultDBContext(), + new(models.PullRequest), + builder.Eq{"base_repo_id": repo.ID}, + func(idx int, bean interface{}) error { + return each(repo, bean.(*models.PullRequest)) + }, + ) +} + +func checkPRMergeBase(logger log.Logger, autofix bool) error { + numRepos := 0 + numPRs := 0 + numPRsUpdated := 0 + err := iterateRepositories(func(repo *models.Repository) error { + numRepos++ + return iteratePRs(repo, func(repo *models.Repository, pr *models.PullRequest) error { + numPRs++ + pr.BaseRepo = repo + repoPath := repo.RepoPath() + + oldMergeBase := pr.MergeBase + + if !pr.HasMerged { + var err error + pr.MergeBase, err = git.NewCommand("merge-base", "--", pr.BaseBranch, pr.GetGitRefName()).RunInDir(repoPath) + if err != nil { + var err2 error + pr.MergeBase, err2 = git.NewCommand("rev-parse", git.BranchPrefix+pr.BaseBranch).RunInDir(repoPath) + if err2 != nil { + logger.Warn("Unable to get merge base for PR ID %d, #%d onto %s in %s/%s. Error: %v & %v", pr.ID, pr.Index, pr.BaseBranch, pr.BaseRepo.OwnerName, pr.BaseRepo.Name, err, err2) + return nil + } + } + } else { + parentsString, err := git.NewCommand("rev-list", "--parents", "-n", "1", pr.MergedCommitID).RunInDir(repoPath) + if err != nil { + logger.Warn("Unable to get parents for merged PR ID %d, #%d onto %s in %s/%s. Error: %v", pr.ID, pr.Index, pr.BaseBranch, pr.BaseRepo.OwnerName, pr.BaseRepo.Name, err) + return nil + } + parents := strings.Split(strings.TrimSpace(parentsString), " ") + if len(parents) < 2 { + return nil + } + + args := append([]string{"merge-base", "--"}, parents[1:]...) + args = append(args, pr.GetGitRefName()) + + pr.MergeBase, err = git.NewCommand(args...).RunInDir(repoPath) + if err != nil { + logger.Warn("Unable to get merge base for merged PR ID %d, #%d onto %s in %s/%s. Error: %v", pr.ID, pr.Index, pr.BaseBranch, pr.BaseRepo.OwnerName, pr.BaseRepo.Name, err) + return nil + } + } + pr.MergeBase = strings.TrimSpace(pr.MergeBase) + if pr.MergeBase != oldMergeBase { + if autofix { + if err := pr.UpdateCols("merge_base"); err != nil { + logger.Critical("Failed to update merge_base. ERROR: %v", err) + return fmt.Errorf("Failed to update merge_base. ERROR: %v", err) + } + } else { + logger.Info("#%d onto %s in %s/%s: MergeBase should be %s but is %s", pr.Index, pr.BaseBranch, pr.BaseRepo.OwnerName, pr.BaseRepo.Name, oldMergeBase, pr.MergeBase) + } + numPRsUpdated++ + } + return nil + }) + }) + + if autofix { + logger.Info("%d PR mergebases updated of %d PRs total in %d repos", numPRsUpdated, numPRs, numRepos) + } else { + if numPRsUpdated > 0 && err == nil { + logger.Critical("%d PRs with incorrect mergebases of %d PRs total in %d repos", numPRsUpdated, numPRs, numRepos) + return fmt.Errorf("%d PRs with incorrect mergebases of %d PRs total in %d repos", numPRsUpdated, numPRs, numRepos) + } + + logger.Warn("%d PRs with incorrect mergebases of %d PRs total in %d repos", numPRsUpdated, numPRs, numRepos) + } + + return err +} + +func init() { + Register(&Check{ + Title: "Recalculate merge bases", + Name: "recalculate-merge-bases", + IsDefault: false, + Run: checkPRMergeBase, + Priority: 7, + }) +} diff --git a/modules/doctor/misc.go b/modules/doctor/misc.go new file mode 100644 index 0000000000..0ca1e841ee --- /dev/null +++ b/modules/doctor/misc.go @@ -0,0 +1,146 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package doctor + +import ( + "fmt" + "os/exec" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/repository" + "code.gitea.io/gitea/modules/setting" + "xorm.io/builder" +) + +func iterateRepositories(each func(*models.Repository) error) error { + err := models.Iterate( + models.DefaultDBContext(), + new(models.Repository), + builder.Gt{"id": 0}, + func(idx int, bean interface{}) error { + return each(bean.(*models.Repository)) + }, + ) + return err +} + +func checkScriptType(logger log.Logger, autofix bool) error { + path, err := exec.LookPath(setting.ScriptType) + if err != nil { + logger.Critical("ScriptType \"%q\" is not on the current PATH. Error: %v", setting.ScriptType, err) + return fmt.Errorf("ScriptType \"%q\" is not on the current PATH. Error: %v", setting.ScriptType, err) + } + logger.Info("ScriptType %s is on the current PATH at %s", setting.ScriptType, path) + return nil +} + +func checkHooks(logger log.Logger, autofix bool) error { + if err := iterateRepositories(func(repo *models.Repository) error { + results, err := repository.CheckDelegateHooks(repo.RepoPath()) + if err != nil { + logger.Critical("Unable to check delegate hooks for repo %-v. ERROR: %v", repo, err) + return fmt.Errorf("Unable to check delegate hooks for repo %-v. ERROR: %v", repo, err) + } + if len(results) > 0 && autofix { + logger.Warn("Regenerated hooks for %s", repo.FullName()) + if err := repository.CreateDelegateHooks(repo.RepoPath()); err != nil { + logger.Critical("Unable to recreate delegate hooks for %-v. ERROR: %v", repo, err) + return fmt.Errorf("Unable to recreate delegate hooks for %-v. ERROR: %v", repo, err) + } + } + for _, result := range results { + logger.Warn(result) + } + return nil + }); err != nil { + logger.Critical("Errors noted whilst checking delegate hooks.") + return err + } + return nil +} + +func checkUserStarNum(logger log.Logger, autofix bool) error { + if err := models.DoctorUserStarNum(); err != nil { + logger.Critical("Unable update User Stars numbers") + return err + } + return nil +} + +func checkEnablePushOptions(logger log.Logger, autofix bool) error { + numRepos := 0 + numNeedUpdate := 0 + if err := iterateRepositories(func(repo *models.Repository) error { + numRepos++ + r, err := git.OpenRepository(repo.RepoPath()) + if err != nil { + return err + } + defer r.Close() + + if autofix { + _, err := git.NewCommand("config", "receive.advertisePushOptions", "true").RunInDir(r.Path) + return err + } + + value, err := git.NewCommand("config", "receive.advertisePushOptions").RunInDir(r.Path) + if err != nil { + return err + } + + result, valid := git.ParseBool(strings.TrimSpace(value)) + if !result || !valid { + numNeedUpdate++ + logger.Info("%s: does not have receive.advertisePushOptions set correctly: %q", repo.FullName(), value) + } + return nil + }); err != nil { + logger.Critical("Unable to EnablePushOptions: %v", err) + return err + } + + if autofix { + logger.Info("Enabled push options for %d repositories.", numRepos) + } else { + logger.Info("Checked %d repositories, %d need updates.", numRepos, numNeedUpdate) + + } + + return nil +} + +func init() { + Register(&Check{ + Title: "Check if SCRIPT_TYPE is available", + Name: "script-type", + IsDefault: false, + Run: checkScriptType, + Priority: 5, + }) + Register(&Check{ + Title: "Check if hook files are up-to-date and executable", + Name: "hooks", + IsDefault: false, + Run: checkHooks, + Priority: 6, + }) + Register(&Check{ + Title: "Recalculate Stars number for all user", + Name: "recalculate-stars-number", + IsDefault: false, + Run: checkUserStarNum, + Priority: 6, + }) + Register(&Check{ + Title: "Enable push options", + Name: "enable-push-options", + IsDefault: false, + Run: checkEnablePushOptions, + Priority: 7, + }) +} diff --git a/modules/doctor/paths.go b/modules/doctor/paths.go new file mode 100644 index 0000000000..53409b5fa4 --- /dev/null +++ b/modules/doctor/paths.go @@ -0,0 +1,126 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package doctor + +import ( + "fmt" + "io/ioutil" + "os" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/options" + "code.gitea.io/gitea/modules/setting" +) + +type configurationFile struct { + Name string + Path string + IsDirectory bool + Required bool + Writable bool +} + +func checkConfigurationFile(logger log.Logger, autofix bool, fileOpts configurationFile) error { + logger.Info(`%-26s %q`, log.NewColoredValue(fileOpts.Name+":", log.Reset), fileOpts.Path) + fi, err := os.Stat(fileOpts.Path) + if err != nil { + if os.IsNotExist(err) && autofix && fileOpts.IsDirectory { + if err := os.MkdirAll(fileOpts.Path, 0777); err != nil { + logger.Error(" Directory does not exist and could not be created. ERROR: %v", err) + return fmt.Errorf("Configuration directory: \"%q\" does not exist and could not be created. ERROR: %v", fileOpts.Path, err) + } + fi, err = os.Stat(fileOpts.Path) + } + } + if err != nil { + if fileOpts.Required { + logger.Error(" Is REQUIRED but is not accessible. ERROR: %v", err) + return fmt.Errorf("Configuration file \"%q\" is not accessible but is required. Error: %v", fileOpts.Path, err) + } + logger.Warn(" NOTICE: is not accessible (Error: %v)", err) + // this is a non-critical error + return nil + } + + if fileOpts.IsDirectory && !fi.IsDir() { + logger.Error(" ERROR: not a directory") + return fmt.Errorf("Configuration directory \"%q\" is not a directory. Error: %v", fileOpts.Path, err) + } else if !fileOpts.IsDirectory && !fi.Mode().IsRegular() { + logger.Error(" ERROR: not a regular file") + return fmt.Errorf("Configuration file \"%q\" is not a regular file. Error: %v", fileOpts.Path, err) + } else if fileOpts.Writable { + if err := isWritableDir(fileOpts.Path); err != nil { + logger.Error(" ERROR: is required to be writable but is not writable: %v", err) + return fmt.Errorf("Configuration file \"%q\" is required to be writable but is not. Error: %v", fileOpts.Path, err) + } + } + return nil +} + +func checkConfigurationFiles(logger log.Logger, autofix bool) error { + if fi, err := os.Stat(setting.CustomConf); err != nil || !fi.Mode().IsRegular() { + logger.Error("Failed to find configuration file at '%s'.", setting.CustomConf) + logger.Error("If you've never ran Gitea yet, this is normal and '%s' will be created for you on first run.", setting.CustomConf) + logger.Error("Otherwise check that you are running this command from the correct path and/or provide a `--config` parameter.") + logger.Critical("Cannot proceed without a configuration file") + return err + } + + setting.NewContext() + + configurationFiles := []configurationFile{ + {"Configuration File Path", setting.CustomConf, false, true, false}, + {"Repository Root Path", setting.RepoRootPath, true, true, true}, + {"Data Root Path", setting.AppDataPath, true, true, true}, + {"Custom File Root Path", setting.CustomPath, true, false, false}, + {"Work directory", setting.AppWorkPath, true, true, false}, + {"Log Root Path", setting.LogRootPath, true, true, true}, + } + + if options.IsDynamic() { + configurationFiles = append(configurationFiles, configurationFile{"Static File Root Path", setting.StaticRootPath, true, true, false}) + } + + numberOfErrors := 0 + for _, configurationFile := range configurationFiles { + if err := checkConfigurationFile(logger, autofix, configurationFile); err != nil { + numberOfErrors++ + } + } + + if numberOfErrors > 0 { + logger.Critical("Please check your configuration files and try again.") + return fmt.Errorf("%d configuration files with errors", numberOfErrors) + } + + return nil +} + +func isWritableDir(path string) error { + // There's no platform-independent way of checking if a directory is writable + // https://stackoverflow.com/questions/20026320/how-to-tell-if-folder-exists-and-is-writable + + tmpFile, err := ioutil.TempFile(path, "doctors-order") + if err != nil { + return err + } + if err := os.Remove(tmpFile.Name()); err != nil { + fmt.Printf("Warning: can't remove temporary file: '%s'\n", tmpFile.Name()) + } + tmpFile.Close() + return nil +} + +func init() { + Register(&Check{ + Title: "Check paths and basic configuration", + Name: "paths", + IsDefault: true, + Run: checkConfigurationFiles, + AbortIfFailed: true, + SkipDatabaseInitialization: true, + Priority: 1, + }) +}