Queue: Make WorkerPools and Queues flushable (#10001)

* Make WorkerPools and Queues flushable

Adds Flush methods to Queues and the WorkerPool
Further abstracts the WorkerPool
Adds a final step to Flush the queues in the defer from PrintCurrentTest
Fixes an issue with Settings inheritance in queues

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Change to for loop

* Add IsEmpty and begin just making the queues composed WorkerPools

* subsume workerpool into the queues and create a flushable interface

* Add manager command

* Move flushall to queue.Manager and add to testlogger

* As per @guillep2k

* as per @guillep2k

* Just make queues all implement flushable and clean up the wrapped queue flushes

* cope with no timeout

Co-authored-by: Lauris BH <lauris@nix.lv>

routers/private/internal.go

@@ -89,5 +89,9 @@ func RegisterRoutes(m *macaron.Macaron) {
 		m.Post("/hook/set-default-branch/:owner/:repo/:branch", SetDefaultBranch)
 		m.Get("/serv/none/:keyid", ServNoCommand)
 		m.Get("/serv/command/:keyid/:owner/:repo", ServCommand)
+		m.Post("/manager/shutdown", Shutdown)
+		m.Post("/manager/restart", Restart)
+		m.Post("/manager/flush-queues", bind(private.FlushOptions{}), FlushQueues)
+
 	}, CheckInternalToken)
 }

routers/private/manager.go

@@ -0,0 +1,41 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package private
+
+import (
+	"net/http"
+
+	"code.gitea.io/gitea/modules/graceful"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/private"
+	"code.gitea.io/gitea/modules/queue"
+
+	"gitea.com/macaron/macaron"
+)
+
+// FlushQueues flushes all the Queues
+func FlushQueues(ctx *macaron.Context, opts private.FlushOptions) {
+	if opts.NonBlocking {
+		// Save the hammer ctx here - as a new one is created each time you call this.
+		baseCtx := graceful.GetManager().HammerContext()
+		go func() {
+			err := queue.GetManager().FlushAll(baseCtx, opts.Timeout)
+			if err != nil {
+				log.Error("Flushing request timed-out with error: %v", err)
+			}
+		}()
+		ctx.JSON(http.StatusAccepted, map[string]interface{}{
+			"err": "Flushing",
+		})
+		return
+	}
+	err := queue.GetManager().FlushAll(ctx.Req.Request.Context(), opts.Timeout)
+	if err != nil {
+		ctx.JSON(http.StatusRequestTimeout, map[string]interface{}{
+			"err": err,
+		})
+	}
+	ctx.PlainText(http.StatusOK, []byte("success"))
+}

routers/private/manager_unix.go

@@ -0,0 +1,28 @@
+// +build !windows
+
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package private
+
+import (
+	"net/http"
+
+	"code.gitea.io/gitea/modules/graceful"
+
+	"gitea.com/macaron/macaron"
+)
+
+// Restart causes the server to perform a graceful restart
+func Restart(ctx *macaron.Context) {
+	graceful.GetManager().DoGracefulRestart()
+	ctx.PlainText(http.StatusOK, []byte("success"))
+
+}
+
+// Shutdown causes the server to perform a graceful shutdown
+func Shutdown(ctx *macaron.Context) {
+	graceful.GetManager().DoGracefulShutdown()
+	ctx.PlainText(http.StatusOK, []byte("success"))
+}

routers/private/manager_windows.go

@@ -0,0 +1,28 @@
+// +build windows
+
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package private
+
+import (
+	"net/http"
+
+	"code.gitea.io/gitea/modules/graceful"
+
+	"gitea.com/macaron/macaron"
+)
+
+// Restart is not implemented for Windows based servers as they can't fork
+func Restart(ctx *macaron.Context) {
+	ctx.JSON(http.StatusNotImplemented, map[string]interface{}{
+		"err": "windows servers cannot be gracefully restarted - shutdown and restart manually",
+	})
+}
+
+// Shutdown causes the server to perform a graceful shutdown
+func Shutdown(ctx *macaron.Context) {
+	graceful.GetManager().DoGracefulShutdown()
+	ctx.PlainText(http.StatusOK, []byte("success"))
+}