From 538efb9df7d729c381044fd02fd1e51a3f78ea79 Mon Sep 17 00:00:00 2001 From: sillyguodong <33891828+sillyguodong@users.noreply.github.com> Date: Thu, 14 Mar 2024 10:19:01 +0800 Subject: [PATCH] Make runs-on support variable expression (#29468) (#29782) backport #29468 Close issue: https://gitea.com/gitea/act_runner/issues/445 Follow: https://gitea.com/gitea/act/pulls/91 Move `getSecretsOfTask` and `getVariablesOfTask` under models because of circular dependency issues. --- go.mod | 8 ++-- go.sum | 16 +++---- models/actions/variable.go | 33 +++++++++++++ models/secret/secret.go | 39 +++++++++++++++ routers/api/actions/runner/utils.go | 74 +++++------------------------ services/actions/notifier_helper.go | 13 ++++- 6 files changed, 108 insertions(+), 75 deletions(-) diff --git a/go.mod b/go.mod index a5134685a3..a93fcfef81 100644 --- a/go.mod +++ b/go.mod @@ -68,11 +68,11 @@ require ( github.com/json-iterator/go v1.1.12 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/keybase/go-crypto v0.0.0-20200123153347-de78d2cb44f4 - github.com/klauspost/compress v1.17.0 + github.com/klauspost/compress v1.17.2 github.com/klauspost/cpuid/v2 v2.2.5 github.com/lib/pq v1.10.9 github.com/markbates/goth v1.78.0 - github.com/mattn/go-isatty v0.0.19 + github.com/mattn/go-isatty v0.0.20 github.com/mattn/go-sqlite3 v1.14.17 github.com/meilisearch/meilisearch-go v0.25.1 github.com/mholt/archiver/v3 v3.5.1 @@ -281,7 +281,7 @@ require ( github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/zeebo/blake3 v0.2.3 // indirect - go.etcd.io/bbolt v1.3.7 // indirect + go.etcd.io/bbolt v1.3.8 // indirect go.mongodb.org/mongo-driver v1.12.1 // indirect go.opentelemetry.io/otel v1.19.0 // indirect go.opentelemetry.io/otel/trace v1.19.0 // indirect @@ -303,7 +303,7 @@ replace github.com/hashicorp/go-version => github.com/6543/go-version v1.3.1 replace github.com/shurcooL/vfsgen => github.com/lunny/vfsgen v0.0.0-20220105142115-2c99e1ffdfa0 -replace github.com/nektos/act => gitea.com/gitea/act v0.2.51 +replace github.com/nektos/act => gitea.com/gitea/act v0.259.1 exclude github.com/gofrs/uuid v3.2.0+incompatible diff --git a/go.sum b/go.sum index 421611ffa0..0aca0255f6 100644 --- a/go.sum +++ b/go.sum @@ -53,8 +53,8 @@ dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= git.sr.ht/~mariusor/go-xsd-duration v0.0.0-20220703122237-02e73435a078 h1:cliQ4HHsCo6xi2oWZYKWW4bly/Ory9FuTpFPRxj/mAg= git.sr.ht/~mariusor/go-xsd-duration v0.0.0-20220703122237-02e73435a078/go.mod h1:g/V2Hjas6Z1UHUp4yIx6bATpNzJ7DYtD0FG3+xARWxs= -gitea.com/gitea/act v0.2.51 h1:gXc/B4OlTciTTzAx9cmNyw04n2SDO7exPjAsR5Idu+c= -gitea.com/gitea/act v0.2.51/go.mod h1:CoaX2053jqBlD6JMgu4d4UgFL/rp2I14Kt5mMqcs0Z0= +gitea.com/gitea/act v0.259.1 h1:8GG1o/xtUHl3qjn5f0h/2FXrT5ubBn05TJOM5ry+FBw= +gitea.com/gitea/act v0.259.1/go.mod h1:UxZWRYqQG2Yj4+4OqfGWW5a3HELwejyWFQyU7F1jUD8= gitea.com/go-chi/binding v0.0.0-20230415142243-04b515c6d669 h1:RUBX+MK/TsDxpHmymaOaydfigEbbzqUnG1OTZU/HAeo= gitea.com/go-chi/binding v0.0.0-20230415142243-04b515c6d669/go.mod h1:77TZu701zMXWJFvB8gvTbQ92zQ3DQq/H7l5wAEjQRKc= gitea.com/go-chi/cache v0.0.0-20210110083709-82c4c9ce2d5e/go.mod h1:k2V/gPDEtXGjjMGuBJiapffAXTv76H4snSmlJRLUhH0= @@ -654,8 +654,8 @@ github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= -github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= @@ -710,8 +710,8 @@ github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kN github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= @@ -1010,8 +1010,8 @@ github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvv github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= -go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= +go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8= diff --git a/models/actions/variable.go b/models/actions/variable.go index e0bb59ccbe..66e2007946 100644 --- a/models/actions/variable.go +++ b/models/actions/variable.go @@ -10,6 +10,7 @@ import ( "strings" "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" @@ -95,3 +96,35 @@ func UpdateVariable(ctx context.Context, variable *ActionVariable) (bool, error) }) return count != 0, err } + +func GetVariablesOfRun(ctx context.Context, run *ActionRun) (map[string]string, error) { + variables := map[string]string{} + + // Global + globalVariables, err := FindVariables(ctx, FindVariablesOpts{}) + if err != nil { + log.Error("find global variables: %v", err) + return nil, err + } + + // Org / User level + ownerVariables, err := FindVariables(ctx, FindVariablesOpts{OwnerID: run.Repo.OwnerID}) + if err != nil { + log.Error("find variables of org: %d, error: %v", run.Repo.OwnerID, err) + return nil, err + } + + // Repo level + repoVariables, err := FindVariables(ctx, FindVariablesOpts{RepoID: run.RepoID}) + if err != nil { + log.Error("find variables of repo: %d, error: %v", run.RepoID, err) + return nil, err + } + + // Level precedence: Repo > Org / User > Global + for _, v := range append(globalVariables, append(ownerVariables, repoVariables...)...) { + variables[v.Name] = v.Data + } + + return variables, nil +} diff --git a/models/secret/secret.go b/models/secret/secret.go index 8df46b6c38..7d7aef8640 100644 --- a/models/secret/secret.go +++ b/models/secret/secret.go @@ -9,7 +9,10 @@ import ( "fmt" "strings" + actions_model "code.gitea.io/gitea/models/actions" "code.gitea.io/gitea/models/db" + actions_module "code.gitea.io/gitea/modules/actions" + "code.gitea.io/gitea/modules/log" secret_module "code.gitea.io/gitea/modules/secret" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" @@ -128,3 +131,39 @@ func UpdateSecret(ctx context.Context, secretID int64, data string) error { } return err } + +func GetSecretsOfTask(ctx context.Context, task *actions_model.ActionTask) (map[string]string, error) { + secrets := map[string]string{} + + secrets["GITHUB_TOKEN"] = task.Token + secrets["GITEA_TOKEN"] = task.Token + + if task.Job.Run.IsForkPullRequest && task.Job.Run.TriggerEvent != actions_module.GithubEventPullRequestTarget { + // ignore secrets for fork pull request, except GITHUB_TOKEN and GITEA_TOKEN which are automatically generated. + // for the tasks triggered by pull_request_target event, they could access the secrets because they will run in the context of the base branch + // see the documentation: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target + return secrets, nil + } + + ownerSecrets, err := FindSecrets(ctx, FindSecretsOptions{OwnerID: task.Job.Run.Repo.OwnerID}) + if err != nil { + log.Error("find secrets of owner %v: %v", task.Job.Run.Repo.OwnerID, err) + return nil, err + } + repoSecrets, err := FindSecrets(ctx, FindSecretsOptions{RepoID: task.Job.Run.RepoID}) + if err != nil { + log.Error("find secrets of repo %v: %v", task.Job.Run.RepoID, err) + return nil, err + } + + for _, secret := range append(ownerSecrets, repoSecrets...) { + v, err := secret_module.DecryptSecret(setting.SecretKey, secret.Data) + if err != nil { + log.Error("decrypt secret %v %q: %v", secret.ID, secret.Name, err) + return nil, err + } + secrets[secret.Name] = v + } + + return secrets, nil +} diff --git a/routers/api/actions/runner/utils.go b/routers/api/actions/runner/utils.go index 24432ab6b2..5dfca8e9c3 100644 --- a/routers/api/actions/runner/utils.go +++ b/routers/api/actions/runner/utils.go @@ -14,7 +14,6 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" - secret_module "code.gitea.io/gitea/modules/secret" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/services/actions" @@ -31,14 +30,24 @@ func pickTask(ctx context.Context, runner *actions_model.ActionRunner) (*runnerv return nil, false, nil } + secrets, err := secret_model.GetSecretsOfTask(ctx, t) + if err != nil { + return nil, false, fmt.Errorf("GetSecretsOfTask: %w", err) + } + + vars, err := actions_model.GetVariablesOfRun(ctx, t.Job.Run) + if err != nil { + return nil, false, fmt.Errorf("GetVariablesOfRun: %w", err) + } + actions.CreateCommitStatus(ctx, t.Job) task := &runnerv1.Task{ Id: t.ID, WorkflowPayload: t.Job.WorkflowPayload, Context: generateTaskContext(t), - Secrets: getSecretsOfTask(ctx, t), - Vars: getVariablesOfTask(ctx, t), + Secrets: secrets, + Vars: vars, } if needs, err := findTaskNeeds(ctx, t); err != nil { @@ -54,65 +63,6 @@ func pickTask(ctx context.Context, runner *actions_model.ActionRunner) (*runnerv return task, true, nil } -func getSecretsOfTask(ctx context.Context, task *actions_model.ActionTask) map[string]string { - secrets := map[string]string{} - - secrets["GITHUB_TOKEN"] = task.Token - secrets["GITEA_TOKEN"] = task.Token - - if task.Job.Run.IsForkPullRequest && task.Job.Run.TriggerEvent != actions_module.GithubEventPullRequestTarget { - // ignore secrets for fork pull request, except GITHUB_TOKEN and GITEA_TOKEN which are automatically generated. - // for the tasks triggered by pull_request_target event, they could access the secrets because they will run in the context of the base branch - // see the documentation: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target - return secrets - } - - ownerSecrets, err := secret_model.FindSecrets(ctx, secret_model.FindSecretsOptions{OwnerID: task.Job.Run.Repo.OwnerID}) - if err != nil { - log.Error("find secrets of owner %v: %v", task.Job.Run.Repo.OwnerID, err) - // go on - } - repoSecrets, err := secret_model.FindSecrets(ctx, secret_model.FindSecretsOptions{RepoID: task.Job.Run.RepoID}) - if err != nil { - log.Error("find secrets of repo %v: %v", task.Job.Run.RepoID, err) - // go on - } - - for _, secret := range append(ownerSecrets, repoSecrets...) { - if v, err := secret_module.DecryptSecret(setting.SecretKey, secret.Data); err != nil { - log.Error("decrypt secret %v %q: %v", secret.ID, secret.Name, err) - // go on - } else { - secrets[secret.Name] = v - } - } - - return secrets -} - -func getVariablesOfTask(ctx context.Context, task *actions_model.ActionTask) map[string]string { - variables := map[string]string{} - - // Org / User level - ownerVariables, err := actions_model.FindVariables(ctx, actions_model.FindVariablesOpts{OwnerID: task.Job.Run.Repo.OwnerID}) - if err != nil { - log.Error("find variables of org: %d, error: %v", task.Job.Run.Repo.OwnerID, err) - } - - // Repo level - repoVariables, err := actions_model.FindVariables(ctx, actions_model.FindVariablesOpts{RepoID: task.Job.Run.RepoID}) - if err != nil { - log.Error("find variables of repo: %d, error: %v", task.Job.Run.RepoID, err) - } - - // Level precedence: Repo > Org / User - for _, v := range append(ownerVariables, repoVariables...) { - variables[v.Name] = v.Data - } - - return variables -} - func generateTaskContext(t *actions_model.ActionTask) *structpb.Struct { event := map[string]any{} _ = json.Unmarshal([]byte(t.Job.Run.EventPayload), &event) diff --git a/services/actions/notifier_helper.go b/services/actions/notifier_helper.go index 807494fb04..bf00868a83 100644 --- a/services/actions/notifier_helper.go +++ b/services/actions/notifier_helper.go @@ -269,7 +269,18 @@ func handleWorkflows( run.NeedApproval = need } - jobs, err := jobparser.Parse(dwf.Content) + if err := run.LoadAttributes(ctx); err != nil { + log.Error("LoadAttributes: %v", err) + continue + } + + vars, err := actions_model.GetVariablesOfRun(ctx, run) + if err != nil { + log.Error("GetVariablesOfRun: %v", err) + continue + } + + jobs, err := jobparser.Parse(dwf.Content, jobparser.WithVars(vars)) if err != nil { log.Error("jobparser.Parse: %v", err) continue