Skip to content

Commit cd86752

Browse files
motatoesmotatoes
authored
feat/next config workflows (#1840)
* supporting workflows in next
1 parent 0ad85ea commit cd86752

File tree

17 files changed

+169
-75
lines changed

17 files changed

+169
-75
lines changed

‎backend/controllers/cache.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ func (d DiggerController) UpdateRepoCache(c *gin.Context) {
6767

6868
// update the cache here, do it async for immediate response
6969
go func() {
70-
err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, *token, func(dir string) error {
70+
err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, func(dir string) error {
7171
diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
7272
diggerYmlStr = string(diggerYmlBytes)
7373
config, _, _, err = dg_configuration.LoadDiggerConfig(dir, true, nil)

‎backend/controllers/github.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -553,7 +553,7 @@ func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6
553553
var diggerYmlStr string
554554
var dependencyGraph graph.Graph[string, dg_configuration.Project]
555555

556-
err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, *token, func(dir string) error {
556+
err = utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, func(dir string) error {
557557
diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
558558
diggerYmlStr = string(diggerYmlBytes)
559559
config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles)

‎backend/utils/github.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,18 @@ func createTempDir() string {
2828

2929
type action func(string) error
3030

31-
func CloneGitRepoAndDoAction(repoUrl string, branch string, token string, action action) error {
31+
func CloneGitRepoAndDoAction(repoUrl string, branch string, commitHash string, token string, action action) error {
3232
dir := createTempDir()
3333
git := NewGitShellWithTokenAuth(dir, token)
3434
err := git.Clone(repoUrl, branch)
3535
if err != nil {
3636
return err
3737
}
3838

39+
if commitHash != "" {
40+
git.Checkout(commitHash)
41+
}
42+
3943
defer func() {
4044
log.Printf("removing cloned directory %v", dir)
4145
ferr := os.RemoveAll(dir)

‎backend/utils/github_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,14 @@ func init() {
1414

1515
func TestGithubCloneWithInvalidTokenThrowsErr(t *testing.T) {
1616
f := func(d string) error { return nil }
17-
err := CloneGitRepoAndDoAction("https://github.com/diggerhq/private-repo", "main", "invalid-token", f)
17+
err := CloneGitRepoAndDoAction("https://github.com/diggerhq/private-repo", "main", "", "invalid-token", f)
1818
assert.NotNil(t, err)
1919
}
2020

2121
func TestGithubCloneWithPublicRepoThrowsNoError(t *testing.T) {
2222
token := os.Getenv("GITHUB_PAT_TOKEN")
2323
f := func(d string) error { return nil }
24-
err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "develop", token, f)
24+
err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "develop", "", token, f)
2525
assert.Nil(t, err)
2626
}
2727

@@ -32,13 +32,13 @@ func TestGithubCloneWithPrivateRepoAndValidTokenThrowsNoError(t *testing.T) {
3232
return
3333
}
3434
f := func(d string) error { return nil }
35-
err := CloneGitRepoAndDoAction("https://github.com/diggerhq/infra-gcp", "main", token, f)
35+
err := CloneGitRepoAndDoAction("https://github.com/diggerhq/infra-gcp", "main", "", token, f)
3636
assert.Nil(t, err)
3737
}
3838

3939
func TestGithubCloneWithInvalidBranchThrowsError(t *testing.T) {
4040
token := os.Getenv("GITHUB_PAT_TOKEN")
4141
f := func(d string) error { return nil }
42-
err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "not-a-branch", token, f)
42+
err := CloneGitRepoAndDoAction("https://github.com/diggerhq/digger", "not-a-branch", "", token, f)
4343
assert.NotNil(t, err)
4444
}

‎backend/utils/gitlab.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ func GetDiggerConfigForBranch(gh GitlabProvider, projectId int, repoFullName str
6262
log.Printf("Error getting changed files: %v", err)
6363
return "", nil, nil, fmt.Errorf("error getting changed files")
6464
}
65-
err = CloneGitRepoAndDoAction(cloneUrl, branch, token, func(dir string) error {
65+
err = CloneGitRepoAndDoAction(cloneUrl, branch, "", token, func(dir string) error {
6666
diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
6767
diggerYmlStr = string(diggerYmlBytes)
6868
config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles)

‎backend/utils/gitshell.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,13 @@ func (g *GitShell) runCommand(args ...string) (string, error) {
103103
return strings.TrimSpace(stdout.String()), nil
104104
}
105105

106+
func (g *GitShell) Checkout(branchOrCommit string) error {
107+
args := []string{"checkout"}
108+
args = append(args, branchOrCommit)
109+
_, err := g.runCommand(args...)
110+
return err
111+
}
112+
106113
// Clone with authentication
107114
func (g *GitShell) Clone(repoURL, branch string) error {
108115
authURL, err := g.formatAuthURL(repoURL)

‎cli/pkg/spec/spec.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,13 @@ func RunSpec(
4343
usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get backend api: %v", err), 1)
4444
}
4545

46+
// for additional output reporting
47+
diggerOutPath := os.Getenv("DIGGER_OUT")
48+
if diggerOutPath == "" {
49+
diggerOutPath = os.Getenv("RUNNER_TEMP") + "/digger-out.log"
50+
os.Setenv("DIGGER_OUT", diggerOutPath)
51+
}
52+
4653
if spec.Job.Commit != "" {
4754
// checking out to the commit ID
4855
log.Printf("fetching commit ID %v", spec.Job.Commit)

‎ee/backend/controllers/gitlab.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ func handlePushEvent(gh utils.GitlabProvider, payload *gitlab.PushEvent, organis
156156
isMainBranch = false
157157
}
158158

159-
err = utils.CloneGitRepoAndDoAction(cloneURL, pushBranch, token, func(dir string) error {
159+
err = utils.CloneGitRepoAndDoAction(cloneURL, pushBranch, "", token, func(dir string) error {
160160
config, err := dg_configuration.LoadDiggerConfigYaml(dir, true, nil)
161161
if err != nil {
162162
log.Printf("ERROR load digger.yml: %v", err)

‎ee/drift/tasks/github.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ func LoadProjectsFromGithubRepo(gh utils2.GithubClientProvider, installationId s
4242
return fmt.Errorf("error getting github service")
4343
}
4444

45-
err = utils3.CloneGitRepoAndDoAction(cloneUrl, branch, *token, func(dir string) error {
45+
err = utils3.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, func(dir string) error {
4646
config, err := dg_configuration.LoadDiggerConfigYaml(dir, true, nil)
4747
if err != nil {
4848
log.Printf("ERROR load digger.yml: %v", err)

‎ee/drift/utils/github.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6
103103

104104
var changedFiles []string = nil
105105

106-
err = utils2.CloneGitRepoAndDoAction(cloneUrl, branch, *token, func(dir string) error {
106+
err = utils2.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, func(dir string) error {
107107
diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
108108
diggerYmlStr = string(diggerYmlBytes)
109109
config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles)

‎libs/digger_config/digger_config.go

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,30 @@ type FileSystemTerragruntDirWalker struct {
3030
type FileSystemModuleDirWalker struct {
3131
}
3232

33+
func CheckOrCreateDiggerFile(dir string) error {
34+
// Check for digger.yml
35+
ymlPath := filepath.Join(dir, "digger.yml")
36+
yamlPath := filepath.Join(dir, "digger.yaml")
37+
38+
// Check if either file exists
39+
if _, err := os.Stat(ymlPath); err == nil {
40+
return nil // digger.yml exists
41+
}
42+
if _, err := os.Stat(yamlPath); err == nil {
43+
return nil // digger.yaml exists
44+
}
45+
46+
// Neither file exists, create digger.yml
47+
file, err := os.Create(ymlPath)
48+
if err != nil {
49+
return err
50+
}
51+
defer file.Close()
52+
53+
// File is created empty by default
54+
return nil
55+
}
56+
3357
func GetFilesWithExtension(workingDir string, ext string) ([]string, error) {
3458
var files []string
3559
listOfFiles, err := os.ReadDir(workingDir)
@@ -363,9 +387,6 @@ func LoadDiggerConfigYaml(workingDir string, generateProjects bool, changedFiles
363387
}
364388

365389
func ValidateDiggerConfigYaml(configYaml *DiggerConfigYaml, fileName string) error {
366-
if (configYaml.Projects == nil || len(configYaml.Projects) == 0) && configYaml.GenerateProjectsConfig == nil {
367-
return fmt.Errorf("no projects digger_config found in '%s'", fileName)
368-
}
369390
if configYaml.DependencyConfiguration != nil {
370391
if configYaml.DependencyConfiguration.Mode != DependencyConfigurationHard && configYaml.DependencyConfiguration.Mode != DependencyConfigurationSoft {
371392
return fmt.Errorf("dependency digger_config mode can only be '%s' or '%s'", DependencyConfigurationHard, DependencyConfigurationSoft)

‎libs/digger_config/digger_config_test.go

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -419,8 +419,9 @@ func TestMissingProjectsReturnsError(t *testing.T) {
419419
`
420420
deleteFile := createFile(path.Join(tempDir, "digger.yaml"), diggerCfg)
421421
defer deleteFile()
422-
_, _, _, err := LoadDiggerConfig(tempDir, true, nil)
423-
assert.ErrorContains(t, err, "no projects digger_config found")
422+
config, _, _, err := LoadDiggerConfig(tempDir, true, nil)
423+
assert.Nil(t, err)
424+
assert.Equal(t, len(config.Projects), 0)
424425
}
425426

426427
func TestDiggerConfigCustomWorkflow(t *testing.T) {
@@ -941,9 +942,9 @@ func TestDiggerGenerateProjectsEmptyParameters(t *testing.T) {
941942
diggerCfg := `
942943
generate_projects:
943944
`
944-
_, _, _, err := LoadDiggerConfigFromString(diggerCfg, "./")
945-
assert.Error(t, err)
946-
assert.Equal(t, "no projects digger_config found in 'loaded_yaml_string'", err.Error())
945+
config, _, _, err := LoadDiggerConfigFromString(diggerCfg, "./")
946+
assert.Nil(t, err)
947+
assert.Equal(t, 0, len(config.Projects))
947948
}
948949

949950
// TestDiggerGenerateProjectsTooManyParameters include/exclude and blocks of include/exclude can't be used together

‎next/controllers/github.go

Lines changed: 17 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -80,17 +80,19 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) {
8080
err := handlePullRequestEvent(gh, event, d.CiBackendProvider)
8181
if err != nil {
8282
log.Printf("handlePullRequestEvent error: %v", err)
83-
c.String(http.StatusInternalServerError, err.Error())
84-
return
8583
}
84+
c.String(http.StatusAccepted, err.Error())
85+
return
86+
8687
case *github.PushEvent:
8788
log.Printf("Got push event for %d", event.Repo.URL)
8889
handlePushEventApplyAfterMerge(gh, event)
8990
if err != nil {
9091
log.Printf("handlePushEvent error: %v", err)
91-
c.String(http.StatusInternalServerError, err.Error())
92-
return
9392
}
93+
c.String(http.StatusAccepted, err.Error())
94+
return
95+
9496
default:
9597
log.Printf("Unhandled event, event type %v", reflect.TypeOf(event))
9698
}
@@ -357,30 +359,23 @@ func handlePullRequestEvent(gh next_utils.GithubClientProvider, payload *github.
357359
}
358360

359361
projectsGraph, err := dg_configuration.CreateProjectDependencyGraph(dgprojects)
362+
workflows, err := services.GetWorkflowsForRepoAndBranch(gh, repo.ID, sourceBranch, commitSha)
363+
if err != nil {
364+
log.Printf("error getting workflows from config: %v", err)
365+
return fmt.Errorf("error getting workflows from config")
366+
}
360367
var config *dg_configuration.DiggerConfig = &dg_configuration.DiggerConfig{
361368
ApplyAfterMerge: true,
362369
AllowDraftPRs: false,
363370
CommentRenderMode: "",
364371
DependencyConfiguration: dg_configuration.DependencyConfiguration{
365372
Mode: dg_configuration.DependencyConfigurationHard,
366373
},
367-
PrLocks: false,
368-
Projects: dgprojects,
369-
AutoMerge: false,
370-
Telemetry: false,
371-
Workflows: map[string]dg_configuration.Workflow{
372-
"default": dg_configuration.Workflow{
373-
EnvVars: nil,
374-
Plan: nil,
375-
Apply: nil,
376-
Configuration: &dg_configuration.WorkflowConfiguration{
377-
OnPullRequestPushed: []string{"digger plan"},
378-
OnPullRequestClosed: []string{},
379-
OnPullRequestConvertedToDraft: []string{},
380-
OnCommitToDefault: []string{},
381-
},
382-
},
383-
},
374+
PrLocks: false,
375+
Projects: dgprojects,
376+
AutoMerge: false,
377+
Telemetry: false,
378+
Workflows: workflows,
384379
MentionDriftedProjectsInPR: false,
385380
TraverseToNestedProjects: false,
386381
}
@@ -540,7 +535,7 @@ func getDiggerConfigForBranch(gh next_utils.GithubClientProvider, installationId
540535
log.Printf("Error getting changed files: %v", err)
541536
return "", nil, nil, nil, fmt.Errorf("error getting changed files")
542537
}
543-
err = backend_utils.CloneGitRepoAndDoAction(cloneUrl, branch, *token, func(dir string) error {
538+
err = backend_utils.CloneGitRepoAndDoAction(cloneUrl, branch, "", *token, func(dir string) error {
544539
diggerYmlBytes, err := os.ReadFile(path.Join(dir, "digger.yml"))
545540
diggerYmlStr = string(diggerYmlBytes)
546541
config, _, dependencyGraph, err = dg_configuration.LoadDiggerConfig(dir, true, changedFiles)

‎next/controllers/github_after_merge.go

Lines changed: 11 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
dg_configuration "github.com/diggerhq/digger/libs/digger_config"
99
"github.com/diggerhq/digger/libs/scheduler"
1010
"github.com/diggerhq/digger/next/dbmodels"
11+
"github.com/diggerhq/digger/next/services"
1112
nextutils "github.com/diggerhq/digger/next/utils"
1213
"github.com/google/go-github/v61/github"
1314
"log"
@@ -66,30 +67,23 @@ func handlePushEventApplyAfterMerge(gh nextutils.GithubClientProvider, payload *
6667
}
6768
}
6869
projectsGraph, err := dg_configuration.CreateProjectDependencyGraph(dgprojects)
70+
workflows, err := services.GetWorkflowsForRepoAndBranch(gh, repo.ID, targetBranch, commitId)
71+
if err != nil {
72+
log.Printf("error getting workflows from config: %v", err)
73+
return fmt.Errorf("error getting workflows from config")
74+
}
6975
var config *dg_configuration.DiggerConfig = &dg_configuration.DiggerConfig{
7076
ApplyAfterMerge: true,
7177
AllowDraftPRs: false,
7278
CommentRenderMode: "",
7379
DependencyConfiguration: dg_configuration.DependencyConfiguration{
7480
Mode: dg_configuration.DependencyConfigurationHard,
7581
},
76-
PrLocks: false,
77-
Projects: dgprojects,
78-
AutoMerge: false,
79-
Telemetry: false,
80-
Workflows: map[string]dg_configuration.Workflow{
81-
"default": dg_configuration.Workflow{
82-
EnvVars: nil,
83-
Plan: nil,
84-
Apply: nil,
85-
Configuration: &dg_configuration.WorkflowConfiguration{
86-
OnPullRequestPushed: []string{"digger plan"},
87-
OnPullRequestClosed: []string{},
88-
OnPullRequestConvertedToDraft: []string{},
89-
OnCommitToDefault: []string{},
90-
},
91-
},
92-
},
82+
PrLocks: false,
83+
Projects: dgprojects,
84+
AutoMerge: false,
85+
Telemetry: false,
86+
Workflows: workflows,
9387
MentionDriftedProjectsInPR: false,
9488
TraverseToNestedProjects: false,
9589
}

‎next/services/config.go

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
package services
2+
3+
import (
4+
"fmt"
5+
utils3 "github.com/diggerhq/digger/backend/utils"
6+
dg_configuration "github.com/diggerhq/digger/libs/digger_config"
7+
"github.com/diggerhq/digger/next/dbmodels"
8+
"github.com/diggerhq/digger/next/utils"
9+
"log"
10+
)
11+
12+
func GetWorkflowsForRepoAndBranch(gh utils.GithubClientProvider, repoId int64, branch string, commitHash string) (map[string]dg_configuration.Workflow, error) {
13+
r := dbmodels.DB.Query.Repo
14+
repo, err := dbmodels.DB.Query.Repo.Where(r.ID.Eq(repoId)).First()
15+
if err != nil {
16+
log.Printf("could not find repo: %v : %v", repoId, err)
17+
return nil, fmt.Errorf("could not find repo: %v: %v", repoId, err)
18+
}
19+
repoOwner := repo.RepoOrganisation
20+
repoFullName := repo.RepoFullName
21+
repoName := repo.RepoName
22+
orgId := repo.OrganizationID
23+
24+
appInstallation, err := dbmodels.DB.GetGithubAppInstallationByOrgAndRepo(orgId, repoFullName, dbmodels.GithubAppInstallActive)
25+
if err != nil {
26+
log.Printf("error retrieving app installation")
27+
return nil, fmt.Errorf("error retrieving app installation %v", err)
28+
}
29+
installationId := appInstallation.GithubInstallationID
30+
log.Printf("installation id is: %v", installationId)
31+
32+
cloneUrl := fmt.Sprintf("https://%v/%v", utils.GetGithubHostname(), repo.RepoFullName)
33+
34+
_, token, err := utils.GetGithubService(gh, installationId, repoFullName, repoOwner, repoName)
35+
if err != nil {
36+
log.Printf("could not get github service :%v", err)
37+
return nil, fmt.Errorf("could not get github service :%v", err)
38+
}
39+
40+
var config *dg_configuration.DiggerConfig
41+
42+
err = utils3.CloneGitRepoAndDoAction(cloneUrl, branch, commitHash, *token, func(dir string) error {
43+
// we create a blank file if it does not exist
44+
err := dg_configuration.CheckOrCreateDiggerFile(dir)
45+
if err != nil {
46+
log.Printf("Error creating blank digger.yml if not exists: %v", err)
47+
return err
48+
}
49+
config, _, _, err = dg_configuration.LoadDiggerConfig(dir, false, nil)
50+
if err != nil {
51+
log.Printf("Error loading digger config: %v", err)
52+
return err
53+
}
54+
return nil
55+
})
56+
57+
if err != nil {
58+
log.Printf("could not load digger config :%v", err)
59+
return nil, fmt.Errorf("could not load digger config :%v", err)
60+
}
61+
62+
return config.Workflows, nil
63+
}

0 commit comments

Comments
 (0)