From be236ba7f07460ce065565b6052bfd68f8445fd7 Mon Sep 17 00:00:00 2001 From: Martin Dobrev Date: Tue, 17 Jun 2025 17:34:19 +0200 Subject: [PATCH] feat: add relative TaskRef fetching functionality The current implementation doesn't allow to reference tasks relative to a remote pipeline (e.g., the task definition is in the same remote location as the pipeline). The issue is described here: https://issues.redhat.com/browse/SRVKP-4332 This patch introduces the necessary functionality to enable the behaviour above. It also includes: - a multi-PipelineRun unit test - adapted docs Signed-off-by: Chmouel Boudjnah --- docs/content/docs/guide/resolver.md | 64 ++++++++ pkg/resolve/remote.go | 44 +++++- pkg/resolve/remote_test.go | 145 ++++++++++++++++-- pkg/resolve/resolve.go | 5 +- ...remote-pipeline-with-relative-tasks-1.yaml | 31 ++++ .../remote-pipeline-with-relative-tasks.yaml | 31 ++++ pkg/test/tekton/genz.go | 9 ++ 7 files changed, 314 insertions(+), 15 deletions(-) create mode 100644 pkg/resolve/testdata/remote-pipeline-with-relative-tasks-1.yaml create mode 100644 pkg/resolve/testdata/remote-pipeline-with-relative-tasks.yaml diff --git a/docs/content/docs/guide/resolver.md b/docs/content/docs/guide/resolver.md index 51016f0b4..ac2a0c8bb 100644 --- a/docs/content/docs/guide/resolver.md +++ b/docs/content/docs/guide/resolver.md @@ -201,6 +201,70 @@ out and not process the pipeline. If the object fetched cannot be parsed as a Tekton `Task` it will error out. +### Relative Tasks + +`Pipeline-as-Code` also supports fetching relative +to a remote pipeline tasks (see Section [Remote Pipeline Annotations](#remote-pipeline-annotations)). + +Consider the following scenario: + +* Repository A (where the event is originating from) contains: + +```yaml +# .tekton/pipelinerun.yaml + +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + name: hello-world + annotations: + pipelinesascode.tekton.dev/on-target-branch: "[main]" + pipelinesascode.tekton.dev/on-event: "[push]" + pipelinesascode.tekton.dev/pipeline: "https://github.com/user/repositoryb/blob/main/pipeline.yaml" +spec: + pipelineRef: + name: hello-world +``` + +* Repository B contains: + +```yaml +# pipeline.yaml + +apiVersion: tekton.dev/v1beta1 +kind: Pipeline +metadata: + name: hello-world + annotations: + pipelinesascode.tekton.dev/task: "./task.yaml" +spec: + tasks: + - name: say-hello + taskRef: + name: hello +``` + +```yaml +# task.yaml + +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: hello +spec: + steps: + - name: echo + image: alpine + script: | + #!/bin/sh + echo "Hello, World!" +``` + +The Resolver will fetch the remote pipeline and then attempt to retrieve each task. +The task paths are relative to the path where the remote pipeline, referencing them +resides (if the pipeline is at `/foo/bar/pipeline.yaml`, and the specified task path is `../task.yaml`, the +assembled target URL for fetching the task is `/foo/task.yaml`). + ## Remote Pipeline annotations Remote Pipeline can be referenced by annotation, allowing you to share a Pipeline across multiple repositories. diff --git a/pkg/resolve/remote.go b/pkg/resolve/remote.go index 501a836a2..f16296621 100644 --- a/pkg/resolve/remote.go +++ b/pkg/resolve/remote.go @@ -3,6 +3,8 @@ package resolve import ( "context" "fmt" + "net/url" + "path" "github.com/openshift-pipelines/pipelines-as-code/pkg/matcher" tektonv1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1" @@ -19,6 +21,38 @@ func alreadyFetchedResource[T NamedItem](resources map[string]T, resourceName st return false } +// Tries to assemble task FQDNs based on the base URL +// of a remote pipeline. +// +// If there isn't a remote pipeline reference for the current +// run, tasks are returned as they are. Any task with an already +// valid URL is skipped. +func assembleTaskFQDNs(pipelineURL string, tasks []string) ([]string, error) { + if pipelineURL == "" { + return tasks, nil // no pipeline URL, return tasks as is + } + + pURL, err := url.Parse(pipelineURL) + if err != nil { + return tasks, err + } + // pop the pipeline file path from the URL + pURL.Path = path.Dir(pURL.Path) + + taskURLS := make([]string, len(tasks)) + for i, t := range tasks { + tURL, err := url.Parse(t) + if err == nil && tURL.Scheme != "" && tURL.Host != "" { + taskURLS[i] = t + continue // it's already an absolute URL + } + tURL = pURL + tURL = tURL.JoinPath(t) + taskURLS[i] = tURL.String() + } + return taskURLS, nil +} + // resolveRemoteResources will get remote tasks or Pipelines from annotations. // // It already has some tasks or pipeline coming from the tekton directory stored in [types] @@ -41,7 +75,8 @@ func resolveRemoteResources(ctx context.Context, rt *matcher.RemoteTasks, types for _, pipelinerun := range types.PipelineRuns { // contain Resources specific to run fetchedResourcesForPipelineRun := FetchedResourcesForRun{ - Tasks: map[string]*tektonv1.Task{}, + Tasks: map[string]*tektonv1.Task{}, + PipelineURL: "", } var pipeline *tektonv1.Pipeline var err error @@ -76,6 +111,8 @@ func resolveRemoteResources(ctx context.Context, rt *matcher.RemoteTasks, types } // add the pipeline to the Resources fetched for the Event fetchedResourcesForEvent.Pipelines[remotePipeline] = pipeline + // add the pipeline URL to the run specific Resources + fetchedResourcesForPipelineRun.PipelineURL = remotePipeline } } } @@ -98,6 +135,11 @@ func resolveRemoteResources(ctx context.Context, rt *matcher.RemoteTasks, types if err != nil { return []*tektonv1.PipelineRun{}, fmt.Errorf("error getting remote task from pipeline annotations: %w", err) } + // check for relative task references and assemble FQDNs + pipelineTasks, err = assembleTaskFQDNs(fetchedResourcesForPipelineRun.PipelineURL, pipelineTasks) + if err != nil { + return []*tektonv1.PipelineRun{}, err + } } } diff --git a/pkg/resolve/remote_test.go b/pkg/resolve/remote_test.go index 314997632..d12612aa0 100644 --- a/pkg/resolve/remote_test.go +++ b/pkg/resolve/remote_test.go @@ -75,6 +75,62 @@ func TestRemote(t *testing.T) { pipelinewithTaskRefYamlB, err := yaml.Marshal(pipelinewithTaskRef) assert.NilError(t, err) + pipelineRelativeTaskRef := []tektonv1.PipelineTask{ + { + Name: remoteTaskName + "-a", + TaskRef: &tektonv1.TaskRef{ + Name: remoteTaskName + "-a", + }, + }, + { + Name: remoteTaskName + "-b", + TaskRef: &tektonv1.TaskRef{ + Name: remoteTaskName + "-b", + }, + }, + { + Name: remoteTaskName + "-c", + TaskRef: &tektonv1.TaskRef{ + Name: remoteTaskName + "-c", + }, + }, + { + Name: remoteTaskName + "-d", + TaskRef: &tektonv1.TaskRef{ + Name: remoteTaskName + "-d", + }, + }, + } + pipelineWithRelativeTaskRef := ttkn.MakePipeline(remotePipelineName, pipelineRelativeTaskRef[:3], map[string]string{ + apipac.Task: "./" + remoteTaskName + "-a", + apipac.Task + "-1": "../" + remoteTaskName + "-b", + apipac.Task + "-2": "../../../../" + remoteTaskName + "-c", + }) + + pipelineWithRelativeTaskRefYamlB, err := yaml.Marshal(pipelineWithRelativeTaskRef) + assert.NilError(t, err) + + pipelineWithRelativeTaskRef1 := ttkn.MakePipeline(remotePipelineName, pipelineRelativeTaskRef[1:], map[string]string{ + apipac.Task: remoteTaskName + "-b", + apipac.Task + "-1": "utils/" + remoteTaskName + "-c", + apipac.Task + "-2": " " + remoteTaskName + "-d", + }) + + pipelineWithRelativeTaskRefYamlB1, err := yaml.Marshal(pipelineWithRelativeTaskRef1) + assert.NilError(t, err) + + singleRelativeTaskBa, err := ttkn.MakeTaskB(remoteTaskName+"-a", taskFromPipelineSpec) + assert.NilError(t, err) + + singleRelativeTaskBb, err := ttkn.MakeTaskB(remoteTaskName+"-b", taskFromPipelineSpec) + assert.NilError(t, err) + + singleRelativeTaskBc, err := ttkn.MakeTaskB(remoteTaskName+"-c", taskFromPipelineSpec) + assert.NilError(t, err) + + singleRelativeTaskBd, err := ttkn.MakeTaskB(remoteTaskName+"-d", taskFromPipelineSpec) + assert.NilError(t, err) + singleTask := ttkn.MakeTask(remoteTaskName, taskFromPipelineSpec) singleTaskB, err := yaml.Marshal(singleTask) assert.NilError(t, err) @@ -92,7 +148,7 @@ func TestRemote(t *testing.T) { remoteURLS map[string]map[string]string expectedLogsSnippets []string expectedTaskSpec tektonv1.TaskSpec - expectedPipelineRun string + expectedPipelineRun []string noPipelineRun bool }{ { @@ -127,7 +183,66 @@ func TestRemote(t *testing.T) { fmt.Sprintf("successfully fetched %s from remote https url", remotePipelineURL), fmt.Sprintf("successfully fetched %s from remote https url", remoteTaskURL), }, - expectedPipelineRun: "remote-pipeline-with-remote-task-from-pipeline.yaml", + expectedPipelineRun: []string{"remote-pipeline-with-remote-task-from-pipeline.yaml"}, + }, + { + name: "remote pipelines with relative tasks", + pipelineruns: []*tektonv1.PipelineRun{ + ttkn.MakePR(randomPipelineRunName, map[string]string{ + apipac.Pipeline: remotePipelineURL, + }, + tektonv1.PipelineRunSpec{ + PipelineRef: &tektonv1.PipelineRef{ + Name: remotePipelineName, + }, + }, + ), + ttkn.MakePR(randomPipelineRunName, map[string]string{ + apipac.Pipeline: remotePipelineURL + "-1", + }, + tektonv1.PipelineRunSpec{ + PipelineRef: &tektonv1.PipelineRef{ + Name: remotePipelineName, + }, + }, + ), + }, + remoteURLS: map[string]map[string]string{ + remotePipelineURL: { + "body": string(pipelineWithRelativeTaskRefYamlB), + "code": "200", + }, + remotePipelineURL + "-1": { + "body": string(pipelineWithRelativeTaskRefYamlB1), + "code": "200", + }, + remoteTaskURL + "-a": { + "body": string(singleRelativeTaskBa), + "code": "200", + }, + remoteTaskURL + "-b": { + "body": string(singleRelativeTaskBb), + "code": "200", + }, + remoteTaskURL + "-c": { + "body": string(singleRelativeTaskBc), + "code": "200", + }, + "http://remote/utils/remote-task-c": { + "body": string(singleRelativeTaskBc), + "code": "200", + }, + remoteTaskURL + "-d": { + "body": string(singleRelativeTaskBd), + "code": "200", + }, + }, + expectedTaskSpec: taskFromPipelineSpec, + expectedLogsSnippets: []string{}, + expectedPipelineRun: []string{ + "remote-pipeline-with-relative-tasks.yaml", + "remote-pipeline-with-relative-tasks-1.yaml", + }, }, { name: "remote pipeline with remote task in pipeline overridden from pipelinerun", @@ -162,7 +277,7 @@ func TestRemote(t *testing.T) { fmt.Sprintf("successfully fetched %s from remote https url", remotePipelineURL), fmt.Sprintf("successfully fetched %s from remote https url", taskFromPipelineRunURL), }, - expectedPipelineRun: "remote-pipeline-with-remote-task-from-pipelinerun.yaml", + expectedPipelineRun: []string{"remote-pipeline-with-remote-task-from-pipelinerun.yaml"}, }, { name: "remote pipelinerun no annotations", @@ -222,7 +337,7 @@ func TestRemote(t *testing.T) { fmt.Sprintf("successfully fetched %s from remote https url", remotePipelineURL), fmt.Sprintf("successfully fetched %s from remote https url", remoteTaskURL), }, - expectedPipelineRun: "skip-fetching-multiple-tasks-of-the-same-name-from-pipelinerun-annotations-and-pipeline-annotation.yaml", + expectedPipelineRun: []string{"skip-fetching-multiple-tasks-of-the-same-name-from-pipelinerun-annotations-and-pipeline-annotation.yaml"}, }, { name: "skip fetching multiple tasks of the same name from pipelinerun annotations and tektondir", @@ -258,7 +373,7 @@ func TestRemote(t *testing.T) { fmt.Sprintf("skipping remote task %s as already fetched task %s for pipelinerun %s", remoteTaskURL, remoteTaskName, randomPipelineRunName), fmt.Sprintf("overriding task %s coming from .tekton directory by an annotation task for pipelinerun %s", remoteTaskName, randomPipelineRunName), }, - expectedPipelineRun: "skip-fetching-multiple-tasks-of-the-same-name-from-pipelinerun-annotations-and-tektondir.yaml", + expectedPipelineRun: []string{"skip-fetching-multiple-tasks-of-the-same-name-from-pipelinerun-annotations-and-tektondir.yaml"}, }, { name: "skip fetching multiple pipelines of the same name from pipelinerun annotations and tektondir", @@ -293,7 +408,7 @@ func TestRemote(t *testing.T) { fmt.Sprintf("successfully fetched %s from remote https url", remoteTaskURL), fmt.Sprintf("skipping remote task %s as already fetched task %s for pipelinerun %s", remoteTaskURL, remoteTaskName, randomPipelineRunName), }, - expectedPipelineRun: "skip-fetching-multiple-pipelines-of-the-same-name-from-pipelinerun-annotations-and-tektondir.yaml", + expectedPipelineRun: []string{"skip-fetching-multiple-pipelines-of-the-same-name-from-pipelinerun-annotations-and-tektondir.yaml"}, }, } for _, tt := range tests { @@ -334,12 +449,18 @@ func TestRemote(t *testing.T) { assert.Assert(t, len(ret) == 0, "not expecting any pipelinerun") return } - expectedData, err := os.ReadFile("testdata/" + tt.expectedPipelineRun) - assert.NilError(t, err) - pipelineRun := &tektonv1.PipelineRun{} - err = yaml.Unmarshal(expectedData, pipelineRun) - assert.NilError(t, err) - assert.DeepEqual(t, pipelineRun, ret[0]) + for i, pr := range ret { + if len(tt.expectedPipelineRun) < len(ret) { + assert.NilError(t, fmt.Errorf("insufficient amount of expectedPipelineRuns was provided, got %d but want %d; or set noPipelineRun to true", + len(tt.expectedPipelineRun), len(ret))) + } + expectedData, err := os.ReadFile("testdata/" + tt.expectedPipelineRun[i]) + assert.NilError(t, err) + pipelineRun := &tektonv1.PipelineRun{} + err = yaml.Unmarshal(expectedData, pipelineRun) + assert.NilError(t, err) + assert.DeepEqual(t, pipelineRun, pr) + } }) } } diff --git a/pkg/resolve/resolve.go b/pkg/resolve/resolve.go index f8f853eef..55ac2c3aa 100644 --- a/pkg/resolve/resolve.go +++ b/pkg/resolve/resolve.go @@ -38,8 +38,9 @@ type FetchedResources struct { // Contains Fetched Resources for Run, with key equals to resource name from metadata.name field. type FetchedResourcesForRun struct { - Tasks map[string]*tektonv1.Task - Pipeline *tektonv1.Pipeline + Tasks map[string]*tektonv1.Task + Pipeline *tektonv1.Pipeline + PipelineURL string } func NewTektonTypes() TektonTypes { diff --git a/pkg/resolve/testdata/remote-pipeline-with-relative-tasks-1.yaml b/pkg/resolve/testdata/remote-pipeline-with-relative-tasks-1.yaml new file mode 100644 index 000000000..514906afb --- /dev/null +++ b/pkg/resolve/testdata/remote-pipeline-with-relative-tasks-1.yaml @@ -0,0 +1,31 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + pipelinesascode.tekton.dev/pipeline: http://remote/remote-pipeline-1 + generateName: pipelinerun-abc- +spec: + pipelineSpec: + tasks: + - name: remote-task-b + taskSpec: + steps: + - name: step1 + image: scratch + command: + - "true" + - name: remote-task-c + taskSpec: + steps: + - name: step1 + image: scratch + command: + - "true" + - name: remote-task-d + taskSpec: + steps: + - name: step1 + image: scratch + command: + - "true" + finally: [] \ No newline at end of file diff --git a/pkg/resolve/testdata/remote-pipeline-with-relative-tasks.yaml b/pkg/resolve/testdata/remote-pipeline-with-relative-tasks.yaml new file mode 100644 index 000000000..54179ac36 --- /dev/null +++ b/pkg/resolve/testdata/remote-pipeline-with-relative-tasks.yaml @@ -0,0 +1,31 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + pipelinesascode.tekton.dev/pipeline: http://remote/remote-pipeline + generateName: pipelinerun-abc- +spec: + pipelineSpec: + tasks: + - name: remote-task-a + taskSpec: + steps: + - name: step1 + image: scratch + command: + - "true" + - name: remote-task-b + taskSpec: + steps: + - name: step1 + image: scratch + command: + - "true" + - name: remote-task-c + taskSpec: + steps: + - name: step1 + image: scratch + command: + - "true" + finally: [] \ No newline at end of file diff --git a/pkg/test/tekton/genz.go b/pkg/test/tekton/genz.go index 5cb8bae39..9f9a1b7f0 100644 --- a/pkg/test/tekton/genz.go +++ b/pkg/test/tekton/genz.go @@ -9,6 +9,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" knativeapi "knative.dev/pkg/apis" knativeduckv1 "knative.dev/pkg/apis/duck/v1" + "sigs.k8s.io/yaml" ) func MakePrTrStatus(ptaskname, displayName string, completionmn int) *tektonv1.PipelineRunTaskRunStatus { @@ -149,6 +150,14 @@ func MakeTask(name string, taskSpec tektonv1.TaskSpec) *tektonv1.Task { } } +func MakeTaskB(name string, taskSpec tektonv1.TaskSpec) ([]byte, error) { + objB, err := yaml.Marshal(MakeTask(name, taskSpec)) + if err != nil { + return nil, err + } + return objB, nil +} + func MakeTaskRunCompletion(clock *clockwork.FakeClock, name, namespace, runstatus string, annotation map[string]string, taskStatus tektonv1.TaskRunStatusFields, conditions knativeduckv1.Conditions, timeshift int) *tektonv1.TaskRun { starttime := time.Duration((timeshift - 5*-1) * int(time.Minute)) endtime := time.Duration((timeshift * -1) * int(time.Minute))