From 4192d6aa375420549391a656cf3537450648def8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20L=C3=A9one?= Date: Mon, 3 Feb 2025 12:08:25 +0100 Subject: [PATCH 1/3] ci: add support to test all cassettes --- .github/workflows/nightly.yml | 3 + .../testhelpers/cassette_validators_test.go | 118 ++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 internal/testhelpers/cassette_validators_test.go diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 17a5ebde1d..c6915593a1 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -67,6 +67,9 @@ jobs: SCW_SECRET_KEY: ${{ secrets.SCW_SECRET_KEY }} SCW_DEFAULT_ORGANIZATION_ID: ${{ secrets.SCW_DEFAULT_ORGANIZATION_ID }} SCW_DEFAULT_PROJECT_ID: ${{ secrets.SCW_DEFAULT_PROJECT_ID }} + - name: Run acceptance test for cassettes + if: success() || failure() # If the job is not cancelled, run it regardless of the result of the previous step + run: go test -v github.com/scaleway/scaleway-cli/v2/internal/testhelpers -run TestAccCassettes_Validator - name: Ping on failure if: ${{ failure() }} run: | diff --git a/internal/testhelpers/cassette_validators_test.go b/internal/testhelpers/cassette_validators_test.go new file mode 100644 index 0000000000..3fe4a326e6 --- /dev/null +++ b/internal/testhelpers/cassette_validators_test.go @@ -0,0 +1,118 @@ +package testhelpers_test + +import ( + "encoding/json" + "fmt" + "io/fs" + "net/http" + "path/filepath" + "strings" + "testing" + + "github.com/dnaeon/go-vcr/cassette" + "github.com/stretchr/testify/require" +) + +func exceptionsCassettesCases() map[string]struct{} { + return map[string]struct{}{ + "../namespaces/baremetal/v1/testdata/test-reboot-server-errors-error-cannot-be-rebooted-while-not-delivered.cassette.yaml": {}, + "../namespaces/baremetal/v1/testdata/test-start-server-errors-error-cannot-be-started-while-not-delivered.cassette.yaml": {}, + "../namespaces/baremetal/v1/testdata/test-stop-server-errors-error-cannot-be-stopped-while-not-delivered.cassette.yaml": {}, + "../namespaces/init/testdata/test-init-cl-iv2-config-no-prompt-overwrite-for-new-profile.cassette.yaml": {}, + "../namespaces/init/testdata/test-init-cl-iv2-config-prompt-overwrite-for-existing-profile.cassette.yaml": {}, + "../namespaces/init/testdata/test-init-ssh-key-unregistered.cassette.yaml": {}, + "../namespaces/init/testdata/test-init-ssh-with-local-ed25519-key.cassette.yaml": {}, + "../namespaces/instance/v1/testdata/test-server-update-no-initial-placement-group&-placement-group-id=invalid-pg-id.cassette.yaml": {}, + "../namespaces/mnq/v1beta1/testdata/test-create-context-with-wrond-id-simple.cassette.yaml": {}, + "../namespaces/mnq/v1beta1/testdata/test-create-context-with-wrong-id-wrong-account-id.cassette.yaml": {}, + "../namespaces/redis/v1/testdata/test-endpoints-edge-cases-private-endpoint-with-both-attributes-set.cassette.yaml": {}, + "../namespaces/redis/v1/testdata/test-endpoints-edge-cases-private-endpoint-with-none-set.cassette.yaml": {}, + "../namespaces/registry/v1/testdata/test-registry-install-docker-helper-command-simple.cassette.yaml": {}, + "../namespaces/registry/v1/testdata/test-registry-install-docker-helper-command-with-profile.cassette.yaml": {}, + } +} + +func fileNameWithoutExtSuffix(fileName string) string { + return strings.TrimSuffix(fileName, filepath.Ext(fileName)) +} + +// getTestFiles returns a map of cassettes files +func getTestFiles() (map[string]struct{}, error) { + filesMap := make(map[string]struct{}) + exceptions := exceptionsCassettesCases() + err := filepath.WalkDir("../namespaces", func(path string, _ fs.DirEntry, _ error) error { + isCassette := strings.Contains(path, "cassette") + _, isException := exceptions[path] + if isCassette && !isException { + filesMap[fileNameWithoutExtSuffix(path)] = struct{}{} + } + return nil + }) + if err != nil { + return nil, err + } + + return filesMap, nil +} + +func checkErrCodeExcept(i *cassette.Interaction, c *cassette.Cassette, codes ...int) bool { + exceptions := exceptionsCassettesCases() + _, isException := exceptions[c.File] + if isException { + return isException + } + if i.Response.Code >= 400 { + for _, httpCode := range codes { + if i.Response.Code == httpCode { + return true + } + } + return false + } + return true +} + +// isTransientStateError checks if the interaction response is a transient state error +// Transient state error are expected when creating resource linked to each other +// example: +// creating a gateway_network will set its public gateway to a transient state +// when creating 2 gateway_network, one will fail with a transient state error +// but the transient state error will be caught, it will wait again for the resource to be ready +func isTransientStateError(i *cassette.Interaction) bool { + if i.Response.Code != 409 { + return false + } + + scwError := struct { + Type string `json:"type"` + }{} + + err := json.Unmarshal([]byte(i.Response.Body), &scwError) + if err != nil { + return false + } + + return scwError.Type == "transient_state" +} + +func checkErrorCode(c *cassette.Cassette) error { + for _, i := range c.Interactions { + if !checkErrCodeExcept(i, c, http.StatusNotFound, http.StatusTooManyRequests, http.StatusForbidden, http.StatusGone) && + !isTransientStateError(i) { + return fmt.Errorf("status: %v found on %s. method: %s, url %s\nrequest body = %v\nresponse body = %v", i.Response.Code, c.Name, i.Request.Method, i.Request.URL, i.Request.Body, i.Response.Body) + } + } + + return nil +} + +func TestAccCassettes_Validator(t *testing.T) { + paths, err := getTestFiles() + require.NoError(t, err) + + for path := range paths { + c, err := cassette.Load(path) + require.NoError(t, err) + require.NoError(t, checkErrorCode(c)) + } +} From 19d35b5100e9687711bd055f177f0fa214190d4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20L=C3=A9one?= Date: Wed, 4 Jun 2025 15:20:15 +0200 Subject: [PATCH 2/3] Fix lint --- .../testhelpers/cassette_validators_test.go | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/internal/testhelpers/cassette_validators_test.go b/internal/testhelpers/cassette_validators_test.go index 3fe4a326e6..6ad60a1caf 100644 --- a/internal/testhelpers/cassette_validators_test.go +++ b/internal/testhelpers/cassette_validators_test.go @@ -29,6 +29,7 @@ func exceptionsCassettesCases() map[string]struct{} { "../namespaces/redis/v1/testdata/test-endpoints-edge-cases-private-endpoint-with-none-set.cassette.yaml": {}, "../namespaces/registry/v1/testdata/test-registry-install-docker-helper-command-simple.cassette.yaml": {}, "../namespaces/registry/v1/testdata/test-registry-install-docker-helper-command-with-profile.cassette.yaml": {}, + "../namespaces/config/testdata/test-config-delete-profile-command-simple.cassette.yaml": {}, } } @@ -46,6 +47,7 @@ func getTestFiles() (map[string]struct{}, error) { if isCassette && !isException { filesMap[fileNameWithoutExtSuffix(path)] = struct{}{} } + return nil }) if err != nil { @@ -67,8 +69,10 @@ func checkErrCodeExcept(i *cassette.Interaction, c *cassette.Cassette, codes ... return true } } + return false } + return true } @@ -97,9 +101,24 @@ func isTransientStateError(i *cassette.Interaction) bool { func checkErrorCode(c *cassette.Cassette) error { for _, i := range c.Interactions { - if !checkErrCodeExcept(i, c, http.StatusNotFound, http.StatusTooManyRequests, http.StatusForbidden, http.StatusGone) && + if !checkErrCodeExcept( + i, + c, + http.StatusNotFound, + http.StatusTooManyRequests, + http.StatusForbidden, + http.StatusGone, + ) && !isTransientStateError(i) { - return fmt.Errorf("status: %v found on %s. method: %s, url %s\nrequest body = %v\nresponse body = %v", i.Response.Code, c.Name, i.Request.Method, i.Request.URL, i.Request.Body, i.Response.Body) + return fmt.Errorf( + "status: %v found on %s. method: %s, url %s\nrequest body = %v\nresponse body = %v", + i.Response.Code, + c.Name, + i.Request.Method, + i.Request.URL, + i.Request.Body, + i.Response.Body, + ) } } From b3ac4780104834960b9058ef7871d9df86067c39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20L=C3=A9one?= Date: Wed, 4 Jun 2025 15:32:18 +0200 Subject: [PATCH 3/3] Fix --- internal/testhelpers/cassette_validators_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/testhelpers/cassette_validators_test.go b/internal/testhelpers/cassette_validators_test.go index 6ad60a1caf..bac7944353 100644 --- a/internal/testhelpers/cassette_validators_test.go +++ b/internal/testhelpers/cassette_validators_test.go @@ -30,6 +30,7 @@ func exceptionsCassettesCases() map[string]struct{} { "../namespaces/registry/v1/testdata/test-registry-install-docker-helper-command-simple.cassette.yaml": {}, "../namespaces/registry/v1/testdata/test-registry-install-docker-helper-command-with-profile.cassette.yaml": {}, "../namespaces/config/testdata/test-config-delete-profile-command-simple.cassette.yaml": {}, + "../namespaces/alias/testdata/test-alias-list-aliases.cassette.yaml": {}, } }