From 1078fe67c817795a52162d79da734e8b6748daa1 Mon Sep 17 00:00:00 2001 From: Chris Bandy Date: Wed, 6 Nov 2024 17:03:51 -0600 Subject: [PATCH 1/3] Move pgbackrest-restore test to Kyverno Chainsaw This test has a number of scripts and jobs that pass and share data. Chainsaw's bindings and templates are a nice way to break this test up, and its "catch" operations provide good context when a step fails. Tested with Kyverno Chainsaw 0.2.12 See: https://kyverno.github.io/chainsaw/main --- .github/workflows/test.yaml | 12 +- Makefile | 13 ++ testing/chainsaw/e2e/config.yaml | 11 ++ .../pgbackrest-restore/01-create-cluster.yaml | 55 ++++++++ .../pgbackrest-restore/02-create-data.yaml | 52 ++++++++ .../pgbackrest-restore/03-create-backup.yaml | 33 +++++ .../pgbackrest-restore/11-update-cluster.yaml | 83 ++++++++++++ .../pgbackrest-restore/12-update-data.yaml | 74 +++++++++++ .../e2e/pgbackrest-restore/21-lose-data.yaml | 125 ++++++++++++++++++ .../22-point-in-time-restore.yaml | 70 ++++++++++ .../pgbackrest-restore/23-verify-primary.yaml | 62 +++++++++ .../pgbackrest-restore/23-verify-replica.yaml | 120 +++++++++++++++++ .../e2e/pgbackrest-restore/chainsaw-test.yaml | 105 +++++++++++++++ .../e2e/pgbackrest-restore/clone-cluster.yaml | 50 +++++++ .../pgbackrest-restore/verify-02-data.yaml | 83 ++++++++++++ .../pgbackrest-restore/verify-12-data.yaml | 85 ++++++++++++ testing/chainsaw/e2e/values.yaml | 5 + .../01--create-cluster.yaml | 24 ---- .../e2e/pgbackrest-restore/01-assert.yaml | 12 -- .../pgbackrest-restore/02--create-data.yaml | 32 ----- .../e2e/pgbackrest-restore/02-assert.yaml | 7 - .../e2e/pgbackrest-restore/03--backup.yaml | 8 -- .../e2e/pgbackrest-restore/03-assert.yaml | 13 -- .../pgbackrest-restore/04--clone-cluster.yaml | 22 --- .../e2e/pgbackrest-restore/04-assert.yaml | 12 -- .../pgbackrest-restore/05--check-data.yaml | 49 ------- .../e2e/pgbackrest-restore/05-assert.yaml | 7 - .../pgbackrest-restore/06--delete-clone.yaml | 8 -- .../e2e/pgbackrest-restore/07--annotate.yaml | 18 --- .../07--update-cluster.yaml | 23 ---- .../pgbackrest-restore/08--wait-restart.yaml | 29 ---- .../e2e/pgbackrest-restore/09--add-data.yaml | 31 ----- .../e2e/pgbackrest-restore/09-assert.yaml | 7 - .../pgbackrest-restore/10--wait-archived.yaml | 18 --- .../pgbackrest-restore/11--clone-cluster.yaml | 22 --- .../e2e/pgbackrest-restore/11-assert.yaml | 12 -- .../pgbackrest-restore/12--check-data.yaml | 51 ------- .../e2e/pgbackrest-restore/12-assert.yaml | 7 - .../pgbackrest-restore/13--delete-clone.yaml | 8 -- .../e2e/pgbackrest-restore/14--lose-data.yaml | 50 ------- .../pgbackrest-restore/15--in-place-pitr.yaml | 50 ------- .../e2e/pgbackrest-restore/15-assert.yaml | 16 --- .../pgbackrest-restore/16--check-data.yaml | 100 -------------- .../e2e/pgbackrest-restore/16-assert.yaml | 15 --- .../17--check-replication.yaml | 22 --- 45 files changed, 1032 insertions(+), 679 deletions(-) create mode 100644 testing/chainsaw/e2e/config.yaml create mode 100644 testing/chainsaw/e2e/pgbackrest-restore/01-create-cluster.yaml create mode 100644 testing/chainsaw/e2e/pgbackrest-restore/02-create-data.yaml create mode 100644 testing/chainsaw/e2e/pgbackrest-restore/03-create-backup.yaml create mode 100644 testing/chainsaw/e2e/pgbackrest-restore/11-update-cluster.yaml create mode 100644 testing/chainsaw/e2e/pgbackrest-restore/12-update-data.yaml create mode 100644 testing/chainsaw/e2e/pgbackrest-restore/21-lose-data.yaml create mode 100644 testing/chainsaw/e2e/pgbackrest-restore/22-point-in-time-restore.yaml create mode 100644 testing/chainsaw/e2e/pgbackrest-restore/23-verify-primary.yaml create mode 100644 testing/chainsaw/e2e/pgbackrest-restore/23-verify-replica.yaml create mode 100644 testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml create mode 100644 testing/chainsaw/e2e/pgbackrest-restore/clone-cluster.yaml create mode 100644 testing/chainsaw/e2e/pgbackrest-restore/verify-02-data.yaml create mode 100644 testing/chainsaw/e2e/pgbackrest-restore/verify-12-data.yaml create mode 100644 testing/chainsaw/e2e/values.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/01--create-cluster.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/01-assert.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/02--create-data.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/02-assert.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/03--backup.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/03-assert.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/04--clone-cluster.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/04-assert.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/05--check-data.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/05-assert.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/06--delete-clone.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/07--annotate.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/07--update-cluster.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/08--wait-restart.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/09--add-data.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/09-assert.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/10--wait-archived.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/11--clone-cluster.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/11-assert.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/12--check-data.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/12-assert.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/13--delete-clone.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/14--lose-data.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/15--in-place-pitr.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/15-assert.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/16--check-data.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/16-assert.yaml delete mode 100644 testing/kuttl/e2e/pgbackrest-restore/17--check-replication.yaml diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1f6b754518..9f86493c13 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -144,10 +144,12 @@ jobs: --env 'RELATED_IMAGE_COLLECTOR=registry.developers.crunchydata.com/crunchydata/postgres-operator:ubi9-5.8.2-0' \ --env 'PGO_FEATURE_GATES=TablespaceVolumes=true,OpenTelemetryLogs=true,OpenTelemetryMetrics=true' \ --name 'postgres-operator' localhost/postgres-operator - - name: Install kuttl - run: | - curl -Lo /usr/local/bin/kubectl-kuttl https://github.com/kudobuilder/kuttl/releases/download/v0.13.0/kubectl-kuttl_0.13.0_linux_x86_64 - chmod +x /usr/local/bin/kubectl-kuttl + + - run: | + make check-chainsaw && exit + failed=$? + echo '::group::PGO logs'; docker logs 'postgres-operator'; echo '::endgroup::' + exit $failed - run: make generate-kuttl env: @@ -161,8 +163,6 @@ jobs: failed=$? echo '::group::PGO logs'; docker logs 'postgres-operator'; echo '::endgroup::' exit $failed - env: - KUTTL: kubectl-kuttl - name: Stop PGO run: docker stop 'postgres-operator' || true diff --git a/Makefile b/Makefile index 2e553d101b..680884e0f8 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,8 @@ GO_TEST ?= $(GO) test CONTROLLER ?= $(GO) tool sigs.k8s.io/controller-tools/cmd/controller-gen # Run tests using the latest tools. +CHAINSAW ?= $(GO) run github.com/kyverno/chainsaw@latest +CHAINSAW_TEST ?= $(CHAINSAW) test ENVTEST ?= $(GO) run sigs.k8s.io/controller-runtime/tools/setup-envtest@latest KUTTL ?= $(GO) run github.com/kudobuilder/kuttl/cmd/kubectl-kuttl@latest KUTTL_TEST ?= $(KUTTL) test @@ -170,6 +172,17 @@ check-envtest-existing: createnamespaces $(GO_TEST) -count=1 -cover -p=1 ./... kubectl delete -k ./config/dev +# Expects operator to be running +# +# Chainsaw runs with a single kubectl context named "chainsaw". +# If you experience `cluster "minikube" does not exist`, try `MINIKUBE_PROFILE=chainsaw`. +# +# https://kyverno.github.io/chainsaw/latest/operations/script#kubeconfig +# +.PHONY: check-chainsaw +check-chainsaw: + $(CHAINSAW_TEST) --config testing/chainsaw/e2e/config.yaml --values testing/chainsaw/e2e/values.yaml testing/chainsaw/e2e + # Expects operator to be running # # KUTTL runs with a single kubectl context named "cluster". diff --git a/testing/chainsaw/e2e/config.yaml b/testing/chainsaw/e2e/config.yaml new file mode 100644 index 0000000000..d3ab36599e --- /dev/null +++ b/testing/chainsaw/e2e/config.yaml @@ -0,0 +1,11 @@ +apiVersion: chainsaw.kyverno.io/v1alpha2 +kind: Configuration +metadata: + name: end-to-end +spec: + namespace: + template: + metadata: + labels: { postgres-operator-test: chainsaw } + timeouts: + assert: 3m diff --git a/testing/chainsaw/e2e/pgbackrest-restore/01-create-cluster.yaml b/testing/chainsaw/e2e/pgbackrest-restore/01-create-cluster.yaml new file mode 100644 index 0000000000..5cbd750ce7 --- /dev/null +++ b/testing/chainsaw/e2e/pgbackrest-restore/01-create-cluster.yaml @@ -0,0 +1,55 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: 01-create-cluster +spec: + try: + - + description: > + Create a cluster with a single pgBackRest repository + and some parameters that require attention during PostgreSQL recovery + apply: + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: original + spec: + postgresVersion: ($postgres.version) + config: + parameters: + archive_timeout: 15 + checkpoint_timeout: 30 + max_connections: 200 + instances: + - dataVolumeClaimSpec: ($volume) + replicas: 2 + backups: + pgbackrest: + manual: + repoName: repo1 + repos: + - name: repo1 + volume: + volumeClaimSpec: ($volume) + + - + description: > + Wait for the replica backup to complete + assert: + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: original + status: + pgbackrest: + repos: + - name: repo1 + replicaCreateBackupComplete: true + stanzaCreated: true + + catch: + - podLogs: + selector: postgres-operator.crunchydata.com/cluster in (original) + tail: 50 diff --git a/testing/chainsaw/e2e/pgbackrest-restore/02-create-data.yaml b/testing/chainsaw/e2e/pgbackrest-restore/02-create-data.yaml new file mode 100644 index 0000000000..d5b328fc28 --- /dev/null +++ b/testing/chainsaw/e2e/pgbackrest-restore/02-create-data.yaml @@ -0,0 +1,52 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: 02-create-data +spec: + try: + - + description: > + Create some data that will be restored + apply: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: original-data + spec: + backoffLimit: 3 + template: + spec: + restartPolicy: Never + containers: + - name: psql + image: ($psql.image) + env: + - ($psql.connect) + - name: PGURI + valueFrom: { secretKeyRef: { name: original-pguser-original, key: uri } } + command: + - psql + - $(PGURI) + - --set=ON_ERROR_STOP=1 + - --command + - | + CREATE SCHEMA IF NOT EXISTS "original"; + CREATE TABLE important (data) AS VALUES ('treasure'); + + - assert: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: original-data + status: + succeeded: 1 + + catch: + - + description: > + Read all log lines from the job pods + podLogs: + selector: batch.kubernetes.io/job-name in (original-data) + tail: -1 diff --git a/testing/chainsaw/e2e/pgbackrest-restore/03-create-backup.yaml b/testing/chainsaw/e2e/pgbackrest-restore/03-create-backup.yaml new file mode 100644 index 0000000000..f234c58805 --- /dev/null +++ b/testing/chainsaw/e2e/pgbackrest-restore/03-create-backup.yaml @@ -0,0 +1,33 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: 03-create-backup +spec: + try: + - + description: > + Annotate the cluster to trigger a backup + patch: + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: original + annotations: + postgres-operator.crunchydata.com/pgbackrest-backup: one + + - + description: > + Wait for it to complete + assert: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + annotations: + postgres-operator.crunchydata.com/pgbackrest-backup: one + labels: + postgres-operator.crunchydata.com/cluster: original + postgres-operator.crunchydata.com/pgbackrest-backup: manual + status: + succeeded: 1 diff --git a/testing/chainsaw/e2e/pgbackrest-restore/11-update-cluster.yaml b/testing/chainsaw/e2e/pgbackrest-restore/11-update-cluster.yaml new file mode 100644 index 0000000000..5eeffd7875 --- /dev/null +++ b/testing/chainsaw/e2e/pgbackrest-restore/11-update-cluster.yaml @@ -0,0 +1,83 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: 11-update-cluster +spec: + try: + - + description: > + Find the primary pod + command: + outputs: + - name: primary + value: (trim_space($stdout)) + entrypoint: kubectl + args: + - get + - pod + - --namespace=${NAMESPACE} + - --output=name + - --selector + - >- + postgres-operator.crunchydata.com/cluster=original, + postgres-operator.crunchydata.com/role=master + + - + description: > + Read the timestamp at which PostgreSQL last started + command: + outputs: + - name: start_before + value: (trim_space($stdout)) + env: + - name: PRIMARY + value: ($primary) + entrypoint: kubectl + args: + - exec + - --namespace=${NAMESPACE} + - ${PRIMARY} + - -- + - psql + - -qAt + - --command + - SELECT pg_postmaster_start_time() + + - + description: > + Update the cluster with parameters that require attention during recovery + patch: + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: original + spec: + config: + parameters: + max_connections: 1000 + + - + description: > + Wait for Postgres to restart + script: + skipCommandOutput: true + timeout: 30s + env: + - name: BEFORE + value: ($start_before) + - name: PRIMARY + value: ($primary) + content: | + while true; do + START=$( + kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \ + -- psql -qAt --command 'SELECT pg_postmaster_start_time()' + ) + if [ "${START}" ] && [ "${START}" != "${BEFORE}" ]; then break; else sleep 1; fi + done + echo "${START} != ${BEFORE}" + + # Reset counters in the "pg_stat_archiver" view. + kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \ + -- psql -qb --command "SELECT pg_stat_reset_shared('archiver')" --output /dev/null diff --git a/testing/chainsaw/e2e/pgbackrest-restore/12-update-data.yaml b/testing/chainsaw/e2e/pgbackrest-restore/12-update-data.yaml new file mode 100644 index 0000000000..60bfc5fe1e --- /dev/null +++ b/testing/chainsaw/e2e/pgbackrest-restore/12-update-data.yaml @@ -0,0 +1,74 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: 12-update-data +spec: + try: + - + description: > + Add more data to the WAL archive + apply: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: original-more-data + spec: + backoffLimit: 3 + template: + spec: + restartPolicy: Never + containers: + - name: psql + image: ($psql.image) + env: + - ($psql.connect) + - name: PGURI + valueFrom: { secretKeyRef: { name: original-pguser-original, key: uri } } + + command: + - psql + - $(PGURI) + - --set=ON_ERROR_STOP=1 + - --command + - | + INSERT INTO important (data) VALUES ('water'), ('socks'); + + - assert: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: original-more-data + status: + succeeded: 1 + + - + description: > + Wait for the data to be sent to the WAL archive + script: + skipCommandOutput: true + content: | + PRIMARY=$( + kubectl get pod --namespace "${NAMESPACE}" \ + --output name --selector ' + postgres-operator.crunchydata.com/cluster=original, + postgres-operator.crunchydata.com/role=master' + ) + + kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \ + -- psql --command 'SELECT pg_switch_wal()' --pset footer=off + + # A prior step reset the "pg_stat_archiver" counters, so anything more than zero should suffice. + while [ 0 = "$( + kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \ + -- psql -qAt --command 'SELECT archived_count FROM pg_stat_archiver' + )" ]; do sleep 1; done + + catch: + - + description: > + Read all log lines from the job pods + podLogs: + selector: batch.kubernetes.io/job-name in (original-more-data) + tail: -1 diff --git a/testing/chainsaw/e2e/pgbackrest-restore/21-lose-data.yaml b/testing/chainsaw/e2e/pgbackrest-restore/21-lose-data.yaml new file mode 100644 index 0000000000..90a9cc9e13 --- /dev/null +++ b/testing/chainsaw/e2e/pgbackrest-restore/21-lose-data.yaml @@ -0,0 +1,125 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: 21-lose-data +spec: + try: + - + description: > + Find the primary pod + command: + outputs: + - name: primary + value: (trim_space($stdout)) + entrypoint: kubectl + args: + - get + - pod + - --namespace=${NAMESPACE} + - --output=name + - --selector + - >- + postgres-operator.crunchydata.com/cluster=original, + postgres-operator.crunchydata.com/role=master + + - + description: > + Store the timestamp reported by PostgreSQL + script: + skipCommandOutput: true + env: + - name: PRIMARY + value: ($primary) + content: | + OBJECTIVE=$( + kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \ + -- psql -qAt --command 'SELECT clock_timestamp()' + ) + + # Store the recovery objective for later steps. + kubectl annotate --namespace "${NAMESPACE}" postgrescluster/original \ + "testing/objective=${OBJECTIVE}" + + - + description: > + A reason to restore + command: + skipCommandOutput: true + env: + - name: PRIMARY + value: ($primary) + entrypoint: kubectl + args: + - exec + - --namespace=${NAMESPACE} + - ${PRIMARY} + - -- + - psql + - -qb + - original + - --set=ON_ERROR_STOP=1 + - --pset=footer=off + - --command + - DROP TABLE original.important + - --command + - SELECT pg_stat_reset_shared('archiver') + - --command + - SELECT pg_switch_wal() + + - + description: > + Wait for the change to be sent to the WAL archive + script: + skipCommandOutput: true + env: + - name: PRIMARY + value: ($primary) + content: | + while [ 0 = "$( + kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \ + -- psql -qAt --command 'SELECT archived_count FROM pg_stat_archiver' + )" ]; do sleep 1; done + + - + description: > + Find the replica pod + command: + outputs: + - name: replica + value: (trim_space($stdout)) + entrypoint: kubectl + args: + - get + - pod + - --namespace=${NAMESPACE} + - --output=name + - --selector + - >- + postgres-operator.crunchydata.com/cluster=original, + postgres-operator.crunchydata.com/role=replica + + - + description: > + Confirm that the replica also needs to be restored + command: + skipCommandOutput: true + env: + - name: REPLICA + value: ($replica) + entrypoint: kubectl + args: + - exec + - --namespace=${NAMESPACE} + - ${REPLICA} + - -- + - psql + - -qb + - original + - --set=ON_ERROR_STOP=1 + - --command + - | + DO $$$$ + BEGIN + PERFORM * FROM information_schema.tables WHERE table_name = 'important'; + ASSERT NOT FOUND, 'expected no table'; + END $$$$ diff --git a/testing/chainsaw/e2e/pgbackrest-restore/22-point-in-time-restore.yaml b/testing/chainsaw/e2e/pgbackrest-restore/22-point-in-time-restore.yaml new file mode 100644 index 0000000000..714227ab48 --- /dev/null +++ b/testing/chainsaw/e2e/pgbackrest-restore/22-point-in-time-restore.yaml @@ -0,0 +1,70 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: 22-point-in-time-restore +spec: + try: + - + description: > + Read the recovery objective from a prior step + # https://github.com/kyverno/chainsaw/issues/1034 + command: + outputs: + - name: objective + value: ($stdout) + entrypoint: kubectl + args: + - get + - --namespace=${NAMESPACE} + - postgrescluster/original + - --output + - go-template={{ index .metadata.annotations "testing/objective" }} + + - + description: > + Trigger an in-place point-in-time restore (PITR) + patch: + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: original + annotations: + postgres-operator.crunchydata.com/pgbackrest-restore: one + spec: + backups: + pgbackrest: + restore: + enabled: true + repoName: repo1 + options: + - --type=time + - (join(`"'"`, ['--target=', $objective, ''])) + + - + description: > + Wait for the restore to complete and the cluster to come online + assert: + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: original + status: + instances: + - name: '00' + replicas: 2 + readyReplicas: 2 + updatedReplicas: 2 + pgbackrest: + restore: + id: one + finished: true + + catch: + - + description: > + Read all log lines from the restore job pods + podLogs: + selector: postgres-operator.crunchydata.com/pgbackrest-restore + tail: -1 diff --git a/testing/chainsaw/e2e/pgbackrest-restore/23-verify-primary.yaml b/testing/chainsaw/e2e/pgbackrest-restore/23-verify-primary.yaml new file mode 100644 index 0000000000..7fbf99cf0f --- /dev/null +++ b/testing/chainsaw/e2e/pgbackrest-restore/23-verify-primary.yaml @@ -0,0 +1,62 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: 23-verify-primary +spec: + try: + - + description: > + Confirm the primary is restored + apply: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: original-pitr-primary + spec: + backoffLimit: 3 + template: + spec: + restartPolicy: Never + containers: + - name: psql + image: ($psql.image) + env: + - ($psql.connect) + - name: PGURI + valueFrom: { secretKeyRef: { name: original-pguser-original, key: uri } } + + # NOTE: the `$$$$` is reduced to `$$` by Kubernetes. + command: + - psql + - $(PGURI) + - -qa + - --set=ON_ERROR_STOP=1 + - --command + - | + DO $$$$ + DECLARE + restored jsonb; + BEGIN + SELECT jsonb_agg(important) INTO restored FROM important; + ASSERT restored = '[ + {"data":"treasure"}, {"data":"water"}, {"data":"socks"} + ]', format('got %L', restored); + END $$$$; + + - assert: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: original-pitr-primary + status: + succeeded: 1 + + catch: + - + description: > + Read all log lines from the job pods + podLogs: + selector: batch.kubernetes.io/job-name in (original-pitr-primary) + tail: -1 diff --git a/testing/chainsaw/e2e/pgbackrest-restore/23-verify-replica.yaml b/testing/chainsaw/e2e/pgbackrest-restore/23-verify-replica.yaml new file mode 100644 index 0000000000..33c65cb871 --- /dev/null +++ b/testing/chainsaw/e2e/pgbackrest-restore/23-verify-replica.yaml @@ -0,0 +1,120 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: 23-verify-replica +spec: + try: + - + description: > + Confirm the replica is also restored and streaming from the primary + apply: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: original-pitr-replica + spec: + backoffLimit: 3 + template: + spec: + restartPolicy: Never + containers: + - name: psql + image: ($psql.image) + env: + - ($psql.connect) + - name: PGPORT + valueFrom: { secretKeyRef: { name: original-pguser-original, key: port } } + - name: PGDATABASE + valueFrom: { secretKeyRef: { name: original-pguser-original, key: dbname } } + - name: PGUSER + valueFrom: { secretKeyRef: { name: original-pguser-original, key: user } } + - name: PGPASSWORD + valueFrom: { secretKeyRef: { name: original-pguser-original, key: password } } + + # The user secret does not contain the replica service. + - name: NAMESPACE + valueFrom: { fieldRef: { fieldPath: metadata.namespace } } + - name: PGHOST + value: "original-replicas.$(NAMESPACE).svc" + + # NOTE: the `$$$$` is reduced to `$$` by Kubernetes. + command: + - psql + - -qa + - --set=ON_ERROR_STOP=1 + - --command + - | + DO $$$$ + DECLARE + restored jsonb; + BEGIN + ASSERT pg_is_in_recovery(), 'expected replica'; + + SELECT jsonb_agg(important) INTO restored FROM important; + ASSERT restored = '[ + {"data":"treasure"}, {"data":"water"}, {"data":"socks"} + ]', format('got %L', restored); + END $$$$; + + - assert: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: original-pitr-replica + status: + succeeded: 1 + + - + description: > + Find the replica pod + command: + outputs: + - name: replica + value: (trim_space($stdout)) + entrypoint: kubectl + args: + - get + - pod + - --namespace=${NAMESPACE} + - --output=name + - --selector + - >- + postgres-operator.crunchydata.com/cluster=original, + postgres-operator.crunchydata.com/role=replica + + - + description: > + Confirm that the replica is streaming from the primary + command: + skipCommandOutput: true + env: + - name: REPLICA + value: ($replica) + entrypoint: kubectl + args: + - exec + - --namespace=${NAMESPACE} + - ${REPLICA} + - -- + - psql + - -qb + - original + - --set=ON_ERROR_STOP=1 + - --command + - | + DO $$$$ + BEGIN + -- only users with "pg_read_all_stats" role may examine "pg_stat_wal_receiver" + PERFORM * FROM pg_stat_wal_receiver WHERE status = 'streaming'; + ASSERT FOUND, 'expected streaming replication'; + END $$$$ + + catch: + - + description: > + Read all log lines from the job pods + podLogs: + selector: batch.kubernetes.io/job-name in (original-pitr-replica) + tail: -1 diff --git a/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml b/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml new file mode 100644 index 0000000000..909ca9cf16 --- /dev/null +++ b/testing/chainsaw/e2e/pgbackrest-restore/chainsaw-test.yaml @@ -0,0 +1,105 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: pgbackrest-restore + labels: + pgbackrest: '' +spec: + failFast: true + bindings: + - name: postgres + value: + version: (to_number(as_string($values.versions.postgres))) + + - name: psql + value: + image: ($values.images.psql) + connect: { name: PGCONNECT_TIMEOUT, value: '5' } + + - name: volume + value: { accessModes: [ReadWriteOnce], resources: { requests: { storage: 1Gi } } } + + steps: + - name: 'Create Cluster' + use: { template: '01-create-cluster.yaml' } + + - name: 'Create Data' + use: { template: '02-create-data.yaml' } + + - name: 'Create Backup' + use: { template: '03-create-backup.yaml' } + + - name: 'Clone Cluster #1' + skipDelete: true + use: + template: 'clone-cluster.yaml' + with: + bindings: + - { name: target, value: clone-one } + + - name: 'Verify Data' + use: + template: 'verify-02-data.yaml' + with: + bindings: + - { name: target, value: clone-one } + + - name: 'Delete Cluster #1' + description: > + Delete this clone in the background to free up resources + try: + - delete: + deletionPropagationPolicy: Background + expect: [{ check: { (`true`): true } }] + ref: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + name: clone-one + + - name: 'Update Cluster' + use: { template: '11-update-cluster.yaml' } + + - name: 'Update Data' + use: { template: '12-update-data.yaml' } + + - name: 'Clone Cluster #2' + skipDelete: true + use: + template: 'clone-cluster.yaml' + with: + bindings: + - { name: source, value: original } + - { name: target, value: clone-two } + + - name: 'Verify Data' + use: + template: 'verify-12-data.yaml' + with: + bindings: + - { name: target, value: clone-two } + + - name: 'Delete Cluster #2' + description: > + Delete this clone in the background to free up resources + try: + - delete: + deletionPropagationPolicy: Background + expect: [{ check: { (`true`): true } }] + ref: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + name: clone-two + + - name: 'Lose Data' + use: { template: '21-lose-data.yaml' } + + - name: 'Point-In-Time Restore' + use: { template: '22-point-in-time-restore.yaml' } + + - name: 'Verify Primary' + description: > + Confirm that data was restored to the point-in-time and the cluster is healthy + use: { template: '23-verify-primary.yaml' } + + - name: 'Confirm Replica' + use: { template: '23-verify-replica.yaml' } diff --git a/testing/chainsaw/e2e/pgbackrest-restore/clone-cluster.yaml b/testing/chainsaw/e2e/pgbackrest-restore/clone-cluster.yaml new file mode 100644 index 0000000000..3a5fe0af74 --- /dev/null +++ b/testing/chainsaw/e2e/pgbackrest-restore/clone-cluster.yaml @@ -0,0 +1,50 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: clone-cluster +spec: + bindings: + - name: target + value: 'The name of the new PostgresCluster' + + try: + - + description: > + Clone the cluster using a pgBackRest restore + apply: + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: ($target) + spec: + dataSource: + postgresCluster: + clusterName: original + repoName: repo1 + + postgresVersion: ($postgres.version) + instances: + - dataVolumeClaimSpec: ($volume) + backups: + pgbackrest: + repos: + - name: repo1 + volume: + volumeClaimSpec: ($volume) + + - + description: > + Wait for the cluster to come online + assert: + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: ($target) + status: + instances: + - name: '00' + replicas: 1 + readyReplicas: 1 + updatedReplicas: 1 diff --git a/testing/chainsaw/e2e/pgbackrest-restore/verify-02-data.yaml b/testing/chainsaw/e2e/pgbackrest-restore/verify-02-data.yaml new file mode 100644 index 0000000000..2b874de7bd --- /dev/null +++ b/testing/chainsaw/e2e/pgbackrest-restore/verify-02-data.yaml @@ -0,0 +1,83 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: verify-02-data +spec: + bindings: + - name: target + value: 'The name of the PostgresCluster' + + try: + # Create bindings derived from the template input. + - command: + entrypoint: 'true' + skipCommandOutput: true + outputs: + - name: job + value: (join('-', [$target, 'data'])) + - name: secret + value: (join('-', [$target, 'pguser', $target])) + + - + description: > + Confirm that all the data was restored + apply: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: ($job) + spec: + backoffLimit: 3 + template: + spec: + restartPolicy: Never + containers: + - name: psql + image: ($psql.image) + env: + - ($psql.connect) + # Connect to the cluster using the restored database and original credentials. + - name: PGHOST + valueFrom: { secretKeyRef: { name: ($secret), key: host } } + - name: PGPORT + valueFrom: { secretKeyRef: { name: ($secret), key: port } } + - name: PGDATABASE + valueFrom: { secretKeyRef: { name: original-pguser-original, key: dbname } } + - name: PGUSER + valueFrom: { secretKeyRef: { name: original-pguser-original, key: user } } + - name: PGPASSWORD + valueFrom: { secretKeyRef: { name: original-pguser-original, key: password } } + + # Confirm that all the data was restored. + # NOTE: the `$$$$` is reduced to `$$` by Kubernetes. + command: + - psql + - -qa + - --set=ON_ERROR_STOP=1 + - --command + - | + DO $$$$ + DECLARE + restored jsonb; + BEGIN + SELECT jsonb_agg(important) INTO restored FROM important; + ASSERT restored = '[{"data":"treasure"}]', format('got %L', restored); + END $$$$; + + - assert: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: ($job) + status: + succeeded: 1 + + catch: + - + description: > + Read all log lines from the job pods + podLogs: + selector: (join('', ['batch.kubernetes.io/job-name in (', $job, ')'])) + tail: -1 diff --git a/testing/chainsaw/e2e/pgbackrest-restore/verify-12-data.yaml b/testing/chainsaw/e2e/pgbackrest-restore/verify-12-data.yaml new file mode 100644 index 0000000000..9a625bd462 --- /dev/null +++ b/testing/chainsaw/e2e/pgbackrest-restore/verify-12-data.yaml @@ -0,0 +1,85 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: verify-12-data +spec: + bindings: + - name: target + value: 'The name of the PostgresCluster' + + try: + # Create bindings derived from the template input. + - command: + entrypoint: 'true' + skipCommandOutput: true + outputs: + - name: job + value: (join('-', [$target, 'data'])) + - name: secret + value: (join('-', [$target, 'pguser', $target])) + + - + description: > + Confirm that all the data was restored + apply: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: ($job) + spec: + backoffLimit: 3 + template: + spec: + restartPolicy: Never + containers: + - name: psql + image: ($psql.image) + env: + - ($psql.connect) + # Connect to the cluster using the restored database and original credentials. + - name: PGHOST + valueFrom: { secretKeyRef: { name: ($secret), key: host } } + - name: PGPORT + valueFrom: { secretKeyRef: { name: ($secret), key: port } } + - name: PGDATABASE + valueFrom: { secretKeyRef: { name: original-pguser-original, key: dbname } } + - name: PGUSER + valueFrom: { secretKeyRef: { name: original-pguser-original, key: user } } + - name: PGPASSWORD + valueFrom: { secretKeyRef: { name: original-pguser-original, key: password } } + + # Confirm that all the data was restored. + # NOTE: the `$$$$` is reduced to `$$` by Kubernetes. + command: + - psql + - -qa + - --set=ON_ERROR_STOP=1 + - --command + - | + DO $$$$ + DECLARE + restored jsonb; + BEGIN + SELECT jsonb_agg(important) INTO restored FROM important; + ASSERT restored = '[ + {"data":"treasure"}, {"data":"water"}, {"data":"socks"} + ]', format('got %L', restored); + END $$$$; + + - assert: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: ($job) + status: + succeeded: 1 + + catch: + - + description: > + Read all log lines from the job pods + podLogs: + selector: (join('', ['batch.kubernetes.io/job-name in (', $job, ')'])) + tail: -1 diff --git a/testing/chainsaw/e2e/values.yaml b/testing/chainsaw/e2e/values.yaml new file mode 100644 index 0000000000..0c8a3ce580 --- /dev/null +++ b/testing/chainsaw/e2e/values.yaml @@ -0,0 +1,5 @@ +versions: + postgres: '17' + +images: + psql: 'registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-17.5-2520' diff --git a/testing/kuttl/e2e/pgbackrest-restore/01--create-cluster.yaml b/testing/kuttl/e2e/pgbackrest-restore/01--create-cluster.yaml deleted file mode 100644 index 5c562189f4..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/01--create-cluster.yaml +++ /dev/null @@ -1,24 +0,0 @@ ---- -# Create a cluster with a single pgBackRest repository and some parameters that -# require attention during PostgreSQL recovery. -apiVersion: postgres-operator.crunchydata.com/v1beta1 -kind: PostgresCluster -metadata: - name: original - labels: { postgres-operator-test: kuttl } -spec: - postgresVersion: ${KUTTL_PG_VERSION} - config: - parameters: - max_connections: 200 - instances: - - dataVolumeClaimSpec: { accessModes: [ReadWriteOnce], resources: { requests: { storage: 1Gi } } } - replicas: 2 - backups: - pgbackrest: - manual: - repoName: repo1 - repos: - - name: repo1 - volume: - volumeClaimSpec: { accessModes: [ReadWriteOnce], resources: { requests: { storage: 1Gi } } } diff --git a/testing/kuttl/e2e/pgbackrest-restore/01-assert.yaml b/testing/kuttl/e2e/pgbackrest-restore/01-assert.yaml deleted file mode 100644 index 25b5bbee76..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/01-assert.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -# Wait for the replica backup to complete. -apiVersion: postgres-operator.crunchydata.com/v1beta1 -kind: PostgresCluster -metadata: - name: original -status: - pgbackrest: - repos: - - name: repo1 - replicaCreateBackupComplete: true - stanzaCreated: true diff --git a/testing/kuttl/e2e/pgbackrest-restore/02--create-data.yaml b/testing/kuttl/e2e/pgbackrest-restore/02--create-data.yaml deleted file mode 100644 index 6801edbf61..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/02--create-data.yaml +++ /dev/null @@ -1,32 +0,0 @@ ---- -# Create some data that will be restored. -apiVersion: batch/v1 -kind: Job -metadata: - name: original-data - labels: { postgres-operator-test: kuttl } -spec: - backoffLimit: 3 - template: - metadata: - labels: { postgres-operator-test: kuttl } - spec: - restartPolicy: Never - containers: - - name: psql - image: ${KUTTL_PSQL_IMAGE} - env: - - name: PGURI - valueFrom: { secretKeyRef: { name: original-pguser-original, key: uri } } - - # Do not wait indefinitely. - - { name: PGCONNECT_TIMEOUT, value: '5' } - - command: - - psql - - $(PGURI) - - --set=ON_ERROR_STOP=1 - - --command - - | - CREATE SCHEMA "original"; - CREATE TABLE important (data) AS VALUES ('treasure'); diff --git a/testing/kuttl/e2e/pgbackrest-restore/02-assert.yaml b/testing/kuttl/e2e/pgbackrest-restore/02-assert.yaml deleted file mode 100644 index 5115ba97c9..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/02-assert.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: original-data -status: - succeeded: 1 diff --git a/testing/kuttl/e2e/pgbackrest-restore/03--backup.yaml b/testing/kuttl/e2e/pgbackrest-restore/03--backup.yaml deleted file mode 100644 index b759dd0fc4..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/03--backup.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - # Annotate the cluster to trigger a backup. - - script: | - kubectl annotate --namespace="${NAMESPACE}" postgrescluster/original \ - 'postgres-operator.crunchydata.com/pgbackrest-backup=one' diff --git a/testing/kuttl/e2e/pgbackrest-restore/03-assert.yaml b/testing/kuttl/e2e/pgbackrest-restore/03-assert.yaml deleted file mode 100644 index a2c5b3bb22..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/03-assert.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -# Wait for the backup job to complete. -apiVersion: batch/v1 -kind: Job -metadata: - annotations: - postgres-operator.crunchydata.com/pgbackrest-backup: one - labels: - postgres-operator.crunchydata.com/cluster: original - postgres-operator.crunchydata.com/pgbackrest-backup: manual - postgres-operator.crunchydata.com/pgbackrest-repo: repo1 -status: - succeeded: 1 diff --git a/testing/kuttl/e2e/pgbackrest-restore/04--clone-cluster.yaml b/testing/kuttl/e2e/pgbackrest-restore/04--clone-cluster.yaml deleted file mode 100644 index 4bc1ce56a9..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/04--clone-cluster.yaml +++ /dev/null @@ -1,22 +0,0 @@ ---- -# Clone the cluster using a pgBackRest restore. -apiVersion: postgres-operator.crunchydata.com/v1beta1 -kind: PostgresCluster -metadata: - name: clone-one - labels: { postgres-operator-test: kuttl } -spec: - dataSource: - postgresCluster: - clusterName: original - repoName: repo1 - - postgresVersion: ${KUTTL_PG_VERSION} - instances: - - dataVolumeClaimSpec: { accessModes: [ReadWriteOnce], resources: { requests: { storage: 1Gi } } } - backups: - pgbackrest: - repos: - - name: repo1 - volume: - volumeClaimSpec: { accessModes: [ReadWriteOnce], resources: { requests: { storage: 1Gi } } } diff --git a/testing/kuttl/e2e/pgbackrest-restore/04-assert.yaml b/testing/kuttl/e2e/pgbackrest-restore/04-assert.yaml deleted file mode 100644 index 8aa51fc440..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/04-assert.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -# Wait for the clone cluster to come online. -apiVersion: postgres-operator.crunchydata.com/v1beta1 -kind: PostgresCluster -metadata: - name: clone-one -status: - instances: - - name: '00' - replicas: 1 - readyReplicas: 1 - updatedReplicas: 1 diff --git a/testing/kuttl/e2e/pgbackrest-restore/05--check-data.yaml b/testing/kuttl/e2e/pgbackrest-restore/05--check-data.yaml deleted file mode 100644 index 1ee6fe9c32..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/05--check-data.yaml +++ /dev/null @@ -1,49 +0,0 @@ ---- -# Confirm that all the data was restored. -apiVersion: batch/v1 -kind: Job -metadata: - name: clone-one-data - labels: { postgres-operator-test: kuttl } -spec: - backoffLimit: 3 - template: - metadata: - labels: { postgres-operator-test: kuttl } - spec: - restartPolicy: Never - containers: - - name: psql - image: ${KUTTL_PSQL_IMAGE} - env: - # Connect to the cluster using the restored database and original credentials. - - name: PGHOST - valueFrom: { secretKeyRef: { name: clone-one-pguser-clone-one, key: host } } - - name: PGPORT - valueFrom: { secretKeyRef: { name: clone-one-pguser-clone-one, key: port } } - - name: PGDATABASE - valueFrom: { secretKeyRef: { name: original-pguser-original, key: dbname } } - - name: PGUSER - valueFrom: { secretKeyRef: { name: original-pguser-original, key: user } } - - name: PGPASSWORD - valueFrom: { secretKeyRef: { name: original-pguser-original, key: password } } - - # Do not wait indefinitely. - - { name: PGCONNECT_TIMEOUT, value: '5' } - - # Confirm that all the data was restored. - # Note: the `$$$$` is reduced to `$$` by Kubernetes. - # - https://kubernetes.io/docs/tasks/inject-data-application/ - command: - - psql - - -qa - - --set=ON_ERROR_STOP=1 - - --command - - | - DO $$$$ - DECLARE - restored jsonb; - BEGIN - SELECT jsonb_agg(important) INTO restored FROM important; - ASSERT restored = '[{"data":"treasure"}]', format('got %L', restored); - END $$$$; diff --git a/testing/kuttl/e2e/pgbackrest-restore/05-assert.yaml b/testing/kuttl/e2e/pgbackrest-restore/05-assert.yaml deleted file mode 100644 index 1b6fad318b..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/05-assert.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: clone-one-data -status: - succeeded: 1 diff --git a/testing/kuttl/e2e/pgbackrest-restore/06--delete-clone.yaml b/testing/kuttl/e2e/pgbackrest-restore/06--delete-clone.yaml deleted file mode 100644 index 69ebc06c9d..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/06--delete-clone.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -# Remove the cloned cluster. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -delete: - - apiVersion: postgres-operator.crunchydata.com/v1beta1 - kind: PostgresCluster - name: clone-one diff --git a/testing/kuttl/e2e/pgbackrest-restore/07--annotate.yaml b/testing/kuttl/e2e/pgbackrest-restore/07--annotate.yaml deleted file mode 100644 index 279c216ed0..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/07--annotate.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -# Annotate the cluster with the timestamp at which PostgreSQL last started. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: | - PRIMARY=$( - kubectl get pod --namespace "${NAMESPACE}" \ - --output name --selector ' - postgres-operator.crunchydata.com/cluster=original, - postgres-operator.crunchydata.com/role=master' - ) - START=$( - kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \ - -- psql -qAt --command 'SELECT pg_postmaster_start_time()' - ) - kubectl annotate --namespace "${NAMESPACE}" postgrescluster/original \ - "testing/start-before=${START}" diff --git a/testing/kuttl/e2e/pgbackrest-restore/07--update-cluster.yaml b/testing/kuttl/e2e/pgbackrest-restore/07--update-cluster.yaml deleted file mode 100644 index 0c8cb99b98..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/07--update-cluster.yaml +++ /dev/null @@ -1,23 +0,0 @@ ---- -# Update the cluster with PostgreSQL parameters that require attention during recovery. -apiVersion: postgres-operator.crunchydata.com/v1beta1 -kind: PostgresCluster -metadata: - name: original - labels: { postgres-operator-test: kuttl } -spec: - postgresVersion: ${KUTTL_PG_VERSION} - config: - parameters: - max_connections: 1000 - instances: - - dataVolumeClaimSpec: { accessModes: [ReadWriteOnce], resources: { requests: { storage: 1Gi } } } - replicas: 2 - backups: - pgbackrest: - manual: - repoName: repo1 - repos: - - name: repo1 - volume: - volumeClaimSpec: { accessModes: [ReadWriteOnce], resources: { requests: { storage: 1Gi } } } diff --git a/testing/kuttl/e2e/pgbackrest-restore/08--wait-restart.yaml b/testing/kuttl/e2e/pgbackrest-restore/08--wait-restart.yaml deleted file mode 100644 index 305d757386..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/08--wait-restart.yaml +++ /dev/null @@ -1,29 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: | - BEFORE=$( - kubectl get --namespace "${NAMESPACE}" postgrescluster/original \ - --output 'go-template={{ index .metadata.annotations "testing/start-before" }}' - ) - PRIMARY=$( - kubectl get pod --namespace "${NAMESPACE}" \ - --output name --selector ' - postgres-operator.crunchydata.com/cluster=original, - postgres-operator.crunchydata.com/role=master' - ) - - # Wait for PostgreSQL to restart. - while true; do - START=$( - kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \ - -- psql -qAt --command 'SELECT pg_postmaster_start_time()' - ) - if [ "${START}" ] && [ "${START}" != "${BEFORE}" ]; then break; else sleep 1; fi - done - echo "${START} != ${BEFORE}" - - # Reset counters in the "pg_stat_archiver" view. - kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \ - -- psql -qb --command "SELECT pg_stat_reset_shared('archiver')" diff --git a/testing/kuttl/e2e/pgbackrest-restore/09--add-data.yaml b/testing/kuttl/e2e/pgbackrest-restore/09--add-data.yaml deleted file mode 100644 index 41c2255239..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/09--add-data.yaml +++ /dev/null @@ -1,31 +0,0 @@ ---- -# Add more data to the WAL archive. -apiVersion: batch/v1 -kind: Job -metadata: - name: original-more-data - labels: { postgres-operator-test: kuttl } -spec: - backoffLimit: 3 - template: - metadata: - labels: { postgres-operator-test: kuttl } - spec: - restartPolicy: Never - containers: - - name: psql - image: ${KUTTL_PSQL_IMAGE} - env: - - name: PGURI - valueFrom: { secretKeyRef: { name: original-pguser-original, key: uri } } - - # Do not wait indefinitely. - - { name: PGCONNECT_TIMEOUT, value: '5' } - - command: - - psql - - $(PGURI) - - --set=ON_ERROR_STOP=1 - - --command - - | - INSERT INTO important (data) VALUES ('water'), ('socks'); diff --git a/testing/kuttl/e2e/pgbackrest-restore/09-assert.yaml b/testing/kuttl/e2e/pgbackrest-restore/09-assert.yaml deleted file mode 100644 index a60cd9ab8f..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/09-assert.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: original-more-data -status: - succeeded: 1 diff --git a/testing/kuttl/e2e/pgbackrest-restore/10--wait-archived.yaml b/testing/kuttl/e2e/pgbackrest-restore/10--wait-archived.yaml deleted file mode 100644 index 446886ead3..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/10--wait-archived.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: | - PRIMARY=$( - kubectl get pod --namespace "${NAMESPACE}" \ - --output name --selector ' - postgres-operator.crunchydata.com/cluster=original, - postgres-operator.crunchydata.com/role=master' - ) - - # Wait for the data to be sent to the WAL archive. A prior step reset the - # "pg_stat_archiver" counters, so anything more than zero should suffice. - kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" -- psql -c 'SELECT pg_switch_wal()' - while [ 0 = "$( - kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" -- psql -qAt -c 'SELECT archived_count FROM pg_stat_archiver' - )" ]; do sleep 1; done diff --git a/testing/kuttl/e2e/pgbackrest-restore/11--clone-cluster.yaml b/testing/kuttl/e2e/pgbackrest-restore/11--clone-cluster.yaml deleted file mode 100644 index fcbdde4ea7..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/11--clone-cluster.yaml +++ /dev/null @@ -1,22 +0,0 @@ ---- -# Clone the cluster using a pgBackRest restore. -apiVersion: postgres-operator.crunchydata.com/v1beta1 -kind: PostgresCluster -metadata: - name: clone-two - labels: { postgres-operator-test: kuttl } -spec: - dataSource: - postgresCluster: - clusterName: original - repoName: repo1 - - postgresVersion: ${KUTTL_PG_VERSION} - instances: - - dataVolumeClaimSpec: { accessModes: [ReadWriteOnce], resources: { requests: { storage: 1Gi } } } - backups: - pgbackrest: - repos: - - name: repo1 - volume: - volumeClaimSpec: { accessModes: [ReadWriteOnce], resources: { requests: { storage: 1Gi } } } diff --git a/testing/kuttl/e2e/pgbackrest-restore/11-assert.yaml b/testing/kuttl/e2e/pgbackrest-restore/11-assert.yaml deleted file mode 100644 index 0ad9669a62..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/11-assert.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -# Wait for the clone cluster to come online. -apiVersion: postgres-operator.crunchydata.com/v1beta1 -kind: PostgresCluster -metadata: - name: clone-two -status: - instances: - - name: '00' - replicas: 1 - readyReplicas: 1 - updatedReplicas: 1 diff --git a/testing/kuttl/e2e/pgbackrest-restore/12--check-data.yaml b/testing/kuttl/e2e/pgbackrest-restore/12--check-data.yaml deleted file mode 100644 index 2cd2e4932b..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/12--check-data.yaml +++ /dev/null @@ -1,51 +0,0 @@ ---- -# Confirm that all the data was restored. -apiVersion: batch/v1 -kind: Job -metadata: - name: clone-two-data - labels: { postgres-operator-test: kuttl } -spec: - backoffLimit: 3 - template: - metadata: - labels: { postgres-operator-test: kuttl } - spec: - restartPolicy: Never - containers: - - name: psql - image: ${KUTTL_PSQL_IMAGE} - env: - # Connect to the cluster using the restored database and original credentials. - - name: PGHOST - valueFrom: { secretKeyRef: { name: clone-two-pguser-clone-two, key: host } } - - name: PGPORT - valueFrom: { secretKeyRef: { name: clone-two-pguser-clone-two, key: port } } - - name: PGDATABASE - valueFrom: { secretKeyRef: { name: original-pguser-original, key: dbname } } - - name: PGUSER - valueFrom: { secretKeyRef: { name: original-pguser-original, key: user } } - - name: PGPASSWORD - valueFrom: { secretKeyRef: { name: original-pguser-original, key: password } } - - # Do not wait indefinitely. - - { name: PGCONNECT_TIMEOUT, value: '5' } - - # Confirm that all the data was restored. - # Note: the `$$$$` is reduced to `$$` by Kubernetes. - # - https://kubernetes.io/docs/tasks/inject-data-application/ - command: - - psql - - -qa - - --set=ON_ERROR_STOP=1 - - --command - - | - DO $$$$ - DECLARE - restored jsonb; - BEGIN - SELECT jsonb_agg(important) INTO restored FROM important; - ASSERT restored = '[ - {"data":"treasure"}, {"data":"water"}, {"data":"socks"} - ]', format('got %L', restored); - END $$$$; diff --git a/testing/kuttl/e2e/pgbackrest-restore/12-assert.yaml b/testing/kuttl/e2e/pgbackrest-restore/12-assert.yaml deleted file mode 100644 index 198d196836..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/12-assert.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: clone-two-data -status: - succeeded: 1 diff --git a/testing/kuttl/e2e/pgbackrest-restore/13--delete-clone.yaml b/testing/kuttl/e2e/pgbackrest-restore/13--delete-clone.yaml deleted file mode 100644 index 9646f66f35..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/13--delete-clone.yaml +++ /dev/null @@ -1,8 +0,0 @@ ---- -# Remove the cloned cluster. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -delete: - - apiVersion: postgres-operator.crunchydata.com/v1beta1 - kind: PostgresCluster - name: clone-two diff --git a/testing/kuttl/e2e/pgbackrest-restore/14--lose-data.yaml b/testing/kuttl/e2e/pgbackrest-restore/14--lose-data.yaml deleted file mode 100644 index 4f1eaeaa53..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/14--lose-data.yaml +++ /dev/null @@ -1,50 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: | - PRIMARY=$( - kubectl get pod --namespace "${NAMESPACE}" \ - --output name --selector ' - postgres-operator.crunchydata.com/cluster=original, - postgres-operator.crunchydata.com/role=master' - ) - OBJECTIVE=$( - kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \ - -- psql -qAt --command 'SELECT clock_timestamp()' - ) - - # Store the recovery objective for later steps. - kubectl annotate --namespace "${NAMESPACE}" postgrescluster/original \ - "testing/objective=${OBJECTIVE}" - - # A reason to restore. Wait for the change to be sent to the WAL archive. - kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \ - -- psql -qb original --set ON_ERROR_STOP=1 \ - --command 'DROP TABLE original.important' \ - --command "SELECT pg_stat_reset_shared('archiver')" \ - --command 'SELECT pg_switch_wal()' - - while [ 0 = "$( - kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" -- psql -qAt -c 'SELECT archived_count FROM pg_stat_archiver' - )" ]; do sleep 1; done - - # The replica should also need to be restored. - - script: | - REPLICA=$( - kubectl get pod --namespace "${NAMESPACE}" \ - --output name --selector ' - postgres-operator.crunchydata.com/cluster=original, - postgres-operator.crunchydata.com/role=replica' - ) - - kubectl exec --stdin --namespace "${NAMESPACE}" "${REPLICA}" \ - -- psql -qb original --set ON_ERROR_STOP=1 \ - --file=- <<'SQL' - DO $$ - BEGIN - ASSERT to_regclass('important') IS NULL, 'expected no table'; - PERFORM * FROM information_schema.tables WHERE table_name = 'important'; - ASSERT NOT FOUND, 'expected no table'; - END $$ - SQL diff --git a/testing/kuttl/e2e/pgbackrest-restore/15--in-place-pitr.yaml b/testing/kuttl/e2e/pgbackrest-restore/15--in-place-pitr.yaml deleted file mode 100644 index 3e647946db..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/15--in-place-pitr.yaml +++ /dev/null @@ -1,50 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: | - TARGET_JSON=$( - kubectl get --namespace "${NAMESPACE}" postgrescluster/original \ - --output 'go-template={{ index .metadata.annotations "testing/objective" | printf "--target=%q" | printf "%q" }}' - ) - - # Configure the cluster for an in-place point-in-time restore (PITR). - kubectl patch --namespace "${NAMESPACE}" postgrescluster/original \ - --type 'merge' --patch ' - {"spec":{"backups":{"pgbackrest":{"restore":{ - "enabled": true, - "repoName": "repo1", - "options": ["--type=time", '"${TARGET_JSON}"'] - }}}}}' - - # Annotate the cluster to trigger the restore. - kubectl annotate --namespace="${NAMESPACE}" postgrescluster/original \ - 'postgres-operator.crunchydata.com/pgbackrest-restore=one' - - # TODO(benjaminjb): remove this when PG10 is no longer being supported - # For PG10, we need to run a patronictl reinit for the replica when that is running - # Get the replica name--the replica will exist during the PITR process so we don't need to wait - if [[ ${KUTTL_PG_VERSION} == 10 ]]; then - # Find replica - REPLICA=$(kubectl get pods --namespace "${NAMESPACE}" \ - --selector=' - postgres-operator.crunchydata.com/cluster=original, - postgres-operator.crunchydata.com/data=postgres, - postgres-operator.crunchydata.com/role!=master' \ - --output=jsonpath={.items..metadata.name}) - - # Wait for replica to be deleted - kubectl wait pod/"${REPLICA}" --namespace "${NAMESPACE}" --for=delete --timeout=-1s - - # Wait for the restarted replica to be started - NOT_RUNNING="" - while [[ "${NOT_RUNNING}" == "" ]]; do - kubectl get pods --namespace "${NAMESPACE}" "${REPLICA}" || (sleep 1 && continue) - - NOT_RUNNING=$(kubectl get pods --namespace "${NAMESPACE}" "${REPLICA}" \ - --output jsonpath="{.status.containerStatuses[?(@.name=='database')].state.running.startedAt}") - sleep 1 - done - - kubectl exec --namespace "${NAMESPACE}" "${REPLICA}" -- patronictl reinit original-ha "${REPLICA}" --force - fi diff --git a/testing/kuttl/e2e/pgbackrest-restore/15-assert.yaml b/testing/kuttl/e2e/pgbackrest-restore/15-assert.yaml deleted file mode 100644 index c408b75a60..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/15-assert.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -# Wait for the restore to complete and the cluster to come online. -apiVersion: postgres-operator.crunchydata.com/v1beta1 -kind: PostgresCluster -metadata: - name: original -status: - instances: - - name: '00' - replicas: 2 - readyReplicas: 2 - updatedReplicas: 2 - pgbackrest: - restore: - id: one - finished: true diff --git a/testing/kuttl/e2e/pgbackrest-restore/16--check-data.yaml b/testing/kuttl/e2e/pgbackrest-restore/16--check-data.yaml deleted file mode 100644 index b0ae252831..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/16--check-data.yaml +++ /dev/null @@ -1,100 +0,0 @@ ---- -# Confirm that data was restored to the point-in-time. -apiVersion: batch/v1 -kind: Job -metadata: - name: original-pitr-primary-data - labels: { postgres-operator-test: kuttl } -spec: - backoffLimit: 3 - template: - metadata: - labels: { postgres-operator-test: kuttl } - spec: - restartPolicy: Never - containers: - - name: psql - image: ${KUTTL_PSQL_IMAGE} - env: - - name: PGURI - valueFrom: { secretKeyRef: { name: original-pguser-original, key: uri } } - - # Do not wait indefinitely. - - { name: PGCONNECT_TIMEOUT, value: '5' } - - # Note: the `$$$$` is reduced to `$$` by Kubernetes. - # - https://kubernetes.io/docs/tasks/inject-data-application/ - command: - - psql - - $(PGURI) - - -qa - - --set=ON_ERROR_STOP=1 - - --command - - | - DO $$$$ - DECLARE - restored jsonb; - BEGIN - SELECT jsonb_agg(important) INTO restored FROM important; - ASSERT restored = '[ - {"data":"treasure"}, {"data":"water"}, {"data":"socks"} - ]', format('got %L', restored); - END $$$$; - ---- -# Confirm that replicas are also restored and streaming from the primary. -apiVersion: batch/v1 -kind: Job -metadata: - name: original-pitr-replica-data - labels: { postgres-operator-test: kuttl } -spec: - backoffLimit: 3 - template: - metadata: - labels: { postgres-operator-test: kuttl } - spec: - restartPolicy: Never - containers: - - name: psql - image: ${KUTTL_PSQL_IMAGE} - env: - - name: PGPORT - valueFrom: { secretKeyRef: { name: original-pguser-original, key: port } } - - name: PGDATABASE - valueFrom: { secretKeyRef: { name: original-pguser-original, key: dbname } } - - name: PGUSER - valueFrom: { secretKeyRef: { name: original-pguser-original, key: user } } - - name: PGPASSWORD - valueFrom: { secretKeyRef: { name: original-pguser-original, key: password } } - - # The user secret does not contain the replica service. - - name: NAMESPACE - valueFrom: { fieldRef: { fieldPath: metadata.namespace } } - - name: PGHOST - value: "original-replicas.$(NAMESPACE).svc" - - # Do not wait indefinitely. - - { name: PGCONNECT_TIMEOUT, value: '5' } - - # Note: the `$$$$` is reduced to `$$` by Kubernetes. - # - https://kubernetes.io/docs/tasks/inject-data-application/ - command: - - psql - - -qa - - --set=ON_ERROR_STOP=1 - - --command - - | - DO $$$$ - DECLARE - restored jsonb; - BEGIN - ASSERT pg_is_in_recovery(), 'expected replica'; - -- only users with "pg_read_all_settings" role may examine "primary_conninfo" - -- ASSERT current_setting('primary_conninfo') <> '', 'expected streaming'; - - SELECT jsonb_agg(important) INTO restored FROM important; - ASSERT restored = '[ - {"data":"treasure"}, {"data":"water"}, {"data":"socks"} - ]', format('got %L', restored); - END $$$$; diff --git a/testing/kuttl/e2e/pgbackrest-restore/16-assert.yaml b/testing/kuttl/e2e/pgbackrest-restore/16-assert.yaml deleted file mode 100644 index 0baadef25b..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/16-assert.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: original-pitr-primary-data -status: - succeeded: 1 - ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: original-pitr-replica-data -status: - succeeded: 1 diff --git a/testing/kuttl/e2e/pgbackrest-restore/17--check-replication.yaml b/testing/kuttl/e2e/pgbackrest-restore/17--check-replication.yaml deleted file mode 100644 index f6c813c8b1..0000000000 --- a/testing/kuttl/e2e/pgbackrest-restore/17--check-replication.yaml +++ /dev/null @@ -1,22 +0,0 @@ ---- -# Confirm that the replica is streaming from the primary. -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: | - REPLICA=$( - kubectl get pod --namespace "${NAMESPACE}" \ - --output name --selector ' - postgres-operator.crunchydata.com/cluster=original, - postgres-operator.crunchydata.com/role=replica' - ) - - kubectl exec --stdin --namespace "${NAMESPACE}" "${REPLICA}" \ - -- psql -qb original --set ON_ERROR_STOP=1 \ - --file=- <<'SQL' - DO $$ - BEGIN - PERFORM * FROM pg_stat_wal_receiver WHERE status = 'streaming'; - ASSERT FOUND, 'expected streaming replication'; - END $$ - SQL From ca9a5b6b766f1cd6438d7041dd3681e2e8876684 Mon Sep 17 00:00:00 2001 From: Chris Bandy Date: Thu, 7 Nov 2024 10:17:27 -0600 Subject: [PATCH 2/3] Add a tablespace volume to the restore E2E test --- .../pgbackrest-restore/01-create-cluster.yaml | 2 ++ .../pgbackrest-restore/02-create-data.yaml | 19 +++++++++++++++++++ .../e2e/pgbackrest-restore/clone-cluster.yaml | 2 ++ .../pgbackrest-restore/verify-02-data.yaml | 3 +++ 4 files changed, 26 insertions(+) diff --git a/testing/chainsaw/e2e/pgbackrest-restore/01-create-cluster.yaml b/testing/chainsaw/e2e/pgbackrest-restore/01-create-cluster.yaml index 5cbd750ce7..f0a42b4d0f 100644 --- a/testing/chainsaw/e2e/pgbackrest-restore/01-create-cluster.yaml +++ b/testing/chainsaw/e2e/pgbackrest-restore/01-create-cluster.yaml @@ -23,6 +23,8 @@ spec: max_connections: 200 instances: - dataVolumeClaimSpec: ($volume) + tablespaceVolumes: + - { name: barn, dataVolumeClaimSpec: ($volume) } replicas: 2 backups: pgbackrest: diff --git a/testing/chainsaw/e2e/pgbackrest-restore/02-create-data.yaml b/testing/chainsaw/e2e/pgbackrest-restore/02-create-data.yaml index d5b328fc28..97bd802807 100644 --- a/testing/chainsaw/e2e/pgbackrest-restore/02-create-data.yaml +++ b/testing/chainsaw/e2e/pgbackrest-restore/02-create-data.yaml @@ -4,6 +4,24 @@ metadata: name: 02-create-data spec: try: + - + description: > + Create a tablespace and grant access to all Postgres users + script: + skipCommandOutput: true + content: | + PRIMARY=$( + kubectl get pod --namespace "${NAMESPACE}" \ + --output name --selector ' + postgres-operator.crunchydata.com/cluster=original, + postgres-operator.crunchydata.com/role=master' + ) + + kubectl exec --stdin --namespace "${NAMESPACE}" "${PRIMARY}" -- psql -q --file=- <<'SQL' + CREATE TABLESPACE barn LOCATION '/tablespaces/barn/data'; + GRANT ALL ON TABLESPACE barn TO public; + SQL + - description: > Create some data that will be restored @@ -33,6 +51,7 @@ spec: - | CREATE SCHEMA IF NOT EXISTS "original"; CREATE TABLE important (data) AS VALUES ('treasure'); + CREATE TABLE cows (name) TABLESPACE barn AS VALUES ('nellie'); - assert: resource: diff --git a/testing/chainsaw/e2e/pgbackrest-restore/clone-cluster.yaml b/testing/chainsaw/e2e/pgbackrest-restore/clone-cluster.yaml index 3a5fe0af74..756a785e80 100644 --- a/testing/chainsaw/e2e/pgbackrest-restore/clone-cluster.yaml +++ b/testing/chainsaw/e2e/pgbackrest-restore/clone-cluster.yaml @@ -26,6 +26,8 @@ spec: postgresVersion: ($postgres.version) instances: - dataVolumeClaimSpec: ($volume) + tablespaceVolumes: + - { name: barn, dataVolumeClaimSpec: ($volume) } backups: pgbackrest: repos: diff --git a/testing/chainsaw/e2e/pgbackrest-restore/verify-02-data.yaml b/testing/chainsaw/e2e/pgbackrest-restore/verify-02-data.yaml index 2b874de7bd..242222bd10 100644 --- a/testing/chainsaw/e2e/pgbackrest-restore/verify-02-data.yaml +++ b/testing/chainsaw/e2e/pgbackrest-restore/verify-02-data.yaml @@ -63,6 +63,9 @@ spec: BEGIN SELECT jsonb_agg(important) INTO restored FROM important; ASSERT restored = '[{"data":"treasure"}]', format('got %L', restored); + + SELECT jsonb_agg(cows) INTO restored FROM cows; + ASSERT restored = '[{"name":"nellie"}]', format('got %L', restored); END $$$$; - assert: From eb10022665c7b16659d2c6e80c0cbdb747cff9d5 Mon Sep 17 00:00:00 2001 From: Chris Bandy Date: Fri, 8 Nov 2024 16:33:55 -0600 Subject: [PATCH 3/3] Add a template to the restore E2E test This helped me reproduce a race in "pg_ctl start" during Postgres recovery. Issue: PGO-1945 --- .../pgbackrest-restore/change-parameters.yaml | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 testing/chainsaw/e2e/pgbackrest-restore/change-parameters.yaml diff --git a/testing/chainsaw/e2e/pgbackrest-restore/change-parameters.yaml b/testing/chainsaw/e2e/pgbackrest-restore/change-parameters.yaml new file mode 100644 index 0000000000..7a755012c6 --- /dev/null +++ b/testing/chainsaw/e2e/pgbackrest-restore/change-parameters.yaml @@ -0,0 +1,62 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: change-parameters +spec: + try: + - + description: > + Read the timestamp at which PostgreSQL last started + script: + outputs: [{ name: before, value: (trim_space($stdout)) }] + content: | + PRIMARY=$( + kubectl get pod --namespace "${NAMESPACE}" \ + --output name --selector ' + postgres-operator.crunchydata.com/cluster=original, + postgres-operator.crunchydata.com/role=master' + ) + + kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \ + -- psql -qAt --command 'SELECT pg_postmaster_start_time()' + + - patch: + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: original + spec: + config: + parameters: ($parameters) + + - + description: > + Wait for Postgres to restart, signal the WAL archiver, then wait for WAL to archive + script: + timeout: 30s + env: [{ name: BEFORE, value: ($before) }] + content: | + PRIMARY=$( + kubectl get pod --namespace "${NAMESPACE}" \ + --output name --selector ' + postgres-operator.crunchydata.com/cluster=original, + postgres-operator.crunchydata.com/role=master' + ) + + while true; do + START=$( + kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \ + -- psql -qAt --command 'SELECT pg_postmaster_start_time()' + ) + if [ "${START}" ] && [ "${START}" != "${BEFORE}" ]; then break; else sleep 1; fi + done + echo "${START} != ${BEFORE}" + + kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \ + -- psql -qb --command "SELECT pg_stat_reset_shared('archiver')" --command 'SELECT pg_switch_wal()' + + while [ 0 = "$( + kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \ + -- psql -qAt --command 'SELECT archived_count FROM pg_stat_archiver' + )" ]; do sleep 1; done