Skip to content

Commit bf5fb88

Browse files
authored
feat(#3097): create warning translation failure events for CA secrets (#3125)
It starts creating Kubernetes Events for all TranslationFailures detected during the translation phase. It will create one event per causing object. An integration test suite is also added in order to track all translation failure cases - which should make eventual future refactors safer (e.g. moving validation rules from the parser to the controllers).
1 parent 551528e commit bf5fb88

File tree

5 files changed

+169
-2
lines changed

5 files changed

+169
-2
lines changed

CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,8 @@ Adding a new version? You'll need three changes:
110110
[#3121](https://github.com/Kong/kubernetes-ingress-controller/pull/3121)
111111
- Routes support annotations for path handling.
112112
[#3121](https://github.com/Kong/kubernetes-ingress-controller/pull/3121)
113+
- Warning events are recorded when CA secrets cannot be properly translated into Kong configuration.
114+
[#3125](https://github.com/Kong/kubernetes-ingress-controller/pull/3125)
113115

114116
### Fixed
115117

internal/dataplane/kong_client.go

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ import (
1111
"github.com/kong/go-kong/kong"
1212
"github.com/prometheus/client_golang/prometheus"
1313
"github.com/sirupsen/logrus"
14+
corev1 "k8s.io/api/core/v1"
15+
"k8s.io/client-go/tools/record"
1416
"sigs.k8s.io/controller-runtime/pkg/client"
1517

1618
"github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane/deckgen"
@@ -24,6 +26,9 @@ import (
2426
"github.com/kong/kubernetes-ingress-controller/v2/internal/versions"
2527
)
2628

29+
// KongConfigurationTranslationFailedEventReason defines an event reason used for creating all translation failure events.
30+
const KongConfigurationTranslationFailedEventReason = "KongConfigurationTranslationFailed"
31+
2732
// -----------------------------------------------------------------------------
2833
// Dataplane Client - Kong - Public Types
2934
// -----------------------------------------------------------------------------
@@ -106,6 +111,9 @@ type KongClient struct {
106111
// whether a Kubernetes object has corresponding data-plane configuration that
107112
// is actively configured (e.g. to know how to set the object status).
108113
kubernetesObjectReportsFilter k8sobj.Set
114+
115+
// eventRecorder is used to record warning events for translation failures.
116+
eventRecorder record.EventRecorder
109117
}
110118

111119
// NewKongClient provides a new KongClient object after connecting to the
@@ -118,6 +126,7 @@ func NewKongClient(
118126
skipCACertificates bool,
119127
diagnostic util.ConfigDumpDiagnostic,
120128
kongConfig sendconfig.Kong,
129+
eventRecorder record.EventRecorder,
121130
) (*KongClient, error) {
122131
// build the client object
123132
cache := store.NewCacheStores()
@@ -131,6 +140,7 @@ func NewKongClient(
131140
prometheusMetrics: metrics.NewCtrlFuncMetrics(),
132141
cache: &cache,
133142
kongConfig: kongConfig,
143+
eventRecorder: eventRecorder,
134144
}
135145

136146
// download the kong root configuration (and validate connectivity to the proxy API)
@@ -320,6 +330,7 @@ func (c *KongClient) Update(ctx context.Context) error {
320330
c.prometheusMetrics.TranslationCount.With(prometheus.Labels{
321331
metrics.SuccessKey: metrics.SuccessFalse,
322332
}).Inc()
333+
c.recordTranslationFailureWarningEvents(translationFailures)
323334
c.logger.Debugf("%d translation failures have occurred when building data-plane configuration", failuresCount)
324335
} else {
325336
c.prometheusMetrics.TranslationCount.With(prometheus.Labels{
@@ -447,3 +458,13 @@ func (c *KongClient) updateKubernetesObjectReportFilter(set k8sobj.Set) {
447458
defer c.kubernetesObjectReportLock.Unlock()
448459
c.kubernetesObjectReportsFilter = set
449460
}
461+
462+
// recordTranslationFailureWarningEvents records a warning KongConfigurationTranslationFailedEventReason events,
463+
// one per a translation failure causing object.
464+
func (c *KongClient) recordTranslationFailureWarningEvents(translationFailures []parser.TranslationFailure) {
465+
for _, failure := range translationFailures {
466+
for _, obj := range failure.CausingObjects() {
467+
c.eventRecorder.Event(obj, corev1.EventTypeWarning, KongConfigurationTranslationFailedEventReason, failure.Reason())
468+
}
469+
}
470+
}

internal/manager/consts.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,6 @@ const MetricsPort = 10255
1919

2020
// DiagnosticsPort is the default port of the manager's diagnostics service listens on.
2121
const DiagnosticsPort = 10256
22+
23+
// KongClientEventRecorderComponentName is a KongClient component name used to identify the events recording component.
24+
const KongClientEventRecorderComponentName = "kong-client"

internal/manager/run.go

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,18 @@ func RunWithLogger(ctx context.Context, c *Config, diagnostic util.ConfigDumpDia
151151
if err != nil {
152152
return fmt.Errorf("%f is not a valid number of seconds to the timeout config for the kong client: %w", c.ProxyTimeoutSeconds, err)
153153
}
154-
dataplaneClient, err := dataplane.NewKongClient(deprecatedLogger, timeoutDuration, c.IngressClassName, c.EnableReverseSync, c.SkipCACertificates, diagnostic, kongConfig)
154+
155+
eventRecorder := mgr.GetEventRecorderFor(KongClientEventRecorderComponentName)
156+
dataplaneClient, err := dataplane.NewKongClient(
157+
deprecatedLogger,
158+
timeoutDuration,
159+
c.IngressClassName,
160+
c.EnableReverseSync,
161+
c.SkipCACertificates,
162+
diagnostic,
163+
kongConfig,
164+
eventRecorder,
165+
)
155166
if err != nil {
156167
return fmt.Errorf("failed to initialize kong data-plane client: %w", err)
157168
}
@@ -200,7 +211,7 @@ func RunWithLogger(ctx context.Context, c *Config, diagnostic util.ConfigDumpDia
200211

201212
// BUG: kubebuilder (at the time of writing - 3.0.0-rc.1) does not allow this tag anywhere else than main.go
202213
// See https://github.com/kubernetes-sigs/kubebuilder/issues/932
203-
//+kubebuilder:scaffold:builder
214+
// +kubebuilder:scaffold:builder
204215

205216
setupLog.Info("Starting health check servers")
206217
if err := mgr.AddHealthzCheck("health", healthz.Ping); err != nil {
Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
//go:build integration_tests
2+
// +build integration_tests
3+
4+
package integration
5+
6+
import (
7+
"fmt"
8+
"testing"
9+
"time"
10+
11+
"github.com/stretchr/testify/assert"
12+
"github.com/stretchr/testify/require"
13+
corev1 "k8s.io/api/core/v1"
14+
v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
15+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
16+
"sigs.k8s.io/controller-runtime/pkg/client"
17+
18+
"github.com/kong/kubernetes-ingress-controller/v2/internal/annotations"
19+
"github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane"
20+
kongv1 "github.com/kong/kubernetes-ingress-controller/v2/pkg/apis/configuration/v1"
21+
"github.com/kong/kubernetes-ingress-controller/v2/pkg/clientset"
22+
)
23+
24+
// TestTranslationFailures ensures that proper warning Kubernetes events are recorded in case of translation failures
25+
// encountered.
26+
func TestTranslationFailures(t *testing.T) {
27+
testCases := []struct {
28+
name string
29+
// translationFailureTrigger should create objects that trigger translation failure and return the objects
30+
// that we expect translation failure warning events to be created for.
31+
translationFailureTrigger func(t *testing.T, ns string) []client.Object
32+
}{
33+
{
34+
name: "invalid CA secret",
35+
translationFailureTrigger: func(t *testing.T, ns string) []client.Object {
36+
createdSecret, err := env.Cluster().Client().CoreV1().Secrets(ns).Create(ctx, invalidCASecret(ns), metav1.CreateOptions{})
37+
require.NoError(t, err)
38+
39+
return []client.Object{createdSecret}
40+
},
41+
},
42+
{
43+
name: "invalid CA secret referred by a plugin",
44+
translationFailureTrigger: func(t *testing.T, ns string) []client.Object {
45+
createdSecret, err := env.Cluster().Client().CoreV1().Secrets(ns).Create(ctx, invalidCASecret(ns), metav1.CreateOptions{})
46+
require.NoError(t, err)
47+
48+
c, err := clientset.NewForConfig(env.Cluster().Config())
49+
require.NoError(t, err)
50+
createdPlugin, err := c.ConfigurationV1().KongPlugins(ns).Create(ctx, pluginUsingInvalidCACert(ns), metav1.CreateOptions{})
51+
require.NoError(t, err)
52+
53+
// expect events for both: a faulty secret and a plugin referring it
54+
return []client.Object{createdSecret, createdPlugin}
55+
},
56+
},
57+
}
58+
59+
for _, tt := range testCases {
60+
tt := tt
61+
t.Run(tt.name, func(t *testing.T) {
62+
t.Parallel()
63+
ns, cleaner := setup(t)
64+
defer func() { assert.NoError(t, cleaner.Cleanup(ctx)) }()
65+
66+
expectedCausingObjects := tt.translationFailureTrigger(t, ns.GetName())
67+
68+
require.Eventually(t, func() bool {
69+
eventsForAllObjectsFound := true
70+
71+
for _, expectedCausingObject := range expectedCausingObjects {
72+
events, err := env.Cluster().Client().CoreV1().Events(ns.GetName()).List(ctx, metav1.ListOptions{
73+
FieldSelector: fmt.Sprintf(
74+
"reason=%s,type=%s,involvedObject.name=%s",
75+
dataplane.KongConfigurationTranslationFailedEventReason,
76+
corev1.EventTypeWarning,
77+
expectedCausingObject.GetName(),
78+
),
79+
})
80+
if err != nil {
81+
t.Logf("failed to list events: %s", err)
82+
eventsForAllObjectsFound = false
83+
}
84+
85+
if len(events.Items) == 0 {
86+
t.Logf("waiting for events related to '%s' to be created", expectedCausingObject.GetName())
87+
eventsForAllObjectsFound = false
88+
}
89+
}
90+
91+
return eventsForAllObjectsFound
92+
}, time.Minute*5, time.Second)
93+
})
94+
}
95+
}
96+
97+
const invalidCASecretID = "8214a145-a328-4c56-ab72-2973a56d4eae" //nolint:gosec
98+
99+
func invalidCASecret(ns string) *corev1.Secret {
100+
return &corev1.Secret{
101+
ObjectMeta: metav1.ObjectMeta{
102+
GenerateName: "ca-secret-",
103+
Namespace: ns,
104+
Labels: map[string]string{
105+
"konghq.com/ca-cert": "true",
106+
},
107+
Annotations: map[string]string{
108+
annotations.IngressClassKey: ingressClass,
109+
},
110+
},
111+
Data: map[string][]byte{
112+
"id": []byte(invalidCASecretID),
113+
// missing cert key
114+
},
115+
}
116+
}
117+
118+
func pluginUsingInvalidCACert(ns string) *kongv1.KongPlugin {
119+
return &kongv1.KongPlugin{
120+
ObjectMeta: metav1.ObjectMeta{
121+
GenerateName: "kong-plugin-",
122+
Namespace: ns,
123+
Annotations: map[string]string{
124+
annotations.IngressClassKey: ingressClass,
125+
},
126+
},
127+
Config: v1.JSON{Raw: []byte(fmt.Sprintf(`{"ca_certificates": ["%s"]}`, invalidCASecretID))},
128+
PluginName: "mtls-auth",
129+
}
130+
}

0 commit comments

Comments
 (0)