Skip to content

Commit 4fac9a8

Browse files
bastjanbastjan
authored
Add controller to deploy machine-api-controllers for full functionality (#7)
The original deployment is done by [machine-api-operator](https://github.com/openshift/machine-api-operator/blob/9c3e4a04009ae84958c25b4cbb380a24e7260761/pkg/operator/sync.go#L70-L164), but it there is no possibility of using this on with non-inlined providers. The controller refuses to create a deployment if the upstream one exists and uses jsonnet for easier rendering. Upstream images are taken from the upstream image configmap. The deployment was taken from a vSphere cluster with some hidden dependencies like [machine-controller-manager](https://github.com/openshift/machine-api-operator/blob/a6fe8378811630a030c8509b180fcec5f53ce3d5/Dockerfile#L12) removed.
1 parent d4e22d7 commit 4fac9a8

7 files changed

+704
-2
lines changed

‎Makefile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,9 @@ clean: ## Cleans up the generated resources
5757
rm -rf .tmpvendor
5858

5959
.PHONY: run
60+
RUN_TARGET ?= manager
6061
run: generate fmt vet ## Run a controller from your host.
61-
go run ./main.go
62+
go run ./main.go "-target=$(RUN_TARGET)"
6263

6364
###
6465
### Assets

‎config/samples/machineset-cloudscale-known-working.yml

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
apiVersion: machine.openshift.io/v1beta1
2+
kind: MachineSet
3+
metadata:
4+
name: app
5+
namespace: openshift-machine-api
6+
labels:
7+
machine.openshift.io/cluster-api-cluster: c-appuio-lab-cloudscale-rma-0
8+
name: app
9+
spec:
10+
deletePolicy: Oldest
11+
replicas: 0
12+
selector:
13+
matchLabels:
14+
machine.openshift.io/cluster-api-cluster: c-appuio-lab-cloudscale-rma-0
15+
machine.openshift.io/cluster-api-machineset: app
16+
template:
17+
metadata:
18+
labels:
19+
machine.openshift.io/cluster-api-cluster: c-appuio-lab-cloudscale-rma-0
20+
machine.openshift.io/cluster-api-machine-role: app
21+
machine.openshift.io/cluster-api-machine-type: app
22+
machine.openshift.io/cluster-api-machineset: app
23+
spec:
24+
lifecycleHooks: {}
25+
metadata:
26+
labels:
27+
node-role.kubernetes.io/app: ""
28+
node-role.kubernetes.io/worker: ""
29+
providerSpec:
30+
value:
31+
zone: rma1
32+
baseDomain: lab-cloudscale-rma-0.appuio.cloud
33+
flavor: flex-16-4
34+
image: custom:rhcos-4.15
35+
rootVolumeSizeGB: 100
36+
antiAffinityKey: app
37+
interfaces:
38+
- type: Private
39+
networkUUID: fd2b132d-f5d0-4024-b99f-68e5321ab4d1
40+
userDataSecret:
41+
name: cloudscale-user-data
42+
tokenSecret:
43+
name: cloudscale-rw-token

‎controllers/machine_api_controllers_controller.go

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
package controllers
2+
3+
import (
4+
"context"
5+
_ "embed"
6+
"encoding/json"
7+
"fmt"
8+
9+
"github.com/google/go-jsonnet"
10+
appsv1 "k8s.io/api/apps/v1"
11+
corev1 "k8s.io/api/core/v1"
12+
apierrors "k8s.io/apimachinery/pkg/api/errors"
13+
"k8s.io/apimachinery/pkg/runtime"
14+
ctrl "sigs.k8s.io/controller-runtime"
15+
"sigs.k8s.io/controller-runtime/pkg/client"
16+
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
17+
"sigs.k8s.io/controller-runtime/pkg/log"
18+
)
19+
20+
const (
21+
imagesConfigMapName = "machine-api-operator-images"
22+
originalUpstreamDeploymentName = "machine-api-controllers"
23+
imageKey = "images.json"
24+
25+
caBundleConfigMapName = "appuio-machine-api-ca-bundle"
26+
)
27+
28+
//go:embed machine_api_controllers_deployment.jsonnet
29+
var deploymentTemplate string
30+
31+
// MachineAPIControllersReconciler creates a appuio-machine-api-controllers deployment based on the images.json ConfigMap
32+
// if the upstream machine-api-controllers does not exist.
33+
type MachineAPIControllersReconciler struct {
34+
client.Client
35+
Scheme *runtime.Scheme
36+
37+
Namespace string
38+
}
39+
40+
func (r *MachineAPIControllersReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
41+
if req.Name != imagesConfigMapName {
42+
return ctrl.Result{}, nil
43+
}
44+
45+
l := log.FromContext(ctx).WithName("UpstreamDeploymentReconciler.Reconcile")
46+
l.Info("Reconciling")
47+
48+
var imageCM corev1.ConfigMap
49+
if err := r.Get(ctx, req.NamespacedName, &imageCM); err != nil {
50+
return ctrl.Result{}, client.IgnoreNotFound(err)
51+
}
52+
53+
ij, ok := imageCM.Data[imageKey]
54+
if !ok {
55+
return ctrl.Result{}, fmt.Errorf("%q key not found in ConfigMap %q", imageKey, imagesConfigMapName)
56+
}
57+
images := make(map[string]string)
58+
if err := json.Unmarshal([]byte(ij), &images); err != nil {
59+
return ctrl.Result{}, fmt.Errorf("failed to unmarshal %q from %q: %w", imageKey, imagesConfigMapName, err)
60+
}
61+
62+
// Check that the original upstream deployment does not exist
63+
// If it does, we should not create the new deployment
64+
var upstreamDeployment appsv1.Deployment
65+
err := r.Get(ctx, client.ObjectKey{
66+
Name: originalUpstreamDeploymentName,
67+
Namespace: r.Namespace,
68+
}, &upstreamDeployment)
69+
if err == nil {
70+
return ctrl.Result{}, fmt.Errorf("original upstream deployment %s already exists", originalUpstreamDeploymentName)
71+
} else if !apierrors.IsNotFound(err) {
72+
return ctrl.Result{}, fmt.Errorf("failed to check for original upstream deployment %s: %w", originalUpstreamDeploymentName, err)
73+
}
74+
75+
vm, err := jsonnetVMWithContext(images)
76+
if err != nil {
77+
return ctrl.Result{}, fmt.Errorf("failed to create jsonnet VM: %w", err)
78+
}
79+
80+
ud, err := vm.EvaluateAnonymousSnippet("controllers_deployment.jsonnet", deploymentTemplate)
81+
if err != nil {
82+
return ctrl.Result{}, fmt.Errorf("failed to evaluate jsonnet: %w", err)
83+
}
84+
85+
// TODO(bastjan) this could be way more generic and support any kind of object.
86+
// We don't need any other object types right now, so we're keeping it simple.
87+
var toDeploy appsv1.Deployment
88+
if err := json.Unmarshal([]byte(ud), &toDeploy); err != nil {
89+
return ctrl.Result{}, fmt.Errorf("failed to unmarshal jsonnet output: %w", err)
90+
}
91+
if toDeploy.APIVersion != "apps/v1" || toDeploy.Kind != "Deployment" {
92+
return ctrl.Result{}, fmt.Errorf("expected Deployment, got %s/%s", toDeploy.APIVersion, toDeploy.Kind)
93+
}
94+
toDeploy.Namespace = r.Namespace
95+
if err := controllerutil.SetControllerReference(&imageCM, &toDeploy, r.Scheme); err != nil {
96+
return ctrl.Result{}, fmt.Errorf("failed to set controller reference: %w", err)
97+
}
98+
if err := r.Client.Patch(ctx, &toDeploy, client.Apply, client.FieldOwner("upstream-deployment-controller")); err != nil {
99+
return ctrl.Result{}, fmt.Errorf("failed to apply Deployment %q: %w", toDeploy.GetName(), err)
100+
}
101+
102+
return ctrl.Result{}, nil
103+
}
104+
105+
// SetupWithManager sets up the controller with the Manager.
106+
func (r *MachineAPIControllersReconciler) SetupWithManager(mgr ctrl.Manager) error {
107+
return ctrl.NewControllerManagedBy(mgr).
108+
For(&corev1.ConfigMap{}).
109+
Owns(&appsv1.Deployment{}).
110+
Complete(r)
111+
}
112+
113+
func jsonnetVMWithContext(images map[string]string) (*jsonnet.VM, error) {
114+
jcr, err := json.Marshal(map[string]any{
115+
"images": images,
116+
})
117+
if err != nil {
118+
return nil, fmt.Errorf("unable to marshal jsonnet context: %w", err)
119+
}
120+
jvm := jsonnet.MakeVM()
121+
jvm.ExtCode("context", string(jcr))
122+
// Don't allow imports
123+
jvm.Importer(&jsonnet.MemoryImporter{})
124+
return jvm, nil
125+
}

‎controllers/machine_api_controllers_controller_test.go

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
package controllers
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"errors"
7+
"fmt"
8+
"testing"
9+
10+
"github.com/stretchr/testify/assert"
11+
"github.com/stretchr/testify/require"
12+
appsv1 "k8s.io/api/apps/v1"
13+
corev1 "k8s.io/api/core/v1"
14+
apierrors "k8s.io/apimachinery/pkg/api/errors"
15+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
16+
"k8s.io/apimachinery/pkg/runtime"
17+
"k8s.io/apimachinery/pkg/types"
18+
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
19+
ctrl "sigs.k8s.io/controller-runtime"
20+
"sigs.k8s.io/controller-runtime/pkg/client"
21+
"sigs.k8s.io/controller-runtime/pkg/client/fake"
22+
)
23+
24+
func Test_MachineAPIControllersReconciler_Reconcile(t *testing.T) {
25+
t.Parallel()
26+
27+
ctx := context.Background()
28+
29+
const namespace = "openshift-machine-api"
30+
31+
scheme := runtime.NewScheme()
32+
require.NoError(t, clientgoscheme.AddToScheme(scheme))
33+
34+
images := map[string]string{
35+
"machineAPIOperator": "registry.io/machine-api-operator:v1.0.0",
36+
"kubeRBACProxy": "registry.io/kube-rbac-proxy:v1.0.0",
37+
}
38+
imagesJSON, err := json.Marshal(images)
39+
require.NoError(t, err)
40+
41+
ucm := &corev1.ConfigMap{
42+
ObjectMeta: metav1.ObjectMeta{
43+
Name: imagesConfigMapName,
44+
Namespace: namespace,
45+
},
46+
Data: map[string]string{
47+
imageKey: string(imagesJSON),
48+
},
49+
}
50+
51+
c := &fakeSSA{
52+
fake.NewClientBuilder().
53+
WithScheme(scheme).
54+
WithRuntimeObjects(ucm).
55+
Build(),
56+
}
57+
58+
r := &MachineAPIControllersReconciler{
59+
Client: c,
60+
Scheme: scheme,
61+
62+
Namespace: namespace,
63+
}
64+
65+
_, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(ucm)})
66+
require.NoError(t, err)
67+
68+
var deployment appsv1.Deployment
69+
require.NoError(t, c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: "appuio-" + originalUpstreamDeploymentName}, &deployment))
70+
71+
assert.Equal(t, "system-node-critical", deployment.Spec.Template.Spec.PriorityClassName)
72+
for _, c := range deployment.Spec.Template.Spec.Containers {
73+
if c.Image == images["machineAPIOperator"] || c.Image == images["kubeRBACProxy"] {
74+
continue
75+
}
76+
t.Errorf("expected image %q or %q, got %q", images["machineAPIOperator"], images["kubeRBACProxy"], c.Image)
77+
}
78+
}
79+
80+
func Test_MachineAPIControllersReconciler_OriginalDeploymentExists(t *testing.T) {
81+
t.Parallel()
82+
83+
ctx := context.Background()
84+
85+
const namespace = "openshift-machine-api"
86+
87+
scheme := runtime.NewScheme()
88+
require.NoError(t, clientgoscheme.AddToScheme(scheme))
89+
90+
images := map[string]string{
91+
"machineAPIOperator": "registry.io/machine-api-operator:v1.0.0",
92+
"kubeRBACProxy": "registry.io/kube-rbac-proxy:v1.0.0",
93+
}
94+
imagesJSON, err := json.Marshal(images)
95+
require.NoError(t, err)
96+
97+
ucm := &corev1.ConfigMap{
98+
ObjectMeta: metav1.ObjectMeta{
99+
Name: imagesConfigMapName,
100+
Namespace: namespace,
101+
},
102+
Data: map[string]string{
103+
imageKey: string(imagesJSON),
104+
},
105+
}
106+
107+
origDeploy := &appsv1.Deployment{
108+
ObjectMeta: metav1.ObjectMeta{
109+
Name: originalUpstreamDeploymentName,
110+
Namespace: namespace,
111+
},
112+
}
113+
114+
c := &fakeSSA{
115+
fake.NewClientBuilder().
116+
WithScheme(scheme).
117+
WithRuntimeObjects(ucm, origDeploy).
118+
Build(),
119+
}
120+
121+
r := &MachineAPIControllersReconciler{
122+
Client: c,
123+
Scheme: scheme,
124+
125+
Namespace: namespace,
126+
}
127+
128+
_, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(ucm)})
129+
require.ErrorContains(t, err, "machine-api-controllers already exists")
130+
}
131+
132+
// fakeSSA is a fake client that approximates SSA.
133+
// It creates objects that don't exist yet and _updates_ them if they exist.
134+
// This is completely kaputt since the object is overwritten with the new object.
135+
// See https://github.com/kubernetes-sigs/controller-runtime/issues/2341
136+
type fakeSSA struct {
137+
client.WithWatch
138+
}
139+
140+
// Patch approximates SSA by creating objects that don't exist yet.
141+
func (f *fakeSSA) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
142+
// Apply patches are supposed to upsert, but fake client fails if the object doesn't exist,
143+
// if an apply patch occurs for an object that doesn't yet exist, create it.
144+
if patch.Type() != types.ApplyPatchType {
145+
return f.WithWatch.Patch(ctx, obj, patch, opts...)
146+
}
147+
check, ok := obj.DeepCopyObject().(client.Object)
148+
if !ok {
149+
return errors.New("could not check for object in fake client")
150+
}
151+
if err := f.WithWatch.Get(ctx, client.ObjectKeyFromObject(obj), check); apierrors.IsNotFound(err) {
152+
if err := f.WithWatch.Create(ctx, check); err != nil {
153+
return fmt.Errorf("could not inject object creation for fake: %w", err)
154+
}
155+
} else if err != nil {
156+
return fmt.Errorf("could not check for object in fake client: %w", err)
157+
}
158+
return f.WithWatch.Update(ctx, obj)
159+
}

0 commit comments

Comments
 (0)