Skip to content

Commit c7237f8

Browse files
feat(api): add resourcepools and claims (#1333)
* feat: functional appsets * feat(api): add resourcepools api Signed-off-by: Oliver Bähler <[email protected]> * chore: fix gomod Signed-off-by: Oliver Bähler <[email protected]> * chore: correct webhooks Signed-off-by: Oliver Bähler <[email protected]> * chore: fix harpoon image Signed-off-by: Oliver Bähler <[email protected]> * chore: improve e2e Signed-off-by: Oliver Bähler <[email protected]> * chore: add labels to e2e test Signed-off-by: Oliver Bähler <[email protected]> * chore: fix status handling Signed-off-by: Oliver Bähler <[email protected]> * chore: fix racing conditions Signed-off-by: Oliver Bähler <[email protected]> * chore: make values compatible Signed-off-by: Oliver Bähler <[email protected]> * chore: fix custom resources test Signed-off-by: Oliver Bähler <[email protected]> * chore: correct metrics Signed-off-by: Oliver Bähler <[email protected]> --------- Signed-off-by: Oliver Bähler <[email protected]>
1 parent f143abc commit c7237f8

File tree

115 files changed

+7221
-116
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

115 files changed

+7221
-116
lines changed

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
*.dylib
88
bin
99
dist/
10+
config/
1011

1112
# Test binary, build with `go test -c`
1213
*.test

.golangci.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,10 @@ linters:
5656
- third_party$
5757
- builtin$
5858
- examples$
59+
rules:
60+
- path: pkg/meta/
61+
linters:
62+
- dupl
5963
formatters:
6064
enable:
6165
- gci

.ko.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,6 @@ defaultPlatforms:
44
- linux/arm
55
builds:
66
- id: capsule
7-
main: ./
7+
main: ./cmd/
88
ldflags:
99
- '{{ if index .Env "LD_FLAGS" }}{{ .Env.LD_FLAGS }}{{ end }}'

.pre-commit-config.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,3 +39,8 @@ repos:
3939
entry: make golint
4040
language: system
4141
files: \.go$
42+
- id: go-test
43+
name: Execute go test
44+
entry: make test
45+
language: system
46+
files: \.go$

Dockerfile.tracing

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ FROM ${TARGET_IMAGE} AS target
55
# Inject Harpoon Image
66
FROM ghcr.io/alegrey91/harpoon:latest
77
WORKDIR /
8-
COPY --from=target /ko-app/capsule ./manager
8+
COPY --from=target /ko-app/cmd ./manager
99
RUN chmod +x ./harpoon
1010
ENTRYPOINT ["/harpoon", \
1111
"capture", \

Makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ LD_FLAGS := "-X main.Version=$(VERSION) \
178178
ko-build-capsule: ko
179179
@echo Building Capsule $(KO_TAGS) for $(KO_PLATFORM) >&2
180180
@LD_FLAGS=$(LD_FLAGS) KOCACHE=$(KOCACHE) KO_DOCKER_REPO=$(CAPSULE_IMG) \
181-
$(KO) build ./ --bare --tags=$(KO_TAGS) --push=false --local --platform=$(KO_PLATFORM)
181+
$(KO) build ./cmd/ --bare --tags=$(KO_TAGS) --push=false --local --platform=$(KO_PLATFORM)
182182

183183
.PHONY: ko-build-all
184184
ko-build-all: ko-build-capsule
@@ -204,7 +204,7 @@ ko-login: ko
204204
.PHONY: ko-publish-capsule
205205
ko-publish-capsule: ko-login ## Build and publish kyvernopre image (with ko)
206206
@LD_FLAGS=$(LD_FLAGS) KOCACHE=$(KOCACHE) KO_DOCKER_REPO=$(CAPSULE_IMG) \
207-
$(KO) build ./ --bare --tags=$(KO_TAGS)
207+
$(KO) build ./cmd/ --bare --tags=$(KO_TAGS)
208208

209209
.PHONY: ko-publish-all
210210
ko-publish-all: ko-publish-capsule

PROJECT

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
1+
# Code generated by tool. DO NOT EDIT.
2+
# This file is used to track the info used to scaffold your project
3+
# and allow the plugins properly work.
4+
# More info: https://book.kubebuilder.io/reference/project-config.html
15
domain: clastix.io
26
layout:
3-
- go.kubebuilder.io/v3
7+
- go.kubebuilder.io/v4
48
plugins:
59
manifests.sdk.operatorframework.io/v2: {}
610
scorecard.sdk.operatorframework.io/v2: {}
@@ -44,4 +48,20 @@ resources:
4448
kind: GlobalTenantResource
4549
path: github.com/projectcapsule/capsule/api/v1beta2
4650
version: v1beta2
51+
- api:
52+
crdVersion: v1
53+
domain: clastix.io
54+
group: capsule
55+
kind: ResourcePool
56+
path: github.com/projectcapsule/capsule/api/v1beta2
57+
version: v1beta2
58+
- api:
59+
crdVersion: v1
60+
namespaced: true
61+
controller: true
62+
domain: clastix.io
63+
group: capsule
64+
kind: ResourcePoolClaim
65+
path: github.com/projectcapsule/capsule/api/v1beta2
66+
version: v1beta2
4767
version: "3"

api/v1beta2/resourcepool_func.go

Lines changed: 276 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,276 @@
1+
// Copyright 2020-2023 Project Capsule Authors.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package v1beta2
5+
6+
import (
7+
"errors"
8+
"sort"
9+
10+
corev1 "k8s.io/api/core/v1"
11+
"k8s.io/apimachinery/pkg/api/resource"
12+
13+
"github.com/projectcapsule/capsule/pkg/api"
14+
)
15+
16+
func (r *ResourcePool) AssignNamespaces(namespaces []corev1.Namespace) {
17+
var l []string
18+
19+
for _, ns := range namespaces {
20+
if ns.Status.Phase == corev1.NamespaceActive && ns.DeletionTimestamp == nil {
21+
l = append(l, ns.GetName())
22+
}
23+
}
24+
25+
sort.Strings(l)
26+
27+
r.Status.NamespaceSize = uint(len(l))
28+
r.Status.Namespaces = l
29+
}
30+
31+
func (r *ResourcePool) AssignClaims() {
32+
var size uint
33+
34+
for _, claims := range r.Status.Claims {
35+
for range claims {
36+
size++
37+
}
38+
}
39+
40+
r.Status.ClaimSize = size
41+
}
42+
43+
func (r *ResourcePool) GetClaimFromStatus(cl *ResourcePoolClaim) *ResourcePoolClaimsItem {
44+
ns := cl.Namespace
45+
46+
claims := r.Status.Claims[ns]
47+
if claims == nil {
48+
return nil
49+
}
50+
51+
for _, claim := range claims {
52+
if claim.UID == cl.UID {
53+
return claim
54+
}
55+
}
56+
57+
return nil
58+
}
59+
60+
func (r *ResourcePool) AddClaimToStatus(claim *ResourcePoolClaim) {
61+
ns := claim.Namespace
62+
63+
if r.Status.Claims == nil {
64+
r.Status.Claims = ResourcePoolNamespaceClaimsStatus{}
65+
}
66+
67+
if r.Status.Allocation.Claimed == nil {
68+
r.Status.Allocation.Claimed = corev1.ResourceList{}
69+
}
70+
71+
claims := r.Status.Claims[ns]
72+
if claims == nil {
73+
claims = ResourcePoolClaimsList{}
74+
}
75+
76+
scl := &ResourcePoolClaimsItem{
77+
StatusNameUID: api.StatusNameUID{
78+
UID: claim.UID,
79+
Name: api.Name(claim.Name),
80+
},
81+
Claims: claim.Spec.ResourceClaims,
82+
}
83+
84+
// Try to update existing entry if UID matches
85+
exists := false
86+
87+
for i, cl := range claims {
88+
if cl.UID == claim.UID {
89+
claims[i] = scl
90+
91+
exists = true
92+
93+
break
94+
}
95+
}
96+
97+
if !exists {
98+
claims = append(claims, scl)
99+
}
100+
101+
r.Status.Claims[ns] = claims
102+
103+
r.CalculateClaimedResources()
104+
}
105+
106+
func (r *ResourcePool) RemoveClaimFromStatus(claim *ResourcePoolClaim) {
107+
newClaims := ResourcePoolClaimsList{}
108+
109+
claims, ok := r.Status.Claims[claim.Namespace]
110+
if !ok {
111+
return
112+
}
113+
114+
for _, cl := range claims {
115+
if cl.UID != claim.UID {
116+
newClaims = append(newClaims, cl)
117+
}
118+
}
119+
120+
r.Status.Claims[claim.Namespace] = newClaims
121+
122+
if len(newClaims) == 0 {
123+
delete(r.Status.Claims, claim.Namespace)
124+
}
125+
}
126+
127+
func (r *ResourcePool) CalculateClaimedResources() {
128+
usage := corev1.ResourceList{}
129+
130+
for res := range r.Status.Allocation.Hard {
131+
usage[res] = resource.MustParse("0")
132+
}
133+
134+
for _, claims := range r.Status.Claims {
135+
for _, claim := range claims {
136+
for resourceName, qt := range claim.Claims {
137+
amount, exists := usage[resourceName]
138+
if !exists {
139+
amount = resource.MustParse("0")
140+
}
141+
142+
amount.Add(qt)
143+
usage[resourceName] = amount
144+
}
145+
}
146+
}
147+
148+
r.Status.Allocation.Claimed = usage
149+
150+
r.CalculateAvailableResources()
151+
}
152+
153+
func (r *ResourcePool) CalculateAvailableResources() {
154+
available := corev1.ResourceList{}
155+
156+
for res, qt := range r.Status.Allocation.Hard {
157+
amount, exists := r.Status.Allocation.Claimed[res]
158+
if exists {
159+
qt.Sub(amount)
160+
}
161+
162+
available[res] = qt
163+
}
164+
165+
r.Status.Allocation.Available = available
166+
}
167+
168+
func (r *ResourcePool) CanClaimFromPool(claim corev1.ResourceList) []error {
169+
claimable := r.GetAvailableClaimableResources()
170+
errs := []error{}
171+
172+
for resourceName, req := range claim {
173+
available, exists := claimable[resourceName]
174+
if !exists || available.IsZero() || available.Cmp(req) < 0 {
175+
errs = append(errs, errors.New("not enough resources"+string(resourceName)+"available"))
176+
}
177+
}
178+
179+
return errs
180+
}
181+
182+
func (r *ResourcePool) GetAvailableClaimableResources() corev1.ResourceList {
183+
hard := r.Status.Allocation.Hard.DeepCopy()
184+
185+
for resourceName, qt := range hard {
186+
claimed, exists := r.Status.Allocation.Claimed[resourceName]
187+
if !exists {
188+
claimed = resource.MustParse("0")
189+
}
190+
191+
qt.Sub(claimed)
192+
193+
hard[resourceName] = qt
194+
}
195+
196+
return hard
197+
}
198+
199+
// Gets the Hard specification for the resourcequotas
200+
// This takes into account the default resources being used. However they don't count towards the claim usage
201+
// This can be changed in the future, the default is not calculated as usage because this might interrupt the namespace management
202+
// As we would need to verify if a new namespace with it's defaults still has place in the Pool. Same with attempting to join existing namespaces.
203+
func (r *ResourcePool) GetResourceQuotaHardResources(namespace string) corev1.ResourceList {
204+
_, claimed := r.GetNamespaceClaims(namespace)
205+
206+
for resourceName, amount := range claimed {
207+
if amount.IsZero() {
208+
delete(claimed, resourceName)
209+
}
210+
}
211+
212+
// Only Consider Default, when enabled
213+
for resourceName, amount := range r.Spec.Defaults {
214+
usedValue := claimed[resourceName]
215+
usedValue.Add(amount)
216+
217+
claimed[resourceName] = usedValue
218+
}
219+
220+
return claimed
221+
}
222+
223+
// Gets the total amount of claimed resources for a namespace.
224+
func (r *ResourcePool) GetNamespaceClaims(namespace string) (claims map[string]*ResourcePoolClaimsItem, claimedResources corev1.ResourceList) {
225+
claimedResources = corev1.ResourceList{}
226+
claims = map[string]*ResourcePoolClaimsItem{}
227+
228+
// First, check if quota exists in the status
229+
for ns, cl := range r.Status.Claims {
230+
if ns != namespace {
231+
continue
232+
}
233+
234+
for _, claim := range cl {
235+
for resourceName, claimed := range claim.Claims {
236+
usedValue, usedExists := claimedResources[resourceName]
237+
if !usedExists {
238+
usedValue = resource.MustParse("0") // Default to zero if no used value is found
239+
}
240+
241+
// Combine with claim
242+
usedValue.Add(claimed)
243+
claimedResources[resourceName] = usedValue
244+
}
245+
246+
claims[string(claim.UID)] = claim
247+
}
248+
}
249+
250+
return
251+
}
252+
253+
// Calculate usage for each namespace.
254+
func (r *ResourcePool) GetClaimedByNamespaceClaims() (claims map[string]corev1.ResourceList) {
255+
claims = map[string]corev1.ResourceList{}
256+
257+
// First, check if quota exists in the status
258+
for ns, cl := range r.Status.Claims {
259+
claims[ns] = corev1.ResourceList{}
260+
nsScope := claims[ns]
261+
262+
for _, claim := range cl {
263+
for resourceName, claimed := range claim.Claims {
264+
usedValue, usedExists := nsScope[resourceName]
265+
if !usedExists {
266+
usedValue = resource.MustParse("0")
267+
}
268+
269+
usedValue.Add(claimed)
270+
nsScope[resourceName] = usedValue
271+
}
272+
}
273+
}
274+
275+
return
276+
}

0 commit comments

Comments
 (0)