Merge pull request #29 from neicnordic/feature/interceptor

Added the interceptor

Author: jbygdell

.github/ci_tests/lint_helper.sh

@@ -9,7 +9,7 @@ touch $1/files/server.crt
 fi
 
 if [ $1 = "sda-svc" ]; then
-for n in ca doa finalize inbox ingest verify
+for n in ca doa finalize inbox ingest interceptor verify
 do
 touch $1/files/$n.crt
 done

.github/ci_tests/svc.conf

@@ -6,6 +6,7 @@
     {"name":"htsget", "ns": "default"},
     {"name":"inbox", "ns": "default"},
     {"name":"ingest", "ns": "default"},
+    {"name":"interceptor", "ns": "default"},
     {"name":"finalize", "ns": "default"},
     {"name":"verify", "ns": "default"},
     {"name":"mq-server", "dns":"broker-sda-mq", "ns": "default"},

.github/workflows/deploy.yml

@@ -129,10 +129,10 @@ jobs:
         cp LocalEGA-deploy-init/config/certs/*.p12 sda-svc/files/
         cp LocalEGA-deploy-init/config/certs/cacerts sda-svc/files/
         cp LocalEGA-deploy-init/config/certs/root.ca.crt sda-svc/files/ca.crt
-        for n in doa finalize ingest verify
+        for n in doa finalize ingest interceptor verify
           do cp LocalEGA-deploy-init/config/certs/$n.ca.crt sda-svc/files/$(echo $n.ca.crt | cut -d '.' -f1,3)
         done
-        for n in doa finalize ingest verify
+        for n in doa finalize ingest interceptor verify
           do cp LocalEGA-deploy-init/config/certs/$n.ca.key sda-svc/files/$(echo $n.ca.key | cut -d '.' -f1,3)
         done
     - name: Deploy the SDA stack
@@ -333,10 +333,10 @@ jobs:
         cp LocalEGA-deploy-init/config/certs/doa.p12 sda-svc/files/
         cp LocalEGA-deploy-init/config/certs/cacerts sda-svc/files/
         cp LocalEGA-deploy-init/config/certs/root.ca.crt sda-svc/files/ca.crt
-        for n in doa finalize ingest verify inbox
+        for n in doa finalize ingest verify inbox interceptor
           do cp LocalEGA-deploy-init/config/certs/$n.ca.crt sda-svc/files/$(echo $n.ca.crt | cut -d '.' -f1,3)
         done
-        for n in doa finalize ingest verify inbox
+        for n in doa finalize ingest verify inbox interceptor
           do cp LocalEGA-deploy-init/config/certs/$n.ca.key sda-svc/files/$(echo $n.ca.key | cut -d '.' -f1,3)
         done
         cp LocalEGA-deploy-init/config/certs/res.ca.crt sda-svc/files/auth.crt

sda-svc/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: sda-svc
-version: 0.2
+version: 0.3
 description: Components for Sensitive Data Archive (SDA) installation
 home: https://neic-sda.readthedocs.io
 icon: https://neic.no/assets/images/logo.png

sda-svc/README.md

@@ -108,6 +108,8 @@ Parameter | Description | Default
 `credentials.ingest.dbPassword` | Database password for ingest | `""`
 `credentials.ingest.mqUser` | Broker user for ingest  | `""`
 `credentials.ingest.mqPassword` | Broker password for ingest | `""`
+`credentials.interceptor.mqUser` | Broker user for interceptor  | `""`
+`credentials.interceptor.mqPassword` | Broker password for interceptor | `""`
 `credentials.verify.dbUser` | Databse user for verify | `""`
 `credentials.verify.dbPassword` | Database password for verify | `""`
 `credentials.verify.mqUser` | Broker user for verify | `""`
@@ -137,6 +139,12 @@ Parameter | Description | Default
 `ingest.imagePullPolicy` | inbox container image pull policy | `Always`
 `ingest.replicaCount` | desired number of ingest workers | `1`
 `ingest.annotations` | Specific annotation for the ingest pod | `{}`
+`interceptor.repository` | interceptor container image repository | `neicnordic/sda-pipeline`
+`interceptor.imageTag` | interceptor container image version | `latest`
+`interceptor.imagePullPolicy` | interceptor container image pull policy | `Always`
+`interceptor.replicaCount` | desired number of interceptor workers | `1`
+`interceptor.annotations` | Specific annotation for the interceptor pod | `{}`
+`interceptor.deploy` | Set to false in a non federated deployment | `true`
 `s3Inbox.repository` | S3inbox container image repository | `neicnordic/sda-s3proxy`
 `s3Inbox.imageTag` | S3inbox container image version | `latest`
 `s3Inbox.imagePullPolicy` | S3inbox container image pull policy | `Always`

sda-svc/templates/_helpers.yaml

@@ -173,6 +173,14 @@ Create chart name and version as used by the chart label.
 {{- ternary .Values.global.broker.password .Values.credentials.inbox.mqPassword (empty .Values.credentials.inbox.mqPassword) -}}
 {{- end -}}
 
+{{/**/}}
+{{- define "mqUserInterceptor" -}}
+{{- ternary .Values.global.broker.username .Values.credentials.interceptor.mqUser (empty .Values.credentials.interceptor.mqUser) -}}
+{{- end -}}
+{{- define "mqPassInterceptor" -}}
+{{- ternary .Values.global.broker.password .Values.credentials.interceptor.mqPassword (empty .Values.credentials.interceptor.mqPassword) -}}
+{{- end -}}
+
 {{/**/}}
 {{- define "dbUserVerify" -}}
 {{- ternary "lega_in" .Values.credentials.verify.dbUser (empty .Values.credentials.verify.dbUser) -}}

sda-svc/templates/interceptor-deploy.yaml

@@ -0,0 +1,111 @@
+{{- if .Values.interceptor.deploy}}
+{{- if or (or (eq "all" .Values.global.deploymentType) (eq "internal" .Values.global.deploymentType) ) (not .Values.global.deploymentType) }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "sda.fullname" . }}-interceptor
+  labels:
+    role: interceptor
+    app: {{ template "sda.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: {{ .Release.Name }}-interceptor
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  replicas: 1
+  revisionHistoryLimit: {{ default "3" .Values.global.revisionHistory }}
+  selector:
+    matchLabels:
+      app: {{ template "sda.name" . }}-interceptor
+      release: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: {{ template "sda.name" . }}-interceptor
+        role: interceptor
+        release: {{ .Release.Name }}
+      annotations:
+        {{- if not .Values.global.secretsService }}
+        checksum/config: {{ include (print $.Template.BasePath "/interceptor-secrets.yaml") . | sha256sum }}
+        {{- end }}
+{{- if .Values.global.podAnnotations }}
+{{- toYaml .Values.global.podAnnotations | nindent 8 -}}
+{{- end }}
+{{- if .Values.interceptor.annotations }}
+{{- toYaml .Values.interceptor.annotations | nindent 8 -}}
+{{- end }}
+    spec:
+    {{- if .Values.global.rbacEnabled}}
+      serviceAccountName: {{ .Release.Name }}
+    {{- end }}
+      securityContext:
+        runAsUser: 65534
+        runAsGroup: 65534
+        fsGroup: 65534
+      containers:
+      - name: interceptor
+        image: "{{ .Values.interceptor.repository }}:{{ .Values.interceptor.imageTag }}"
+        imagePullPolicy: {{ .Values.interceptor.imagePullPolicy | quote }}
+        command: ["sda-interceptor"]
+        securityContext:
+          allowPrivilegeEscalation: false
+        env:
+      {{- if not .Values.global.secretsService }}
+        - name: BROKER_PASSWORD
+          valueFrom:
+              secretKeyRef:
+                name: {{ template "sda.fullname" . }}-interceptor
+                key: mqPassword
+        - name: BROKER_USER
+          valueFrom:
+              secretKeyRef:
+                name: {{ template "sda.fullname" . }}-interceptor
+                key: mqUser
+      {{- end }}
+        {{- if .Values.global.broker.ssl }}
+        - name: BROKER_CACERT
+          value: {{ include "tlsPath" . }}/ca.crt
+        {{- if .Values.global.broker.verifyPeer }}
+        - name: BROKER_CLIENTCERT
+          value: {{ include "tlsPath" . }}/interceptor.crt
+        - name: BROKER_CLIENTKEY
+          value: {{ include "tlsPath" . }}/interceptor.key
+        {{- end }}
+        {{- end }}
+        - name: BROKER_DURABLE
+          value: "true"
+        - name: BROKER_EXCHANGE
+          value: {{ default "lega" .Values.global.broker.exchange | quote}}
+        - name: BROKER_HOST
+          value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }}
+        - name: BROKER_PORT
+          value: {{ .Values.global.broker.port | quote }}
+        - name: BROKER_QUEUE
+          value: "cega_files"
+        - name: BROKER_ROUTINGERROR
+          value: {{ .Values.global.broker.routingError | quote }}
+        - name: BROKER_SSL
+          value: {{ .Values.global.broker.ssl | quote}}
+        - name: BROKER_VERIFYPEER
+          value: {{ .Values.global.broker.verifyPeer | quote }}
+        - name: BROKER_VHOST
+          value: {{ .Values.global.broker.vhost | quote }}
+      {{- if .Values.global.log }}
+        - name: LOG_LEVEL
+          value: {{ .Values.global.logLevel | quote }}
+      {{- end }}
+        resources:
+{{ toYaml .Values.interceptor.resources | trim | indent 10 }}
+  {{- if not .Values.global.pkiService }}
+        volumeMounts:
+        - name: tls
+          mountPath: {{ template "tlsPath" . }}
+      volumes:
+        - name: {{ "tls"  }}
+          secret:
+            defaultMode: 0440
+            secretName: {{ template "sda.fullname" . }}-interceptor-certs
+    {{- end }}
+      restartPolicy: Always
+{{- end }}
+{{- end }}

sda-svc/templates/interceptor-secrets.yaml

@@ -0,0 +1,26 @@
+{{- if .Values.interceptor.deploy}}
+{{- if or (or (eq "all" .Values.global.deploymentType) (eq "internal" .Values.global.deploymentType) ) (not .Values.global.deploymentType) }}
+{{- if not .Values.global.secretsService }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ template "sda.fullname" . }}-interceptor
+type: Opaque
+data:
+  mqPassword: {{ include "mqPassInterceptor" . | b64enc }}
+  mqUser: {{ include "mqUserInterceptor" . | b64enc }}
+{{- end }}
+{{- if not .Values.global.pkiService }}
+---
+{{- $interceptor := .Files.Glob "files/interceptor*" }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ template "sda.fullname" . }}-interceptor-certs
+data:
+{{ ( .Files.Glob "files/ca.crt" ).AsSecrets | trim | indent 2 }}
+{{ ( $interceptor ).AsSecrets | indent 2 }}
+{{- end }}
+{{- end }}
+{{- end }}

sda-svc/values.yaml

@@ -200,6 +200,10 @@ credentials:
     dbUser: ""
     dbPassword: ""
 
+  interceptor:
+    mqUser: ""
+    mqPassword: ""
+
   verify:
     mqUser: ""
     mqPassword: ""
@@ -284,6 +288,25 @@ ingest:
 # the annotations to apply to the service pods
   annotations: {}
 
+interceptor:
+  deploy: true
+  name: ingest
+  replicaCount: 1
+  repository: neicnordic/sda-pipeline
+  imageTag: latest
+  imagePullPolicy: Always
+  resources:
+    requests:
+      memory: "32Mi"
+      cpu: "100m"
+    limits:
+      memory: "128Mi"
+      cpu: "2000m"
+# Extra annotations to attach to the service pods
+# This should be a multi-line string mapping directly to the a map of
+# the annotations to apply to the service pods
+  annotations: {}
+
 s3Inbox:
   name: s3Inbox
   repository: neicnordic/sda-s3proxy