feat: Integrate listener operator (#784)

* add support for listeners

* add very basic integration test

* add pr number to changelog

* add external-access test

* chore: Add NiFi 2.4.0 and remove 2.2.0 (#797)

add NiFi 2.4.0 and remove 2.2.0

* test: Add test for Apache Iceberg integration (#785)

* Clean up smoke test

* clean up smoke test part 2

* Add working test :)

* Move files

* Add and test HDFS functionality

* Kerbize HDFS and HMS

* Add Kerberos test

* Use nightly image

* linter

* Update Iceberg docs

* changelog

* Small bumps

* Update docs/modules/nifi/pages/usage_guide/writing-to-iceberg-tables.adoc

Co-authored-by: Nick <[email protected]>

---------

Co-authored-by: Nick <[email protected]>

* wip: update listener implementation based on implementation for superset operator

* use single listener with pvc  per rolegroup

* update integration tests with crd change

* restore iceberg test file

* fix listener class in iceberg test

* expose https port in headless service

* update comment

* address feedback from review

* remove unused error variants

* create headless service name in function

* remove unused functions

* move listenerClass to roleConfig

* use new headless service name in integration tests

* move listener constants to listener module

* remove duplicate iceberg test

* remove hard-coded names

* fix integration tests

* remove hardcoded role name

* improve code quality

* set rolegroup label on listener pvcs to none

* fix app version label on listener

* set rolegroup label on listener pvcs

* remove listener class from trino in iceberg test

* add note on custom ListenerClasses to docs

* remove version argument in reporting task

* rename headless service

* fix iceberg test

* create separate headless services

* add missing file

* fix iceberg test

* use listener scope for tls

---------

Co-authored-by: Sebastian Bernauer <[email protected]>
Co-authored-by: Nick <[email protected]>

Author: 3 people

CHANGELOG.md

@@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file.
 ### Added
 
 - Add rolling upgrade support for upgrades between NiFi 2 versions ([#771]).
+- Added Listener support for NiFi ([#784]).
 - Adds new telemetry CLI arguments and environment variables ([#782]).
   - Use `--file-log-max-files` (or `FILE_LOG_MAX_FILES`) to limit the number of log files kept.
   - Use `--file-log-rotation-period` (or `FILE_LOG_ROTATION_PERIOD`) to configure the frequency of rotation.
@@ -52,6 +53,7 @@ All notable changes to this project will be documented in this file.
 [#782]: https://github.com/stackabletech/nifi-operator/pull/782
 [#785]: https://github.com/stackabletech/nifi-operator/pull/785
 [#787]: https://github.com/stackabletech/nifi-operator/pull/787
+[#784]: https://github.com/stackabletech/nifi-operator/pull/784
 [#789]: https://github.com/stackabletech/nifi-operator/pull/789
 [#793]: https://github.com/stackabletech/nifi-operator/pull/793
 [#794]: https://github.com/stackabletech/nifi-operator/pull/794

deploy/helm/nifi-operator/crds/crds.yaml

@@ -194,20 +194,6 @@ spec:
                           description: Allow all proxy hosts by turning off host header validation. See <https://github.com/stackabletech/docker-images/pull/694>
                           type: boolean
                       type: object
-                    listenerClass:
-                      default: cluster-internal
-                      description: |-
-                        This field controls which type of Service the Operator creates for this NifiCluster:
-
-                        * cluster-internal: Use a ClusterIP service
-
-                        * external-unstable: Use a NodePort service
-
-                        This is a temporary solution with the goal to keep yaml manifests forward compatible. In the future, this setting will control which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change.
-                      enum:
-                        - cluster-internal
-                        - external-unstable
-                      type: string
                     sensitiveProperties:
                       description: These settings configure the encryption of sensitive properties in NiFi processors. NiFi supports encrypting sensitive properties in processors as they are written to disk. You can configure the encryption algorithm and the key to use. You can also let the operator generate an encryption key for you.
                       properties:
@@ -790,11 +776,15 @@ spec:
                       x-kubernetes-preserve-unknown-fields: true
                     roleConfig:
                       default:
+                        listenerClass: cluster-internal
                         podDisruptionBudget:
                           enabled: true
                           maxUnavailable: null
                       description: This is a product-agnostic RoleConfig, which is sufficient for most of the products.
                       properties:
+                        listenerClass:
+                          default: cluster-internal
+                          type: string
                         podDisruptionBudget:
                           default:
                             enabled: true

deploy/helm/nifi-operator/templates/roles.yaml

@@ -90,6 +90,17 @@ rules:
     verbs:
       - create
       - patch
+  - apiGroups:
+      - listeners.stackable.tech
+    resources:
+      - listeners
+    verbs:
+      - get
+      - list
+      - watch
+      - patch
+      - create
+      - delete
   - apiGroups:
       - {{ include "operator.name" . }}.stackable.tech
     resources:

docs/modules/nifi/examples/getting_started/getting_started.sh

@@ -143,15 +143,17 @@ spec:
   clusterConfig:
     authentication:
       - authenticationClass: simple-nifi-users
-    listenerClass: external-unstable
     sensitiveProperties:
       keySecret: nifi-sensitive-property-key
       autoGenerate: true
     zookeeperConfigMapName: simple-nifi-znode
   nodes:
+    roleConfig:
+      listenerClass: external-unstable
     roleGroups:
       default:
         replicas: 1
+
 EOF
 # end::install-nifi[]
 

docs/modules/nifi/examples/getting_started/getting_started.sh.j2

@@ -143,12 +143,13 @@ spec:
   clusterConfig:
     authentication:
       - authenticationClass: simple-nifi-users
-    listenerClass: external-unstable
     sensitiveProperties:
       keySecret: nifi-sensitive-property-key
       autoGenerate: true
     zookeeperConfigMapName: simple-nifi-znode
   nodes:
+    roleConfig:
+      listenerClass: external-unstable
     roleGroups:
       default:
         replicas: 1

docs/modules/nifi/pages/usage_guide/custom-components.adoc

@@ -162,12 +162,13 @@ spec:
     - name: nifi-processors
       configMap:
         name: nifi-processors
-    listenerClass: external-unstable
     sensitiveProperties:
       keySecret: nifi-sensitive-property-key
       autoGenerate: true
     zookeeperConfigMapName: simple-nifi-znode
   nodes:
+    roleConfig:
+      listenerClass: external-unstable
     configOverrides:
       nifi.properties:
         nifi.nar.library.directory.myCustomLibs: /stackable/userdata/nifi-processors/  # <2>
@@ -281,12 +282,13 @@ spec:
     - name: nifi-processors
       persistentVolumeClaim:
         claimName: nifi-processors
-    listenerClass: external-unstable
     sensitiveProperties:
       keySecret: nifi-sensitive-property-key
       autoGenerate: true
     zookeeperConfigMapName: simple-nifi-znode
   nodes:
+    roleConfig:
+      listenerClass: external-unstable
     configOverrides:
       nifi.properties:
         nifi.nar.library.directory.myCustomLibs: /stackable/userdata/nifi-processors/  # <2>

docs/modules/nifi/pages/usage_guide/index.adoc

@@ -26,11 +26,12 @@ spec:
       - name: nifi-client-certs
         secret:
           secretName: nifi-client-certs
-    listenerClass: external-unstable
     sensitiveProperties:
       keySecret: nifi-sensitive-property-key
       autoGenerate: true
   nodes:
+    roleConfig:
+      listenerClass: external-unstable
   roleGroups:
     default:
       config:

docs/modules/nifi/pages/usage_guide/listenerclass.adoc

@@ -1,19 +1,14 @@
 = Service exposition with ListenerClasses
 :description: Configure Apache NiFi service exposure with cluster-internal or external-unstable listener classes.
 
-Apache NiFi offers a web UI and an API.
-The Operator deploys a service called `<name>` (where `<name>` is the name of the NifiCluster) through which NiFi can be reached.
-
-This service can have either the `cluster-internal` or `external-unstable` type.
-`external-stable` is not supported for NiFi at the moment.
-Read more about the types in the xref:concepts:service-exposition.adoc[service exposition] documentation at platform level.
-
-This is how the listener class is configured:
+The operator deploys a xref:listener-operator:listener.adoc[Listener] for the Node pod.
+The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.nodes.roleConfig.listenerClass`:
 
 [source,yaml]
 ----
 spec:
-  clusterConfig:
-    listenerClass: cluster-internal  # <1>
+  nodes:
+    roleConfig:
+      listenerClass: external-unstable  # <1>
 ----
-<1> The default `cluster-internal` setting.
+<1> Specify one of `external-stable`, `external-unstable`, `cluster-internal` or a custom ListenerClass (the default setting is `cluster-internal`).

docs/modules/nifi/pages/usage_guide/monitoring.adoc

@@ -127,7 +127,7 @@ spec:
           - __meta_kubernetes_pod_container_port_number
         targetLabel: __address__
         replacement: ${1}.${2}.${3}.svc.cluster.local:${4}
-        regex: (.+);(.+?)(?:-metrics)?;(.+);(.+)
+        regex: (.+);(.+?)(?:-headless)?;(.+);(.+)
   selector:
     matchLabels:
       prometheus.io/scrape: "true"
@@ -138,4 +138,4 @@ spec:
 <1> Authorization via Bearer Token stored in a secret
 <2> Relabel \\__address__ to be a FQDN rather then the IP-Address of target pod
 
-NOTE: As of xref:listener-operator:listener.adoc[Listener] integration, SDP exposes a Service with `-metrics` thus we need to regex this suffix.
+NOTE: As of xref:listener-operator:listener.adoc[Listener] integration, SDP exposes a Service with `-headless` thus we need to regex this suffix.

examples/simple-nifi-cluster.yaml

@@ -51,13 +51,13 @@ spec:
   clusterConfig:
     authentication:
       - authenticationClass: simple-nifi-admin-user
-    listenerClass: external-unstable
     sensitiveProperties:
       keySecret: nifi-sensitive-property-key
       autoGenerate: true
     zookeeperConfigMapName: simple-nifi-znode
   nodes:
-    config:
+    roleConfig:
+      listenerClass: external-unstable
     roleGroups:
       default:
         replicas: 1