From 60eb387e29752255466c416c210f3362b6a735f5 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 25 Apr 2025 14:57:17 +0200 Subject: [PATCH 01/37] add support for listeners --- CHANGELOG.md | 1 + deploy/helm/nifi-operator/crds/crds.yaml | 30 +-- .../helm/nifi-operator/templates/roles.yaml | 6 + .../nifi/pages/usage_guide/listenerclass.adoc | 17 +- rust/operator-binary/src/controller.rs | 199 ++++++------------ rust/operator-binary/src/crd/mod.rs | 192 +++++++++++++---- rust/operator-binary/src/crd/utils.rs | 104 +++++++++ 7 files changed, 352 insertions(+), 197 deletions(-) create mode 100644 rust/operator-binary/src/crd/utils.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a52449c..eccd725e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file. ### Added - Add rolling upgrade support for upgrades between NiFi 2 versions ([#771]). +- Added listener support for Nifi ([#XXX]). ### Changed diff --git a/deploy/helm/nifi-operator/crds/crds.yaml b/deploy/helm/nifi-operator/crds/crds.yaml index 7511f55a..cef05d1d 100644 --- a/deploy/helm/nifi-operator/crds/crds.yaml +++ b/deploy/helm/nifi-operator/crds/crds.yaml @@ -94,20 +94,6 @@ spec: description: Allow all proxy hosts by turning off host header validation. See type: boolean type: object - listenerClass: - default: cluster-internal - description: |- - This field controls which type of Service the Operator creates for this NifiCluster: - - * cluster-internal: Use a ClusterIP service - - * external-unstable: Use a NodePort service - - This is a temporary solution with the goal to keep yaml manifests forward compatible. In the future, this setting will control which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change. - enum: - - cluster-internal - - external-unstable - type: string sensitiveProperties: description: These settings configure the encryption of sensitive properties in NiFi processors. NiFi supports encrypting sensitive properties in processors as they are written to disk. You can configure the encryption algorithm and the key to use. You can also let the operator generate an encryption key for you. properties: @@ -274,6 +260,14 @@ spec: description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + enum: + - cluster-internal + - external-unstable + - external-stable + nullable: true + type: string logging: default: containers: {} @@ -761,6 +755,14 @@ spec: description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + enum: + - cluster-internal + - external-unstable + - external-stable + nullable: true + type: string logging: default: containers: {} diff --git a/deploy/helm/nifi-operator/templates/roles.yaml b/deploy/helm/nifi-operator/templates/roles.yaml index 53b90c67..69b4c9c9 100644 --- a/deploy/helm/nifi-operator/templates/roles.yaml +++ b/deploy/helm/nifi-operator/templates/roles.yaml @@ -90,6 +90,12 @@ rules: verbs: - create - patch + - apiGroups: + - listeners.stackable.tech + resources: + - listeners + verbs: + - get - apiGroups: - {{ include "operator.name" . }}.stackable.tech resources: diff --git a/docs/modules/nifi/pages/usage_guide/listenerclass.adoc b/docs/modules/nifi/pages/usage_guide/listenerclass.adoc index 8ff77c87..55dcbbb1 100644 --- a/docs/modules/nifi/pages/usage_guide/listenerclass.adoc +++ b/docs/modules/nifi/pages/usage_guide/listenerclass.adoc @@ -1,19 +1,14 @@ = Service exposition with ListenerClasses :description: Configure Apache NiFi service exposure with cluster-internal or external-unstable listener classes. -Apache NiFi offers a web UI and an API. -The Operator deploys a service called `` (where `` is the name of the NifiCluster) through which NiFi can be reached. - -This service can have either the `cluster-internal` or `external-unstable` type. -`external-stable` is not supported for NiFi at the moment. -Read more about the types in the xref:concepts:service-exposition.adoc[service exposition] documentation at platform level. - -This is how the listener class is configured: +The operator deploys a xref:listener-operator:listener.adoc[Listener] for the Node pod. +The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.nodes.config.listenerClass`: [source,yaml] ---- spec: - clusterConfig: - listenerClass: cluster-internal # <1> + nodes: + config: + listenerClass: external-unstable # <1> ---- -<1> The default `cluster-internal` setting. +<1> Specify one of `external-stable`, `external-unstable`, `cluster-internal` (the default setting is `cluster-internal`). diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index cca71af6..4d2baa79 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -20,8 +20,11 @@ use stackable_operator::{ configmap::ConfigMapBuilder, meta::ObjectMetaBuilder, pod::{ - PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, - security::PodSecurityContextBuilder, volume::SecretFormat, + PodBuilder, + container::ContainerBuilder, + resources::ResourceRequirementsBuilder, + security::PodSecurityContextBuilder, + volume::{ListenerOperatorVolumeSourceBuilderError, SecretFormat}, }, }, client::Client, @@ -37,15 +40,14 @@ use stackable_operator::{ apps::v1::{StatefulSet, StatefulSetSpec, StatefulSetUpdateStrategy}, core::v1::{ ConfigMap, ConfigMapKeySelector, ConfigMapVolumeSource, EmptyDirVolumeSource, - EnvVar, EnvVarSource, Node, ObjectFieldSelector, Probe, SecretVolumeSource, - Service, ServicePort, ServiceSpec, TCPSocketAction, Volume, + EnvVar, EnvVarSource, ObjectFieldSelector, Probe, SecretVolumeSource, Service, + ServicePort, ServiceSpec, TCPSocketAction, Volume, }, }, apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, }, kube::{ Resource, ResourceExt, - api::ListParams, core::{DeserializeGuard, error_boundary}, runtime::{controller::Action, reflector::ObjectRef}, }, @@ -81,11 +83,11 @@ use crate::{ build_nifi_properties, build_state_management_xml, validated_product_config, }, crd::{ - APP_NAME, BALANCE_PORT, BALANCE_PORT_NAME, Container, CurrentlySupportedListenerClasses, - HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT, METRICS_PORT_NAME, NifiConfig, + APP_NAME, BALANCE_PORT, BALANCE_PORT_NAME, Container, HTTPS_PORT, HTTPS_PORT_NAME, + LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, METRICS_PORT, METRICS_PORT_NAME, NifiConfig, NifiConfigFragment, NifiRole, NifiStatus, PROTOCOL_PORT, PROTOCOL_PORT_NAME, - STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, authentication::AuthenticationClassResolved, - v1alpha1, + STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, SupportedListenerClasses, + authentication::AuthenticationClassResolved, v1alpha1, }, operations::{ graceful_shutdown::add_graceful_shutdown_config, @@ -353,6 +355,11 @@ pub enum Error { #[snafu(display("Failed to determine the state of the version upgrade procedure"))] ClusterVersionUpdateState { source: upgrade::Error }, + + #[snafu(display("failed to build listener volume"))] + BuildListenerVolume { + source: ListenerOperatorVolumeSourceBuilderError, + }, } type Result = std::result::Result; @@ -375,11 +382,6 @@ pub async fn reconcile_nifi( .context(InvalidNifiClusterSnafu)?; let client = &ctx.client; - let namespace = &nifi - .metadata - .namespace - .clone() - .with_context(|| ObjectHasNoNamespaceSnafu {})?; let resolved_product_image: ResolvedProductImage = nifi .spec @@ -441,20 +443,6 @@ pub async fn reconcile_nifi( .map(Cow::Borrowed) .unwrap_or_default(); - let node_role_service = build_node_role_service(nifi, &resolved_product_image)?; - cluster_resources - .add(client, node_role_service) - .await - .context(ApplyRoleServiceSnafu)?; - - // This is read back to obtain the hosts that we later need to fill in the proxy_hosts variable - let updated_role_service = client - .get::(&nifi.name_any(), namespace) - .await - .with_context(|_| MissingServiceSnafu { - obj_ref: ObjectRef::new(&nifi.name_any()).within(namespace), - })?; - let nifi_authentication_config = NifiAuthenticationConfig::try_from( AuthenticationClassResolved::from(nifi, client) .await @@ -511,7 +499,7 @@ pub async fn reconcile_nifi( // Since we cannot predict which of the addresses a user might decide to use we will simply // add all of them to the setting for now. // For more information see - let proxy_hosts = get_proxy_hosts(client, nifi, &updated_role_service).await?; + let proxy_hosts = get_proxy_hosts(client, nifi, &merged_config).await?; let rg_configmap = build_node_rolegroup_config_map( nifi, @@ -648,52 +636,6 @@ pub async fn reconcile_nifi( Ok(Action::await_change()) } -/// The node-role service is the primary endpoint that should be used by clients that do not -/// perform internal load balancing including targets outside of the cluster. -pub fn build_node_role_service( - nifi: &v1alpha1::NifiCluster, - resolved_product_image: &ResolvedProductImage, -) -> Result { - let role_name = NifiRole::Node.to_string(); - - let role_svc_name = nifi.node_role_service_name(); - Ok(Service { - metadata: ObjectMetaBuilder::new() - .name_and_namespace(nifi) - .name(&role_svc_name) - .ownerreference_from_resource(nifi, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(build_recommended_labels( - nifi, - &resolved_product_image.app_version_label, - &role_name, - "global", - )) - .context(MetadataBuildSnafu)? - .build(), - spec: Some(ServiceSpec { - type_: Some(nifi.spec.cluster_config.listener_class.k8s_service_type()), - ports: Some(vec![ServicePort { - name: Some(HTTPS_PORT_NAME.to_string()), - port: HTTPS_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }]), - selector: Some( - Labels::role_selector(nifi, APP_NAME, &role_name) - .context(LabelBuildSnafu)? - .into(), - ), - external_traffic_policy: match nifi.spec.cluster_config.listener_class { - CurrentlySupportedListenerClasses::ClusterInternal => None, - CurrentlySupportedListenerClasses::ExternalUnstable => Some("Local".to_string()), - }, - ..ServiceSpec::default() - }), - status: None, - }) -} - /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator #[allow(clippy::too_many_arguments)] async fn build_node_rolegroup_config_map( @@ -870,7 +812,7 @@ async fn build_node_rolegroup_statefulset( role: &Role, rolegroup_config: &HashMap>, merged_config: &NifiConfig, - nifi_auth_config: &NifiAuthenticationConfig, + nifi_authentication_config: &NifiAuthenticationConfig, rolling_update_supported: bool, replicas: Option, sa_name: &str, @@ -922,7 +864,7 @@ async fn build_node_rolegroup_statefulset( &nifi.spec.cluster_config.zookeeper_config_map_name, )); - if let NifiAuthenticationConfig::Oidc { oidc, .. } = nifi_auth_config { + if let NifiAuthenticationConfig::Oidc { oidc, .. } = nifi_authentication_config { env_vars.extend(AuthenticationProvider::client_credentials_env_var_mounts( oidc.client_credentials_secret_ref.clone(), )); @@ -973,7 +915,22 @@ async fn build_node_rolegroup_statefulset( ]); // This commands needs to go first, as they might set env variables needed by the templating - prepare_args.extend_from_slice(nifi_auth_config.get_additional_container_args().as_slice()); + prepare_args.extend_from_slice( + nifi_authentication_config + .get_additional_container_args() + .as_slice(), + ); + + if merged_config.listener_class == SupportedListenerClasses::ExternalUnstable { + prepare_args.extend(vec![ + "export LISTENER_DEFAULT_ADDRESS=$(cat /stackable/listener/default-address/address)" + .to_string(), + ]); + prepare_args.extend(vec![ + "export LISTENER_DEFAULT_PORT_HTTPS=$(cat /stackable/listener/default-address/ports/https)" + .to_string(), + ]); + } prepare_args.extend(vec![ "echo Templating config files".to_string(), @@ -1038,6 +995,8 @@ async fn build_node_rolegroup_statefulset( .context(AddVolumeMountSnafu)? .add_volume_mount(TRUSTSTORE_VOLUME_NAME, STACKABLE_SERVER_TLS_DIR) .context(AddVolumeMountSnafu)? + .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) + .context(AddVolumeMountSnafu)? .resources( ResourceRequirementsBuilder::new() .with_cpu_request("500m") @@ -1114,6 +1073,8 @@ async fn build_node_rolegroup_statefulset( .context(AddVolumeMountSnafu)? .add_volume_mount(TRUSTSTORE_VOLUME_NAME, STACKABLE_SERVER_TLS_DIR) .context(AddVolumeMountSnafu)? + .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) + .context(AddVolumeMountSnafu)? .add_container_port(HTTPS_PORT_NAME, HTTPS_PORT.into()) .add_container_port(PROTOCOL_PORT_NAME, PROTOCOL_PORT.into()) .add_container_port(BALANCE_PORT_NAME, BALANCE_PORT.into()) @@ -1140,6 +1101,28 @@ async fn build_node_rolegroup_statefulset( .resources(merged_config.resources.clone().into()); let mut pod_builder = PodBuilder::new(); + + let recommended_object_labels = build_recommended_labels( + nifi, + &resolved_product_image.app_version_label, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ); + let recommended_labels = + Labels::recommended(recommended_object_labels.clone()).context(LabelBuildSnafu)?; + + let listener_class = &merged_config.listener_class; + // all listeners will use ephemeral volumes as they can/should + // be removed when the pods are *terminated* (ephemeral PVCs will + // survive re-starts) + pod_builder + .add_listener_volume_by_listener_class( + LISTENER_VOLUME_NAME, + &listener_class.to_string(), + &recommended_labels, + ) + .context(AddVolumeSnafu)?; + add_graceful_shutdown_config(merged_config, &mut pod_builder).context(GracefulShutdownSnafu)?; // Add user configured extra volumes if any are specified @@ -1222,7 +1205,7 @@ async fn build_node_rolegroup_statefulset( } } - nifi_auth_config + nifi_authentication_config .add_volumes_and_mounts(&mut pod_builder, vec![ &mut container_prepare, container_nifi, @@ -1346,12 +1329,7 @@ async fn build_node_rolegroup_statefulset( .name(rolegroup_ref.object_name()) .ownerreference_from_resource(nifi, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(build_recommended_labels( - nifi, - &resolved_product_image.app_version_label, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - )) + .with_recommended_labels(recommended_object_labels) .context(MetadataBuildSnafu)? .build(), spec: Some(StatefulSetSpec { @@ -1408,25 +1386,6 @@ async fn build_node_rolegroup_statefulset( }) } -fn external_node_port(nifi_service: &Service) -> Result { - let external_ports = nifi_service - .spec - .as_ref() - .with_context(|| ObjectHasNoSpecSnafu {})? - .ports - .as_ref() - .with_context(|| ExternalPortSnafu {})? - .iter() - .filter(|p| p.name == Some(HTTPS_PORT_NAME.to_string())) - .collect::>(); - - let port = external_ports - .first() - .with_context(|| ExternalPortSnafu {})?; - - port.node_port.with_context(|| ExternalPortSnafu {}) -} - /// Used for the `ZOOKEEPER_HOSTS` and `ZOOKEEPER_CHROOT` env vars. fn zookeeper_env_var(name: &str, configmap_name: &str) -> EnvVar { EnvVar { @@ -1446,7 +1405,7 @@ fn zookeeper_env_var(name: &str, configmap_name: &str) -> EnvVar { async fn get_proxy_hosts( client: &Client, nifi: &v1alpha1::NifiCluster, - nifi_service: &Service, + merged_config: &NifiConfig, ) -> Result { let host_header_check = nifi.spec.cluster_config.host_header_check.clone(); @@ -1478,32 +1437,10 @@ async fn get_proxy_hosts( proxy_hosts_set.extend(host_header_check.additional_allowed_hosts); - // In case NodePort is used add them as well - if nifi.spec.cluster_config.listener_class - == CurrentlySupportedListenerClasses::ExternalUnstable - { - let external_port = external_node_port(nifi_service)?; - - let cluster_nodes = client - .list::(&(), &ListParams::default()) - .await - .with_context(|_| MissingNodesSnafu { - obj_ref: ObjectRef::from_obj(nifi), - })?; - - // We need the addresses of all nodes to add these to the NiFi proxy setting - // Since there is no real convention about how to label these addresses we will simply - // take all published addresses for now to be on the safe side. - proxy_hosts_set.extend( - cluster_nodes - .into_iter() - .flat_map(|node| { - node.status - .unwrap_or_default() - .addresses - .unwrap_or_default() - }) - .map(|node_address| format!("{}:{external_port}", node_address.address)), + // If NodePort is used inject the address and port from the listener volume in the prepare container + if merged_config.listener_class == SupportedListenerClasses::ExternalUnstable { + proxy_hosts_set.insert( + "${env:LISTENER_DEFAULT_ADDRESS}:${env:LISTENER_DEFAULT_PORT_HTTPS}".to_string(), ); } diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index c3227e77..270b5902 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,8 +1,9 @@ pub mod affinity; pub mod authentication; pub mod tls; +pub mod utils; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use affinity::get_affinity; use serde::{Deserialize, Serialize}; @@ -20,7 +21,7 @@ use stackable_operator::{ }, config::{ fragment::{self, Fragment, ValidationError}, - merge::Merge, + merge::{Atomic, Merge}, }, k8s_openapi::{ api::core::v1::{PodTemplateSpec, Volume}, @@ -42,6 +43,7 @@ use stackable_operator::{ }; use strum::Display; use tls::NifiTls; +use utils::{PodRef, get_listener_podrefs}; pub const APP_NAME: &str = "nifi"; @@ -54,6 +56,8 @@ pub const BALANCE_PORT: u16 = 6243; pub const METRICS_PORT_NAME: &str = "metrics"; pub const METRICS_PORT: u16 = 8081; +pub const LISTENER_VOLUME_NAME: &str = "listener"; +pub const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; pub const STACKABLE_LOG_CONFIG_DIR: &str = "/stackable/log_config"; @@ -74,6 +78,12 @@ pub enum Error { #[snafu(display("fragment validation failure"))] FragmentValidationFailure { source: ValidationError }, + + #[snafu(display("object has no nodes defined"))] + NoNodesDefined, + + #[snafu(display("listener podrefs could not be resolved"))] + ListenerPodRef { source: utils::Error }, } #[versioned(version(name = "v1alpha1"))] @@ -152,18 +162,6 @@ pub mod versioned { #[schemars(schema_with = "raw_object_list_schema")] pub extra_volumes: Vec, - /// This field controls which type of Service the Operator creates for this NifiCluster: - /// - /// * cluster-internal: Use a ClusterIP service - /// - /// * external-unstable: Use a NodePort service - /// - /// This is a temporary solution with the goal to keep yaml manifests forward compatible. - /// In the future, this setting will control which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) - /// will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change. - #[serde(default)] - pub listener_class: CurrentlySupportedListenerClasses, - // Docs are on the struct #[serde(default)] pub create_reporting_task_job: CreateReportingTaskJob, @@ -236,6 +234,11 @@ impl v1alpha1::NifiCluster { namespace: ns.clone(), role_group_service_name: rolegroup_ref.object_name(), pod_name: format!("{}-{}", rolegroup_ref.object_name(), i), + ports: HashMap::from([ + (HTTPS_PORT_NAME.to_owned(), HTTPS_PORT), + (METRICS_PORT_NAME.to_owned(), METRICS_PORT), + ]), + fqdn_override: None, }) })) } @@ -270,6 +273,100 @@ impl v1alpha1::NifiCluster { tracing::debug!("Merged config: {:?}", conf_rolegroup); fragment::validate(conf_rolegroup).context(FragmentValidationFailureSnafu) } + + pub fn merged_listener_class( + &self, + rolegroup_name: &String, + ) -> Result, Error> { + let listener_class_default = Some(SupportedListenerClasses::ClusterInternal); + let role = self.spec.nodes.as_ref().context(NoNodesDefinedSnafu)?; + + let mut listener_class_role = role.config.config.listener_class.to_owned(); + let mut listener_class_rolegroup = role + .role_groups + .get(rolegroup_name) + .map(|rg| rg.config.config.listener_class.clone()) + .unwrap_or_default(); + listener_class_role.merge(&listener_class_default); + listener_class_rolegroup.merge(&listener_class_role); + tracing::debug!("Merged listener-class: {:?}", listener_class_rolegroup); + Ok(listener_class_rolegroup) + } + + pub fn rolegroup_ref( + &self, + role_name: impl Into, + group_name: impl Into, + ) -> RoleGroupRef { + RoleGroupRef { + cluster: ObjectRef::from_obj(self), + role: role_name.into(), + role_group: group_name.into(), + } + } + + pub fn rolegroup_ref_and_replicas(&self) -> Vec<(RoleGroupRef, u16)> { + self.spec + .nodes + .iter() + .flat_map(|role| &role.role_groups) + // Order rolegroups consistently, to avoid spurious downstream rewrites + .collect::>() + .into_iter() + .filter(|(rolegroup_name, _)| self.resolved_listener_class_discoverable(rolegroup_name)) + .map(|(rolegroup_name, role_group)| { + ( + self.rolegroup_ref(NifiRole::Node.to_string(), rolegroup_name), + role_group.replicas.unwrap_or_default(), + ) + }) + .collect() + } + + fn resolved_listener_class_discoverable(&self, rolegroup_name: &&String) -> bool { + if let Ok(Some(listener_class)) = self.merged_listener_class(rolegroup_name) { + listener_class.discoverable() + } else { + // merged_listener_class returns an error if one of the roles was not found: + // all roles are mandatory for airflow to work, but a missing role will by + // definition not have a listener class + false + } + } + + pub fn pod_refs(&self) -> Result, Error> { + let ns = self.metadata.namespace.clone().context(NoNamespaceSnafu)?; + let rolegroup_ref_and_replicas = self.rolegroup_ref_and_replicas(); + + Ok(rolegroup_ref_and_replicas + .iter() + .flat_map(|(rolegroup_ref, replicas)| { + let ns = ns.clone(); + (0..*replicas).map(move |i| PodRef { + namespace: ns.clone(), + role_group_service_name: rolegroup_ref.object_name(), + pod_name: format!("{}-{}", rolegroup_ref.object_name(), i), + ports: HashMap::from([ + (HTTPS_PORT_NAME.to_owned(), HTTPS_PORT), + (METRICS_PORT_NAME.to_owned(), METRICS_PORT), + ]), + fqdn_override: None, + }) + }) + .collect()) + } + + pub async fn listener_refs( + &self, + client: &stackable_operator::client::Client, + ) -> Result, Error> { + let pod_refs = self.pod_refs()?; + + tracing::debug!("Pod references: {:#?}", pod_refs); + get_listener_podrefs(client, pod_refs, LISTENER_VOLUME_NAME) + .await + .context(ListenerPodRefSnafu) + } } #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] @@ -297,22 +394,51 @@ pub fn default_allow_all() -> bool { true } -// TODO: Temporary solution until listener-operator is finished +// // TODO: Temporary solution until listener-operator is finished +// #[derive(Clone, Debug, Default, Display, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +// #[serde(rename_all = "PascalCase")] +// pub enum CurrentlySupportedListenerClasses { +// #[default] +// #[serde(rename = "cluster-internal")] +// ClusterInternal, +// #[serde(rename = "external-unstable")] +// ExternalUnstable, +// } + +// impl CurrentlySupportedListenerClasses { +// pub fn k8s_service_type(&self) -> String { +// match self { +// CurrentlySupportedListenerClasses::ClusterInternal => "ClusterIP".to_string(), +// CurrentlySupportedListenerClasses::ExternalUnstable => "NodePort".to_string(), +// } +// } +// } + #[derive(Clone, Debug, Default, Display, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "PascalCase")] -pub enum CurrentlySupportedListenerClasses { +pub enum SupportedListenerClasses { #[default] #[serde(rename = "cluster-internal")] + #[strum(serialize = "cluster-internal")] ClusterInternal, + #[serde(rename = "external-unstable")] + #[strum(serialize = "external-unstable")] ExternalUnstable, + + #[serde(rename = "external-stable")] + #[strum(serialize = "external-stable")] + ExternalStable, } -impl CurrentlySupportedListenerClasses { - pub fn k8s_service_type(&self) -> String { +impl Atomic for SupportedListenerClasses {} + +impl SupportedListenerClasses { + pub fn discoverable(&self) -> bool { match self { - CurrentlySupportedListenerClasses::ClusterInternal => "ClusterIP".to_string(), - CurrentlySupportedListenerClasses::ExternalUnstable => "NodePort".to_string(), + SupportedListenerClasses::ClusterInternal => false, + SupportedListenerClasses::ExternalUnstable => true, + SupportedListenerClasses::ExternalStable => true, } } } @@ -504,6 +630,10 @@ pub struct NifiConfig { /// Please note that this can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. #[fragment_attrs(serde(default))] pub requested_secret_lifetime: Option, + + /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose the webserver. + #[serde(default)] + pub listener_class: SupportedListenerClasses, } impl NifiConfig { @@ -553,6 +683,7 @@ impl NifiConfig { affinity: get_affinity(cluster_name, role), graceful_shutdown_timeout: Some(DEFAULT_NODE_GRACEFUL_SHUTDOWN_TIMEOUT), requested_secret_lifetime: Some(Self::DEFAULT_NODE_SECRET_LIFETIME), + listener_class: Some(SupportedListenerClasses::ClusterInternal), } } } @@ -632,24 +763,3 @@ pub struct NifiStorageConfig { #[fragment_attrs(serde(default))] pub state_repo: PvcConfig, } - -/// Reference to a single `Pod` that is a component of a [`NifiCluster`] -/// Used for service discovery. -// TODO: this should move to operator-rs -pub struct PodRef { - pub namespace: String, - pub role_group_service_name: String, - pub pod_name: String, -} - -impl PodRef { - pub fn fqdn(&self, cluster_info: &KubernetesClusterInfo) -> String { - format!( - "{pod_name}.{service_name}.{namespace}.svc.{cluster_domain}", - pod_name = self.pod_name, - service_name = self.role_group_service_name, - namespace = self.namespace, - cluster_domain = cluster_info.cluster_domain - ) - } -} diff --git a/rust/operator-binary/src/crd/utils.rs b/rust/operator-binary/src/crd/utils.rs new file mode 100644 index 00000000..a54cd0bf --- /dev/null +++ b/rust/operator-binary/src/crd/utils.rs @@ -0,0 +1,104 @@ +use std::{borrow::Cow, collections::HashMap, num::TryFromIntError}; + +use futures::future::try_join_all; +use snafu::{OptionExt, ResultExt, Snafu}; +use stackable_operator::{ + commons::listener::Listener, k8s_openapi::api::core::v1::Pod, + kube::runtime::reflector::ObjectRef, utils::cluster_info::KubernetesClusterInfo, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("unable to get {listener} (for {pod})"))] + GetPodListener { + source: stackable_operator::client::Error, + listener: ObjectRef, + pod: ObjectRef, + }, + + #[snafu(display("{listener} (for {pod}) has no address"))] + PodListenerHasNoAddress { + listener: ObjectRef, + pod: ObjectRef, + }, + + #[snafu(display("port {port} ({port_name:?}) is out of bounds, must be within {range:?}", range = 0..=u16::MAX))] + PortOutOfBounds { + source: TryFromIntError, + port_name: String, + port: i32, + }, +} + +/// Reference to a single `Pod` that is a component of the product cluster +/// +/// Used for service discovery. +#[derive(Debug)] +pub struct PodRef { + pub namespace: String, + pub role_group_service_name: String, + pub pod_name: String, + pub fqdn_override: Option, + pub ports: HashMap, +} + +impl PodRef { + pub fn fqdn(&self, cluster_info: &KubernetesClusterInfo) -> Cow { + self.fqdn_override.as_deref().map_or_else( + || { + Cow::Owned(format!( + "{pod_name}.{role_group_service_name}.{namespace}.svc.{cluster_domain}", + pod_name = self.pod_name, + role_group_service_name = self.role_group_service_name, + namespace = self.namespace, + cluster_domain = cluster_info.cluster_domain, + )) + }, + Cow::Borrowed, + ) + } +} + +pub async fn get_listener_podrefs( + client: &stackable_operator::client::Client, + pod_refs: Vec, + listener_prefix: &str, +) -> Result, Error> { + try_join_all(pod_refs.into_iter().map(|pod_ref| async { + // N.B. use the naming convention for ephemeral listener volumes as we + // have defined all listeners to be so. + let listener_name = format!("{}-{listener_prefix}", pod_ref.pod_name); + let listener_ref = || ObjectRef::::new(&listener_name).within(&pod_ref.namespace); + let pod_obj_ref = || ObjectRef::::new(&pod_ref.pod_name).within(&pod_ref.namespace); + let listener = client + .get::(&listener_name, &pod_ref.namespace) + .await + .context(GetPodListenerSnafu { + listener: listener_ref(), + pod: pod_obj_ref(), + })?; + let listener_address = listener + .status + .and_then(|s| s.ingress_addresses?.into_iter().next()) + .context(PodListenerHasNoAddressSnafu { + listener: listener_ref(), + pod: pod_obj_ref(), + })?; + Ok(PodRef { + fqdn_override: Some(listener_address.address), + ports: listener_address + .ports + .into_iter() + .map(|(port_name, port)| { + let port = u16::try_from(port).context(PortOutOfBoundsSnafu { + port_name: &port_name, + port, + })?; + Ok((port_name, port)) + }) + .collect::>()?, + ..pod_ref + }) + })) + .await +} From 93120b305c3f409660e05e607b4e896251622348 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 25 Apr 2025 20:21:01 +0200 Subject: [PATCH 02/37] add very basic integration test --- .../kuttl/external-access/00-patch-ns.yaml.j2 | 9 ++ .../kuttl/external-access/00-range-limit.yaml | 11 +++ .../kuttl/external-access/00-rbac.yaml.j2 | 29 ++++++ .../kuttl/external-access/10-assert.yaml.j2 | 10 ++ ...tor-aggregator-discovery-configmap.yaml.j2 | 9 ++ .../kuttl/external-access/20-assert.yaml | 12 +++ .../external-access/20-install-zk.yaml.j2 | 29 ++++++ .../kuttl/external-access/30-assert.yaml | 25 +++++ .../kuttl/external-access/30-assert.yaml.j2 | 10 ++ .../external-access/30-install-nifi.yaml.j2 | 96 +++++++++++++++++++ 10 files changed, 240 insertions(+) create mode 100644 tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 create mode 100644 tests/templates/kuttl/external-access/00-range-limit.yaml create mode 100644 tests/templates/kuttl/external-access/00-rbac.yaml.j2 create mode 100644 tests/templates/kuttl/external-access/10-assert.yaml.j2 create mode 100644 tests/templates/kuttl/external-access/10-install-vector-aggregator-discovery-configmap.yaml.j2 create mode 100644 tests/templates/kuttl/external-access/20-assert.yaml create mode 100644 tests/templates/kuttl/external-access/20-install-zk.yaml.j2 create mode 100644 tests/templates/kuttl/external-access/30-assert.yaml create mode 100644 tests/templates/kuttl/external-access/30-assert.yaml.j2 create mode 100644 tests/templates/kuttl/external-access/30-install-nifi.yaml.j2 diff --git a/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 b/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/external-access/00-range-limit.yaml b/tests/templates/kuttl/external-access/00-range-limit.yaml new file mode 100644 index 00000000..8fd02210 --- /dev/null +++ b/tests/templates/kuttl/external-access/00-range-limit.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: LimitRange +metadata: + name: limit-request-ratio +spec: + limits: + - type: "Container" + maxLimitRequestRatio: + cpu: 5 + memory: 1 diff --git a/tests/templates/kuttl/external-access/00-rbac.yaml.j2 b/tests/templates/kuttl/external-access/00-rbac.yaml.j2 new file mode 100644 index 00000000..7ee61d23 --- /dev/null +++ b/tests/templates/kuttl/external-access/00-rbac.yaml.j2 @@ -0,0 +1,29 @@ +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-role +rules: +{% if test_scenario['values']['openshift'] == "true" %} + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["privileged"] + verbs: ["use"] +{% endif %} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-sa +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-rb +subjects: + - kind: ServiceAccount + name: test-sa +roleRef: + kind: Role + name: test-role + apiGroup: rbac.authorization.k8s.io diff --git a/tests/templates/kuttl/external-access/10-assert.yaml.j2 b/tests/templates/kuttl/external-access/10-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/external-access/10-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/external-access/10-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/external-access/10-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/external-access/10-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/external-access/20-assert.yaml b/tests/templates/kuttl/external-access/20-assert.yaml new file mode 100644 index 00000000..e0766c49 --- /dev/null +++ b/tests/templates/kuttl/external-access/20-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-zk-server-default +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/external-access/20-install-zk.yaml.j2 b/tests/templates/kuttl/external-access/20-install-zk.yaml.j2 new file mode 100644 index 00000000..2f1558cc --- /dev/null +++ b/tests/templates/kuttl/external-access/20-install-zk.yaml.j2 @@ -0,0 +1,29 @@ +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperCluster +metadata: + name: test-zk +spec: + image: + productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" + pullPolicy: IfNotPresent + clusterConfig: + listenerClass: {{ test_scenario['values']['listener-class'] }} +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + servers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperZnode +metadata: + name: test-nifi-znode +spec: + clusterRef: + name: test-zk diff --git a/tests/templates/kuttl/external-access/30-assert.yaml b/tests/templates/kuttl/external-access/30-assert.yaml new file mode 100644 index 00000000..32f2b7cc --- /dev/null +++ b/tests/templates/kuttl/external-access/30-assert.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 1200 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-nifi-node-default +spec: + template: + spec: + terminationGracePeriodSeconds: 300 +status: + readyReplicas: 2 + replicas: 2 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: test-nifi-node +status: + expectedPods: 4 + currentHealthy: 4 + disruptionsAllowed: 1 diff --git a/tests/templates/kuttl/external-access/30-assert.yaml.j2 b/tests/templates/kuttl/external-access/30-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/external-access/30-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/external-access/30-install-nifi.yaml.j2 b/tests/templates/kuttl/external-access/30-install-nifi.yaml.j2 new file mode 100644 index 00000000..5838a12f --- /dev/null +++ b/tests/templates/kuttl/external-access/30-install-nifi.yaml.j2 @@ -0,0 +1,96 @@ +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: simple-nifi-users +spec: + provider: + static: + userCredentialsSecret: + name: simple-nifi-admin-credentials +--- +apiVersion: v1 +kind: Secret +metadata: + name: simple-nifi-admin-credentials +stringData: + admin: > + passwordWithSpecialCharacter\@<&>"' +--- +apiVersion: v1 +kind: Secret +metadata: + name: nifi-sensitive-property-key +stringData: + nifiSensitivePropsKey: mYsUp3rS3cr3tk3y +--- +apiVersion: nifi.stackable.tech/v1alpha1 +kind: NifiCluster +metadata: + name: test-nifi +spec: + image: +{% if test_scenario['values']['nifi'].find(",") > 0 %} + custom: "{{ test_scenario['values']['nifi'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['nifi'].split(',')[0] }}" +{% else %} + custom: null + productVersion: "{{ test_scenario['values']['nifi'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + zookeeperConfigMapName: test-nifi-znode + listenerClass: {{ test_scenario['values']['listener-class'] }} + authentication: + - authenticationClass: simple-nifi-users + hostHeaderCheck: + allowAll: false + sensitiveProperties: + keySecret: nifi-sensitive-property-key +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + nodes: + envOverrides: + COMMON_VAR: role-value # overridden by role group below + ROLE_VAR: role-value # only defined here at role level + configOverrides: + "nifi.properties": + "nifi.diagnostics.on.shutdown.enabled": "true" + "nifi.diagnostics.on.shutdown.verbose": "false" + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 2 + envOverrides: + COMMON_VAR: group-value # overrides role value + GROUP_VAR: group-value # only defined here at group level + configOverrides: + "nifi.properties": + "nifi.diagnostics.on.shutdown.enabled": "false" + "nifi.diagnostics.on.shutdown.max.filecount": "20" + roleGroups: + external-unstable: + replicas: 1 + config: + listenerClass: external-unstable + envOverrides: + COMMON_VAR: group-value # overrides role value + GROUP_VAR: group-value # only defined here at group level + configOverrides: + "nifi.properties": + "nifi.diagnostics.on.shutdown.enabled": "false" + "nifi.diagnostics.on.shutdown.max.filecount": "20" + cluster-internal: + replicas: 1 + config: + listenerClass: cluster-internal + envOverrides: + COMMON_VAR: group-value # overrides role value + GROUP_VAR: group-value # only defined here at group level + configOverrides: + "nifi.properties": + "nifi.diagnostics.on.shutdown.enabled": "false" + "nifi.diagnostics.on.shutdown.max.filecount": "20" From eb5136a82bf98194db401c642214662626e6a46c Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 16 May 2025 16:42:42 +0200 Subject: [PATCH 03/37] add pr number to changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dfb501b..ea28a017 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ All notable changes to this project will be documented in this file. ### Added - Add rolling upgrade support for upgrades between NiFi 2 versions ([#771]). -- Added listener support for Nifi ([#XXX]). +- Added listener support for Nifi ([#784]). - Adds new telemetry CLI arguments and environment variables ([#782]). - Use `--file-log-max-files` (or `FILE_LOG_MAX_FILES`) to limit the number of log files kept. - Use `--file-log-rotation-period` (or `FILE_LOG_ROTATION_PERIOD`) to configure the frequency of rotation. @@ -41,6 +41,7 @@ All notable changes to this project will be documented in this file. [#776]: https://github.com/stackabletech/nifi-operator/pull/776 [#782]: https://github.com/stackabletech/nifi-operator/pull/782 [#787]: https://github.com/stackabletech/nifi-operator/pull/787 +[#784]: https://github.com/stackabletech/nifi-operator/pull/784 [#789]: https://github.com/stackabletech/nifi-operator/pull/789 ## [25.3.0] - 2025-03-21 From c1aede3ebd7c8ee5823587fcaa07ee23a4cfe2d8 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 16 May 2025 17:19:17 +0200 Subject: [PATCH 04/37] add external-access test --- tests/test-definition.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index d83e0751..b253b67e 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -90,6 +90,11 @@ tests: - zookeeper-latest - oidc-use-tls - openshift + - name: external-access + dimensions: + - nifi + - zookeeper-latest + - openshift suites: - name: nightly patch: From 46b3c14b90f050cc2643492006dd49d06bfc5e2f Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 20 May 2025 11:19:36 +0200 Subject: [PATCH 05/37] chore: Add NiFi 2.4.0 and remove 2.2.0 (#797) add NiFi 2.4.0 and remove 2.2.0 --- docs/modules/nifi/pages/usage_guide/updating.adoc | 2 +- docs/modules/nifi/partials/supported-versions.adoc | 2 +- tests/test-definition.yaml | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/modules/nifi/pages/usage_guide/updating.adoc b/docs/modules/nifi/pages/usage_guide/updating.adoc index d1bbde6d..44209e1a 100644 --- a/docs/modules/nifi/pages/usage_guide/updating.adoc +++ b/docs/modules/nifi/pages/usage_guide/updating.adoc @@ -24,7 +24,7 @@ NiFi clusters cannot be upgraded or downgraded in a rolling fashion due to a lim When upgrading between NiFi 1 versions or from NiFi 1 to NiFi 2, any change to the NiFi version in the CRD triggers a full cluster restart with brief downtime. However, the Stackable image version can be updated in a rolling manner, provided the NiFi version remains unchanged. -For upgrades between NiFi 2 versions, e.g. from `2.0.0` to `2.2.0`, rolling upgrades are supported. +For upgrades between NiFi 2 versions, e.g. from `2.0.0` to `2.4.0`, rolling upgrades are supported. ==== == NiFi 2.0.0 diff --git a/docs/modules/nifi/partials/supported-versions.adoc b/docs/modules/nifi/partials/supported-versions.adoc index c0e5ea16..c5bad944 100644 --- a/docs/modules/nifi/partials/supported-versions.adoc +++ b/docs/modules/nifi/partials/supported-versions.adoc @@ -2,7 +2,7 @@ // This is a separate file, since it is used by both the direct NiFi-Operator documentation, and the overarching // Stackable Platform documentation. -* 2.2.0 (experimental) - Please note that you need to upgrade to at least 1.27.x before upgrading to 2.x.x! +* 2.4.0 (experimental) - Please note that you need to upgrade to at least 1.27.x before upgrading to 2.x.x! * 1.28.1 * 1.27.0 (LTS) diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index b253b67e..edc5b103 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -4,25 +4,25 @@ dimensions: values: - 1.27.0 - 1.28.1 - - 2.2.0 + - 2.4.0 # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version # as in the example below. - # - 2.2.0,oci.stackable.tech/sandbox/nifi:2.2.0-stackable0.0.0-dev + # - 2.4.0,oci.stackable.tech/sandbox/nifi:2.4.0-stackable0.0.0-dev - name: nifi_old values: - 1.27.0 - name: nifi_new values: - - 2.2.0 + - 2.4.0 # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version # as in the example below. - # - 2.2.0,oci.stackable.tech/sandbox/nifi:2.2.0-stackable0.0.0-dev + # - 2.4.0,oci.stackable.tech/sandbox/nifi:2.4.0-stackable0.0.0-dev - name: nifi-latest values: - - 2.2.0 + - 2.4.0 # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version # as in the example below. - # - 2.2.0,oci.stackable.tech/sandbox/nifi:2.2.0-stackable0.0.0-dev + # - 2.4.0,oci.stackable.tech/sandbox/nifi:2.4.0-stackable0.0.0-dev - name: zookeeper values: - 3.9.3 From d33ffab2342487063774295792d0ce069b7bce30 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Wed, 21 May 2025 10:32:59 +0200 Subject: [PATCH 06/37] test: Add test for Apache Iceberg integration (#785) * Clean up smoke test * clean up smoke test part 2 * Add working test :) * Move files * Add and test HDFS functionality * Kerbize HDFS and HMS * Add Kerberos test * Use nightly image * linter * Update Iceberg docs * changelog * Small bumps * Update docs/modules/nifi/pages/usage_guide/writing-to-iceberg-tables.adoc Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com> --------- Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com> --- CHANGELOG.md | 2 + .../writing-to-iceberg-tables.adoc | 20 +- tests/release.yaml | 8 + .../kuttl/iceberg/00-patch-ns.yaml.j2 | 9 + tests/templates/kuttl/iceberg/00-rbac.yaml.j2 | 29 + .../iceberg/01-create-s3-connection.yaml | 5 + .../kuttl/iceberg/01_s3-connection.yaml | 36 + .../templates/kuttl/iceberg/02-assert.yaml.j2 | 14 + .../kuttl/iceberg/02-install-krb5-kdc.yaml.j2 | 146 +++ .../03-create-kerberos-secretclass.yaml.j2 | 8 + .../iceberg/03_kerberos-secretclass.yaml.j2 | 33 + .../templates/kuttl/iceberg/10-assert.yaml.j2 | 10 + ...tor-aggregator-discovery-configmap.yaml.j2 | 9 + tests/templates/kuttl/iceberg/20-assert.yaml | 19 + .../kuttl/iceberg/20-install-minio.yaml | 5 + tests/templates/kuttl/iceberg/21-assert.yaml | 12 + .../iceberg/21-install-hive-postgres.yaml | 12 + .../21_helm-bitnami-postgresql-values.yaml.j2 | 29 + tests/templates/kuttl/iceberg/30-assert.yaml | 12 + .../iceberg/30-install-zookeeper.yaml.j2 | 21 + tests/templates/kuttl/iceberg/31-opa.yaml.j2 | 17 + tests/templates/kuttl/iceberg/32-assert.yaml | 28 + .../kuttl/iceberg/32-install-hdfs.yaml.j2 | 6 + tests/templates/kuttl/iceberg/32_hdfs.yaml.j2 | 78 ++ tests/templates/kuttl/iceberg/33-assert.yaml | 12 + .../kuttl/iceberg/33-install-hive.yaml.j2 | 6 + tests/templates/kuttl/iceberg/33_hive.yaml.j2 | 47 + tests/templates/kuttl/iceberg/34-assert.yaml | 6 + .../kuttl/iceberg/34-install-trino.yaml.j2 | 6 + .../templates/kuttl/iceberg/34_trino.yaml.j2 | 108 ++ tests/templates/kuttl/iceberg/40-assert.yaml | 7 + .../kuttl/iceberg/40-create-iceberg-tables.j2 | 24 + tests/templates/kuttl/iceberg/50-assert.yaml | 12 + .../kuttl/iceberg/50-install-nifi.yaml.j2 | 6 + tests/templates/kuttl/iceberg/50_nifi.yaml.j2 | 154 +++ .../60-create-nifi-flow-configmap.yaml.j2 | 10 + .../iceberg/60_nifi-flow-with-kerberos.json | 1144 +++++++++++++++++ .../60_nifi-flow-without-kerberos.json | 1068 +++++++++++++++ tests/templates/kuttl/iceberg/61-assert.yaml | 11 + .../kuttl/iceberg/61-provision-nifi-flow.yaml | 114 ++ tests/templates/kuttl/iceberg/70-assert.yaml | 11 + .../iceberg/70-check-iceberg-tables.yaml.j2 | 36 + tests/templates/kuttl/iceberg/README.md | 34 + .../kuttl/iceberg/interactive-nifi.yaml | 88 ++ tests/templates/kuttl/smoke/20-assert.yaml | 2 +- ...k.yaml.j2 => 20-install-zookeeper.yaml.j2} | 10 +- tests/templates/kuttl/smoke/30-assert.yaml | 4 +- .../kuttl/smoke/30-install-nifi.yaml.j2 | 64 +- tests/templates/kuttl/smoke/31-assert.yaml.j2 | 2 +- tests/templates/kuttl/smoke/32-assert.yaml | 12 +- tests/templates/kuttl/smoke/33-assert.yaml | 2 +- tests/templates/kuttl/smoke/40-assert.yaml | 2 +- .../kuttl/smoke/40-scale-up-nifi.yaml.j2 | 2 +- tests/templates/kuttl/smoke/70-assert.yaml | 42 - .../kuttl/smoke/70-enable-anonymous.yaml.j2 | 9 - tests/templates/kuttl/smoke/test_nifi.py | 2 +- .../kuttl/smoke/test_nifi_metrics.py | 2 +- tests/test-definition.yaml | 49 +- 58 files changed, 3562 insertions(+), 114 deletions(-) create mode 100644 tests/templates/kuttl/iceberg/00-patch-ns.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/00-rbac.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/01-create-s3-connection.yaml create mode 100644 tests/templates/kuttl/iceberg/01_s3-connection.yaml create mode 100644 tests/templates/kuttl/iceberg/02-assert.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/02-install-krb5-kdc.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/03-create-kerberos-secretclass.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/03_kerberos-secretclass.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/10-assert.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/10-install-vector-aggregator-discovery-configmap.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/20-assert.yaml create mode 100644 tests/templates/kuttl/iceberg/20-install-minio.yaml create mode 100644 tests/templates/kuttl/iceberg/21-assert.yaml create mode 100644 tests/templates/kuttl/iceberg/21-install-hive-postgres.yaml create mode 100644 tests/templates/kuttl/iceberg/21_helm-bitnami-postgresql-values.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/30-assert.yaml create mode 100644 tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/31-opa.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/32-assert.yaml create mode 100644 tests/templates/kuttl/iceberg/32-install-hdfs.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/32_hdfs.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/33-assert.yaml create mode 100644 tests/templates/kuttl/iceberg/33-install-hive.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/33_hive.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/34-assert.yaml create mode 100644 tests/templates/kuttl/iceberg/34-install-trino.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/34_trino.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/40-assert.yaml create mode 100644 tests/templates/kuttl/iceberg/40-create-iceberg-tables.j2 create mode 100644 tests/templates/kuttl/iceberg/50-assert.yaml create mode 100644 tests/templates/kuttl/iceberg/50-install-nifi.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/50_nifi.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/60-create-nifi-flow-configmap.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json create mode 100644 tests/templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json create mode 100644 tests/templates/kuttl/iceberg/61-assert.yaml create mode 100644 tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml create mode 100644 tests/templates/kuttl/iceberg/70-assert.yaml create mode 100644 tests/templates/kuttl/iceberg/70-check-iceberg-tables.yaml.j2 create mode 100644 tests/templates/kuttl/iceberg/README.md create mode 100644 tests/templates/kuttl/iceberg/interactive-nifi.yaml rename tests/templates/kuttl/smoke/{20-install-zk.yaml.j2 => 20-install-zookeeper.yaml.j2} (78%) delete mode 100644 tests/templates/kuttl/smoke/70-assert.yaml delete mode 100644 tests/templates/kuttl/smoke/70-enable-anonymous.yaml.j2 diff --git a/CHANGELOG.md b/CHANGELOG.md index ea28a017..d766019a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ All notable changes to this project will be documented in this file. - Use `--file-log-max-files` (or `FILE_LOG_MAX_FILES`) to limit the number of log files kept. - Use `--file-log-rotation-period` (or `FILE_LOG_ROTATION_PERIOD`) to configure the frequency of rotation. - Use `--console-log-format` (or `CONSOLE_LOG_FORMAT`) to set the format to `plain` (default) or `json`. +- Add test for Apache Iceberg integration ([#785]). ### Changed @@ -40,6 +41,7 @@ All notable changes to this project will be documented in this file. [#774]: https://github.com/stackabletech/nifi-operator/pull/774 [#776]: https://github.com/stackabletech/nifi-operator/pull/776 [#782]: https://github.com/stackabletech/nifi-operator/pull/782 +[#785]: https://github.com/stackabletech/nifi-operator/pull/785 [#787]: https://github.com/stackabletech/nifi-operator/pull/787 [#784]: https://github.com/stackabletech/nifi-operator/pull/784 [#789]: https://github.com/stackabletech/nifi-operator/pull/789 diff --git a/docs/modules/nifi/pages/usage_guide/writing-to-iceberg-tables.adoc b/docs/modules/nifi/pages/usage_guide/writing-to-iceberg-tables.adoc index a026a5b6..13048014 100644 --- a/docs/modules/nifi/pages/usage_guide/writing-to-iceberg-tables.adoc +++ b/docs/modules/nifi/pages/usage_guide/writing-to-iceberg-tables.adoc @@ -2,13 +2,27 @@ :description: Write to Apache Iceberg tables in NiFi using the PutIceberg processor. Supports integration with S3 and Hive Metastore for scalable data handling. :iceberg: https://iceberg.apache.org/ -WARNING: In NiFi `2.0.0` Iceberg support https://issues.apache.org/jira/browse/NIFI-13938[has been removed]. - {iceberg}[Apache Iceberg] is a high-performance format for huge analytic tables. Iceberg brings the reliability and simplicity of SQL tables to big data, while making it possible for engines like Spark, Trino, Flink, Presto, Hive and Impala to safely work with the same tables, at the same time. NiFi supports a `PutIceberg` processor to add rows to an existing Iceberg table https://issues.apache.org/jira/browse/NIFI-10442[starting from version 1.19.0]. -As of NiFi version `1.23.1` only `PutIceberg` is supported, you need to create and compact your tables with other tools such as Trino or Spark (both included in the Stackable Data Platform). +As of NiFi version `2.4.0` only `PutIceberg` is supported, you need to create and compact your tables with other tools such as Trino or Spark (both included in the Stackable Data Platform). + +== NiFi 2 + +In NiFi `2.0.0` Iceberg support https://issues.apache.org/jira/browse/NIFI-13938[has been removed] from upstream NiFi. + +We forked the `nifi-iceberg-bundle` and made it available at https://github.com/stackabletech/nifi-iceberg-bundle. +Starting with SDP 25.7, we have added the necessary bundle to NiFi by default, you don't need to explicitly add Iceberg support to the Stackable NiFi. + +Please read on https://github.com/stackabletech/nifi-iceberg-bundle[its documentation] on how to ingest data into Iceberg tables. +You don't need any special configs on the `NiFiCluster` in case you are using S3 and no Kerberos. + +HDFS and Kerberos are also supported, please have a look at the https://github.com/stackabletech/nifi-operator/tree/main/tests/templates/kuttl/iceberg[Iceberg integration test] for that. + +== NiFi 1 + +Starting with `1.19.0`, NiFi supports writing to Iceberg tables. The following example shows an example NiFi setup using the Iceberg integration. diff --git a/tests/release.yaml b/tests/release.yaml index b6ecf959..04ee6498 100644 --- a/tests/release.yaml +++ b/tests/release.yaml @@ -12,7 +12,15 @@ releases: operatorVersion: 0.0.0-dev listener: operatorVersion: 0.0.0-dev + opa: + operatorVersion: 0.0.0-dev zookeeper: operatorVersion: 0.0.0-dev + hdfs: + operatorVersion: 0.0.0-dev + hive: + operatorVersion: 0.0.0-dev + trino: + operatorVersion: 0.0.0-dev nifi: operatorVersion: 0.0.0-dev diff --git a/tests/templates/kuttl/iceberg/00-patch-ns.yaml.j2 b/tests/templates/kuttl/iceberg/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/iceberg/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/iceberg/00-rbac.yaml.j2 b/tests/templates/kuttl/iceberg/00-rbac.yaml.j2 new file mode 100644 index 00000000..7ee61d23 --- /dev/null +++ b/tests/templates/kuttl/iceberg/00-rbac.yaml.j2 @@ -0,0 +1,29 @@ +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-role +rules: +{% if test_scenario['values']['openshift'] == "true" %} + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["privileged"] + verbs: ["use"] +{% endif %} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-sa +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-rb +subjects: + - kind: ServiceAccount + name: test-sa +roleRef: + kind: Role + name: test-role + apiGroup: rbac.authorization.k8s.io diff --git a/tests/templates/kuttl/iceberg/01-create-s3-connection.yaml b/tests/templates/kuttl/iceberg/01-create-s3-connection.yaml new file mode 100644 index 00000000..26f46c12 --- /dev/null +++ b/tests/templates/kuttl/iceberg/01-create-s3-connection.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: envsubst '$NAMESPACE' < 01_s3-connection.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/iceberg/01_s3-connection.yaml b/tests/templates/kuttl/iceberg/01_s3-connection.yaml new file mode 100644 index 00000000..56c30d36 --- /dev/null +++ b/tests/templates/kuttl/iceberg/01_s3-connection.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: s3.stackable.tech/v1alpha1 +kind: S3Connection +metadata: + name: minio +spec: + host: "minio.${NAMESPACE}.svc.cluster.local" + port: 9000 + accessStyle: Path + credentials: + secretClass: s3-credentials-class + tls: + verification: + server: + caCert: + secretClass: tls +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: s3-credentials-class +spec: + backend: + k8sSearch: + searchNamespace: + pod: {} +--- +apiVersion: v1 +kind: Secret +metadata: + name: minio-credentials + labels: + secrets.stackable.tech/class: s3-credentials-class +stringData: + accessKey: admin + secretKey: adminadmin diff --git a/tests/templates/kuttl/iceberg/02-assert.yaml.j2 b/tests/templates/kuttl/iceberg/02-assert.yaml.j2 new file mode 100644 index 00000000..9d648d0b --- /dev/null +++ b/tests/templates/kuttl/iceberg/02-assert.yaml.j2 @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: krb5-kdc +status: + readyReplicas: 1 + replicas: 1 +{% endif %} diff --git a/tests/templates/kuttl/iceberg/02-install-krb5-kdc.yaml.j2 b/tests/templates/kuttl/iceberg/02-install-krb5-kdc.yaml.j2 new file mode 100644 index 00000000..15b1e04c --- /dev/null +++ b/tests/templates/kuttl/iceberg/02-install-krb5-kdc.yaml.j2 @@ -0,0 +1,146 @@ +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: krb5-kdc +spec: + selector: + matchLabels: + app: krb5-kdc + template: + metadata: + labels: + app: krb5-kdc + spec: + serviceAccountName: test-sa + initContainers: + - name: init + image: oci.stackable.tech/sdp/krb5:{{ test_scenario['values']['krb5'] }}-stackable0.0.0-dev + args: + - sh + - -euo + - pipefail + - -c + - | + test -e /var/kerberos/krb5kdc/principal || kdb5_util create -s -P asdf + kadmin.local get_principal -terse root/admin || kadmin.local add_principal -pw asdf root/admin + # stackable-secret-operator principal must match the keytab specified in the SecretClass + kadmin.local get_principal -terse stackable-secret-operator || kadmin.local add_principal -e aes256-cts-hmac-sha384-192:normal -pw asdf stackable-secret-operator + env: + - name: KRB5_CONFIG + value: /stackable/config/krb5.conf + volumeMounts: + - mountPath: /stackable/config + name: config + - mountPath: /var/kerberos/krb5kdc + name: data + containers: + - name: kdc + image: oci.stackable.tech/sdp/krb5:{{ test_scenario['values']['krb5'] }}-stackable0.0.0-dev + args: + - krb5kdc + - -n + env: + - name: KRB5_CONFIG + value: /stackable/config/krb5.conf + volumeMounts: + - mountPath: /stackable/config + name: config + - mountPath: /var/kerberos/krb5kdc + name: data +# Root permissions required on Openshift to access internal ports +{% if test_scenario['values']['openshift'] == "true" %} + securityContext: + runAsUser: 0 +{% endif %} + - name: kadmind + image: oci.stackable.tech/sdp/krb5:{{ test_scenario['values']['krb5'] }}-stackable0.0.0-dev + args: + - kadmind + - -nofork + env: + - name: KRB5_CONFIG + value: /stackable/config/krb5.conf + volumeMounts: + - mountPath: /stackable/config + name: config + - mountPath: /var/kerberos/krb5kdc + name: data +# Root permissions required on Openshift to access internal ports +{% if test_scenario['values']['openshift'] == "true" %} + securityContext: + runAsUser: 0 +{% endif %} + - name: client + image: oci.stackable.tech/sdp/krb5:{{ test_scenario['values']['krb5'] }}-stackable0.0.0-dev + tty: true + stdin: true + env: + - name: KRB5_CONFIG + value: /stackable/config/krb5.conf + volumeMounts: + - mountPath: /stackable/config + name: config + volumes: + - name: config + configMap: + name: krb5-kdc + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: krb5-kdc +spec: + selector: + app: krb5-kdc + ports: + - name: kadmin + port: 749 + - name: kdc + port: 88 + - name: kdc-udp + port: 88 + protocol: UDP +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: krb5-kdc +data: + krb5.conf: | + [logging] + default = STDERR + kdc = STDERR + admin_server = STDERR + # default = FILE:/var/log/krb5libs.log + # kdc = FILE:/var/log/krb5kdc.log + # admin_server = FILE:/vaggr/log/kadmind.log + [libdefaults] + dns_lookup_realm = false + ticket_lifetime = 24h + renew_lifetime = 7d + forwardable = true + rdns = false + default_realm = {{ test_scenario['values']['kerberos-realm'] }} + spake_preauth_groups = edwards25519 + [realms] + {{ test_scenario['values']['kerberos-realm'] }} = { + acl_file = /stackable/config/kadm5.acl + disable_encrypted_timestamp = false + } + [domain_realm] + .cluster.local = {{ test_scenario['values']['kerberos-realm'] }} + cluster.local = {{ test_scenario['values']['kerberos-realm'] }} + kadm5.acl: | + root/admin *e + stackable-secret-operator *e +{% endif %} diff --git a/tests/templates/kuttl/iceberg/03-create-kerberos-secretclass.yaml.j2 b/tests/templates/kuttl/iceberg/03-create-kerberos-secretclass.yaml.j2 new file mode 100644 index 00000000..d3061645 --- /dev/null +++ b/tests/templates/kuttl/iceberg/03-create-kerberos-secretclass.yaml.j2 @@ -0,0 +1,8 @@ +--- +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + # We need to replace $NAMESPACE (by KUTTL) + - script: envsubst '$NAMESPACE' < 03_kerberos-secretclass.yaml | kubectl apply -n $NAMESPACE -f - +{% endif %} diff --git a/tests/templates/kuttl/iceberg/03_kerberos-secretclass.yaml.j2 b/tests/templates/kuttl/iceberg/03_kerberos-secretclass.yaml.j2 new file mode 100644 index 00000000..e5033462 --- /dev/null +++ b/tests/templates/kuttl/iceberg/03_kerberos-secretclass.yaml.j2 @@ -0,0 +1,33 @@ +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: kerberos-$NAMESPACE +spec: + backend: + kerberosKeytab: + realmName: {{ test_scenario['values']['kerberos-realm'] }} + kdc: krb5-kdc.$NAMESPACE.svc.cluster.local + admin: + mit: + kadminServer: krb5-kdc.$NAMESPACE.svc.cluster.local + adminKeytabSecret: + namespace: $NAMESPACE + name: secret-operator-keytab + adminPrincipal: stackable-secret-operator +--- +apiVersion: v1 +kind: Secret +metadata: + name: secret-operator-keytab +data: + # To create keytab. When promted enter password asdf + # cat | ktutil << 'EOF' + # list + # add_entry -password -p stackable-secret-operator@CLUSTER.LOCAL -k 1 -e aes256-cts-hmac-sha384-192 + # wkt /tmp/keytab + # EOF +{% if test_scenario['values']['kerberos-realm'] == 'CLUSTER.LOCAL' %} + keytab: BQIAAABdAAEADUNMVVNURVIuTE9DQUwAGXN0YWNrYWJsZS1zZWNyZXQtb3BlcmF0b3IAAAABZAYWIgEAFAAgm8MCZ8B//XF1tH92GciD6/usWUNAmBTZnZQxLua2TkgAAAAB +{% elif test_scenario['values']['kerberos-realm'] == 'PROD.MYCORP' %} + keytab: BQIAAABbAAEAC1BST0QuTVlDT1JQABlzdGFja2FibGUtc2VjcmV0LW9wZXJhdG9yAAAAAWQZa0EBABQAIC/EnFNejq/K5lX6tX+B3/tkI13TCzkPB7d2ggCIEzE8AAAAAQ== +{% endif %} diff --git a/tests/templates/kuttl/iceberg/10-assert.yaml.j2 b/tests/templates/kuttl/iceberg/10-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/iceberg/10-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/iceberg/10-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/iceberg/10-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/iceberg/10-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/iceberg/20-assert.yaml b/tests/templates/kuttl/iceberg/20-assert.yaml new file mode 100644 index 00000000..e1829b77 --- /dev/null +++ b/tests/templates/kuttl/iceberg/20-assert.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: minio-post-job +status: + succeeded: 1 diff --git a/tests/templates/kuttl/iceberg/20-install-minio.yaml b/tests/templates/kuttl/iceberg/20-install-minio.yaml new file mode 100644 index 00000000..bbef7238 --- /dev/null +++ b/tests/templates/kuttl/iceberg/20-install-minio.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl -n $NAMESPACE apply -f https://raw.githubusercontent.com/stackabletech/demos/refs/heads/release-25.3/stacks/_templates/minio-tls/rendered-chart.yaml diff --git a/tests/templates/kuttl/iceberg/21-assert.yaml b/tests/templates/kuttl/iceberg/21-assert.yaml new file mode 100644 index 00000000..1ac12423 --- /dev/null +++ b/tests/templates/kuttl/iceberg/21-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgresql +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/iceberg/21-install-hive-postgres.yaml b/tests/templates/kuttl/iceberg/21-install-hive-postgres.yaml new file mode 100644 index 00000000..fa5698e0 --- /dev/null +++ b/tests/templates/kuttl/iceberg/21-install-hive-postgres.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 300 +commands: + - script: >- + helm upgrade postgresql + --install + --version=12.5.6 + --namespace $NAMESPACE + -f 21_helm-bitnami-postgresql-values.yaml + --repo https://charts.bitnami.com/bitnami postgresql diff --git a/tests/templates/kuttl/iceberg/21_helm-bitnami-postgresql-values.yaml.j2 b/tests/templates/kuttl/iceberg/21_helm-bitnami-postgresql-values.yaml.j2 new file mode 100644 index 00000000..2526c3df --- /dev/null +++ b/tests/templates/kuttl/iceberg/21_helm-bitnami-postgresql-values.yaml.j2 @@ -0,0 +1,29 @@ +--- +volumePermissions: + enabled: false + securityContext: + runAsUser: auto + +primary: + extendedConfiguration: | + password_encryption=md5 + podSecurityContext: +{% if test_scenario['values']['openshift'] == 'true' %} + enabled: false +{% else %} + enabled: true +{% endif %} + containerSecurityContext: + enabled: false + resources: + requests: + memory: "512Mi" + cpu: "512m" + limits: + memory: "512Mi" + cpu: "1" + +auth: + username: hive + password: hive + database: hive diff --git a/tests/templates/kuttl/iceberg/30-assert.yaml b/tests/templates/kuttl/iceberg/30-assert.yaml new file mode 100644 index 00000000..49ba7437 --- /dev/null +++ b/tests/templates/kuttl/iceberg/30-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: zookeeper-server-default +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 b/tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 new file mode 100644 index 00000000..58440246 --- /dev/null +++ b/tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 @@ -0,0 +1,21 @@ +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperCluster +metadata: + name: zookeeper +spec: + image: + productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" + pullPolicy: IfNotPresent + clusterConfig: + listenerClass: {{ test_scenario['values']['listener-class'] }} +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + servers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/iceberg/31-opa.yaml.j2 b/tests/templates/kuttl/iceberg/31-opa.yaml.j2 new file mode 100644 index 00000000..179bfbc2 --- /dev/null +++ b/tests/templates/kuttl/iceberg/31-opa.yaml.j2 @@ -0,0 +1,17 @@ +--- +apiVersion: opa.stackable.tech/v1alpha1 +kind: OpaCluster +metadata: + name: opa +spec: + image: +{% if test_scenario['values']['opa-l'].find(",") > 0 %} + custom: "{{ test_scenario['values']['opa-l'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['opa-l'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['opa-l'] }}" +{% endif %} + servers: + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/iceberg/32-assert.yaml b/tests/templates/kuttl/iceberg/32-assert.yaml new file mode 100644 index 00000000..126b5639 --- /dev/null +++ b/tests/templates/kuttl/iceberg/32-assert.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 900 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hdfs-namenode-default +status: + readyReplicas: 2 + replicas: 2 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hdfs-journalnode-default +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hdfs-datanode-default +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/iceberg/32-install-hdfs.yaml.j2 b/tests/templates/kuttl/iceberg/32-install-hdfs.yaml.j2 new file mode 100644 index 00000000..9bbe6c50 --- /dev/null +++ b/tests/templates/kuttl/iceberg/32-install-hdfs.yaml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + # We need to replace $NAMESPACE (by KUTTL) + - script: envsubst '$NAMESPACE' < 32_hdfs.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/iceberg/32_hdfs.yaml.j2 b/tests/templates/kuttl/iceberg/32_hdfs.yaml.j2 new file mode 100644 index 00000000..7a23c747 --- /dev/null +++ b/tests/templates/kuttl/iceberg/32_hdfs.yaml.j2 @@ -0,0 +1,78 @@ +--- +apiVersion: hdfs.stackable.tech/v1alpha1 +kind: HdfsCluster +metadata: + name: hdfs +spec: + image: +{% if test_scenario['values']['hdfs-l'].find(",") > 0 %} + custom: "{{ test_scenario['values']['hdfs-l'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['hdfs-l'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['hdfs-l'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + dfsReplication: 1 +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} + authentication: + tlsSecretClass: tls + kerberos: + secretClass: kerberos-$NAMESPACE + authorization: + opa: + configMapName: opa + package: hdfs +{% endif %} +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + zookeeperConfigMapName: hdfs-znode + nameNodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + configOverrides: &configOverrides + core-site.xml: + # The idea is that the user "hive" can't do anything in hdfs, + # *but* it can impersonate other users (such as trino), + # that have the needed permissions + hadoop.proxyuser.hive.users: "*" + hadoop.proxyuser.hive.hosts: "*" + roleGroups: + default: + replicas: 2 + dataNodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 + journalNodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperZnode +metadata: + name: hdfs-znode +spec: + clusterRef: + name: zookeeper +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: hdfs-regorules + labels: + opa.stackable.tech/bundle: "true" +data: + hdfs.rego: | + package hdfs + + default allow := true diff --git a/tests/templates/kuttl/iceberg/33-assert.yaml b/tests/templates/kuttl/iceberg/33-assert.yaml new file mode 100644 index 00000000..50c27fd9 --- /dev/null +++ b/tests/templates/kuttl/iceberg/33-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 900 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hive-metastore-default +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/iceberg/33-install-hive.yaml.j2 b/tests/templates/kuttl/iceberg/33-install-hive.yaml.j2 new file mode 100644 index 00000000..126fe2ab --- /dev/null +++ b/tests/templates/kuttl/iceberg/33-install-hive.yaml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + # We need to replace $NAMESPACE (by KUTTL) + - script: envsubst '$NAMESPACE' < 33_hive.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/iceberg/33_hive.yaml.j2 b/tests/templates/kuttl/iceberg/33_hive.yaml.j2 new file mode 100644 index 00000000..ad533522 --- /dev/null +++ b/tests/templates/kuttl/iceberg/33_hive.yaml.j2 @@ -0,0 +1,47 @@ +--- +apiVersion: hive.stackable.tech/v1alpha1 +kind: HiveCluster +metadata: + name: hive +spec: + image: +{% if test_scenario['values']['hive-l'].find(",") > 0 %} + custom: "{{ test_scenario['values']['hive-l'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['hive-l'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['hive-l'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + database: + connString: jdbc:postgresql://postgresql:5432/hive + credentialsSecret: postgres-credentials + dbType: postgres + hdfs: + configMap: hdfs + s3: + reference: minio +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} + authentication: + kerberos: + secretClass: kerberos-$NAMESPACE +{% endif %} +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + metastore: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgres-credentials +type: Opaque +stringData: + username: hive + password: hive diff --git a/tests/templates/kuttl/iceberg/34-assert.yaml b/tests/templates/kuttl/iceberg/34-assert.yaml new file mode 100644 index 00000000..b9f1bbe1 --- /dev/null +++ b/tests/templates/kuttl/iceberg/34-assert.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 720 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino --timeout 301s diff --git a/tests/templates/kuttl/iceberg/34-install-trino.yaml.j2 b/tests/templates/kuttl/iceberg/34-install-trino.yaml.j2 new file mode 100644 index 00000000..3bfdc0d2 --- /dev/null +++ b/tests/templates/kuttl/iceberg/34-install-trino.yaml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + # We need to replace $NAMESPACE (by KUTTL) + - script: envsubst '$NAMESPACE' < 34_trino.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/iceberg/34_trino.yaml.j2 b/tests/templates/kuttl/iceberg/34_trino.yaml.j2 new file mode 100644 index 00000000..e5c1e80c --- /dev/null +++ b/tests/templates/kuttl/iceberg/34_trino.yaml.j2 @@ -0,0 +1,108 @@ +--- +apiVersion: trino.stackable.tech/v1alpha1 +kind: TrinoCluster +metadata: + name: trino +spec: + image: +{% if test_scenario['values']['trino-l'].find(",") > 0 %} + custom: "{{ test_scenario['values']['trino-l'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['trino-l'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['trino-l'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + catalogLabelSelector: + matchLabels: + trino: trino + listenerClass: external-unstable +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + coordinators: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} + podOverrides: &podOverrides + spec: + containers: + - name: trino + env: + - name: KERBEROS_REALM + value: {{ test_scenario['values']['kerberos-realm'] }} + volumeMounts: + - name: kerberos + mountPath: /stackable/kerberos + # Normally we would use a different location and set `-Djava.security.krb5.conf=/example/path/krb5.conf`, + # but we can not influence the JVM arguments (edit: we can now). + - name: kerberos + mountPath: /etc/krb5.conf + subPath: krb5.conf + volumes: + - name: kerberos + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: kerberos-$NAMESPACE + secrets.stackable.tech/kerberos.service.names: trino + secrets.stackable.tech/scope: service=trino + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech +{% endif %} + roleGroups: + default: + replicas: 1 + workers: + config: + gracefulShutdownTimeout: 60s # Let the test run faster + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} + podOverrides: *podOverrides +{% endif %} + roleGroups: + default: + replicas: 1 + config: {} +--- +apiVersion: trino.stackable.tech/v1alpha1 +kind: TrinoCatalog +metadata: + name: iceberg + labels: + trino: trino +spec: + connector: + iceberg: + metastore: + configMap: hive + s3: + reference: minio + hdfs: + configMap: hdfs +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} + configOverrides: + # HDFS configuration + hive.hdfs.authentication.type: KERBEROS + hive.hdfs.trino.principal: trino/trino.$NAMESPACE.svc.cluster.local@{{ test_scenario['values']['kerberos-realm'] }} + hive.hdfs.trino.keytab: /stackable/kerberos/keytab + hive.hdfs.impersonation.enabled: "false" + hive.hdfs.wire-encryption.enabled: "true" + # HMS configuration + hive.metastore.authentication.type: KERBEROS + hive.metastore.client.principal: trino/trino.$NAMESPACE.svc.cluster.local@{{ test_scenario['values']['kerberos-realm'] }} + hive.metastore.client.keytab: /stackable/kerberos/keytab + hive.metastore.service.principal: hive/hive.$NAMESPACE.svc.cluster.local@{{ test_scenario['values']['kerberos-realm'] }} + hive.metastore.thrift.impersonation.enabled: "false" + # By default, Hive views are executed with the RUN AS DEFINER security mode. Set the hive.hive-views.run-as-invoker catalog configuration property to true to use RUN AS INVOKER semantics. + # However, this does *not* work for Iceberg catalogs :/ (I asked on the Trino slack: https://trinodb.slack.com/archives/CJ6UC075E/p1711449384648869) + # hive.hive-views.run-as-invoker: "true" +{% endif %} diff --git a/tests/templates/kuttl/iceberg/40-assert.yaml b/tests/templates/kuttl/iceberg/40-assert.yaml new file mode 100644 index 00000000..485373c8 --- /dev/null +++ b/tests/templates/kuttl/iceberg/40-assert.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: create-iceberg-tables +status: + succeeded: 1 diff --git a/tests/templates/kuttl/iceberg/40-create-iceberg-tables.j2 b/tests/templates/kuttl/iceberg/40-create-iceberg-tables.j2 new file mode 100644 index 00000000..7958931a --- /dev/null +++ b/tests/templates/kuttl/iceberg/40-create-iceberg-tables.j2 @@ -0,0 +1,24 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: create-iceberg-tables +spec: + template: + spec: + containers: + - name: create-iceberg-tables + image: "oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino-l'] }}-stackable0.0.0-dev" + command: + - bash + - -euo + - pipefail + - -c + - | + cat << 'EOF' | java -jar trino-cli-*-executable.jar --server https://trino-coordinator:8443 --insecure --user admin + CREATE SCHEMA IF NOT EXISTS iceberg.s3 WITH (location = 's3a://demo/lakehouse/s3'); + CREATE TABLE IF NOT EXISTS iceberg.s3.greetings (hello varchar); + CREATE SCHEMA IF NOT EXISTS iceberg.hdfs WITH (location = 'hdfs:/lakehouse/hdfs'); + CREATE TABLE IF NOT EXISTS iceberg.hdfs.greetings (hello varchar); + EOF + restartPolicy: OnFailure diff --git a/tests/templates/kuttl/iceberg/50-assert.yaml b/tests/templates/kuttl/iceberg/50-assert.yaml new file mode 100644 index 00000000..35aae31d --- /dev/null +++ b/tests/templates/kuttl/iceberg/50-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 1200 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nifi-node-default +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/iceberg/50-install-nifi.yaml.j2 b/tests/templates/kuttl/iceberg/50-install-nifi.yaml.j2 new file mode 100644 index 00000000..006c39e3 --- /dev/null +++ b/tests/templates/kuttl/iceberg/50-install-nifi.yaml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + # We need to replace $NAMESPACE (by KUTTL) + - script: envsubst '$NAMESPACE' < 50_nifi.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 b/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 new file mode 100644 index 00000000..ac1d6ae3 --- /dev/null +++ b/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 @@ -0,0 +1,154 @@ +--- +apiVersion: nifi.stackable.tech/v1alpha1 +kind: NifiCluster +metadata: + name: nifi +spec: + image: +{% if test_scenario['values']['nifi-iceberg'].find(",") > 0 %} + custom: "{{ test_scenario['values']['nifi-iceberg'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['nifi-iceberg'].split(',')[0] }}" +{% else %} + custom: null + productVersion: "{{ test_scenario['values']['nifi-iceberg'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + zookeeperConfigMapName: nifi-znode + listenerClass: external-unstable + authentication: + - authenticationClass: nifi-users + sensitiveProperties: + keySecret: nifi-sensitive-property-key + extraVolumes: + - name: hdfs-config + configMap: + name: hdfs +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} + - name: hive-config + configMap: + name: nifi-hive-config + - name: kerberos + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: kerberos-$NAMESPACE + secrets.stackable.tech/kerberos.service.names: nifi + secrets.stackable.tech/scope: service=nifi + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech +{% endif %} +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + nodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + configOverrides: + nifi.properties: +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} + nifi.kerberos.krb5.file: /stackable/userdata/kerberos/krb5.conf +{% endif %} + + # Quicker startup, and we only have a single node + nifi.cluster.flow.election.max.wait.time: 5 secs + jvmArgumentOverrides: + add: + # Needed for NiFi to trust the minio cert + - -Djavax.net.ssl.trustStore=/stackable/keystore/truststore.p12 + - -Djavax.net.ssl.trustStorePassword=secret + - -Djavax.net.ssl.trustStoreType=PKCS12 +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} + podOverrides: &podOverrides + spec: + containers: + - name: nifi + env: + - name: KERBEROS_REALM + value: {{ test_scenario['values']['kerberos-realm'] }} + volumeMounts: + - name: kerberos + mountPath: /stackable/kerberos + # Normally we would use a different location and set `-Djava.security.krb5.conf=/example/path/krb5.conf`, + # but we can not influence the JVM arguments (edit: we can now) + - name: kerberos + mountPath: /etc/krb5.conf + subPath: krb5.conf + volumes: + - name: kerberos + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: kerberos-$NAMESPACE + secrets.stackable.tech/kerberos.service.names: nifi + secrets.stackable.tech/scope: service=nifi + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech +{% endif %} + roleGroups: + default: + replicas: 1 +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: nifi-users +spec: + provider: + static: + userCredentialsSecret: + name: nifi-users +--- +apiVersion: v1 +kind: Secret +metadata: + name: nifi-users +stringData: + admin: adminadmin +--- +apiVersion: v1 +kind: Secret +metadata: + name: nifi-sensitive-property-key +stringData: + nifiSensitivePropsKey: mYsUp3rS3cr3tk3y +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperZnode +metadata: + name: nifi-znode +spec: + clusterRef: + name: zookeeper +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nifi-hive-config +data: + hive-site.xml: | + + + hive.metastore.sasl.enabled + true + + + hive.metastore.kerberos.principal + hive/hive.$NAMESPACE.svc.cluster.local@{{ test_scenario['values']['kerberos-realm'] }} + + +{% endif %} diff --git a/tests/templates/kuttl/iceberg/60-create-nifi-flow-configmap.yaml.j2 b/tests/templates/kuttl/iceberg/60-create-nifi-flow-configmap.yaml.j2 new file mode 100644 index 00000000..f6373a77 --- /dev/null +++ b/tests/templates/kuttl/iceberg/60-create-nifi-flow-configmap.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} + - script: cat 60_nifi-flow-with-kerberos.json | envsubst '$NAMESPACE' | kubectl -n $NAMESPACE create configmap nifi-flow --from-file=nifi-flow.json=/dev/stdin +{% else %} + - script: cat 60_nifi-flow-without-kerberos.json | envsubst '$NAMESPACE' | kubectl -n $NAMESPACE create configmap nifi-flow --from-file=nifi-flow.json=/dev/stdin + +{% endif %} diff --git a/tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json b/tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json new file mode 100644 index 00000000..a5f773f1 --- /dev/null +++ b/tests/templates/kuttl/iceberg/60_nifi-flow-with-kerberos.json @@ -0,0 +1,1144 @@ +{ + "externalControllerServices": {}, + "flowContents": { + "comments": "", + "componentType": "PROCESS_GROUP", + "connections": [ + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "0ccd3baf-e0f5-3935-b660-456082596e69", + "instanceIdentifier": "c6381ef4-05d1-323a-9392-dac83fba33b3", + "name": "Funnel", + "type": "FUNNEL" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "a3bbe2a3-2115-3ede-8759-d32c3b7c899e", + "instanceIdentifier": "6872c492-329c-36fe-85b5-9593161ccbeb", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "failure" + ], + "source": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "abd8dd79-b5f5-368a-9562-633fd93c784b", + "instanceIdentifier": "f6e95612-25e7-31c8-8b99-0fb0b09c2d1c", + "name": "PutIceberg into S3", + "type": "PROCESSOR" + }, + "zIndex": 0 + }, + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "c268715d-cccb-3c6b-8499-b9bc2abd6ba1", + "instanceIdentifier": "dd4c3b4d-7ad6-3ed3-b869-24a06f65a3e5", + "name": "Funnel", + "type": "FUNNEL" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "73b06526-5744-3aad-b148-995a910f28c5", + "instanceIdentifier": "c17f8b9d-ab2d-3797-82e2-deb7905f90ed", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "failure" + ], + "source": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "27ba6117-133a-38e6-0000-0000390d1716", + "instanceIdentifier": "2b9de595-8fd5-3abc-a6a9-04a9de2cd439", + "name": "PutIceberg into HDFS", + "type": "PROCESSOR" + }, + "zIndex": 0 + }, + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "27ba6117-133a-38e6-0000-0000390d1716", + "instanceIdentifier": "2b9de595-8fd5-3abc-a6a9-04a9de2cd439", + "name": "PutIceberg into HDFS", + "type": "PROCESSOR" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "bc79d195-d4b2-3531-846e-77318e9f08c0", + "instanceIdentifier": "c197d602-e207-3d65-b7f7-cfc966a09089", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "success" + ], + "source": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "27782850-84c8-37e6-a437-a74b2308e57e", + "instanceIdentifier": "a6286bb0-4644-39c7-b37b-eed047db10f3", + "name": "GenerateFlowFile", + "type": "PROCESSOR" + }, + "zIndex": 0 + }, + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "97cf8d71-95e1-3f0d-b6ad-a437ebaa44c4", + "instanceIdentifier": "5c168bce-0d60-372b-94f4-0690b880ce6c", + "name": "Funnel", + "type": "FUNNEL" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "79b3e396-3be1-3175-8c52-4f93347b1f9d", + "instanceIdentifier": "334a9821-8b6f-302c-a34d-51faa6bb36f1", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "success" + ], + "source": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "27ba6117-133a-38e6-0000-0000390d1716", + "instanceIdentifier": "2b9de595-8fd5-3abc-a6a9-04a9de2cd439", + "name": "PutIceberg into HDFS", + "type": "PROCESSOR" + }, + "zIndex": 0 + }, + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "abd8dd79-b5f5-368a-9562-633fd93c784b", + "instanceIdentifier": "f6e95612-25e7-31c8-8b99-0fb0b09c2d1c", + "name": "PutIceberg into S3", + "type": "PROCESSOR" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "11cf418f-4873-3913-b573-a2b07322b81d", + "instanceIdentifier": "01f6d8a0-cd07-36e0-b417-c757a727915b", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "success" + ], + "source": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "27782850-84c8-37e6-a437-a74b2308e57e", + "instanceIdentifier": "a6286bb0-4644-39c7-b37b-eed047db10f3", + "name": "GenerateFlowFile", + "type": "PROCESSOR" + }, + "zIndex": 0 + }, + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "5a024e6e-7818-3e73-ffff-ffff964a69c4", + "instanceIdentifier": "77e72db2-fe65-31b7-a3b3-f9769051c8ac", + "name": "Funnel", + "type": "FUNNEL" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "6a11518e-a129-3faa-8bf2-71c2ad5d8d6f", + "instanceIdentifier": "5558acd0-942d-34ed-8507-dca4bd42307b", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "success" + ], + "source": { + "comments": "", + "groupId": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "id": "abd8dd79-b5f5-368a-9562-633fd93c784b", + "instanceIdentifier": "f6e95612-25e7-31c8-8b99-0fb0b09c2d1c", + "name": "PutIceberg into S3", + "type": "PROCESSOR" + }, + "zIndex": 0 + } + ], + "controllerServices": [ + { + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-aws-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "comments": "", + "componentType": "CONTROLLER_SERVICE", + "controllerServiceApis": [ + { + "bundle": { + "artifact": "nifi-aws-service-api-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "type": "org.apache.nifi.processors.aws.credentials.provider.AwsCredentialsProviderService" + }, + { + "bundle": { + "artifact": "nifi-aws-service-api-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "type": "org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService" + } + ], + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "d9e8d00a-c387-3064-add2-c6060f158ae7", + "instanceIdentifier": "8bf71c50-5f81-3241-b983-28199b65d3d4", + "name": "AWSCredentialsProviderControllerService", + "properties": { + "anonymous-credentials": "false", + "Assume Role ARN": null, + "Assume Role Session Name": null, + "assume-role-external-id": null, + "assume-role-proxy-configuration-service": null, + "assume-role-ssl-context-service": null, + "assume-role-sts-endpoint": null, + "assume-role-sts-region": "us-west-2", + "assume-role-sts-signer-override": "Default Signature", + "Credentials File": null, + "custom-signer-class-name": null, + "custom-signer-module-location": null, + "default-credentials": "false", + "profile-name": null, + "Session Time": "3600", + "Access Key": "admin", + "Secret Key": "adminadmin" + }, + "propertyDescriptors": { + "Access Key": { + "displayName": "Access Key ID", + "dynamic": false, + "identifiesControllerService": false, + "name": "Access Key", + "sensitive": true + }, + "anonymous-credentials": { + "displayName": "Use Anonymous Credentials", + "dynamic": false, + "identifiesControllerService": false, + "name": "anonymous-credentials", + "sensitive": false + }, + "Assume Role ARN": { + "displayName": "Assume Role ARN", + "dynamic": false, + "identifiesControllerService": false, + "name": "Assume Role ARN", + "sensitive": false + }, + "Assume Role Session Name": { + "displayName": "Assume Role Session Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "Assume Role Session Name", + "sensitive": false + }, + "assume-role-external-id": { + "displayName": "Assume Role External ID", + "dynamic": false, + "identifiesControllerService": false, + "name": "assume-role-external-id", + "sensitive": false + }, + "assume-role-proxy-configuration-service": { + "displayName": "Assume Role Proxy Configuration Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "assume-role-proxy-configuration-service", + "sensitive": false + }, + "assume-role-ssl-context-service": { + "displayName": "Assume Role SSL Context Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "assume-role-ssl-context-service", + "sensitive": false + }, + "assume-role-sts-endpoint": { + "displayName": "Assume Role STS Endpoint Override", + "dynamic": false, + "identifiesControllerService": false, + "name": "assume-role-sts-endpoint", + "sensitive": false + }, + "assume-role-sts-region": { + "displayName": "Assume Role STS Region", + "dynamic": false, + "identifiesControllerService": false, + "name": "assume-role-sts-region", + "sensitive": false + }, + "assume-role-sts-signer-override": { + "displayName": "Assume Role STS Signer Override", + "dynamic": false, + "identifiesControllerService": false, + "name": "assume-role-sts-signer-override", + "sensitive": false + }, + "Credentials File": { + "displayName": "Credentials File", + "dynamic": false, + "identifiesControllerService": false, + "name": "Credentials File", + "resourceDefinition": { + "cardinality": "SINGLE", + "resourceTypes": [ + "FILE" + ] + }, + "sensitive": false + }, + "custom-signer-class-name": { + "displayName": "Custom Signer Class Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "custom-signer-class-name", + "sensitive": false + }, + "custom-signer-module-location": { + "displayName": "Custom Signer Module Location", + "dynamic": false, + "identifiesControllerService": false, + "name": "custom-signer-module-location", + "resourceDefinition": { + "cardinality": "MULTIPLE", + "resourceTypes": [ + "DIRECTORY", + "FILE" + ] + }, + "sensitive": false + }, + "default-credentials": { + "displayName": "Use Default Credentials", + "dynamic": false, + "identifiesControllerService": false, + "name": "default-credentials", + "sensitive": false + }, + "profile-name": { + "displayName": "Profile Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "profile-name", + "sensitive": false + }, + "Secret Key": { + "displayName": "Secret Access Key", + "dynamic": false, + "identifiesControllerService": false, + "name": "Secret Key", + "sensitive": true + }, + "Session Time": { + "displayName": "Assume Role Session Time", + "dynamic": false, + "identifiesControllerService": false, + "name": "Session Time", + "sensitive": false + } + }, + "scheduledState": "DISABLED", + "type": "org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderControllerService" + }, + { + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-record-serialization-services-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "comments": "", + "componentType": "CONTROLLER_SERVICE", + "controllerServiceApis": [ + { + "bundle": { + "artifact": "nifi-standard-services-api-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "type": "org.apache.nifi.serialization.RecordReaderFactory" + } + ], + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "af6f000d-1afb-39a5-89be-2ff9176b53fe", + "instanceIdentifier": "b9cdca6d-40e9-3f45-bd6e-448e9c772985", + "name": "JsonTreeReader", + "properties": { + "Allow Comments": "false", + "Date Format": null, + "Max String Length": "20 MB", + "schema-access-strategy": "infer-schema", + "schema-application-strategy": "SELECTED_PART", + "schema-branch": null, + "schema-inference-cache": null, + "schema-name": "${schema.name}", + "schema-reference-reader": null, + "schema-registry": null, + "schema-text": "${avro.schema}", + "schema-version": null, + "starting-field-name": null, + "starting-field-strategy": "ROOT_NODE", + "Time Format": null, + "Timestamp Format": null + }, + "propertyDescriptors": { + "Allow Comments": { + "displayName": "Allow Comments", + "dynamic": false, + "identifiesControllerService": false, + "name": "Allow Comments", + "sensitive": false + }, + "Date Format": { + "displayName": "Date Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "Date Format", + "sensitive": false + }, + "Max String Length": { + "displayName": "Max String Length", + "dynamic": false, + "identifiesControllerService": false, + "name": "Max String Length", + "sensitive": false + }, + "schema-access-strategy": { + "displayName": "Schema Access Strategy", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-access-strategy", + "sensitive": false + }, + "schema-application-strategy": { + "displayName": "Schema Application Strategy", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-application-strategy", + "sensitive": false + }, + "schema-branch": { + "displayName": "Schema Branch", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-branch", + "sensitive": false + }, + "schema-inference-cache": { + "displayName": "Schema Inference Cache", + "dynamic": false, + "identifiesControllerService": true, + "name": "schema-inference-cache", + "sensitive": false + }, + "schema-name": { + "displayName": "Schema Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-name", + "sensitive": false + }, + "schema-reference-reader": { + "displayName": "Schema Reference Reader", + "dynamic": false, + "identifiesControllerService": true, + "name": "schema-reference-reader", + "sensitive": false + }, + "schema-registry": { + "displayName": "Schema Registry", + "dynamic": false, + "identifiesControllerService": true, + "name": "schema-registry", + "sensitive": false + }, + "schema-text": { + "displayName": "Schema Text", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-text", + "sensitive": false + }, + "schema-version": { + "displayName": "Schema Version", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-version", + "sensitive": false + }, + "starting-field-name": { + "displayName": "Starting Field Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "starting-field-name", + "sensitive": false + }, + "starting-field-strategy": { + "displayName": "Starting Field Strategy", + "dynamic": false, + "identifiesControllerService": false, + "name": "starting-field-strategy", + "sensitive": false + }, + "Time Format": { + "displayName": "Time Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "Time Format", + "sensitive": false + }, + "Timestamp Format": { + "displayName": "Timestamp Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "Timestamp Format", + "sensitive": false + } + }, + "scheduledState": "DISABLED", + "type": "org.apache.nifi.json.JsonTreeReader" + }, + { + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-kerberos-user-service-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "comments": "", + "componentType": "CONTROLLER_SERVICE", + "controllerServiceApis": [ + { + "bundle": { + "artifact": "nifi-standard-services-api-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "type": "org.apache.nifi.kerberos.SelfContainedKerberosUserService" + }, + { + "bundle": { + "artifact": "nifi-standard-services-api-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "type": "org.apache.nifi.kerberos.KerberosUserService" + } + ], + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "79d1509c-17ab-3c2a-a431-51324f50e829", + "instanceIdentifier": "a5bbd59e-0196-1000-0000-0000560be512", + "name": "KerberosKeytabUserService", + "properties": { + "Kerberos Keytab": "/stackable/userdata/kerberos/keytab", + "Kerberos Principal": "nifi/nifi.${NAMESPACE}.svc.cluster.local@PROD.MYCORP" + }, + "propertyDescriptors": { + "Kerberos Keytab": { + "displayName": "Kerberos Keytab", + "dynamic": false, + "identifiesControllerService": false, + "name": "Kerberos Keytab", + "resourceDefinition": { + "cardinality": "SINGLE", + "resourceTypes": [ + "FILE" + ] + }, + "sensitive": false + }, + "Kerberos Principal": { + "displayName": "Kerberos Principal", + "dynamic": false, + "identifiesControllerService": false, + "name": "Kerberos Principal", + "sensitive": false + } + }, + "scheduledState": "DISABLED", + "type": "org.apache.nifi.kerberos.KerberosKeytabUserService" + }, + { + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-iceberg-services-nar", + "group": "tech.stackable.nifi", + "version": "0.0.3" + }, + "comments": "", + "componentType": "CONTROLLER_SERVICE", + "controllerServiceApis": [ + { + "bundle": { + "artifact": "nifi-iceberg-services-api-nar", + "group": "tech.stackable.nifi", + "version": "0.0.3" + }, + "type": "tech.stackable.nifi.services.iceberg.IcebergCatalogService" + } + ], + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "64202e39-4c8e-3ba5-9096-35c75272f5a4", + "instanceIdentifier": "e2c4259a-d4e0-3be9-af1c-c361796e6a0f", + "name": "IcebergHiveCatalogService", + "properties": { + "AWS Credentials Provider service": "d9e8d00a-c387-3064-add2-c6060f158ae7", + "hadoop-config-resources": "/stackable/userdata/hdfs-config/core-site.xml,/stackable/userdata/hdfs-config/hdfs-site.xml,/stackable/userdata/hive-config/hive-site.xml", + "hive-metastore-uri": "thrift://hive:9083", + "s3-endpoint": "https://minio.${NAMESPACE}.svc.cluster.local:9000", + "s3-path-style-access": "true", + "warehouse-location": "s3a://demo/lakehouse" + }, + "propertyDescriptors": { + "AWS Credentials Provider service": { + "displayName": "AWS Credentials Provider Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "AWS Credentials Provider service", + "sensitive": false + }, + "hadoop-config-resources": { + "displayName": "Hadoop Configuration Resources", + "dynamic": false, + "identifiesControllerService": false, + "name": "hadoop-config-resources", + "resourceDefinition": { + "cardinality": "MULTIPLE", + "resourceTypes": [ + "FILE" + ] + }, + "sensitive": false + }, + "hive-metastore-uri": { + "displayName": "Hive Metastore URI", + "dynamic": false, + "identifiesControllerService": false, + "name": "hive-metastore-uri", + "sensitive": false + }, + "s3-endpoint": { + "displayName": "S3 endpoint", + "dynamic": false, + "identifiesControllerService": false, + "name": "s3-endpoint", + "sensitive": false + }, + "s3-path-style-access": { + "displayName": "S3 path style access", + "dynamic": false, + "identifiesControllerService": false, + "name": "s3-path-style-access", + "sensitive": false + }, + "warehouse-location": { + "displayName": "Default Warehouse Location", + "dynamic": false, + "identifiesControllerService": false, + "name": "warehouse-location", + "sensitive": false + } + }, + "scheduledState": "DISABLED", + "type": "tech.stackable.nifi.services.iceberg.IcebergHiveCatalogService" + } + ], + "defaultBackPressureDataSizeThreshold": "1 GB", + "defaultBackPressureObjectThreshold": 10000, + "defaultFlowFileExpiration": "0 sec", + "executionEngine": "INHERITED", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE", + "funnels": [ + { + "componentType": "FUNNEL", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "97cf8d71-95e1-3f0d-b6ad-a437ebaa44c4", + "instanceIdentifier": "5c168bce-0d60-372b-94f4-0690b880ce6c", + "position": { + "x": -880.0, + "y": -200.0 + } + }, + { + "componentType": "FUNNEL", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "c268715d-cccb-3c6b-8499-b9bc2abd6ba1", + "instanceIdentifier": "dd4c3b4d-7ad6-3ed3-b869-24a06f65a3e5", + "position": { + "x": -1344.0, + "y": -432.0 + } + }, + { + "componentType": "FUNNEL", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "5a024e6e-7818-3e73-ffff-ffff964a69c4", + "instanceIdentifier": "77e72db2-fe65-31b7-a3b3-f9769051c8ac", + "position": { + "x": -328.0, + "y": -200.0 + } + }, + { + "componentType": "FUNNEL", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "0ccd3baf-e0f5-3935-b660-456082596e69", + "instanceIdentifier": "c6381ef4-05d1-323a-9392-dac83fba33b3", + "position": { + "x": 160.0, + "y": -432.0 + } + } + ], + "identifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "inputPorts": [], + "instanceIdentifier": "a5a6f0ac-0196-1000-ffff-fffffaba5637", + "labels": [], + "maxConcurrentTasks": 1, + "name": "Iceberg Test", + "outputPorts": [], + "position": { + "x": 100.0, + "y": 10.0 + }, + "processGroups": [], + "processors": [ + { + "autoTerminatedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-iceberg-processors-nar", + "group": "tech.stackable.nifi", + "version": "0.0.3" + }, + "comments": "", + "componentType": "PROCESSOR", + "concurrentlySchedulableTaskCount": 1, + "executionNode": "ALL", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "27ba6117-133a-38e6-0000-0000390d1716", + "instanceIdentifier": "2b9de595-8fd5-3abc-a6a9-04a9de2cd439", + "maxBackoffPeriod": "10 mins", + "name": "PutIceberg into HDFS", + "penaltyDuration": "30 sec", + "position": { + "x": -1032.0, + "y": -472.0 + }, + "properties": { + "catalog-namespace": "hdfs", + "catalog-service": "64202e39-4c8e-3ba5-9096-35c75272f5a4", + "file-format": "ORC", + "kerberos-user-service": "79d1509c-17ab-3c2a-a431-51324f50e829", + "maximum-commit-duration": "30 sec", + "maximum-commit-wait-time": "2 sec", + "maximum-file-size": null, + "minimum-commit-wait-time": "100 ms", + "number-of-commit-retries": "10", + "record-reader": "af6f000d-1afb-39a5-89be-2ff9176b53fe", + "table-name": "greetings", + "unmatched-column-behavior": "FAIL_UNMATCHED_COLUMN" + }, + "propertyDescriptors": { + "catalog-namespace": { + "displayName": "Catalog Namespace", + "dynamic": false, + "identifiesControllerService": false, + "name": "catalog-namespace", + "sensitive": false + }, + "catalog-service": { + "displayName": "Catalog Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "catalog-service", + "sensitive": false + }, + "file-format": { + "displayName": "File Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "file-format", + "sensitive": false + }, + "kerberos-user-service": { + "displayName": "Kerberos User Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "kerberos-user-service", + "sensitive": false + }, + "maximum-commit-duration": { + "displayName": "Maximum Commit Duration", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-commit-duration", + "sensitive": false + }, + "maximum-commit-wait-time": { + "displayName": "Maximum Commit Wait Time", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-commit-wait-time", + "sensitive": false + }, + "maximum-file-size": { + "displayName": "Maximum File Size", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-file-size", + "sensitive": false + }, + "minimum-commit-wait-time": { + "displayName": "Minimum Commit Wait Time", + "dynamic": false, + "identifiesControllerService": false, + "name": "minimum-commit-wait-time", + "sensitive": false + }, + "number-of-commit-retries": { + "displayName": "Number of Commit Retries", + "dynamic": false, + "identifiesControllerService": false, + "name": "number-of-commit-retries", + "sensitive": false + }, + "record-reader": { + "displayName": "Record Reader", + "dynamic": false, + "identifiesControllerService": true, + "name": "record-reader", + "sensitive": false + }, + "table-name": { + "displayName": "Table Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "table-name", + "sensitive": false + }, + "unmatched-column-behavior": { + "displayName": "Unmatched Column Behavior", + "dynamic": false, + "identifiesControllerService": false, + "name": "unmatched-column-behavior", + "sensitive": false + } + }, + "retriedRelationships": [], + "retryCount": 10, + "runDurationMillis": 0, + "scheduledState": "ENABLED", + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "style": {}, + "type": "tech.stackable.nifi.processors.iceberg.PutIceberg", + "yieldDuration": "1 sec" + }, + { + "autoTerminatedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-iceberg-processors-nar", + "group": "tech.stackable.nifi", + "version": "0.0.3" + }, + "comments": "", + "componentType": "PROCESSOR", + "concurrentlySchedulableTaskCount": 1, + "executionNode": "ALL", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "abd8dd79-b5f5-368a-9562-633fd93c784b", + "instanceIdentifier": "f6e95612-25e7-31c8-8b99-0fb0b09c2d1c", + "maxBackoffPeriod": "10 mins", + "name": "PutIceberg into S3", + "penaltyDuration": "30 sec", + "position": { + "x": -480.0, + "y": -472.0 + }, + "properties": { + "catalog-namespace": "s3", + "catalog-service": "64202e39-4c8e-3ba5-9096-35c75272f5a4", + "file-format": "PARQUET", + "kerberos-user-service": "79d1509c-17ab-3c2a-a431-51324f50e829", + "maximum-commit-duration": "30 sec", + "maximum-commit-wait-time": "2 sec", + "maximum-file-size": null, + "minimum-commit-wait-time": "100 ms", + "number-of-commit-retries": "10", + "record-reader": "af6f000d-1afb-39a5-89be-2ff9176b53fe", + "table-name": "greetings", + "unmatched-column-behavior": "FAIL_UNMATCHED_COLUMN" + }, + "propertyDescriptors": { + "catalog-namespace": { + "displayName": "Catalog Namespace", + "dynamic": false, + "identifiesControllerService": false, + "name": "catalog-namespace", + "sensitive": false + }, + "catalog-service": { + "displayName": "Catalog Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "catalog-service", + "sensitive": false + }, + "file-format": { + "displayName": "File Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "file-format", + "sensitive": false + }, + "kerberos-user-service": { + "displayName": "Kerberos User Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "kerberos-user-service", + "sensitive": false + }, + "maximum-commit-duration": { + "displayName": "Maximum Commit Duration", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-commit-duration", + "sensitive": false + }, + "maximum-commit-wait-time": { + "displayName": "Maximum Commit Wait Time", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-commit-wait-time", + "sensitive": false + }, + "maximum-file-size": { + "displayName": "Maximum File Size", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-file-size", + "sensitive": false + }, + "minimum-commit-wait-time": { + "displayName": "Minimum Commit Wait Time", + "dynamic": false, + "identifiesControllerService": false, + "name": "minimum-commit-wait-time", + "sensitive": false + }, + "number-of-commit-retries": { + "displayName": "Number of Commit Retries", + "dynamic": false, + "identifiesControllerService": false, + "name": "number-of-commit-retries", + "sensitive": false + }, + "record-reader": { + "displayName": "Record Reader", + "dynamic": false, + "identifiesControllerService": true, + "name": "record-reader", + "sensitive": false + }, + "table-name": { + "displayName": "Table Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "table-name", + "sensitive": false + }, + "unmatched-column-behavior": { + "displayName": "Unmatched Column Behavior", + "dynamic": false, + "identifiesControllerService": false, + "name": "unmatched-column-behavior", + "sensitive": false + } + }, + "retriedRelationships": [], + "retryCount": 10, + "runDurationMillis": 0, + "scheduledState": "ENABLED", + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "style": {}, + "type": "tech.stackable.nifi.processors.iceberg.PutIceberg", + "yieldDuration": "1 sec" + }, + { + "autoTerminatedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-standard-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "comments": "", + "componentType": "PROCESSOR", + "concurrentlySchedulableTaskCount": 1, + "executionNode": "ALL", + "groupIdentifier": "cc177a6c-d142-3812-bc44-3f79b98f7563", + "identifier": "27782850-84c8-37e6-a437-a74b2308e57e", + "instanceIdentifier": "a6286bb0-4644-39c7-b37b-eed047db10f3", + "maxBackoffPeriod": "10 mins", + "name": "GenerateFlowFile", + "penaltyDuration": "30 sec", + "position": { + "x": -744.0, + "y": -704.0 + }, + "properties": { + "Batch Size": "1", + "character-set": "UTF-8", + "Data Format": "Text", + "File Size": "0B", + "generate-ff-custom-text": "{\"hello\": \"world from NiFi :)\"}", + "mime-type": null, + "Unique FlowFiles": "false" + }, + "propertyDescriptors": { + "Batch Size": { + "displayName": "Batch Size", + "dynamic": false, + "identifiesControllerService": false, + "name": "Batch Size", + "sensitive": false + }, + "character-set": { + "displayName": "Character Set", + "dynamic": false, + "identifiesControllerService": false, + "name": "character-set", + "sensitive": false + }, + "Data Format": { + "displayName": "Data Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "Data Format", + "sensitive": false + }, + "File Size": { + "displayName": "File Size", + "dynamic": false, + "identifiesControllerService": false, + "name": "File Size", + "sensitive": false + }, + "generate-ff-custom-text": { + "displayName": "Custom Text", + "dynamic": false, + "identifiesControllerService": false, + "name": "generate-ff-custom-text", + "sensitive": false + }, + "mime-type": { + "displayName": "Mime Type", + "dynamic": false, + "identifiesControllerService": false, + "name": "mime-type", + "sensitive": false + }, + "Unique FlowFiles": { + "displayName": "Unique FlowFiles", + "dynamic": false, + "identifiesControllerService": false, + "name": "Unique FlowFiles", + "sensitive": false + } + }, + "retriedRelationships": [], + "retryCount": 10, + "runDurationMillis": 0, + "scheduledState": "ENABLED", + "schedulingPeriod": "1 min", + "schedulingStrategy": "TIMER_DRIVEN", + "style": {}, + "type": "org.apache.nifi.processors.standard.GenerateFlowFile", + "yieldDuration": "1 sec" + } + ], + "remoteProcessGroups": [], + "scheduledState": "ENABLED", + "statelessFlowTimeout": "1 min" + }, + "flowEncodingVersion": "1.0", + "latest": false, + "parameterContexts": {}, + "parameterProviders": {} +} diff --git a/tests/templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json b/tests/templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json new file mode 100644 index 00000000..c63158b9 --- /dev/null +++ b/tests/templates/kuttl/iceberg/60_nifi-flow-without-kerberos.json @@ -0,0 +1,1068 @@ +{ + "externalControllerServices": {}, + "flowContents": { + "comments": "", + "componentType": "PROCESS_GROUP", + "connections": [ + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "id": "0ccd3baf-e0f5-3935-b660-456082596e69", + "instanceIdentifier": "9013ea63-2dbe-3774-83bf-97110f8adf27", + "name": "Funnel", + "type": "FUNNEL" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "identifier": "a3bbe2a3-2115-3ede-8759-d32c3b7c899e", + "instanceIdentifier": "1b7d4c80-a27c-32e5-9428-0fce2f8a00bd", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "failure" + ], + "source": { + "comments": "", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "id": "abd8dd79-b5f5-368a-9562-633fd93c784b", + "instanceIdentifier": "392482fb-74d5-3a22-ae3a-71f29acab6d8", + "name": "PutIceberg", + "type": "PROCESSOR" + }, + "zIndex": 0 + }, + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "id": "c268715d-cccb-3c6b-8499-b9bc2abd6ba1", + "instanceIdentifier": "8fbbad6f-0196-1000-0000-0000468ba4b3", + "name": "Funnel", + "type": "FUNNEL" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "identifier": "73b06526-5744-3aad-b148-995a910f28c5", + "instanceIdentifier": "8fbbca9c-0196-1000-0000-00002d9b3b50", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "failure" + ], + "source": { + "comments": "", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "id": "27ba6117-133a-38e6-0000-0000390d1716", + "instanceIdentifier": "dfc6acaf-3478-38ec-bee7-9baae44f9147", + "name": "PutIceberg", + "type": "PROCESSOR" + }, + "zIndex": 0 + }, + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "id": "97cf8d71-95e1-3f0d-b6ad-a437ebaa44c4", + "instanceIdentifier": "8fbcd068-0196-1000-0000-00003242b691", + "name": "Funnel", + "type": "FUNNEL" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "identifier": "79b3e396-3be1-3175-8c52-4f93347b1f9d", + "instanceIdentifier": "8fbcdd98-0196-1000-0000-00003cc6f1be", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "success" + ], + "source": { + "comments": "", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "id": "27ba6117-133a-38e6-0000-0000390d1716", + "instanceIdentifier": "dfc6acaf-3478-38ec-bee7-9baae44f9147", + "name": "PutIceberg", + "type": "PROCESSOR" + }, + "zIndex": 0 + }, + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "id": "27ba6117-133a-38e6-0000-0000390d1716", + "instanceIdentifier": "dfc6acaf-3478-38ec-bee7-9baae44f9147", + "name": "PutIceberg", + "type": "PROCESSOR" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "identifier": "bc79d195-d4b2-3531-846e-77318e9f08c0", + "instanceIdentifier": "8fbb639e-0196-1000-ffff-ffff9b19bf06", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "success" + ], + "source": { + "comments": "", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "id": "27782850-84c8-37e6-a437-a74b2308e57e", + "instanceIdentifier": "a17760a7-4af3-3da3-a31c-24c4cd48e653", + "name": "GenerateFlowFile", + "type": "PROCESSOR" + }, + "zIndex": 0 + }, + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "id": "abd8dd79-b5f5-368a-9562-633fd93c784b", + "instanceIdentifier": "392482fb-74d5-3a22-ae3a-71f29acab6d8", + "name": "PutIceberg", + "type": "PROCESSOR" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "identifier": "11cf418f-4873-3913-b573-a2b07322b81d", + "instanceIdentifier": "c6e38b01-4f7f-34dd-8be1-22aec59413b6", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "success" + ], + "source": { + "comments": "", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "id": "27782850-84c8-37e6-a437-a74b2308e57e", + "instanceIdentifier": "a17760a7-4af3-3da3-a31c-24c4cd48e653", + "name": "GenerateFlowFile", + "type": "PROCESSOR" + }, + "zIndex": 0 + }, + { + "backPressureDataSizeThreshold": "1 GB", + "backPressureObjectThreshold": 10000, + "bends": [], + "componentType": "CONNECTION", + "destination": { + "comments": "", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "id": "5a024e6e-7818-3e73-ffff-ffff964a69c4", + "instanceIdentifier": "6d38af9a-d0e9-3b2e-a776-44da91f4497d", + "name": "Funnel", + "type": "FUNNEL" + }, + "flowFileExpiration": "0 sec", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "identifier": "6a11518e-a129-3faa-8bf2-71c2ad5d8d6f", + "instanceIdentifier": "195765f2-2d5d-3221-9e72-fb39091ad4d0", + "labelIndex": 0, + "loadBalanceCompression": "DO_NOT_COMPRESS", + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "name": "", + "partitioningAttribute": "", + "prioritizers": [], + "selectedRelationships": [ + "success" + ], + "source": { + "comments": "", + "groupId": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "id": "abd8dd79-b5f5-368a-9562-633fd93c784b", + "instanceIdentifier": "392482fb-74d5-3a22-ae3a-71f29acab6d8", + "name": "PutIceberg", + "type": "PROCESSOR" + }, + "zIndex": 0 + } + ], + "controllerServices": [ + { + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-aws-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "comments": "", + "componentType": "CONTROLLER_SERVICE", + "controllerServiceApis": [ + { + "bundle": { + "artifact": "nifi-aws-service-api-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "type": "org.apache.nifi.processors.aws.credentials.provider.AwsCredentialsProviderService" + }, + { + "bundle": { + "artifact": "nifi-aws-service-api-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "type": "org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService" + } + ], + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "identifier": "d9e8d00a-c387-3064-add2-c6060f158ae7", + "instanceIdentifier": "a4049610-9d6c-3a38-a411-06fe8e5b7425", + "name": "AWSCredentialsProviderControllerService", + "properties": { + "anonymous-credentials": "false", + "Assume Role ARN": null, + "Assume Role Session Name": null, + "assume-role-external-id": null, + "assume-role-proxy-configuration-service": null, + "assume-role-ssl-context-service": null, + "assume-role-sts-endpoint": null, + "assume-role-sts-region": "us-west-2", + "assume-role-sts-signer-override": "Default Signature", + "Credentials File": null, + "custom-signer-class-name": null, + "custom-signer-module-location": null, + "default-credentials": "false", + "profile-name": null, + "Session Time": "3600", + "Access Key": "admin", + "Secret Key": "adminadmin" + }, + "propertyDescriptors": { + "Access Key": { + "displayName": "Access Key ID", + "dynamic": false, + "identifiesControllerService": false, + "name": "Access Key", + "sensitive": true + }, + "anonymous-credentials": { + "displayName": "Use Anonymous Credentials", + "dynamic": false, + "identifiesControllerService": false, + "name": "anonymous-credentials", + "sensitive": false + }, + "Assume Role ARN": { + "displayName": "Assume Role ARN", + "dynamic": false, + "identifiesControllerService": false, + "name": "Assume Role ARN", + "sensitive": false + }, + "Assume Role Session Name": { + "displayName": "Assume Role Session Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "Assume Role Session Name", + "sensitive": false + }, + "assume-role-external-id": { + "displayName": "Assume Role External ID", + "dynamic": false, + "identifiesControllerService": false, + "name": "assume-role-external-id", + "sensitive": false + }, + "assume-role-proxy-configuration-service": { + "displayName": "Assume Role Proxy Configuration Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "assume-role-proxy-configuration-service", + "sensitive": false + }, + "assume-role-ssl-context-service": { + "displayName": "Assume Role SSL Context Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "assume-role-ssl-context-service", + "sensitive": false + }, + "assume-role-sts-endpoint": { + "displayName": "Assume Role STS Endpoint Override", + "dynamic": false, + "identifiesControllerService": false, + "name": "assume-role-sts-endpoint", + "sensitive": false + }, + "assume-role-sts-region": { + "displayName": "Assume Role STS Region", + "dynamic": false, + "identifiesControllerService": false, + "name": "assume-role-sts-region", + "sensitive": false + }, + "assume-role-sts-signer-override": { + "displayName": "Assume Role STS Signer Override", + "dynamic": false, + "identifiesControllerService": false, + "name": "assume-role-sts-signer-override", + "sensitive": false + }, + "Credentials File": { + "displayName": "Credentials File", + "dynamic": false, + "identifiesControllerService": false, + "name": "Credentials File", + "resourceDefinition": { + "cardinality": "SINGLE", + "resourceTypes": [ + "FILE" + ] + }, + "sensitive": false + }, + "custom-signer-class-name": { + "displayName": "Custom Signer Class Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "custom-signer-class-name", + "sensitive": false + }, + "custom-signer-module-location": { + "displayName": "Custom Signer Module Location", + "dynamic": false, + "identifiesControllerService": false, + "name": "custom-signer-module-location", + "resourceDefinition": { + "cardinality": "MULTIPLE", + "resourceTypes": [ + "FILE", + "DIRECTORY" + ] + }, + "sensitive": false + }, + "default-credentials": { + "displayName": "Use Default Credentials", + "dynamic": false, + "identifiesControllerService": false, + "name": "default-credentials", + "sensitive": false + }, + "profile-name": { + "displayName": "Profile Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "profile-name", + "sensitive": false + }, + "Secret Key": { + "displayName": "Secret Access Key", + "dynamic": false, + "identifiesControllerService": false, + "name": "Secret Key", + "sensitive": true + }, + "Session Time": { + "displayName": "Assume Role Session Time", + "dynamic": false, + "identifiesControllerService": false, + "name": "Session Time", + "sensitive": false + } + }, + "scheduledState": "DISABLED", + "type": "org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderControllerService" + }, + { + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-record-serialization-services-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "comments": "", + "componentType": "CONTROLLER_SERVICE", + "controllerServiceApis": [ + { + "bundle": { + "artifact": "nifi-standard-services-api-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "type": "org.apache.nifi.serialization.RecordReaderFactory" + } + ], + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "identifier": "af6f000d-1afb-39a5-89be-2ff9176b53fe", + "instanceIdentifier": "15771472-fd85-37b8-9b4d-f496e31f49c2", + "name": "JsonTreeReader", + "properties": { + "Allow Comments": "false", + "Date Format": null, + "Max String Length": "20 MB", + "schema-access-strategy": "infer-schema", + "schema-application-strategy": "SELECTED_PART", + "schema-branch": null, + "schema-inference-cache": null, + "schema-name": "${schema.name}", + "schema-reference-reader": null, + "schema-registry": null, + "schema-text": "${avro.schema}", + "schema-version": null, + "starting-field-name": null, + "starting-field-strategy": "ROOT_NODE", + "Time Format": null, + "Timestamp Format": null + }, + "propertyDescriptors": { + "Allow Comments": { + "displayName": "Allow Comments", + "dynamic": false, + "identifiesControllerService": false, + "name": "Allow Comments", + "sensitive": false + }, + "Date Format": { + "displayName": "Date Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "Date Format", + "sensitive": false + }, + "Max String Length": { + "displayName": "Max String Length", + "dynamic": false, + "identifiesControllerService": false, + "name": "Max String Length", + "sensitive": false + }, + "schema-access-strategy": { + "displayName": "Schema Access Strategy", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-access-strategy", + "sensitive": false + }, + "schema-application-strategy": { + "displayName": "Schema Application Strategy", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-application-strategy", + "sensitive": false + }, + "schema-branch": { + "displayName": "Schema Branch", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-branch", + "sensitive": false + }, + "schema-inference-cache": { + "displayName": "Schema Inference Cache", + "dynamic": false, + "identifiesControllerService": true, + "name": "schema-inference-cache", + "sensitive": false + }, + "schema-name": { + "displayName": "Schema Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-name", + "sensitive": false + }, + "schema-reference-reader": { + "displayName": "Schema Reference Reader", + "dynamic": false, + "identifiesControllerService": true, + "name": "schema-reference-reader", + "sensitive": false + }, + "schema-registry": { + "displayName": "Schema Registry", + "dynamic": false, + "identifiesControllerService": true, + "name": "schema-registry", + "sensitive": false + }, + "schema-text": { + "displayName": "Schema Text", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-text", + "sensitive": false + }, + "schema-version": { + "displayName": "Schema Version", + "dynamic": false, + "identifiesControllerService": false, + "name": "schema-version", + "sensitive": false + }, + "starting-field-name": { + "displayName": "Starting Field Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "starting-field-name", + "sensitive": false + }, + "starting-field-strategy": { + "displayName": "Starting Field Strategy", + "dynamic": false, + "identifiesControllerService": false, + "name": "starting-field-strategy", + "sensitive": false + }, + "Time Format": { + "displayName": "Time Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "Time Format", + "sensitive": false + }, + "Timestamp Format": { + "displayName": "Timestamp Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "Timestamp Format", + "sensitive": false + } + }, + "scheduledState": "DISABLED", + "type": "org.apache.nifi.json.JsonTreeReader" + }, + { + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-iceberg-services-nar", + "group": "tech.stackable.nifi", + "version": "0.0.2" + }, + "comments": "", + "componentType": "CONTROLLER_SERVICE", + "controllerServiceApis": [ + { + "bundle": { + "artifact": "nifi-iceberg-services-api-nar", + "group": "tech.stackable.nifi", + "version": "0.0.2" + }, + "type": "tech.stackable.nifi.services.iceberg.IcebergCatalogService" + } + ], + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "identifier": "64202e39-4c8e-3ba5-9096-35c75272f5a4", + "instanceIdentifier": "127f71a3-94c7-3c71-b761-36f2a09629a7", + "name": "IcebergHiveCatalogService", + "properties": { + "AWS Credentials Provider service": "d9e8d00a-c387-3064-add2-c6060f158ae7", + "hadoop-config-resources": "/stackable/userdata/hdfs-config/core-site.xml,/stackable/userdata/hdfs-config/hdfs-site.xml", + "hive-metastore-uri": "thrift://hive:9083", + "s3-endpoint": "https://minio.${NAMESPACE}.svc.cluster.local:9000", + "s3-path-style-access": "true", + "warehouse-location": "s3a://demo/lakehouse" + }, + "propertyDescriptors": { + "AWS Credentials Provider service": { + "displayName": "AWS Credentials Provider Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "AWS Credentials Provider service", + "sensitive": false + }, + "hadoop-config-resources": { + "displayName": "Hadoop Configuration Resources", + "dynamic": false, + "identifiesControllerService": false, + "name": "hadoop-config-resources", + "resourceDefinition": { + "cardinality": "MULTIPLE", + "resourceTypes": [ + "FILE" + ] + }, + "sensitive": false + }, + "hive-metastore-uri": { + "displayName": "Hive Metastore URI", + "dynamic": false, + "identifiesControllerService": false, + "name": "hive-metastore-uri", + "sensitive": false + }, + "s3-endpoint": { + "displayName": "S3 endpoint", + "dynamic": false, + "identifiesControllerService": false, + "name": "s3-endpoint", + "sensitive": false + }, + "s3-path-style-access": { + "displayName": "S3 path style access", + "dynamic": false, + "identifiesControllerService": false, + "name": "s3-path-style-access", + "sensitive": false + }, + "warehouse-location": { + "displayName": "Default Warehouse Location", + "dynamic": false, + "identifiesControllerService": false, + "name": "warehouse-location", + "sensitive": false + } + }, + "scheduledState": "DISABLED", + "type": "tech.stackable.nifi.services.iceberg.IcebergHiveCatalogService" + } + ], + "defaultBackPressureDataSizeThreshold": "1 GB", + "defaultBackPressureObjectThreshold": 10000, + "defaultFlowFileExpiration": "0 sec", + "executionEngine": "INHERITED", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE", + "funnels": [ + { + "componentType": "FUNNEL", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "identifier": "97cf8d71-95e1-3f0d-b6ad-a437ebaa44c4", + "instanceIdentifier": "8fbcd068-0196-1000-0000-00003242b691", + "position": { + "x": -880.0, + "y": -200.0 + } + }, + { + "componentType": "FUNNEL", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "identifier": "c268715d-cccb-3c6b-8499-b9bc2abd6ba1", + "instanceIdentifier": "8fbbad6f-0196-1000-0000-0000468ba4b3", + "position": { + "x": -1344.0, + "y": -432.0 + } + }, + { + "componentType": "FUNNEL", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "identifier": "5a024e6e-7818-3e73-ffff-ffff964a69c4", + "instanceIdentifier": "6d38af9a-d0e9-3b2e-a776-44da91f4497d", + "position": { + "x": -328.0, + "y": -200.0 + } + }, + { + "componentType": "FUNNEL", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "identifier": "0ccd3baf-e0f5-3935-b660-456082596e69", + "instanceIdentifier": "9013ea63-2dbe-3774-83bf-97110f8adf27", + "position": { + "x": 160.0, + "y": -432.0 + } + } + ], + "identifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "inputPorts": [], + "instanceIdentifier": "8fb66dc6-0196-1000-ffff-ffff98cfc281", + "labels": [], + "maxConcurrentTasks": 1, + "name": "Iceberg Test", + "outputPorts": [], + "position": { + "x": 100.0, + "y": 10.0 + }, + "processGroups": [], + "processors": [ + { + "autoTerminatedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-iceberg-processors-nar", + "group": "tech.stackable.nifi", + "version": "0.0.2" + }, + "comments": "", + "componentType": "PROCESSOR", + "concurrentlySchedulableTaskCount": 1, + "executionNode": "ALL", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "identifier": "27ba6117-133a-38e6-0000-0000390d1716", + "instanceIdentifier": "dfc6acaf-3478-38ec-bee7-9baae44f9147", + "maxBackoffPeriod": "10 mins", + "name": "PutIceberg", + "penaltyDuration": "30 sec", + "position": { + "x": -1032.0, + "y": -472.0 + }, + "properties": { + "catalog-namespace": "hdfs", + "catalog-service": "64202e39-4c8e-3ba5-9096-35c75272f5a4", + "file-format": "ORC", + "maximum-commit-duration": "30 sec", + "maximum-commit-wait-time": "2 sec", + "maximum-file-size": null, + "minimum-commit-wait-time": "100 ms", + "number-of-commit-retries": "10", + "record-reader": "af6f000d-1afb-39a5-89be-2ff9176b53fe", + "table-name": "greetings", + "unmatched-column-behavior": "FAIL_UNMATCHED_COLUMN" + }, + "propertyDescriptors": { + "catalog-namespace": { + "displayName": "Catalog Namespace", + "dynamic": false, + "identifiesControllerService": false, + "name": "catalog-namespace", + "sensitive": false + }, + "catalog-service": { + "displayName": "Catalog Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "catalog-service", + "sensitive": false + }, + "file-format": { + "displayName": "File Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "file-format", + "sensitive": false + }, + "maximum-commit-duration": { + "displayName": "Maximum Commit Duration", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-commit-duration", + "sensitive": false + }, + "maximum-commit-wait-time": { + "displayName": "Maximum Commit Wait Time", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-commit-wait-time", + "sensitive": false + }, + "maximum-file-size": { + "displayName": "Maximum File Size", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-file-size", + "sensitive": false + }, + "minimum-commit-wait-time": { + "displayName": "Minimum Commit Wait Time", + "dynamic": false, + "identifiesControllerService": false, + "name": "minimum-commit-wait-time", + "sensitive": false + }, + "number-of-commit-retries": { + "displayName": "Number of Commit Retries", + "dynamic": false, + "identifiesControllerService": false, + "name": "number-of-commit-retries", + "sensitive": false + }, + "record-reader": { + "displayName": "Record Reader", + "dynamic": false, + "identifiesControllerService": true, + "name": "record-reader", + "sensitive": false + }, + "table-name": { + "displayName": "Table Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "table-name", + "sensitive": false + }, + "unmatched-column-behavior": { + "displayName": "Unmatched Column Behavior", + "dynamic": false, + "identifiesControllerService": false, + "name": "unmatched-column-behavior", + "sensitive": false + } + }, + "retriedRelationships": [], + "retryCount": 10, + "runDurationMillis": 0, + "scheduledState": "ENABLED", + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "style": {}, + "type": "tech.stackable.nifi.processors.iceberg.PutIceberg", + "yieldDuration": "1 sec" + }, + { + "autoTerminatedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-iceberg-processors-nar", + "group": "tech.stackable.nifi", + "version": "0.0.2" + }, + "comments": "", + "componentType": "PROCESSOR", + "concurrentlySchedulableTaskCount": 1, + "executionNode": "ALL", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "identifier": "abd8dd79-b5f5-368a-9562-633fd93c784b", + "instanceIdentifier": "392482fb-74d5-3a22-ae3a-71f29acab6d8", + "maxBackoffPeriod": "10 mins", + "name": "PutIceberg", + "penaltyDuration": "30 sec", + "position": { + "x": -480.0, + "y": -472.0 + }, + "properties": { + "catalog-namespace": "s3", + "catalog-service": "64202e39-4c8e-3ba5-9096-35c75272f5a4", + "file-format": "PARQUET", + "maximum-commit-duration": "30 sec", + "maximum-commit-wait-time": "2 sec", + "maximum-file-size": null, + "minimum-commit-wait-time": "100 ms", + "number-of-commit-retries": "10", + "record-reader": "af6f000d-1afb-39a5-89be-2ff9176b53fe", + "table-name": "greetings", + "unmatched-column-behavior": "FAIL_UNMATCHED_COLUMN" + }, + "propertyDescriptors": { + "catalog-namespace": { + "displayName": "Catalog Namespace", + "dynamic": false, + "identifiesControllerService": false, + "name": "catalog-namespace", + "sensitive": false + }, + "catalog-service": { + "displayName": "Catalog Service", + "dynamic": false, + "identifiesControllerService": true, + "name": "catalog-service", + "sensitive": false + }, + "file-format": { + "displayName": "File Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "file-format", + "sensitive": false + }, + "maximum-commit-duration": { + "displayName": "Maximum Commit Duration", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-commit-duration", + "sensitive": false + }, + "maximum-commit-wait-time": { + "displayName": "Maximum Commit Wait Time", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-commit-wait-time", + "sensitive": false + }, + "maximum-file-size": { + "displayName": "Maximum File Size", + "dynamic": false, + "identifiesControllerService": false, + "name": "maximum-file-size", + "sensitive": false + }, + "minimum-commit-wait-time": { + "displayName": "Minimum Commit Wait Time", + "dynamic": false, + "identifiesControllerService": false, + "name": "minimum-commit-wait-time", + "sensitive": false + }, + "number-of-commit-retries": { + "displayName": "Number of Commit Retries", + "dynamic": false, + "identifiesControllerService": false, + "name": "number-of-commit-retries", + "sensitive": false + }, + "record-reader": { + "displayName": "Record Reader", + "dynamic": false, + "identifiesControllerService": true, + "name": "record-reader", + "sensitive": false + }, + "table-name": { + "displayName": "Table Name", + "dynamic": false, + "identifiesControllerService": false, + "name": "table-name", + "sensitive": false + }, + "unmatched-column-behavior": { + "displayName": "Unmatched Column Behavior", + "dynamic": false, + "identifiesControllerService": false, + "name": "unmatched-column-behavior", + "sensitive": false + } + }, + "retriedRelationships": [], + "retryCount": 10, + "runDurationMillis": 0, + "scheduledState": "ENABLED", + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "style": {}, + "type": "tech.stackable.nifi.processors.iceberg.PutIceberg", + "yieldDuration": "1 sec" + }, + { + "autoTerminatedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "bulletinLevel": "WARN", + "bundle": { + "artifact": "nifi-standard-nar", + "group": "org.apache.nifi", + "version": "2.2.0" + }, + "comments": "", + "componentType": "PROCESSOR", + "concurrentlySchedulableTaskCount": 1, + "executionNode": "ALL", + "groupIdentifier": "d93c30fa-183d-3f8e-afdf-45616b74a969", + "identifier": "27782850-84c8-37e6-a437-a74b2308e57e", + "instanceIdentifier": "a17760a7-4af3-3da3-a31c-24c4cd48e653", + "maxBackoffPeriod": "10 mins", + "name": "GenerateFlowFile", + "penaltyDuration": "30 sec", + "position": { + "x": -744.0, + "y": -704.0 + }, + "properties": { + "Batch Size": "1", + "character-set": "UTF-8", + "Data Format": "Text", + "File Size": "0B", + "generate-ff-custom-text": "{\"hello\": \"world from NiFi :)\"}", + "mime-type": null, + "Unique FlowFiles": "false" + }, + "propertyDescriptors": { + "Batch Size": { + "displayName": "Batch Size", + "dynamic": false, + "identifiesControllerService": false, + "name": "Batch Size", + "sensitive": false + }, + "character-set": { + "displayName": "Character Set", + "dynamic": false, + "identifiesControllerService": false, + "name": "character-set", + "sensitive": false + }, + "Data Format": { + "displayName": "Data Format", + "dynamic": false, + "identifiesControllerService": false, + "name": "Data Format", + "sensitive": false + }, + "File Size": { + "displayName": "File Size", + "dynamic": false, + "identifiesControllerService": false, + "name": "File Size", + "sensitive": false + }, + "generate-ff-custom-text": { + "displayName": "Custom Text", + "dynamic": false, + "identifiesControllerService": false, + "name": "generate-ff-custom-text", + "sensitive": false + }, + "mime-type": { + "displayName": "Mime Type", + "dynamic": false, + "identifiesControllerService": false, + "name": "mime-type", + "sensitive": false + }, + "Unique FlowFiles": { + "displayName": "Unique FlowFiles", + "dynamic": false, + "identifiesControllerService": false, + "name": "Unique FlowFiles", + "sensitive": false + } + }, + "retriedRelationships": [], + "retryCount": 10, + "runDurationMillis": 0, + "scheduledState": "ENABLED", + "schedulingPeriod": "1 min", + "schedulingStrategy": "TIMER_DRIVEN", + "style": {}, + "type": "org.apache.nifi.processors.standard.GenerateFlowFile", + "yieldDuration": "1 sec" + } + ], + "remoteProcessGroups": [], + "scheduledState": "ENABLED", + "statelessFlowTimeout": "1 min" + }, + "flowEncodingVersion": "1.0", + "latest": false, + "parameterContexts": {}, + "parameterProviders": {} +} diff --git a/tests/templates/kuttl/iceberg/61-assert.yaml b/tests/templates/kuttl/iceberg/61-assert.yaml new file mode 100644 index 00000000..813504b1 --- /dev/null +++ b/tests/templates/kuttl/iceberg/61-assert.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: provision-nifi-flow +status: + succeeded: 1 diff --git a/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml b/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml new file mode 100644 index 00000000..7ab671ce --- /dev/null +++ b/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml @@ -0,0 +1,114 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: provision-nifi-flow +spec: + template: + spec: + containers: + - name: provision-nifi-flow + image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev + command: + - bash + - -euo + - pipefail + - -c + - python -u /tmp/script/script.py + volumeMounts: + - name: script + mountPath: /tmp/script + - name: nifi-flow + mountPath: /tmp/nifi-flow + - name: nifi-users + mountPath: /nifi-users + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumes: + - name: script + configMap: + name: provision-nifi-flow-script + - name: nifi-flow + configMap: + name: nifi-flow + - name: nifi-users + secret: + secretName: nifi-users + restartPolicy: OnFailure +--- +# Taken from https://github.com/stackabletech/demos/blob/1744be00054eec2827b2d9ef0a90645843cb0075/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml#L52 +apiVersion: v1 +kind: ConfigMap +metadata: + name: provision-nifi-flow-script +data: + script.py: | + from nipyapi.canvas import get_root_pg_id, schedule_process_group, list_all_controllers, schedule_controller + from nipyapi.security import service_login + import nipyapi + import os + import requests + import urllib3 + + # As of 2022-08-29 we cant use "https://nifi:8443" here because

The request contained an invalid host header [nifi:8443] in the request [/nifi-api]. Check for request manipulation or third-party intercept.

+ ENDPOINT = f"https://nifi-node-default-0.nifi-node-default.{os.environ['NAMESPACE']}.svc.cluster.local:8443" # For local testing / developing replace it, afterwards change back to f"https://nifi-node-default-0.nifi-node-default.{os.environ['NAMESPACE']}.svc.cluster.local:8443" + USERNAME = "admin" + PASSWORD = open("/nifi-users/admin").read() + + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + nipyapi.config.nifi_config.host = f"{ENDPOINT}/nifi-api" + nipyapi.config.nifi_config.verify_ssl = False + + print(f"Logging in as {USERNAME}") + service_login(username=USERNAME, password=PASSWORD) + print("Logged in") + + filename = "/tmp/nifi-flow/nifi-flow.json" + + pg_id = get_root_pg_id() + print(f"Got root process group id: {pg_id}") + + if not nipyapi.config.nifi_config.api_client: + nipyapi.config.nifi_config.api_client = ApiClient() + + header_params = {} + header_params['Accept'] = nipyapi.config.nifi_config.api_client.select_header_accept(['application/json']) + header_params['Content-Type'] = nipyapi.config.nifi_config.api_client.select_header_content_type(['multipart/form-data']) + + print("Uploading process group") + nipyapi.config.nifi_config.api_client.call_api('/process-groups/{pg_id}/process-groups/upload', 'POST', + path_params={'pg_id': pg_id}, + header_params=header_params, + _return_http_data_only=True, + post_params=[ + ('id', pg_id), + ('groupName', 'Iceberg Test'), + ('positionX', 100), + ('positionY', 10), + ('clientId', nipyapi.nifi.FlowApi().generate_client_id()), + ], + files={ + 'file': filename + }, + auth_settings=['tokenAuth']) + print("Process group uploaded") + + # As they are started in the wrong order :D we need to retry + max_retries = 5 + for _ in range(max_retries): + controllers = list_all_controllers(pg_id) + print(f"Found {len(controllers)} controllers") + for controller in controllers: + if controller.component.state != "ENABLED": + try: + print(f"Scheduling controller {controller.component.name}") + schedule_controller(controller, scheduled=True) + print(f"Scheduled controller: {controller.component.name}") + except Exception as e: + print(f"Failed to schedule controller {controller.component.name}: {e}") + + schedule_process_group(pg_id, scheduled=True) diff --git a/tests/templates/kuttl/iceberg/70-assert.yaml b/tests/templates/kuttl/iceberg/70-assert.yaml new file mode 100644 index 00000000..4a7bd272 --- /dev/null +++ b/tests/templates/kuttl/iceberg/70-assert.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: check-iceberg-tables +status: + succeeded: 1 diff --git a/tests/templates/kuttl/iceberg/70-check-iceberg-tables.yaml.j2 b/tests/templates/kuttl/iceberg/70-check-iceberg-tables.yaml.j2 new file mode 100644 index 00000000..079a2722 --- /dev/null +++ b/tests/templates/kuttl/iceberg/70-check-iceberg-tables.yaml.j2 @@ -0,0 +1,36 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: check-iceberg-tables +spec: + template: + spec: + containers: + - name: check-iceberg-tables + image: "oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino-l'] }}-stackable0.0.0-dev" + command: + - bash + - -euo + - pipefail + - -c + - | + for SCHEMA in iceberg.s3 iceberg.hdfs; do + COUNT=$(cat << EOF | java -jar trino-cli-*-executable.jar --server https://trino-coordinator:8443 --insecure --user admin + SELECT COUNT(*) FROM $SCHEMA.greetings WHERE hello = 'world from NiFi :)'; + EOF + ) + + COUNT="${COUNT%\"}" # Remove trailing quote if any + COUNT="${COUNT#\"}" # Remove leading quote if any + echo "Count is $COUNT" + + # Check if it's a number greater than 0 + if ! [[ "$COUNT" =~ ^[0-9]+$ ]] || [ "$COUNT" -le 0 ]; then + echo "Invalid or zero count: $COUNT" + exit 1 + fi + + echo "Count $COUNT was valid" + done + restartPolicy: OnFailure diff --git a/tests/templates/kuttl/iceberg/README.md b/tests/templates/kuttl/iceberg/README.md new file mode 100644 index 00000000..b856148c --- /dev/null +++ b/tests/templates/kuttl/iceberg/README.md @@ -0,0 +1,34 @@ +The file `60_nifi-flow.json` was exported from the NiFi UI. + +*However*, we need to update some stuff, such as adding S3 credentials and templating the namespace of MinIO. + +TIP: I used `JSON: Sort Document` in VScode to somewhat have consistent formatting, which makes reading and diffs easier. + +Notable the following diff has been made (may not be up to date!): + +```diff +diff --git a/tests/templates/kuttl/iceberg/60_nifi-flow.json b/tests/templates/kuttl/iceberg/60_nifi-flow.json +index 09783fa..23c679f 100644 +--- a/tests/templates/kuttl/iceberg/60_nifi-flow.json ++++ b/tests/templates/kuttl/iceberg/60_nifi-flow.json +@@ -160,7 +160,9 @@ + "custom-signer-module-location": null, + "default-credentials": "false", + "profile-name": null, +- "Session Time": "3600" ++ "Session Time": "3600", ++ "Access Key": "admin", ++ "Secret Key": "adminadmin" + }, + "propertyDescriptors": { + "Access Key": { +@@ -483,7 +485,7 @@ + "properties": { + "AWS Credentials Provider service": "d9e8d00a-c387-3064-add2-c6060f158ae7", + "hive-metastore-uri": "thrift://hive:9083", +- "s3-endpoint": "https://minio.kuttl-test-patient-tarpon.svc.cluster.local:9000", ++ "s3-endpoint": "https://minio.${NAMESPACE}.svc.cluster.local:9000", + "s3-path-style-access": "true", + "warehouse-location": "s3a://demo/lakehouse" + }, +``` diff --git a/tests/templates/kuttl/iceberg/interactive-nifi.yaml b/tests/templates/kuttl/iceberg/interactive-nifi.yaml new file mode 100644 index 00000000..d21fff01 --- /dev/null +++ b/tests/templates/kuttl/iceberg/interactive-nifi.yaml @@ -0,0 +1,88 @@ +apiVersion: nifi.stackable.tech/v1alpha1 +kind: NifiCluster +metadata: + name: nifi-interactive +spec: + image: + productVersion: 2.2.0 + clusterConfig: + authentication: + - authenticationClass: nifi-users + listenerClass: external-unstable + sensitiveProperties: + keySecret: nifi-interactive-sensitive-property-key + autoGenerate: true + zookeeperConfigMapName: nifi-interactive-znode + extraVolumes: + - name: nifi-processors + persistentVolumeClaim: + claimName: nifi-interactive-processors + - name: hdfs-config + configMap: + name: hdfs + - name: nifi-interactive-hive-site + configMap: + name: nifi-interactive-hive-site + - name: kerberos + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: kerberos-kuttl-test-united-pheasant + secrets.stackable.tech/kerberos.service.names: nifi + secrets.stackable.tech/scope: service=nifi + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech + nodes: + configOverrides: + nifi.properties: + nifi.nar.library.directory.myCustomLibs: /stackable/userdata/nifi-processors/ + nifi.kerberos.krb5.file: /stackable/userdata/kerberos/krb5.conf + # Quicker startup + nifi.cluster.flow.election.max.wait.time: 3 secs + envOverrides: + KERBEROS_REALM: PROD.MYCORP + roleGroups: + default: + replicas: 1 +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperZnode +metadata: + name: nifi-interactive-znode +spec: + clusterRef: + name: zookeeper +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nifi-interactive-processors +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nifi-interactive-hive-site +data: + hive-site.xml: | + + + hive.metastore.kerberos.principal + hive/hive.kuttl-test-united-pheasant.svc.cluster.local@MY.CORP + + + hive.metastore.sasl.enabled + true + + diff --git a/tests/templates/kuttl/smoke/20-assert.yaml b/tests/templates/kuttl/smoke/20-assert.yaml index e0766c49..49ba7437 100644 --- a/tests/templates/kuttl/smoke/20-assert.yaml +++ b/tests/templates/kuttl/smoke/20-assert.yaml @@ -6,7 +6,7 @@ timeout: 600 apiVersion: apps/v1 kind: StatefulSet metadata: - name: test-zk-server-default + name: zookeeper-server-default status: readyReplicas: 1 replicas: 1 diff --git a/tests/templates/kuttl/smoke/20-install-zk.yaml.j2 b/tests/templates/kuttl/smoke/20-install-zookeeper.yaml.j2 similarity index 78% rename from tests/templates/kuttl/smoke/20-install-zk.yaml.j2 rename to tests/templates/kuttl/smoke/20-install-zookeeper.yaml.j2 index 100927b1..103845c4 100644 --- a/tests/templates/kuttl/smoke/20-install-zk.yaml.j2 +++ b/tests/templates/kuttl/smoke/20-install-zookeeper.yaml.j2 @@ -2,7 +2,7 @@ apiVersion: zookeeper.stackable.tech/v1alpha1 kind: ZookeeperCluster metadata: - name: test-zk + name: zookeeper spec: image: productVersion: "{{ test_scenario['values']['zookeeper'] }}" @@ -19,11 +19,3 @@ spec: roleGroups: default: replicas: 1 ---- -apiVersion: zookeeper.stackable.tech/v1alpha1 -kind: ZookeeperZnode -metadata: - name: test-nifi-znode -spec: - clusterRef: - name: test-zk diff --git a/tests/templates/kuttl/smoke/30-assert.yaml b/tests/templates/kuttl/smoke/30-assert.yaml index ae825d11..d075654f 100644 --- a/tests/templates/kuttl/smoke/30-assert.yaml +++ b/tests/templates/kuttl/smoke/30-assert.yaml @@ -6,7 +6,7 @@ timeout: 1200 apiVersion: apps/v1 kind: StatefulSet metadata: - name: test-nifi-node-default + name: nifi-node-default spec: template: spec: @@ -18,7 +18,7 @@ status: apiVersion: policy/v1 kind: PodDisruptionBudget metadata: - name: test-nifi-node + name: nifi-node status: expectedPods: 2 currentHealthy: 2 diff --git a/tests/templates/kuttl/smoke/30-install-nifi.yaml.j2 b/tests/templates/kuttl/smoke/30-install-nifi.yaml.j2 index aceebaf6..337c37a9 100644 --- a/tests/templates/kuttl/smoke/30-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/smoke/30-install-nifi.yaml.j2 @@ -1,33 +1,8 @@ --- -apiVersion: authentication.stackable.tech/v1alpha1 -kind: AuthenticationClass -metadata: - name: simple-nifi-users -spec: - provider: - static: - userCredentialsSecret: - name: simple-nifi-admin-credentials ---- -apiVersion: v1 -kind: Secret -metadata: - name: simple-nifi-admin-credentials -stringData: - admin: > - passwordWithSpecialCharacter\@<&>"' ---- -apiVersion: v1 -kind: Secret -metadata: - name: nifi-sensitive-property-key -stringData: - nifiSensitivePropsKey: mYsUp3rS3cr3tk3y ---- apiVersion: nifi.stackable.tech/v1alpha1 kind: NifiCluster metadata: - name: test-nifi + name: nifi spec: image: {% if test_scenario['values']['nifi'].find(",") > 0 %} @@ -39,10 +14,10 @@ spec: {% endif %} pullPolicy: IfNotPresent clusterConfig: - zookeeperConfigMapName: test-nifi-znode + zookeeperConfigMapName: nifi-znode listenerClass: {{ test_scenario['values']['listener-class'] }} authentication: - - authenticationClass: simple-nifi-users + - authenticationClass: nifi-users hostHeaderCheck: allowAll: false additionalAllowedHosts: @@ -73,3 +48,36 @@ spec: "nifi.properties": "nifi.diagnostics.on.shutdown.enabled": "false" "nifi.diagnostics.on.shutdown.max.filecount": "20" +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: nifi-users +spec: + provider: + static: + userCredentialsSecret: + name: nifi-users +--- +apiVersion: v1 +kind: Secret +metadata: + name: nifi-users +stringData: + admin: > + passwordWithSpecialCharacter\@<&>"' +--- +apiVersion: v1 +kind: Secret +metadata: + name: nifi-sensitive-property-key +stringData: + nifiSensitivePropsKey: mYsUp3rS3cr3tk3y +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperZnode +metadata: + name: nifi-znode +spec: + clusterRef: + name: zookeeper diff --git a/tests/templates/kuttl/smoke/31-assert.yaml.j2 b/tests/templates/kuttl/smoke/31-assert.yaml.j2 index 06b1dc78..7b2eacd4 100644 --- a/tests/templates/kuttl/smoke/31-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/31-assert.yaml.j2 @@ -3,4 +3,4 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 30 commands: -- script: kubectl get cm -n $NAMESPACE test-nifi-node-default -o yaml | grep -- 'nifi.web.proxy.host=.*example.com:1234' | xargs test ! -z +- script: kubectl get cm -n $NAMESPACE nifi-node-default -o yaml | grep -- 'nifi.web.proxy.host=.*example.com:1234' | xargs test ! -z diff --git a/tests/templates/kuttl/smoke/32-assert.yaml b/tests/templates/kuttl/smoke/32-assert.yaml index 00da1613..960b4789 100644 --- a/tests/templates/kuttl/smoke/32-assert.yaml +++ b/tests/templates/kuttl/smoke/32-assert.yaml @@ -7,13 +7,13 @@ commands: # Test envOverrides # - script: | - kubectl -n $NAMESPACE get sts test-nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")' - kubectl -n $NAMESPACE get sts test-nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")' - kubectl -n $NAMESPACE get sts test-nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")' + kubectl -n $NAMESPACE get sts nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")' + kubectl -n $NAMESPACE get sts nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")' + kubectl -n $NAMESPACE get sts nifi-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "nifi") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")' # # Test configOverrides # - script: | - kubectl -n $NAMESPACE get cm test-nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.enabled=false" - kubectl -n $NAMESPACE get cm test-nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.verbose=false" - kubectl -n $NAMESPACE get cm test-nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.max.filecount=20" + kubectl -n $NAMESPACE get cm nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.enabled=false" + kubectl -n $NAMESPACE get cm nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.verbose=false" + kubectl -n $NAMESPACE get cm nifi-node-default -o yaml | yq -e '.data."nifi.properties"' | grep "nifi.diagnostics.on.shutdown.max.filecount=20" diff --git a/tests/templates/kuttl/smoke/33-assert.yaml b/tests/templates/kuttl/smoke/33-assert.yaml index b2a98140..271a8bec 100644 --- a/tests/templates/kuttl/smoke/33-assert.yaml +++ b/tests/templates/kuttl/smoke/33-assert.yaml @@ -4,4 +4,4 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 600 commands: - - script: kubectl exec -n $NAMESPACE --container nifi test-nifi-node-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' + - script: kubectl exec -n $NAMESPACE --container nifi nifi-node-default-0 -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' diff --git a/tests/templates/kuttl/smoke/40-assert.yaml b/tests/templates/kuttl/smoke/40-assert.yaml index 88f50b77..c3bcd76b 100644 --- a/tests/templates/kuttl/smoke/40-assert.yaml +++ b/tests/templates/kuttl/smoke/40-assert.yaml @@ -6,7 +6,7 @@ timeout: 1200 apiVersion: apps/v1 kind: StatefulSet metadata: - name: test-nifi-node-default + name: nifi-node-default status: readyReplicas: 3 replicas: 3 diff --git a/tests/templates/kuttl/smoke/40-scale-up-nifi.yaml.j2 b/tests/templates/kuttl/smoke/40-scale-up-nifi.yaml.j2 index 987b0745..37ded261 100644 --- a/tests/templates/kuttl/smoke/40-scale-up-nifi.yaml.j2 +++ b/tests/templates/kuttl/smoke/40-scale-up-nifi.yaml.j2 @@ -4,5 +4,5 @@ kind: TestStep commands: - script: >- kubectl --namespace $NAMESPACE - patch nificlusters.nifi.stackable.tech test-nifi + patch nificlusters.nifi.stackable.tech nifi --type=merge --patch '{"spec":{"nodes": {"roleGroups": {"default": {"replicas": 3}}}}}' diff --git a/tests/templates/kuttl/smoke/70-assert.yaml b/tests/templates/kuttl/smoke/70-assert.yaml deleted file mode 100644 index 29b0ff4d..00000000 --- a/tests/templates/kuttl/smoke/70-assert.yaml +++ /dev/null @@ -1,42 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 600 ---- -apiVersion: v1 -kind: Event -reason: Started -source: - component: kubelet -involvedObject: - apiVersion: v1 - kind: Pod - name: test-nifi-node-default-0 ---- -apiVersion: v1 -kind: Event -reason: Started -source: - component: kubelet -involvedObject: - apiVersion: v1 - kind: Pod - name: test-nifi-node-default-1 ---- -apiVersion: v1 -kind: Event -reason: Started -source: - component: kubelet -involvedObject: - apiVersion: v1 - kind: Pod - name: test-nifi-node-default-2 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: test-nifi-node-default -status: - readyReplicas: 3 - replicas: 3 diff --git a/tests/templates/kuttl/smoke/70-enable-anonymous.yaml.j2 b/tests/templates/kuttl/smoke/70-enable-anonymous.yaml.j2 deleted file mode 100644 index f39ce021..00000000 --- a/tests/templates/kuttl/smoke/70-enable-anonymous.yaml.j2 +++ /dev/null @@ -1,9 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: >- - kubectl --namespace $NAMESPACE - patch nificlusters.nifi.stackable.tech test-nifi - --type=merge --patch '{"spec":{"config": {"authentication": {"allowAnonymousAccess": true}}}}' - - command: kubectl rollout restart statefulset test-nifi-node-default --namespace $NAMESPACE diff --git a/tests/templates/kuttl/smoke/test_nifi.py b/tests/templates/kuttl/smoke/test_nifi.py index b885c09f..9d89fb63 100755 --- a/tests/templates/kuttl/smoke/test_nifi.py +++ b/tests/templates/kuttl/smoke/test_nifi.py @@ -42,7 +42,7 @@ def get_token(nifi_host, username, password): # disable warnings as we have specified non-verified https connections urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - host = f"https://test-nifi-node-default-1.test-nifi-node-default.{args['namespace']}.svc.cluster.local:8443" + host = f"https://nifi-node-default-1.nifi-node-default.{args['namespace']}.svc.cluster.local:8443" token = get_token(host, args['user'], args['password']) headers = {'Authorization': token} node_count = int(args['count']) diff --git a/tests/templates/kuttl/smoke/test_nifi_metrics.py b/tests/templates/kuttl/smoke/test_nifi_metrics.py index 86076038..2b0682e1 100755 --- a/tests/templates/kuttl/smoke/test_nifi_metrics.py +++ b/tests/templates/kuttl/smoke/test_nifi_metrics.py @@ -23,7 +23,7 @@ port = args["port"] timeout = int(args["timeout"]) - url = f"http://test-nifi-node-default-0.test-nifi-node-default.{namespace}.svc.cluster.local:{port}/metrics" + url = f"http://nifi-node-default-0.nifi-node-default.{namespace}.svc.cluster.local:{port}/metrics" # wait for 'timeout' seconds t_end = time.time() + timeout diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index edc5b103..29dbade1 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -13,22 +13,36 @@ dimensions: - 1.27.0 - name: nifi_new values: - - 2.4.0 - # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version - # as in the example below. - # - 2.4.0,oci.stackable.tech/sandbox/nifi:2.4.0-stackable0.0.0-dev + - 2.4.0 # oci.stackable.tech/sandbox/nifi:2.4.0-stackable0.0.0-dev - name: nifi-latest values: - - 2.4.0 - # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version - # as in the example below. - # - 2.4.0,oci.stackable.tech/sandbox/nifi:2.4.0-stackable0.0.0-dev + - 2.4.0 # oci.stackable.tech/sandbox/nifi:2.4.0-stackable0.0.0-dev + - name: nifi-iceberg + # Not all NiFi versions support Iceberg with the same functionality! + # E.g. our own implementation started with NiFi 2.2.0 + values: + - 2.4.0 # oci.stackable.tech/sandbox/nifi:2.4.0-stackable0.0.0-dev - name: zookeeper values: - 3.9.3 - name: zookeeper-latest values: - 3.9.3 + - name: opa-l + values: + - 1.4.2 + - name: hdfs-l + values: + - 3.4.1 + - name: hive-l + values: + - 4.0.1 + - name: trino-l + values: + - "470" + - name: krb5 + values: + - 1.21.1 - name: ldap-use-tls values: - "false" @@ -37,6 +51,13 @@ dimensions: values: - "false" - "true" + - name: iceberg-use-kerberos + values: + - "false" + - "true" + - name: kerberos-realm + values: + - "PROD.MYCORP" - name: openshift values: - "false" @@ -95,6 +116,18 @@ tests: - nifi - zookeeper-latest - openshift + - name: iceberg + dimensions: + - nifi-iceberg + - opa-l + - zookeeper-latest + - hdfs-l + - hive-l + - trino-l + - krb5 + - iceberg-use-kerberos + - kerberos-realm + - openshift suites: - name: nightly patch: From 1b8039731dc4858d211737cdb730374ee1190aa0 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 23 May 2025 17:23:23 +0200 Subject: [PATCH 07/37] wip: update listener implementation based on implementation for superset operator --- deploy/helm/nifi-operator/crds/crds.yaml | 8 - rust/operator-binary/src/controller.rs | 182 ++++++++++++------ rust/operator-binary/src/crd/mod.rs | 159 ++------------- rust/operator-binary/src/crd/utils.rs | 47 +---- .../15_listener_classes.yaml.j2 | 21 ++ .../external-access/30-install-nifi.yaml.j2 | 34 +--- tests/test-definition.yaml | 4 +- 7 files changed, 160 insertions(+), 295 deletions(-) create mode 100644 tests/templates/kuttl/external-access/15_listener_classes.yaml.j2 diff --git a/deploy/helm/nifi-operator/crds/crds.yaml b/deploy/helm/nifi-operator/crds/crds.yaml index cef05d1d..85addf9b 100644 --- a/deploy/helm/nifi-operator/crds/crds.yaml +++ b/deploy/helm/nifi-operator/crds/crds.yaml @@ -262,10 +262,6 @@ spec: type: string listenerClass: description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. - enum: - - cluster-internal - - external-unstable - - external-stable nullable: true type: string logging: @@ -757,10 +753,6 @@ spec: type: string listenerClass: description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. - enum: - - cluster-internal - - external-unstable - - external-stable nullable: true type: string logging: diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 4d2baa79..c8c2d1a3 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -31,7 +31,9 @@ use stackable_operator::{ cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, commons::{ authentication::oidc::AuthenticationProvider, - product_image_selection::ResolvedProductImage, rbac::build_rbac_resources, + listener::{Listener, ListenerPort, ListenerSpec}, + product_image_selection::ResolvedProductImage, + rbac::build_rbac_resources, }, config::fragment, k8s_openapi::{ @@ -86,8 +88,8 @@ use crate::{ APP_NAME, BALANCE_PORT, BALANCE_PORT_NAME, Container, HTTPS_PORT, HTTPS_PORT_NAME, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, METRICS_PORT, METRICS_PORT_NAME, NifiConfig, NifiConfigFragment, NifiRole, NifiStatus, PROTOCOL_PORT, PROTOCOL_PORT_NAME, - STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, SupportedListenerClasses, - authentication::AuthenticationClassResolved, v1alpha1, + STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, authentication::AuthenticationClassResolved, + v1alpha1, }, operations::{ graceful_shutdown::add_graceful_shutdown_config, @@ -360,6 +362,12 @@ pub enum Error { BuildListenerVolume { source: ListenerOperatorVolumeSourceBuilderError, }, + + #[snafu(display("failed to apply group listener for {rolegroup}"))] + ApplyGroupListener { + source: stackable_operator::cluster_resources::Error, + rolegroup: RoleGroupRef, + }, } type Result = std::result::Result; @@ -499,7 +507,8 @@ pub async fn reconcile_nifi( // Since we cannot predict which of the addresses a user might decide to use we will simply // add all of them to the setting for now. // For more information see - let proxy_hosts = get_proxy_hosts(client, nifi, &merged_config).await?; + // let proxy_hosts = get_proxy_hosts(client, nifi, &merged_config).await?; + let proxy_hosts = get_proxy_hosts(client, nifi).await?; let rg_configmap = build_node_rolegroup_config_map( nifi, @@ -536,6 +545,19 @@ pub async fn reconcile_nifi( ) .await?; + let rg_group_listener = build_group_listener( + nifi, + &resolved_product_image, + &rolegroup, + merged_config.listener_class, + )?; + + cluster_resources + .add(client, rg_group_listener) + .await + .with_context(|_| ApplyGroupListenerSnafu { + rolegroup: rolegroup.clone(), + })?; cluster_resources .add(client, rg_service) .await @@ -752,49 +774,90 @@ fn build_node_rolegroup_service( resolved_product_image: &ResolvedProductImage, rolegroup: &RoleGroupRef, ) -> Result { - Ok(Service { - metadata: ObjectMetaBuilder::new() - .name_and_namespace(nifi) - .name(rolegroup.object_name()) - .ownerreference_from_resource(nifi, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(build_recommended_labels( - nifi, - &resolved_product_image.app_version_label, - &rolegroup.role, - &rolegroup.role_group, - )) - .context(MetadataBuildSnafu)? - .with_label(Label::try_from(("prometheus.io/scrape", "true")).context(LabelBuildSnafu)?) - .build(), - spec: Some(ServiceSpec { - // Internal communication does not need to be exposed - type_: Some("ClusterIP".to_string()), - cluster_ip: Some("None".to_string()), - ports: Some(vec![ - ServicePort { - name: Some(HTTPS_PORT_NAME.to_string()), - port: HTTPS_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }, - ServicePort { - name: Some(METRICS_PORT_NAME.to_string()), - port: METRICS_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }, - ]), - selector: Some( - Labels::role_group_selector(nifi, APP_NAME, &rolegroup.role, &rolegroup.role_group) - .context(LabelBuildSnafu)? - .into(), - ), - publish_not_ready_addresses: Some(true), - ..ServiceSpec::default() - }), + let metadata = ObjectMetaBuilder::new() + .name_and_namespace(nifi) + .name(format!("{name}-metrics", name = rolegroup.object_name())) + .ownerreference_from_resource(nifi, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(build_recommended_labels( + nifi, + &resolved_product_image.app_version_label, + &rolegroup.role, + &rolegroup.role_group, + )) + .context(MetadataBuildSnafu)? + .with_label(Label::try_from(("prometheus.io/scrape", "true")).context(LabelBuildSnafu)?) + .build(); + + let spec = Some(ServiceSpec { + // Internal communication does not need to be exposed + type_: Some("ClusterIP".to_owned()), + cluster_ip: Some("None".to_owned()), + ports: Some(vec![ServicePort { + name: Some(METRICS_PORT_NAME.to_owned()), + port: METRICS_PORT.into(), + protocol: Some("TCP".to_owned()), + ..ServicePort::default() + }]), + selector: Some( + Labels::role_group_selector(nifi, APP_NAME, &rolegroup.role, &rolegroup.role_group) + .context(LabelBuildSnafu)? + .into(), + ), + publish_not_ready_addresses: Some(true), + ..ServiceSpec::default() + }); + + let service = Service { + metadata, + spec, status: None, - }) + }; + + Ok(service) +} + +pub fn build_group_listener( + nifi: &v1alpha1::NifiCluster, + resolved_product_image: &ResolvedProductImage, + rolegroup: &RoleGroupRef, + listener_class: String, +) -> Result { + let metadata = ObjectMetaBuilder::new() + .name_and_namespace(nifi) + .name(nifi.group_listener_name(rolegroup)) + .ownerreference_from_resource(nifi, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(build_recommended_labels( + nifi, + &resolved_product_image.app_version_label, + &rolegroup.role, + &rolegroup.role_group, + )) + .context(MetadataBuildSnafu)? + .build(); + + let spec = ListenerSpec { + class_name: Some(listener_class), + ports: Some(listener_ports()), + ..Default::default() + }; + + let listener = Listener { + metadata, + spec, + status: None, + }; + + Ok(listener) +} + +fn listener_ports() -> Vec { + vec![ListenerPort { + name: HTTPS_PORT_NAME.to_owned(), + port: HTTPS_PORT.into(), + protocol: Some("TCP".to_owned()), + }] } const USERDATA_MOUNTPOINT: &str = "/stackable/userdata"; @@ -921,16 +984,16 @@ async fn build_node_rolegroup_statefulset( .as_slice(), ); - if merged_config.listener_class == SupportedListenerClasses::ExternalUnstable { - prepare_args.extend(vec![ - "export LISTENER_DEFAULT_ADDRESS=$(cat /stackable/listener/default-address/address)" - .to_string(), - ]); - prepare_args.extend(vec![ + // if merged_config.listener_class == SupportedListenerClasses::ExternalUnstable { + prepare_args.extend(vec![ + "export LISTENER_DEFAULT_ADDRESS=$(cat /stackable/listener/default-address/address)" + .to_string(), + ]); + prepare_args.extend(vec![ "export LISTENER_DEFAULT_PORT_HTTPS=$(cat /stackable/listener/default-address/ports/https)" .to_string(), - ]); - } + ]); + // } prepare_args.extend(vec![ "echo Templating config files".to_string(), @@ -1405,7 +1468,7 @@ fn zookeeper_env_var(name: &str, configmap_name: &str) -> EnvVar { async fn get_proxy_hosts( client: &Client, nifi: &v1alpha1::NifiCluster, - merged_config: &NifiConfig, + // merged_config: &NifiConfig, ) -> Result { let host_header_check = nifi.spec.cluster_config.host_header_check.clone(); @@ -1438,11 +1501,10 @@ async fn get_proxy_hosts( proxy_hosts_set.extend(host_header_check.additional_allowed_hosts); // If NodePort is used inject the address and port from the listener volume in the prepare container - if merged_config.listener_class == SupportedListenerClasses::ExternalUnstable { - proxy_hosts_set.insert( - "${env:LISTENER_DEFAULT_ADDRESS}:${env:LISTENER_DEFAULT_PORT_HTTPS}".to_string(), - ); - } + // if merged_config.listener_class == SupportedListenerClasses::ExternalUnstable { + proxy_hosts_set + .insert("${env:LISTENER_DEFAULT_ADDRESS}:${env:LISTENER_DEFAULT_PORT_HTTPS}".to_string()); + // } let mut proxy_hosts = Vec::from_iter(proxy_hosts_set); proxy_hosts.sort(); diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 270b5902..5bb84b06 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -21,7 +21,7 @@ use stackable_operator::{ }, config::{ fragment::{self, Fragment, ValidationError}, - merge::{Atomic, Merge}, + merge::Merge, }, k8s_openapi::{ api::core::v1::{PodTemplateSpec, Volume}, @@ -41,9 +41,8 @@ use stackable_operator::{ }, versioned::versioned, }; -use strum::Display; use tls::NifiTls; -use utils::{PodRef, get_listener_podrefs}; +use utils::PodRef; pub const APP_NAME: &str = "nifi"; @@ -178,6 +177,13 @@ impl HasStatusCondition for v1alpha1::NifiCluster { } impl v1alpha1::NifiCluster { + /// The name of the group-listener provided for a specific role-group. + /// The UI will use this group listener so that only one load balancer + /// is needed (per role group). + pub fn group_listener_name(&self, rolegroup: &RoleGroupRef) -> String { + rolegroup.object_name() + } + /// The name of the role-level load-balanced Kubernetes `Service` pub fn node_role_service_name(&self) -> String { self.name_any() @@ -273,100 +279,6 @@ impl v1alpha1::NifiCluster { tracing::debug!("Merged config: {:?}", conf_rolegroup); fragment::validate(conf_rolegroup).context(FragmentValidationFailureSnafu) } - - pub fn merged_listener_class( - &self, - rolegroup_name: &String, - ) -> Result, Error> { - let listener_class_default = Some(SupportedListenerClasses::ClusterInternal); - let role = self.spec.nodes.as_ref().context(NoNodesDefinedSnafu)?; - - let mut listener_class_role = role.config.config.listener_class.to_owned(); - let mut listener_class_rolegroup = role - .role_groups - .get(rolegroup_name) - .map(|rg| rg.config.config.listener_class.clone()) - .unwrap_or_default(); - listener_class_role.merge(&listener_class_default); - listener_class_rolegroup.merge(&listener_class_role); - tracing::debug!("Merged listener-class: {:?}", listener_class_rolegroup); - Ok(listener_class_rolegroup) - } - - pub fn rolegroup_ref( - &self, - role_name: impl Into, - group_name: impl Into, - ) -> RoleGroupRef { - RoleGroupRef { - cluster: ObjectRef::from_obj(self), - role: role_name.into(), - role_group: group_name.into(), - } - } - - pub fn rolegroup_ref_and_replicas(&self) -> Vec<(RoleGroupRef, u16)> { - self.spec - .nodes - .iter() - .flat_map(|role| &role.role_groups) - // Order rolegroups consistently, to avoid spurious downstream rewrites - .collect::>() - .into_iter() - .filter(|(rolegroup_name, _)| self.resolved_listener_class_discoverable(rolegroup_name)) - .map(|(rolegroup_name, role_group)| { - ( - self.rolegroup_ref(NifiRole::Node.to_string(), rolegroup_name), - role_group.replicas.unwrap_or_default(), - ) - }) - .collect() - } - - fn resolved_listener_class_discoverable(&self, rolegroup_name: &&String) -> bool { - if let Ok(Some(listener_class)) = self.merged_listener_class(rolegroup_name) { - listener_class.discoverable() - } else { - // merged_listener_class returns an error if one of the roles was not found: - // all roles are mandatory for airflow to work, but a missing role will by - // definition not have a listener class - false - } - } - - pub fn pod_refs(&self) -> Result, Error> { - let ns = self.metadata.namespace.clone().context(NoNamespaceSnafu)?; - let rolegroup_ref_and_replicas = self.rolegroup_ref_and_replicas(); - - Ok(rolegroup_ref_and_replicas - .iter() - .flat_map(|(rolegroup_ref, replicas)| { - let ns = ns.clone(); - (0..*replicas).map(move |i| PodRef { - namespace: ns.clone(), - role_group_service_name: rolegroup_ref.object_name(), - pod_name: format!("{}-{}", rolegroup_ref.object_name(), i), - ports: HashMap::from([ - (HTTPS_PORT_NAME.to_owned(), HTTPS_PORT), - (METRICS_PORT_NAME.to_owned(), METRICS_PORT), - ]), - fqdn_override: None, - }) - }) - .collect()) - } - - pub async fn listener_refs( - &self, - client: &stackable_operator::client::Client, - ) -> Result, Error> { - let pod_refs = self.pod_refs()?; - - tracing::debug!("Pod references: {:#?}", pod_refs); - get_listener_podrefs(client, pod_refs, LISTENER_VOLUME_NAME) - .await - .context(ListenerPodRefSnafu) - } } #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] @@ -394,55 +306,6 @@ pub fn default_allow_all() -> bool { true } -// // TODO: Temporary solution until listener-operator is finished -// #[derive(Clone, Debug, Default, Display, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] -// #[serde(rename_all = "PascalCase")] -// pub enum CurrentlySupportedListenerClasses { -// #[default] -// #[serde(rename = "cluster-internal")] -// ClusterInternal, -// #[serde(rename = "external-unstable")] -// ExternalUnstable, -// } - -// impl CurrentlySupportedListenerClasses { -// pub fn k8s_service_type(&self) -> String { -// match self { -// CurrentlySupportedListenerClasses::ClusterInternal => "ClusterIP".to_string(), -// CurrentlySupportedListenerClasses::ExternalUnstable => "NodePort".to_string(), -// } -// } -// } - -#[derive(Clone, Debug, Default, Display, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "PascalCase")] -pub enum SupportedListenerClasses { - #[default] - #[serde(rename = "cluster-internal")] - #[strum(serialize = "cluster-internal")] - ClusterInternal, - - #[serde(rename = "external-unstable")] - #[strum(serialize = "external-unstable")] - ExternalUnstable, - - #[serde(rename = "external-stable")] - #[strum(serialize = "external-stable")] - ExternalStable, -} - -impl Atomic for SupportedListenerClasses {} - -impl SupportedListenerClasses { - pub fn discoverable(&self) -> bool { - match self { - SupportedListenerClasses::ClusterInternal => false, - SupportedListenerClasses::ExternalUnstable => true, - SupportedListenerClasses::ExternalStable => true, - } - } -} - /// These settings configure the encryption of sensitive properties in NiFi processors. /// NiFi supports encrypting sensitive properties in processors as they are written to disk. /// You can configure the encryption algorithm and the key to use. @@ -633,7 +496,7 @@ pub struct NifiConfig { /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose the webserver. #[serde(default)] - pub listener_class: SupportedListenerClasses, + pub listener_class: String, } impl NifiConfig { @@ -683,7 +546,7 @@ impl NifiConfig { affinity: get_affinity(cluster_name, role), graceful_shutdown_timeout: Some(DEFAULT_NODE_GRACEFUL_SHUTDOWN_TIMEOUT), requested_secret_lifetime: Some(Self::DEFAULT_NODE_SECRET_LIFETIME), - listener_class: Some(SupportedListenerClasses::ClusterInternal), + listener_class: Some("cluster-internal".to_owned()), } } } diff --git a/rust/operator-binary/src/crd/utils.rs b/rust/operator-binary/src/crd/utils.rs index a54cd0bf..4e651160 100644 --- a/rust/operator-binary/src/crd/utils.rs +++ b/rust/operator-binary/src/crd/utils.rs @@ -1,7 +1,6 @@ use std::{borrow::Cow, collections::HashMap, num::TryFromIntError}; -use futures::future::try_join_all; -use snafu::{OptionExt, ResultExt, Snafu}; +use snafu::Snafu; use stackable_operator::{ commons::listener::Listener, k8s_openapi::api::core::v1::Pod, kube::runtime::reflector::ObjectRef, utils::cluster_info::KubernetesClusterInfo, @@ -58,47 +57,3 @@ impl PodRef { ) } } - -pub async fn get_listener_podrefs( - client: &stackable_operator::client::Client, - pod_refs: Vec, - listener_prefix: &str, -) -> Result, Error> { - try_join_all(pod_refs.into_iter().map(|pod_ref| async { - // N.B. use the naming convention for ephemeral listener volumes as we - // have defined all listeners to be so. - let listener_name = format!("{}-{listener_prefix}", pod_ref.pod_name); - let listener_ref = || ObjectRef::::new(&listener_name).within(&pod_ref.namespace); - let pod_obj_ref = || ObjectRef::::new(&pod_ref.pod_name).within(&pod_ref.namespace); - let listener = client - .get::(&listener_name, &pod_ref.namespace) - .await - .context(GetPodListenerSnafu { - listener: listener_ref(), - pod: pod_obj_ref(), - })?; - let listener_address = listener - .status - .and_then(|s| s.ingress_addresses?.into_iter().next()) - .context(PodListenerHasNoAddressSnafu { - listener: listener_ref(), - pod: pod_obj_ref(), - })?; - Ok(PodRef { - fqdn_override: Some(listener_address.address), - ports: listener_address - .ports - .into_iter() - .map(|(port_name, port)| { - let port = u16::try_from(port).context(PortOutOfBoundsSnafu { - port_name: &port_name, - port, - })?; - Ok((port_name, port)) - }) - .collect::>()?, - ..pod_ref - }) - })) - .await -} diff --git a/tests/templates/kuttl/external-access/15_listener_classes.yaml.j2 b/tests/templates/kuttl/external-access/15_listener_classes.yaml.j2 new file mode 100644 index 00000000..4131526a --- /dev/null +++ b/tests/templates/kuttl/external-access/15_listener_classes.yaml.j2 @@ -0,0 +1,21 @@ +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: ListenerClass +metadata: + name: test-cluster-internal-$NAMESPACE +spec: + serviceType: ClusterIP +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: ListenerClass +metadata: + name: test-external-stable-$NAMESPACE +spec: + serviceType: NodePort +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: ListenerClass +metadata: + name: test-external-unstable-$NAMESPACE +spec: + serviceType: NodePort diff --git a/tests/templates/kuttl/external-access/30-install-nifi.yaml.j2 b/tests/templates/kuttl/external-access/30-install-nifi.yaml.j2 index 5838a12f..584fc564 100644 --- a/tests/templates/kuttl/external-access/30-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/external-access/30-install-nifi.yaml.j2 @@ -51,46 +51,18 @@ spec: vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} nodes: - envOverrides: - COMMON_VAR: role-value # overridden by role group below - ROLE_VAR: role-value # only defined here at role level - configOverrides: - "nifi.properties": - "nifi.diagnostics.on.shutdown.enabled": "true" - "nifi.diagnostics.on.shutdown.verbose": "false" config: + listenerClass: test-external-stable-$NAMESPACE logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: default: replicas: 2 - envOverrides: - COMMON_VAR: group-value # overrides role value - GROUP_VAR: group-value # only defined here at group level - configOverrides: - "nifi.properties": - "nifi.diagnostics.on.shutdown.enabled": "false" - "nifi.diagnostics.on.shutdown.max.filecount": "20" - roleGroups: external-unstable: replicas: 1 config: - listenerClass: external-unstable - envOverrides: - COMMON_VAR: group-value # overrides role value - GROUP_VAR: group-value # only defined here at group level - configOverrides: - "nifi.properties": - "nifi.diagnostics.on.shutdown.enabled": "false" - "nifi.diagnostics.on.shutdown.max.filecount": "20" + listenerClass: test-external-unstable-$NAMESPACE cluster-internal: replicas: 1 config: - listenerClass: cluster-internal - envOverrides: - COMMON_VAR: group-value # overrides role value - GROUP_VAR: group-value # only defined here at group level - configOverrides: - "nifi.properties": - "nifi.diagnostics.on.shutdown.enabled": "false" - "nifi.diagnostics.on.shutdown.max.filecount": "20" + listenerClass: test-cluster-internal-$NAMESPACE diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 29dbade1..3308f418 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -2,8 +2,8 @@ dimensions: - name: nifi values: - - 1.27.0 - - 1.28.1 + # - 1.27.0 + # - 1.28.1 - 2.4.0 # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version # as in the example below. From fcb0c4a063e425d326e09ff52b59b77ec03ff12c Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 3 Jun 2025 16:27:49 +0200 Subject: [PATCH 08/37] use single listener with pvc per rolegroup --- deploy/helm/nifi-operator/crds/crds.yaml | 4 +- .../helm/nifi-operator/templates/roles.yaml | 5 ++ rust/operator-binary/src/controller.rs | 45 ++++++++++-------- rust/operator-binary/src/crd/mod.rs | 2 +- .../15-create-listener-classes.yaml | 5 ++ ...asses.yaml.j2 => 15_listener-classes.yaml} | 0 .../kuttl/external-access/30-assert.yaml | 47 +++++++++++++++++-- .../kuttl/external-access/30-assert.yaml.j2 | 10 ---- .../external-access/30-install-nifi.yaml | 5 ++ ...0-install-nifi.yaml.j2 => 30_nifi.yaml.j2} | 1 - 10 files changed, 86 insertions(+), 38 deletions(-) create mode 100644 tests/templates/kuttl/external-access/15-create-listener-classes.yaml rename tests/templates/kuttl/external-access/{15_listener_classes.yaml.j2 => 15_listener-classes.yaml} (100%) delete mode 100644 tests/templates/kuttl/external-access/30-assert.yaml.j2 create mode 100644 tests/templates/kuttl/external-access/30-install-nifi.yaml rename tests/templates/kuttl/external-access/{30-install-nifi.yaml.j2 => 30_nifi.yaml.j2} (96%) diff --git a/deploy/helm/nifi-operator/crds/crds.yaml b/deploy/helm/nifi-operator/crds/crds.yaml index dd3abe31..b920603e 100644 --- a/deploy/helm/nifi-operator/crds/crds.yaml +++ b/deploy/helm/nifi-operator/crds/crds.yaml @@ -363,7 +363,7 @@ spec: nullable: true type: string listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the nodes. nullable: true type: string logging: @@ -854,7 +854,7 @@ spec: nullable: true type: string listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the nodes. nullable: true type: string logging: diff --git a/deploy/helm/nifi-operator/templates/roles.yaml b/deploy/helm/nifi-operator/templates/roles.yaml index b615d5a2..0e0e61fa 100644 --- a/deploy/helm/nifi-operator/templates/roles.yaml +++ b/deploy/helm/nifi-operator/templates/roles.yaml @@ -96,6 +96,11 @@ rules: - listeners verbs: - get + - list + - watch + - patch + - create + - delete - apiGroups: - {{ include "operator.name" . }}.stackable.tech resources: diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index fb3f4504..7038b475 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -24,7 +24,10 @@ use stackable_operator::{ container::ContainerBuilder, resources::ResourceRequirementsBuilder, security::PodSecurityContextBuilder, - volume::{ListenerOperatorVolumeSourceBuilderError, SecretFormat}, + volume::{ + ListenerOperatorVolumeSourceBuilder, ListenerOperatorVolumeSourceBuilderError, + ListenerReference, SecretFormat, + }, }, }, client::Client, @@ -1050,7 +1053,6 @@ async fn build_node_rolegroup_statefulset( .as_slice(), ); - // if merged_config.listener_class == SupportedListenerClasses::ExternalUnstable { prepare_args.extend(vec![ "export LISTENER_DEFAULT_ADDRESS=$(cat /stackable/listener/default-address/address)" .to_string(), @@ -1059,7 +1061,6 @@ async fn build_node_rolegroup_statefulset( "export LISTENER_DEFAULT_PORT_HTTPS=$(cat /stackable/listener/default-address/ports/https)" .to_string(), ]); - // } prepare_args.extend(vec![ "echo Templating config files".to_string(), @@ -1237,20 +1238,27 @@ async fn build_node_rolegroup_statefulset( &rolegroup_ref.role, &rolegroup_ref.role_group, ); - let recommended_labels = - Labels::recommended(recommended_object_labels.clone()).context(LabelBuildSnafu)?; - let listener_class = &merged_config.listener_class; - // all listeners will use ephemeral volumes as they can/should - // be removed when the pods are *terminated* (ephemeral PVCs will - // survive re-starts) - pod_builder - .add_listener_volume_by_listener_class( - LISTENER_VOLUME_NAME, - &listener_class.to_string(), - &recommended_labels, - ) - .context(AddVolumeSnafu)?; + // Used for PVC templates that cannot be modified once they are deployed + let unversioned_recommended_labels = Labels::recommended(build_recommended_labels( + nifi, + // A version value is required, and we do want to use the "recommended" format for the other desired labels + "none", + &rolegroup_ref.role, + &rolegroup_ref.role_group, + )) + .context(LabelBuildSnafu)?; + + // listener endpoints will use persistent volumes + // so that load balancers can hard-code the target addresses and + // that it is possible to connect to a consistent address + let listener_pvc = ListenerOperatorVolumeSourceBuilder::new( + &ListenerReference::ListenerName(nifi.group_listener_name(rolegroup_ref)), + &unversioned_recommended_labels, + ) + .context(BuildListenerVolumeSnafu)? + .build_pvc(LISTENER_VOLUME_NAME.to_owned()) + .context(BuildListenerVolumeSnafu)?; add_graceful_shutdown_config(merged_config, &mut pod_builder).context(GracefulShutdownSnafu)?; @@ -1524,6 +1532,7 @@ async fn build_node_rolegroup_statefulset( &NifiRepository::State.repository(), Some(vec!["ReadWriteOnce"]), ), + listener_pvc, ]), ..StatefulSetSpec::default() }), @@ -1566,11 +1575,9 @@ async fn get_proxy_hosts( proxy_hosts_set.extend(host_header_check.additional_allowed_hosts); - // If NodePort is used inject the address and port from the listener volume in the prepare container - // if merged_config.listener_class == SupportedListenerClasses::ExternalUnstable { + // Inject the address and port from the listener volume during the prepare container proxy_hosts_set .insert("${env:LISTENER_DEFAULT_ADDRESS}:${env:LISTENER_DEFAULT_PORT_HTTPS}".to_string()); - // } let mut proxy_hosts = Vec::from_iter(proxy_hosts_set); proxy_hosts.sort(); diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index c6ae33b5..3635176d 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -477,7 +477,7 @@ pub struct NifiConfig { #[fragment_attrs(serde(default))] pub requested_secret_lifetime: Option, - /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose the webserver. + /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose the nodes. #[serde(default)] pub listener_class: String, } diff --git a/tests/templates/kuttl/external-access/15-create-listener-classes.yaml b/tests/templates/kuttl/external-access/15-create-listener-classes.yaml new file mode 100644 index 00000000..0fad7e51 --- /dev/null +++ b/tests/templates/kuttl/external-access/15-create-listener-classes.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: envsubst < 15_listener-classes.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/external-access/15_listener_classes.yaml.j2 b/tests/templates/kuttl/external-access/15_listener-classes.yaml similarity index 100% rename from tests/templates/kuttl/external-access/15_listener_classes.yaml.j2 rename to tests/templates/kuttl/external-access/15_listener-classes.yaml diff --git a/tests/templates/kuttl/external-access/30-assert.yaml b/tests/templates/kuttl/external-access/30-assert.yaml index 32f2b7cc..c71ac818 100644 --- a/tests/templates/kuttl/external-access/30-assert.yaml +++ b/tests/templates/kuttl/external-access/30-assert.yaml @@ -1,20 +1,36 @@ --- apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 1200 +metadata: + name: install-nifi +timeout: 300 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available=true nificlusters.nifi.stackable.tech/nifi --timeout 301s --- apiVersion: apps/v1 kind: StatefulSet metadata: name: test-nifi-node-default -spec: - template: - spec: - terminationGracePeriodSeconds: 300 status: readyReplicas: 2 replicas: 2 --- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-nifi-node-external-unstable +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-nifi-node-cluster-internal +status: + readyReplicas: 1 + replicas: 1 +--- apiVersion: policy/v1 kind: PodDisruptionBudget metadata: @@ -23,3 +39,24 @@ status: expectedPods: 4 currentHealthy: 4 disruptionsAllowed: 1 +--- +apiVersion: v1 +kind: Service +metadata: + name: test-nifi-node-cluster-internal +spec: + type: ClusterIP # cluster-internal +--- +apiVersion: v1 +kind: Service +metadata: + name: test-nifi-node-default +spec: + type: NodePort # external-stable +--- +apiVersion: v1 +kind: Service +metadata: + name: test-nifi-node-external-unstable +spec: + type: NodePort # external-unstable diff --git a/tests/templates/kuttl/external-access/30-assert.yaml.j2 b/tests/templates/kuttl/external-access/30-assert.yaml.j2 deleted file mode 100644 index 50b1d4c3..00000000 --- a/tests/templates/kuttl/external-access/30-assert.yaml.j2 +++ /dev/null @@ -1,10 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -{% if lookup('env', 'VECTOR_AGGREGATOR') %} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: vector-aggregator-discovery -{% endif %} diff --git a/tests/templates/kuttl/external-access/30-install-nifi.yaml b/tests/templates/kuttl/external-access/30-install-nifi.yaml new file mode 100644 index 00000000..e7126be2 --- /dev/null +++ b/tests/templates/kuttl/external-access/30-install-nifi.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: envsubst < 30_nifi.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/external-access/30-install-nifi.yaml.j2 b/tests/templates/kuttl/external-access/30_nifi.yaml.j2 similarity index 96% rename from tests/templates/kuttl/external-access/30-install-nifi.yaml.j2 rename to tests/templates/kuttl/external-access/30_nifi.yaml.j2 index 584fc564..c1d6c74e 100644 --- a/tests/templates/kuttl/external-access/30-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/external-access/30_nifi.yaml.j2 @@ -40,7 +40,6 @@ spec: pullPolicy: IfNotPresent clusterConfig: zookeeperConfigMapName: test-nifi-znode - listenerClass: {{ test_scenario['values']['listener-class'] }} authentication: - authenticationClass: simple-nifi-users hostHeaderCheck: From b5202ff191770bdac111c93eac8290ce7238b168 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 3 Jun 2025 17:22:23 +0200 Subject: [PATCH 09/37] update integration tests with crd change --- .../nifi/examples/getting_started/getting_started.sh | 4 +++- .../nifi/examples/getting_started/getting_started.sh.j2 | 3 ++- docs/modules/nifi/pages/usage_guide/custom-components.adoc | 6 ++++-- docs/modules/nifi/pages/usage_guide/index.adoc | 3 ++- examples/simple-nifi-cluster.yaml | 2 +- .../custom-components-git-sync/30-install-nifi.yaml.j2 | 2 +- tests/templates/kuttl/external-access/30_nifi.yaml.j2 | 2 +- tests/templates/kuttl/iceberg/interactive-nifi.yaml | 3 ++- tests/templates/kuttl/ldap/12-install-nifi.yaml.j2 | 2 +- tests/templates/kuttl/oidc-opa/30_nifi.yaml.j2 | 2 +- tests/templates/kuttl/smoke_v1/30-install-nifi.yaml.j2 | 2 +- tests/templates/kuttl/smoke_v2/30-install-nifi.yaml.j2 | 2 +- 12 files changed, 20 insertions(+), 13 deletions(-) diff --git a/docs/modules/nifi/examples/getting_started/getting_started.sh b/docs/modules/nifi/examples/getting_started/getting_started.sh index 3aed9fe5..d6caf6a1 100755 --- a/docs/modules/nifi/examples/getting_started/getting_started.sh +++ b/docs/modules/nifi/examples/getting_started/getting_started.sh @@ -143,15 +143,17 @@ spec: clusterConfig: authentication: - authenticationClass: simple-nifi-users - listenerClass: external-unstable sensitiveProperties: keySecret: nifi-sensitive-property-key autoGenerate: true zookeeperConfigMapName: simple-nifi-znode nodes: + config: + listenerClass: external-unstable roleGroups: default: replicas: 1 + EOF # end::install-nifi[] diff --git a/docs/modules/nifi/examples/getting_started/getting_started.sh.j2 b/docs/modules/nifi/examples/getting_started/getting_started.sh.j2 index 668eb772..012ce15c 100755 --- a/docs/modules/nifi/examples/getting_started/getting_started.sh.j2 +++ b/docs/modules/nifi/examples/getting_started/getting_started.sh.j2 @@ -143,12 +143,13 @@ spec: clusterConfig: authentication: - authenticationClass: simple-nifi-users - listenerClass: external-unstable sensitiveProperties: keySecret: nifi-sensitive-property-key autoGenerate: true zookeeperConfigMapName: simple-nifi-znode nodes: + config: + listenerClass: external-unstable roleGroups: default: replicas: 1 diff --git a/docs/modules/nifi/pages/usage_guide/custom-components.adoc b/docs/modules/nifi/pages/usage_guide/custom-components.adoc index 210aa925..3f6a3c2e 100644 --- a/docs/modules/nifi/pages/usage_guide/custom-components.adoc +++ b/docs/modules/nifi/pages/usage_guide/custom-components.adoc @@ -162,12 +162,13 @@ spec: - name: nifi-processors configMap: name: nifi-processors - listenerClass: external-unstable sensitiveProperties: keySecret: nifi-sensitive-property-key autoGenerate: true zookeeperConfigMapName: simple-nifi-znode nodes: + config: + listenerClass: external-unstable configOverrides: nifi.properties: nifi.nar.library.directory.myCustomLibs: /stackable/userdata/nifi-processors/ # <2> @@ -281,12 +282,13 @@ spec: - name: nifi-processors persistentVolumeClaim: claimName: nifi-processors - listenerClass: external-unstable sensitiveProperties: keySecret: nifi-sensitive-property-key autoGenerate: true zookeeperConfigMapName: simple-nifi-znode nodes: + config: + listenerClass: external-unstable configOverrides: nifi.properties: nifi.nar.library.directory.myCustomLibs: /stackable/userdata/nifi-processors/ # <2> diff --git a/docs/modules/nifi/pages/usage_guide/index.adoc b/docs/modules/nifi/pages/usage_guide/index.adoc index 13dea386..419b4f5a 100644 --- a/docs/modules/nifi/pages/usage_guide/index.adoc +++ b/docs/modules/nifi/pages/usage_guide/index.adoc @@ -26,11 +26,12 @@ spec: - name: nifi-client-certs secret: secretName: nifi-client-certs - listenerClass: external-unstable sensitiveProperties: keySecret: nifi-sensitive-property-key autoGenerate: true nodes: + config: + listenerClass: external-unstable roleGroups: default: config: diff --git a/examples/simple-nifi-cluster.yaml b/examples/simple-nifi-cluster.yaml index b084efc5..ca084c80 100644 --- a/examples/simple-nifi-cluster.yaml +++ b/examples/simple-nifi-cluster.yaml @@ -51,13 +51,13 @@ spec: clusterConfig: authentication: - authenticationClass: simple-nifi-admin-user - listenerClass: external-unstable sensitiveProperties: keySecret: nifi-sensitive-property-key autoGenerate: true zookeeperConfigMapName: simple-nifi-znode nodes: config: + listenerClass: external-unstable roleGroups: default: replicas: 1 diff --git a/tests/templates/kuttl/custom-components-git-sync/30-install-nifi.yaml.j2 b/tests/templates/kuttl/custom-components-git-sync/30-install-nifi.yaml.j2 index a6153bd6..16a92f1a 100644 --- a/tests/templates/kuttl/custom-components-git-sync/30-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/custom-components-git-sync/30-install-nifi.yaml.j2 @@ -30,7 +30,6 @@ spec: {% endif %} pullPolicy: IfNotPresent clusterConfig: - listenerClass: external-unstable zookeeperConfigMapName: test-nifi-znode authentication: - authenticationClass: nifi-users @@ -51,6 +50,7 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-unstable podOverrides: spec: initContainers: diff --git a/tests/templates/kuttl/external-access/30_nifi.yaml.j2 b/tests/templates/kuttl/external-access/30_nifi.yaml.j2 index c1d6c74e..989f5109 100644 --- a/tests/templates/kuttl/external-access/30_nifi.yaml.j2 +++ b/tests/templates/kuttl/external-access/30_nifi.yaml.j2 @@ -51,9 +51,9 @@ spec: {% endif %} nodes: config: - listenerClass: test-external-stable-$NAMESPACE logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: test-external-stable-$NAMESPACE roleGroups: default: replicas: 2 diff --git a/tests/templates/kuttl/iceberg/interactive-nifi.yaml b/tests/templates/kuttl/iceberg/interactive-nifi.yaml index d21fff01..4e07d0ea 100644 --- a/tests/templates/kuttl/iceberg/interactive-nifi.yaml +++ b/tests/templates/kuttl/iceberg/interactive-nifi.yaml @@ -8,7 +8,6 @@ spec: clusterConfig: authentication: - authenticationClass: nifi-users - listenerClass: external-unstable sensitiveProperties: keySecret: nifi-interactive-sensitive-property-key autoGenerate: true @@ -39,6 +38,8 @@ spec: storage: "1" storageClassName: secrets.stackable.tech nodes: + config: + listenerClass: external-unstable configOverrides: nifi.properties: nifi.nar.library.directory.myCustomLibs: /stackable/userdata/nifi-processors/ diff --git a/tests/templates/kuttl/ldap/12-install-nifi.yaml.j2 b/tests/templates/kuttl/ldap/12-install-nifi.yaml.j2 index 8df853c6..b8f1b0e8 100644 --- a/tests/templates/kuttl/ldap/12-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/ldap/12-install-nifi.yaml.j2 @@ -44,10 +44,10 @@ spec: vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} zookeeperConfigMapName: nifi-with-ldap-znode - listenerClass: external-unstable nodes: config: gracefulShutdownTimeout: 1m + listenerClass: external-unstable logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} roleGroups: diff --git a/tests/templates/kuttl/oidc-opa/30_nifi.yaml.j2 b/tests/templates/kuttl/oidc-opa/30_nifi.yaml.j2 index bb446c0b..d24403b7 100644 --- a/tests/templates/kuttl/oidc-opa/30_nifi.yaml.j2 +++ b/tests/templates/kuttl/oidc-opa/30_nifi.yaml.j2 @@ -37,12 +37,12 @@ spec: vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} zookeeperConfigMapName: nifi-znode - listenerClass: external-unstable nodes: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} gracefulShutdownTimeout: 1s # let the tests run faster + listenerClass: external-unstable configOverrides: nifi.properties: # speed up tests diff --git a/tests/templates/kuttl/smoke_v1/30-install-nifi.yaml.j2 b/tests/templates/kuttl/smoke_v1/30-install-nifi.yaml.j2 index 2b9adf37..533e3a70 100644 --- a/tests/templates/kuttl/smoke_v1/30-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/smoke_v1/30-install-nifi.yaml.j2 @@ -15,7 +15,6 @@ spec: pullPolicy: IfNotPresent clusterConfig: zookeeperConfigMapName: nifi-znode - listenerClass: {{ test_scenario['values']['listener-class'] }} authentication: - authenticationClass: nifi-users hostHeaderCheck: @@ -38,6 +37,7 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: {{ test_scenario['values']['listener-class'] }} roleGroups: default: replicas: 2 diff --git a/tests/templates/kuttl/smoke_v2/30-install-nifi.yaml.j2 b/tests/templates/kuttl/smoke_v2/30-install-nifi.yaml.j2 index 455a47f4..15d9c414 100644 --- a/tests/templates/kuttl/smoke_v2/30-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/smoke_v2/30-install-nifi.yaml.j2 @@ -17,7 +17,6 @@ spec: {% if test_scenario['values']['use-zookeeper-manager'] == 'true' %} zookeeperConfigMapName: nifi-znode {% endif %} - listenerClass: {{ test_scenario['values']['listener-class'] }} authentication: - authenticationClass: nifi-users hostHeaderCheck: @@ -40,6 +39,7 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: {{ test_scenario['values']['listener-class'] }} roleGroups: default: replicas: 2 From a29e789d3a78d3b243778a61928d67d4c7998863 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 10 Jun 2025 10:41:12 +0200 Subject: [PATCH 10/37] restore iceberg test file --- tests/templates/kuttl/iceberg/50_nifi.yaml.j2 | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) diff --git a/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 b/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 index e69de29b..16705a35 100644 --- a/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 +++ b/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 @@ -0,0 +1,145 @@ +--- +apiVersion: nifi.stackable.tech/v1alpha1 +kind: NifiCluster +metadata: + name: nifi +spec: + image: +{% if test_scenario['values']['nifi-iceberg'].find(",") > 0 %} + custom: "{{ test_scenario['values']['nifi-iceberg'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['nifi-iceberg'].split(',')[0] }}" +{% else %} + custom: null + productVersion: "{{ test_scenario['values']['nifi-iceberg'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + listenerClass: external-unstable + authentication: + - authenticationClass: nifi-users + sensitiveProperties: + keySecret: nifi-sensitive-property-key + extraVolumes: + - name: hdfs-config + configMap: + name: hdfs +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} + - name: hive-config + configMap: + name: nifi-hive-config + - name: kerberos + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: kerberos-$NAMESPACE + secrets.stackable.tech/kerberos.service.names: nifi + secrets.stackable.tech/scope: service=nifi + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech +{% endif %} +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + nodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + configOverrides: + nifi.properties: +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} + nifi.kerberos.krb5.file: /stackable/userdata/kerberos/krb5.conf +{% endif %} + + # Quicker startup, and we only have a single node + nifi.cluster.flow.election.max.wait.time: 5 secs + jvmArgumentOverrides: + add: + # Needed for NiFi to trust the minio cert + - -Djavax.net.ssl.trustStore=/stackable/keystore/truststore.p12 + - -Djavax.net.ssl.trustStorePassword=secret + - -Djavax.net.ssl.trustStoreType=PKCS12 +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} + podOverrides: &podOverrides + spec: + containers: + - name: nifi + env: + - name: KERBEROS_REALM + value: {{ test_scenario['values']['kerberos-realm'] }} + volumeMounts: + - name: kerberos + mountPath: /stackable/kerberos + # Normally we would use a different location and set `-Djava.security.krb5.conf=/example/path/krb5.conf`, + # but we can not influence the JVM arguments (edit: we can now) + - name: kerberos + mountPath: /etc/krb5.conf + subPath: krb5.conf + volumes: + - name: kerberos + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: kerberos-$NAMESPACE + secrets.stackable.tech/kerberos.service.names: nifi + secrets.stackable.tech/scope: service=nifi + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech +{% endif %} + roleGroups: + default: + replicas: 1 +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: nifi-users +spec: + provider: + static: + userCredentialsSecret: + name: nifi-users +--- +apiVersion: v1 +kind: Secret +metadata: + name: nifi-users +stringData: + admin: adminadmin +--- +apiVersion: v1 +kind: Secret +metadata: + name: nifi-sensitive-property-key +stringData: + nifiSensitivePropsKey: mYsUp3rS3cr3tk3y +{% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nifi-hive-config +data: + hive-site.xml: | + + + hive.metastore.sasl.enabled + true + + + hive.metastore.kerberos.principal + hive/hive.$NAMESPACE.svc.cluster.local@{{ test_scenario['values']['kerberos-realm'] }} + + +{% endif %} From a99adffb4369f7442202350e7a0c6925e645f147 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 10 Jun 2025 11:05:31 +0200 Subject: [PATCH 11/37] fix listener class in iceberg test --- tests/templates/kuttl/iceberg/50_nifi.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 b/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 index 16705a35..9d1a8583 100644 --- a/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 +++ b/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 @@ -14,7 +14,6 @@ spec: {% endif %} pullPolicy: IfNotPresent clusterConfig: - listenerClass: external-unstable authentication: - authenticationClass: nifi-users sensitiveProperties: @@ -50,6 +49,7 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-unstable configOverrides: nifi.properties: {% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} From dd63d7597e4a059326eb2c70be9dd2fa11457d42 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 10 Jun 2025 13:45:22 +0200 Subject: [PATCH 12/37] expose https port in headless service --- rust/operator-binary/src/controller.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index b38f0766..f9648bcc 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -830,6 +830,7 @@ fn build_node_rolegroup_service( .with_label(Label::try_from(("prometheus.io/scrape", "true")).context(LabelBuildSnafu)?) .build(); + // In NiFi 2.x metrics are scraped from the HTTPS endpoint let mut service_ports = vec![]; if resolved_product_image.product_version.starts_with("1.") { service_ports.push(ServicePort { @@ -838,6 +839,13 @@ fn build_node_rolegroup_service( protocol: Some("TCP".to_owned()), ..ServicePort::default() }); + } else { + service_ports.push(ServicePort { + name: Some(HTTPS_PORT_NAME.to_owned()), + port: HTTPS_PORT.into(), + protocol: Some("TCP".to_owned()), + ..ServicePort::default() + }); } let spec = Some(ServiceSpec { From afc55aced8ff67b0abdfe8e65e8fdc544ecac9bd Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 10 Jun 2025 13:46:48 +0200 Subject: [PATCH 13/37] update comment --- rust/operator-binary/src/controller.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index f9648bcc..e7ecccdf 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -830,7 +830,7 @@ fn build_node_rolegroup_service( .with_label(Label::try_from(("prometheus.io/scrape", "true")).context(LabelBuildSnafu)?) .build(); - // In NiFi 2.x metrics are scraped from the HTTPS endpoint + // In NiFi 2.x metrics are scraped from the HTTPS port let mut service_ports = vec![]; if resolved_product_image.product_version.starts_with("1.") { service_ports.push(ServicePort { From bf88cc19a6dd2d7d74d439c73c36da954f32ca29 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 10 Jun 2025 16:39:59 +0200 Subject: [PATCH 14/37] address feedback from review --- CHANGELOG.md | 2 +- rust/operator-binary/src/controller.rs | 48 +++++---------- rust/operator-binary/src/crd/mod.rs | 37 +----------- rust/operator-binary/src/crd/utils.rs | 59 ------------------- .../kuttl/external-access/30-assert.yaml | 2 +- 5 files changed, 19 insertions(+), 129 deletions(-) delete mode 100644 rust/operator-binary/src/crd/utils.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index a342ec6d..a9d0d4a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ All notable changes to this project will be documented in this file. ### Added - Add rolling upgrade support for upgrades between NiFi 2 versions ([#771]). -- Added listener support for Nifi ([#784]). +- Added Listener support for NiFi ([#784]). - Adds new telemetry CLI arguments and environment variables ([#782]). - Use `--file-log-max-files` (or `FILE_LOG_MAX_FILES`) to limit the number of log files kept. - Use `--file-log-rotation-period` (or `FILE_LOG_ROTATION_PERIOD`) to configure the frequency of rotation. diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index e7ecccdf..b89fdc03 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -140,9 +140,6 @@ pub enum Error { #[snafu(display("object defines no name"))] ObjectHasNoName, - #[snafu(display("object defines no spec"))] - ObjectHasNoSpec, - #[snafu(display("object defines no namespace"))] ObjectHasNoNamespace, @@ -222,21 +219,12 @@ pub enum Error { #[snafu(display("Failed to find information about file [{}] in product config", kind))] ProductConfigKindNotSpecified { kind: String }, - #[snafu(display("Failed to find any nodes in cluster {obj_ref}",))] - MissingNodes { - source: stackable_operator::client::Error, - obj_ref: ObjectRef, - }, - #[snafu(display("Failed to find service {obj_ref}"))] MissingService { source: stackable_operator::client::Error, obj_ref: ObjectRef, }, - #[snafu(display("Failed to find an external port to use for proxy hosts"))] - ExternalPort, - #[snafu(display("Could not build role service fqdn"))] NoRoleServiceFqdn, @@ -534,8 +522,7 @@ pub async fn reconcile_nifi( // Since we cannot predict which of the addresses a user might decide to use we will simply // add all of them to the setting for now. // For more information see - // let proxy_hosts = get_proxy_hosts(client, nifi, &merged_config).await?; - let proxy_hosts = get_proxy_hosts(client, nifi).await?; + let proxy_hosts = get_proxy_hosts(client, nifi, &resolved_product_image).await?; let rg_configmap = build_node_rolegroup_config_map( nifi, @@ -1560,7 +1547,7 @@ async fn build_node_rolegroup_statefulset( async fn get_proxy_hosts( client: &Client, nifi: &v1alpha1::NifiCluster, - // merged_config: &NifiConfig, + resolved_product_image: &ResolvedProductImage, ) -> Result { let host_header_check = nifi.spec.cluster_config.host_header_check.clone(); @@ -1576,27 +1563,24 @@ async fn get_proxy_hosts( return Ok("*".to_string()); } - let node_role_service_fqdn = nifi - .node_role_service_fqdn(&client.kubernetes_cluster_info) - .context(NoRoleServiceFqdnSnafu)?; - let reporting_task_service_name = reporting_task::build_reporting_task_fqdn_service_name( - nifi, - &client.kubernetes_cluster_info, - ) - .context(ReportingTaskSnafu)?; - let mut proxy_hosts_set = HashSet::from([ - node_role_service_fqdn.clone(), - format!("{node_role_service_fqdn}:{HTTPS_PORT}"), - format!("{reporting_task_service_name}:{HTTPS_PORT}"), + // Address and port are injected from the listener volume during the prepare container + let mut proxy_hosts = HashSet::from([ + "${env:LISTENER_DEFAULT_ADDRESS}:${env:LISTENER_DEFAULT_PORT_HTTPS}".to_string(), ]); + proxy_hosts.extend(host_header_check.additional_allowed_hosts); - proxy_hosts_set.extend(host_header_check.additional_allowed_hosts); + // Reporting task only exists for NiFi 1.x + if resolved_product_image.product_version.starts_with("1.") { + let reporting_task_service_name = reporting_task::build_reporting_task_fqdn_service_name( + nifi, + &client.kubernetes_cluster_info, + ) + .context(ReportingTaskSnafu)?; - // Inject the address and port from the listener volume during the prepare container - proxy_hosts_set - .insert("${env:LISTENER_DEFAULT_ADDRESS}:${env:LISTENER_DEFAULT_PORT_HTTPS}".to_string()); + proxy_hosts.insert(format!("{reporting_task_service_name}:{HTTPS_PORT}")); + } - let mut proxy_hosts = Vec::from_iter(proxy_hosts_set); + let mut proxy_hosts = Vec::from_iter(proxy_hosts); proxy_hosts.sort(); Ok(proxy_hosts.join(",")) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 3635176d..c351b0eb 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -2,9 +2,8 @@ pub mod affinity; pub mod authentication; pub mod sensitive_properties; pub mod tls; -pub mod utils; -use std::collections::{BTreeMap, HashMap}; +use std::collections::BTreeMap; use affinity::get_affinity; use sensitive_properties::NifiSensitivePropertiesConfig; @@ -46,7 +45,6 @@ use stackable_operator::{ versioned::versioned, }; use tls::NifiTls; -use utils::PodRef; pub const APP_NAME: &str = "nifi"; @@ -84,9 +82,6 @@ pub enum Error { #[snafu(display("object has no nodes defined"))] NoNodesDefined, - - #[snafu(display("listener podrefs could not be resolved"))] - ListenerPodRef { source: utils::Error }, } #[versioned(version(name = "v1alpha1"))] @@ -250,36 +245,6 @@ impl v1alpha1::NifiCluster { &self.spec.cluster_config.tls.server_secret_class } - /// List all pods expected to form the cluster - /// - /// We try to predict the pods here rather than looking at the current cluster state in order to - /// avoid instance churn. - pub fn pods(&self) -> Result + '_, Error> { - let ns = self.metadata.namespace.clone().context(NoNamespaceSnafu)?; - Ok(self - .spec - .nodes - .iter() - .flat_map(|role| &role.role_groups) - // Order rolegroups consistently, to avoid spurious downstream rewrites - .collect::>() - .into_iter() - .flat_map(move |(rolegroup_name, rolegroup)| { - let rolegroup_ref = self.node_rolegroup_ref(rolegroup_name); - let ns = ns.clone(); - (0..rolegroup.replicas.unwrap_or(0)).map(move |i| PodRef { - namespace: ns.clone(), - role_group_service_name: rolegroup_ref.object_name(), - pod_name: format!("{}-{}", rolegroup_ref.object_name(), i), - ports: HashMap::from([ - (HTTPS_PORT_NAME.to_owned(), HTTPS_PORT), - (METRICS_PORT_NAME.to_owned(), METRICS_PORT), - ]), - fqdn_override: None, - }) - })) - } - /// Retrieve and merge resource configs for role and role groups pub fn merged_config(&self, role: &NifiRole, role_group: &str) -> Result { // Initialize the result with all default values as baseline diff --git a/rust/operator-binary/src/crd/utils.rs b/rust/operator-binary/src/crd/utils.rs deleted file mode 100644 index 8466d1ba..00000000 --- a/rust/operator-binary/src/crd/utils.rs +++ /dev/null @@ -1,59 +0,0 @@ -use std::{borrow::Cow, collections::HashMap, num::TryFromIntError}; - -use snafu::Snafu; -use stackable_operator::{ - crd::listener::v1alpha1::Listener, k8s_openapi::api::core::v1::Pod, - kube::runtime::reflector::ObjectRef, utils::cluster_info::KubernetesClusterInfo, -}; - -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("unable to get {listener} (for {pod})"))] - GetPodListener { - source: stackable_operator::client::Error, - listener: ObjectRef, - pod: ObjectRef, - }, - - #[snafu(display("{listener} (for {pod}) has no address"))] - PodListenerHasNoAddress { - listener: ObjectRef, - pod: ObjectRef, - }, - - #[snafu(display("port {port} ({port_name:?}) is out of bounds, must be within {range:?}", range = 0..=u16::MAX))] - PortOutOfBounds { - source: TryFromIntError, - port_name: String, - port: i32, - }, -} - -/// Reference to a single `Pod` that is a component of the product cluster -/// -/// Used for service discovery. -#[derive(Debug)] -pub struct PodRef { - pub namespace: String, - pub role_group_service_name: String, - pub pod_name: String, - pub fqdn_override: Option, - pub ports: HashMap, -} - -impl PodRef { - pub fn fqdn(&self, cluster_info: &KubernetesClusterInfo) -> Cow { - self.fqdn_override.as_deref().map_or_else( - || { - Cow::Owned(format!( - "{pod_name}.{role_group_service_name}.{namespace}.svc.{cluster_domain}", - pod_name = self.pod_name, - role_group_service_name = self.role_group_service_name, - namespace = self.namespace, - cluster_domain = cluster_info.cluster_domain, - )) - }, - Cow::Borrowed, - ) - } -} diff --git a/tests/templates/kuttl/external-access/30-assert.yaml b/tests/templates/kuttl/external-access/30-assert.yaml index c71ac818..cc1c5125 100644 --- a/tests/templates/kuttl/external-access/30-assert.yaml +++ b/tests/templates/kuttl/external-access/30-assert.yaml @@ -5,7 +5,7 @@ metadata: name: install-nifi timeout: 300 commands: - - script: kubectl -n $NAMESPACE wait --for=condition=available=true nificlusters.nifi.stackable.tech/nifi --timeout 301s + - script: kubectl -n $NAMESPACE wait --for=condition=available=true nificlusters.nifi.stackable.tech/test-nifi --timeout 301s --- apiVersion: apps/v1 kind: StatefulSet From 7262d8970830998b4a717dd083f91b63276b5840 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 12 Jun 2025 14:26:57 +0200 Subject: [PATCH 15/37] remove unused error variants --- rust/operator-binary/src/controller.rs | 23 +---------------------- rust/operator-binary/src/crd/mod.rs | 9 --------- 2 files changed, 1 insertion(+), 31 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index b89fdc03..54b2988a 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -33,7 +33,6 @@ use stackable_operator::{ client::Client, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, commons::{product_image_selection::ResolvedProductImage, rbac::build_rbac_resources}, - config::fragment, crd::{ authentication::oidc::v1alpha1::AuthenticationProvider, git_sync, @@ -54,7 +53,7 @@ use stackable_operator::{ kube::{ Resource, ResourceExt, core::{DeserializeGuard, error_boundary}, - runtime::{controller::Action, reflector::ObjectRef}, + runtime::controller::Action, }, kvp::{Label, Labels, ObjectLabels}, logging::controller::ReconcilerError, @@ -153,11 +152,6 @@ pub enum Error { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("failed to apply global Service"))] - ApplyRoleService { - source: stackable_operator::cluster_resources::Error, - }, - #[snafu(display("failed to fetch deployed StatefulSets"))] FetchStatefulsets { source: stackable_operator::client::Error, @@ -219,15 +213,6 @@ pub enum Error { #[snafu(display("Failed to find information about file [{}] in product config", kind))] ProductConfigKindNotSpecified { kind: String }, - #[snafu(display("Failed to find service {obj_ref}"))] - MissingService { - source: stackable_operator::client::Error, - obj_ref: ObjectRef, - }, - - #[snafu(display("Could not build role service fqdn"))] - NoRoleServiceFqdn, - #[snafu(display("Bootstrap configuration error"))] BootstrapConfig { #[snafu(source(from(config::Error, Box::new)))] @@ -247,12 +232,6 @@ pub enum Error { container_name: String, }, - #[snafu(display("failed to validate resources for {rolegroup}"))] - ResourceValidation { - source: fragment::ValidationError, - rolegroup: RoleGroupRef, - }, - #[snafu(display("failed to resolve and merge config for role and role group"))] FailedToResolveConfig { source: crate::crd::Error }, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index c351b0eb..f04b8225 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -68,20 +68,11 @@ const DEFAULT_NODE_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_ #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("object has no namespace associated"))] - NoNamespace, - #[snafu(display("the NiFi role [{role}] is missing from spec"))] MissingNifiRole { role: String }, - #[snafu(display("the NiFi node role group [{role_group}] is missing from spec"))] - MissingNifiRoleGroup { role_group: String }, - #[snafu(display("fragment validation failure"))] FragmentValidationFailure { source: ValidationError }, - - #[snafu(display("object has no nodes defined"))] - NoNodesDefined, } #[versioned(version(name = "v1alpha1"))] From e950a5a2876a98137f8768cafb1c4f30052b477b Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 12 Jun 2025 14:42:12 +0200 Subject: [PATCH 16/37] create headless service name in function --- rust/operator-binary/src/controller.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 54b2988a..d4e859b6 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -783,7 +783,7 @@ fn build_node_rolegroup_service( ) -> Result { let metadata = ObjectMetaBuilder::new() .name_and_namespace(nifi) - .name(format!("{name}-metrics", name = rolegroup.object_name())) + .name(rolegroup_service_name(rolegroup)) .ownerreference_from_resource(nifi, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(build_recommended_labels( @@ -1484,7 +1484,7 @@ async fn build_node_rolegroup_statefulset( ), ..LabelSelector::default() }, - service_name: Some(rolegroup_ref.object_name()), + service_name: Some(rolegroup_service_name(rolegroup_ref)), template: pod_template, update_strategy: Some(StatefulSetUpdateStrategy { type_: if rolling_update_supported { @@ -1523,6 +1523,10 @@ async fn build_node_rolegroup_statefulset( }) } +pub fn rolegroup_service_name(rolegroup: &RoleGroupRef) -> String { + format!("{name}-metrics", name = rolegroup.object_name()) +} + async fn get_proxy_hosts( client: &Client, nifi: &v1alpha1::NifiCluster, From 41819bbd98a909f9c843f2f09d0199734fe153dd Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 12 Jun 2025 14:46:22 +0200 Subject: [PATCH 17/37] remove unused functions --- rust/operator-binary/src/crd/mod.rs | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index f04b8225..1a428afb 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -38,10 +38,7 @@ use stackable_operator::{ schemars::{self, JsonSchema}, status::condition::{ClusterCondition, HasStatusCondition}, time::Duration, - utils::{ - cluster_info::KubernetesClusterInfo, - crds::{raw_object_list_schema, raw_object_schema}, - }, + utils::crds::{raw_object_list_schema, raw_object_schema}, versioned::versioned, }; use tls::NifiTls; @@ -201,21 +198,6 @@ impl v1alpha1::NifiCluster { rolegroup.object_name() } - /// The name of the role-level load-balanced Kubernetes `Service` - pub fn node_role_service_name(&self) -> String { - self.name_any() - } - - /// The fully-qualified domain name of the role-level load-balanced Kubernetes `Service` - pub fn node_role_service_fqdn(&self, cluster_info: &KubernetesClusterInfo) -> Option { - Some(format!( - "{}.{}.svc.{}", - self.node_role_service_name(), - self.metadata.namespace.as_ref()?, - cluster_info.cluster_domain, - )) - } - /// Metadata about a metastore rolegroup pub fn node_rolegroup_ref(&self, group_name: impl Into) -> RoleGroupRef { RoleGroupRef { From 190ca21e7a11c02c9c3342eab70a5c8457d7fa9e Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 18 Jun 2025 19:15:13 +0200 Subject: [PATCH 18/37] move listenerClass to roleConfig --- deploy/helm/nifi-operator/crds/crds.yaml | 12 +- .../getting_started/getting_started.sh | 2 +- .../getting_started/getting_started.sh.j2 | 2 +- .../pages/usage_guide/custom-components.adoc | 4 +- .../modules/nifi/pages/usage_guide/index.adoc | 2 +- .../nifi/pages/usage_guide/listenerclass.adoc | 4 +- examples/simple-nifi-cluster.yaml | 2 +- rust/operator-binary/src/config/jvm.rs | 6 +- rust/operator-binary/src/config/mod.rs | 11 +- rust/operator-binary/src/controller.rs | 117 +++++------------- rust/operator-binary/src/crd/mod.rs | 39 +++--- rust/operator-binary/src/listener.rs | 76 ++++++++++++ rust/operator-binary/src/main.rs | 1 + .../30-install-nifi.yaml.j2 | 1 + .../15-create-listener-class.yaml | 5 + .../15-create-listener-classes.yaml | 5 - .../external-access/15_listener-class.yaml | 7 ++ .../external-access/15_listener-classes.yaml | 21 ---- .../kuttl/external-access/30-assert.yaml | 35 +----- .../kuttl/external-access/30_nifi.yaml.j2 | 11 +- .../iceberg/30-install-zookeeper.yaml.j2 | 4 +- .../templates/kuttl/iceberg/34_trino.yaml.j2 | 3 +- tests/templates/kuttl/iceberg/50_nifi.yaml.j2 | 1 + .../kuttl/iceberg/interactive-nifi.yaml | 2 +- .../kuttl/ldap/12-install-nifi.yaml.j2 | 3 +- .../templates/kuttl/oidc-opa/30_nifi.yaml.j2 | 1 + .../smoke_v1/20-install-zookeeper.yaml.j2 | 3 +- .../kuttl/smoke_v1/30-install-nifi.yaml.j2 | 1 + .../smoke_v2/20-install-zookeeper.yaml.j2 | 3 +- .../kuttl/smoke_v2/30-install-nifi.yaml.j2 | 1 + 30 files changed, 191 insertions(+), 194 deletions(-) create mode 100644 rust/operator-binary/src/listener.rs create mode 100644 tests/templates/kuttl/external-access/15-create-listener-class.yaml delete mode 100644 tests/templates/kuttl/external-access/15-create-listener-classes.yaml create mode 100644 tests/templates/kuttl/external-access/15_listener-class.yaml delete mode 100644 tests/templates/kuttl/external-access/15_listener-classes.yaml diff --git a/deploy/helm/nifi-operator/crds/crds.yaml b/deploy/helm/nifi-operator/crds/crds.yaml index b920603e..1fa6e8d0 100644 --- a/deploy/helm/nifi-operator/crds/crds.yaml +++ b/deploy/helm/nifi-operator/crds/crds.yaml @@ -362,10 +362,6 @@ spec: description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true type: string - listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the nodes. - nullable: true - type: string logging: default: containers: {} @@ -780,11 +776,15 @@ spec: x-kubernetes-preserve-unknown-fields: true roleConfig: default: + listenerClass: cluster-internal podDisruptionBudget: enabled: true maxUnavailable: null description: This is a product-agnostic RoleConfig, which is sufficient for most of the products. properties: + listenerClass: + default: cluster-internal + type: string podDisruptionBudget: default: enabled: true @@ -853,10 +853,6 @@ spec: description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true type: string - listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the nodes. - nullable: true - type: string logging: default: containers: {} diff --git a/docs/modules/nifi/examples/getting_started/getting_started.sh b/docs/modules/nifi/examples/getting_started/getting_started.sh index d6caf6a1..fb25f97b 100755 --- a/docs/modules/nifi/examples/getting_started/getting_started.sh +++ b/docs/modules/nifi/examples/getting_started/getting_started.sh @@ -148,7 +148,7 @@ spec: autoGenerate: true zookeeperConfigMapName: simple-nifi-znode nodes: - config: + roleConfig: listenerClass: external-unstable roleGroups: default: diff --git a/docs/modules/nifi/examples/getting_started/getting_started.sh.j2 b/docs/modules/nifi/examples/getting_started/getting_started.sh.j2 index 012ce15c..b9248c9e 100755 --- a/docs/modules/nifi/examples/getting_started/getting_started.sh.j2 +++ b/docs/modules/nifi/examples/getting_started/getting_started.sh.j2 @@ -148,7 +148,7 @@ spec: autoGenerate: true zookeeperConfigMapName: simple-nifi-znode nodes: - config: + roleConfig: listenerClass: external-unstable roleGroups: default: diff --git a/docs/modules/nifi/pages/usage_guide/custom-components.adoc b/docs/modules/nifi/pages/usage_guide/custom-components.adoc index 3f6a3c2e..f9f1e305 100644 --- a/docs/modules/nifi/pages/usage_guide/custom-components.adoc +++ b/docs/modules/nifi/pages/usage_guide/custom-components.adoc @@ -167,7 +167,7 @@ spec: autoGenerate: true zookeeperConfigMapName: simple-nifi-znode nodes: - config: + roleConfig: listenerClass: external-unstable configOverrides: nifi.properties: @@ -287,7 +287,7 @@ spec: autoGenerate: true zookeeperConfigMapName: simple-nifi-znode nodes: - config: + roleConfig: listenerClass: external-unstable configOverrides: nifi.properties: diff --git a/docs/modules/nifi/pages/usage_guide/index.adoc b/docs/modules/nifi/pages/usage_guide/index.adoc index 419b4f5a..adaf1f60 100644 --- a/docs/modules/nifi/pages/usage_guide/index.adoc +++ b/docs/modules/nifi/pages/usage_guide/index.adoc @@ -30,7 +30,7 @@ spec: keySecret: nifi-sensitive-property-key autoGenerate: true nodes: - config: + roleConfig: listenerClass: external-unstable roleGroups: default: diff --git a/docs/modules/nifi/pages/usage_guide/listenerclass.adoc b/docs/modules/nifi/pages/usage_guide/listenerclass.adoc index 55dcbbb1..daf8595f 100644 --- a/docs/modules/nifi/pages/usage_guide/listenerclass.adoc +++ b/docs/modules/nifi/pages/usage_guide/listenerclass.adoc @@ -2,13 +2,13 @@ :description: Configure Apache NiFi service exposure with cluster-internal or external-unstable listener classes. The operator deploys a xref:listener-operator:listener.adoc[Listener] for the Node pod. -The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.nodes.config.listenerClass`: +The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.nodes.roleConfig.listenerClass`: [source,yaml] ---- spec: nodes: - config: + roleConfig: listenerClass: external-unstable # <1> ---- <1> Specify one of `external-stable`, `external-unstable`, `cluster-internal` (the default setting is `cluster-internal`). diff --git a/examples/simple-nifi-cluster.yaml b/examples/simple-nifi-cluster.yaml index ca084c80..a8b2d033 100644 --- a/examples/simple-nifi-cluster.yaml +++ b/examples/simple-nifi-cluster.yaml @@ -56,7 +56,7 @@ spec: autoGenerate: true zookeeperConfigMapName: simple-nifi-znode nodes: - config: + roleConfig: listenerClass: external-unstable roleGroups: default: diff --git a/rust/operator-binary/src/config/jvm.rs b/rust/operator-binary/src/config/jvm.rs index f9d3c25c..7960a2e9 100644 --- a/rust/operator-binary/src/config/jvm.rs +++ b/rust/operator-binary/src/config/jvm.rs @@ -1,12 +1,12 @@ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ memory::{BinaryMultiple, MemoryQuantity}, - role_utils::{self, GenericRoleConfig, JavaCommonConfig, JvmArgumentOverrides, Role}, + role_utils::{self, JavaCommonConfig, JvmArgumentOverrides, Role}, }; use crate::{ config::{JVM_SECURITY_PROPERTIES_FILE, NIFI_CONFIG_DIRECTORY}, - crd::{NifiConfig, NifiConfigFragment}, + crd::{NifiConfig, NifiConfigFragment, NifiNodeRoleConfig}, }; // Part of memory resources allocated for Java heap @@ -29,7 +29,7 @@ pub enum Error { /// Create the NiFi bootstrap.conf pub fn build_merged_jvm_config( merged_config: &NifiConfig, - role: &Role, + role: &Role, role_group: &str, ) -> Result { let heap_size = MemoryQuantity::try_from( diff --git a/rust/operator-binary/src/config/mod.rs b/rust/operator-binary/src/config/mod.rs index d1df2426..3ddfa6da 100644 --- a/rust/operator-binary/src/config/mod.rs +++ b/rust/operator-binary/src/config/mod.rs @@ -14,14 +14,15 @@ use stackable_operator::{ ValidatedRoleConfigByPropertyKind, transform_all_roles_to_config, validate_all_roles_and_groups_config, }, - role_utils::{GenericRoleConfig, JavaCommonConfig, Role}, + role_utils::{JavaCommonConfig, Role}, }; use strum::{Display, EnumIter}; use crate::{ crd::{ - HTTPS_PORT, NifiConfig, NifiConfigFragment, NifiRole, NifiStorageConfig, PROTOCOL_PORT, - sensitive_properties, v1alpha1, v1alpha1::NifiClusteringBackend, + HTTPS_PORT, NifiConfig, NifiConfigFragment, NifiNodeRoleConfig, NifiRole, + NifiStorageConfig, PROTOCOL_PORT, sensitive_properties, + v1alpha1::{self, NifiClusteringBackend}, }, operations::graceful_shutdown::graceful_shutdown_config_properties, security::{ @@ -112,7 +113,7 @@ pub enum Error { pub fn build_bootstrap_conf( merged_config: &NifiConfig, overrides: BTreeMap, - role: &Role, + role: &Role, role_group: &str, ) -> Result { let mut bootstrap = BTreeMap::new(); @@ -736,7 +737,7 @@ pub fn build_state_management_xml(clustering_backend: &NifiClusteringBackend) -> pub fn validated_product_config( resource: &v1alpha1::NifiCluster, version: &str, - role: &Role, + role: &Role, product_config: &ProductConfigManager, ) -> Result { let mut roles = HashMap::new(); diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index d4e859b6..85e8bf74 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -24,20 +24,13 @@ use stackable_operator::{ container::ContainerBuilder, resources::ResourceRequirementsBuilder, security::PodSecurityContextBuilder, - volume::{ - ListenerOperatorVolumeSourceBuilder, ListenerOperatorVolumeSourceBuilderError, - ListenerReference, SecretFormat, - }, + volume::{ListenerOperatorVolumeSourceBuilderError, SecretFormat}, }, }, client::Client, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, commons::{product_image_selection::ResolvedProductImage, rbac::build_rbac_resources}, - crd::{ - authentication::oidc::v1alpha1::AuthenticationProvider, - git_sync, - listener::v1alpha1::{Listener, ListenerPort, ListenerSpec}, - }, + crd::{authentication::oidc::v1alpha1::AuthenticationProvider, git_sync}, k8s_openapi::{ DeepMerge, api::{ @@ -91,10 +84,11 @@ use crate::{ crd::{ APP_NAME, BALANCE_PORT, BALANCE_PORT_NAME, Container, HTTPS_PORT, HTTPS_PORT_NAME, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, METRICS_PORT, METRICS_PORT_NAME, NifiConfig, - NifiConfigFragment, NifiRole, NifiStatus, PROTOCOL_PORT, PROTOCOL_PORT_NAME, - STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, authentication::AuthenticationClassResolved, - v1alpha1, + NifiConfigFragment, NifiNodeRoleConfig, NifiRole, NifiStatus, PROTOCOL_PORT, + PROTOCOL_PORT_NAME, STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, + authentication::AuthenticationClassResolved, v1alpha1, }, + listener::{build_group_listener, build_group_listener_pvc, group_listener_name}, operations::{ graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs, @@ -343,12 +337,13 @@ pub enum Error { BuildListenerVolume { source: ListenerOperatorVolumeSourceBuilderError, }, - - #[snafu(display("failed to apply group listener for {rolegroup}"))] + #[snafu(display("failed to apply group listener"))] ApplyGroupListener { source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, }, + + #[snafu(display("failed to configure listener"))] + ListenerConfiguration { source: crate::listener::Error }, } type Result = std::result::Result; @@ -542,19 +537,6 @@ pub async fn reconcile_nifi( ) .await?; - let rg_group_listener = build_group_listener( - nifi, - &resolved_product_image, - &rolegroup, - merged_config.listener_class, - )?; - - cluster_resources - .add(client, rg_group_listener) - .await - .with_context(|_| ApplyGroupListenerSnafu { - rolegroup: rolegroup.clone(), - })?; cluster_resources .add(client, rg_service) .await @@ -583,13 +565,29 @@ pub async fn reconcile_nifi( } let role_config = nifi.role_config(&nifi_role); - if let Some(GenericRoleConfig { - pod_disruption_budget: pdb, + if let Some(NifiNodeRoleConfig { + common: GenericRoleConfig { + pod_disruption_budget: pdb, + }, + listener_class, }) = role_config { add_pdbs(pdb, nifi, &nifi_role, client, &mut cluster_resources) .await .context(FailedToCreatePdbSnafu)?; + + let role_group_listener = build_group_listener( + nifi, + build_recommended_labels(nifi, NIFI_CONTROLLER_NAME, "node", "none"), + listener_class.to_owned(), + group_listener_name(nifi), + ) + .context(ListenerConfigurationSnafu)?; + + cluster_resources + .add(client, role_group_listener) + .await + .with_context(|_| ApplyGroupListenerSnafu)?; } // Only add the reporting task in case it is enabled. @@ -662,7 +660,7 @@ async fn build_node_rolegroup_config_map( resolved_product_image: &ResolvedProductImage, authentication_config: &NifiAuthenticationConfig, authorization_config: &NifiAuthorizationConfig, - role: &Role, + role: &Role, rolegroup: &RoleGroupRef, rolegroup_config: &HashMap>, merged_config: &NifiConfig, @@ -837,49 +835,6 @@ fn build_node_rolegroup_service( Ok(service) } -pub fn build_group_listener( - nifi: &v1alpha1::NifiCluster, - resolved_product_image: &ResolvedProductImage, - rolegroup: &RoleGroupRef, - listener_class: String, -) -> Result { - let metadata = ObjectMetaBuilder::new() - .name_and_namespace(nifi) - .name(nifi.group_listener_name(rolegroup)) - .ownerreference_from_resource(nifi, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(build_recommended_labels( - nifi, - &resolved_product_image.app_version_label, - &rolegroup.role, - &rolegroup.role_group, - )) - .context(MetadataBuildSnafu)? - .build(); - - let spec = ListenerSpec { - class_name: Some(listener_class), - ports: Some(listener_ports()), - ..Default::default() - }; - - let listener = Listener { - metadata, - spec, - status: None, - }; - - Ok(listener) -} - -fn listener_ports() -> Vec { - vec![ListenerPort { - name: HTTPS_PORT_NAME.to_owned(), - port: HTTPS_PORT.into(), - protocol: Some("TCP".to_owned()), - }] -} - const USERDATA_MOUNTPOINT: &str = "/stackable/userdata"; /// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. @@ -892,7 +847,7 @@ async fn build_node_rolegroup_statefulset( resolved_product_image: &ResolvedProductImage, cluster_info: &KubernetesClusterInfo, rolegroup_ref: &RoleGroupRef, - role: &Role, + role: &Role, rolegroup_config: &HashMap>, merged_config: &NifiConfig, authentication_config: &NifiAuthenticationConfig, @@ -976,7 +931,7 @@ async fn build_node_rolegroup_statefulset( env_vars.extend(authorization_config.get_env_vars()); let node_address = format!( - "$POD_NAME.{}-node-{}.{}.svc.{}", + "$POD_NAME.{}-node-{}-metrics.{}.svc.{}", rolegroup_ref.cluster.name, rolegroup_ref.role_group, &nifi @@ -1235,13 +1190,9 @@ async fn build_node_rolegroup_statefulset( // listener endpoints will use persistent volumes // so that load balancers can hard-code the target addresses and // that it is possible to connect to a consistent address - let listener_pvc = ListenerOperatorVolumeSourceBuilder::new( - &ListenerReference::ListenerName(nifi.group_listener_name(rolegroup_ref)), - &unversioned_recommended_labels, - ) - .context(BuildListenerVolumeSnafu)? - .build_pvc(LISTENER_VOLUME_NAME.to_owned()) - .context(BuildListenerVolumeSnafu)?; + let listener_pvc = + build_group_listener_pvc(&group_listener_name(nifi), &unversioned_recommended_labels) + .context(ListenerConfigurationSnafu)?; add_graceful_shutdown_config(merged_config, &mut pod_builder).context(GracefulShutdownSnafu)?; diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 1a428afb..38afe308 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -97,7 +97,7 @@ pub mod versioned { // no doc - docs in Role struct. #[serde(default, skip_serializing_if = "Option::is_none")] - pub nodes: Option>, + pub nodes: Option>, // no doc - docs in ProductImage struct. pub image: ProductImage, @@ -191,13 +191,6 @@ impl HasStatusCondition for v1alpha1::NifiCluster { } impl v1alpha1::NifiCluster { - /// The name of the group-listener provided for a specific role-group. - /// The UI will use this group listener so that only one load balancer - /// is needed (per role group). - pub fn group_listener_name(&self, rolegroup: &RoleGroupRef) -> String { - rolegroup.object_name() - } - /// Metadata about a metastore rolegroup pub fn node_rolegroup_ref(&self, group_name: impl Into) -> RoleGroupRef { RoleGroupRef { @@ -207,7 +200,7 @@ impl v1alpha1::NifiCluster { } } - pub fn role_config(&self, role: &NifiRole) -> Option<&GenericRoleConfig> { + pub fn role_config(&self, role: &NifiRole) -> Option<&NifiNodeRoleConfig> { match role { NifiRole::Node => self.spec.nodes.as_ref().map(|n| &n.role_config), } @@ -414,10 +407,6 @@ pub struct NifiConfig { /// Please note that this can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. #[fragment_attrs(serde(default))] pub requested_secret_lifetime: Option, - - /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose the nodes. - #[serde(default)] - pub listener_class: String, } impl NifiConfig { @@ -467,7 +456,6 @@ impl NifiConfig { affinity: get_affinity(cluster_name, role), graceful_shutdown_timeout: Some(DEFAULT_NODE_GRACEFUL_SHUTDOWN_TIMEOUT), requested_secret_lifetime: Some(Self::DEFAULT_NODE_SECRET_LIFETIME), - listener_class: Some("cluster-internal".to_owned()), } } } @@ -547,3 +535,26 @@ pub struct NifiStorageConfig { #[fragment_attrs(serde(default))] pub state_repo: PvcConfig, } + +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct NifiNodeRoleConfig { + #[serde(flatten)] + pub common: GenericRoleConfig, + + #[serde(default = "node_default_listener_class")] + pub listener_class: String, +} + +impl Default for NifiNodeRoleConfig { + fn default() -> Self { + NifiNodeRoleConfig { + listener_class: node_default_listener_class(), + common: Default::default(), + } + } +} + +fn node_default_listener_class() -> String { + "cluster-internal".to_string() +} diff --git a/rust/operator-binary/src/listener.rs b/rust/operator-binary/src/listener.rs new file mode 100644 index 00000000..51f197a7 --- /dev/null +++ b/rust/operator-binary/src/listener.rs @@ -0,0 +1,76 @@ +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::{ + meta::ObjectMetaBuilder, + pod::volume::{ListenerOperatorVolumeSourceBuilder, ListenerReference}, + }, + crd::listener::v1alpha1::{Listener, ListenerPort, ListenerSpec}, + k8s_openapi::api::core::v1::PersistentVolumeClaim, + kube::ResourceExt, + kvp::{Labels, ObjectLabels}, +}; + +use crate::crd::{HTTPS_PORT, HTTPS_PORT_NAME, LISTENER_VOLUME_NAME, v1alpha1}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("listener object is missing metadata to build owner reference"))] + ObjectMissingMetadataForOwnerRef { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to build listener object meta data"))] + BuildObjectMeta { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to build listener volume"))] + BuildListenerPersistentVolume { + source: stackable_operator::builder::pod::volume::ListenerOperatorVolumeSourceBuilderError, + }, +} + +pub fn build_group_listener( + nifi: &v1alpha1::NifiCluster, + object_labels: ObjectLabels, + listener_class: String, + listener_group_name: String, +) -> Result { + Ok(Listener { + metadata: ObjectMetaBuilder::new() + .name_and_namespace(nifi) + .name(listener_group_name) + .ownerreference_from_resource(nifi, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(object_labels) + .context(BuildObjectMetaSnafu)? + .build(), + spec: ListenerSpec { + class_name: Some(listener_class), + ports: Some(vec![ListenerPort { + name: HTTPS_PORT_NAME.into(), + port: HTTPS_PORT.into(), + protocol: Some("TCP".into()), + }]), + ..Default::default() + }, + status: None, + }) +} + +pub fn build_group_listener_pvc( + group_listener_name: &String, + unversioned_recommended_labels: &Labels, +) -> Result { + ListenerOperatorVolumeSourceBuilder::new( + &ListenerReference::ListenerName(group_listener_name.to_string()), + unversioned_recommended_labels, + ) + .context(BuildListenerPersistentVolumeSnafu)? + .build_pvc(LISTENER_VOLUME_NAME.to_string()) + .context(BuildListenerPersistentVolumeSnafu) +} + +pub fn group_listener_name(nifi: &v1alpha1::NifiCluster) -> String { + format!("{cluster_name}-node", cluster_name = nifi.name_any()) +} diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 3beb2105..8ea4413f 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -34,6 +34,7 @@ use crate::{ mod config; mod controller; mod crd; +mod listener; mod operations; mod product_logging; mod reporting_task; diff --git a/tests/templates/kuttl/custom-components-git-sync/30-install-nifi.yaml.j2 b/tests/templates/kuttl/custom-components-git-sync/30-install-nifi.yaml.j2 index 16a92f1a..9319e116 100644 --- a/tests/templates/kuttl/custom-components-git-sync/30-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/custom-components-git-sync/30-install-nifi.yaml.j2 @@ -50,6 +50,7 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleConfig: listenerClass: external-unstable podOverrides: spec: diff --git a/tests/templates/kuttl/external-access/15-create-listener-class.yaml b/tests/templates/kuttl/external-access/15-create-listener-class.yaml new file mode 100644 index 00000000..7d5c2ee4 --- /dev/null +++ b/tests/templates/kuttl/external-access/15-create-listener-class.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: envsubst < 15_listener-class.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/external-access/15-create-listener-classes.yaml b/tests/templates/kuttl/external-access/15-create-listener-classes.yaml deleted file mode 100644 index 0fad7e51..00000000 --- a/tests/templates/kuttl/external-access/15-create-listener-classes.yaml +++ /dev/null @@ -1,5 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - - script: envsubst < 15_listener-classes.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/external-access/15_listener-class.yaml b/tests/templates/kuttl/external-access/15_listener-class.yaml new file mode 100644 index 00000000..46f0c64a --- /dev/null +++ b/tests/templates/kuttl/external-access/15_listener-class.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: ListenerClass +metadata: + name: test-external-unstable-$NAMESPACE +spec: + serviceType: NodePort diff --git a/tests/templates/kuttl/external-access/15_listener-classes.yaml b/tests/templates/kuttl/external-access/15_listener-classes.yaml deleted file mode 100644 index 4131526a..00000000 --- a/tests/templates/kuttl/external-access/15_listener-classes.yaml +++ /dev/null @@ -1,21 +0,0 @@ ---- -apiVersion: listeners.stackable.tech/v1alpha1 -kind: ListenerClass -metadata: - name: test-cluster-internal-$NAMESPACE -spec: - serviceType: ClusterIP ---- -apiVersion: listeners.stackable.tech/v1alpha1 -kind: ListenerClass -metadata: - name: test-external-stable-$NAMESPACE -spec: - serviceType: NodePort ---- -apiVersion: listeners.stackable.tech/v1alpha1 -kind: ListenerClass -metadata: - name: test-external-unstable-$NAMESPACE -spec: - serviceType: NodePort diff --git a/tests/templates/kuttl/external-access/30-assert.yaml b/tests/templates/kuttl/external-access/30-assert.yaml index cc1c5125..25907e6a 100644 --- a/tests/templates/kuttl/external-access/30-assert.yaml +++ b/tests/templates/kuttl/external-access/30-assert.yaml @@ -15,48 +15,21 @@ status: readyReplicas: 2 replicas: 2 --- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: test-nifi-node-external-unstable -status: - readyReplicas: 1 - replicas: 1 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: test-nifi-node-cluster-internal -status: - readyReplicas: 1 - replicas: 1 --- apiVersion: policy/v1 kind: PodDisruptionBudget metadata: name: test-nifi-node status: - expectedPods: 4 - currentHealthy: 4 + expectedPods: 2 + currentHealthy: 2 disruptionsAllowed: 1 --- -apiVersion: v1 -kind: Service -metadata: - name: test-nifi-node-cluster-internal -spec: - type: ClusterIP # cluster-internal ---- -apiVersion: v1 -kind: Service -metadata: - name: test-nifi-node-default -spec: - type: NodePort # external-stable --- apiVersion: v1 kind: Service metadata: - name: test-nifi-node-external-unstable + name: test-nifi-node spec: type: NodePort # external-unstable +--- diff --git a/tests/templates/kuttl/external-access/30_nifi.yaml.j2 b/tests/templates/kuttl/external-access/30_nifi.yaml.j2 index 989f5109..dd621497 100644 --- a/tests/templates/kuttl/external-access/30_nifi.yaml.j2 +++ b/tests/templates/kuttl/external-access/30_nifi.yaml.j2 @@ -53,15 +53,8 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} - listenerClass: test-external-stable-$NAMESPACE + roleConfig: + listenerClass: test-external-unstable-$NAMESPACE roleGroups: default: replicas: 2 - external-unstable: - replicas: 1 - config: - listenerClass: test-external-unstable-$NAMESPACE - cluster-internal: - replicas: 1 - config: - listenerClass: test-cluster-internal-$NAMESPACE diff --git a/tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 b/tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 index 58440246..08935314 100644 --- a/tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 +++ b/tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 @@ -7,8 +7,6 @@ spec: image: productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" pullPolicy: IfNotPresent - clusterConfig: - listenerClass: {{ test_scenario['values']['listener-class'] }} {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} @@ -16,6 +14,8 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleConfig: + listenerClass: {{ test_scenario['values']['listener-class'] }} roleGroups: default: replicas: 1 diff --git a/tests/templates/kuttl/iceberg/34_trino.yaml.j2 b/tests/templates/kuttl/iceberg/34_trino.yaml.j2 index e5c1e80c..b5d00488 100644 --- a/tests/templates/kuttl/iceberg/34_trino.yaml.j2 +++ b/tests/templates/kuttl/iceberg/34_trino.yaml.j2 @@ -16,7 +16,6 @@ spec: catalogLabelSelector: matchLabels: trino: trino - listenerClass: external-unstable {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} @@ -24,6 +23,8 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleConfig: + listenerClass: external-unstable {% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} podOverrides: &podOverrides spec: diff --git a/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 b/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 index 9d1a8583..11097e5c 100644 --- a/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 +++ b/tests/templates/kuttl/iceberg/50_nifi.yaml.j2 @@ -49,6 +49,7 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleConfig: listenerClass: external-unstable configOverrides: nifi.properties: diff --git a/tests/templates/kuttl/iceberg/interactive-nifi.yaml b/tests/templates/kuttl/iceberg/interactive-nifi.yaml index 4e07d0ea..81e42630 100644 --- a/tests/templates/kuttl/iceberg/interactive-nifi.yaml +++ b/tests/templates/kuttl/iceberg/interactive-nifi.yaml @@ -38,7 +38,7 @@ spec: storage: "1" storageClassName: secrets.stackable.tech nodes: - config: + roleConfig: listenerClass: external-unstable configOverrides: nifi.properties: diff --git a/tests/templates/kuttl/ldap/12-install-nifi.yaml.j2 b/tests/templates/kuttl/ldap/12-install-nifi.yaml.j2 index b8f1b0e8..7bc54924 100644 --- a/tests/templates/kuttl/ldap/12-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/ldap/12-install-nifi.yaml.j2 @@ -47,9 +47,10 @@ spec: nodes: config: gracefulShutdownTimeout: 1m - listenerClass: external-unstable logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleConfig: + listenerClass: external-unstable roleGroups: default: config: {} diff --git a/tests/templates/kuttl/oidc-opa/30_nifi.yaml.j2 b/tests/templates/kuttl/oidc-opa/30_nifi.yaml.j2 index d24403b7..659476a1 100644 --- a/tests/templates/kuttl/oidc-opa/30_nifi.yaml.j2 +++ b/tests/templates/kuttl/oidc-opa/30_nifi.yaml.j2 @@ -42,6 +42,7 @@ spec: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} gracefulShutdownTimeout: 1s # let the tests run faster + roleConfig: listenerClass: external-unstable configOverrides: nifi.properties: diff --git a/tests/templates/kuttl/smoke_v1/20-install-zookeeper.yaml.j2 b/tests/templates/kuttl/smoke_v1/20-install-zookeeper.yaml.j2 index 103845c4..2080faf6 100644 --- a/tests/templates/kuttl/smoke_v1/20-install-zookeeper.yaml.j2 +++ b/tests/templates/kuttl/smoke_v1/20-install-zookeeper.yaml.j2 @@ -8,7 +8,6 @@ spec: productVersion: "{{ test_scenario['values']['zookeeper'] }}" pullPolicy: IfNotPresent clusterConfig: - listenerClass: {{ test_scenario['values']['listener-class'] }} {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} @@ -16,6 +15,8 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleConfig: + listenerClass: {{ test_scenario['values']['listener-class'] }} roleGroups: default: replicas: 1 diff --git a/tests/templates/kuttl/smoke_v1/30-install-nifi.yaml.j2 b/tests/templates/kuttl/smoke_v1/30-install-nifi.yaml.j2 index 533e3a70..9e6ddfe1 100644 --- a/tests/templates/kuttl/smoke_v1/30-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/smoke_v1/30-install-nifi.yaml.j2 @@ -37,6 +37,7 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleConfig: listenerClass: {{ test_scenario['values']['listener-class'] }} roleGroups: default: diff --git a/tests/templates/kuttl/smoke_v2/20-install-zookeeper.yaml.j2 b/tests/templates/kuttl/smoke_v2/20-install-zookeeper.yaml.j2 index 201b102f..cda0e02f 100644 --- a/tests/templates/kuttl/smoke_v2/20-install-zookeeper.yaml.j2 +++ b/tests/templates/kuttl/smoke_v2/20-install-zookeeper.yaml.j2 @@ -9,7 +9,6 @@ spec: productVersion: "{{ test_scenario['values']['zookeeper'] }}" pullPolicy: IfNotPresent clusterConfig: - listenerClass: {{ test_scenario['values']['listener-class'] }} {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} @@ -17,6 +16,8 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleConfig: + listenerClass: {{ test_scenario['values']['listener-class'] }} roleGroups: default: replicas: 1 diff --git a/tests/templates/kuttl/smoke_v2/30-install-nifi.yaml.j2 b/tests/templates/kuttl/smoke_v2/30-install-nifi.yaml.j2 index 15d9c414..06cfae19 100644 --- a/tests/templates/kuttl/smoke_v2/30-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/smoke_v2/30-install-nifi.yaml.j2 @@ -39,6 +39,7 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleConfig: listenerClass: {{ test_scenario['values']['listener-class'] }} roleGroups: default: From ef07f0c457e3f1c61778f7c6f0f8971dc2ed71bf Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 20 Jun 2025 13:32:50 +0200 Subject: [PATCH 19/37] use new headless service name in integration tests --- tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml | 2 +- tests/templates/kuttl/smoke_v1/test_nifi.py | 2 +- tests/templates/kuttl/smoke_v1/test_nifi_metrics.py | 2 +- tests/templates/kuttl/smoke_v2/test_nifi.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml b/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml index 7ab671ce..33045ae4 100644 --- a/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml +++ b/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml @@ -54,7 +54,7 @@ data: import urllib3 # As of 2022-08-29 we cant use "https://nifi:8443" here because

The request contained an invalid host header [nifi:8443] in the request [/nifi-api]. Check for request manipulation or third-party intercept.

- ENDPOINT = f"https://nifi-node-default-0.nifi-node-default.{os.environ['NAMESPACE']}.svc.cluster.local:8443" # For local testing / developing replace it, afterwards change back to f"https://nifi-node-default-0.nifi-node-default.{os.environ['NAMESPACE']}.svc.cluster.local:8443" + ENDPOINT = f"https://nifi-node-default-0.nifi-node-default-metrics.{os.environ['NAMESPACE']}.svc.cluster.local:8443" # For local testing / developing replace it, afterwards change back to f"https://nifi-node-default-0.nifi-node-default-metrics.{os.environ['NAMESPACE']}.svc.cluster.local:8443" USERNAME = "admin" PASSWORD = open("/nifi-users/admin").read() diff --git a/tests/templates/kuttl/smoke_v1/test_nifi.py b/tests/templates/kuttl/smoke_v1/test_nifi.py index 60740afe..29fb0242 100755 --- a/tests/templates/kuttl/smoke_v1/test_nifi.py +++ b/tests/templates/kuttl/smoke_v1/test_nifi.py @@ -46,7 +46,7 @@ def get_token(nifi_host, username, password): # disable warnings as we have specified non-verified https connections urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - host = f"https://nifi-node-default-1.nifi-node-default.{args['namespace']}.svc.cluster.local:8443" + host = f"https://nifi-node-default-1.nifi-node-default-metrics.{args['namespace']}.svc.cluster.local:8443" token = get_token(host, args["user"], args["password"]) headers = {"Authorization": token} node_count = int(args["count"]) diff --git a/tests/templates/kuttl/smoke_v1/test_nifi_metrics.py b/tests/templates/kuttl/smoke_v1/test_nifi_metrics.py index fd383ab0..f8cd20df 100755 --- a/tests/templates/kuttl/smoke_v1/test_nifi_metrics.py +++ b/tests/templates/kuttl/smoke_v1/test_nifi_metrics.py @@ -39,7 +39,7 @@ port = args["port"] timeout = int(args["timeout"]) - url = f"http://nifi-node-default-0.nifi-node-default.{namespace}.svc.cluster.local:{port}/metrics" + url = f"http://nifi-node-default-0.nifi-node-default-metrics.{namespace}.svc.cluster.local:{port}/metrics" # wait for 'timeout' seconds t_end = time.time() + timeout diff --git a/tests/templates/kuttl/smoke_v2/test_nifi.py b/tests/templates/kuttl/smoke_v2/test_nifi.py index 60740afe..29fb0242 100755 --- a/tests/templates/kuttl/smoke_v2/test_nifi.py +++ b/tests/templates/kuttl/smoke_v2/test_nifi.py @@ -46,7 +46,7 @@ def get_token(nifi_host, username, password): # disable warnings as we have specified non-verified https connections urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - host = f"https://nifi-node-default-1.nifi-node-default.{args['namespace']}.svc.cluster.local:8443" + host = f"https://nifi-node-default-1.nifi-node-default-metrics.{args['namespace']}.svc.cluster.local:8443" token = get_token(host, args["user"], args["password"]) headers = {"Authorization": token} node_count = int(args["count"]) From b02944088ce4e4b507f6de698bb3c249ea7f5ec1 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Mon, 23 Jun 2025 10:36:12 +0200 Subject: [PATCH 20/37] move listener constants to listener module --- rust/operator-binary/src/controller.rs | 12 +++++++----- rust/operator-binary/src/crd/mod.rs | 2 -- rust/operator-binary/src/listener.rs | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 85e8bf74..565e02ae 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -83,12 +83,14 @@ use crate::{ }, crd::{ APP_NAME, BALANCE_PORT, BALANCE_PORT_NAME, Container, HTTPS_PORT, HTTPS_PORT_NAME, - LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, METRICS_PORT, METRICS_PORT_NAME, NifiConfig, - NifiConfigFragment, NifiNodeRoleConfig, NifiRole, NifiStatus, PROTOCOL_PORT, - PROTOCOL_PORT_NAME, STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, - authentication::AuthenticationClassResolved, v1alpha1, + METRICS_PORT, METRICS_PORT_NAME, NifiConfig, NifiConfigFragment, NifiNodeRoleConfig, + NifiRole, NifiStatus, PROTOCOL_PORT, PROTOCOL_PORT_NAME, STACKABLE_LOG_CONFIG_DIR, + STACKABLE_LOG_DIR, authentication::AuthenticationClassResolved, v1alpha1, + }, + listener::{ + LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_group_listener, build_group_listener_pvc, + group_listener_name, }, - listener::{build_group_listener, build_group_listener_pvc, group_listener_name}, operations::{ graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 38afe308..122b4ea7 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -54,8 +54,6 @@ pub const BALANCE_PORT: u16 = 6243; pub const METRICS_PORT_NAME: &str = "metrics"; pub const METRICS_PORT: u16 = 8081; -pub const LISTENER_VOLUME_NAME: &str = "listener"; -pub const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; pub const STACKABLE_LOG_CONFIG_DIR: &str = "/stackable/log_config"; diff --git a/rust/operator-binary/src/listener.rs b/rust/operator-binary/src/listener.rs index 51f197a7..6da9dba9 100644 --- a/rust/operator-binary/src/listener.rs +++ b/rust/operator-binary/src/listener.rs @@ -10,7 +10,10 @@ use stackable_operator::{ kvp::{Labels, ObjectLabels}, }; -use crate::crd::{HTTPS_PORT, HTTPS_PORT_NAME, LISTENER_VOLUME_NAME, v1alpha1}; +use crate::crd::{HTTPS_PORT, HTTPS_PORT_NAME, v1alpha1}; + +pub const LISTENER_VOLUME_NAME: &str = "listener"; +pub const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; #[derive(Snafu, Debug)] pub enum Error { From a4e33135088601d342b787b663ea37a74db54bfe Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Mon, 23 Jun 2025 10:36:54 +0200 Subject: [PATCH 21/37] remove duplicate iceberg test --- tests/test-definition.yaml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 830887a6..c35b2f99 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -157,18 +157,6 @@ tests: - nifi - zookeeper-latest - openshift - - name: iceberg - dimensions: - - nifi-iceberg - - opa-l - - zookeeper-latest - - hdfs-l - - hive-l - - trino-l - - krb5 - - iceberg-use-kerberos - - kerberos-realm - - openshift suites: - name: nightly patch: From eb062911ddb072f0ec3878e8820319fcb5b41a85 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Mon, 23 Jun 2025 10:55:12 +0200 Subject: [PATCH 22/37] remove hard-coded names --- rust/operator-binary/src/controller.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 565e02ae..7bd5f8ef 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -580,7 +580,7 @@ pub async fn reconcile_nifi( let role_group_listener = build_group_listener( nifi, - build_recommended_labels(nifi, NIFI_CONTROLLER_NAME, "node", "none"), + build_recommended_labels(nifi, NIFI_CONTROLLER_NAME, &nifi_role.to_string(), "none"), listener_class.to_owned(), group_listener_name(nifi), ) @@ -933,9 +933,8 @@ async fn build_node_rolegroup_statefulset( env_vars.extend(authorization_config.get_env_vars()); let node_address = format!( - "$POD_NAME.{}-node-{}-metrics.{}.svc.{}", - rolegroup_ref.cluster.name, - rolegroup_ref.role_group, + "$POD_NAME.{}.{}.svc.{}", + rolegroup_service_name(rolegroup_ref), &nifi .metadata .namespace From 3c7850f5fbf77171769edbd52d5ff9f2ebc69e6e Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Mon, 23 Jun 2025 16:26:54 +0200 Subject: [PATCH 23/37] fix integration tests --- .../operator-binary/src/reporting_task/mod.rs | 2 +- .../custom-components-git-sync/20-assert.yaml | 2 +- .../templates/kuttl/iceberg/34_trino.yaml.j2 | 3 +- tests/templates/kuttl/ldap/test_nifi.py | 60 +++++++++++-------- tests/templates/kuttl/oidc-opa/test.py | 2 +- .../templates/kuttl/upgrade/04-assert.yaml.j2 | 4 +- .../templates/kuttl/upgrade/07-assert.yaml.j2 | 2 +- tests/templates/kuttl/upgrade/test_nifi.py | 60 +++++++++++-------- .../kuttl/upgrade/test_nifi_metrics.py | 40 +++++++++---- 9 files changed, 106 insertions(+), 69 deletions(-) diff --git a/rust/operator-binary/src/reporting_task/mod.rs b/rust/operator-binary/src/reporting_task/mod.rs index b36841b8..c0ff7928 100644 --- a/rust/operator-binary/src/reporting_task/mod.rs +++ b/rust/operator-binary/src/reporting_task/mod.rs @@ -8,7 +8,7 @@ //! Due to changes in the JWT validation in 1.25.0, the issuer refers to the FQDN of the Pod that was created, e.g.: //! { //! "sub": "admin", -//! "iss": "test-nifi-node-default-0.test-nifi-node-default.default.svc.cluster.local:8443", +//! "iss": "test-nifi-node-default-0.test-nifi-node-default-metrics.default.svc.cluster.local:8443", //! } //! which was different in e.g. 1.23.2 //! { diff --git a/tests/templates/kuttl/custom-components-git-sync/20-assert.yaml b/tests/templates/kuttl/custom-components-git-sync/20-assert.yaml index 49ba7437..e0766c49 100644 --- a/tests/templates/kuttl/custom-components-git-sync/20-assert.yaml +++ b/tests/templates/kuttl/custom-components-git-sync/20-assert.yaml @@ -6,7 +6,7 @@ timeout: 600 apiVersion: apps/v1 kind: StatefulSet metadata: - name: zookeeper-server-default + name: test-zk-server-default status: readyReplicas: 1 replicas: 1 diff --git a/tests/templates/kuttl/iceberg/34_trino.yaml.j2 b/tests/templates/kuttl/iceberg/34_trino.yaml.j2 index b5d00488..e5c1e80c 100644 --- a/tests/templates/kuttl/iceberg/34_trino.yaml.j2 +++ b/tests/templates/kuttl/iceberg/34_trino.yaml.j2 @@ -16,6 +16,7 @@ spec: catalogLabelSelector: matchLabels: trino: trino + listenerClass: external-unstable {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} @@ -23,8 +24,6 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} - roleConfig: - listenerClass: external-unstable {% if test_scenario['values']['iceberg-use-kerberos'] == 'true' %} podOverrides: &podOverrides spec: diff --git a/tests/templates/kuttl/ldap/test_nifi.py b/tests/templates/kuttl/ldap/test_nifi.py index b885c09f..c98b5d8f 100755 --- a/tests/templates/kuttl/ldap/test_nifi.py +++ b/tests/templates/kuttl/ldap/test_nifi.py @@ -8,61 +8,71 @@ def get_token(nifi_host, username, password): nifi_headers = { - 'content-type': 'application/x-www-form-urlencoded; charset=UTF-8', + "content-type": "application/x-www-form-urlencoded; charset=UTF-8", } - data = {'username': username, 'password': password} + data = {"username": username, "password": password} # TODO: handle actual errors when connecting properly - nifi_url = nifi_host + '/nifi-api/access/token' - response = requests.post(nifi_url, headers=nifi_headers, data=data, verify=False) # , cert='./tmp/cacert.pem') + nifi_url = nifi_host + "/nifi-api/access/token" + response = requests.post( + nifi_url, headers=nifi_headers, data=data, verify=False + ) # , cert='./tmp/cacert.pem') if response.ok: - nifi_token = response.content.decode('utf-8') + nifi_token = response.content.decode("utf-8") return "Bearer " + nifi_token else: print(f"Failed to get token: {response.status_code}: {response.content}") exit(-1) -if __name__ == '__main__': +if __name__ == "__main__": # Construct an argument parser all_args = argparse.ArgumentParser() # Add arguments to the parser - all_args.add_argument("-u", "--user", required=True, - help="Username to connect as") - all_args.add_argument("-p", "--password", required=True, - help="Password for the user") - all_args.add_argument("-n", "--namespace", required=True, - help="Namespace the test is running in") - all_args.add_argument("-c", "--count", required=True, - help="The expected number of Nodes") + all_args.add_argument("-u", "--user", required=True, help="Username to connect as") + all_args.add_argument( + "-p", "--password", required=True, help="Password for the user" + ) + all_args.add_argument( + "-n", "--namespace", required=True, help="Namespace the test is running in" + ) + all_args.add_argument( + "-c", "--count", required=True, help="The expected number of Nodes" + ) args = vars(all_args.parse_args()) # disable warnings as we have specified non-verified https connections urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - host = f"https://test-nifi-node-default-1.test-nifi-node-default.{args['namespace']}.svc.cluster.local:8443" - token = get_token(host, args['user'], args['password']) - headers = {'Authorization': token} - node_count = int(args['count']) + host = f"https://test-nifi-node-default-1.test-nifi-node-default-metrics.{args['namespace']}.svc.cluster.local:8443" + token = get_token(host, args["user"], args["password"]) + headers = {"Authorization": token} + node_count = int(args["count"]) x = 0 while x < 15: - url = host + '/nifi-api/controller/cluster' - cluster = requests.get(url, headers=headers, verify=False) # , cert='/tmp/cacert.pem') + url = host + "/nifi-api/controller/cluster" + cluster = requests.get( + url, headers=headers, verify=False + ) # , cert='/tmp/cacert.pem') if cluster.status_code != 200: print("Waiting for cluster...") else: - cluster_data = json.loads(cluster.content.decode('utf-8')) - nodes = cluster_data['cluster']['nodes'] + cluster_data = json.loads(cluster.content.decode("utf-8")) + nodes = cluster_data["cluster"]["nodes"] if len(nodes) != node_count: - print(f"Cluster should have {node_count} nodes at this stage, but has: {len(nodes)}") + print( + f"Cluster should have {node_count} nodes at this stage, but has: {len(nodes)}" + ) else: connected = True for node in nodes: - if node['status'] != "CONNECTED": - print(f"Node {node['nodeId']} is in state {node['status']} but should have been CONNECTED") + if node["status"] != "CONNECTED": + print( + f"Node {node['nodeId']} is in state {node['status']} but should have been CONNECTED" + ) connected = False if connected: print("Test succeeded!") diff --git a/tests/templates/kuttl/oidc-opa/test.py b/tests/templates/kuttl/oidc-opa/test.py index a7e96d55..7b7e43fd 100644 --- a/tests/templates/kuttl/oidc-opa/test.py +++ b/tests/templates/kuttl/oidc-opa/test.py @@ -17,7 +17,7 @@ namespace = os.environ["NAMESPACE"] tls = os.environ["OIDC_USE_TLS"] nifi_version = os.environ["NIFI_VERSION"] -nifi = f"test-nifi-node-default-0.test-nifi-node-default.{namespace}.svc.cluster.local" +nifi = f"test-nifi-node-default-0.test-nifi-node-default-metrics.{namespace}.svc.cluster.local" keycloak_service = f"keycloak.{namespace}.svc.cluster.local" keycloak_base_url = ( diff --git a/tests/templates/kuttl/upgrade/04-assert.yaml.j2 b/tests/templates/kuttl/upgrade/04-assert.yaml.j2 index 8bcf1c8e..3e538737 100644 --- a/tests/templates/kuttl/upgrade/04-assert.yaml.j2 +++ b/tests/templates/kuttl/upgrade/04-assert.yaml.j2 @@ -8,9 +8,9 @@ commands: - script: kubectl exec -n $NAMESPACE test-nifi-0 -- python /tmp/test_nifi_metrics.py -n $NAMESPACE {% endif %} {% if test_scenario['values']['nifi_old'].split(',')[0] == '2.0.0' %} - - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default.$NAMESPACE.svc.cluster.local:8443 run json /tmp/generate-and-log-flowfiles.json > /tmp/old_input" + - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default-metrics.$NAMESPACE.svc.cluster.local:8443 run json /tmp/generate-and-log-flowfiles.json > /tmp/old_input" {% else %} - - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default.$NAMESPACE.svc.cluster.local:8443 run template /tmp/generate-and-log-flowfiles.xml > /tmp/old_input" + - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default-metrics.$NAMESPACE.svc.cluster.local:8443 run template /tmp/generate-and-log-flowfiles.xml > /tmp/old_input" {% endif %} # This tests if the output contains an Error or zero flow files are queued, which also indicates that something went wrong - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "cat /tmp/old_input | grep -Eov 'Error|\b0\b'" diff --git a/tests/templates/kuttl/upgrade/07-assert.yaml.j2 b/tests/templates/kuttl/upgrade/07-assert.yaml.j2 index a71c5a37..c0d0f7ad 100644 --- a/tests/templates/kuttl/upgrade/07-assert.yaml.j2 +++ b/tests/templates/kuttl/upgrade/07-assert.yaml.j2 @@ -9,7 +9,7 @@ commands: {% if test_scenario['values']['nifi_new'].split(',')[0].startswith('1.') %} - script: kubectl exec -n $NAMESPACE test-nifi-0 -- python /tmp/test_nifi_metrics.py -n $NAMESPACE {% endif %} - - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default.$NAMESPACE.svc.cluster.local:8443 query > /tmp/new_input" + - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default-metrics.$NAMESPACE.svc.cluster.local:8443 query > /tmp/new_input" # This tests if the output contains an Error or zero flow files are queued, which also indicates that something went wrong - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "cat /tmp/new_input | grep -Eov 'Error|\b0\b'" # This tests that the number of input records stays the same after the upgrade. diff --git a/tests/templates/kuttl/upgrade/test_nifi.py b/tests/templates/kuttl/upgrade/test_nifi.py index b885c09f..c98b5d8f 100755 --- a/tests/templates/kuttl/upgrade/test_nifi.py +++ b/tests/templates/kuttl/upgrade/test_nifi.py @@ -8,61 +8,71 @@ def get_token(nifi_host, username, password): nifi_headers = { - 'content-type': 'application/x-www-form-urlencoded; charset=UTF-8', + "content-type": "application/x-www-form-urlencoded; charset=UTF-8", } - data = {'username': username, 'password': password} + data = {"username": username, "password": password} # TODO: handle actual errors when connecting properly - nifi_url = nifi_host + '/nifi-api/access/token' - response = requests.post(nifi_url, headers=nifi_headers, data=data, verify=False) # , cert='./tmp/cacert.pem') + nifi_url = nifi_host + "/nifi-api/access/token" + response = requests.post( + nifi_url, headers=nifi_headers, data=data, verify=False + ) # , cert='./tmp/cacert.pem') if response.ok: - nifi_token = response.content.decode('utf-8') + nifi_token = response.content.decode("utf-8") return "Bearer " + nifi_token else: print(f"Failed to get token: {response.status_code}: {response.content}") exit(-1) -if __name__ == '__main__': +if __name__ == "__main__": # Construct an argument parser all_args = argparse.ArgumentParser() # Add arguments to the parser - all_args.add_argument("-u", "--user", required=True, - help="Username to connect as") - all_args.add_argument("-p", "--password", required=True, - help="Password for the user") - all_args.add_argument("-n", "--namespace", required=True, - help="Namespace the test is running in") - all_args.add_argument("-c", "--count", required=True, - help="The expected number of Nodes") + all_args.add_argument("-u", "--user", required=True, help="Username to connect as") + all_args.add_argument( + "-p", "--password", required=True, help="Password for the user" + ) + all_args.add_argument( + "-n", "--namespace", required=True, help="Namespace the test is running in" + ) + all_args.add_argument( + "-c", "--count", required=True, help="The expected number of Nodes" + ) args = vars(all_args.parse_args()) # disable warnings as we have specified non-verified https connections urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - host = f"https://test-nifi-node-default-1.test-nifi-node-default.{args['namespace']}.svc.cluster.local:8443" - token = get_token(host, args['user'], args['password']) - headers = {'Authorization': token} - node_count = int(args['count']) + host = f"https://test-nifi-node-default-1.test-nifi-node-default-metrics.{args['namespace']}.svc.cluster.local:8443" + token = get_token(host, args["user"], args["password"]) + headers = {"Authorization": token} + node_count = int(args["count"]) x = 0 while x < 15: - url = host + '/nifi-api/controller/cluster' - cluster = requests.get(url, headers=headers, verify=False) # , cert='/tmp/cacert.pem') + url = host + "/nifi-api/controller/cluster" + cluster = requests.get( + url, headers=headers, verify=False + ) # , cert='/tmp/cacert.pem') if cluster.status_code != 200: print("Waiting for cluster...") else: - cluster_data = json.loads(cluster.content.decode('utf-8')) - nodes = cluster_data['cluster']['nodes'] + cluster_data = json.loads(cluster.content.decode("utf-8")) + nodes = cluster_data["cluster"]["nodes"] if len(nodes) != node_count: - print(f"Cluster should have {node_count} nodes at this stage, but has: {len(nodes)}") + print( + f"Cluster should have {node_count} nodes at this stage, but has: {len(nodes)}" + ) else: connected = True for node in nodes: - if node['status'] != "CONNECTED": - print(f"Node {node['nodeId']} is in state {node['status']} but should have been CONNECTED") + if node["status"] != "CONNECTED": + print( + f"Node {node['nodeId']} is in state {node['status']} but should have been CONNECTED" + ) connected = False if connected: print("Test succeeded!") diff --git a/tests/templates/kuttl/upgrade/test_nifi_metrics.py b/tests/templates/kuttl/upgrade/test_nifi_metrics.py index 86076038..d8619361 100755 --- a/tests/templates/kuttl/upgrade/test_nifi_metrics.py +++ b/tests/templates/kuttl/upgrade/test_nifi_metrics.py @@ -4,18 +4,34 @@ import time from requests.exceptions import ConnectionError -if __name__ == '__main__': +if __name__ == "__main__": # Construct an argument parser all_args = argparse.ArgumentParser() # Add arguments to the parser - all_args.add_argument("-m", "--metric", required=False, default="nifi_amount_bytes_read", - help="The name of a certain metric to check") - all_args.add_argument("-n", "--namespace", required=True, - help="The namespace the test is running in") - all_args.add_argument("-p", "--port", required=False, default="8081", - help="The port where metrics are exposed") - all_args.add_argument("-t", "--timeout", required=False, default="120", - help="The timeout in seconds to wait for the metrics port to be opened") + all_args.add_argument( + "-m", + "--metric", + required=False, + default="nifi_amount_bytes_read", + help="The name of a certain metric to check", + ) + all_args.add_argument( + "-n", "--namespace", required=True, help="The namespace the test is running in" + ) + all_args.add_argument( + "-p", + "--port", + required=False, + default="8081", + help="The port where metrics are exposed", + ) + all_args.add_argument( + "-t", + "--timeout", + required=False, + default="120", + help="The timeout in seconds to wait for the metrics port to be opened", + ) args = vars(all_args.parse_args()) metric_name = args["metric"] @@ -23,7 +39,7 @@ port = args["port"] timeout = int(args["timeout"]) - url = f"http://test-nifi-node-default-0.test-nifi-node-default.{namespace}.svc.cluster.local:{port}/metrics" + url = f"http://test-nifi-node-default-0.test-nifi-node-default-metrics.{namespace}.svc.cluster.local:{port}/metrics" # wait for 'timeout' seconds t_end = time.time() + timeout @@ -35,7 +51,9 @@ print("Test metrics succeeded!") exit(0) else: - print(f"Could not find metric [{metric_name}] in response:\n {response.text}") + print( + f"Could not find metric [{metric_name}] in response:\n {response.text}" + ) time.sleep(timeout) except ConnectionError: # NewConnectionError is expected until metrics are available From 79a193081d3cd1b21f843c9c0018d1b17af0c84a Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Mon, 23 Jun 2025 16:29:25 +0200 Subject: [PATCH 24/37] remove hardcoded role name --- rust/operator-binary/src/controller.rs | 21 +++++++++++++-------- rust/operator-binary/src/listener.rs | 4 ++-- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 7bd5f8ef..af3192b1 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -578,11 +578,14 @@ pub async fn reconcile_nifi( .await .context(FailedToCreatePdbSnafu)?; + let role_name = match &nifi_role { + NifiRole::Node => "node".to_string(), + }; let role_group_listener = build_group_listener( nifi, build_recommended_labels(nifi, NIFI_CONTROLLER_NAME, &nifi_role.to_string(), "none"), listener_class.to_owned(), - group_listener_name(nifi), + group_listener_name(nifi, &role_name), ) .context(ListenerConfigurationSnafu)?; @@ -933,14 +936,14 @@ async fn build_node_rolegroup_statefulset( env_vars.extend(authorization_config.get_env_vars()); let node_address = format!( - "$POD_NAME.{}.{}.svc.{}", - rolegroup_service_name(rolegroup_ref), - &nifi + "$POD_NAME.{service_name}.{namespace}.svc.{cluster_domain}", + service_name = rolegroup_service_name(rolegroup_ref), + namespace = &nifi .metadata .namespace .as_ref() .context(ObjectHasNoNamespaceSnafu)?, - cluster_info.cluster_domain, + cluster_domain = cluster_info.cluster_domain, ); let sensitive_key_secret = &nifi.spec.cluster_config.sensitive_properties.key_secret; @@ -1191,9 +1194,11 @@ async fn build_node_rolegroup_statefulset( // listener endpoints will use persistent volumes // so that load balancers can hard-code the target addresses and // that it is possible to connect to a consistent address - let listener_pvc = - build_group_listener_pvc(&group_listener_name(nifi), &unversioned_recommended_labels) - .context(ListenerConfigurationSnafu)?; + let listener_pvc = build_group_listener_pvc( + &group_listener_name(nifi, &rolegroup_ref.role), + &unversioned_recommended_labels, + ) + .context(ListenerConfigurationSnafu)?; add_graceful_shutdown_config(merged_config, &mut pod_builder).context(GracefulShutdownSnafu)?; diff --git a/rust/operator-binary/src/listener.rs b/rust/operator-binary/src/listener.rs index 6da9dba9..fb0e4ab8 100644 --- a/rust/operator-binary/src/listener.rs +++ b/rust/operator-binary/src/listener.rs @@ -74,6 +74,6 @@ pub fn build_group_listener_pvc( .context(BuildListenerPersistentVolumeSnafu) } -pub fn group_listener_name(nifi: &v1alpha1::NifiCluster) -> String { - format!("{cluster_name}-node", cluster_name = nifi.name_any()) +pub fn group_listener_name(nifi: &v1alpha1::NifiCluster, role_name: &String) -> String { + format!("{cluster_name}-{role_name}", cluster_name = nifi.name_any(),) } From f38deac7e422c1494e54a4d750c62d7758a652ed Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Mon, 23 Jun 2025 16:52:31 +0200 Subject: [PATCH 25/37] improve code quality --- rust/operator-binary/src/controller.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index af3192b1..7ae15d11 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -578,21 +578,18 @@ pub async fn reconcile_nifi( .await .context(FailedToCreatePdbSnafu)?; - let role_name = match &nifi_role { - NifiRole::Node => "node".to_string(), - }; let role_group_listener = build_group_listener( nifi, build_recommended_labels(nifi, NIFI_CONTROLLER_NAME, &nifi_role.to_string(), "none"), listener_class.to_owned(), - group_listener_name(nifi, &role_name), + group_listener_name(nifi, &nifi_role.to_string()), ) .context(ListenerConfigurationSnafu)?; cluster_resources .add(client, role_group_listener) .await - .with_context(|_| ApplyGroupListenerSnafu)?; + .context(ApplyGroupListenerSnafu)?; } // Only add the reporting task in case it is enabled. From bbbc4ad60b2de208f3dee65d30ab43aed06d4ff1 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Mon, 23 Jun 2025 18:27:40 +0200 Subject: [PATCH 26/37] set rolegroup label on listener pvcs to none --- rust/operator-binary/src/controller.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 7ae15d11..53b57213 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -1184,7 +1184,7 @@ async fn build_node_rolegroup_statefulset( // A version value is required, and we do want to use the "recommended" format for the other desired labels "none", &rolegroup_ref.role, - &rolegroup_ref.role_group, + "none", )) .context(LabelBuildSnafu)?; From c5c2264cacc8bbb16d6c9a3b5f7d497bae47b07b Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 24 Jun 2025 11:21:18 +0200 Subject: [PATCH 27/37] fix app version label on listener --- rust/operator-binary/src/controller.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 53b57213..977f1c05 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -580,7 +580,12 @@ pub async fn reconcile_nifi( let role_group_listener = build_group_listener( nifi, - build_recommended_labels(nifi, NIFI_CONTROLLER_NAME, &nifi_role.to_string(), "none"), + build_recommended_labels( + nifi, + &resolved_product_image.app_version_label, + &nifi_role.to_string(), + "none", + ), listener_class.to_owned(), group_listener_name(nifi, &nifi_role.to_string()), ) From 5629842dc773d870a4eca8a036d4dd11b395a5d1 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 24 Jun 2025 11:26:22 +0200 Subject: [PATCH 28/37] set rolegroup label on listener pvcs --- rust/operator-binary/src/controller.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 977f1c05..c77b7a54 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -1189,7 +1189,7 @@ async fn build_node_rolegroup_statefulset( // A version value is required, and we do want to use the "recommended" format for the other desired labels "none", &rolegroup_ref.role, - "none", + &rolegroup_ref.role_group, )) .context(LabelBuildSnafu)?; From 286072073a4c575fe5c2114c403016c90c5b46af Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 24 Jun 2025 17:04:41 +0200 Subject: [PATCH 29/37] remove listener class from trino in iceberg test --- tests/templates/kuttl/iceberg/34_trino.yaml.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/templates/kuttl/iceberg/34_trino.yaml.j2 b/tests/templates/kuttl/iceberg/34_trino.yaml.j2 index e5c1e80c..9d8b5415 100644 --- a/tests/templates/kuttl/iceberg/34_trino.yaml.j2 +++ b/tests/templates/kuttl/iceberg/34_trino.yaml.j2 @@ -16,7 +16,6 @@ spec: catalogLabelSelector: matchLabels: trino: trino - listenerClass: external-unstable {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} From b8c87e32f94c7a6c21f3c700bf927da96baf07f5 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 26 Jun 2025 11:20:17 +0200 Subject: [PATCH 30/37] add note on custom ListenerClasses to docs --- docs/modules/nifi/pages/usage_guide/listenerclass.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/nifi/pages/usage_guide/listenerclass.adoc b/docs/modules/nifi/pages/usage_guide/listenerclass.adoc index daf8595f..03d0d47f 100644 --- a/docs/modules/nifi/pages/usage_guide/listenerclass.adoc +++ b/docs/modules/nifi/pages/usage_guide/listenerclass.adoc @@ -11,4 +11,4 @@ spec: roleConfig: listenerClass: external-unstable # <1> ---- -<1> Specify one of `external-stable`, `external-unstable`, `cluster-internal` (the default setting is `cluster-internal`). +<1> Specify one of `external-stable`, `external-unstable`, `cluster-internal` or a custom ListenerClass (the default setting is `cluster-internal`). From 074d138fef1e6d2472ff7b265e1651faba0677ce Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 26 Jun 2025 17:18:42 +0200 Subject: [PATCH 31/37] remove version argument in reporting task --- rust/operator-binary/src/reporting_task/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rust/operator-binary/src/reporting_task/mod.rs b/rust/operator-binary/src/reporting_task/mod.rs index c0ff7928..44121031 100644 --- a/rust/operator-binary/src/reporting_task/mod.rs +++ b/rust/operator-binary/src/reporting_task/mod.rs @@ -295,7 +295,6 @@ fn build_reporting_task_job( format!("-n {nifi_connect_url}"), user_name_command, format!("-p \"$(cat {admin_password_file})\""), - format!("-v {product_version}"), format!("-m {METRICS_PORT}"), format!("-c {REPORTING_TASK_CERT_VOLUME_MOUNT}/ca.crt"), ]; From 3f8d8407ffa3250b4880cfabe2cf6c6432094614 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Mon, 30 Jun 2025 10:49:36 +0200 Subject: [PATCH 32/37] rename headless service --- docs/modules/nifi/pages/usage_guide/monitoring.adoc | 4 ++-- rust/operator-binary/src/controller.rs | 2 +- rust/operator-binary/src/reporting_task/mod.rs | 2 +- tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml | 2 +- tests/templates/kuttl/ldap/test_nifi.py | 2 +- tests/templates/kuttl/oidc-opa/test.py | 2 +- tests/templates/kuttl/smoke_v1/test_nifi.py | 2 +- tests/templates/kuttl/smoke_v1/test_nifi_metrics.py | 2 +- tests/templates/kuttl/smoke_v2/test_nifi.py | 2 +- tests/templates/kuttl/upgrade/04-assert.yaml.j2 | 4 ++-- tests/templates/kuttl/upgrade/07-assert.yaml.j2 | 2 +- tests/templates/kuttl/upgrade/test_nifi.py | 2 +- tests/templates/kuttl/upgrade/test_nifi_metrics.py | 2 +- 13 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/modules/nifi/pages/usage_guide/monitoring.adoc b/docs/modules/nifi/pages/usage_guide/monitoring.adoc index 355f96b6..e3e102ea 100644 --- a/docs/modules/nifi/pages/usage_guide/monitoring.adoc +++ b/docs/modules/nifi/pages/usage_guide/monitoring.adoc @@ -127,7 +127,7 @@ spec: - __meta_kubernetes_pod_container_port_number targetLabel: __address__ replacement: ${1}.${2}.${3}.svc.cluster.local:${4} - regex: (.+);(.+?)(?:-metrics)?;(.+);(.+) + regex: (.+);(.+?)(?:-headless)?;(.+);(.+) selector: matchLabels: prometheus.io/scrape: "true" @@ -138,4 +138,4 @@ spec: <1> Authorization via Bearer Token stored in a secret <2> Relabel \\__address__ to be a FQDN rather then the IP-Address of target pod -NOTE: As of xref:listener-operator:listener.adoc[Listener] integration, SDP exposes a Service with `-metrics` thus we need to regex this suffix. +NOTE: As of xref:listener-operator:listener.adoc[Listener] integration, SDP exposes a Service with `-headless` thus we need to regex this suffix. diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index c77b7a54..33f2a681 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -1483,7 +1483,7 @@ async fn build_node_rolegroup_statefulset( } pub fn rolegroup_service_name(rolegroup: &RoleGroupRef) -> String { - format!("{name}-metrics", name = rolegroup.object_name()) + format!("{name}-headless", name = rolegroup.object_name()) } async fn get_proxy_hosts( diff --git a/rust/operator-binary/src/reporting_task/mod.rs b/rust/operator-binary/src/reporting_task/mod.rs index 44121031..be7e1446 100644 --- a/rust/operator-binary/src/reporting_task/mod.rs +++ b/rust/operator-binary/src/reporting_task/mod.rs @@ -8,7 +8,7 @@ //! Due to changes in the JWT validation in 1.25.0, the issuer refers to the FQDN of the Pod that was created, e.g.: //! { //! "sub": "admin", -//! "iss": "test-nifi-node-default-0.test-nifi-node-default-metrics.default.svc.cluster.local:8443", +//! "iss": "test-nifi-node-default-0.test-nifi-node-default-headless.default.svc.cluster.local:8443", //! } //! which was different in e.g. 1.23.2 //! { diff --git a/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml b/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml index 33045ae4..3666b063 100644 --- a/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml +++ b/tests/templates/kuttl/iceberg/61-provision-nifi-flow.yaml @@ -54,7 +54,7 @@ data: import urllib3 # As of 2022-08-29 we cant use "https://nifi:8443" here because

The request contained an invalid host header [nifi:8443] in the request [/nifi-api]. Check for request manipulation or third-party intercept.

- ENDPOINT = f"https://nifi-node-default-0.nifi-node-default-metrics.{os.environ['NAMESPACE']}.svc.cluster.local:8443" # For local testing / developing replace it, afterwards change back to f"https://nifi-node-default-0.nifi-node-default-metrics.{os.environ['NAMESPACE']}.svc.cluster.local:8443" + ENDPOINT = f"https://nifi-node-default-0.nifi-node-default-headless.{os.environ['NAMESPACE']}.svc.cluster.local:8443" # For local testing / developing replace it, afterwards change back to f"https://nifi-node-default-0.nifi-node-default-headless.{os.environ['NAMESPACE']}.svc.cluster.local:8443" USERNAME = "admin" PASSWORD = open("/nifi-users/admin").read() diff --git a/tests/templates/kuttl/ldap/test_nifi.py b/tests/templates/kuttl/ldap/test_nifi.py index c98b5d8f..3b5cee3e 100755 --- a/tests/templates/kuttl/ldap/test_nifi.py +++ b/tests/templates/kuttl/ldap/test_nifi.py @@ -46,7 +46,7 @@ def get_token(nifi_host, username, password): # disable warnings as we have specified non-verified https connections urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - host = f"https://test-nifi-node-default-1.test-nifi-node-default-metrics.{args['namespace']}.svc.cluster.local:8443" + host = f"https://test-nifi-node-default-1.test-nifi-node-default-headless.{args['namespace']}.svc.cluster.local:8443" token = get_token(host, args["user"], args["password"]) headers = {"Authorization": token} node_count = int(args["count"]) diff --git a/tests/templates/kuttl/oidc-opa/test.py b/tests/templates/kuttl/oidc-opa/test.py index 7b7e43fd..608561d5 100644 --- a/tests/templates/kuttl/oidc-opa/test.py +++ b/tests/templates/kuttl/oidc-opa/test.py @@ -17,7 +17,7 @@ namespace = os.environ["NAMESPACE"] tls = os.environ["OIDC_USE_TLS"] nifi_version = os.environ["NIFI_VERSION"] -nifi = f"test-nifi-node-default-0.test-nifi-node-default-metrics.{namespace}.svc.cluster.local" +nifi = f"test-nifi-node-default-0.test-nifi-node-default-headless.{namespace}.svc.cluster.local" keycloak_service = f"keycloak.{namespace}.svc.cluster.local" keycloak_base_url = ( diff --git a/tests/templates/kuttl/smoke_v1/test_nifi.py b/tests/templates/kuttl/smoke_v1/test_nifi.py index 29fb0242..5c72c1a6 100755 --- a/tests/templates/kuttl/smoke_v1/test_nifi.py +++ b/tests/templates/kuttl/smoke_v1/test_nifi.py @@ -46,7 +46,7 @@ def get_token(nifi_host, username, password): # disable warnings as we have specified non-verified https connections urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - host = f"https://nifi-node-default-1.nifi-node-default-metrics.{args['namespace']}.svc.cluster.local:8443" + host = f"https://nifi-node-default-1.nifi-node-default-headless.{args['namespace']}.svc.cluster.local:8443" token = get_token(host, args["user"], args["password"]) headers = {"Authorization": token} node_count = int(args["count"]) diff --git a/tests/templates/kuttl/smoke_v1/test_nifi_metrics.py b/tests/templates/kuttl/smoke_v1/test_nifi_metrics.py index f8cd20df..336660c4 100755 --- a/tests/templates/kuttl/smoke_v1/test_nifi_metrics.py +++ b/tests/templates/kuttl/smoke_v1/test_nifi_metrics.py @@ -39,7 +39,7 @@ port = args["port"] timeout = int(args["timeout"]) - url = f"http://nifi-node-default-0.nifi-node-default-metrics.{namespace}.svc.cluster.local:{port}/metrics" + url = f"http://nifi-node-default-0.nifi-node-default-headless.{namespace}.svc.cluster.local:{port}/metrics" # wait for 'timeout' seconds t_end = time.time() + timeout diff --git a/tests/templates/kuttl/smoke_v2/test_nifi.py b/tests/templates/kuttl/smoke_v2/test_nifi.py index 29fb0242..5c72c1a6 100755 --- a/tests/templates/kuttl/smoke_v2/test_nifi.py +++ b/tests/templates/kuttl/smoke_v2/test_nifi.py @@ -46,7 +46,7 @@ def get_token(nifi_host, username, password): # disable warnings as we have specified non-verified https connections urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - host = f"https://nifi-node-default-1.nifi-node-default-metrics.{args['namespace']}.svc.cluster.local:8443" + host = f"https://nifi-node-default-1.nifi-node-default-headless.{args['namespace']}.svc.cluster.local:8443" token = get_token(host, args["user"], args["password"]) headers = {"Authorization": token} node_count = int(args["count"]) diff --git a/tests/templates/kuttl/upgrade/04-assert.yaml.j2 b/tests/templates/kuttl/upgrade/04-assert.yaml.j2 index 3e538737..1f66464c 100644 --- a/tests/templates/kuttl/upgrade/04-assert.yaml.j2 +++ b/tests/templates/kuttl/upgrade/04-assert.yaml.j2 @@ -8,9 +8,9 @@ commands: - script: kubectl exec -n $NAMESPACE test-nifi-0 -- python /tmp/test_nifi_metrics.py -n $NAMESPACE {% endif %} {% if test_scenario['values']['nifi_old'].split(',')[0] == '2.0.0' %} - - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default-metrics.$NAMESPACE.svc.cluster.local:8443 run json /tmp/generate-and-log-flowfiles.json > /tmp/old_input" + - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default-headless.$NAMESPACE.svc.cluster.local:8443 run json /tmp/generate-and-log-flowfiles.json > /tmp/old_input" {% else %} - - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default-metrics.$NAMESPACE.svc.cluster.local:8443 run template /tmp/generate-and-log-flowfiles.xml > /tmp/old_input" + - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default-headless.$NAMESPACE.svc.cluster.local:8443 run template /tmp/generate-and-log-flowfiles.xml > /tmp/old_input" {% endif %} # This tests if the output contains an Error or zero flow files are queued, which also indicates that something went wrong - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "cat /tmp/old_input | grep -Eov 'Error|\b0\b'" diff --git a/tests/templates/kuttl/upgrade/07-assert.yaml.j2 b/tests/templates/kuttl/upgrade/07-assert.yaml.j2 index c0d0f7ad..6f10d137 100644 --- a/tests/templates/kuttl/upgrade/07-assert.yaml.j2 +++ b/tests/templates/kuttl/upgrade/07-assert.yaml.j2 @@ -9,7 +9,7 @@ commands: {% if test_scenario['values']['nifi_new'].split(',')[0].startswith('1.') %} - script: kubectl exec -n $NAMESPACE test-nifi-0 -- python /tmp/test_nifi_metrics.py -n $NAMESPACE {% endif %} - - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default-metrics.$NAMESPACE.svc.cluster.local:8443 query > /tmp/new_input" + - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "python /tmp/flow.py -e https://test-nifi-node-default-0.test-nifi-node-default-headless.$NAMESPACE.svc.cluster.local:8443 query > /tmp/new_input" # This tests if the output contains an Error or zero flow files are queued, which also indicates that something went wrong - script: kubectl exec -n $NAMESPACE test-nifi-0 -- sh -c "cat /tmp/new_input | grep -Eov 'Error|\b0\b'" # This tests that the number of input records stays the same after the upgrade. diff --git a/tests/templates/kuttl/upgrade/test_nifi.py b/tests/templates/kuttl/upgrade/test_nifi.py index c98b5d8f..3b5cee3e 100755 --- a/tests/templates/kuttl/upgrade/test_nifi.py +++ b/tests/templates/kuttl/upgrade/test_nifi.py @@ -46,7 +46,7 @@ def get_token(nifi_host, username, password): # disable warnings as we have specified non-verified https connections urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - host = f"https://test-nifi-node-default-1.test-nifi-node-default-metrics.{args['namespace']}.svc.cluster.local:8443" + host = f"https://test-nifi-node-default-1.test-nifi-node-default-headless.{args['namespace']}.svc.cluster.local:8443" token = get_token(host, args["user"], args["password"]) headers = {"Authorization": token} node_count = int(args["count"]) diff --git a/tests/templates/kuttl/upgrade/test_nifi_metrics.py b/tests/templates/kuttl/upgrade/test_nifi_metrics.py index d8619361..f0f00d0c 100755 --- a/tests/templates/kuttl/upgrade/test_nifi_metrics.py +++ b/tests/templates/kuttl/upgrade/test_nifi_metrics.py @@ -39,7 +39,7 @@ port = args["port"] timeout = int(args["timeout"]) - url = f"http://test-nifi-node-default-0.test-nifi-node-default-metrics.{namespace}.svc.cluster.local:{port}/metrics" + url = f"http://test-nifi-node-default-0.test-nifi-node-default-headless.{namespace}.svc.cluster.local:{port}/metrics" # wait for 'timeout' seconds t_end = time.time() + timeout From c7ce9b6cbe1b37e6233912932711443a5d8a339b Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Mon, 30 Jun 2025 17:05:43 +0200 Subject: [PATCH 33/37] fix iceberg test --- tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 b/tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 index 08935314..ae514a09 100644 --- a/tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 +++ b/tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 @@ -14,8 +14,6 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} - roleConfig: - listenerClass: {{ test_scenario['values']['listener-class'] }} roleGroups: default: replicas: 1 From 8658f8bb4a94b022e8fa7801c287c83b68e40ad8 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 2 Jul 2025 11:29:53 +0200 Subject: [PATCH 34/37] create separate headless services --- rust/operator-binary/src/controller.rs | 129 ++++++++++--------------- rust/operator-binary/src/main.rs | 1 + 2 files changed, 53 insertions(+), 77 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 33f2a681..4cc527e3 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -37,8 +37,8 @@ use stackable_operator::{ apps::v1::{StatefulSet, StatefulSetSpec, StatefulSetUpdateStrategy}, core::v1::{ ConfigMap, ConfigMapKeySelector, ConfigMapVolumeSource, EmptyDirVolumeSource, - EnvVar, EnvVarSource, ObjectFieldSelector, Probe, SecretVolumeSource, Service, - ServicePort, ServiceSpec, TCPSocketAction, Volume, + EnvVar, EnvVarSource, ObjectFieldSelector, Probe, SecretVolumeSource, + TCPSocketAction, Volume, }, }, apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, @@ -48,7 +48,7 @@ use stackable_operator::{ core::{DeserializeGuard, error_boundary}, runtime::controller::Action, }, - kvp::{Label, Labels, ObjectLabels}, + kvp::{Labels, ObjectLabels}, logging::controller::ReconcilerError, memory::{BinaryMultiple, MemoryQuantity}, product_config_utils::env_vars_from_rolegroup_config, @@ -107,6 +107,10 @@ use crate::{ build_tls_volume, check_or_generate_oidc_admin_password, check_or_generate_sensitive_key, tls::{KEYSTORE_NIFI_CONTAINER_MOUNT, KEYSTORE_VOLUME_NAME, TRUSTSTORE_VOLUME_NAME}, }, + service::{ + build_rolegroup_headless_service, build_rolegroup_metrics_service, + rolegroup_headless_service_name, + }, }; pub const NIFI_CONTROLLER_NAME: &str = "nificluster"; @@ -346,6 +350,9 @@ pub enum Error { #[snafu(display("failed to configure listener"))] ListenerConfiguration { source: crate::listener::Error }, + + #[snafu(display("failed to configure service"))] + ServiceConfiguration { source: crate::service::Error }, } type Result = std::result::Result; @@ -488,8 +495,24 @@ pub async fn reconcile_nifi( ) .context(InvalidGitSyncSpecSnafu)?; - let rg_service = - build_node_rolegroup_service(nifi, &resolved_product_image, &rolegroup)?; + let role_group_service_recommended_labels = build_recommended_labels( + nifi, + &resolved_product_image.app_version_label, + &rolegroup.role, + &rolegroup.role_group, + ); + + let role_group_service_selector = + Labels::role_group_selector(nifi, APP_NAME, &rolegroup.role, &rolegroup.role_group) + .context(LabelBuildSnafu)?; + + let rg_headless_service = build_rolegroup_headless_service( + nifi, + &rolegroup, + role_group_service_recommended_labels.clone(), + role_group_service_selector.clone().into(), + ) + .context(ServiceConfigurationSnafu)?; let role = nifi.spec.nodes.as_ref().context(NoNodesDefinedSnafu)?; @@ -539,12 +562,30 @@ pub async fn reconcile_nifi( ) .await?; + if resolved_product_image.product_version.starts_with("1.") { + let rg_metrics_service = build_rolegroup_metrics_service( + nifi, + &rolegroup, + role_group_service_recommended_labels, + role_group_service_selector.into(), + ) + .context(ServiceConfigurationSnafu)?; + + cluster_resources + .add(client, rg_metrics_service) + .await + .with_context(|_| ApplyRoleGroupServiceSnafu { + rolegroup: rolegroup.clone(), + })?; + } + cluster_resources - .add(client, rg_service) + .add(client, rg_headless_service) .await .with_context(|_| ApplyRoleGroupServiceSnafu { rolegroup: rolegroup.clone(), })?; + cluster_resources .add(client, rg_configmap) .await @@ -778,76 +819,12 @@ async fn build_node_rolegroup_config_map( }) } -/// The rolegroup [`Service`] is a headless service that allows direct access to the instances of a certain rolegroup -/// -/// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. -fn build_node_rolegroup_service( - nifi: &v1alpha1::NifiCluster, - resolved_product_image: &ResolvedProductImage, - rolegroup: &RoleGroupRef, -) -> Result { - let metadata = ObjectMetaBuilder::new() - .name_and_namespace(nifi) - .name(rolegroup_service_name(rolegroup)) - .ownerreference_from_resource(nifi, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(build_recommended_labels( - nifi, - &resolved_product_image.app_version_label, - &rolegroup.role, - &rolegroup.role_group, - )) - .context(MetadataBuildSnafu)? - .with_label(Label::try_from(("prometheus.io/scrape", "true")).context(LabelBuildSnafu)?) - .build(); - - // In NiFi 2.x metrics are scraped from the HTTPS port - let mut service_ports = vec![]; - if resolved_product_image.product_version.starts_with("1.") { - service_ports.push(ServicePort { - name: Some(METRICS_PORT_NAME.to_owned()), - port: METRICS_PORT.into(), - protocol: Some("TCP".to_owned()), - ..ServicePort::default() - }); - } else { - service_ports.push(ServicePort { - name: Some(HTTPS_PORT_NAME.to_owned()), - port: HTTPS_PORT.into(), - protocol: Some("TCP".to_owned()), - ..ServicePort::default() - }); - } - - let spec = Some(ServiceSpec { - // Internal communication does not need to be exposed - type_: Some("ClusterIP".to_owned()), - cluster_ip: Some("None".to_owned()), - ports: Some(service_ports), - selector: Some( - Labels::role_group_selector(nifi, APP_NAME, &rolegroup.role, &rolegroup.role_group) - .context(LabelBuildSnafu)? - .into(), - ), - publish_not_ready_addresses: Some(true), - ..ServiceSpec::default() - }); - - let service = Service { - metadata, - spec, - status: None, - }; - - Ok(service) -} - const USERDATA_MOUNTPOINT: &str = "/stackable/userdata"; /// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. /// /// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the -/// corresponding [`Service`] (from [`build_node_rolegroup_service`]). +/// corresponding [`stackable_operator::k8s_openapi::api::core::v1::Service`] (from [`build_rolegroup_headless_service`]). #[allow(clippy::too_many_arguments)] async fn build_node_rolegroup_statefulset( nifi: &v1alpha1::NifiCluster, @@ -939,7 +916,7 @@ async fn build_node_rolegroup_statefulset( let node_address = format!( "$POD_NAME.{service_name}.{namespace}.svc.{cluster_domain}", - service_name = rolegroup_service_name(rolegroup_ref), + service_name = rolegroup_headless_service_name(&rolegroup_ref.object_name()), namespace = &nifi .metadata .namespace @@ -1443,7 +1420,9 @@ async fn build_node_rolegroup_statefulset( ), ..LabelSelector::default() }, - service_name: Some(rolegroup_service_name(rolegroup_ref)), + service_name: Some(rolegroup_headless_service_name( + &rolegroup_ref.object_name(), + )), template: pod_template, update_strategy: Some(StatefulSetUpdateStrategy { type_: if rolling_update_supported { @@ -1482,10 +1461,6 @@ async fn build_node_rolegroup_statefulset( }) } -pub fn rolegroup_service_name(rolegroup: &RoleGroupRef) -> String { - format!("{name}-headless", name = rolegroup.object_name()) -} - async fn get_proxy_hosts( client: &Client, nifi: &v1alpha1::NifiCluster, diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 8ea4413f..65dfebec 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -39,6 +39,7 @@ mod operations; mod product_logging; mod reporting_task; mod security; +mod service; mod built_info { include!(concat!(env!("OUT_DIR"), "/built.rs")); From b83120de1d86fc6c3a166dec04b129c9ce396851 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 2 Jul 2025 16:03:00 +0200 Subject: [PATCH 35/37] add missing file --- rust/operator-binary/src/service.rs | 124 ++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 rust/operator-binary/src/service.rs diff --git a/rust/operator-binary/src/service.rs b/rust/operator-binary/src/service.rs new file mode 100644 index 00000000..2e0e9a79 --- /dev/null +++ b/rust/operator-binary/src/service.rs @@ -0,0 +1,124 @@ +use std::collections::BTreeMap; + +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::meta::ObjectMetaBuilder, + k8s_openapi::api::core::v1::{Service, ServicePort, ServiceSpec}, + kvp::{Label, ObjectLabels}, + role_utils::RoleGroupRef, +}; + +use crate::crd::{HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT, METRICS_PORT_NAME, v1alpha1}; + +const METRICS_SERVICE_SUFFIX: &str = "metrics"; +const HEADLESS_SERVICE_SUFFIX: &str = "headless"; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("object is missing metadata to build owner reference"))] + ObjectMissingMetadataForOwnerRef { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to build Metadata"))] + MetadataBuild { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to build Labels"))] + LabelBuild { + source: stackable_operator::kvp::LabelError, + }, +} + +/// The rolegroup headless [`Service`] is a service that allows direct access to the instances of a certain rolegroup +/// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. +pub fn build_rolegroup_headless_service( + nifi: &v1alpha1::NifiCluster, + role_group_ref: &RoleGroupRef, + object_labels: ObjectLabels, + selector: BTreeMap, +) -> Result { + Ok(Service { + metadata: ObjectMetaBuilder::new() + .name_and_namespace(nifi) + .name(rolegroup_headless_service_name( + &role_group_ref.object_name(), + )) + .ownerreference_from_resource(nifi, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(object_labels) + .context(MetadataBuildSnafu)? + .build(), + spec: Some(ServiceSpec { + // Internal communication does not need to be exposed + type_: Some("ClusterIP".to_string()), + cluster_ip: Some("None".to_string()), + ports: Some(headless_service_ports()), + selector: Some(selector), + publish_not_ready_addresses: Some(true), + ..ServiceSpec::default() + }), + status: None, + }) +} + +/// The rolegroup metrics [`Service`] is a service that exposes metrics and a prometheus scraping label. +pub fn build_rolegroup_metrics_service( + nifi: &v1alpha1::NifiCluster, + role_group_ref: &RoleGroupRef, + object_labels: ObjectLabels, + selector: BTreeMap, +) -> Result { + Ok(Service { + metadata: ObjectMetaBuilder::new() + .name_and_namespace(nifi) + .name(rolegroup_metrics_service_name( + &role_group_ref.object_name(), + )) + .ownerreference_from_resource(nifi, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(object_labels) + .context(MetadataBuildSnafu)? + .with_label(Label::try_from(("prometheus.io/scrape", "true")).context(LabelBuildSnafu)?) + .build(), + spec: Some(ServiceSpec { + // Internal communication does not need to be exposed + type_: Some("ClusterIP".to_string()), + cluster_ip: Some("None".to_string()), + ports: Some(metrics_service_ports()), + selector: Some(selector), + publish_not_ready_addresses: Some(true), + ..ServiceSpec::default() + }), + status: None, + }) +} + +fn headless_service_ports() -> Vec { + vec![ServicePort { + name: Some(HTTPS_PORT_NAME.into()), + port: HTTPS_PORT.into(), + protocol: Some("TCP".to_string()), + ..ServicePort::default() + }] +} + +fn metrics_service_ports() -> Vec { + vec![ServicePort { + name: Some(METRICS_PORT_NAME.to_string()), + port: METRICS_PORT.into(), + protocol: Some("TCP".to_string()), + ..ServicePort::default() + }] +} + +/// Returns the metrics rolegroup service name `---`. +fn rolegroup_metrics_service_name(role_group_ref_object_name: &str) -> String { + format!("{role_group_ref_object_name}-{METRICS_SERVICE_SUFFIX}") +} + +/// Returns the headless rolegroup service name `---`. +pub fn rolegroup_headless_service_name(role_group_ref_object_name: &str) -> String { + format!("{role_group_ref_object_name}-{HEADLESS_SERVICE_SUFFIX}") +} From bb49f4359506ca6c8effe35c8f1a128df9351afb Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 2 Jul 2025 16:32:46 +0200 Subject: [PATCH 36/37] fix iceberg test --- tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 b/tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 index ae514a09..bfca700f 100644 --- a/tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 +++ b/tests/templates/kuttl/iceberg/30-install-zookeeper.yaml.j2 @@ -8,6 +8,7 @@ spec: productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" pullPolicy: IfNotPresent {% if lookup('env', 'VECTOR_AGGREGATOR') %} + clusterConfig: vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} servers: From 05ada01df025765df113cd3b727e1c106b33d528 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 2 Jul 2025 18:21:02 +0200 Subject: [PATCH 37/37] use listener scope for tls --- rust/operator-binary/src/controller.rs | 6 ++---- rust/operator-binary/src/reporting_task/mod.rs | 2 ++ rust/operator-binary/src/security/mod.rs | 2 ++ rust/operator-binary/src/security/tls.rs | 3 ++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 4cc527e3..c16a591a 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -1346,12 +1346,10 @@ async fn build_node_rolegroup_statefulset( build_tls_volume( nifi, KEYSTORE_VOLUME_NAME, - vec![ - &nifi_cluster_name, - &build_reporting_task_service_name(&nifi_cluster_name), - ], + vec![&build_reporting_task_service_name(&nifi_cluster_name)], SecretFormat::TlsPkcs12, &requested_secret_lifetime, + LISTENER_VOLUME_NAME, ) .context(SecuritySnafu)?, ) diff --git a/rust/operator-binary/src/reporting_task/mod.rs b/rust/operator-binary/src/reporting_task/mod.rs index be7e1446..1efd71b3 100644 --- a/rust/operator-binary/src/reporting_task/mod.rs +++ b/rust/operator-binary/src/reporting_task/mod.rs @@ -51,6 +51,7 @@ use stackable_operator::{ use crate::{ controller::build_recommended_labels, crd::{APP_NAME, HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT, NifiRole, v1alpha1}, + listener::LISTENER_VOLUME_NAME, security::{ authentication::{NifiAuthenticationConfig, STACKABLE_ADMIN_USERNAME}, build_tls_volume, @@ -356,6 +357,7 @@ fn build_reporting_task_job( // There is no correct way to configure this job since it's an implementation detail. // Also it will be dropped when support for 1.x is removed. &Duration::from_days_unchecked(1), + LISTENER_VOLUME_NAME, ) .context(SecretVolumeBuildFailureSnafu)?, ) diff --git a/rust/operator-binary/src/security/mod.rs b/rust/operator-binary/src/security/mod.rs index eebc0a30..bc304b30 100644 --- a/rust/operator-binary/src/security/mod.rs +++ b/rust/operator-binary/src/security/mod.rs @@ -50,6 +50,7 @@ pub fn build_tls_volume( service_scopes: Vec<&str>, secret_format: SecretFormat, requested_secret_lifetime: &Duration, + listener_scope: &str, ) -> Result { tls::build_tls_volume( nifi, @@ -57,6 +58,7 @@ pub fn build_tls_volume( service_scopes, secret_format, requested_secret_lifetime, + listener_scope, ) .context(TlsSnafu) } diff --git a/rust/operator-binary/src/security/tls.rs b/rust/operator-binary/src/security/tls.rs index 8559a4ef..932ab381 100644 --- a/rust/operator-binary/src/security/tls.rs +++ b/rust/operator-binary/src/security/tls.rs @@ -27,6 +27,7 @@ pub(crate) fn build_tls_volume( service_scopes: Vec<&str>, secret_format: SecretFormat, requested_secret_lifetime: &Duration, + listener_scope: &str, ) -> Result { let mut secret_volume_source_builder = SecretOperatorVolumeSourceBuilder::new(nifi.server_tls_secret_class()); @@ -42,8 +43,8 @@ pub(crate) fn build_tls_volume( Ok(VolumeBuilder::new(volume_name) .ephemeral( secret_volume_source_builder - .with_node_scope() .with_pod_scope() + .with_listener_volume_scope(listener_scope) .with_format(secret_format) .with_auto_tls_cert_lifetime(*requested_secret_lifetime) .build()