Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
- Environment variable overrides (`envOverrides`) are now applied after all environment
variables set by the operator. In particular, `CONTAINERDEBUG_LOG_DIRECTORY` can now be
overridden, whereas previously the operator's value always took precedence ([#838]).
- Make operations infallible where appropriate ([#852]).
Comment thread
siegfriedweber marked this conversation as resolved.

### Fixed

Expand All @@ -58,6 +59,7 @@
[#844]: https://github.com/stackabletech/airflow-operator/pull/844
[#847]: https://github.com/stackabletech/airflow-operator/pull/847
[#849]: https://github.com/stackabletech/airflow-operator/pull/849
[#852]: https://github.com/stackabletech/airflow-operator/pull/852

## [26.7.0] - 2026-07-21

Expand Down
45 changes: 45 additions & 0 deletions rust/operator-binary/src/controller/build/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -693,4 +693,49 @@ mod tests {
assert_eq!(env.get("AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX"), None);
assert_eq!(env.get("AIRFLOW__WEBSERVER__PROXY_FIX_X_FOR"), None);
}

/// A cluster with the given `spec.clusterConfig.volumeMounts` (as standalone YAML).
fn cluster_with_user_volume_mounts(
executor_key: &str,
executor_config: &str,
volume_mounts: &str,
) -> ValidatedCluster {
validated_cluster_with(executor_key, executor_config, |cluster| {
cluster["spec"]["clusterConfig"]
.as_mapping_mut()
.expect("clusterConfig is a mapping")
.insert(
"volumeMounts".into(),
serde_yaml::from_str(volume_mounts).expect("valid volumeMounts YAML"),
);
})
}

/// A user-supplied volumeMount that collides with an operator-managed mount path must be
/// reported as an error (the operator's own mounts are added first and are infallible).
#[test]
fn user_volume_mount_colliding_with_config_path_is_an_error() {
let cluster = cluster_with_user_volume_mounts(
"kubernetesExecutors",
"{config: {}}",
"[{name: user-volume, mountPath: /stackable/app/config}]",
);

let Err(error) = build(&cluster) else {
panic!("the colliding mount must be rejected");
};
assert!(
matches!(
error,
super::Error::StatefulSet {
source:
crate::controller::build::resource::statefulset::Error::AddVolumeMount { .. },
..
} | super::Error::ExecutorTemplate {
source: crate::controller::build::resource::executor::Error::AddVolumeMount { .. },
}
),
"unexpected error: {error:?}"
);
}
}
11 changes: 1 addition & 10 deletions rust/operator-binary/src/controller/build/properties/env_vars.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
use std::{collections::BTreeSet, path::PathBuf, str::FromStr};
use std::{collections::BTreeSet, str::FromStr};

use snafu::Snafu;
use stackable_operator::{
constant,
crd::{authentication::oidc, git_sync},
Expand Down Expand Up @@ -81,14 +80,6 @@ constant!(PYTHONPATH: EnvVarName = "PYTHONPATH");
constant!(CONTAINERDEBUG_LOG_DIRECTORY: EnvVarName = "CONTAINERDEBUG_LOG_DIRECTORY");
constant!(STACKABLE_POST_HOOK: EnvVarName = "_STACKABLE_POST_HOOK");

#[derive(Snafu, Debug)]
pub enum Error {
#[snafu(display(
"failed to construct Git DAG folder - Is the git folder a valid path?: {dag_folder:?}"
))]
ConstructGitDagFolder { dag_folder: PathBuf },
}

/// Return environment variables to be applied to the statefulsets for the scheduler, webserver (and worker,
/// for clusters utilizing `celeryExecutor`: for clusters using `kubernetesExecutor` a different set will be
/// used which is defined in [`build_airflow_template_envs`]).
Expand Down
12 changes: 3 additions & 9 deletions rust/operator-binary/src/controller/build/resource/config_map.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,6 @@ pub enum Error {
source: webserver_config::Error,
role_group: RoleGroupName,
},

#[snafu(display("failed to build ConfigMap for role group {role_group}"))]
BuildConfigMap {
source: stackable_operator::builder::configmap::Error,
role_group: RoleGroupName,
},
}

/// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator
Expand Down Expand Up @@ -98,7 +92,7 @@ pub fn build_rolegroup_config_map(
cm_builder.add_data(VECTOR_CONFIG_FILE, vector_config_file_content());
}

cm_builder.build().with_context(|_| BuildConfigMapSnafu {
role_group: role_group_name.clone(),
})
Ok(cm_builder
.build()
.expect("The ConfigMap metadata is set in this function."))
}
67 changes: 35 additions & 32 deletions rust/operator-binary/src/controller/build/resource/executor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@ use stackable_operator::{
meta::ObjectMetaBuilder,
pod::{PodBuilder, security::PodSecurityContextBuilder},
},
constants::RESTART_CONTROLLER_ENABLED_LABEL,
k8s_openapi::{
DeepMerge,
api::core::v1::{ConfigMap, PodTemplateSpec},
},
kvp::{Label, LabelError},
v2::{
builder::pod::container::{EnvVarSet, new_container_builder},
product_logging::framework::STACKABLE_LOG_DIR,
Expand Down Expand Up @@ -55,17 +55,9 @@ pub enum Error {
source: stackable_operator::builder::pod::container::Error,
},

#[snafu(display("failed to build label"))]
BuildLabel { source: LabelError },

#[snafu(display("pod template serialization"))]
PodTemplateSerde { source: serde_yaml::Error },

#[snafu(display("failed to build the pod template config map"))]
PodTemplateConfigMap {
source: stackable_operator::builder::configmap::Error,
},

#[snafu(display("failed to build shared pod resources"))]
Pod {
source: crate::controller::build::resource::pod::Error,
Expand Down Expand Up @@ -118,12 +110,6 @@ pub fn build_executor_template_config_map(
// See https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.4.0/kubernetes_executor.html#base-image
let mut airflow_container = new_container_builder(&Container::Base.to_container_name());

add_authentication_volumes_and_volume_mounts(
authentication_config,
&mut airflow_container,
&mut pb,
)
.context(PodSnafu)?;
airflow_container
.image_from_product_image(resolved_product_image)
.resources(executor_config.resources.clone().into())
Expand All @@ -133,13 +119,37 @@ pub fn build_executor_template_config_map(
&executor_config.logging,
git_sync_resources,
))
.add_volume_mounts(cluster.volume_mounts())
.context(AddVolumeMountSnafu)?
// Statically named operator mounts first: their names and paths are distinct
// constants, so they cannot collide with each other.
.add_volume_mount(&*CONFIG_VOLUME_NAME, CONFIG_PATH)
.context(AddVolumeMountSnafu)?
.expect("The mount paths are statically defined and there should be no duplicates.")
.add_volume_mount(&*LOG_CONFIG_VOLUME_NAME, LOG_CONFIG_DIR)
.context(AddVolumeMountSnafu)?
.expect("The mount paths are statically defined and there should be no duplicates.")
.add_volume_mount(&*LOG_VOLUME_NAME, STACKABLE_LOG_DIR)
.expect("The mount paths are statically defined and there should be no duplicates.");

// Statically named operator volumes first, for the same reason.
pb.add_volumes(volumes::create_volumes(
cluster
.role_group_resource_names(&EXECUTOR_ROLE_NAME, &EXECUTOR_ROLE_GROUP_NAME)
.role_group_config_map()
.as_ref(),
&executor_config.logging.product_container,
))
.expect("The volume names are statically defined and there should be no duplicates.");

// Authentication mounts and volumes next: their names derive from the AuthenticationClass
// contents, so a collision with the static names above surfaces as an error, not a panic.
add_authentication_volumes_and_volume_mounts(
authentication_config,
&mut airflow_container,
&mut pb,
)
.context(PodSnafu)?;

// User-supplied mounts last: these can collide with the ones above, so this stays fallible.
airflow_container
.add_volume_mounts(cluster.volume_mounts())
.context(AddVolumeMountSnafu)?;

add_git_sync_resources(
Expand All @@ -157,16 +167,10 @@ pub fn build_executor_template_config_map(
.add_to_container(&mut airflow_container);

pb.add_container(airflow_container.build());
// User-supplied volumes last (fallible); operator-managed and authentication volumes were
// added above.
pb.add_volumes(cluster.volumes().clone())
.context(AddVolumeSnafu)?;
pb.add_volumes(volumes::create_volumes(
cluster
.role_group_resource_names(&EXECUTOR_ROLE_NAME, &EXECUTOR_ROLE_GROUP_NAME)
.role_group_config_map()
.as_ref(),
&executor_config.logging.product_container,
))
.context(AddVolumeSnafu)?;

if let Some(vector_log_config) = &executor_config.logging.vector_container {
pb.add_container(build_logging_container(
Expand All @@ -182,9 +186,6 @@ pub fn build_executor_template_config_map(

let mut cm_builder = ConfigMapBuilder::new();

let restarter_label =
Label::try_from(("restarter.stackable.tech/enabled", "true")).context(BuildLabelSnafu)?;

cm_builder
.metadata(
object_meta(
Expand All @@ -196,13 +197,15 @@ pub fn build_executor_template_config_map(
&EXECUTOR_TEMPLATE_ROLE_GROUP_NAME,
),
)
.with_label(restarter_label)
.with_label(RESTART_CONTROLLER_ENABLED_LABEL.clone())
.build(),
)
.add_data(
TEMPLATE_NAME,
serde_yaml::to_string(&pod_template).context(PodTemplateSerdeSnafu)?,
);

cm_builder.build().context(PodTemplateConfigMapSnafu)
Ok(cm_builder
.build()
.expect("The ConfigMap metadata is set in this function."))
}
11 changes: 11 additions & 0 deletions rust/operator-binary/src/controller/build/resource/pod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -148,3 +148,14 @@ pub(crate) fn build_logging_container(
EnvVarSet::new(),
)
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn test_constants() {
// Test that dereferencing the constants does not panic.
let _ = *VECTOR_CONTAINER_NAME;
}
}
Loading
Loading