From ef64d52a804c1b631f8db79427c636ebd9f303b4 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Jun 2026 19:59:51 +0200 Subject: [PATCH 01/39] chore: point stackable-operator at smooth-operator branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add [patch] section to Cargo.toml pointing stackable-operator at the smooth-operator branch of operator-rs (v0.111.1). No import-path relocations were needed — all existing paths remain valid on this branch. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 308 ++++++++++++++++++++++++++++------------------------- Cargo.toml | 4 +- 2 files changed, 163 insertions(+), 149 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 25ac572e..1b5b6eff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -163,9 +163,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "axum" @@ -265,9 +265,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "2.11.1" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "84d7ced0ae9557296835c32bf1b1e02b44c746701f898460fb000d7eaa84f00a" [[package]] name = "block-buffer" @@ -280,9 +280,9 @@ dependencies = [ [[package]] name = "built" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4ad8f11f288f48ca24471bbd51ac257aaeaaa07adae295591266b792902ae64" +checksum = "5c0e531d93d39c34eef561e929e8a7f86d77a5af08aac4f6d6e39976c51858e9" dependencies = [ "chrono", "git2", @@ -290,9 +290,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.20.2" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytes" @@ -302,9 +302,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "cc" -version = "1.2.60" +version = "1.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" dependencies = [ "find-msvc-tools", "jobserver", @@ -633,9 +633,9 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", @@ -696,9 +696,9 @@ dependencies = [ [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "elliptic-curve" @@ -771,7 +771,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -950,9 +950,9 @@ checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-timer" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" [[package]] name = "futures-util" @@ -1023,15 +1023,14 @@ dependencies = [ [[package]] name = "git2" -version = "0.20.4" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" +checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" dependencies = [ "bitflags", "libc", "libgit2-sys", "log", - "url", ] [[package]] @@ -1065,9 +1064,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" dependencies = [ "atomic-waker", "bytes", @@ -1104,9 +1103,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.17.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "heck" @@ -1136,9 +1135,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" dependencies = [ "bytes", "itoa", @@ -1187,9 +1186,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hyper" -version = "1.9.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" dependencies = [ "atomic-waker", "bytes", @@ -1391,9 +1390,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", @@ -1406,7 +1405,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.17.0", + "hashbrown 0.17.1", "serde", "serde_core", ] @@ -1426,16 +1425,6 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" -[[package]] -name = "iri-string" -version = "0.7.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -1470,9 +1459,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.23" +version = "0.2.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" +checksum = "4603d3033e49e2b0e31229fcab20a5d40089c607d975cd9c80551dc69eed9102" dependencies = [ "jiff-static", "jiff-tzdb-platform", @@ -1480,14 +1469,14 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.61.2", + "windows-link", ] [[package]] name = "jiff-static" -version = "0.2.23" +version = "0.2.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47" dependencies = [ "proc-macro2", "quote", @@ -1521,9 +1510,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.95" +version = "0.3.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" +checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" dependencies = [ "cfg-if", "futures-util", @@ -1533,14 +1522,15 @@ dependencies = [ [[package]] name = "json-patch" -version = "4.1.0" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f300e415e2134745ef75f04562dd0145405c2f7fd92065db029ac4b16b57fe90" +checksum = "7421438de105a0827e44fadd05377727847d717c80ce29a229f85fd04c427b72" dependencies = [ "jsonptr", + "schemars", "serde", "serde_json", - "thiserror 1.0.69", + "thiserror 2.0.18", ] [[package]] @@ -1582,11 +1572,11 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "darling", "regex", - "snafu 0.9.0", + "snafu 0.9.1", ] [[package]] @@ -1729,15 +1719,15 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.185" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libgit2-sys" -version = "0.18.3+1.9.2" +version = "0.18.5+1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487" +checksum = "005d6ae6eac1912906073e069f7db60b1fa98e052a68227824afe3e3a1c59ca2" dependencies = [ "cc", "libc", @@ -1753,9 +1743,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libz-sys" -version = "1.1.28" +version = "1.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc3a226e576f50782b3305c5ccf458698f92798987f551c6a02efe8276721e22" +checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" dependencies = [ "cc", "libc", @@ -1780,9 +1770,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "113b30b4cd05f7c06868fdb2854f66a7b9fece9a48425351cd532e810d74024f" [[package]] name = "matchers" @@ -1801,9 +1791,9 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" [[package]] name = "mime" @@ -1823,9 +1813,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi", @@ -1859,9 +1849,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" @@ -2125,18 +2115,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.11" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.11" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", @@ -2570,9 +2560,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.38" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "log", "once_cell", @@ -2585,9 +2575,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -2597,9 +2587,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ "zeroize", ] @@ -2773,9 +2763,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -2853,9 +2843,9 @@ dependencies = [ [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "signal-hook-registry" @@ -2916,11 +2906,11 @@ dependencies = [ [[package]] name = "snafu" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1d4bced6a69f90b2056c03dcff2c4737f98d6fb9e0853493996e1d253ca29c6" +checksum = "d1a012328be2e3f5d5f6f3218147ca02588cea4cb865e876849ab6debcf36522" dependencies = [ - "snafu-derive 0.9.0", + "snafu-derive 0.9.1", ] [[package]] @@ -2948,9 +2938,9 @@ dependencies = [ [[package]] name = "snafu-derive" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" +checksum = "5f103c50866b8743da9429b8a581d81a27c2d3a9c4ac7df8f8571c1dd7896eda" dependencies = [ "heck", "proc-macro2", @@ -2960,9 +2950,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", "windows-sys 0.61.2", @@ -2993,7 +2983,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "const-oid", "ecdsa", @@ -3005,7 +2995,7 @@ dependencies = [ "rsa", "sha2", "signature", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-shared", "tokio", "tokio-rustls", @@ -3033,7 +3023,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-operator", "strum", "tokio", @@ -3043,8 +3033,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.111.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +version = "0.111.1" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "base64", "clap", @@ -3068,7 +3058,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-operator-derive", "stackable-shared", "stackable-telemetry", @@ -3080,12 +3070,13 @@ dependencies = [ "tracing-appender", "tracing-subscriber", "url", + "uuid", ] [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "darling", "proc-macro2", @@ -3096,7 +3087,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "jiff", "k8s-openapi", @@ -3105,7 +3096,7 @@ dependencies = [ "semver", "serde", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "strum", "time", ] @@ -3113,7 +3104,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "axum", "clap", @@ -3124,7 +3115,7 @@ dependencies = [ "opentelemetry-semantic-conventions", "opentelemetry_sdk", "pin-project", - "snafu 0.9.0", + "snafu 0.9.1", "strum", "tokio", "tower", @@ -3137,21 +3128,21 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "kube", "schemars", "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-versioned-macros", ] [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "convert_case", "convert_case_extras", @@ -3169,7 +3160,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "arc-swap", "async-trait", @@ -3185,7 +3176,7 @@ dependencies = [ "rand 0.9.4", "serde", "serde_json", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-certs", "stackable-shared", "stackable-telemetry", @@ -3231,6 +3222,12 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + [[package]] name = "syn" version = "1.0.109" @@ -3386,9 +3383,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.0" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a91135f59b1cbf38c91e73cf3386fca9bb77915c45ce2771460c9d92f0f3d776" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -3458,9 +3455,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.11+spec-1.1.0" +version = "0.25.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" +checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" dependencies = [ "indexmap", "toml_datetime", @@ -3479,9 +3476,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "base64", @@ -3506,9 +3503,9 @@ dependencies = [ [[package]] name = "tonic-prost" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" dependencies = [ "bytes", "prost", @@ -3536,9 +3533,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "base64", "bitflags", @@ -3546,13 +3543,13 @@ dependencies = [ "futures-util", "http", "http-body", - "iri-string", "mime", "pin-project-lite", "tower", "tower-layer", "tower-service", "tracing", + "url", ] [[package]] @@ -3581,11 +3578,12 @@ dependencies = [ [[package]] name = "tracing-appender" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" dependencies = [ "crossbeam-channel", + "symlink", "thiserror 2.0.18", "time", "tracing-subscriber", @@ -3678,9 +3676,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "ucd-trie" @@ -3696,9 +3694,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-segmentation" -version = "1.13.2" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] name = "unicode-xid" @@ -3743,6 +3741,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -3778,11 +3786,11 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.2+wasi-0.2.9" +version = "1.0.3+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.57.1", ] [[package]] @@ -3791,14 +3799,14 @@ version = "0.4.0+wasi-0.3.0-rc-2026-01-06" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.51.0", ] [[package]] name = "wasm-bindgen" -version = "0.2.118" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" +checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" dependencies = [ "cfg-if", "once_cell", @@ -3809,9 +3817,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.68" +version = "0.4.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8" +checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" dependencies = [ "js-sys", "wasm-bindgen", @@ -3819,9 +3827,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.118" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" +checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3829,9 +3837,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.118" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" +checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" dependencies = [ "bumpalo", "proc-macro2", @@ -3842,9 +3850,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.118" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" +checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" dependencies = [ "unicode-ident", ] @@ -3885,9 +3893,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.95" +version = "0.3.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" +checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" dependencies = [ "js-sys", "wasm-bindgen", @@ -4046,9 +4054,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" dependencies = [ "memchr", ] @@ -4062,6 +4070,12 @@ dependencies = [ "wit-bindgen-rust-macro", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "wit-bindgen-core" version = "0.51.0" @@ -4163,9 +4177,9 @@ dependencies = [ [[package]] name = "xml" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8aa498d22c9bbaf482329839bc5620c46be275a19a812e9a22a2b07529a642a" +checksum = "636f85e5ca6488e96401b61eb7de54f4e44755c988af0f52cf90230c312a1a89" [[package]] name = "yoke" @@ -4192,18 +4206,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.48" +version = "0.8.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.48" +version = "0.8.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" dependencies = [ "proc-macro2", "quote", @@ -4212,9 +4226,9 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ "zerofrom-derive", ] diff --git a/Cargo.toml b/Cargo.toml index 7ad6543a..b2b9cc1d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,6 @@ tracing = "0.1" url = { version = "2.5.7" } xml-rs = "1.0" -# [patch."https://github.com/stackabletech/operator-rs.git"] -# stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } +[patch."https://github.com/stackabletech/operator-rs.git"] +stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "smooth-operator"} # stackable-operator = { path = "../operator-rs/crates/stackable-operator" } From 925766d24f6150bd92591462adaccf7418290f9f Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Jun 2026 20:18:38 +0200 Subject: [PATCH 02/39] feat: vendor java-properties writer and add ConfigFileName MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Vendor the `to_java_properties_string` / `to_hadoop_xml` writers from `product-config` into `controller::build::properties::writer` so the operator no longer relies on product-config for config serialization. Introduce `ConfigFileName` (the enum of NiFi config file names) and skeleton helpers `defined_entries` / `resolved_overrides` in `controller::build::properties` — these are dead-code-allowed stubs that will be wired up in Task 4. Swap the single existing call site (JVM security properties) to use the vendored writer; output is byte-identical. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 1 + Cargo.toml | 1 + rust/operator-binary/Cargo.toml | 1 + rust/operator-binary/src/controller.rs | 10 +-- rust/operator-binary/src/controller/build.rs | 5 ++ .../src/controller/build/properties.rs | 47 +++++++++++ .../src/controller/build/properties/writer.rs | 78 +++++++++++++++++++ 7 files changed, 138 insertions(+), 5 deletions(-) create mode 100644 rust/operator-binary/src/controller/build.rs create mode 100644 rust/operator-binary/src/controller/build/properties.rs create mode 100644 rust/operator-binary/src/controller/build/properties/writer.rs diff --git a/Cargo.lock b/Cargo.lock index 1b5b6eff..7d7f04c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3015,6 +3015,7 @@ dependencies = [ "fnv", "futures 0.3.32", "indoc", + "java-properties", "pin-project", "product-config", "rand 0.10.1", diff --git a/Cargo.toml b/Cargo.toml index b2b9cc1d..f14b7085 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ const_format = "0.2" fnv = "1.0" futures = { version = "0.3", features = ["compat"] } indoc = "2.0" +java-properties = "2.0" pin-project = "1.1" rand = "0.10" rstest = "0.26" diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index dd6a415c..cd7fc147 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -18,6 +18,7 @@ const_format.workspace = true fnv.workspace = true futures.workspace = true indoc.workspace = true +java-properties.workspace = true pin-project.workspace = true rand.workspace = true semver.workspace = true diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index a2b0cd52..59fabcbd 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -6,13 +6,12 @@ use std::{ sync::Arc, }; +use crate::controller::build::properties::writer::{ + PropertiesWriterError, to_java_properties_string, +}; use const_format::concatcp; use indoc::formatdoc; -use product_config::{ - ProductConfigManager, - types::PropertyNameKind, - writer::{PropertiesWriterError, to_java_properties_string}, -}; +use product_config::{ProductConfigManager, types::PropertyNameKind}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{ @@ -72,6 +71,7 @@ use stackable_operator::{ use strum::{EnumDiscriminants, IntoStaticStr}; use tracing::Instrument; +mod build; mod dereference; mod validate; diff --git a/rust/operator-binary/src/controller/build.rs b/rust/operator-binary/src/controller/build.rs new file mode 100644 index 00000000..bb77c4fc --- /dev/null +++ b/rust/operator-binary/src/controller/build.rs @@ -0,0 +1,5 @@ +//! Builders that assemble Kubernetes resources from a [`ValidatedCluster`]. +//! +//! [`ValidatedCluster`]: crate::controller::validate::ValidatedCluster + +pub mod properties; diff --git a/rust/operator-binary/src/controller/build/properties.rs b/rust/operator-binary/src/controller/build/properties.rs new file mode 100644 index 00000000..3f33a1bf --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties.rs @@ -0,0 +1,47 @@ +//! Per-file builders for the NiFi rolegroup ConfigMap. +//! +//! Each `` module produces the rendered content for one NiFi config file. +//! The shared [`writer`] module serializes `.properties`/`.conf` key/value maps to +//! the Java-properties on-wire format. + +use std::collections::BTreeMap; + +use stackable_operator::config_overrides::KeyValueConfigOverrides; + +pub mod writer; + +/// The names of the files assembled into the NiFi rolegroup ConfigMap. +#[allow(dead_code)] // used once the per-file builders land in Task 4 +#[derive(Clone, Copy, Debug, strum::Display)] +pub enum ConfigFileName { + #[strum(serialize = "bootstrap.conf")] + BootstrapConf, + #[strum(serialize = "nifi.properties")] + NifiProperties, + #[strum(serialize = "state-management.xml")] + StateManagementXml, + #[strum(serialize = "security.properties")] + SecurityProperties, + #[strum(serialize = "login-identity-providers.xml")] + LoginIdentityProviders, + #[strum(serialize = "authorizers.xml")] + Authorizers, +} + +/// Keep only the set (`Some`) entries of a `key -> optional value` map, as `(key, value)` pairs. +#[allow(dead_code)] // used once the per-file builders land in Task 4 +fn defined_entries( + entries: BTreeMap>, +) -> impl Iterator { + entries + .into_iter() + .filter_map(|(key, value)| value.map(|value| (key, value))) +} + +/// Resolve user-provided [`KeyValueConfigOverrides`] into key/value pairs. +#[allow(dead_code)] // used once the per-file builders land in Task 4 +fn resolved_overrides( + overrides: KeyValueConfigOverrides, +) -> impl Iterator { + overrides.overrides.into_iter() +} diff --git a/rust/operator-binary/src/controller/build/properties/writer.rs b/rust/operator-binary/src/controller/build/properties/writer.rs new file mode 100644 index 00000000..a74babf0 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/writer.rs @@ -0,0 +1,78 @@ +//! Writer for Java `.properties` files. +//! +//! Vendored from the `product-config` crate's `writer` module so the operator no +//! longer depends on `product-config` for rendering. + +use std::io::Write; + +use java_properties::{PropertiesError, PropertiesWriter}; +use snafu::{ResultExt, Snafu}; + +#[derive(Debug, Snafu)] +pub enum PropertiesWriterError { + #[snafu(display("failed to create properties file"))] + Properties { source: PropertiesError }, + + #[snafu(display("failed to convert properties file byte array to UTF-8"))] + FromUtf8 { source: std::string::FromUtf8Error }, +} + +/// Creates a common Java properties file string in the format: +/// `property_1=value_1\nproperty_2=value_2\n`. +pub fn to_java_properties_string<'a, T>(properties: T) -> Result +where + T: Iterator)>, +{ + let mut output = Vec::new(); + write_java_properties(&mut output, properties)?; + String::from_utf8(output).context(FromUtf8Snafu) +} + +/// Writes Java properties to the given writer. A `None` value is written as an +/// empty value (`key=`). +fn write_java_properties<'a, W, T>(writer: W, properties: T) -> Result<(), PropertiesWriterError> +where + W: Write, + T: Iterator)>, +{ + let mut writer = PropertiesWriter::new(writer); + for (k, v) in properties { + let property_value = v.as_deref().unwrap_or_default(); + writer.write(k, property_value).context(PropertiesSnafu)?; + } + writer.flush().context(PropertiesSnafu)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use super::*; + + fn props(pairs: &[(&str, Option<&str>)]) -> String { + let map: BTreeMap> = pairs + .iter() + .map(|(k, v)| (k.to_string(), v.map(str::to_string))) + .collect(); + to_java_properties_string(map.iter()).unwrap() + } + + #[test] + fn java_properties_renders_key_value() { + assert_eq!(props(&[("a", Some("1")), ("b", Some("2"))]), "a=1\nb=2\n"); + } + + #[test] + fn java_properties_renders_none_as_empty() { + assert_eq!(props(&[("none", None)]), "none=\n"); + } + + #[test] + fn java_properties_escapes_colon_in_value() { + assert_eq!( + props(&[("url", Some("file://this/location/file.abc"))]), + "url=file\\://this/location/file.abc\n" + ); + } +} From b1906abaf99b6935328ba0691fa2945f0d488028 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Jun 2026 20:50:32 +0200 Subject: [PATCH 03/39] feat: validate rolegroups via vendored v2 role_utils shim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces a typed, merged-per-rolegroup config path using a vendored trino-style framework/role_utils.rs shim. Adds NifiRoleGroupConfig type alias, build_role_group_configs() helper, and role_group_configs field to ValidatedInputs (strangler pattern — old validated_role_config kept intact). Also adds Merge impl for NifiConfigOverrides and Ord/Eq/Hash derives to NifiRole to satisfy generic bounds required by with_validated_config. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/controller/validate.rs | 41 ++++- rust/operator-binary/src/crd/mod.rs | 112 ++++++++++++- rust/operator-binary/src/framework.rs | 8 + .../src/framework/role_utils.rs | 156 ++++++++++++++++++ rust/operator-binary/src/main.rs | 1 + 5 files changed, 315 insertions(+), 3 deletions(-) create mode 100644 rust/operator-binary/src/framework.rs create mode 100644 rust/operator-binary/src/framework/role_utils.rs diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index e0b6d3e4..8f7e436c 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -3,14 +3,16 @@ //! Synchronously validates inputs that don't require Kubernetes API calls. Produces //! [`ValidatedInputs`], consumed by the rest of `reconcile_nifi`. -use std::collections::HashSet; +use std::collections::{BTreeMap, HashSet}; use product_config::ProductConfigManager; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, commons::product_image_selection::{self, ResolvedProductImage}, + kube::ResourceExt as _, product_config_utils::ValidatedRoleConfigByPropertyKind, + role_utils::JavaCommonConfig, utils::cluster_info::KubernetesClusterInfo, }; use strum::{EnumDiscriminants, IntoStaticStr}; @@ -18,7 +20,8 @@ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ config::{self, validated_product_config}, controller::dereference::DereferencedObjects, - crd::{HTTPS_PORT, v1alpha1}, + crd::{HTTPS_PORT, NifiConfig, NifiRole, v1alpha1}, + framework::role_utils::with_validated_config, reporting_task, security::{ authentication::{self, NifiAuthenticationConfig}, @@ -51,8 +54,19 @@ pub enum Error { ReportingTask { source: crate::reporting_task::Error, }, + + #[snafu(display("failed to validate config fragment for a rolegroup"))] + InvalidConfigFragment { + source: stackable_operator::config::fragment::ValidationError, + }, } +pub type NifiRoleGroupConfig = crate::framework::role_utils::RoleGroupConfig< + NifiConfig, + JavaCommonConfig, + v1alpha1::NifiConfigOverrides, +>; + type Result = std::result::Result; /// Synchronous inputs the rest of `reconcile_nifi` needs after dereferencing. @@ -63,6 +77,9 @@ pub struct ValidatedInputs { pub validated_role_config: ValidatedRoleConfigByPropertyKind, // Comma-separated NiFi proxy hosts, or `"*"` if `spec.clusterConfig.hostHeaderCheck.allowAll` is set. pub proxy_hosts: String, + // Not yet consumed — Tasks 4-6 will use this to replace the product-config pipeline. + #[allow(dead_code)] + pub role_group_configs: BTreeMap>, } /// Validates the cluster spec and the dereferenced inputs. @@ -108,9 +125,29 @@ pub fn validate( authorization_config, validated_role_config, proxy_hosts, + role_group_configs: build_role_group_configs(nifi)?, }) } +fn build_role_group_configs( + nifi: &v1alpha1::NifiCluster, +) -> Result>> { + let role = nifi.spec.nodes.as_ref().context(NoNodesDefinedSnafu)?; + let default_config = NifiConfig::default_config(&nifi.name_any(), &NifiRole::Node); + + let mut groups: BTreeMap = BTreeMap::new(); + for (rg_name, rg) in &role.role_groups { + let validated_rg = + with_validated_config::(rg, role, &default_config) + .context(InvalidConfigFragmentSnafu)?; + groups.insert(rg_name.clone(), validated_rg); + } + + let mut role_group_configs = BTreeMap::new(); + role_group_configs.insert(NifiRole::Node, groups); + Ok(role_group_configs) +} + fn compute_proxy_hosts( nifi: &v1alpha1::NifiCluster, cluster_info: &KubernetesClusterInfo, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index b84b5da7..d2209b0f 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -229,6 +229,28 @@ impl KeyValueOverridesProvider for v1alpha1::NifiConfigOverrides { } } +impl Merge for v1alpha1::NifiConfigOverrides { + /// Merges per-file overrides: individual key-value pairs from `defaults` are + /// inserted only when the same key is absent from `self`. + fn merge(&mut self, defaults: &Self) { + fn merge_kv( + target: &mut Option, + default: &Option, + ) { + if let Some(default_kv) = default { + let target_kv = target.get_or_insert_with(Default::default); + for (k, v) in &default_kv.overrides { + target_kv.overrides.entry(k.clone()).or_insert(v.clone()); + } + } + } + + merge_kv(&mut self.bootstrap_conf, &defaults.bootstrap_conf); + merge_kv(&mut self.nifi_properties, &defaults.nifi_properties); + merge_kv(&mut self.security_properties, &defaults.security_properties); + } +} + impl HasStatusCondition for v1alpha1::NifiCluster { fn conditions(&self) -> Vec { match &self.status { @@ -353,7 +375,10 @@ impl CreateReportingTaskJob { } } -#[derive(strum::Display)] +#[derive( + Clone, Debug, Deserialize, Eq, JsonSchema, Ord, PartialEq, PartialOrd, Serialize, strum::Display, +)] +#[serde(rename_all = "camelCase")] #[strum(serialize_all = "camelCase")] pub enum NifiRole { #[strum(serialize = "node")] @@ -586,6 +611,91 @@ fn node_default_listener_class() -> String { "cluster-internal".to_string() } +#[cfg(test)] +mod merge_tests { + use std::collections::BTreeMap; + + use stackable_operator::config::merge::Merge as _; + use stackable_operator::config_overrides::KeyValueConfigOverrides; + + use super::v1alpha1::NifiConfigOverrides; + + fn kv(pairs: &[(&str, &str)]) -> KeyValueConfigOverrides { + KeyValueConfigOverrides { + overrides: pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect::>(), + } + } + + fn overrides( + bootstrap: Option, + nifi: Option, + security: Option, + ) -> NifiConfigOverrides { + NifiConfigOverrides { + bootstrap_conf: bootstrap, + nifi_properties: nifi, + security_properties: security, + } + } + + #[test] + fn rolegroup_key_wins_over_role_key() { + let mut rg = overrides(Some(kv(&[("nifi.bootstrap.key", "rg-value")])), None, None); + let role = overrides( + Some(kv(&[("nifi.bootstrap.key", "role-value")])), + None, + None, + ); + rg.merge(&role); + assert_eq!( + rg.bootstrap_conf.unwrap().overrides["nifi.bootstrap.key"], + "rg-value" + ); + } + + #[test] + fn role_key_fills_gap_absent_from_rolegroup() { + let mut rg = overrides(Some(kv(&[("rg.only.key", "rg-value")])), None, None); + let role = overrides( + Some(kv(&[ + ("rg.only.key", "role-value"), + ("role.only.key", "role-default"), + ])), + None, + None, + ); + rg.merge(&role); + let result = rg.bootstrap_conf.unwrap(); + assert_eq!(result.overrides["rg.only.key"], "rg-value"); + assert_eq!(result.overrides["role.only.key"], "role-default"); + } + + #[test] + fn none_field_adopts_role_values() { + let mut rg = overrides(None, Some(kv(&[("nifi.some.prop", "rg-val")])), None); + let role = overrides( + Some(kv(&[("nifi.bootstrap.key", "role-default")])), + Some(kv(&[ + ("nifi.some.prop", "role-val"), + ("nifi.other.prop", "role-other"), + ])), + None, + ); + rg.merge(&role); + assert_eq!( + rg.bootstrap_conf.as_ref().unwrap().overrides["nifi.bootstrap.key"], + "role-default" + ); + let nifi = rg.nifi_properties.as_ref().unwrap(); + assert_eq!(nifi.overrides["nifi.some.prop"], "rg-val"); + assert_eq!(nifi.overrides["nifi.other.prop"], "role-other"); + assert!(rg.security_properties.is_none()); + } +} + #[cfg(test)] mod tests { use stackable_operator::versioned::test_utils::RoundtripTestData; diff --git a/rust/operator-binary/src/framework.rs b/rust/operator-binary/src/framework.rs new file mode 100644 index 00000000..0f5717f4 --- /dev/null +++ b/rust/operator-binary/src/framework.rs @@ -0,0 +1,8 @@ +//! Local additions to `stackable-operator` that are not yet (well) generalized in +//! `stackable_operator::v2::*`. +//! +//! Follow-up: replace these with `stackable_operator::v2::*` imports once upstream +//! reconciles `with_validated_config` (it currently returns a bare `RoleGroup`, and +//! the upstream `RoleGroupConfig` uses `EnvVarSet` rather than a plain map). + +pub mod role_utils; diff --git a/rust/operator-binary/src/framework/role_utils.rs b/rust/operator-binary/src/framework/role_utils.rs new file mode 100644 index 00000000..d426b7a2 --- /dev/null +++ b/rust/operator-binary/src/framework/role_utils.rs @@ -0,0 +1,156 @@ +//! Vendored variant of `stackable_operator::v2::role_utils` from the +//! `smooth-operator` branch, with simplifications appropriate for nifi-operator. +//! +//! Differences from upstream: +//! - `env_overrides` is `HashMap` instead of `EnvVarSet`. +//! - No `cli_overrides_to_vec` helper, `ResourceNames`, or service-account helpers. +//! - The `CommonConfig` (a.k.a. `product_specific_common_config`) does NOT need to +//! implement `Merge`. Upstream Trino uses `JavaCommonConfig`, which intentionally +//! does not implement `Merge` because its inner `JvmArgumentOverrides::try_merge` +//! is fallible (regex validation). Merging JVM argument overrides for Trino is +//! handled separately via `Role::get_merged_jvm_argument_overrides`. The +//! `RoleGroupConfig::product_specific_common_config` field here simply carries +//! the role-group level value through. +//! +//! Replace with `stackable_operator::v2::role_utils::*` once upstream publishes +//! the module. + +use std::collections::BTreeMap; + +use serde::Serialize; +use stackable_operator::{ + config::{ + fragment::{self, FromFragment}, + merge::{Merge, merge}, + }, + k8s_openapi::{DeepMerge, api::core::v1::PodTemplateSpec}, + role_utils::{Role, RoleGroup}, + schemars::JsonSchema, +}; + +/// Trino-friendly view of a validated, merged `RoleGroup`. +/// +/// Mirrors `stackable_operator::v2::role_utils::RoleGroupConfig` on the +/// `smooth-operator` branch, with `env_overrides: BTreeMap` +/// instead of the upstream `EnvVarSet`. +#[derive(Clone, Debug, PartialEq)] +pub struct RoleGroupConfig { + pub replicas: u16, + pub config: Config, + pub config_overrides: ConfigOverrides, + pub env_overrides: BTreeMap, + pub cli_overrides: BTreeMap, + pub pod_overrides: PodTemplateSpec, + pub product_specific_common_config: CommonConfig, +} + +/// Merges and validates the `RoleGroup` with the given `role` and `default_config`, +/// returning a `RoleGroupConfig`. +/// +/// Merge order matches `with_validated_config` on `smooth-operator`: +/// - `Config` (Fragment): `default_config <- role.config <- rg.config` via `Merge::merge`, +/// then validated to `ValidatedConfig` via `FromFragment`. +/// - `ConfigOverrides`: `role.config_overrides <- rg.config_overrides` via `Merge::merge`. +/// - `env_overrides` / `cli_overrides`: `extend` (rg keys overwrite role keys). +/// - `pod_overrides`: `DeepMerge::merge_from` (rg overrides role). +/// - `product_specific_common_config`: passes through the role-group level value +/// (see module docs for rationale). +pub fn with_validated_config( + role_group: &RoleGroup, + role: &Role, + default_config: &Config, +) -> Result< + RoleGroupConfig, + fragment::ValidationError, +> +where + ValidatedConfig: FromFragment, + CommonConfig: Clone + Default + JsonSchema + Serialize, + Config: Clone + Merge, + RoleConfig: Default + JsonSchema + Serialize, + ConfigOverrides: Clone + Default + JsonSchema + Merge + Serialize, +{ + let validated_config = validate_config(role_group, role, default_config)?; + Ok(RoleGroupConfig { + replicas: role_group.replicas.unwrap_or(1), + config: validated_config, + config_overrides: merged_config_overrides( + &role.config.config_overrides, + role_group.config.config_overrides.clone(), + ), + env_overrides: merged_env_overrides( + role.config + .env_overrides + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + role_group + .config + .env_overrides + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + ), + cli_overrides: merged_cli_overrides( + role.config.cli_overrides.clone(), + role_group.config.cli_overrides.clone(), + ), + pod_overrides: merged_pod_overrides( + role.config.pod_overrides.clone(), + role_group.config.pod_overrides.clone(), + ), + product_specific_common_config: role_group.config.product_specific_common_config.clone(), + }) +} + +fn validate_config( + role_group: &RoleGroup, + role: &Role, + default_config: &Config, +) -> Result +where + ValidatedConfig: FromFragment, + CommonConfig: Default + JsonSchema + Serialize, + Config: Clone + Merge, + RoleConfig: Default + JsonSchema + Serialize, + ConfigOverrides: Default + JsonSchema + Serialize, +{ + role_group.validate_config(role, default_config) +} + +fn merged_config_overrides( + role_config_overrides: &ConfigOverrides, + role_group_config_overrides: ConfigOverrides, +) -> ConfigOverrides +where + ConfigOverrides: Merge, +{ + merge(role_group_config_overrides, role_config_overrides) +} + +fn merged_env_overrides( + role_env_overrides: BTreeMap, + role_group_env_overrides: BTreeMap, +) -> BTreeMap { + let mut merged = role_env_overrides; + merged.extend(role_group_env_overrides); + merged +} + +fn merged_cli_overrides( + role_cli_overrides: BTreeMap, + role_group_cli_overrides: BTreeMap, +) -> BTreeMap { + let mut merged = role_cli_overrides; + merged.extend(role_group_cli_overrides); + merged +} + +fn merged_pod_overrides( + role_pod_overrides: PodTemplateSpec, + role_group_pod_overrides: PodTemplateSpec, +) -> PodTemplateSpec { + let mut merged = role_pod_overrides; + merged.merge_from(role_group_pod_overrides); + merged +} diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index d20eea0d..94d50ca9 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -41,6 +41,7 @@ use crate::{ mod config; mod controller; mod crd; +mod framework; mod listener; mod operations; mod product_logging; From 12cde33ca5ec08f2f5ecb06fb27f753453716156 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Jun 2026 21:09:03 +0200 Subject: [PATCH 04/39] refactor: ValidatedInputs -> ValidatedCluster with nested cluster_config Rename ValidatedInputs to ValidatedCluster and introduce a nested ValidatedClusterConfig grouping authentication, authorization, and proxy_hosts. Add a name field (dead_code for now). Add #[derive(Clone)] to NifiAuthenticationConfig and ResolvedNifiAuthorizationConfig so reconcile_nifi can clone fields while keeping the validated binding alive for later tasks. Co-Authored-By: Claude Opus 4.8 (1M context) --- rust/operator-binary/src/controller.rs | 16 +++++--- .../src/controller/validate.rs | 40 ++++++++++++------- .../src/security/authentication.rs | 1 + .../src/security/authorization.rs | 1 + 4 files changed, 39 insertions(+), 19 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 59fabcbd..e5b58ef0 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -379,11 +379,17 @@ pub async fn reconcile_nifi( ) .context(ValidateClusterSnafu)?; - let resolved_product_image = validated.image; - let authentication_config = validated.authentication_config; - let authorization_config = validated.authorization_config; - let validated_config = validated.validated_role_config; - let proxy_hosts = validated.proxy_hosts; + let validate::ValidatedCluster { + image: resolved_product_image, + cluster_config: + validate::ValidatedClusterConfig { + authentication: authentication_config, + authorization: authorization_config, + proxy_hosts, + }, + validated_role_config: validated_config, + .. + } = validated; tracing::info!("Checking for sensitive key configuration"); check_or_generate_sensitive_key(client, nifi) diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 8f7e436c..f90542c8 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -1,7 +1,7 @@ //! The validate step in the NifiCluster controller //! //! Synchronously validates inputs that don't require Kubernetes API calls. Produces -//! [`ValidatedInputs`], consumed by the rest of `reconcile_nifi`. +//! [`ValidatedCluster`], consumed by the rest of `reconcile_nifi`. use std::collections::{BTreeMap, HashSet}; @@ -69,17 +69,26 @@ pub type NifiRoleGroupConfig = crate::framework::role_utils::RoleGroupConfig< type Result = std::result::Result; -/// Synchronous inputs the rest of `reconcile_nifi` needs after dereferencing. -pub struct ValidatedInputs { +/// The validated NifiCluster: everything `reconcile_nifi` needs after dereferencing, +/// in fail-safe / resolved form. The raw `NifiCluster` should only be needed for +/// OwnerReferences after this point. +pub struct ValidatedCluster { + #[allow(dead_code)] + pub name: String, pub image: ResolvedProductImage, - pub authentication_config: NifiAuthenticationConfig, - pub authorization_config: ResolvedNifiAuthorizationConfig, - pub validated_role_config: ValidatedRoleConfigByPropertyKind, - // Comma-separated NiFi proxy hosts, or `"*"` if `spec.clusterConfig.hostHeaderCheck.allowAll` is set. - pub proxy_hosts: String, // Not yet consumed — Tasks 4-6 will use this to replace the product-config pipeline. #[allow(dead_code)] pub role_group_configs: BTreeMap>, + pub cluster_config: ValidatedClusterConfig, + // Temporary: retained until a later task migrates the configmap builder off product-config. + pub validated_role_config: ValidatedRoleConfigByPropertyKind, +} + +pub struct ValidatedClusterConfig { + pub authentication: NifiAuthenticationConfig, + pub authorization: ResolvedNifiAuthorizationConfig, + /// Comma-separated NiFi proxy hosts, or `"*"` if `hostHeaderCheck.allowAll` is set. + pub proxy_hosts: String, } /// Validates the cluster spec and the dereferenced inputs. @@ -89,7 +98,7 @@ pub fn validate( operator_environment: &OperatorEnvironmentOptions, product_config: &ProductConfigManager, cluster_info: &KubernetesClusterInfo, -) -> Result { +) -> Result { let image = nifi .spec .image @@ -119,13 +128,16 @@ pub fn validate( let proxy_hosts = compute_proxy_hosts(nifi, cluster_info)?; - Ok(ValidatedInputs { + Ok(ValidatedCluster { + name: nifi.name_any(), image, - authentication_config, - authorization_config, - validated_role_config, - proxy_hosts, role_group_configs: build_role_group_configs(nifi)?, + cluster_config: ValidatedClusterConfig { + authentication: authentication_config, + authorization: authorization_config, + proxy_hosts, + }, + validated_role_config, }) } diff --git a/rust/operator-binary/src/security/authentication.rs b/rust/operator-binary/src/security/authentication.rs index e91a5e3b..44cd8ac5 100644 --- a/rust/operator-binary/src/security/authentication.rs +++ b/rust/operator-binary/src/security/authentication.rs @@ -119,6 +119,7 @@ impl DereferencedAuthenticationClasses { } #[allow(clippy::large_enum_variant)] +#[derive(Clone)] pub enum NifiAuthenticationConfig { SingleUser { provider: r#static::v1alpha1::AuthenticationProvider, diff --git a/rust/operator-binary/src/security/authorization.rs b/rust/operator-binary/src/security/authorization.rs index df8d1d99..cd63bf57 100644 --- a/rust/operator-binary/src/security/authorization.rs +++ b/rust/operator-binary/src/security/authorization.rs @@ -37,6 +37,7 @@ pub enum Error { }, } +#[derive(Clone)] pub enum ResolvedNifiAuthorizationConfig { Opa { config: OpaConfig, From 8df8d18439e2e81f1553f5b07285904895acb405 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Jun 2026 21:31:07 +0200 Subject: [PATCH 05/39] feat: per-file build() modules under controller/build/properties Create one build() module per NiFi config file (bootstrap.conf, nifi.properties, state-management.xml, security.properties, login-identity-providers.xml, authorizers.xml) behind the ConfigFileName enum, sourcing user overrides from the typed NifiRoleGroupConfig instead of the product-config PropertyNameKind::File maps. Wire the new builders into build_node_rolegroup_config_map, keeping validated alive in reconcile_nifi and looking up the typed rg per loop iteration. Add clustering_backend and sensitive_properties_algorithm to ValidatedClusterConfig; validate the algorithm at the validate() step. Move bootstrap.conf tests (now byte-identical) to bootstrap_conf.rs. Co-Authored-By: Claude Opus 4.8 (1M context) --- rust/operator-binary/src/config/mod.rs | 818 +----------------- rust/operator-binary/src/controller.rs | 147 ++-- .../src/controller/build/properties.rs | 25 +- .../build/properties/authorizers.rs | 11 + .../build/properties/bootstrap_conf.rs | 199 +++++ .../properties/login_identity_providers.rs | 10 + .../build/properties/nifi_properties.rs | 586 +++++++++++++ .../build/properties/security_properties.rs | 70 ++ .../build/properties/state_management_xml.rs | 68 ++ .../src/controller/validate.rs | 25 +- .../src/security/authentication.rs | 3 - 11 files changed, 1052 insertions(+), 910 deletions(-) create mode 100644 rust/operator-binary/src/controller/build/properties/authorizers.rs create mode 100644 rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs create mode 100644 rust/operator-binary/src/controller/build/properties/login_identity_providers.rs create mode 100644 rust/operator-binary/src/controller/build/properties/nifi_properties.rs create mode 100644 rust/operator-binary/src/controller/build/properties/security_properties.rs create mode 100644 rust/operator-binary/src/controller/build/properties/state_management_xml.rs diff --git a/rust/operator-binary/src/config/mod.rs b/rust/operator-binary/src/config/mod.rs index 5e4d1945..cff95f96 100644 --- a/rust/operator-binary/src/config/mod.rs +++ b/rust/operator-binary/src/config/mod.rs @@ -3,33 +3,17 @@ use std::{ fmt::Write, }; -use jvm::build_merged_jvm_config; use product_config::{ProductConfigManager, types::PropertyNameKind}; -use snafu::{ResultExt, Snafu, ensure}; -use stackable_operator::{ - commons::resources::Resources, - crd::git_sync, - memory::MemoryQuantity, - product_config_utils::{ - ValidatedRoleConfigByPropertyKind, transform_all_roles_to_config, - validate_all_roles_and_groups_config, - }, +use snafu::{ResultExt, Snafu}; +use stackable_operator::product_config_utils::{ + ValidatedRoleConfigByPropertyKind, transform_all_roles_to_config, + validate_all_roles_and_groups_config, }; use strum::{Display, EnumIter}; use crate::{ - crd::{ - HTTPS_PORT, NifiConfig, NifiRole, NifiRoleType, NifiStorageConfig, PROTOCOL_PORT, - sensitive_properties, - v1alpha1::{self, NifiClusteringBackend}, - }, - operations::graceful_shutdown::graceful_shutdown_config_properties, - security::{ - authentication::{ - NifiAuthenticationConfig, STACKABLE_SERVER_TLS_DIR, STACKABLE_TLS_STORE_PASSWORD, - }, - oidc::{self, add_oidc_config_to_properties}, - }, + crd::{NifiRole, NifiRoleType, v1alpha1}, + security::oidc, }; pub mod jvm; @@ -43,12 +27,6 @@ pub const NIFI_PROPERTIES: &str = "nifi.properties"; pub const NIFI_STATE_MANAGEMENT_XML: &str = "state-management.xml"; pub const JVM_SECURITY_PROPERTIES_FILE: &str = "security.properties"; -// Keep some overhead for NiFi volumes, since cleanup is an asynchronous process that can stall active jobs -const STORAGE_PROVENANCE_UTILIZATION_FACTOR: f32 = 0.9; -const STORAGE_FLOW_ARCHIVE_UTILIZATION_FACTOR: f32 = 0.9; -// Content archive only counts _old_ data, so we want to allow some space for active data as well -const STORAGE_CONTENT_ARCHIVE_UTILIZATION_FACTOR: f32 = 0.5; - #[derive(Debug, Display, EnumIter)] pub enum NifiRepository { #[strum(serialize = "filebased")] @@ -76,6 +54,7 @@ impl NifiRepository { } #[derive(Snafu, Debug)] +#[snafu(visibility(pub(crate)))] pub enum Error { #[snafu(display("invalid product config"))] InvalidProductConfig { @@ -106,642 +85,6 @@ pub enum Error { "NiFi 1.x requires ZooKeeper (hint: upgrade to NiFi 2.x or set .spec.clusterConfig.zookeeperConfigMapName)" ))] Nifi1RequiresZookeeper, - - #[snafu(display("failed to configure sensitive properties"))] - ConfigureSensitiveProperties { source: sensitive_properties::Error }, -} - -/// Create the NiFi bootstrap.conf -pub fn build_bootstrap_conf( - merged_config: &NifiConfig, - overrides: BTreeMap, - role: &NifiRoleType, - role_group: &str, - authorization_config: Option<&crate::security::authorization::ResolvedNifiAuthorizationConfig>, -) -> Result { - let mut bootstrap = BTreeMap::new(); - // Java command to use when running NiFi - bootstrap.insert("java".to_string(), "java".to_string()); - // Username to use when running NiFi. This value will be ignored on Windows. - bootstrap.insert("run.as".to_string(), "".to_string()); - // Preserve shell environment while running as "run.as" user - bootstrap.insert("preserve.environment".to_string(), "false".to_string()); - // Configure where NiFi's lib and conf directories live - bootstrap.insert("lib.dir".to_string(), "./lib".to_string()); - bootstrap.insert("conf.dir".to_string(), "./conf".to_string()); - bootstrap.extend(graceful_shutdown_config_properties(merged_config)); - - let merged_jvm_config = - build_merged_jvm_config(merged_config, role, role_group, authorization_config) - .context(InvalidJVMConfigSnafu)?; - - for (index, argument) in merged_jvm_config - .effective_jvm_config_after_merging() - .iter() - .enumerate() - { - bootstrap.insert(format!("java.arg.{}", index + 1), argument.clone()); - } - - // configOverrides come last - bootstrap.extend(overrides); - - Ok(format_properties(bootstrap)) -} - -/// Create the NiFi nifi.properties -pub fn build_nifi_properties( - spec: &v1alpha1::NifiClusterSpec, - resource_config: &Resources, - proxy_hosts: &str, - auth_config: &NifiAuthenticationConfig, - overrides: BTreeMap, - product_version: &str, - git_sync_resources: &git_sync::v1alpha2::GitSyncResources, -) -> Result { - // TODO: Remove once we dropped support for all NiFi 1.x versions - let is_nifi_1 = product_version.starts_with("1."); - - let mut properties = BTreeMap::new(); - // Core Properties - // According to https://cwiki.apache.org/confluence/display/NIFI/Migration+Guidance#MigrationGuidance-Migratingto2.0.0-M1 - // The nifi.flow.configuration.file property in nifi.properties must be changed to reference - // "flow.json.gz" instead of "flow.xml.gz" - // TODO: Remove once we dropped support for all 1.x.x versions - // TODO(malte): In order to use CLI tools like: ./bin/nifi.sh set-sensitive-properties-algorithm NIFI_PBKDF2_AES_GCM_256 - // we have to set both "nifi.flow.configuration.file" and "nifi.flow.configuration.json.file" in NiFi 1.x.x. - if is_nifi_1 { - properties.insert( - "nifi.flow.configuration.file".to_string(), - NifiRepository::Database.mount_path() + "/flow.xml.gz", - ); - properties.insert( - "nifi.flow.configuration.json.file".to_string(), - NifiRepository::Database.mount_path() + "/flow.json.gz", - ); - } else { - properties.insert( - "nifi.flow.configuration.file".to_string(), - NifiRepository::Database.mount_path() + "/flow.json.gz", - ); - } - - properties.insert( - "nifi.flow.configuration.archive.enabled".to_string(), - "true".to_string(), - ); - properties.insert( - "nifi.flow.configuration.archive.dir".to_string(), - "/stackable/nifi/conf/archive/".to_string(), - ); - properties.insert( - "nifi.flow.configuration.archive.max.time".to_string(), - "".to_string(), - ); - if let Some(capacity) = resource_config.storage.flowfile_repo.capacity.as_ref() { - properties.insert( - "nifi.flow.configuration.archive.max.storage".to_string(), - storage_quantity_to_nifi( - MemoryQuantity::try_from(capacity).context(CalculateStorageQuotaSnafu { - repo: NifiRepository::Flowfile, - })? * STORAGE_FLOW_ARCHIVE_UTILIZATION_FACTOR, - ), - ); - } - properties.insert( - "nifi.flow.configuration.archive.max.count".to_string(), - "".to_string(), - ); - properties.insert( - "nifi.flowcontroller.autoResumeState".to_string(), - "true".to_string(), - ); - properties.insert( - "nifi.flowcontroller.graceful.shutdown.period".to_string(), - "10 sec".to_string(), - ); - properties.insert( - "nifi.flowservice.writedelay.interval".to_string(), - "500 ms".to_string(), - ); - properties.insert( - "nifi.administrative.yield.duration".to_string(), - "30 sec".to_string(), - ); - - properties.insert( - "nifi.authorizer.configuration.file".to_string(), - "/stackable/nifi/conf/authorizers.xml".to_string(), - ); - properties.insert( - "nifi.login.identity.provider.configuration.file".to_string(), - "/stackable/nifi/conf/login-identity-providers.xml".to_string(), - ); - properties.insert( - "nifi.templates.directory".to_string(), - "./conf/templates".to_string(), - ); - properties.insert("nifi.ui.banner.text".to_string(), "".to_string()); - properties.insert( - "nifi.ui.autorefresh.interval".to_string(), - "30 sec".to_string(), - ); - properties.insert( - "nifi.nar.library.directory".to_string(), - "./lib".to_string(), - ); - properties.insert( - "nifi.nar.library.autoload.directory".to_string(), - "./extensions".to_string(), - ); - properties.insert( - "nifi.nar.working.directory".to_string(), - "./work/nar/".to_string(), - ); - properties.insert( - "nifi.documentation.working.directory".to_string(), - "./work/docs/components".to_string(), - ); - - //################### - // State Management # - //################### - properties.insert( - "nifi.state.management.configuration.file".to_string(), - "./conf/state-management.xml".to_string(), - ); - // The ID of the local state provider - properties.insert( - "nifi.state.management.provider.local".to_string(), - "local-provider".to_string(), - ); - // The ID of the cluster-wide state provider. This will be ignored if NiFi is not clustered but must be populated if running in a cluster. - properties.insert( - "nifi.state.management.provider.cluster".to_string(), - match spec.cluster_config.clustering_backend { - v1alpha1::NifiClusteringBackend::ZooKeeper { .. } => "zk-provider".to_string(), - v1alpha1::NifiClusteringBackend::Kubernetes { .. } => "kubernetes-provider".to_string(), - }, - ); - // Specifies whether or not this instance of NiFi should run an embedded ZooKeeper server - properties.insert( - "nifi.state.management.embedded.zookeeper.start".to_string(), - "false".to_string(), - ); - - // H2 Settings - properties.insert( - "nifi.database.directory".to_string(), - NifiRepository::Database.mount_path(), - ); - properties.insert( - "nifi.h2.url.append".to_string(), - ";LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE".to_string(), - ); - - // FlowFile Repository - properties.insert( - "nifi.flowfile.repository.implementation".to_string(), - "org.apache.nifi.controller.repository.WriteAheadFlowFileRepository".to_string(), - ); - properties.insert( - "nifi.flowfile.repository.wal.implementation".to_string(), - "org.apache.nifi.wali.SequentialAccessWriteAheadLog".to_string(), - ); - properties.insert( - "nifi.flowfile.repository.directory".to_string(), - NifiRepository::Flowfile.mount_path(), - ); - properties.insert( - "nifi.flowfile.repository.checkpoint.interval".to_string(), - "20 secs".to_string(), - ); - properties.insert( - "nifi.flowfile.repository.always.sync".to_string(), - "false".to_string(), - ); - properties.insert( - "nifi.flowfile.repository.retain.orphaned.flowfiles".to_string(), - "true".to_string(), - ); - - properties.insert( - "nifi.swap.manager.implementation".to_string(), - "org.apache.nifi.controller.FileSystemSwapManager".to_string(), - ); - properties.insert("nifi.queue.swap.threshold".to_string(), "20000".to_string()); - - // Content Repository - properties.insert( - "nifi.content.repository.implementation".to_string(), - "org.apache.nifi.controller.repository.FileSystemRepository".to_string(), - ); - properties.insert( - "nifi.content.claim.max.appendable.size".to_string(), - "1 MB".to_string(), - ); - properties.insert( - "nifi.content.repository.directory.default".to_string(), - NifiRepository::Content.mount_path(), - ); - // Cap archived content age so the archive directory stays bounded in - // file count. NiFi treats empty as Long.MAX_VALUE, leaving size-based - // purge as the only trigger; that lets the archive grow to whatever - // half the PVC holds, and the startup directory scan in - // FileSystemRepository.initializeRepository scales with file count. - // 3 days covers a Friday-incident-investigated-Monday window for - // content replay; users with longer requirements can extend via - // configOverrides. The percentage-based threshold below acts as a - // safety net if write rate outpaces time-based purge. - // Also see https://github.com/stackabletech/nifi-operator/issues/354 - properties.insert( - "nifi.content.repository.archive.max.retention.period".to_string(), - "3 days".to_string(), - ); - properties.insert( - "nifi.content.repository.archive.max.usage.percentage".to_string(), - format!("{}%", STORAGE_CONTENT_ARCHIVE_UTILIZATION_FACTOR * 100.0), - ); - properties.insert( - "nifi.content.repository.archive.enabled".to_string(), - "true".to_string(), - ); - properties.insert( - "nifi.content.repository.always.sync".to_string(), - "false".to_string(), - ); - properties.insert( - "nifi.content.viewer.url".to_string(), - "../nifi-content-viewer/".to_string(), - ); - - // Provenance Repository Properties - properties.insert( - "nifi.provenance.repository.implementation".to_string(), - "org.apache.nifi.provenance.WriteAheadProvenanceRepository".to_string(), - ); - - // Persistent Provenance Repository Properties - properties.insert( - "nifi.provenance.repository.directory.default".to_string(), - NifiRepository::Provenance.mount_path(), - ); - properties.insert( - "nifi.provenance.repository.max.storage.time".to_string(), - "".to_string(), - ); - if let Some(capacity) = resource_config.storage.provenance_repo.capacity.as_ref() { - properties.insert( - "nifi.provenance.repository.max.storage.size".to_string(), - storage_quantity_to_nifi( - MemoryQuantity::try_from(capacity).context(CalculateStorageQuotaSnafu { - repo: NifiRepository::Provenance, - })? * STORAGE_PROVENANCE_UTILIZATION_FACTOR, - ), - ); - } - properties.insert( - "nifi.provenance.repository.rollover.time".to_string(), - "10 mins".to_string(), - ); - properties.insert( - "nifi.provenance.repository.rollover.size".to_string(), - "100 MB".to_string(), - ); - properties.insert( - "nifi.provenance.repository.query.threads".to_string(), - "2".to_string(), - ); - properties.insert( - "nifi.provenance.repository.index.threads".to_string(), - "2".to_string(), - ); - properties.insert( - "nifi.provenance.repository.compress.on.rollover".to_string(), - "true".to_string(), - ); - properties.insert( - "nifi.provenance.repository.always.sync".to_string(), - "false".to_string(), - ); - // Comma-separated list of fields. Fields that are not indexed will not be searchable. Valid fields are: - // EventType, FlowFileUUID, Filename, TransitURI, ProcessorID, AlternateIdentifierURI, Relationship, Details - properties.insert( - "nifi.provenance.repository.indexed.fields".to_string(), - "EventType, FlowFileUUID, Filename, ProcessorID, Relationship".to_string(), - ); - // FlowFile Attributes that should be indexed and made searchable. Some examples to consider are filename, uuid, mime.type - properties.insert( - "nifi.provenance.repository.indexed.attributes".to_string(), - "".to_string(), - ); - // Large values for the shard size will result in more Java heap usage when searching the Provenance Repository - // but should provide better performance - properties.insert( - "nifi.provenance.repository.index.shard.size".to_string(), - "500 MB".to_string(), - ); - // Indicates the maximum length that a FlowFile attribute can be when retrieving a Provenance Event from - // the repository. If the length of any attribute exceeds this value, it will be truncated when the event is retrieved. - properties.insert( - "nifi.provenance.repository.max.attribute.length".to_string(), - "65536".to_string(), - ); - properties.insert( - "nifi.provenance.repository.concurrent.merge.threads".to_string(), - "2".to_string(), - ); - - // Volatile Provenance Repository Properties - properties.insert( - "nifi.provenance.repository.buffer.size".to_string(), - "100000".to_string(), - ); - - // Component Status Repository - properties.insert( - "nifi.components.status.repository.implementation".to_string(), - "org.apache.nifi.controller.status.history.VolatileComponentStatusRepository".to_string(), - ); - properties.insert( - "nifi.components.status.repository.buffer.size".to_string(), - "1440".to_string(), - ); - properties.insert( - "nifi.components.status.snapshot.frequency".to_string(), - "1 min".to_string(), - ); - - // QuestDB Status History Repository Properties - properties.insert( - "nifi.status.repository.questdb.persist.node.days".to_string(), - "14".to_string(), - ); - properties.insert( - "nifi.status.repository.questdb.persist.component.days".to_string(), - "3".to_string(), - ); - properties.insert( - "nifi.status.repository.questdb.persist.location".to_string(), - "./status_repository".to_string(), - ); - - //############################################# - properties.insert( - "nifi.web.https.host".to_string(), - "${env:NODE_ADDRESS}".to_string(), - ); - properties.insert("nifi.web.https.port".to_string(), HTTPS_PORT.to_string()); - properties.insert( - "nifi.web.https.network.interface.default".to_string(), - "".to_string(), - ); - // Specifically listen on eth0 and lo interfaces. - // Listening on lo allows k8s port-forward to work. - // Once we listen on lo, we need to explicitly listen on eth0 so the server can be exposed (including health probes). - // NOTE: We assume "eth0" is always the external interface in containers launched in Kubernetes. - // It is possible that some container runtime will name it differently, but we haven't yet observed that. - properties.insert( - "nifi.web.https.network.interface.eth0".to_string(), - "eth0".to_string(), - ); - properties.insert( - "nifi.web.https.network.interface.lo".to_string(), - "lo".to_string(), - ); - //############################################# - properties.insert( - "nifi.web.jetty.working.directory".to_string(), - "./work/jetty".to_string(), - ); - properties.insert("nifi.web.jetty.threads".to_string(), "200".to_string()); - properties.insert("nifi.web.max.header.size".to_string(), "16 KB".to_string()); - properties.insert("nifi.web.proxy.context.path".to_string(), "".to_string()); - properties.insert("nifi.web.proxy.host".to_string(), proxy_hosts.to_string()); - - properties.insert( - "nifi.sensitive.props.key".to_string(), - "${file:UTF-8:/stackable/sensitiveproperty/nifiSensitivePropsKey}".to_string(), - ); - properties.insert( - "nifi.sensitive.props.key.protected".to_string(), - "".to_string(), - ); - - let sensitive_properties_algorithm = &spec - .cluster_config - .sensitive_properties - .algorithm - .clone() - .unwrap_or_default(); - - sensitive_properties_algorithm - .check_for_nifi_version(spec.image.product_version()) - .context(ConfigureSensitivePropertiesSnafu)?; - - properties.insert( - "nifi.sensitive.props.algorithm".to_string(), - sensitive_properties_algorithm.to_string(), - ); - - // key and trust store - // these properties are ok to hard code here, because the cannot be configured and are - // generated with fixed values in the init container - properties.insert( - "nifi.security.keystore".to_string(), - format!( - "{keystore_path}/keystore.p12", - keystore_path = STACKABLE_SERVER_TLS_DIR - ), - ); - properties.insert( - "nifi.security.keystoreType".to_string(), - "PKCS12".to_string(), - ); - properties.insert( - "nifi.security.keystorePasswd".to_string(), - STACKABLE_TLS_STORE_PASSWORD.to_string(), - ); - properties.insert( - "nifi.security.truststore".to_string(), - format!( - "{keystore_path}/truststore.p12", - keystore_path = STACKABLE_SERVER_TLS_DIR - ), - ); - properties.insert( - "nifi.security.truststoreType".to_string(), - "PKCS12".to_string(), - ); - properties.insert( - "nifi.security.truststorePasswd".to_string(), - STACKABLE_TLS_STORE_PASSWORD.to_string(), - ); - properties.insert( - "nifi.security.user.login.identity.provider".to_string(), - "login-identity-provider".to_string(), - ); - properties.insert( - "nifi.security.user.authorizer".to_string(), - "authorizer".to_string(), - ); - properties.insert( - "nifi.security.allow.anonymous.authentication".to_string(), - "false".to_string(), - ); - properties.insert( - "nifi.cluster.protocol.is.secure".to_string(), - "true".to_string(), - ); - - if let NifiAuthenticationConfig::Oidc { provider, oidc, .. } = auth_config { - add_oidc_config_to_properties(provider, oidc, &mut properties) - .context(GenerateOidcConfigSnafu)?; - }; - - // cluster node properties (only configure for cluster nodes) - properties.insert("nifi.cluster.is.node".to_string(), "true".to_string()); - properties.insert( - "nifi.cluster.node.address".to_string(), - "${env:NODE_ADDRESS}".to_string(), - ); - properties.insert( - "nifi.cluster.node.protocol.port".to_string(), - PROTOCOL_PORT.to_string(), - ); - properties.insert( - "nifi.cluster.flow.election.max.candidates".to_string(), - "".to_string(), - ); - - match spec.cluster_config.clustering_backend { - v1alpha1::NifiClusteringBackend::ZooKeeper { .. } => { - properties.insert( - "nifi.cluster.leader.election.implementation".to_string(), - "CuratorLeaderElectionManager".to_string(), - ); - - // this will be replaced via a container command script - properties.insert( - "nifi.zookeeper.connect.string".to_string(), - "${env:ZOOKEEPER_HOSTS}".to_string(), - ); - - // this will be replaced via a container command script - properties.insert( - "nifi.zookeeper.root.node".to_string(), - "${env:ZOOKEEPER_CHROOT}".to_string(), - ); - } - - v1alpha1::NifiClusteringBackend::Kubernetes {} => { - ensure!(!is_nifi_1, Nifi1RequiresZookeeperSnafu); - - properties.insert( - "nifi.cluster.leader.election.implementation".to_string(), - "KubernetesLeaderElectionManager".to_string(), - ); - - // this will be replaced via a container command script - properties.insert( - "nifi.cluster.leader.election.kubernetes.lease.prefix".to_string(), - "${env:STACKLET_NAME}".to_string(), - ); - } - } - - //#################### - // Custom components # - //#################### - // NiFi 1.x does not support Python components and the Python configuration below is just - // ignored. - - // The command used to launch Python. - // This property must be set to enable Python-based processors. - properties.insert("nifi.python.command".to_string(), "python3".to_string()); - - // The directory that contains the Python framework for communicating between the Python and - // Java processes. - properties.insert( - "nifi.python.framework.source.directory".to_string(), - "/stackable/nifi/python/framework/".to_string(), - ); - - // The working directory where NiFi should store artifacts; - // This property defaults to ./work/python but if you want to mount an emptyDir for the working - // directory then another directory has to be set to avoid ownership conflicts with ./work/nar. - properties.insert( - "nifi.python.working.directory".to_string(), - NIFI_PYTHON_WORKING_DIRECTORY.to_string(), - ); - - // The default directory that NiFi should look in to find custom Python-based components. - // This directory is mentioned in the documentation - // (docs/modules/nifi/pages/usage_guide/custom-components.adoc), so do not change it! - properties.insert( - "nifi.python.extensions.source.directory.default".to_string(), - "/stackable/nifi/python/extensions/".to_string(), - ); - - for (i, git_folder) in git_sync_resources - .git_content_folders_as_string() - .into_iter() - .enumerate() - { - // The directory that NiFi should look in to find custom Python-based components. - properties.insert( - format!("nifi.python.extensions.source.directory.{i}"), - git_folder.clone(), - ); - - // The directory that NiFi should look in to find custom Java-based components. - properties.insert(format!("nifi.nar.library.directory.{i}"), git_folder); - } - //########################## - - // override with config overrides - properties.extend(overrides); - - Ok(format_properties(properties)) -} - -pub fn build_state_management_xml(clustering_backend: &NifiClusteringBackend) -> String { - // Inert providers are ignored by NiFi itself, but templating still fails if they refer to invalid environment variables, - // so only include the actually used provider. - let cluster_provider = match clustering_backend { - NifiClusteringBackend::ZooKeeper { .. } => { - r#" - zk-provider - org.apache.nifi.controller.state.providers.zookeeper.ZooKeeperStateProvider - ${env:ZOOKEEPER_HOSTS} - ${env:ZOOKEEPER_CHROOT} - 10 seconds - Open - "# - } - NifiClusteringBackend::Kubernetes {} => { - r#" - kubernetes-provider - org.apache.nifi.kubernetes.state.provider.KubernetesConfigMapStateProvider - ${env:STACKLET_NAME} - "# - } - }; - format!( - r#" - - - local-provider - org.apache.nifi.controller.state.providers.local.WriteAheadLocalStateProvider - {local_state_path} - false - 16 - 2 mins - - {cluster_provider} - "#, - local_state_path = NifiRepository::State.mount_path(), - ) } /// Defines all required roles and their required configuration. In this case we need three files: @@ -787,7 +130,7 @@ pub fn validated_product_config( // TODO: Use crate like https://crates.io/crates/java-properties (currently does not work for Nifi // because of escapes), to have save handling of escapes etc. -fn format_properties(properties: BTreeMap) -> String { +pub(crate) fn format_properties(properties: BTreeMap) -> String { let mut result = String::new(); for (key, value) in properties { @@ -796,148 +139,3 @@ fn format_properties(properties: BTreeMap) -> String { result } - -fn storage_quantity_to_nifi(quantity: MemoryQuantity) -> String { - format!( - "{}MB", - quantity - .scale_to(stackable_operator::memory::BinaryMultiple::Mebi) - .value - ) -} - -#[cfg(test)] -mod tests { - use indoc::indoc; - - use super::*; - use crate::{config::build_bootstrap_conf, crd::v1alpha1}; - - #[test] - fn test_build_bootstrap_conf_defaults() { - let input = r#" - apiVersion: nifi.stackable.tech/v1alpha1 - kind: NifiCluster - metadata: - name: simple-nifi - spec: - image: - productVersion: 2.9.0 - clusterConfig: - authentication: - - authenticationClass: nifi-admin-credentials-simple - sensitiveProperties: - keySecret: simple-nifi-sensitive-property-key - autoGenerate: true - nodes: - roleGroups: - default: - replicas: 1 - "#; - let bootstrap_conf = construct_bootstrap_conf(input); - - assert_eq!( - bootstrap_conf, - indoc! {" - conf.dir=./conf - graceful.shutdown.seconds=300 - java=java - java.arg.1=-Xmx3276m - java.arg.10=-Djavax.security.auth.useSubjectCredsOnly=true - java.arg.11=-Dzookeeper.admin.enableServer=false - java.arg.12=-Djava.security.properties=/stackable/nifi/conf/security.properties - java.arg.2=-Xms3276m - java.arg.3=-XX:+UseG1GC - java.arg.4=-Djava.awt.headless=true - java.arg.5=-Dorg.apache.jasper.compiler.disablejsr199=true - java.arg.6=-Djava.net.preferIPv4Stack=true - java.arg.7=-Dsun.net.http.allowRestrictedHeaders=true - java.arg.8=-Djava.protocol.handler.pkgs=sun.net.www.protocol - java.arg.9=-Djava.security.egd=file:/dev/urandom - lib.dir=./lib - preserve.environment=false - run.as= - "} - ); - } - - #[test] - fn test_build_bootstrap_conf_jvm_argument_overrides() { - let input = r#" - apiVersion: nifi.stackable.tech/v1alpha1 - kind: NifiCluster - metadata: - name: simple-nifi - spec: - image: - productVersion: 2.9.0 - clusterConfig: - authentication: - - authenticationClass: nifi-admin-credentials-simple - sensitiveProperties: - keySecret: simple-nifi-sensitive-property-key - autoGenerate: true - nodes: - config: - resources: - memory: - limit: 42Gi - jvmArgumentOverrides: - remove: - - -XX:+UseG1GC - add: - - -Dhttps.proxyHost=proxy.my.corp - - -Dhttps.proxyPort=8080 - - -Djava.net.preferIPv4Stack=true - roleGroups: - default: - replicas: 1 - jvmArgumentOverrides: - # We need more memory! - removeRegex: - - -Xmx.* - - -Dhttps.proxyPort=.* - add: - - -Xmx40000m - - -Dhttps.proxyPort=1234 - "#; - let bootstrap_conf = construct_bootstrap_conf(input); - - assert_eq!( - bootstrap_conf, - indoc! {" - conf.dir=./conf - graceful.shutdown.seconds=300 - java=java - java.arg.1=-Xms34406m - java.arg.10=-Djava.security.properties=/stackable/nifi/conf/security.properties - java.arg.11=-Dhttps.proxyHost=proxy.my.corp - java.arg.12=-Djava.net.preferIPv4Stack=true - java.arg.13=-Xmx40000m - java.arg.14=-Dhttps.proxyPort=1234 - java.arg.2=-Djava.awt.headless=true - java.arg.3=-Dorg.apache.jasper.compiler.disablejsr199=true - java.arg.4=-Djava.net.preferIPv4Stack=true - java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true - java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol - java.arg.7=-Djava.security.egd=file:/dev/urandom - java.arg.8=-Djavax.security.auth.useSubjectCredsOnly=true - java.arg.9=-Dzookeeper.admin.enableServer=false - lib.dir=./lib - preserve.environment=false - run.as= - "} - ); - } - - fn construct_bootstrap_conf(nifi_cluster: &str) -> String { - let nifi: v1alpha1::NifiCluster = - serde_yaml::from_str(nifi_cluster).expect("illegal test input"); - - let nifi_role = NifiRole::Node; - let role = nifi.spec.nodes.as_ref().unwrap(); - let merged_config = nifi.merged_config(&nifi_role, "default").unwrap(); - - build_bootstrap_conf(&merged_config, BTreeMap::new(), role, "default", None).unwrap() - } -} diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index e5b58ef0..099a4197 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -6,9 +6,7 @@ use std::{ sync::Arc, }; -use crate::controller::build::properties::writer::{ - PropertiesWriterError, to_java_properties_string, -}; +use crate::controller::build::properties::writer::PropertiesWriterError; use const_format::concatcp; use indoc::formatdoc; use product_config::{ProductConfigManager, types::PropertyNameKind}; @@ -78,9 +76,12 @@ mod validate; use crate::{ OPERATOR_NAME, config::{ - self, JVM_SECURITY_PROPERTIES_FILE, NIFI_BOOTSTRAP_CONF, NIFI_CONFIG_DIRECTORY, - NIFI_PROPERTIES, NIFI_PYTHON_WORKING_DIRECTORY, NIFI_STATE_MANAGEMENT_XML, NifiRepository, - build_bootstrap_conf, build_nifi_properties, build_state_management_xml, + self, JVM_SECURITY_PROPERTIES_FILE, NIFI_CONFIG_DIRECTORY, NIFI_PYTHON_WORKING_DIRECTORY, + NifiRepository, + }, + controller::build::properties::{ + ConfigFileName, authorizers, bootstrap_conf, login_identity_providers, nifi_properties, + security_properties, state_management_xml, }, crd::{ APP_NAME, BALANCE_PORT, BALANCE_PORT_NAME, Container, HTTPS_PORT, HTTPS_PORT_NAME, @@ -101,7 +102,6 @@ use crate::{ reporting_task::{build_maybe_reporting_task, build_reporting_task_service_name}, security::{ authentication::{ - AUTHORIZERS_XML_FILE_NAME, LOGIN_IDENTITY_PROVIDERS_XML_FILE_NAME, NifiAuthenticationConfig, STACKABLE_SERVER_TLS_DIR, STACKABLE_TLS_STORE_PASSWORD, }, authorization::{self, OPA_TLS_MOUNT_PATH, ResolvedNifiAuthorizationConfig}, @@ -110,6 +110,7 @@ use crate::{ }, service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, }; +use validate::{NifiRoleGroupConfig, ValidatedCluster}; pub const NIFI_CONTROLLER_NAME: &str = "nificluster"; pub const NIFI_FULL_CONTROLLER_NAME: &str = concatcp!(NIFI_CONTROLLER_NAME, '.', OPERATOR_NAME); @@ -341,6 +342,9 @@ pub enum Error { #[snafu(display("failed to build authorization configuration"))] AuthorizationConfiguration { source: authorization::Error }, + + #[snafu(display("missing role group config for rolegroup {rolegroup_name}"))] + MissingRoleGroupConfig { rolegroup_name: String }, } type Result = std::result::Result; @@ -379,17 +383,10 @@ pub async fn reconcile_nifi( ) .context(ValidateClusterSnafu)?; - let validate::ValidatedCluster { - image: resolved_product_image, - cluster_config: - validate::ValidatedClusterConfig { - authentication: authentication_config, - authorization: authorization_config, - proxy_hosts, - }, - validated_role_config: validated_config, - .. - } = validated; + let resolved_product_image = &validated.image; + let authentication_config = &validated.cluster_config.authentication; + let authorization_config = &validated.cluster_config.authorization; + let validated_config = &validated.validated_role_config; tracing::info!("Checking for sensitive key configuration"); check_or_generate_sensitive_key(client, nifi) @@ -478,9 +475,17 @@ pub async fn reconcile_nifi( .merged_config(&NifiRole::Node, rolegroup_name) .context(FailedToResolveConfigSnafu)?; + let rg = validated + .role_group_configs + .get(&NifiRole::Node) + .and_then(|g| g.get(rolegroup_name)) + .context(MissingRoleGroupConfigSnafu { + rolegroup_name: rolegroup_name.clone(), + })?; + let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new( &nifi.spec.cluster_config.custom_components_git_sync, - &resolved_product_image, + resolved_product_image, &env_vars_from_rolegroup_config(rolegroup_config), &[], LOG_VOLUME_NAME, @@ -514,14 +519,13 @@ pub async fn reconcile_nifi( // For more information see let rg_configmap = build_node_rolegroup_config_map( nifi, - &resolved_product_image, - &authentication_config, - &authorization_config, + &validated, + rg, + resolved_product_image, role, &rolegroup, rolegroup_config, &merged_config, - &proxy_hosts, &git_sync_resources, ) .await?; @@ -536,14 +540,14 @@ pub async fn reconcile_nifi( let rg_statefulset = build_node_rolegroup_statefulset( nifi, - &resolved_product_image, + resolved_product_image, &client.kubernetes_cluster_info, &rolegroup, role, rolegroup_config, &merged_config, - &authentication_config, - &authorization_config, + authentication_config, + authorization_config, rolling_upgrade_supported, replicas, &rbac_sa.name_any(), @@ -634,9 +638,9 @@ pub async fn reconcile_nifi( if nifi.spec.cluster_config.create_reporting_task_job.enabled { if let Some((reporting_task_job, reporting_task_service)) = build_maybe_reporting_task( nifi, - &resolved_product_image, + resolved_product_image, &client.kubernetes_cluster_info, - &authentication_config, + authentication_config, &rbac_sa.name_any(), ) .context(ReportingTaskSnafu)? @@ -672,7 +676,7 @@ pub async fn reconcile_nifi( // we are still in the process of updating let status = if cluster_version_update_state != ClusterVersionUpdateState::UpdateRequested { NifiStatus { - deployed_version: Some(resolved_product_image.product_version), + deployed_version: Some(resolved_product_image.product_version.clone()), conditions, } } else { @@ -697,34 +701,17 @@ pub async fn reconcile_nifi( #[allow(clippy::too_many_arguments)] async fn build_node_rolegroup_config_map( nifi: &v1alpha1::NifiCluster, + cluster: &ValidatedCluster, + rg: &NifiRoleGroupConfig, resolved_product_image: &ResolvedProductImage, - authentication_config: &NifiAuthenticationConfig, - authorization_config: &ResolvedNifiAuthorizationConfig, role: &NifiRoleType, rolegroup: &RoleGroupRef, - rolegroup_config: &HashMap>, - merged_config: &NifiConfig, - proxy_hosts: &str, + _rolegroup_config: &HashMap>, + _merged_config: &NifiConfig, git_sync_resources: &git_sync::v1alpha2::GitSyncResources, ) -> Result { tracing::debug!("building rolegroup configmaps"); - let login_identity_provider_xml = authentication_config - .get_authentication_config() - .context(InvalidNifiAuthenticationConfigSnafu)?; - - let authorizers_xml = authorization_config.get_authorizers_config(nifi); - - let jvm_sec_props: BTreeMap> = rolegroup_config - .get(&PropertyNameKind::File( - JVM_SECURITY_PROPERTIES_FILE.to_string(), - )) - .cloned() - .unwrap_or_default() - .into_iter() - .map(|(k, v)| (k, Some(v))) - .collect(); - let mut cm_builder = ConfigMapBuilder::new(); cm_builder @@ -744,60 +731,44 @@ async fn build_node_rolegroup_config_map( .build(), ) .add_data( - NIFI_BOOTSTRAP_CONF, - build_bootstrap_conf( - merged_config, - rolegroup_config - .get(&PropertyNameKind::File(NIFI_BOOTSTRAP_CONF.to_string())) - .with_context(|| ProductConfigKindNotSpecifiedSnafu { - kind: NIFI_BOOTSTRAP_CONF.to_string(), - })? - .clone(), + ConfigFileName::BootstrapConf.to_string(), + bootstrap_conf::build( + rg, role, &rolegroup.role_group, - Some(authorization_config), + Some(&cluster.cluster_config.authorization), ) .context(BootstrapConfigSnafu)?, ) .add_data( - NIFI_PROPERTIES, - build_nifi_properties( - &nifi.spec, - &merged_config.resources, - proxy_hosts, - authentication_config, - rolegroup_config - .get(&PropertyNameKind::File(NIFI_PROPERTIES.to_string())) - .with_context(|| ProductConfigKindNotSpecifiedSnafu { - kind: NIFI_PROPERTIES.to_string(), - })? - .clone(), - resolved_product_image.product_version.as_ref(), - git_sync_resources, - ) - .with_context(|_| BuildProductConfigSnafu { - rolegroup: rolegroup.clone(), + ConfigFileName::NifiProperties.to_string(), + nifi_properties::build(cluster, rg, git_sync_resources).with_context(|_| { + BuildProductConfigSnafu { + rolegroup: rolegroup.clone(), + } })?, ) .add_data( - NIFI_STATE_MANAGEMENT_XML, - build_state_management_xml(&nifi.spec.cluster_config.clustering_backend), + ConfigFileName::StateManagementXml.to_string(), + state_management_xml::build(&cluster.cluster_config.clustering_backend), ) .add_data( - LOGIN_IDENTITY_PROVIDERS_XML_FILE_NAME, - login_identity_provider_xml, + ConfigFileName::LoginIdentityProviders.to_string(), + login_identity_providers::build(cluster) + .context(InvalidNifiAuthenticationConfigSnafu)?, ) - .add_data(AUTHORIZERS_XML_FILE_NAME, authorizers_xml) .add_data( - JVM_SECURITY_PROPERTIES_FILE, - to_java_properties_string(jvm_sec_props.iter()).with_context(|_| { - JvmSecurityPropertiesSnafu { - rolegroup: rolegroup.role_group.clone(), - } + ConfigFileName::Authorizers.to_string(), + authorizers::build(cluster, nifi), + ) + .add_data( + ConfigFileName::SecurityProperties.to_string(), + security_properties::build(rg).with_context(|_| JvmSecurityPropertiesSnafu { + rolegroup: rolegroup.role_group.clone(), })?, ); - extend_role_group_config_map(rolegroup, &merged_config.logging, &mut cm_builder).context( + extend_role_group_config_map(rolegroup, &rg.config.logging, &mut cm_builder).context( InvalidLoggingConfigSnafu { cm_name: rolegroup.object_name(), }, diff --git a/rust/operator-binary/src/controller/build/properties.rs b/rust/operator-binary/src/controller/build/properties.rs index 3f33a1bf..cf384e95 100644 --- a/rust/operator-binary/src/controller/build/properties.rs +++ b/rust/operator-binary/src/controller/build/properties.rs @@ -6,12 +6,19 @@ use std::collections::BTreeMap; -use stackable_operator::config_overrides::KeyValueConfigOverrides; +use stackable_operator::config_overrides::KeyValueOverridesProvider; +use crate::controller::validate::NifiRoleGroupConfig; + +pub mod authorizers; +pub mod bootstrap_conf; +pub mod login_identity_providers; +pub mod nifi_properties; +pub mod security_properties; +pub mod state_management_xml; pub mod writer; /// The names of the files assembled into the NiFi rolegroup ConfigMap. -#[allow(dead_code)] // used once the per-file builders land in Task 4 #[derive(Clone, Copy, Debug, strum::Display)] pub enum ConfigFileName { #[strum(serialize = "bootstrap.conf")] @@ -29,7 +36,6 @@ pub enum ConfigFileName { } /// Keep only the set (`Some`) entries of a `key -> optional value` map, as `(key, value)` pairs. -#[allow(dead_code)] // used once the per-file builders land in Task 4 fn defined_entries( entries: BTreeMap>, ) -> impl Iterator { @@ -38,10 +44,13 @@ fn defined_entries( .filter_map(|(key, value)| value.map(|value| (key, value))) } -/// Resolve user-provided [`KeyValueConfigOverrides`] into key/value pairs. -#[allow(dead_code)] // used once the per-file builders land in Task 4 -fn resolved_overrides( - overrides: KeyValueConfigOverrides, +/// Resolve the user overrides for `file` from a rolegroup's config overrides, dropping unset values. +pub(crate) fn resolved_overrides_for( + rg: &NifiRoleGroupConfig, + file: ConfigFileName, ) -> impl Iterator { - overrides.overrides.into_iter() + defined_entries( + rg.config_overrides + .get_key_value_overrides(&file.to_string()), + ) } diff --git a/rust/operator-binary/src/controller/build/properties/authorizers.rs b/rust/operator-binary/src/controller/build/properties/authorizers.rs new file mode 100644 index 00000000..aae33f4d --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/authorizers.rs @@ -0,0 +1,11 @@ +//! Builder for `authorizers.xml`. + +use crate::{controller::validate::ValidatedCluster, crd::v1alpha1}; + +pub fn build(cluster: &ValidatedCluster, nifi: &v1alpha1::NifiCluster) -> String { + // TODO(follow-up PR): narrow get_authorizers_config to resolved fields on ValidatedCluster instead of taking the full NifiCluster. + cluster + .cluster_config + .authorization + .get_authorizers_config(nifi) +} diff --git a/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs b/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs new file mode 100644 index 00000000..c91490dd --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs @@ -0,0 +1,199 @@ +//! Builder for `bootstrap.conf`. + +use std::collections::BTreeMap; + +use snafu::ResultExt; + +use super::ConfigFileName; +use crate::{ + config::{Error, InvalidJVMConfigSnafu, jvm::build_merged_jvm_config}, + controller::validate::NifiRoleGroupConfig, + crd::NifiRoleType, + operations::graceful_shutdown::graceful_shutdown_config_properties, + security::authorization::ResolvedNifiAuthorizationConfig, +}; + +pub fn build( + rg: &NifiRoleGroupConfig, + role: &NifiRoleType, + role_group: &str, + authorization_config: Option<&ResolvedNifiAuthorizationConfig>, +) -> Result { + let mut bootstrap = BTreeMap::new(); + // Java command to use when running NiFi + bootstrap.insert("java".to_string(), "java".to_string()); + // Username to use when running NiFi. This value will be ignored on Windows. + bootstrap.insert("run.as".to_string(), "".to_string()); + // Preserve shell environment while running as "run.as" user + bootstrap.insert("preserve.environment".to_string(), "false".to_string()); + // Configure where NiFi's lib and conf directories live + bootstrap.insert("lib.dir".to_string(), "./lib".to_string()); + bootstrap.insert("conf.dir".to_string(), "./conf".to_string()); + bootstrap.extend(graceful_shutdown_config_properties(&rg.config)); + + let merged_jvm_config = + build_merged_jvm_config(&rg.config, role, role_group, authorization_config) + .context(InvalidJVMConfigSnafu)?; + + for (index, argument) in merged_jvm_config + .effective_jvm_config_after_merging() + .iter() + .enumerate() + { + bootstrap.insert(format!("java.arg.{}", index + 1), argument.clone()); + } + + // configOverrides come last + for (k, v) in super::resolved_overrides_for(rg, ConfigFileName::BootstrapConf) { + bootstrap.insert(k, v); + } + + Ok(crate::config::format_properties(bootstrap)) +} + +#[cfg(test)] +mod tests { + use indoc::indoc; + + use stackable_operator::kube::ResourceExt as _; + + use super::*; + use crate::{ + crd::{NifiConfig, NifiRole, v1alpha1}, + framework::role_utils::with_validated_config, + }; + + fn construct_bootstrap_conf(nifi_cluster: &str) -> String { + let nifi: v1alpha1::NifiCluster = + serde_yaml::from_str(nifi_cluster).expect("illegal test input"); + + let nifi_role = NifiRole::Node; + let role = nifi.spec.nodes.as_ref().unwrap(); + let default_config = NifiConfig::default_config(&nifi.name_any(), &nifi_role); + let rg = with_validated_config::( + role.role_groups.get("default").unwrap(), + role, + &default_config, + ) + .expect("failed to build role group config"); + + build(&rg, role, "default", None).unwrap() + } + + #[test] + fn test_build_bootstrap_conf_defaults() { + let input = r#" + apiVersion: nifi.stackable.tech/v1alpha1 + kind: NifiCluster + metadata: + name: simple-nifi + spec: + image: + productVersion: 2.9.0 + clusterConfig: + authentication: + - authenticationClass: nifi-admin-credentials-simple + sensitiveProperties: + keySecret: simple-nifi-sensitive-property-key + autoGenerate: true + nodes: + roleGroups: + default: + replicas: 1 + "#; + let bootstrap_conf = construct_bootstrap_conf(input); + + assert_eq!( + bootstrap_conf, + indoc! {" + conf.dir=./conf + graceful.shutdown.seconds=300 + java=java + java.arg.1=-Xmx3276m + java.arg.10=-Djavax.security.auth.useSubjectCredsOnly=true + java.arg.11=-Dzookeeper.admin.enableServer=false + java.arg.12=-Djava.security.properties=/stackable/nifi/conf/security.properties + java.arg.2=-Xms3276m + java.arg.3=-XX:+UseG1GC + java.arg.4=-Djava.awt.headless=true + java.arg.5=-Dorg.apache.jasper.compiler.disablejsr199=true + java.arg.6=-Djava.net.preferIPv4Stack=true + java.arg.7=-Dsun.net.http.allowRestrictedHeaders=true + java.arg.8=-Djava.protocol.handler.pkgs=sun.net.www.protocol + java.arg.9=-Djava.security.egd=file:/dev/urandom + lib.dir=./lib + preserve.environment=false + run.as= + "} + ); + } + + #[test] + fn test_build_bootstrap_conf_jvm_argument_overrides() { + let input = r#" + apiVersion: nifi.stackable.tech/v1alpha1 + kind: NifiCluster + metadata: + name: simple-nifi + spec: + image: + productVersion: 2.9.0 + clusterConfig: + authentication: + - authenticationClass: nifi-admin-credentials-simple + sensitiveProperties: + keySecret: simple-nifi-sensitive-property-key + autoGenerate: true + nodes: + config: + resources: + memory: + limit: 42Gi + jvmArgumentOverrides: + remove: + - -XX:+UseG1GC + add: + - -Dhttps.proxyHost=proxy.my.corp + - -Dhttps.proxyPort=8080 + - -Djava.net.preferIPv4Stack=true + roleGroups: + default: + replicas: 1 + jvmArgumentOverrides: + # We need more memory! + removeRegex: + - -Xmx.* + - -Dhttps.proxyPort=.* + add: + - -Xmx40000m + - -Dhttps.proxyPort=1234 + "#; + let bootstrap_conf = construct_bootstrap_conf(input); + + assert_eq!( + bootstrap_conf, + indoc! {" + conf.dir=./conf + graceful.shutdown.seconds=300 + java=java + java.arg.1=-Xms34406m + java.arg.10=-Djava.security.properties=/stackable/nifi/conf/security.properties + java.arg.11=-Dhttps.proxyHost=proxy.my.corp + java.arg.12=-Djava.net.preferIPv4Stack=true + java.arg.13=-Xmx40000m + java.arg.14=-Dhttps.proxyPort=1234 + java.arg.2=-Djava.awt.headless=true + java.arg.3=-Dorg.apache.jasper.compiler.disablejsr199=true + java.arg.4=-Djava.net.preferIPv4Stack=true + java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true + java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + java.arg.7=-Djava.security.egd=file:/dev/urandom + java.arg.8=-Djavax.security.auth.useSubjectCredsOnly=true + java.arg.9=-Dzookeeper.admin.enableServer=false + lib.dir=./lib + preserve.environment=false + run.as= + "} + ); + } +} diff --git a/rust/operator-binary/src/controller/build/properties/login_identity_providers.rs b/rust/operator-binary/src/controller/build/properties/login_identity_providers.rs new file mode 100644 index 00000000..02e6a12b --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/login_identity_providers.rs @@ -0,0 +1,10 @@ +//! Builder for `login-identity-providers.xml`. + +use crate::controller::validate::ValidatedCluster; + +pub fn build(cluster: &ValidatedCluster) -> Result { + cluster + .cluster_config + .authentication + .get_authentication_config() +} diff --git a/rust/operator-binary/src/controller/build/properties/nifi_properties.rs b/rust/operator-binary/src/controller/build/properties/nifi_properties.rs new file mode 100644 index 00000000..7e1afbf5 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/nifi_properties.rs @@ -0,0 +1,586 @@ +//! Builder for `nifi.properties`. + +use std::collections::BTreeMap; + +use snafu::{ResultExt, ensure}; +use stackable_operator::{crd::git_sync, memory::MemoryQuantity}; + +use super::ConfigFileName; +use crate::{ + config::{ + CalculateStorageQuotaSnafu, Error, GenerateOidcConfigSnafu, NIFI_PYTHON_WORKING_DIRECTORY, + Nifi1RequiresZookeeperSnafu, NifiRepository, format_properties, + }, + controller::validate::{NifiRoleGroupConfig, ValidatedCluster}, + crd::{HTTPS_PORT, v1alpha1}, + security::{ + authentication::{ + NifiAuthenticationConfig, STACKABLE_SERVER_TLS_DIR, STACKABLE_TLS_STORE_PASSWORD, + }, + oidc::add_oidc_config_to_properties, + }, +}; + +const STORAGE_PROVENANCE_UTILIZATION_FACTOR: f32 = 0.9; +const STORAGE_FLOW_ARCHIVE_UTILIZATION_FACTOR: f32 = 0.9; +const STORAGE_CONTENT_ARCHIVE_UTILIZATION_FACTOR: f32 = 0.5; + +pub fn build( + cluster: &ValidatedCluster, + rg: &NifiRoleGroupConfig, + git_sync_resources: &git_sync::v1alpha2::GitSyncResources, +) -> Result { + let product_version = &cluster.image.product_version; + let proxy_hosts = &cluster.cluster_config.proxy_hosts; + let auth_config = &cluster.cluster_config.authentication; + let resource_config = &rg.config.resources; + + // TODO: Remove once we dropped support for all NiFi 1.x versions + let is_nifi_1 = product_version.starts_with("1."); + + let mut properties = BTreeMap::new(); + // Core Properties + // According to https://cwiki.apache.org/confluence/display/NIFI/Migration+Guidance#MigrationGuidance-Migratingto2.0.0-M1 + // The nifi.flow.configuration.file property in nifi.properties must be changed to reference + // "flow.json.gz" instead of "flow.xml.gz" + // TODO: Remove once we dropped support for all 1.x.x versions + // TODO(malte): In order to use CLI tools like: ./bin/nifi.sh set-sensitive-properties-algorithm NIFI_PBKDF2_AES_GCM_256 + // we have to set both "nifi.flow.configuration.file" and "nifi.flow.configuration.json.file" in NiFi 1.x.x. + if is_nifi_1 { + properties.insert( + "nifi.flow.configuration.file".to_string(), + NifiRepository::Database.mount_path() + "/flow.xml.gz", + ); + properties.insert( + "nifi.flow.configuration.json.file".to_string(), + NifiRepository::Database.mount_path() + "/flow.json.gz", + ); + } else { + properties.insert( + "nifi.flow.configuration.file".to_string(), + NifiRepository::Database.mount_path() + "/flow.json.gz", + ); + } + + properties.insert( + "nifi.flow.configuration.archive.enabled".to_string(), + "true".to_string(), + ); + properties.insert( + "nifi.flow.configuration.archive.dir".to_string(), + "/stackable/nifi/conf/archive/".to_string(), + ); + properties.insert( + "nifi.flow.configuration.archive.max.time".to_string(), + "".to_string(), + ); + if let Some(capacity) = resource_config.storage.flowfile_repo.capacity.as_ref() { + properties.insert( + "nifi.flow.configuration.archive.max.storage".to_string(), + storage_quantity_to_nifi( + MemoryQuantity::try_from(capacity).context(CalculateStorageQuotaSnafu { + repo: NifiRepository::Flowfile, + })? * STORAGE_FLOW_ARCHIVE_UTILIZATION_FACTOR, + ), + ); + } + properties.insert( + "nifi.flow.configuration.archive.max.count".to_string(), + "".to_string(), + ); + properties.insert( + "nifi.flowcontroller.autoResumeState".to_string(), + "true".to_string(), + ); + properties.insert( + "nifi.flowcontroller.graceful.shutdown.period".to_string(), + "10 sec".to_string(), + ); + properties.insert( + "nifi.flowservice.writedelay.interval".to_string(), + "500 ms".to_string(), + ); + properties.insert( + "nifi.administrative.yield.duration".to_string(), + "30 sec".to_string(), + ); + + properties.insert( + "nifi.authorizer.configuration.file".to_string(), + "/stackable/nifi/conf/authorizers.xml".to_string(), + ); + properties.insert( + "nifi.login.identity.provider.configuration.file".to_string(), + "/stackable/nifi/conf/login-identity-providers.xml".to_string(), + ); + properties.insert( + "nifi.templates.directory".to_string(), + "./conf/templates".to_string(), + ); + properties.insert("nifi.ui.banner.text".to_string(), "".to_string()); + properties.insert( + "nifi.ui.autorefresh.interval".to_string(), + "30 sec".to_string(), + ); + properties.insert( + "nifi.nar.library.directory".to_string(), + "./lib".to_string(), + ); + properties.insert( + "nifi.nar.library.autoload.directory".to_string(), + "./extensions".to_string(), + ); + properties.insert( + "nifi.nar.working.directory".to_string(), + "./work/nar/".to_string(), + ); + properties.insert( + "nifi.documentation.working.directory".to_string(), + "./work/docs/components".to_string(), + ); + + //################### + // State Management # + //################### + properties.insert( + "nifi.state.management.configuration.file".to_string(), + "./conf/state-management.xml".to_string(), + ); + // The ID of the local state provider + properties.insert( + "nifi.state.management.provider.local".to_string(), + "local-provider".to_string(), + ); + // The ID of the cluster-wide state provider. This will be ignored if NiFi is not clustered but must be populated if running in a cluster. + properties.insert( + "nifi.state.management.provider.cluster".to_string(), + match cluster.cluster_config.clustering_backend { + v1alpha1::NifiClusteringBackend::ZooKeeper { .. } => "zk-provider".to_string(), + v1alpha1::NifiClusteringBackend::Kubernetes { .. } => "kubernetes-provider".to_string(), + }, + ); + // Specifies whether or not this instance of NiFi should run an embedded ZooKeeper server + properties.insert( + "nifi.state.management.embedded.zookeeper.start".to_string(), + "false".to_string(), + ); + + // H2 Settings + properties.insert( + "nifi.database.directory".to_string(), + NifiRepository::Database.mount_path(), + ); + properties.insert( + "nifi.h2.url.append".to_string(), + ";LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE".to_string(), + ); + + // FlowFile Repository + properties.insert( + "nifi.flowfile.repository.implementation".to_string(), + "org.apache.nifi.controller.repository.WriteAheadFlowFileRepository".to_string(), + ); + properties.insert( + "nifi.flowfile.repository.wal.implementation".to_string(), + "org.apache.nifi.wali.SequentialAccessWriteAheadLog".to_string(), + ); + properties.insert( + "nifi.flowfile.repository.directory".to_string(), + NifiRepository::Flowfile.mount_path(), + ); + properties.insert( + "nifi.flowfile.repository.checkpoint.interval".to_string(), + "20 secs".to_string(), + ); + properties.insert( + "nifi.flowfile.repository.always.sync".to_string(), + "false".to_string(), + ); + properties.insert( + "nifi.flowfile.repository.retain.orphaned.flowfiles".to_string(), + "true".to_string(), + ); + + properties.insert( + "nifi.swap.manager.implementation".to_string(), + "org.apache.nifi.controller.FileSystemSwapManager".to_string(), + ); + properties.insert("nifi.queue.swap.threshold".to_string(), "20000".to_string()); + + // Content Repository + properties.insert( + "nifi.content.repository.implementation".to_string(), + "org.apache.nifi.controller.repository.FileSystemRepository".to_string(), + ); + properties.insert( + "nifi.content.claim.max.appendable.size".to_string(), + "1 MB".to_string(), + ); + properties.insert( + "nifi.content.repository.directory.default".to_string(), + NifiRepository::Content.mount_path(), + ); + // Cap archived content age so the archive directory stays bounded in + // file count. NiFi treats empty as Long.MAX_VALUE, leaving size-based + // purge as the only trigger; that lets the archive grow to whatever + // half the PVC holds, and the startup directory scan in + // FileSystemRepository.initializeRepository scales with file count. + // 3 days covers a Friday-incident-investigated-Monday window for + // content replay; users with longer requirements can extend via + // configOverrides. The percentage-based threshold below acts as a + // safety net if write rate outpaces time-based purge. + // Also see https://github.com/stackabletech/nifi-operator/issues/354 + properties.insert( + "nifi.content.repository.archive.max.retention.period".to_string(), + "3 days".to_string(), + ); + properties.insert( + "nifi.content.repository.archive.max.usage.percentage".to_string(), + format!("{}%", STORAGE_CONTENT_ARCHIVE_UTILIZATION_FACTOR * 100.0), + ); + properties.insert( + "nifi.content.repository.archive.enabled".to_string(), + "true".to_string(), + ); + properties.insert( + "nifi.content.repository.always.sync".to_string(), + "false".to_string(), + ); + properties.insert( + "nifi.content.viewer.url".to_string(), + "../nifi-content-viewer/".to_string(), + ); + + // Provenance Repository Properties + properties.insert( + "nifi.provenance.repository.implementation".to_string(), + "org.apache.nifi.provenance.WriteAheadProvenanceRepository".to_string(), + ); + + // Persistent Provenance Repository Properties + properties.insert( + "nifi.provenance.repository.directory.default".to_string(), + NifiRepository::Provenance.mount_path(), + ); + properties.insert( + "nifi.provenance.repository.max.storage.time".to_string(), + "".to_string(), + ); + if let Some(capacity) = resource_config.storage.provenance_repo.capacity.as_ref() { + properties.insert( + "nifi.provenance.repository.max.storage.size".to_string(), + storage_quantity_to_nifi( + MemoryQuantity::try_from(capacity).context(CalculateStorageQuotaSnafu { + repo: NifiRepository::Provenance, + })? * STORAGE_PROVENANCE_UTILIZATION_FACTOR, + ), + ); + } + properties.insert( + "nifi.provenance.repository.rollover.time".to_string(), + "10 mins".to_string(), + ); + properties.insert( + "nifi.provenance.repository.rollover.size".to_string(), + "100 MB".to_string(), + ); + properties.insert( + "nifi.provenance.repository.query.threads".to_string(), + "2".to_string(), + ); + properties.insert( + "nifi.provenance.repository.index.threads".to_string(), + "2".to_string(), + ); + properties.insert( + "nifi.provenance.repository.compress.on.rollover".to_string(), + "true".to_string(), + ); + properties.insert( + "nifi.provenance.repository.always.sync".to_string(), + "false".to_string(), + ); + // Comma-separated list of fields. Fields that are not indexed will not be searchable. Valid fields are: + // EventType, FlowFileUUID, Filename, TransitURI, ProcessorID, AlternateIdentifierURI, Relationship, Details + properties.insert( + "nifi.provenance.repository.indexed.fields".to_string(), + "EventType, FlowFileUUID, Filename, ProcessorID, Relationship".to_string(), + ); + // FlowFile Attributes that should be indexed and made searchable. Some examples to consider are filename, uuid, mime.type + properties.insert( + "nifi.provenance.repository.indexed.attributes".to_string(), + "".to_string(), + ); + // Large values for the shard size will result in more Java heap usage when searching the Provenance Repository + // but should provide better performance + properties.insert( + "nifi.provenance.repository.index.shard.size".to_string(), + "500 MB".to_string(), + ); + // Indicates the maximum length that a FlowFile attribute can be when retrieving a Provenance Event from + // the repository. If the length of any attribute exceeds this value, it will be truncated when the event is retrieved. + properties.insert( + "nifi.provenance.repository.max.attribute.length".to_string(), + "65536".to_string(), + ); + properties.insert( + "nifi.provenance.repository.concurrent.merge.threads".to_string(), + "2".to_string(), + ); + + // Volatile Provenance Repository Properties + properties.insert( + "nifi.provenance.repository.buffer.size".to_string(), + "100000".to_string(), + ); + + // Component Status Repository + properties.insert( + "nifi.components.status.repository.implementation".to_string(), + "org.apache.nifi.controller.status.history.VolatileComponentStatusRepository".to_string(), + ); + properties.insert( + "nifi.components.status.repository.buffer.size".to_string(), + "1440".to_string(), + ); + properties.insert( + "nifi.components.status.snapshot.frequency".to_string(), + "1 min".to_string(), + ); + + // QuestDB Status History Repository Properties + properties.insert( + "nifi.status.repository.questdb.persist.node.days".to_string(), + "14".to_string(), + ); + properties.insert( + "nifi.status.repository.questdb.persist.component.days".to_string(), + "3".to_string(), + ); + properties.insert( + "nifi.status.repository.questdb.persist.location".to_string(), + "./status_repository".to_string(), + ); + + //############################################# + properties.insert( + "nifi.web.https.host".to_string(), + "${env:NODE_ADDRESS}".to_string(), + ); + properties.insert("nifi.web.https.port".to_string(), HTTPS_PORT.to_string()); + properties.insert( + "nifi.web.https.network.interface.default".to_string(), + "".to_string(), + ); + // Specifically listen on eth0 and lo interfaces. + // Listening on lo allows k8s port-forward to work. + // Once we listen on lo, we need to explicitly listen on eth0 so the server can be exposed (including health probes). + // NOTE: We assume "eth0" is always the external interface in containers launched in Kubernetes. + // It is possible that some container runtime will name it differently, but we haven't yet observed that. + properties.insert( + "nifi.web.https.network.interface.eth0".to_string(), + "eth0".to_string(), + ); + properties.insert( + "nifi.web.https.network.interface.lo".to_string(), + "lo".to_string(), + ); + //############################################# + properties.insert( + "nifi.web.jetty.working.directory".to_string(), + "./work/jetty".to_string(), + ); + properties.insert("nifi.web.jetty.threads".to_string(), "200".to_string()); + properties.insert("nifi.web.max.header.size".to_string(), "16 KB".to_string()); + properties.insert("nifi.web.proxy.context.path".to_string(), "".to_string()); + properties.insert("nifi.web.proxy.host".to_string(), proxy_hosts.to_string()); + + properties.insert( + "nifi.sensitive.props.key".to_string(), + "${file:UTF-8:/stackable/sensitiveproperty/nifiSensitivePropsKey}".to_string(), + ); + properties.insert( + "nifi.sensitive.props.key.protected".to_string(), + "".to_string(), + ); + + // The algorithm has already been validated in the validate step (check_for_nifi_version). + properties.insert( + "nifi.sensitive.props.algorithm".to_string(), + cluster + .cluster_config + .sensitive_properties_algorithm + .to_string(), + ); + + // key and trust store + // these properties are ok to hard code here, because the cannot be configured and are + // generated with fixed values in the init container + properties.insert( + "nifi.security.keystore".to_string(), + format!( + "{keystore_path}/keystore.p12", + keystore_path = STACKABLE_SERVER_TLS_DIR + ), + ); + properties.insert( + "nifi.security.keystoreType".to_string(), + "PKCS12".to_string(), + ); + properties.insert( + "nifi.security.keystorePasswd".to_string(), + STACKABLE_TLS_STORE_PASSWORD.to_string(), + ); + properties.insert( + "nifi.security.truststore".to_string(), + format!( + "{keystore_path}/truststore.p12", + keystore_path = STACKABLE_SERVER_TLS_DIR + ), + ); + properties.insert( + "nifi.security.truststoreType".to_string(), + "PKCS12".to_string(), + ); + properties.insert( + "nifi.security.truststorePasswd".to_string(), + STACKABLE_TLS_STORE_PASSWORD.to_string(), + ); + properties.insert( + "nifi.security.user.login.identity.provider".to_string(), + "login-identity-provider".to_string(), + ); + properties.insert( + "nifi.security.user.authorizer".to_string(), + "authorizer".to_string(), + ); + properties.insert( + "nifi.security.allow.anonymous.authentication".to_string(), + "false".to_string(), + ); + properties.insert( + "nifi.cluster.protocol.is.secure".to_string(), + "true".to_string(), + ); + + if let NifiAuthenticationConfig::Oidc { provider, oidc, .. } = auth_config { + add_oidc_config_to_properties(provider, oidc, &mut properties) + .context(GenerateOidcConfigSnafu)?; + }; + + // cluster node properties (only configure for cluster nodes) + properties.insert("nifi.cluster.is.node".to_string(), "true".to_string()); + properties.insert( + "nifi.cluster.node.address".to_string(), + "${env:NODE_ADDRESS}".to_string(), + ); + properties.insert( + "nifi.cluster.node.protocol.port".to_string(), + crate::crd::PROTOCOL_PORT.to_string(), + ); + properties.insert( + "nifi.cluster.flow.election.max.candidates".to_string(), + "".to_string(), + ); + + match cluster.cluster_config.clustering_backend { + v1alpha1::NifiClusteringBackend::ZooKeeper { .. } => { + properties.insert( + "nifi.cluster.leader.election.implementation".to_string(), + "CuratorLeaderElectionManager".to_string(), + ); + + // this will be replaced via a container command script + properties.insert( + "nifi.zookeeper.connect.string".to_string(), + "${env:ZOOKEEPER_HOSTS}".to_string(), + ); + + // this will be replaced via a container command script + properties.insert( + "nifi.zookeeper.root.node".to_string(), + "${env:ZOOKEEPER_CHROOT}".to_string(), + ); + } + + v1alpha1::NifiClusteringBackend::Kubernetes {} => { + ensure!(!is_nifi_1, Nifi1RequiresZookeeperSnafu); + + properties.insert( + "nifi.cluster.leader.election.implementation".to_string(), + "KubernetesLeaderElectionManager".to_string(), + ); + + // this will be replaced via a container command script + properties.insert( + "nifi.cluster.leader.election.kubernetes.lease.prefix".to_string(), + "${env:STACKLET_NAME}".to_string(), + ); + } + } + + //#################### + // Custom components # + //#################### + // NiFi 1.x does not support Python components and the Python configuration below is just + // ignored. + + // The command used to launch Python. + // This property must be set to enable Python-based processors. + properties.insert("nifi.python.command".to_string(), "python3".to_string()); + + // The directory that contains the Python framework for communicating between the Python and + // Java processes. + properties.insert( + "nifi.python.framework.source.directory".to_string(), + "/stackable/nifi/python/framework/".to_string(), + ); + + // The working directory where NiFi should store artifacts; + // This property defaults to ./work/python but if you want to mount an emptyDir for the working + // directory then another directory has to be set to avoid ownership conflicts with ./work/nar. + properties.insert( + "nifi.python.working.directory".to_string(), + NIFI_PYTHON_WORKING_DIRECTORY.to_string(), + ); + + // The default directory that NiFi should look in to find custom Python-based components. + // This directory is mentioned in the documentation + // (docs/modules/nifi/pages/usage_guide/custom-components.adoc), so do not change it! + properties.insert( + "nifi.python.extensions.source.directory.default".to_string(), + "/stackable/nifi/python/extensions/".to_string(), + ); + + for (i, git_folder) in git_sync_resources + .git_content_folders_as_string() + .into_iter() + .enumerate() + { + // The directory that NiFi should look in to find custom Python-based components. + properties.insert( + format!("nifi.python.extensions.source.directory.{i}"), + git_folder.clone(), + ); + + // The directory that NiFi should look in to find custom Java-based components. + properties.insert(format!("nifi.nar.library.directory.{i}"), git_folder); + } + //########################## + + // override with config overrides + for (k, v) in super::resolved_overrides_for(rg, ConfigFileName::NifiProperties) { + properties.insert(k, v); + } + + Ok(format_properties(properties)) +} + +fn storage_quantity_to_nifi(quantity: MemoryQuantity) -> String { + format!( + "{}MB", + quantity + .scale_to(stackable_operator::memory::BinaryMultiple::Mebi) + .value + ) +} diff --git a/rust/operator-binary/src/controller/build/properties/security_properties.rs b/rust/operator-binary/src/controller/build/properties/security_properties.rs new file mode 100644 index 00000000..6643eb6c --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/security_properties.rs @@ -0,0 +1,70 @@ +//! Builder for `security.properties`. + +use std::collections::BTreeMap; + +use super::{ConfigFileName, writer}; +use crate::controller::validate::NifiRoleGroupConfig; + +pub fn build(rg: &NifiRoleGroupConfig) -> Result { + let mut props: BTreeMap> = BTreeMap::new(); + // Defaults previously injected by deploy/config-spec/properties.yaml: + props.insert( + "networkaddress.cache.ttl".to_string(), + Some("30".to_string()), + ); + props.insert( + "networkaddress.cache.negative.ttl".to_string(), + Some("0".to_string()), + ); + for (k, v) in super::resolved_overrides_for(rg, ConfigFileName::SecurityProperties) { + props.insert(k, Some(v)); + } + writer::to_java_properties_string(props.iter()) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use stackable_operator::config_overrides::KeyValueConfigOverrides; + + use super::*; + use crate::{ + controller::validate::NifiRoleGroupConfig, + crd::{NifiConfig, v1alpha1::NifiConfigOverrides}, + }; + + fn make_rg(overrides: Option>) -> NifiRoleGroupConfig { + use stackable_operator::role_utils::JavaCommonConfig; + NifiRoleGroupConfig { + replicas: 1, + config: NifiConfig::default(), + config_overrides: NifiConfigOverrides { + security_properties: overrides.map(|o| KeyValueConfigOverrides { overrides: o }), + ..Default::default() + }, + env_overrides: BTreeMap::new(), + cli_overrides: BTreeMap::new(), + pod_overrides: Default::default(), + product_specific_common_config: JavaCommonConfig::default(), + } + } + + #[test] + fn test_default_keys_present() { + let rg = make_rg(None); + let result = build(&rg).unwrap(); + assert!(result.contains("networkaddress.cache.ttl=30")); + assert!(result.contains("networkaddress.cache.negative.ttl=0")); + } + + #[test] + fn test_user_override_wins() { + let mut overrides = BTreeMap::new(); + overrides.insert("networkaddress.cache.ttl".to_string(), "60".to_string()); + let rg = make_rg(Some(overrides)); + let result = build(&rg).unwrap(); + assert!(result.contains("networkaddress.cache.ttl=60")); + assert!(!result.contains("networkaddress.cache.ttl=30")); + } +} diff --git a/rust/operator-binary/src/controller/build/properties/state_management_xml.rs b/rust/operator-binary/src/controller/build/properties/state_management_xml.rs new file mode 100644 index 00000000..53747c9d --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/state_management_xml.rs @@ -0,0 +1,68 @@ +//! Builder for `state-management.xml`. + +use crate::{config::NifiRepository, crd::v1alpha1::NifiClusteringBackend}; + +pub fn build(clustering_backend: &NifiClusteringBackend) -> String { + // Inert providers are ignored by NiFi itself, but templating still fails if they refer to invalid environment variables, + // so only include the actually used provider. + let cluster_provider = match clustering_backend { + NifiClusteringBackend::ZooKeeper { .. } => { + r#" + zk-provider + org.apache.nifi.controller.state.providers.zookeeper.ZooKeeperStateProvider + ${env:ZOOKEEPER_HOSTS} + ${env:ZOOKEEPER_CHROOT} + 10 seconds + Open + "# + } + NifiClusteringBackend::Kubernetes {} => { + r#" + kubernetes-provider + org.apache.nifi.kubernetes.state.provider.KubernetesConfigMapStateProvider + ${env:STACKLET_NAME} + "# + } + }; + format!( + r#" + + + local-provider + org.apache.nifi.controller.state.providers.local.WriteAheadLocalStateProvider + {local_state_path} + false + 16 + 2 mins + + {cluster_provider} + "#, + local_state_path = NifiRepository::State.mount_path(), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_build_state_management_xml_kubernetes() { + let xml = build(&NifiClusteringBackend::Kubernetes {}); + assert!(xml.contains("kubernetes-provider")); + assert!(xml.contains("KubernetesConfigMapStateProvider")); + assert!(xml.contains("${env:STACKLET_NAME}")); + assert!(!xml.contains("zk-provider")); + } + + #[test] + fn test_build_state_management_xml_zookeeper() { + let xml = build(&NifiClusteringBackend::ZooKeeper { + zookeeper_config_map_name: "my-zk".to_string(), + }); + assert!(xml.contains("zk-provider")); + assert!(xml.contains("ZooKeeperStateProvider")); + assert!(xml.contains("${env:ZOOKEEPER_HOSTS}")); + assert!(xml.contains("${env:ZOOKEEPER_CHROOT}")); + assert!(!xml.contains("kubernetes-provider")); + } +} diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index f90542c8..47a424f0 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -20,7 +20,10 @@ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ config::{self, validated_product_config}, controller::dereference::DereferencedObjects, - crd::{HTTPS_PORT, NifiConfig, NifiRole, v1alpha1}, + crd::{ + HTTPS_PORT, NifiConfig, NifiRole, sensitive_properties, + sensitive_properties::NifiSensitiveKeyAlgorithm, v1alpha1, + }, framework::role_utils::with_validated_config, reporting_task, security::{ @@ -59,6 +62,9 @@ pub enum Error { InvalidConfigFragment { source: stackable_operator::config::fragment::ValidationError, }, + + #[snafu(display("invalid sensitive properties algorithm"))] + InvalidSensitivePropertiesAlgorithm { source: sensitive_properties::Error }, } pub type NifiRoleGroupConfig = crate::framework::role_utils::RoleGroupConfig< @@ -89,6 +95,10 @@ pub struct ValidatedClusterConfig { pub authorization: ResolvedNifiAuthorizationConfig, /// Comma-separated NiFi proxy hosts, or `"*"` if `hostHeaderCheck.allowAll` is set. pub proxy_hosts: String, + /// The clustering backend (ZooKeeper or Kubernetes), copied from the spec. + pub clustering_backend: v1alpha1::NifiClusteringBackend, + /// The validated sensitive properties algorithm. + pub sensitive_properties_algorithm: NifiSensitiveKeyAlgorithm, } /// Validates the cluster spec and the dereferenced inputs. @@ -128,6 +138,17 @@ pub fn validate( let proxy_hosts = compute_proxy_hosts(nifi, cluster_info)?; + let sensitive_properties_algorithm = nifi + .spec + .cluster_config + .sensitive_properties + .algorithm + .clone() + .unwrap_or_default(); + sensitive_properties_algorithm + .check_for_nifi_version(&image.product_version) + .context(InvalidSensitivePropertiesAlgorithmSnafu)?; + Ok(ValidatedCluster { name: nifi.name_any(), image, @@ -136,6 +157,8 @@ pub fn validate( authentication: authentication_config, authorization: authorization_config, proxy_hosts, + clustering_backend: nifi.spec.cluster_config.clustering_backend.clone(), + sensitive_properties_algorithm, }, validated_role_config, }) diff --git a/rust/operator-binary/src/security/authentication.rs b/rust/operator-binary/src/security/authentication.rs index 44cd8ac5..11d6b6fc 100644 --- a/rust/operator-binary/src/security/authentication.rs +++ b/rust/operator-binary/src/security/authentication.rs @@ -17,9 +17,6 @@ pub const STACKABLE_ADMIN_USERNAME: &str = "admin"; const STACKABLE_USER_VOLUME_MOUNT_PATH: &str = "/stackable/users"; -pub const LOGIN_IDENTITY_PROVIDERS_XML_FILE_NAME: &str = "login-identity-providers.xml"; -pub const AUTHORIZERS_XML_FILE_NAME: &str = "authorizers.xml"; - pub const STACKABLE_SERVER_TLS_DIR: &str = "/stackable/server_tls"; pub const STACKABLE_TLS_STORE_PASSWORD: &str = "secret"; From 5d1b146c140a17a9e25e01056a7ec11bf535713b Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Jun 2026 21:52:28 +0200 Subject: [PATCH 06/39] refactor: move configmap build into controller/build/config_map.rs Co-Authored-By: Claude Opus 4.8 (1M context) --- rust/operator-binary/src/controller.rs | 150 ++--------------- rust/operator-binary/src/controller/build.rs | 1 + .../src/controller/build/config_map.rs | 156 ++++++++++++++++++ 3 files changed, 168 insertions(+), 139 deletions(-) create mode 100644 rust/operator-binary/src/controller/build/config_map.rs diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 099a4197..dc1bc55a 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -6,7 +6,6 @@ use std::{ sync::Arc, }; -use crate::controller::build::properties::writer::PropertiesWriterError; use const_format::concatcp; use indoc::formatdoc; use product_config::{ProductConfigManager, types::PropertyNameKind}; @@ -14,7 +13,6 @@ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{ self, - configmap::ConfigMapBuilder, meta::ObjectMetaBuilder, pod::{ PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, @@ -32,8 +30,8 @@ use stackable_operator::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec, StatefulSetUpdateStrategy}, core::v1::{ - ConfigMap, ConfigMapKeySelector, ConfigMapVolumeSource, EmptyDirVolumeSource, - EnvVar, EnvVarSource, ObjectFieldSelector, PersistentVolumeClaim, Probe, + ConfigMapKeySelector, ConfigMapVolumeSource, EmptyDirVolumeSource, EnvVar, + EnvVarSource, ObjectFieldSelector, PersistentVolumeClaim, Probe, SecretVolumeSource, TCPSocketAction, Volume, }, }, @@ -75,14 +73,7 @@ mod validate; use crate::{ OPERATOR_NAME, - config::{ - self, JVM_SECURITY_PROPERTIES_FILE, NIFI_CONFIG_DIRECTORY, NIFI_PYTHON_WORKING_DIRECTORY, - NifiRepository, - }, - controller::build::properties::{ - ConfigFileName, authorizers, bootstrap_conf, login_identity_providers, nifi_properties, - security_properties, state_management_xml, - }, + config::{NIFI_CONFIG_DIRECTORY, NIFI_PYTHON_WORKING_DIRECTORY, NifiRepository}, crd::{ APP_NAME, BALANCE_PORT, BALANCE_PORT_NAME, Container, HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT, METRICS_PORT_NAME, NifiConfig, NifiNodeRoleConfig, NifiRole, NifiRoleType, @@ -98,7 +89,6 @@ use crate::{ pdb::add_pdbs, upgrade::{self, ClusterVersionUpdateState}, }, - product_logging::extend_role_group_config_map, reporting_task::{build_maybe_reporting_task, build_reporting_task_service_name}, security::{ authentication::{ @@ -110,7 +100,6 @@ use crate::{ }, service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, }; -use validate::{NifiRoleGroupConfig, ValidatedCluster}; pub const NIFI_CONTROLLER_NAME: &str = "nificluster"; pub const NIFI_FULL_CONTROLLER_NAME: &str = concatcp!(NIFI_CONTROLLER_NAME, '.', OPERATOR_NAME); @@ -169,9 +158,9 @@ pub enum Error { rolegroup: RoleGroupRef, }, - #[snafu(display("failed to build ConfigMap for {}", rolegroup))] - BuildRoleGroupConfig { - source: stackable_operator::builder::configmap::Error, + #[snafu(display("failed to build rolegroup ConfigMap for {}", rolegroup))] + BuildRoleGroupConfigMap { + source: build::config_map::Error, rolegroup: RoleGroupRef, }, @@ -208,19 +197,6 @@ pub enum Error { #[snafu(display("Failed to find information about file [{}] in product config", kind))] ProductConfigKindNotSpecified { kind: String }, - #[snafu(display("Bootstrap configuration error"))] - BootstrapConfig { - #[snafu(source(from(config::Error, Box::new)))] - source: Box, - }, - - #[snafu(display("failed to prepare NiFi configuration for rolegroup {rolegroup}"))] - BuildProductConfig { - #[snafu(source(from(config::Error, Box::new)))] - source: Box, - rolegroup: RoleGroupRef, - }, - #[snafu(display("illegal container name: [{container_name}]"))] IllegalContainerName { source: stackable_operator::builder::pod::container::Error, @@ -236,12 +212,6 @@ pub enum Error { #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] VectorAggregatorConfigMapMissing, - #[snafu(display("failed to add the logging configuration to the ConfigMap [{cm_name}]"))] - InvalidLoggingConfig { - source: crate::product_logging::Error, - cm_name: String, - }, - #[snafu(display("failed to patch service account"))] ApplyServiceAccount { source: stackable_operator::cluster_resources::Error, @@ -257,20 +227,6 @@ pub enum Error { source: stackable_operator::commons::rbac::Error, }, - #[snafu(display( - "failed to serialize [{JVM_SECURITY_PROPERTIES_FILE}] for {}", - rolegroup - ))] - JvmSecurityProperties { - source: PropertiesWriterError, - rolegroup: String, - }, - - #[snafu(display("Invalid NiFi Authentication Configuration"))] - InvalidNifiAuthenticationConfig { - source: crate::security::authentication::Error, - }, - #[snafu(display("Invalid NiFi Authorization Configuration"))] InvalidNifiAuthorizationConfig { source: crate::security::authorization::Error, @@ -517,18 +473,18 @@ pub async fn reconcile_nifi( // The proxy hosts allow-list lets external users access NiFi via addresses we cannot // predict, so all of them are added to the setting. // For more information see - let rg_configmap = build_node_rolegroup_config_map( + let rg_configmap = build::config_map::build_rolegroup_config_map( nifi, &validated, rg, - resolved_product_image, role, &rolegroup, - rolegroup_config, - &merged_config, &git_sync_resources, + &role_group_service_recommended_labels, ) - .await?; + .context(BuildRoleGroupConfigMapSnafu { + rolegroup: rolegroup.clone(), + })?; let role_group = role.role_groups.get(&rolegroup.role_group); let replicas = @@ -697,90 +653,6 @@ pub async fn reconcile_nifi( Ok(Action::await_change()) } -/// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator -#[allow(clippy::too_many_arguments)] -async fn build_node_rolegroup_config_map( - nifi: &v1alpha1::NifiCluster, - cluster: &ValidatedCluster, - rg: &NifiRoleGroupConfig, - resolved_product_image: &ResolvedProductImage, - role: &NifiRoleType, - rolegroup: &RoleGroupRef, - _rolegroup_config: &HashMap>, - _merged_config: &NifiConfig, - git_sync_resources: &git_sync::v1alpha2::GitSyncResources, -) -> Result { - tracing::debug!("building rolegroup configmaps"); - - let mut cm_builder = ConfigMapBuilder::new(); - - cm_builder - .metadata( - ObjectMetaBuilder::new() - .name_and_namespace(nifi) - .name(rolegroup.object_name()) - .ownerreference_from_resource(nifi, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&build_recommended_labels( - nifi, - &resolved_product_image.app_version_label_value, - &rolegroup.role, - &rolegroup.role_group, - )) - .context(MetadataBuildSnafu)? - .build(), - ) - .add_data( - ConfigFileName::BootstrapConf.to_string(), - bootstrap_conf::build( - rg, - role, - &rolegroup.role_group, - Some(&cluster.cluster_config.authorization), - ) - .context(BootstrapConfigSnafu)?, - ) - .add_data( - ConfigFileName::NifiProperties.to_string(), - nifi_properties::build(cluster, rg, git_sync_resources).with_context(|_| { - BuildProductConfigSnafu { - rolegroup: rolegroup.clone(), - } - })?, - ) - .add_data( - ConfigFileName::StateManagementXml.to_string(), - state_management_xml::build(&cluster.cluster_config.clustering_backend), - ) - .add_data( - ConfigFileName::LoginIdentityProviders.to_string(), - login_identity_providers::build(cluster) - .context(InvalidNifiAuthenticationConfigSnafu)?, - ) - .add_data( - ConfigFileName::Authorizers.to_string(), - authorizers::build(cluster, nifi), - ) - .add_data( - ConfigFileName::SecurityProperties.to_string(), - security_properties::build(rg).with_context(|_| JvmSecurityPropertiesSnafu { - rolegroup: rolegroup.role_group.clone(), - })?, - ); - - extend_role_group_config_map(rolegroup, &rg.config.logging, &mut cm_builder).context( - InvalidLoggingConfigSnafu { - cm_name: rolegroup.object_name(), - }, - )?; - - cm_builder - .build() - .with_context(|_| BuildRoleGroupConfigSnafu { - rolegroup: rolegroup.clone(), - }) -} - const USERDATA_MOUNTPOINT: &str = "/stackable/userdata"; /// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. diff --git a/rust/operator-binary/src/controller/build.rs b/rust/operator-binary/src/controller/build.rs index bb77c4fc..ad7b4079 100644 --- a/rust/operator-binary/src/controller/build.rs +++ b/rust/operator-binary/src/controller/build.rs @@ -2,4 +2,5 @@ //! //! [`ValidatedCluster`]: crate::controller::validate::ValidatedCluster +pub mod config_map; pub mod properties; diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs new file mode 100644 index 00000000..1964c5fe --- /dev/null +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -0,0 +1,156 @@ +//! Build per-rolegroup `ConfigMap` for the NiFi cluster. + +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, + crd::git_sync, + k8s_openapi::api::core::v1::ConfigMap, + kvp::ObjectLabels, + role_utils::RoleGroupRef, +}; + +use crate::{ + controller::{ + build::properties::{ + ConfigFileName, authorizers, bootstrap_conf, login_identity_providers, nifi_properties, + security_properties, state_management_xml, + }, + validate::{NifiRoleGroupConfig, ValidatedCluster}, + }, + crd::{NifiRoleType, v1alpha1}, + product_logging::extend_role_group_config_map, +}; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("object is missing metadata to build owner reference"))] + ObjectMissingMetadataForOwnerRef { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to build metadata"))] + MetadataBuild { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to build bootstrap.conf"))] + BootstrapConfig { + #[snafu(source(from(crate::config::Error, Box::new)))] + source: Box, + }, + + #[snafu(display("failed to prepare NiFi configuration for rolegroup {rolegroup}"))] + BuildProductConfig { + #[snafu(source(from(crate::config::Error, Box::new)))] + source: Box, + rolegroup: RoleGroupRef, + }, + + #[snafu(display("failed to add the logging configuration to the ConfigMap [{cm_name}]"))] + InvalidLoggingConfig { + source: crate::product_logging::Error, + cm_name: String, + }, + + #[snafu(display("failed to build ConfigMap for {rolegroup}"))] + BuildRoleGroupConfig { + source: stackable_operator::builder::configmap::Error, + rolegroup: RoleGroupRef, + }, + + #[snafu(display("failed to serialize JVM security properties for {}", rolegroup))] + JvmSecurityProperties { + source: crate::controller::build::properties::writer::PropertiesWriterError, + rolegroup: String, + }, + + #[snafu(display("failed to build login-identity-providers configuration"))] + InvalidNifiAuthenticationConfig { + source: crate::security::authentication::Error, + }, +} + +type Result = std::result::Result; + +/// Build the rolegroup [`ConfigMap`] configuring the rolegroup based on the +/// resolved cluster configuration. +/// +/// The only use of `owner` is for the OwnerReference and `name_and_namespace`. +/// All other NiFi configuration is sourced from `cluster` or `rg`. +/// `recommended_labels` must be built by the caller (typically via `build_recommended_labels`). +pub fn build_rolegroup_config_map( + owner: &v1alpha1::NifiCluster, + cluster: &ValidatedCluster, + rg: &NifiRoleGroupConfig, + role: &NifiRoleType, + rolegroup: &RoleGroupRef, + git_sync_resources: &git_sync::v1alpha2::GitSyncResources, + recommended_labels: &ObjectLabels<'_, v1alpha1::NifiCluster>, +) -> Result { + tracing::debug!("building rolegroup ConfigMap"); + + let mut cm_builder = ConfigMapBuilder::new(); + + cm_builder + .metadata( + ObjectMetaBuilder::new() + .name_and_namespace(owner) + .name(rolegroup.object_name()) + .ownerreference_from_resource(owner, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(recommended_labels) + .context(MetadataBuildSnafu)? + .build(), + ) + .add_data( + ConfigFileName::BootstrapConf.to_string(), + bootstrap_conf::build( + rg, + role, + &rolegroup.role_group, + Some(&cluster.cluster_config.authorization), + ) + .context(BootstrapConfigSnafu)?, + ) + .add_data( + ConfigFileName::NifiProperties.to_string(), + nifi_properties::build(cluster, rg, git_sync_resources).with_context(|_| { + BuildProductConfigSnafu { + rolegroup: rolegroup.clone(), + } + })?, + ) + .add_data( + ConfigFileName::StateManagementXml.to_string(), + state_management_xml::build(&cluster.cluster_config.clustering_backend), + ) + .add_data( + ConfigFileName::LoginIdentityProviders.to_string(), + login_identity_providers::build(cluster) + .context(InvalidNifiAuthenticationConfigSnafu)?, + ) + .add_data( + ConfigFileName::Authorizers.to_string(), + // TODO: authorizers::build currently takes a raw &NifiCluster; once migrated + // to ValidatedCluster this `owner` arg can be removed. + authorizers::build(cluster, owner), + ) + .add_data( + ConfigFileName::SecurityProperties.to_string(), + security_properties::build(rg).with_context(|_| JvmSecurityPropertiesSnafu { + rolegroup: rolegroup.role_group.clone(), + })?, + ); + + extend_role_group_config_map(rolegroup, &rg.config.logging, &mut cm_builder).context( + InvalidLoggingConfigSnafu { + cm_name: rolegroup.object_name(), + }, + )?; + + cm_builder + .build() + .with_context(|_| BuildRoleGroupConfigSnafu { + rolegroup: rolegroup.clone(), + }) +} From baaba90a7829277f83a1dd210202bca6dcb455f3 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Jun 2026 22:09:45 +0200 Subject: [PATCH 07/39] refactor: source statefulset and git-sync env vars from ValidatedCluster Replace the product-config validated_role_config map as the env-var source for the rolegroup loop, git-sync, and statefulset builder with the typed NifiRoleGroupConfig.env_overrides from ValidatedCluster.role_group_configs. - Loop now iterates node_role_group_configs (typed BTreeMap) directly - git-sync gets env vars via env_vars_from_overrides(&rg.env_overrides) - build_node_rolegroup_statefulset takes &NifiRoleGroupConfig instead of &HashMap> - validated_role_config field is now populated but unread; annotated with #[allow(dead_code)] pending removal in Task 7 - Removes: nifi_node_config derivation, MissingRoleGroupConfig error, ProductConfigKindNotSpecified error, env_vars_from_rolegroup_config import, PropertyNameKind and HashMap imports from controller.rs Co-Authored-By: Claude Opus 4.8 (1M context) --- rust/operator-binary/src/controller.rs | 67 +++++++------------ .../src/controller/validate.rs | 5 +- 2 files changed, 26 insertions(+), 46 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index dc1bc55a..2fead5af 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -1,14 +1,10 @@ //! Ensures that `Pod`s are configured and running for each [`v1alpha1::NifiCluster`]. -use std::{ - borrow::Cow, - collections::{BTreeMap, HashMap}, - sync::Arc, -}; +use std::{collections::BTreeMap, sync::Arc}; use const_format::concatcp; use indoc::formatdoc; -use product_config::{ProductConfigManager, types::PropertyNameKind}; +use product_config::ProductConfigManager; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{ @@ -45,7 +41,6 @@ use stackable_operator::{ kvp::{Labels, ObjectLabels}, logging::controller::ReconcilerError, memory::{BinaryMultiple, MemoryQuantity}, - product_config_utils::env_vars_from_rolegroup_config, product_logging::{ self, framework::{ @@ -100,6 +95,7 @@ use crate::{ }, service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, }; +use validate::NifiRoleGroupConfig; pub const NIFI_CONTROLLER_NAME: &str = "nificluster"; pub const NIFI_FULL_CONTROLLER_NAME: &str = concatcp!(NIFI_CONTROLLER_NAME, '.', OPERATOR_NAME); @@ -194,9 +190,6 @@ pub enum Error { source: stackable_operator::builder::meta::Error, }, - #[snafu(display("Failed to find information about file [{}] in product config", kind))] - ProductConfigKindNotSpecified { kind: String }, - #[snafu(display("illegal container name: [{container_name}]"))] IllegalContainerName { source: stackable_operator::builder::pod::container::Error, @@ -298,9 +291,6 @@ pub enum Error { #[snafu(display("failed to build authorization configuration"))] AuthorizationConfiguration { source: authorization::Error }, - - #[snafu(display("missing role group config for rolegroup {rolegroup_name}"))] - MissingRoleGroupConfig { rolegroup_name: String }, } type Result = std::result::Result; @@ -342,7 +332,6 @@ pub async fn reconcile_nifi( let resolved_product_image = &validated.image; let authentication_config = &validated.cluster_config.authentication; let authorization_config = &validated.cluster_config.authorization; - let validated_config = &validated.validated_role_config; tracing::info!("Checking for sensitive key configuration"); check_or_generate_sensitive_key(client, nifi) @@ -387,11 +376,6 @@ pub async fn reconcile_nifi( ) .context(CreateClusterResourcesSnafu)?; - let nifi_node_config = validated_config - .get(&NifiRole::Node.to_string()) - .map(Cow::Borrowed) - .unwrap_or_default(); - if let NifiAuthenticationConfig::Oidc { .. } = authentication_config { check_or_generate_oidc_admin_password(client, nifi) .await @@ -420,7 +404,11 @@ pub async fn reconcile_nifi( let mut ss_cond_builder = StatefulSetConditionBuilder::default(); let nifi_role = NifiRole::Node; - for (rolegroup_name, rolegroup_config) in nifi_node_config.iter() { + let node_role_group_configs = validated + .role_group_configs + .get(&NifiRole::Node) + .context(NoNodesDefinedSnafu)?; + for (rolegroup_name, rg) in node_role_group_configs.iter() { let rg_span = tracing::info_span!("rolegroup_span", rolegroup = rolegroup_name.as_str()); async { let rolegroup = nifi.node_rolegroup_ref(rolegroup_name); @@ -431,18 +419,10 @@ pub async fn reconcile_nifi( .merged_config(&NifiRole::Node, rolegroup_name) .context(FailedToResolveConfigSnafu)?; - let rg = validated - .role_group_configs - .get(&NifiRole::Node) - .and_then(|g| g.get(rolegroup_name)) - .context(MissingRoleGroupConfigSnafu { - rolegroup_name: rolegroup_name.clone(), - })?; - let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new( &nifi.spec.cluster_config.custom_components_git_sync, resolved_product_image, - &env_vars_from_rolegroup_config(rolegroup_config), + &env_vars_from_overrides(&rg.env_overrides), &[], LOG_VOLUME_NAME, &merged_config.logging.for_container(&Container::GitSync), @@ -500,7 +480,7 @@ pub async fn reconcile_nifi( &client.kubernetes_cluster_info, &rolegroup, role, - rolegroup_config, + rg, &merged_config, authentication_config, authorization_config, @@ -655,6 +635,18 @@ pub async fn reconcile_nifi( const USERDATA_MOUNTPOINT: &str = "/stackable/userdata"; +/// Build a `Vec` from a plain `BTreeMap` of env overrides. +fn env_vars_from_overrides(env_overrides: &BTreeMap) -> Vec { + env_overrides + .iter() + .map(|(name, value)| EnvVar { + name: name.clone(), + value: Some(value.clone()), + ..EnvVar::default() + }) + .collect() +} + /// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. /// /// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the @@ -666,7 +658,7 @@ async fn build_node_rolegroup_statefulset( cluster_info: &KubernetesClusterInfo, rolegroup_ref: &RoleGroupRef, role: &NifiRoleType, - rolegroup_config: &HashMap>, + rg: &NifiRoleGroupConfig, merged_config: &NifiConfig, authentication_config: &NifiAuthenticationConfig, authorization_config: &ResolvedNifiAuthorizationConfig, @@ -679,18 +671,7 @@ async fn build_node_rolegroup_statefulset( let role_group = role.role_groups.get(&rolegroup_ref.role_group); // get env vars and env overrides - let mut env_vars: Vec = rolegroup_config - .get(&PropertyNameKind::Env) - .with_context(|| ProductConfigKindNotSpecifiedSnafu { - kind: "ENV".to_string(), - })? - .iter() - .map(|(k, v)| EnvVar { - name: k.clone(), - value: Some(v.clone()), - ..EnvVar::default() - }) - .collect(); + let mut env_vars: Vec = env_vars_from_overrides(&rg.env_overrides); // we need the POD_NAME env var to overwrite `nifi.cluster.node.address` later env_vars.push(EnvVar { diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 47a424f0..c1533e46 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -82,11 +82,10 @@ pub struct ValidatedCluster { #[allow(dead_code)] pub name: String, pub image: ResolvedProductImage, - // Not yet consumed — Tasks 4-6 will use this to replace the product-config pipeline. - #[allow(dead_code)] pub role_group_configs: BTreeMap>, pub cluster_config: ValidatedClusterConfig, - // Temporary: retained until a later task migrates the configmap builder off product-config. + // Populated but no longer read — removed in Task 7 along with the product-config dependency. + #[allow(dead_code)] // removed in Task 7 along with the product-config dependency pub validated_role_config: ValidatedRoleConfigByPropertyKind, } From f4fb9eb5923a8f9bf33fcb7bfef6d6de76c75bd7 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Jun 2026 22:31:04 +0200 Subject: [PATCH 08/39] chore: remove product-config dependency Drop the `product-config` crate from workspace and operator-binary Cargo.toml, delete the now-unused `validated_role_config` field and `validated_product_config()` pipeline, remove ProductConfigManager from Ctx and main.rs, empty both properties.yaml files to their stub form, and apply two small nits (use `nifi_role` binding consistently, drop stale env-var comment). Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 1 - Cargo.toml | 1 - deploy/config-spec/properties.yaml | 41 +---------- .../nifi-operator/configs/properties.yaml | 41 +---------- rust/operator-binary/Cargo.toml | 1 - rust/operator-binary/src/config/mod.rs | 71 ++----------------- rust/operator-binary/src/controller.rs | 8 +-- .../src/controller/validate.rs | 22 ------ rust/operator-binary/src/crd/mod.rs | 30 -------- rust/operator-binary/src/main.rs | 8 +-- 10 files changed, 11 insertions(+), 213 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7d7f04c7..123bd3ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3017,7 +3017,6 @@ dependencies = [ "indoc", "java-properties", "pin-project", - "product-config", "rand 0.10.1", "rstest", "semver", diff --git a/Cargo.toml b/Cargo.toml index f14b7085..ab7f24b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,6 @@ edition = "2021" repository = "https://github.com/stackabletech/nifi-operator" [workspace.dependencies] -product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.8.0" } stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.111.0", features = ["webhook"] } anyhow = "1.0" diff --git a/deploy/config-spec/properties.yaml b/deploy/config-spec/properties.yaml index 1fcb04f3..9bd8c3b2 100644 --- a/deploy/config-spec/properties.yaml +++ b/deploy/config-spec/properties.yaml @@ -1,42 +1,5 @@ +--- version: 0.1.0 spec: units: [] - -properties: - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "node" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "node" - required: true - asOfVersion: "0.0.0" - comment: "TTL for host names that cannot be resolved." - description: "TTL for host names that cannot be resolved." +properties: [] diff --git a/deploy/helm/nifi-operator/configs/properties.yaml b/deploy/helm/nifi-operator/configs/properties.yaml index 1fcb04f3..9bd8c3b2 100644 --- a/deploy/helm/nifi-operator/configs/properties.yaml +++ b/deploy/helm/nifi-operator/configs/properties.yaml @@ -1,42 +1,5 @@ +--- version: 0.1.0 spec: units: [] - -properties: - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "node" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "node" - required: true - asOfVersion: "0.0.0" - comment: "TTL for host names that cannot be resolved." - description: "TTL for host names that cannot be resolved." +properties: [] diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index cd7fc147..2539a0d4 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -9,7 +9,6 @@ repository.workspace = true publish = false [dependencies] -product-config.workspace = true stackable-operator.workspace = true anyhow.workspace = true diff --git a/rust/operator-binary/src/config/mod.rs b/rust/operator-binary/src/config/mod.rs index cff95f96..78e6c47f 100644 --- a/rust/operator-binary/src/config/mod.rs +++ b/rust/operator-binary/src/config/mod.rs @@ -1,20 +1,9 @@ -use std::{ - collections::{BTreeMap, HashMap}, - fmt::Write, -}; - -use product_config::{ProductConfigManager, types::PropertyNameKind}; -use snafu::{ResultExt, Snafu}; -use stackable_operator::product_config_utils::{ - ValidatedRoleConfigByPropertyKind, transform_all_roles_to_config, - validate_all_roles_and_groups_config, -}; +use std::{collections::BTreeMap, fmt::Write}; + +use snafu::Snafu; use strum::{Display, EnumIter}; -use crate::{ - crd::{NifiRole, NifiRoleType, v1alpha1}, - security::oidc, -}; +use crate::security::oidc; pub mod jvm; @@ -24,7 +13,6 @@ pub const NIFI_PVC_STORAGE_DIRECTORY: &str = "/stackable/data"; pub const NIFI_BOOTSTRAP_CONF: &str = "bootstrap.conf"; pub const NIFI_PROPERTIES: &str = "nifi.properties"; -pub const NIFI_STATE_MANAGEMENT_XML: &str = "state-management.xml"; pub const JVM_SECURITY_PROPERTIES_FILE: &str = "security.properties"; #[derive(Debug, Display, EnumIter)] @@ -56,22 +44,12 @@ impl NifiRepository { #[derive(Snafu, Debug)] #[snafu(visibility(pub(crate)))] pub enum Error { - #[snafu(display("invalid product config"))] - InvalidProductConfig { - source: stackable_operator::product_config_utils::Error, - }, - #[snafu(display("invalid memory resource configuration - missing default or value in crd?"))] MissingMemoryResourceConfig, #[snafu(display("invalid JVM config"))] InvalidJVMConfig { source: jvm::Error }, - #[snafu(display("failed to transform product configs"))] - ProductConfigTransform { - source: stackable_operator::product_config_utils::Error, - }, - #[snafu(display("failed to calculate storage quota for {repo} repository"))] CalculateStorageQuota { source: stackable_operator::memory::Error, @@ -87,47 +65,6 @@ pub enum Error { Nifi1RequiresZookeeper, } -/// Defines all required roles and their required configuration. In this case we need three files: -/// `bootstrap.conf`, `nifi.properties` and `state-management.xml`. -/// -/// We do not require any env variables yet. We will however utilize them to change the -/// configuration directory - check for more detail. -/// -/// The roles and their configs are then validated and complemented by the product config. -/// -/// # Arguments -/// * `resource` - The NifiCluster containing the role definitions. -/// * `version` - The NifiCluster version. -/// * `product_config` - The product config to validate and complement the user config. -/// -pub fn validated_product_config( - resource: &v1alpha1::NifiCluster, - version: &str, - role: &NifiRoleType, - product_config: &ProductConfigManager, -) -> Result { - let mut roles = HashMap::new(); - roles.insert( - NifiRole::Node.to_string(), - ( - vec![ - PropertyNameKind::File(NIFI_BOOTSTRAP_CONF.to_string()), - PropertyNameKind::File(NIFI_PROPERTIES.to_string()), - PropertyNameKind::File(NIFI_STATE_MANAGEMENT_XML.to_string()), - PropertyNameKind::File(JVM_SECURITY_PROPERTIES_FILE.to_string()), - PropertyNameKind::Env, - ], - role.clone(), - ), - ); - - let role_config = - transform_all_roles_to_config(resource, &roles).context(ProductConfigTransformSnafu)?; - - validate_all_roles_and_groups_config(version, &role_config, product_config, false, false) - .context(InvalidProductConfigSnafu) -} - // TODO: Use crate like https://crates.io/crates/java-properties (currently does not work for Nifi // because of escapes), to have save handling of escapes etc. pub(crate) fn format_properties(properties: BTreeMap) -> String { diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 2fead5af..8c96eeac 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -4,7 +4,6 @@ use std::{collections::BTreeMap, sync::Arc}; use const_format::concatcp; use indoc::formatdoc; -use product_config::ProductConfigManager; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{ @@ -105,7 +104,6 @@ const LOG_VOLUME_NAME: &str = "log"; pub struct Ctx { pub client: Client, - pub product_config: ProductConfigManager, pub operator_environment: OperatorEnvironmentOptions, } @@ -324,7 +322,6 @@ pub async fn reconcile_nifi( nifi, &dereferenced_objects, &ctx.operator_environment, - &ctx.product_config, &client.kubernetes_cluster_info, ) .context(ValidateClusterSnafu)?; @@ -406,7 +403,7 @@ pub async fn reconcile_nifi( let nifi_role = NifiRole::Node; let node_role_group_configs = validated .role_group_configs - .get(&NifiRole::Node) + .get(&nifi_role) .context(NoNodesDefinedSnafu)?; for (rolegroup_name, rg) in node_role_group_configs.iter() { let rg_span = tracing::info_span!("rolegroup_span", rolegroup = rolegroup_name.as_str()); @@ -416,7 +413,7 @@ pub async fn reconcile_nifi( tracing::debug!("Processing rolegroup {}", rolegroup); let merged_config = nifi - .merged_config(&NifiRole::Node, rolegroup_name) + .merged_config(&nifi_role, rolegroup_name) .context(FailedToResolveConfigSnafu)?; let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new( @@ -670,7 +667,6 @@ async fn build_node_rolegroup_statefulset( tracing::debug!("Building statefulset"); let role_group = role.role_groups.get(&rolegroup_ref.role_group); - // get env vars and env overrides let mut env_vars: Vec = env_vars_from_overrides(&rg.env_overrides); // we need the POD_NAME env var to overwrite `nifi.cluster.node.address` later diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index c1533e46..4c592f8f 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -5,20 +5,17 @@ use std::collections::{BTreeMap, HashSet}; -use product_config::ProductConfigManager; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, commons::product_image_selection::{self, ResolvedProductImage}, kube::ResourceExt as _, - product_config_utils::ValidatedRoleConfigByPropertyKind, role_utils::JavaCommonConfig, utils::cluster_info::KubernetesClusterInfo, }; use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ - config::{self, validated_product_config}, controller::dereference::DereferencedObjects, crd::{ HTTPS_PORT, NifiConfig, NifiRole, sensitive_properties, @@ -47,12 +44,6 @@ pub enum Error { #[snafu(display("invalid NiFi authentication configuration"))] InvalidAuthenticationConfig { source: authentication::Error }, - #[snafu(display("failed to load product config"))] - ProductConfigLoadFailed { - #[snafu(source(from(config::Error, Box::new)))] - source: Box, - }, - #[snafu(display("failed to build reporting task service name"))] ReportingTask { source: crate::reporting_task::Error, @@ -84,9 +75,6 @@ pub struct ValidatedCluster { pub image: ResolvedProductImage, pub role_group_configs: BTreeMap>, pub cluster_config: ValidatedClusterConfig, - // Populated but no longer read — removed in Task 7 along with the product-config dependency. - #[allow(dead_code)] // removed in Task 7 along with the product-config dependency - pub validated_role_config: ValidatedRoleConfigByPropertyKind, } pub struct ValidatedClusterConfig { @@ -105,7 +93,6 @@ pub fn validate( nifi: &v1alpha1::NifiCluster, dereferenced_objects: &DereferencedObjects, operator_environment: &OperatorEnvironmentOptions, - product_config: &ProductConfigManager, cluster_info: &KubernetesClusterInfo, ) -> Result { let image = nifi @@ -127,14 +114,6 @@ pub fn validate( &dereferenced_objects.authorization, ); - let validated_role_config = validated_product_config( - nifi, - &image.product_version, - nifi.spec.nodes.as_ref().context(NoNodesDefinedSnafu)?, - product_config, - ) - .context(ProductConfigLoadFailedSnafu)?; - let proxy_hosts = compute_proxy_hosts(nifi, cluster_info)?; let sensitive_properties_algorithm = nifi @@ -159,7 +138,6 @@ pub fn validate( clustering_backend: nifi.spec.cluster_config.clustering_backend.clone(), sensitive_properties_algorithm, }, - validated_role_config, }) } diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index d2209b0f..1539d9be 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -33,7 +33,6 @@ use stackable_operator::{ }, kube::{CustomResource, ResourceExt, runtime::reflector::ObjectRef}, memory::MemoryQuantity, - product_config_utils::{self, Configuration}, product_logging::{self, spec::Logging}, role_utils::{GenericRoleConfig, JavaCommonConfig, Role, RoleGroupRef}, schemars::{self, JsonSchema}, @@ -507,35 +506,6 @@ impl NifiConfig { } } -impl Configuration for NifiConfigFragment { - type Configurable = v1alpha1::NifiCluster; - - fn compute_env( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, product_config_utils::Error> { - Ok(BTreeMap::new()) - } - - fn compute_cli( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, product_config_utils::Error> { - Ok(BTreeMap::new()) - } - - fn compute_files( - &self, - _resource: &Self::Configurable, - _role_name: &str, - _file: &str, - ) -> Result>, product_config_utils::Error> { - Ok(BTreeMap::new()) - } -} - #[derive(Clone, Debug, Default, JsonSchema, PartialEq, Fragment)] #[fragment_attrs( derive( diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 94d50ca9..c1eee997 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -73,7 +73,7 @@ async fn main() -> anyhow::Result<()> { Command::Run(RunArguments { operator_environment, watch_namespace, - product_config, + product_config: _, maintenance, common, }) => { @@ -120,11 +120,6 @@ async fn main() -> anyhow::Result<()> { .run(sigterm_watcher.handle()) .map_err(|err| anyhow!(err).context("failed to run webhook server")); - let product_config = product_config.load(&[ - "deploy/config-spec/properties.yaml", - "/etc/stackable/nifi-operator/config-spec/properties.yaml", - ])?; - let event_recorder = Arc::new(Recorder::new( client.as_kube_client(), Reporter { @@ -183,7 +178,6 @@ async fn main() -> anyhow::Result<()> { Arc::new(controller::Ctx { client: client.clone(), operator_environment, - product_config, }), ) // We can let the reporting happen in the background From b188b0677f059f929e53cadb87d6381aace1aaf7 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Jun 2026 22:45:18 +0200 Subject: [PATCH 09/39] test: add unit tests for build modules and validate Add a test_support helper in controller/build/properties.rs for constructing a minimal ValidatedCluster without Kubernetes API access (direct construction chosen over validate::validate() because the latter requires DereferencedAuthenticationClasses fetched from the API). Tests added: - nifi_properties: test_stable_keys_present (HTTPS port, cluster node, Kubernetes election impl, sensitive-props algorithm, proxy host wildcard) - nifi_properties: test_config_override_wins (configOverrides flow-through) - login_identity_providers: test_build_returns_ok_with_expected_structure - authorizers: test_build_returns_non_empty_xml_with_authorizers_root Drive-by rename: BuildProductConfig -> BuildNifiProperties (and its Snafu selector) in config_map.rs to remove the stale product-config name. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/controller/build/config_map.rs | 4 +- .../src/controller/build/properties.rs | 138 ++++++++++++++++++ .../build/properties/authorizers.rs | 32 ++++ .../properties/login_identity_providers.rs | 25 ++++ .../build/properties/nifi_properties.rs | 114 +++++++++++++++ 5 files changed, 311 insertions(+), 2 deletions(-) diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 1964c5fe..9fe7c757 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -40,7 +40,7 @@ pub enum Error { }, #[snafu(display("failed to prepare NiFi configuration for rolegroup {rolegroup}"))] - BuildProductConfig { + BuildNifiProperties { #[snafu(source(from(crate::config::Error, Box::new)))] source: Box, rolegroup: RoleGroupRef, @@ -115,7 +115,7 @@ pub fn build_rolegroup_config_map( .add_data( ConfigFileName::NifiProperties.to_string(), nifi_properties::build(cluster, rg, git_sync_resources).with_context(|_| { - BuildProductConfigSnafu { + BuildNifiPropertiesSnafu { rolegroup: rolegroup.clone(), } })?, diff --git a/rust/operator-binary/src/controller/build/properties.rs b/rust/operator-binary/src/controller/build/properties.rs index cf384e95..96d781f8 100644 --- a/rust/operator-binary/src/controller/build/properties.rs +++ b/rust/operator-binary/src/controller/build/properties.rs @@ -54,3 +54,141 @@ pub(crate) fn resolved_overrides_for( .get_key_value_overrides(&file.to_string()), ) } + +/// Test helpers for constructing a minimal [`ValidatedCluster`] and related types without +/// requiring Kubernetes API access. +/// +/// # Design choice — direct construction vs. `validate::validate()` +/// +/// NiFi's `validate::validate()` calls `NifiAuthenticationConfig::validate()`, which requires a +/// `DereferencedAuthenticationClasses` value populated with real `AuthenticationClass` objects +/// fetched from the Kubernetes API. Fabricating those objects in unit tests would require +/// pulling in serialized CRD YAML for operator-rs types that are not part of the nifi-operator +/// crate and would be fragile to upstream changes. +/// +/// Instead, we construct [`ValidatedCluster`] directly from its public fields. The +/// `NifiAuthenticationConfig::SingleUser` variant contains only an +/// `r#static::v1alpha1::AuthenticationProvider` (a small struct with a single `Secret` name), +/// which we can build without any Kubernetes interaction. For +/// `ResolvedNifiAuthorizationConfig` and `proxy_hosts` we pick the simplest variants. +/// +/// Role-group configs are built via `with_validated_config` on a parsed `NifiCluster`, +/// exactly as the existing `bootstrap_conf` tests do — the YAML fixture is minimal and +/// self-contained. +#[cfg(test)] +pub(crate) mod test_support { + use std::collections::BTreeMap; + + use stackable_operator::{ + commons::product_image_selection::ResolvedProductImage, + crd::authentication::r#static::v1alpha1::{ + AuthenticationProvider as StaticAuthProvider, UserCredentialsSecretRef, + }, + kube::ResourceExt as _, + kvp::LabelValue, + }; + + use crate::{ + controller::validate::{NifiRoleGroupConfig, ValidatedCluster, ValidatedClusterConfig}, + crd::{NifiConfig, NifiRole, v1alpha1}, + framework::role_utils::with_validated_config, + security::{ + authentication::NifiAuthenticationConfig, + authorization::ResolvedNifiAuthorizationConfig, + }, + }; + + /// A minimal NiFi cluster YAML. Mirrors the fixture used by bootstrap_conf tests, + /// stripped down to the mandatory fields only (NiFi 2.x, Kubernetes clustering backend, + /// SingleUser auth). + pub const MINIMAL_NIFI_YAML: &str = r#" + apiVersion: nifi.stackable.tech/v1alpha1 + kind: NifiCluster + metadata: + name: simple-nifi + namespace: default + spec: + image: + productVersion: 2.9.0 + clusterConfig: + authentication: + - authenticationClass: nifi-admin-credentials-simple + sensitiveProperties: + keySecret: simple-nifi-sensitive-property-key + autoGenerate: true + nodes: + roleGroups: + default: + replicas: 1 + "#; + + /// Build a minimal [`ValidatedCluster`] directly (without Kubernetes API access). + /// + /// The cluster uses: + /// - NiFi 2.9.0 (product version) + /// - `SingleUser` authentication + /// - `SingleUser` authorization (no OPA, no file-based) + /// - `allow_all = true` proxy hosts (i.e. `"*"`) + /// - Kubernetes clustering backend + /// - Default `NifiArgon2AesGcm256` sensitive-properties algorithm + pub fn minimal_validated_cluster() -> ValidatedCluster { + let nifi: v1alpha1::NifiCluster = + serde_yaml::from_str(MINIMAL_NIFI_YAML).expect("invalid test YAML"); + + let nifi_role = NifiRole::Node; + let role = nifi.spec.nodes.as_ref().unwrap(); + let default_config = NifiConfig::default_config(&nifi.name_any(), &nifi_role); + + let mut role_groups: BTreeMap = BTreeMap::new(); + for (rg_name, rg) in &role.role_groups { + let validated_rg = + with_validated_config::(rg, role, &default_config) + .expect("with_validated_config should succeed for minimal fixture"); + role_groups.insert(rg_name.clone(), validated_rg); + } + let mut role_group_configs = BTreeMap::new(); + role_group_configs.insert(NifiRole::Node, role_groups); + + let image = ResolvedProductImage { + product_version: "2.9.0".to_string(), + app_version_label_value: "2.9.0".parse::().unwrap(), + image: "oci.stackable.tech/sdp/nifi:2.9.0-stackable0.0.0-dev".to_string(), + image_pull_policy: "IfNotPresent".to_string(), + pull_secrets: None, + }; + + ValidatedCluster { + name: "simple-nifi".to_string(), + image, + role_group_configs, + cluster_config: ValidatedClusterConfig { + authentication: NifiAuthenticationConfig::SingleUser { + provider: StaticAuthProvider { + user_credentials_secret: UserCredentialsSecretRef { + name: "nifi-admin-credentials-simple".to_string(), + }, + }, + }, + authorization: ResolvedNifiAuthorizationConfig::SingleUser, + proxy_hosts: "*".to_string(), + clustering_backend: v1alpha1::NifiClusteringBackend::Kubernetes {}, + sensitive_properties_algorithm: Default::default(), // NifiArgon2AesGcm256 + }, + } + } + + /// Return the "default" role-group config from a [`ValidatedCluster`]. + pub fn default_rg(cluster: &ValidatedCluster) -> &NifiRoleGroupConfig { + cluster + .role_group_configs + .get(&NifiRole::Node) + .and_then(|rgs| rgs.get("default")) + .expect("minimal_validated_cluster must contain a 'default' role group") + } + + /// Build an empty [`GitSyncResources`] (no git-sync configured). + pub fn empty_git_sync_resources() + -> stackable_operator::crd::git_sync::v1alpha2::GitSyncResources { + stackable_operator::crd::git_sync::v1alpha2::GitSyncResources::default() + } +} diff --git a/rust/operator-binary/src/controller/build/properties/authorizers.rs b/rust/operator-binary/src/controller/build/properties/authorizers.rs index aae33f4d..48e8b563 100644 --- a/rust/operator-binary/src/controller/build/properties/authorizers.rs +++ b/rust/operator-binary/src/controller/build/properties/authorizers.rs @@ -9,3 +9,35 @@ pub fn build(cluster: &ValidatedCluster, nifi: &v1alpha1::NifiCluster) -> String .authorization .get_authorizers_config(nifi) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::controller::build::properties::test_support::{ + MINIMAL_NIFI_YAML, minimal_validated_cluster, + }; + + #[test] + fn test_build_returns_non_empty_xml_with_authorizers_root() { + let cluster = minimal_validated_cluster(); + let nifi: v1alpha1::NifiCluster = + serde_yaml::from_str(MINIMAL_NIFI_YAML).expect("invalid test YAML"); + + let xml = build(&cluster, &nifi); + + assert!(!xml.is_empty(), "authorizers.xml should not be empty"); + assert!( + xml.contains(""), + "output must contain root element" + ); + assert!( + xml.contains(""), + "output must contain closing tag" + ); + // For SingleUser authorization we expect the SingleUserAuthorizer class + assert!( + xml.contains("SingleUserAuthorizer"), + "expected SingleUserAuthorizer for SingleUser authorization" + ); + } +} diff --git a/rust/operator-binary/src/controller/build/properties/login_identity_providers.rs b/rust/operator-binary/src/controller/build/properties/login_identity_providers.rs index 02e6a12b..3993a67a 100644 --- a/rust/operator-binary/src/controller/build/properties/login_identity_providers.rs +++ b/rust/operator-binary/src/controller/build/properties/login_identity_providers.rs @@ -8,3 +8,28 @@ pub fn build(cluster: &ValidatedCluster) -> Result"), + "output must contain root element" + ); + assert!( + xml.contains("SingleUserLoginIdentityProvider"), + "expected SingleUserLoginIdentityProvider class for SingleUser auth" + ); + assert!( + xml.contains(""), + "output must contain closing tag" + ); + } +} diff --git a/rust/operator-binary/src/controller/build/properties/nifi_properties.rs b/rust/operator-binary/src/controller/build/properties/nifi_properties.rs index 7e1afbf5..a8b66f19 100644 --- a/rust/operator-binary/src/controller/build/properties/nifi_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/nifi_properties.rs @@ -584,3 +584,117 @@ fn storage_quantity_to_nifi(quantity: MemoryQuantity) -> String { .value ) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + controller::build::properties::test_support::{ + default_rg, empty_git_sync_resources, minimal_validated_cluster, + }, + crd::HTTPS_PORT, + }; + + /// Verify that core stable keys are present in the rendered nifi.properties with their + /// expected values. Assertions are on substrings — they do NOT assert the full file. + #[test] + fn test_stable_keys_present() { + let cluster = minimal_validated_cluster(); + let rg = default_rg(&cluster); + let git_sync = empty_git_sync_resources(); + + let props = build(&cluster, rg, &git_sync).expect("build should succeed"); + + // HTTPS port + assert!( + props.contains(&format!("nifi.web.https.port={HTTPS_PORT}")), + "expected nifi.web.https.port={HTTPS_PORT} in output, got:\n{props}" + ); + + // Clustering enabled + assert!( + props.contains("nifi.cluster.is.node=true"), + "expected nifi.cluster.is.node=true in output" + ); + + // Kubernetes clustering backend sets the Kubernetes election implementation + assert!( + props.contains( + "nifi.cluster.leader.election.implementation=KubernetesLeaderElectionManager" + ), + "expected KubernetesLeaderElectionManager in output" + ); + + // Sensitive-properties algorithm default (NifiArgon2AesGcm256) + assert!( + props.contains("nifi.sensitive.props.algorithm=NIFI_ARGON2_AES_GCM_256"), + "expected default algorithm NIFI_ARGON2_AES_GCM_256 in output" + ); + + // Proxy hosts wildcard from allow_all + assert!( + props.contains("nifi.web.proxy.host=*"), + "expected nifi.web.proxy.host=* in output" + ); + } + + /// Verify that a user configOverride for `nifi.properties` flows through to the output. + #[test] + fn test_config_override_wins() { + use stackable_operator::kube::ResourceExt as _; + + use crate::crd::{NifiConfig, NifiRole, v1alpha1}; + use crate::framework::role_utils::with_validated_config; + + let yaml = r#" + apiVersion: nifi.stackable.tech/v1alpha1 + kind: NifiCluster + metadata: + name: simple-nifi + namespace: default + spec: + image: + productVersion: 2.9.0 + clusterConfig: + authentication: + - authenticationClass: nifi-admin-credentials-simple + sensitiveProperties: + keySecret: simple-nifi-sensitive-property-key + autoGenerate: true + nodes: + roleGroups: + default: + replicas: 1 + configOverrides: + nifi.properties: + some.custom.key: some-custom-value + "#; + let nifi: v1alpha1::NifiCluster = serde_yaml::from_str(yaml).expect("invalid test YAML"); + let role = nifi.spec.nodes.as_ref().unwrap(); + let default_config = NifiConfig::default_config(&nifi.name_any(), &NifiRole::Node); + let rg = with_validated_config::( + role.role_groups.get("default").unwrap(), + role, + &default_config, + ) + .expect("with_validated_config should succeed"); + + // Build a cluster with this rg substituted in + let mut cluster = minimal_validated_cluster(); + cluster + .role_group_configs + .get_mut(&NifiRole::Node) + .unwrap() + .insert("default".to_string(), rg.clone()); + + let git_sync = empty_git_sync_resources(); + let props = build(&cluster, &rg, &git_sync).expect("build with override should succeed"); + + assert!( + props.contains("some.custom.key=some-custom-value"), + "expected user override some.custom.key=some-custom-value to appear in output" + ); + // The HTTPS port should still be present + assert!(props.contains(&format!("nifi.web.https.port={HTTPS_PORT}"))); + } +} From c91ba38b71e52c74b01094d3d0d035db0ecf9ffd Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Jun 2026 22:57:45 +0200 Subject: [PATCH 10/39] docs: fix vendored role_utils doc (BTreeMap, nifi) Co-Authored-By: Claude Opus 4.8 (1M context) --- rust/operator-binary/src/framework/role_utils.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/operator-binary/src/framework/role_utils.rs b/rust/operator-binary/src/framework/role_utils.rs index d426b7a2..ed5a04b8 100644 --- a/rust/operator-binary/src/framework/role_utils.rs +++ b/rust/operator-binary/src/framework/role_utils.rs @@ -2,7 +2,7 @@ //! `smooth-operator` branch, with simplifications appropriate for nifi-operator. //! //! Differences from upstream: -//! - `env_overrides` is `HashMap` instead of `EnvVarSet`. +//! - `env_overrides` is `BTreeMap` instead of `EnvVarSet`. //! - No `cli_overrides_to_vec` helper, `ResourceNames`, or service-account helpers. //! - The `CommonConfig` (a.k.a. `product_specific_common_config`) does NOT need to //! implement `Merge`. Upstream Trino uses `JavaCommonConfig`, which intentionally @@ -28,7 +28,7 @@ use stackable_operator::{ schemars::JsonSchema, }; -/// Trino-friendly view of a validated, merged `RoleGroup`. +/// NiFi-friendly view of a validated, merged `RoleGroup`. /// /// Mirrors `stackable_operator::v2::role_utils::RoleGroupConfig` on the /// `smooth-operator` branch, with `env_overrides: BTreeMap` From b3722937254b58c3933924773c9289623072d6f3 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Jun 2026 15:30:41 +0200 Subject: [PATCH 11/39] docs: remove product-config from cli and env --- .../reference/commandline-parameters.adoc | 13 ---------- .../reference/environment-variables.adoc | 26 ------------------- 2 files changed, 39 deletions(-) diff --git a/docs/modules/nifi/pages/reference/commandline-parameters.adoc b/docs/modules/nifi/pages/reference/commandline-parameters.adoc index a2dae146..65812f3b 100644 --- a/docs/modules/nifi/pages/reference/commandline-parameters.adoc +++ b/docs/modules/nifi/pages/reference/commandline-parameters.adoc @@ -2,19 +2,6 @@ This operator accepts the following command line parameters: -== product-config - -*Default value*: `/etc/stackable/nifi-operator/config-spec/properties.yaml` - -*Required*: false - -*Multiple values:* false - -[source] ----- -stackable-nifi-operator run --product-config /foo/bar/properties.yaml ----- - == watch-namespace *Default value*: All namespaces diff --git a/docs/modules/nifi/pages/reference/environment-variables.adoc b/docs/modules/nifi/pages/reference/environment-variables.adoc index 2bfaf328..181ec54e 100644 --- a/docs/modules/nifi/pages/reference/environment-variables.adoc +++ b/docs/modules/nifi/pages/reference/environment-variables.adoc @@ -33,32 +33,6 @@ docker run \ oci.stackable.tech/sdp/nifi-operator:0.0.0-dev ---- -== PRODUCT_CONFIG - -*Default value*: `/etc/stackable/nifi-operator/config-spec/properties.yaml` - -*Required*: false - -*Multiple values*: false - -[source] ----- -export PRODUCT_CONFIG=/foo/bar/properties.yaml -stackable-nifi-operator run ----- - -or via Docker: - ----- -docker run \ - --name nifi-operator \ - --network host \ - --env KUBECONFIG=/home/stackable/.kube/config \ - --env PRODUCT_CONFIG=/my/product/config.yaml \ - --mount type=bind,source="$HOME/.kube/config",target="/home/stackable/.kube/config" \ - oci.stackable.tech/sdp/nifi-operator:0.0.0-dev ----- - == WATCH_NAMESPACE *Default value*: All namespaces From 4c30f1b80e7adc16e23a62cee10efe465c6ba7f3 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Jun 2026 16:02:06 +0200 Subject: [PATCH 12/39] refactor: fix parameters in build configmap --- rust/operator-binary/src/controller.rs | 41 +++++++--------- .../src/controller/build/config_map.rs | 47 ++++++++++++++----- .../src/controller/build/properties.rs | 1 + .../src/controller/validate.rs | 46 ++++++++++++++++-- 4 files changed, 98 insertions(+), 37 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 8c96eeac..cfb9869b 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -65,6 +65,8 @@ mod build; mod dereference; mod validate; +use validate::NifiRoleGroupConfig; + use crate::{ OPERATOR_NAME, config::{NIFI_CONFIG_DIRECTORY, NIFI_PYTHON_WORKING_DIRECTORY, NifiRepository}, @@ -94,7 +96,6 @@ use crate::{ }, service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, }; -use validate::NifiRoleGroupConfig; pub const NIFI_CONTROLLER_NAME: &str = "nificluster"; pub const NIFI_FULL_CONTROLLER_NAME: &str = concatcp!(NIFI_CONTROLLER_NAME, '.', OPERATOR_NAME); @@ -197,8 +198,8 @@ pub enum Error { #[snafu(display("failed to resolve and merge config for role and role group"))] FailedToResolveConfig { source: crate::crd::Error }, - #[snafu(display("invalid git-sync specification"))] - InvalidGitSyncSpec { source: git_sync::v1alpha2::Error }, + #[snafu(display("missing git-sync resources for rolegroup [{rolegroup}]"))] + MissingGitSyncResources { rolegroup: String }, #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] VectorAggregatorConfigMapMissing, @@ -318,7 +319,7 @@ pub async fn reconcile_nifi( .context(DereferenceSnafu)?; // validate (no Kubernetes API calls required) - let validated = validate::validate( + let validated_cluster = validate::validate( nifi, &dereferenced_objects, &ctx.operator_environment, @@ -326,9 +327,9 @@ pub async fn reconcile_nifi( ) .context(ValidateClusterSnafu)?; - let resolved_product_image = &validated.image; - let authentication_config = &validated.cluster_config.authentication; - let authorization_config = &validated.cluster_config.authorization; + let resolved_product_image = &validated_cluster.image; + let authentication_config = &validated_cluster.cluster_config.authentication; + let authorization_config = &validated_cluster.cluster_config.authorization; tracing::info!("Checking for sensitive key configuration"); check_or_generate_sensitive_key(client, nifi) @@ -401,7 +402,7 @@ pub async fn reconcile_nifi( let mut ss_cond_builder = StatefulSetConditionBuilder::default(); let nifi_role = NifiRole::Node; - let node_role_group_configs = validated + let node_role_group_configs = validated_cluster .role_group_configs .get(&nifi_role) .context(NoNodesDefinedSnafu)?; @@ -416,15 +417,12 @@ pub async fn reconcile_nifi( .merged_config(&nifi_role, rolegroup_name) .context(FailedToResolveConfigSnafu)?; - let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new( - &nifi.spec.cluster_config.custom_components_git_sync, - resolved_product_image, - &env_vars_from_overrides(&rg.env_overrides), - &[], - LOG_VOLUME_NAME, - &merged_config.logging.for_container(&Container::GitSync), - ) - .context(InvalidGitSyncSpecSnafu)?; + let git_sync_resources = validated_cluster + .git_sync_resources + .get(rolegroup_name) + .context(MissingGitSyncResourcesSnafu { + rolegroup: rolegroup_name.clone(), + })?; let role_group_service_recommended_labels = build_recommended_labels( nifi, @@ -451,13 +449,10 @@ pub async fn reconcile_nifi( // predict, so all of them are added to the setting. // For more information see let rg_configmap = build::config_map::build_rolegroup_config_map( - nifi, - &validated, - rg, - role, + &validated_cluster, &rolegroup, - &git_sync_resources, &role_group_service_recommended_labels, + nifi, ) .context(BuildRoleGroupConfigMapSnafu { rolegroup: rolegroup.clone(), @@ -484,7 +479,7 @@ pub async fn reconcile_nifi( rolling_upgrade_supported, replicas, &rbac_sa.name_any(), - &git_sync_resources, + git_sync_resources, ) .await?; diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 9fe7c757..74659f7e 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -1,9 +1,8 @@ //! Build per-rolegroup `ConfigMap` for the NiFi cluster. -use snafu::{ResultExt, Snafu}; +use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, - crd::git_sync, k8s_openapi::api::core::v1::ConfigMap, kvp::ObjectLabels, role_utils::RoleGroupRef, @@ -15,9 +14,9 @@ use crate::{ ConfigFileName, authorizers, bootstrap_conf, login_identity_providers, nifi_properties, security_properties, state_management_xml, }, - validate::{NifiRoleGroupConfig, ValidatedCluster}, + validate::ValidatedCluster, }, - crd::{NifiRoleType, v1alpha1}, + crd::{NifiRole, v1alpha1}, product_logging::extend_role_group_config_map, }; @@ -68,6 +67,15 @@ pub enum Error { InvalidNifiAuthenticationConfig { source: crate::security::authentication::Error, }, + + #[snafu(display("object has no nodes defined"))] + NoNodesDefined, + + #[snafu(display("the cluster has no rolegroup [{role_group}] in role [{role}]"))] + MissingRoleGroup { role: String, role_group: String }, + + #[snafu(display("missing git-sync resources for rolegroup [{role_group}]"))] + MissingGitSyncResources { role_group: String }, } type Result = std::result::Result; @@ -75,20 +83,37 @@ type Result = std::result::Result; /// Build the rolegroup [`ConfigMap`] configuring the rolegroup based on the /// resolved cluster configuration. /// -/// The only use of `owner` is for the OwnerReference and `name_and_namespace`. -/// All other NiFi configuration is sourced from `cluster` or `rg`. -/// `recommended_labels` must be built by the caller (typically via `build_recommended_labels`). +/// All NiFi configuration is sourced from `cluster`. The only use of `owner` is for +/// the OwnerReference, `name_and_namespace`, and the raw role spec used for JVM +/// argument merging. `recommended_labels` must be built by the caller (typically via +/// `build_recommended_labels`). pub fn build_rolegroup_config_map( - owner: &v1alpha1::NifiCluster, cluster: &ValidatedCluster, - rg: &NifiRoleGroupConfig, - role: &NifiRoleType, rolegroup: &RoleGroupRef, - git_sync_resources: &git_sync::v1alpha2::GitSyncResources, recommended_labels: &ObjectLabels<'_, v1alpha1::NifiCluster>, + owner: &v1alpha1::NifiCluster, ) -> Result { tracing::debug!("building rolegroup ConfigMap"); + let rg = cluster + .role_group_configs + .get(&NifiRole::Node) + .and_then(|groups| groups.get(&rolegroup.role_group)) + .with_context(|| MissingRoleGroupSnafu { + role: NifiRole::Node.to_string(), + role_group: rolegroup.role_group.clone(), + })?; + + // The raw role spec is only needed for JVM argument merging in `bootstrap_conf`. + let role = owner.spec.nodes.as_ref().context(NoNodesDefinedSnafu)?; + + let git_sync_resources = cluster + .git_sync_resources + .get(&rolegroup.role_group) + .with_context(|| MissingGitSyncResourcesSnafu { + role_group: rolegroup.role_group.clone(), + })?; + let mut cm_builder = ConfigMapBuilder::new(); cm_builder diff --git a/rust/operator-binary/src/controller/build/properties.rs b/rust/operator-binary/src/controller/build/properties.rs index 96d781f8..6e8ecb1f 100644 --- a/rust/operator-binary/src/controller/build/properties.rs +++ b/rust/operator-binary/src/controller/build/properties.rs @@ -161,6 +161,7 @@ pub(crate) mod test_support { name: "simple-nifi".to_string(), image, role_group_configs, + git_sync_resources: Default::default(), cluster_config: ValidatedClusterConfig { authentication: NifiAuthenticationConfig::SingleUser { provider: StaticAuthProvider { diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 4c592f8f..7b0270d1 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -9,6 +9,7 @@ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, commons::product_image_selection::{self, ResolvedProductImage}, + crd::git_sync, kube::ResourceExt as _, role_utils::JavaCommonConfig, utils::cluster_info::KubernetesClusterInfo, @@ -16,9 +17,9 @@ use stackable_operator::{ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ - controller::dereference::DereferencedObjects, + controller::{LOG_VOLUME_NAME, dereference::DereferencedObjects, env_vars_from_overrides}, crd::{ - HTTPS_PORT, NifiConfig, NifiRole, sensitive_properties, + Container, HTTPS_PORT, NifiConfig, NifiRole, sensitive_properties, sensitive_properties::NifiSensitiveKeyAlgorithm, v1alpha1, }, framework::role_utils::with_validated_config, @@ -56,6 +57,9 @@ pub enum Error { #[snafu(display("invalid sensitive properties algorithm"))] InvalidSensitivePropertiesAlgorithm { source: sensitive_properties::Error }, + + #[snafu(display("invalid git-sync specification"))] + InvalidGitSyncSpec { source: git_sync::v1alpha2::Error }, } pub type NifiRoleGroupConfig = crate::framework::role_utils::RoleGroupConfig< @@ -74,6 +78,10 @@ pub struct ValidatedCluster { pub name: String, pub image: ResolvedProductImage, pub role_group_configs: BTreeMap>, + /// The git-sync resources (volumes, mounts, containers) for each Node rolegroup, + /// keyed by rolegroup name. Precomputed here so both the ConfigMap and StatefulSet + /// builders can source them from `ValidatedCluster`. + pub git_sync_resources: BTreeMap, pub cluster_config: ValidatedClusterConfig, } @@ -127,10 +135,14 @@ pub fn validate( .check_for_nifi_version(&image.product_version) .context(InvalidSensitivePropertiesAlgorithmSnafu)?; + let role_group_configs = build_role_group_configs(nifi)?; + let git_sync_resources = build_git_sync_resources(nifi, &image, &role_group_configs)?; + Ok(ValidatedCluster { name: nifi.name_any(), image, - role_group_configs: build_role_group_configs(nifi)?, + role_group_configs, + git_sync_resources, cluster_config: ValidatedClusterConfig { authentication: authentication_config, authorization: authorization_config, @@ -160,6 +172,34 @@ fn build_role_group_configs( Ok(role_group_configs) } +/// Builds the [`git_sync::v1alpha2::GitSyncResources`] for every Node rolegroup, keyed by +/// rolegroup name. The env vars and logging configuration differ per rolegroup, so the +/// resources are computed per rolegroup rather than once for the whole cluster. +fn build_git_sync_resources( + nifi: &v1alpha1::NifiCluster, + image: &ResolvedProductImage, + role_group_configs: &BTreeMap>, +) -> Result> { + let mut resources = BTreeMap::new(); + + if let Some(groups) = role_group_configs.get(&NifiRole::Node) { + for (rg_name, rg) in groups { + let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new( + &nifi.spec.cluster_config.custom_components_git_sync, + image, + &env_vars_from_overrides(&rg.env_overrides), + &[], + LOG_VOLUME_NAME, + &rg.config.logging.for_container(&Container::GitSync), + ) + .context(InvalidGitSyncSpecSnafu)?; + resources.insert(rg_name.clone(), git_sync_resources); + } + } + + Ok(resources) +} + fn compute_proxy_hosts( nifi: &v1alpha1::NifiCluster, cluster_info: &KubernetesClusterInfo, From 4adfbcad1b40674fdc7ab81012e8388e58268489 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Jun 2026 16:29:57 +0200 Subject: [PATCH 13/39] refactor: move product logging & lint --- .../usage_guide/exposing-processors/http.adoc | 2 +- .../src/controller/build/config_map.rs | 26 ++--- .../src/controller/build/properties.rs | 8 +- .../build/properties/bootstrap_conf.rs | 1 - .../controller/build/properties/logging.rs | 86 +++++++++++++++ .../build/properties/nifi_properties.rs | 6 +- .../build/properties/security_properties.rs | 4 +- rust/operator-binary/src/crd/mod.rs | 5 +- rust/operator-binary/src/framework.rs | 1 + .../build/properties => framework}/writer.rs | 0 rust/operator-binary/src/main.rs | 1 - rust/operator-binary/src/product_logging.rs | 103 ------------------ 12 files changed, 113 insertions(+), 130 deletions(-) create mode 100644 rust/operator-binary/src/controller/build/properties/logging.rs rename rust/operator-binary/src/{controller/build/properties => framework}/writer.rs (100%) delete mode 100644 rust/operator-binary/src/product_logging.rs diff --git a/docs/modules/nifi/pages/usage_guide/exposing-processors/http.adoc b/docs/modules/nifi/pages/usage_guide/exposing-processors/http.adoc index 0222b163..c17fe88f 100644 --- a/docs/modules/nifi/pages/usage_guide/exposing-processors/http.adoc +++ b/docs/modules/nifi/pages/usage_guide/exposing-processors/http.adoc @@ -68,7 +68,7 @@ spec: number: 8042 ---- -=== 3. Route +=== 3. Route The next step is to handle different kind of messages coming in, based on the HTTP path. diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 74659f7e..c5d4d926 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -5,19 +5,19 @@ use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, k8s_openapi::api::core::v1::ConfigMap, kvp::ObjectLabels, + product_logging::framework::VECTOR_CONFIG_FILE, role_utils::RoleGroupRef, }; use crate::{ controller::{ build::properties::{ - ConfigFileName, authorizers, bootstrap_conf, login_identity_providers, nifi_properties, - security_properties, state_management_xml, + ConfigFileName, authorizers, bootstrap_conf, logging, login_identity_providers, + nifi_properties, security_properties, state_management_xml, }, validate::ValidatedCluster, }, crd::{NifiRole, v1alpha1}, - product_logging::extend_role_group_config_map, }; #[derive(Debug, Snafu)] @@ -45,12 +45,6 @@ pub enum Error { rolegroup: RoleGroupRef, }, - #[snafu(display("failed to add the logging configuration to the ConfigMap [{cm_name}]"))] - InvalidLoggingConfig { - source: crate::product_logging::Error, - cm_name: String, - }, - #[snafu(display("failed to build ConfigMap for {rolegroup}"))] BuildRoleGroupConfig { source: stackable_operator::builder::configmap::Error, @@ -59,7 +53,7 @@ pub enum Error { #[snafu(display("failed to serialize JVM security properties for {}", rolegroup))] JvmSecurityProperties { - source: crate::controller::build::properties::writer::PropertiesWriterError, + source: crate::framework::writer::PropertiesWriterError, rolegroup: String, }, @@ -167,11 +161,13 @@ pub fn build_rolegroup_config_map( })?, ); - extend_role_group_config_map(rolegroup, &rg.config.logging, &mut cm_builder).context( - InvalidLoggingConfigSnafu { - cm_name: rolegroup.object_name(), - }, - )?; + if let Some(logback_config) = logging::build_logback_config(&rg.config.logging) { + cm_builder.add_data(ConfigFileName::Logback.to_string(), logback_config); + } + + if let Some(vector_config) = logging::build_vector_config(rolegroup, &rg.config.logging) { + cm_builder.add_data(VECTOR_CONFIG_FILE, vector_config); + } cm_builder .build() diff --git a/rust/operator-binary/src/controller/build/properties.rs b/rust/operator-binary/src/controller/build/properties.rs index 6e8ecb1f..81d8969b 100644 --- a/rust/operator-binary/src/controller/build/properties.rs +++ b/rust/operator-binary/src/controller/build/properties.rs @@ -1,8 +1,8 @@ //! Per-file builders for the NiFi rolegroup ConfigMap. //! //! Each `` module produces the rendered content for one NiFi config file. -//! The shared [`writer`] module serializes `.properties`/`.conf` key/value maps to -//! the Java-properties on-wire format. +//! The shared [`crate::framework::writer`] module serializes `.properties`/`.conf` +//! key/value maps to the Java-properties on-wire format. use std::collections::BTreeMap; @@ -12,11 +12,11 @@ use crate::controller::validate::NifiRoleGroupConfig; pub mod authorizers; pub mod bootstrap_conf; +pub mod logging; pub mod login_identity_providers; pub mod nifi_properties; pub mod security_properties; pub mod state_management_xml; -pub mod writer; /// The names of the files assembled into the NiFi rolegroup ConfigMap. #[derive(Clone, Copy, Debug, strum::Display)] @@ -33,6 +33,8 @@ pub enum ConfigFileName { LoginIdentityProviders, #[strum(serialize = "authorizers.xml")] Authorizers, + #[strum(serialize = "logback.xml")] + Logback, } /// Keep only the set (`Some`) entries of a `key -> optional value` map, as `(key, value)` pairs. diff --git a/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs b/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs index c91490dd..f274db81 100644 --- a/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs +++ b/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs @@ -54,7 +54,6 @@ pub fn build( #[cfg(test)] mod tests { use indoc::indoc; - use stackable_operator::kube::ResourceExt as _; use super::*; diff --git a/rust/operator-binary/src/controller/build/properties/logging.rs b/rust/operator-binary/src/controller/build/properties/logging.rs new file mode 100644 index 00000000..32d8086c --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/logging.rs @@ -0,0 +1,86 @@ +//! Builds the logback and Vector logging configuration for the rolegroup `ConfigMap`. + +use stackable_operator::{ + memory::BinaryMultiple, + product_logging::{ + self, + spec::{ContainerLogConfig, ContainerLogConfigChoice, Logging}, + }, + role_utils::RoleGroupRef, +}; + +use crate::crd::{Container, MAX_NIFI_LOG_FILES_SIZE, STACKABLE_LOG_DIR, v1alpha1}; + +pub const NIFI_LOG_FILE: &str = "nifi.log4j.xml"; + +const CONSOLE_CONVERSION_PATTERN: &str = "%date %level [%thread] %logger{40} %msg%n"; +// This is required to remove double entries in the nifi.log4j.xml as well as nested +// console output like: " ... ... +const ADDITIONAL_LOGBACK_CONFIG: &str = r#" + + %msg%n + + + + + + + + + + +"#; + +/// Renders the `logback.xml` for the NiFi container. +/// +/// Returns `None` when the container uses a custom log ConfigMap instead of the operator's +/// automatic logging configuration. +pub fn build_logback_config(logging: &Logging) -> Option { + let ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Automatic(log_config)), + } = logging.containers.get(&Container::Nifi)? + else { + return None; + }; + + Some(product_logging::framework::create_logback_config( + &format!( + "{STACKABLE_LOG_DIR}/{container}", + container = Container::Nifi + ), + NIFI_LOG_FILE, + MAX_NIFI_LOG_FILES_SIZE + .scale_to(BinaryMultiple::Mebi) + .floor() + .value as u32, + CONSOLE_CONVERSION_PATTERN, + log_config, + Some(ADDITIONAL_LOGBACK_CONFIG), + )) +} + +/// Renders the Vector agent config (`vector.yaml`). +/// +/// Returns `None` when the Vector agent is disabled for this role group. +pub fn build_vector_config( + rolegroup: &RoleGroupRef, + logging: &Logging, +) -> Option { + if !logging.enable_vector_agent { + return None; + } + + let vector_log_config = if let Some(ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Automatic(log_config)), + }) = logging.containers.get(&Container::Vector) + { + Some(log_config) + } else { + None + }; + + Some(product_logging::framework::create_vector_config( + rolegroup, + vector_log_config, + )) +} diff --git a/rust/operator-binary/src/controller/build/properties/nifi_properties.rs b/rust/operator-binary/src/controller/build/properties/nifi_properties.rs index a8b66f19..04c0ee64 100644 --- a/rust/operator-binary/src/controller/build/properties/nifi_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/nifi_properties.rs @@ -643,8 +643,10 @@ mod tests { fn test_config_override_wins() { use stackable_operator::kube::ResourceExt as _; - use crate::crd::{NifiConfig, NifiRole, v1alpha1}; - use crate::framework::role_utils::with_validated_config; + use crate::{ + crd::{NifiConfig, NifiRole, v1alpha1}, + framework::role_utils::with_validated_config, + }; let yaml = r#" apiVersion: nifi.stackable.tech/v1alpha1 diff --git a/rust/operator-binary/src/controller/build/properties/security_properties.rs b/rust/operator-binary/src/controller/build/properties/security_properties.rs index 6643eb6c..97c9304c 100644 --- a/rust/operator-binary/src/controller/build/properties/security_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/security_properties.rs @@ -2,8 +2,8 @@ use std::collections::BTreeMap; -use super::{ConfigFileName, writer}; -use crate::controller::validate::NifiRoleGroupConfig; +use super::ConfigFileName; +use crate::{controller::validate::NifiRoleGroupConfig, framework::writer}; pub fn build(rg: &NifiRoleGroupConfig) -> Result { let mut props: BTreeMap> = BTreeMap::new(); diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 1539d9be..18c2af8b 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -585,8 +585,9 @@ fn node_default_listener_class() -> String { mod merge_tests { use std::collections::BTreeMap; - use stackable_operator::config::merge::Merge as _; - use stackable_operator::config_overrides::KeyValueConfigOverrides; + use stackable_operator::{ + config::merge::Merge as _, config_overrides::KeyValueConfigOverrides, + }; use super::v1alpha1::NifiConfigOverrides; diff --git a/rust/operator-binary/src/framework.rs b/rust/operator-binary/src/framework.rs index 0f5717f4..2c9e9ea6 100644 --- a/rust/operator-binary/src/framework.rs +++ b/rust/operator-binary/src/framework.rs @@ -6,3 +6,4 @@ //! the upstream `RoleGroupConfig` uses `EnvVarSet` rather than a plain map). pub mod role_utils; +pub mod writer; diff --git a/rust/operator-binary/src/controller/build/properties/writer.rs b/rust/operator-binary/src/framework/writer.rs similarity index 100% rename from rust/operator-binary/src/controller/build/properties/writer.rs rename to rust/operator-binary/src/framework/writer.rs diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index c1eee997..77622403 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -44,7 +44,6 @@ mod crd; mod framework; mod listener; mod operations; -mod product_logging; mod reporting_task; mod security; mod service; diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs deleted file mode 100644 index b9fb012a..00000000 --- a/rust/operator-binary/src/product_logging.rs +++ /dev/null @@ -1,103 +0,0 @@ -use snafu::Snafu; -use stackable_operator::{ - builder::configmap::ConfigMapBuilder, - memory::BinaryMultiple, - product_logging::{ - self, - spec::{ContainerLogConfig, ContainerLogConfigChoice, Logging}, - }, - role_utils::RoleGroupRef, -}; - -use crate::crd::{Container, MAX_NIFI_LOG_FILES_SIZE, STACKABLE_LOG_DIR, v1alpha1}; - -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("object has no namespace"))] - ObjectHasNoNamespace, - #[snafu(display("failed to retrieve the ConfigMap {cm_name}"))] - ConfigMapNotFound { - source: stackable_operator::client::Error, - cm_name: String, - }, - #[snafu(display("failed to retrieve the entry {entry} for ConfigMap {cm_name}"))] - MissingConfigMapEntry { - entry: &'static str, - cm_name: String, - }, - #[snafu(display("crd validation failure"))] - CrdValidationFailure { source: crate::crd::Error }, - #[snafu(display("vectorAggregatorConfigMapName must be set"))] - MissingVectorAggregatorAddress, -} - -type Result = std::result::Result; - -pub const LOGBACK_CONFIG_FILE: &str = "logback.xml"; -pub const NIFI_LOG_FILE: &str = "nifi.log4j.xml"; - -const CONSOLE_CONVERSION_PATTERN: &str = "%date %level [%thread] %logger{40} %msg%n"; -// This is required to remove double entries in the nifi.log4j.xml as well as nested -// console output like: " ... ... -const ADDITIONAL_LOGBACK_CONFIG: &str = r#" - - %msg%n - - - - - - - - - - -"#; - -/// Extend the role group ConfigMap with logging and Vector configurations -pub fn extend_role_group_config_map( - rolegroup: &RoleGroupRef, - logging: &Logging, - cm_builder: &mut ConfigMapBuilder, -) -> Result<()> { - if let Some(ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic(log_config)), - }) = logging.containers.get(&Container::Nifi) - { - cm_builder.add_data( - LOGBACK_CONFIG_FILE, - product_logging::framework::create_logback_config( - &format!( - "{STACKABLE_LOG_DIR}/{container}", - container = Container::Nifi - ), - NIFI_LOG_FILE, - MAX_NIFI_LOG_FILES_SIZE - .scale_to(BinaryMultiple::Mebi) - .floor() - .value as u32, - CONSOLE_CONVERSION_PATTERN, - log_config, - Some(ADDITIONAL_LOGBACK_CONFIG), - ), - ); - } - - let vector_log_config = if let Some(ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic(log_config)), - }) = logging.containers.get(&Container::Vector) - { - Some(log_config) - } else { - None - }; - - if logging.enable_vector_agent { - cm_builder.add_data( - product_logging::framework::VECTOR_CONFIG_FILE, - product_logging::framework::create_vector_config(rolegroup, vector_log_config), - ); - } - - Ok(()) -} From 377ca973f611e35ee06eef22593eeb14762b4ba2 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Jun 2026 16:30:09 +0200 Subject: [PATCH 14/39] chore: regenerate hashes --- Cargo.nix | 577 +++++++++++++++++++++++++--------------------- crate-hashes.json | 18 +- 2 files changed, 324 insertions(+), 271 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index cf857536..45b7b36e 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -489,9 +489,9 @@ rec { }; "autocfg" = rec { crateName = "autocfg"; - version = "1.5.0"; + version = "1.5.1"; edition = "2015"; - sha256 = "1s77f98id9l4af4alklmzq46f21c980v13z2r1pcxx6bqgw0d1n0"; + sha256 = "0lqasy5i30flcgih1b50kvsk6z32g09r1q4ql7q81pj6228jy0zj"; authors = [ "Josh Stone " ]; @@ -866,9 +866,9 @@ rec { }; "bitflags" = rec { crateName = "bitflags"; - version = "2.11.1"; + version = "2.12.1"; edition = "2021"; - sha256 = "1cvqijg3rvwgis20a66vfdxannjsxfy5fgjqkaq3l13gyfcj4lf4"; + sha256 = "02phhjm7w380zdh8928zf13cfi1bw2qz2ay36ml2jmwmmv8cxmw4"; authors = [ "The Rust Project Developers" ]; @@ -898,9 +898,9 @@ rec { }; "built" = rec { crateName = "built"; - version = "0.8.0"; - edition = "2021"; - sha256 = "0r5f08lpjsr6j5ajkbmd0ymfmajpq8ddbfvi8ji8rx48y88qzbgl"; + version = "0.8.1"; + edition = "2024"; + sha256 = "1saq332pd6g3svvc9ah8myjpfvgqlzl2ksb1ypp3976kjcfm63jw"; authors = [ "Lukas Lueg " ]; @@ -924,15 +924,16 @@ rec { "chrono" = [ "dep:chrono" ]; "dependency-tree" = [ "cargo-lock/dependency-tree" ]; "git2" = [ "dep:git2" ]; + "gix" = [ "dep:gix" ]; "semver" = [ "dep:semver" ]; }; resolvedDefaultFeatures = [ "chrono" "git2" ]; }; "bumpalo" = rec { crateName = "bumpalo"; - version = "3.20.2"; + version = "3.20.3"; edition = "2021"; - sha256 = "1jrgxlff76k9glam0akhwpil2fr1w32gbjdf5hpipc7ld2c7h82x"; + sha256 = "0jc6va3nwcqikm7chnpdv1s87my3gs2j7g1sc7g3k91brg3arxbj"; authors = [ "Nick Fitzgerald " ]; @@ -961,9 +962,9 @@ rec { }; "cc" = rec { crateName = "cc"; - version = "1.2.60"; + version = "1.2.63"; edition = "2018"; - sha256 = "084a8ziprdlyrj865f3303qr0b7aaggilkl18slncss6m4yp1ia3"; + sha256 = "0zy2bqc4nvj6bv2cipx4h4bn65wf1zqf1fw1hsh64mmvg1hh2vjm"; authors = [ "Alex Crichton " ]; @@ -1977,9 +1978,9 @@ rec { }; "displaydoc" = rec { crateName = "displaydoc"; - version = "0.2.5"; + version = "0.2.6"; edition = "2021"; - sha256 = "1q0alair462j21iiqwrr21iabkfnb13d6x5w95lkdg21q2xrqdlp"; + sha256 = "0kyxwfbdmagd8afzb2pzja7wj8dhah7smxdsgw00iq8pa2jhmiqs"; procMacro = true; authors = [ "Jane Lusby " @@ -2179,12 +2180,9 @@ rec { }; "either" = rec { crateName = "either"; - version = "1.15.0"; + version = "1.16.0"; edition = "2021"; - sha256 = "069p1fknsmzn9llaizh77kip0pqmcwpdsykv2x30xpjyija5gis8"; - authors = [ - "bluss" - ]; + sha256 = "17k7jfbdz7k440h6lws9baz8p9zlxgb41sig3w81h80nwzsjyqli"; features = { "default" = [ "std" ]; "serde" = [ "dep:serde" ]; @@ -2411,7 +2409,7 @@ rec { } { name = "windows-sys"; - packageId = "windows-sys 0.61.2"; + packageId = "windows-sys 0.52.0"; target = { target, features }: (target."windows" or false); features = [ "Win32_Foundation" "Win32_System_Diagnostics_Debug" ]; } @@ -2913,9 +2911,9 @@ rec { }; "futures-timer" = rec { crateName = "futures-timer"; - version = "3.0.3"; + version = "3.0.4"; edition = "2018"; - sha256 = "094vw8k37djpbwv74bwf2qb7n6v6ghif4myss6smd6hgyajb127j"; + sha256 = "0s39in8ivw7g4d37pf31q02y44zd1hpfkd1pgra2slcqibdzlhxg"; libName = "futures_timer"; authors = [ "Alex Crichton " @@ -3263,9 +3261,9 @@ rec { }; "git2" = rec { crateName = "git2"; - version = "0.20.4"; - edition = "2018"; - sha256 = "0azykjpk3j6s354z23jkyq3r3pbmlw9ha1zsxkw5cnnpi1h2b23v"; + version = "0.21.0"; + edition = "2021"; + sha256 = "0bmqga9vlyx5sdlr0i28z0362s89xv9i4qcv20vvx9j54y9vzpfx"; authors = [ "Josh Triplett " "Alex Crichton " @@ -3287,17 +3285,14 @@ rec { name = "log"; packageId = "log"; } - { - name = "url"; - packageId = "url"; - } ]; features = { - "default" = [ "ssh" "https" ]; - "https" = [ "libgit2-sys/https" "openssl-sys" "openssl-probe" ]; + "cred" = [ "dep:url" ]; + "https" = [ "libgit2-sys/https" "openssl-sys" "openssl-probe" "cred" ]; "openssl-probe" = [ "dep:openssl-probe" ]; "openssl-sys" = [ "dep:openssl-sys" ]; - "ssh" = [ "libgit2-sys/ssh" ]; + "ssh" = [ "libgit2-sys/ssh" "cred" ]; + "unstable-sha256" = [ "libgit2-sys/unstable-sha256" ]; "vendored-libgit2" = [ "libgit2-sys/vendored" ]; "vendored-openssl" = [ "openssl-sys/vendored" "libgit2-sys/vendored-openssl" ]; "zlib-ng-compat" = [ "libgit2-sys/zlib-ng-compat" ]; @@ -3387,9 +3382,9 @@ rec { }; "h2" = rec { crateName = "h2"; - version = "0.4.13"; + version = "0.4.14"; edition = "2021"; - sha256 = "0m6w5gg0n0m1m5915bxrv8n4rlazhx5icknkslz719jhh4xdli1g"; + sha256 = "0cw7jk7kn2vn6f8w8ssh6gis1mljnfjxd606gvi4sjpyjayfy7qp"; authors = [ "Carl Lerche " "Sean McArthur " @@ -3530,14 +3525,11 @@ rec { }; resolvedDefaultFeatures = [ "allocator-api2" "default" "default-hasher" "equivalent" "inline-more" "raw-entry" ]; }; - "hashbrown 0.17.0" = rec { + "hashbrown 0.17.1" = rec { crateName = "hashbrown"; - version = "0.17.0"; + version = "0.17.1"; edition = "2024"; - sha256 = "0l8gvcz80lvinb7x22h53cqbi2y1fm603y2jhhh9qwygvkb7sijg"; - authors = [ - "Amanieu d'Antras " - ]; + sha256 = "0jmqz7i4yl6cm7rbn0i2ffkfrmwi6xkmzkaldr2v8bcsx2v0jngd"; features = { "alloc" = [ "dep:alloc" ]; "allocator-api2" = [ "dep:allocator-api2" ]; @@ -3612,9 +3604,9 @@ rec { }; "http" = rec { crateName = "http"; - version = "1.4.0"; + version = "1.4.1"; edition = "2021"; - sha256 = "06iind4cwsj1d6q8c2xgq8i2wka4ps74kmws24gsi1bzdlw2mfp3"; + sha256 = "1l7k2ia57z3q7q3ka497krzps795kd3fymm2k12lr623y4nldrwb"; authors = [ "Alex Crichton " "Carl Lerche " @@ -3731,9 +3723,9 @@ rec { }; "hyper" = rec { crateName = "hyper"; - version = "1.9.0"; + version = "1.10.1"; edition = "2021"; - sha256 = "1jmwbwqcaficskg76kq402gbymbnh2z4v99xwq3l5aa6n8bg16b2"; + sha256 = "1624nwrh1ci34psqcl3q8q266kha8kd6fmqjj14qck49l59iqa2m"; authors = [ "Sean McArthur " ]; @@ -4520,9 +4512,9 @@ rec { }; "idna_adapter" = rec { crateName = "idna_adapter"; - version = "1.2.1"; - edition = "2021"; - sha256 = "0i0339pxig6mv786nkqcxnwqa87v4m94b2653f6k3aj0jmhfkjis"; + version = "1.2.2"; + edition = "2024"; + sha256 = "0557p76l8hj35r9zn1yv7c6x1c0qbrsffmg80n0yy8361ly3fs6b"; authors = [ "The rust-url developers" ]; @@ -4556,7 +4548,7 @@ rec { } { name = "hashbrown"; - packageId = "hashbrown 0.17.0"; + packageId = "hashbrown 0.17.1"; usesDefaultFeatures = false; } { @@ -4635,39 +4627,6 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; - "iri-string" = rec { - crateName = "iri-string"; - version = "0.7.12"; - edition = "2021"; - sha256 = "082fpx6c5ghvmqpwxaf2b268m47z2ic3prajqbmi1s1qpfj5kri5"; - libName = "iri_string"; - authors = [ - "YOSHIOKA Takuma " - ]; - dependencies = [ - { - name = "memchr"; - packageId = "memchr"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "serde"; - packageId = "serde"; - optional = true; - usesDefaultFeatures = false; - features = [ "derive" ]; - } - ]; - features = { - "alloc" = [ "serde?/alloc" ]; - "default" = [ "std" ]; - "memchr" = [ "dep:memchr" ]; - "serde" = [ "dep:serde" ]; - "std" = [ "alloc" "memchr?/std" "serde?/std" ]; - }; - resolvedDefaultFeatures = [ "alloc" "default" "std" ]; - }; "is_terminal_polyfill" = rec { crateName = "is_terminal_polyfill"; version = "1.70.2"; @@ -4737,9 +4696,9 @@ rec { }; "jiff" = rec { crateName = "jiff"; - version = "0.2.23"; + version = "0.2.28"; edition = "2021"; - sha256 = "0nc37n7jvgrzxdkcgc2hsfdf70lfagigjalh4igjrm5njvf4cd8s"; + sha256 = "00lixngcc7amh2fcsxfr0z38j06lllhapz192biv1qj97q1x60s6"; authors = [ "Andrew Gallant " ]; @@ -4785,12 +4744,10 @@ rec { usesDefaultFeatures = false; } { - name = "windows-sys"; - packageId = "windows-sys 0.61.2"; + name = "windows-link"; + packageId = "windows-link"; optional = true; - usesDefaultFeatures = false; target = { target, features }: (target."windows" or false); - features = [ "Win32_Foundation" "Win32_System_Time" ]; } ]; devDependencies = [ @@ -4809,7 +4766,7 @@ rec { "static-tz" = [ "dep:jiff-static" ]; "std" = [ "alloc" "log?/std" "serde_core?/std" ]; "tz-fat" = [ "jiff-static?/tz-fat" ]; - "tz-system" = [ "std" "dep:windows-sys" ]; + "tz-system" = [ "std" "dep:windows-link" ]; "tzdb-bundle-always" = [ "dep:jiff-tzdb" "alloc" ]; "tzdb-bundle-platform" = [ "dep:jiff-tzdb-platform" "alloc" ]; "tzdb-concatenated" = [ "std" ]; @@ -4819,9 +4776,9 @@ rec { }; "jiff-static" = rec { crateName = "jiff-static"; - version = "0.2.23"; + version = "0.2.28"; edition = "2021"; - sha256 = "192ss3cnixvg79cpa76clwkhn4mmz10vnwsbf7yjw8i484s8p31a"; + sha256 = "0irbhfh2f4i9w5l53jcmh6ssnhdd92wfy76978chgwnxilvk4bbq"; procMacro = true; libName = "jiff_static"; authors = [ @@ -4901,9 +4858,9 @@ rec { }; "js-sys" = rec { crateName = "js-sys"; - version = "0.3.95"; + version = "0.3.99"; edition = "2021"; - sha256 = "1jhj3kgxxgwm0cpdjiz7i2qapqr7ya9qswadmr63dhwx3lnyjr19"; + sha256 = "04azrzsz91gr5s3z0ij36lz0kj9ry4lw3jz0mmbiwb251rsc8aql"; libName = "js_sys"; authors = [ "The wasm-bindgen Developers" @@ -4912,7 +4869,6 @@ rec { { name = "cfg-if"; packageId = "cfg-if"; - optional = true; } { name = "futures-util"; @@ -4934,17 +4890,16 @@ rec { ]; features = { "default" = [ "std" "unsafe-eval" ]; - "futures" = [ "dep:cfg-if" "dep:futures-util" ]; - "futures-core-03-stream" = [ "futures" "dep:futures-core" ]; - "std" = [ "wasm-bindgen/std" ]; + "futures-core-03-stream" = [ "dep:futures-util" "dep:futures-core" ]; + "std" = [ "wasm-bindgen/std" "dep:futures-util" ]; }; - resolvedDefaultFeatures = [ "default" "futures" "std" "unsafe-eval" ]; + resolvedDefaultFeatures = [ "default" "std" "unsafe-eval" ]; }; "json-patch" = rec { crateName = "json-patch"; - version = "4.1.0"; + version = "4.2.0"; edition = "2021"; - sha256 = "147yaxmv3i4s0bdna86rgwpmqh2507fn4ighfpplaiqkw8ay807k"; + sha256 = "0wkv896d0pzq56i2kkl0giqpv117fwvhbpgs8iz85805w66l68bl"; libName = "json_patch"; authors = [ "Ivan Dubrov " @@ -4954,6 +4909,11 @@ rec { name = "jsonptr"; packageId = "jsonptr"; } + { + name = "schemars"; + packageId = "schemars"; + optional = true; + } { name = "serde"; packageId = "serde"; @@ -4965,10 +4925,14 @@ rec { } { name = "thiserror"; - packageId = "thiserror 1.0.69"; + packageId = "thiserror 2.0.18"; } ]; devDependencies = [ + { + name = "schemars"; + packageId = "schemars"; + } { name = "serde_json"; packageId = "serde_json"; @@ -4980,7 +4944,7 @@ rec { "schemars" = [ "dep:schemars" ]; "utoipa" = [ "dep:utoipa" ]; }; - resolvedDefaultFeatures = [ "default" "diff" ]; + resolvedDefaultFeatures = [ "default" "diff" "schemars" ]; }; "jsonpath-rust" = rec { crateName = "jsonpath-rust"; @@ -5106,9 +5070,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "k8s_version"; authors = [ @@ -5126,7 +5090,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } ]; features = { @@ -5770,9 +5734,9 @@ rec { }; "libc" = rec { crateName = "libc"; - version = "0.2.185"; + version = "0.2.186"; edition = "2021"; - sha256 = "13rbdaa59l3w92q7kfcxx8zbikm99zzw54h59aqvcv5wx47jrzsj"; + sha256 = "0rnyhzjyqq9x56skkllbjzzzwym3r61lq3l4hqj64v71gw0r3av8"; authors = [ "The Rust Project Developers" ]; @@ -5786,10 +5750,10 @@ rec { }; "libgit2-sys" = rec { crateName = "libgit2-sys"; - version = "0.18.3+1.9.2"; + version = "0.18.5+1.9.4"; edition = "2021"; links = "git2"; - sha256 = "11rlbyihj3k35mnkxxz4yvsnlx33a4r9srl66c5vp08pp72arcy9"; + sha256 = "18lwqnhy7qxg4iw24s1a0n7aj7qbnryry1iy0w32k4f1xbk6lp80"; libName = "libgit2_sys"; libPath = "lib.rs"; authors = [ @@ -5847,10 +5811,10 @@ rec { }; "libz-sys" = rec { crateName = "libz-sys"; - version = "1.1.28"; + version = "1.1.29"; edition = "2018"; links = "z"; - sha256 = "08hyf9v85zifl3353xc7i5wr53v9b3scri856cmphl3gaxp24fpw"; + sha256 = "1n98kqya7a7a0cxf5n5z3g13rj7a1vqxynk2xc7bja1qfxbrdg45"; libName = "libz_sys"; authors = [ "Alex Crichton " @@ -5927,9 +5891,9 @@ rec { }; "log" = rec { crateName = "log"; - version = "0.4.29"; + version = "0.4.31"; edition = "2021"; - sha256 = "15q8j9c8g5zpkcw0hnd6cf2z7fxqnvsjh3rw5mv5q10r83i34l2y"; + sha256 = "0kq2fh6q2bjkrm8m6hj8kb7gxfd7cr7qbcpxd1lc1xq5rns30fqi"; authors = [ "The Rust Project Developers" ]; @@ -5983,9 +5947,9 @@ rec { }; "memchr" = rec { crateName = "memchr"; - version = "2.8.0"; + version = "2.8.1"; edition = "2021"; - sha256 = "0y9zzxcqxvdqg6wyag7vc3h0blhdn7hkq164bxyx2vph8zs5ijpq"; + sha256 = "1n448jx01h5z2xknj6x2dhxgr8s8fb717cf6vfqj5lmhkpj7m53b"; authors = [ "Andrew Gallant " "bluss" @@ -6046,9 +6010,9 @@ rec { }; "mio" = rec { crateName = "mio"; - version = "1.2.0"; + version = "1.2.1"; edition = "2021"; - sha256 = "1hanrh4fwsfkdqdaqfidz48zz1wdix23zwn3r2x78am0garfbdsh"; + sha256 = "1nkggmrlnjs93w8rja4lvjj4aml1xqahgimv1h0p7d373kvhmg82"; authors = [ "Carl Lerche " "Thomas de Zeeuw " @@ -6184,9 +6148,9 @@ rec { }; "num-conv" = rec { crateName = "num-conv"; - version = "0.2.1"; + version = "0.2.2"; edition = "2021"; - sha256 = "0rqrr29brafaa2za352pbmhkk556n7f8z9rrkgmjp1idvdl3fry6"; + sha256 = "0hg4f9bwmy7cwpxdkm165dmkfc8jhkkayci234jsmi5ssb33j5sj"; libName = "num_conv"; authors = [ "Jacob Pratt " @@ -7134,9 +7098,9 @@ rec { }; "pin-project" = rec { crateName = "pin-project"; - version = "1.1.11"; + version = "1.1.13"; edition = "2021"; - sha256 = "05zm3y3bl83ypsr6favxvny2kys4i19jiz1y18ylrbxwsiz9qx7i"; + sha256 = "09091qp946lpmjz4yp0xil1r5v4hgc91fi19dg5csayhdqrv4ri4"; libName = "pin_project"; dependencies = [ { @@ -7148,9 +7112,9 @@ rec { }; "pin-project-internal" = rec { crateName = "pin-project-internal"; - version = "1.1.11"; + version = "1.1.13"; edition = "2021"; - sha256 = "1ik4mpb92da75inmjvxf2qm61vrnwml3x24wddvrjlqh1z9hxcnr"; + sha256 = "12rzlh07i1sdgrvzj6wgkka5bjqyvbfsl8knq6qi7g16m7q9aqy9"; procMacro = true; libName = "pin_project_internal"; dependencies = [ @@ -8601,9 +8565,9 @@ rec { }; "rustls" = rec { crateName = "rustls"; - version = "0.23.38"; + version = "0.23.40"; edition = "2021"; - sha256 = "089ssmhd79f0kd22brh6lkaadql2p3pi6579ax1s0kn1n9pldyb9"; + sha256 = "12qnv3ag4wrw7aj8jng74kgrilpjm2b1rfcjaac8h691frccv1pg"; dependencies = [ { name = "log"; @@ -8670,9 +8634,9 @@ rec { }; "rustls-native-certs" = rec { crateName = "rustls-native-certs"; - version = "0.8.3"; + version = "0.8.4"; edition = "2021"; - sha256 = "0qrajg2n90bcr3bcq6j95gjm7a9lirfkkdmjj32419dyyzan0931"; + sha256 = "0kgazl8zc1sv63qg179bz96ilzh56lzfa5k92ji7d265f4kibdfs"; libName = "rustls_native_certs"; dependencies = [ { @@ -8701,9 +8665,9 @@ rec { }; "rustls-pki-types" = rec { crateName = "rustls-pki-types"; - version = "1.14.0"; + version = "1.14.1"; edition = "2021"; - sha256 = "1p9zsgslvwzzkzhm6bqicffqndr4jpx67992b0vl0pi21a5hy15y"; + sha256 = "1a9pr54y0f3qr97bxpd3ahjldq0gqdld0h799xbnwdzbwxx1k9rh"; libName = "rustls_pki_types"; dependencies = [ { @@ -9239,9 +9203,9 @@ rec { }; "serde_json" = rec { crateName = "serde_json"; - version = "1.0.149"; + version = "1.0.150"; edition = "2021"; - sha256 = "11jdx4vilzrjjd1dpgy67x5lgzr0laplz30dhv75lnf5ffa07z43"; + sha256 = "1ffgfhy9kndjnrz8lmy95pr758p2zk8dxv6yi99x0vkkni24w0g8"; authors = [ "Erick Tryzelaar " "David Tolnay " @@ -9482,9 +9446,9 @@ rec { }; "shlex" = rec { crateName = "shlex"; - version = "1.3.0"; - edition = "2015"; - sha256 = "0r1y6bv26c1scpxvhg2cabimrmwgbp4p3wy6syj9n0c4s3q2znhg"; + version = "2.0.1"; + edition = "2018"; + sha256 = "1fjsll1cd7d2bcpdij9kd6w62rpbc7qqzvydvs021vsmr1cxvypq"; authors = [ "comex " "Fenhl " @@ -9663,29 +9627,25 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "rust_1_61" "rust_1_65" "std" ]; }; - "snafu 0.9.0" = rec { + "snafu 0.9.1" = rec { crateName = "snafu"; - version = "0.9.0"; + version = "0.9.1"; edition = "2018"; - sha256 = "1ii9r99x5qcn754m624yzgb9hzvkqkrcygf0aqh0pyb9dbnvrm6i"; + sha256 = "08k5yfydxdlshivfhrdq9km8qn02r93q28gkyvazbqz2icr1586i"; authors = [ "Jake Goulding " ]; dependencies = [ { name = "snafu-derive"; - packageId = "snafu-derive 0.9.0"; + packageId = "snafu-derive 0.9.1"; } ]; features = { - "backtrace" = [ "dep:backtrace" ]; - "backtraces-impl-backtrace-crate" = [ "backtrace" ]; + "backtraces-impl-backtrace-crate" = [ "dep:backtrace" ]; "default" = [ "std" "rust_1_81" ]; - "futures" = [ "futures-core-crate" "pin-project" ]; - "futures-core-crate" = [ "dep:futures-core-crate" ]; - "futures-crate" = [ "dep:futures-crate" ]; - "internal-dev-dependencies" = [ "futures-crate" ]; - "pin-project" = [ "dep:pin-project" ]; + "futures" = [ "dep:futures-core" "dep:pin-project" ]; + "internal-dev-dependencies" = [ "dep:futures" ]; "std" = [ "alloc" ]; "unstable-provider-api" = [ "snafu-derive/unstable-provider-api" ]; }; @@ -9753,11 +9713,11 @@ rec { }; resolvedDefaultFeatures = [ "rust_1_61" ]; }; - "snafu-derive 0.9.0" = rec { + "snafu-derive 0.9.1" = rec { crateName = "snafu-derive"; - version = "0.9.0"; + version = "0.9.1"; edition = "2018"; - sha256 = "0h0x61kyj4fvilcr2nj02l85shw1ika64vq9brf2gyna662ln9al"; + sha256 = "1nkfi7bis72pz3w7vb64m79w49qsv20sbf19jkd471vbhr83q42z"; procMacro = true; libName = "snafu_derive"; authors = [ @@ -9783,7 +9743,7 @@ rec { name = "syn"; packageId = "syn 2.0.117"; usesDefaultFeatures = false; - features = [ "clone-impls" "derive" "full" "parsing" "printing" "proc-macro" ]; + features = [ "clone-impls" "derive" "full" "parsing" "printing" "proc-macro" "visit-mut" ]; } ]; features = { @@ -9791,9 +9751,9 @@ rec { }; "socket2" = rec { crateName = "socket2"; - version = "0.6.3"; + version = "0.6.4"; edition = "2021"; - sha256 = "0gkjjcyn69hqhhlh5kl8byk5m0d7hyrp2aqwzbs3d33q208nwxis"; + sha256 = "0ldyp5rhba15spwxj1n94xh7sjks1398c3vwpwkxkd1087nwzlaj"; authors = [ "Alex Crichton " "Thomas de Zeeuw " @@ -9891,9 +9851,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_certs"; authors = [ @@ -9951,7 +9911,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-shared"; @@ -10030,12 +9990,12 @@ rec { packageId = "indoc"; } { - name = "pin-project"; - packageId = "pin-project"; + name = "java-properties"; + packageId = "java-properties"; } { - name = "product-config"; - packageId = "product-config"; + name = "pin-project"; + packageId = "pin-project"; } { name = "rand"; @@ -10056,7 +10016,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-operator"; @@ -10103,13 +10063,13 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.111.0"; + version = "0.111.1"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_operator"; authors = [ @@ -10166,6 +10126,7 @@ rec { { name = "json-patch"; packageId = "json-patch"; + features = [ "schemars" ]; } { name = "k8s-openapi"; @@ -10215,7 +10176,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-operator-derive"; @@ -10269,6 +10230,10 @@ rec { packageId = "url"; features = [ "serde" ]; } + { + name = "uuid"; + packageId = "uuid"; + } ]; features = { "certs" = [ "dep:stackable-certs" ]; @@ -10287,9 +10252,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -10322,9 +10287,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_shared"; authors = [ @@ -10368,7 +10333,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "strum"; @@ -10403,9 +10368,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_telemetry"; authors = [ @@ -10456,7 +10421,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "strum"; @@ -10513,9 +10478,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_versioned"; authors = [ @@ -10548,7 +10513,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-versioned-macros"; @@ -10563,9 +10528,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -10631,9 +10596,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_webhook"; authors = [ @@ -10705,7 +10670,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-certs"; @@ -10835,6 +10800,16 @@ rec { }; resolvedDefaultFeatures = [ "i128" ]; }; + "symlink" = rec { + crateName = "symlink"; + version = "0.1.0"; + edition = "2015"; + sha256 = "02h1i0b81mxb4vns4xrvrfibpcvs7jqqav8p3yilwik8cv73r5x7"; + authors = [ + "Chris Morgan " + ]; + + }; "syn 1.0.109" = rec { crateName = "syn"; version = "1.0.109"; @@ -11267,9 +11242,9 @@ rec { }; "tokio" = rec { crateName = "tokio"; - version = "1.52.0"; + version = "1.52.3"; edition = "2021"; - sha256 = "0xnpygq9578c8rqjgkj5bj8pgfx9zj337kvk3v4kigqwkgska4d9"; + sha256 = "1zpzazypkg61sw91na1m85x5s4rsjym335fwwhwm1hcs70dz1iwg"; authors = [ "Tokio Contributors " ]; @@ -11579,9 +11554,9 @@ rec { }; "toml_edit" = rec { crateName = "toml_edit"; - version = "0.25.11+spec-1.1.0"; + version = "0.25.12+spec-1.1.0"; edition = "2024"; - sha256 = "0awzffbkx33v9x4h19b5mfrwp3sn4ifr16y58sbk6j6l5v9c8n8b"; + sha256 = "1mx5paq837rjw7w51zprrjynk1vaig9yzxfqz9ac79jmd7f3w5fj"; dependencies = [ { name = "indexmap"; @@ -11634,9 +11609,9 @@ rec { }; "tonic" = rec { crateName = "tonic"; - version = "0.14.5"; - edition = "2021"; - sha256 = "1v4k7aa28m7722gz9qak2jiy7lis1ycm4fdmq63iip4m0qdcdizy"; + version = "0.14.6"; + edition = "2024"; + sha256 = "1vs5ci6z6b9xhfsnx4s8qx6bqi1zzcrxncjp71147a0gqwc5aamc"; authors = [ "Lucio Franco " ]; @@ -11763,9 +11738,9 @@ rec { }; "tonic-prost" = rec { crateName = "tonic-prost"; - version = "0.14.5"; - edition = "2021"; - sha256 = "02fkg2bv87q0yds2wz3w0s7i1x6qcgbrl00dy6ipajdapfh7clx5"; + version = "0.14.6"; + edition = "2024"; + sha256 = "184y40nf0iyzc5rg32ivgd88snv68sqy1kchynn55r1vhml9z12h"; libName = "tonic_prost"; authors = [ "Lucio Franco " @@ -11907,9 +11882,9 @@ rec { }; "tower-http" = rec { crateName = "tower-http"; - version = "0.6.8"; + version = "0.6.11"; edition = "2018"; - sha256 = "1y514jwzbyrmrkbaajpwmss4rg0mak82k16d6588w9ncaffmbrnl"; + sha256 = "0h08wjgs3hwnq11iwwzlmnabn1h4cl0fzd48svaccvqffkiggz2c"; libName = "tower_http"; authors = [ "Tower Maintainers " @@ -11943,11 +11918,6 @@ rec { packageId = "http-body"; optional = true; } - { - name = "iri-string"; - packageId = "iri-string"; - optional = true; - } { name = "mime"; packageId = "mime"; @@ -11977,6 +11947,11 @@ rec { optional = true; usesDefaultFeatures = false; } + { + name = "url"; + packageId = "url"; + optional = true; + } ]; devDependencies = [ { @@ -11998,35 +11973,33 @@ rec { } ]; features = { - "async-compression" = [ "dep:async-compression" ]; "auth" = [ "base64" "validate-request" ]; "base64" = [ "dep:base64" ]; "catch-panic" = [ "tracing" "futures-util/std" "dep:http-body" "dep:http-body-util" ]; - "compression-br" = [ "async-compression/brotli" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; - "compression-deflate" = [ "async-compression/zlib" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; + "compression-br" = [ "dep:async-compression" "async-compression?/brotli" "futures-core" "dep:http-body" "tokio-util" "dep:tokio" ]; + "compression-deflate" = [ "dep:async-compression" "async-compression?/zlib" "futures-core" "dep:http-body" "tokio-util" "dep:tokio" ]; "compression-full" = [ "compression-br" "compression-deflate" "compression-gzip" "compression-zstd" ]; - "compression-gzip" = [ "async-compression/gzip" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; - "compression-zstd" = [ "async-compression/zstd" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; - "decompression-br" = [ "async-compression/brotli" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; - "decompression-deflate" = [ "async-compression/zlib" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; + "compression-gzip" = [ "dep:async-compression" "async-compression?/gzip" "futures-core" "dep:http-body" "tokio-util" "dep:tokio" ]; + "compression-zstd" = [ "dep:async-compression" "async-compression?/zstd" "futures-core" "dep:http-body" "tokio-util" "dep:tokio" ]; + "decompression-br" = [ "dep:async-compression" "async-compression?/brotli" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "dep:tokio" ]; + "decompression-deflate" = [ "dep:async-compression" "async-compression?/zlib" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "dep:tokio" ]; "decompression-full" = [ "decompression-br" "decompression-deflate" "decompression-gzip" "decompression-zstd" ]; - "decompression-gzip" = [ "async-compression/gzip" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; - "decompression-zstd" = [ "async-compression/zstd" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; - "follow-redirect" = [ "futures-util" "dep:http-body" "iri-string" "tower/util" ]; - "fs" = [ "futures-core" "futures-util" "dep:http-body" "dep:http-body-util" "tokio/fs" "tokio-util/io" "tokio/io-util" "dep:http-range-header" "mime_guess" "mime" "percent-encoding" "httpdate" "set-status" "futures-util/alloc" "tracing" ]; - "full" = [ "add-extension" "auth" "catch-panic" "compression-full" "cors" "decompression-full" "follow-redirect" "fs" "limit" "map-request-body" "map-response-body" "metrics" "normalize-path" "propagate-header" "redirect" "request-id" "sensitive-headers" "set-header" "set-status" "timeout" "trace" "util" "validate-request" ]; + "decompression-gzip" = [ "dep:async-compression" "async-compression?/gzip" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "dep:tokio" ]; + "decompression-zstd" = [ "dep:async-compression" "async-compression?/zstd" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "dep:tokio" ]; + "follow-redirect" = [ "futures-util" "dep:http-body" "dep:url" "tower/util" ]; + "fs" = [ "dep:tokio" "tokio?/fs" "tokio?/io-util" "futures-core" "futures-util" "dep:http-body" "dep:http-body-util" "tokio-util/io" "dep:http-range-header" "mime_guess" "mime" "percent-encoding" "httpdate" "set-status" "futures-util/alloc" ]; + "full" = [ "add-extension" "auth" "catch-panic" "compression-full" "cors" "decompression-full" "follow-redirect" "fs" "limit" "map-request-body" "map-response-body" "metrics" "normalize-path" "on-early-drop" "propagate-header" "redirect" "request-id" "sensitive-headers" "set-header" "set-status" "timeout" "trace" "util" "validate-request" ]; "futures-core" = [ "dep:futures-core" ]; "futures-util" = [ "dep:futures-util" ]; "httpdate" = [ "dep:httpdate" ]; - "iri-string" = [ "dep:iri-string" ]; "limit" = [ "dep:http-body" "dep:http-body-util" ]; - "metrics" = [ "dep:http-body" "tokio/time" ]; + "metrics" = [ "dep:http-body" "dep:tokio" "tokio?/time" ]; "mime" = [ "dep:mime" ]; "mime_guess" = [ "dep:mime_guess" ]; + "on-early-drop" = [ "dep:http-body" ]; "percent-encoding" = [ "dep:percent-encoding" ]; "request-id" = [ "uuid" ]; - "timeout" = [ "dep:http-body" "tokio/time" ]; - "tokio" = [ "dep:tokio" ]; + "timeout" = [ "dep:http-body" "dep:tokio" "tokio?/time" ]; "tokio-util" = [ "dep:tokio-util" ]; "tower" = [ "dep:tower" ]; "trace" = [ "dep:http-body" "tracing" ]; @@ -12035,7 +12008,7 @@ rec { "uuid" = [ "dep:uuid" ]; "validate-request" = [ "mime" ]; }; - resolvedDefaultFeatures = [ "auth" "base64" "default" "follow-redirect" "futures-util" "iri-string" "map-response-body" "mime" "tower" "trace" "tracing" "util" "validate-request" ]; + resolvedDefaultFeatures = [ "auth" "base64" "default" "follow-redirect" "futures-util" "map-response-body" "mime" "tower" "trace" "tracing" "util" "validate-request" ]; }; "tower-layer" = rec { crateName = "tower-layer"; @@ -12108,9 +12081,9 @@ rec { }; "tracing-appender" = rec { crateName = "tracing-appender"; - version = "0.2.4"; + version = "0.2.5"; edition = "2018"; - sha256 = "1bxf7xvsr89glbq174cx0b9pinaacbhlmc85y1ssniv2rq5lhvbq"; + sha256 = "0g4a6q5s3wafid5lqw1ljzvh1nhk3a4zmb627fxv96dr7qcqc1h5"; libName = "tracing_appender"; authors = [ "Zeki Sherif " @@ -12121,6 +12094,10 @@ rec { name = "crossbeam-channel"; packageId = "crossbeam-channel"; } + { + name = "symlink"; + packageId = "symlink"; + } { name = "thiserror"; packageId = "thiserror 2.0.18"; @@ -12475,13 +12452,9 @@ rec { }; "typenum" = rec { crateName = "typenum"; - version = "1.19.0"; + version = "1.20.1"; edition = "2018"; - sha256 = "1fw2mpbn2vmqan56j1b3fbpcdg80mz26fm53fs16bq5xcq84hban"; - authors = [ - "Paho Lurie-Gregg " - "Andre Bogus " - ]; + sha256 = "086s9ly0906kw5yw41249fba97w5zfxf03pyfwdkffvcprqfixdn"; features = { "scale-info" = [ "dep:scale-info" ]; "scale_info" = [ "scale-info/derive" ]; @@ -12514,9 +12487,9 @@ rec { }; "unicode-segmentation" = rec { crateName = "unicode-segmentation"; - version = "1.13.2"; + version = "1.13.3"; edition = "2018"; - sha256 = "135a26m4a0wj319gcw28j6a5aqvz00jmgwgmcs6szgxjf942facn"; + sha256 = "1a47zaq83p386r3baq4m018xd5q4q0grdg56i1x042dzn71x7xf6"; libName = "unicode_segmentation"; authors = [ "kwantam " @@ -12642,6 +12615,66 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; + "uuid" = rec { + crateName = "uuid"; + version = "1.23.2"; + edition = "2021"; + sha256 = "1xy942s4z0bi8p3441wvd4ry3hx6ry1c7s6fgrr38462xqybhn6j"; + authors = [ + "Ashley Mannix" + "Dylan DPC" + "Hunar Roop Kahlon" + ]; + dependencies = [ + { + name = "js-sys"; + packageId = "js-sys"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)) && (builtins.elem "atomics" targetFeatures)); + } + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); + } + ]; + devDependencies = [ + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + "atomic" = [ "dep:atomic" ]; + "borsh" = [ "dep:borsh" "dep:borsh-derive" ]; + "bytemuck" = [ "dep:bytemuck" ]; + "default" = [ "std" ]; + "fast-rng" = [ "rng" "dep:rand" ]; + "js" = [ "dep:wasm-bindgen" "dep:js-sys" ]; + "md5" = [ "dep:md-5" ]; + "rng" = [ "dep:getrandom" ]; + "rng-getrandom" = [ "rng" "dep:getrandom" "uuid-rng-internal-lib" "uuid-rng-internal-lib/getrandom" ]; + "rng-rand" = [ "rng" "dep:rand" "uuid-rng-internal-lib" "uuid-rng-internal-lib/rand" ]; + "serde" = [ "dep:serde_core" ]; + "sha1" = [ "dep:sha1_smol" ]; + "slog" = [ "dep:slog" ]; + "std" = [ "wasm-bindgen?/std" "js-sys?/std" ]; + "uuid-rng-internal-lib" = [ "dep:uuid-rng-internal-lib" ]; + "v1" = [ "atomic" ]; + "v3" = [ "md5" ]; + "v4" = [ "rng" ]; + "v5" = [ "sha1" ]; + "v6" = [ "atomic" ]; + "v7" = [ "rng" ]; + "zerocopy" = [ "dep:zerocopy" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "valuable" = rec { crateName = "valuable"; version = "0.1.1"; @@ -12709,13 +12742,13 @@ rec { }; "wasip2" = rec { crateName = "wasip2"; - version = "1.0.2+wasi-0.2.9"; + version = "1.0.3+wasi-0.2.9"; edition = "2021"; - sha256 = "1xdw7v08jpfjdg94sp4lbdgzwa587m5ifpz6fpdnkh02kwizj5wm"; + sha256 = "1mi3w855dz99xzjqc4aa8c9q5b6z1y5c963pkk4cvmr6vdr4c1i0"; dependencies = [ { name = "wit-bindgen"; - packageId = "wit-bindgen"; + packageId = "wit-bindgen 0.57.1"; usesDefaultFeatures = false; } ]; @@ -12735,7 +12768,7 @@ rec { dependencies = [ { name = "wit-bindgen"; - packageId = "wit-bindgen"; + packageId = "wit-bindgen 0.51.0"; usesDefaultFeatures = false; features = [ "async" ]; } @@ -12743,7 +12776,7 @@ rec { devDependencies = [ { name = "wit-bindgen"; - packageId = "wit-bindgen"; + packageId = "wit-bindgen 0.51.0"; usesDefaultFeatures = false; features = [ "async-spawn" ]; } @@ -12754,9 +12787,9 @@ rec { }; "wasm-bindgen" = rec { crateName = "wasm-bindgen"; - version = "0.2.118"; + version = "0.2.122"; edition = "2021"; - sha256 = "129s5r14fx4v4xrzpx2c6l860nkxpl48j50y7kl6j16bpah3iy8b"; + sha256 = "02flix96brsb2r1i3grnikii302iqpdm337kl3xv5lklz5v4bl1y"; libName = "wasm_bindgen"; authors = [ "The wasm-bindgen Developers" @@ -12805,9 +12838,9 @@ rec { }; "wasm-bindgen-futures" = rec { crateName = "wasm-bindgen-futures"; - version = "0.4.68"; + version = "0.4.72"; edition = "2021"; - sha256 = "1y7bq5d9fk7s9xaayx38bgs9ns35na0kpb5zw19944zvya1x6wgk"; + sha256 = "03qb24gfr072rk8hb69glfdc8yhqqqq2rhy3j5i0ps8sk79dnwwl"; libName = "wasm_bindgen_futures"; authors = [ "The wasm-bindgen Developers" @@ -12817,7 +12850,6 @@ rec { name = "js-sys"; packageId = "js-sys"; usesDefaultFeatures = false; - features = [ "futures" ]; } { name = "wasm-bindgen"; @@ -12834,9 +12866,9 @@ rec { }; "wasm-bindgen-macro" = rec { crateName = "wasm-bindgen-macro"; - version = "0.2.118"; + version = "0.2.122"; edition = "2021"; - sha256 = "1v98r8vs17cj8918qsg0xx4nlg4nxk1g0jd4nwnyrh1687w29zzf"; + sha256 = "1inyl55bvdifx7l60q9wl0ivmw7236jg7jqmcqpxhsx3knq52qci"; procMacro = true; libName = "wasm_bindgen_macro"; authors = [ @@ -12858,9 +12890,9 @@ rec { }; "wasm-bindgen-macro-support" = rec { crateName = "wasm-bindgen-macro-support"; - version = "0.2.118"; + version = "0.2.122"; edition = "2021"; - sha256 = "0169jr0q469hfx5zqxfyywf2h2f4aj17vn4zly02nfwqmxghc24x"; + sha256 = "0pjw5kc2mbfz59agk5l21kh4hxzp94rygdvsnr4f3z6b5hv4g419"; libName = "wasm_bindgen_macro_support"; authors = [ "The wasm-bindgen Developers" @@ -12894,10 +12926,10 @@ rec { }; "wasm-bindgen-shared" = rec { crateName = "wasm-bindgen-shared"; - version = "0.2.118"; + version = "0.2.122"; edition = "2021"; links = "wasm_bindgen"; - sha256 = "0ag1vvdzi4334jlzilsy14y3nyzwddf1ndn62fyhf6bg62g4vl2z"; + sha256 = "0ds4mmfqvxwc5fp33hn0jblf0f6b4lghrd9mpkls66zic4n9p4ls"; libName = "wasm_bindgen_shared"; authors = [ "The wasm-bindgen Developers" @@ -13022,9 +13054,9 @@ rec { }; "web-sys" = rec { crateName = "web-sys"; - version = "0.3.95"; + version = "0.3.99"; edition = "2021"; - sha256 = "0zfr2jy5bpkkggl88i43yy37p538hg20i56kwn421yj9g6qznbag"; + sha256 = "0dilfvl9jnyhi4skl6cry9wc300r693j0w82jjbq8yy3rx0i8qkd"; libName = "web_sys"; authors = [ "The wasm-bindgen Developers" @@ -13108,6 +13140,7 @@ rec { "CssStyleSheet" = [ "StyleSheet" ]; "CssSupportsRule" = [ "CssConditionRule" "CssGroupingRule" "CssRule" ]; "CssTransition" = [ "Animation" "EventTarget" ]; + "CssViewTransitionRule" = [ "CssRule" ]; "CustomEvent" = [ "Event" ]; "DedicatedWorkerGlobalScope" = [ "EventTarget" "WorkerGlobalScope" ]; "DelayNode" = [ "AudioNode" "EventTarget" ]; @@ -13922,7 +13955,7 @@ rec { "Win32_Web" = [ "Win32" ]; "Win32_Web_InternetExplorer" = [ "Win32_Web" ]; }; - resolvedDefaultFeatures = [ "Win32" "Win32_Foundation" "Win32_System" "Win32_System_Threading" "default" ]; + resolvedDefaultFeatures = [ "Win32" "Win32_Foundation" "Win32_System" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_Threading" "default" ]; }; "windows-sys 0.61.2" = rec { crateName = "windows-sys"; @@ -14184,7 +14217,7 @@ rec { "Win32_Web" = [ "Win32" ]; "Win32_Web_InternetExplorer" = [ "Win32_Web" ]; }; - resolvedDefaultFeatures = [ "Wdk" "Wdk_Foundation" "Wdk_Storage" "Wdk_Storage_FileSystem" "Wdk_System" "Wdk_System_IO" "Win32" "Win32_Foundation" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_IO" "Win32_System_LibraryLoader" "Win32_System_Memory" "Win32_System_Pipes" "Win32_System_SystemInformation" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_Time" "Win32_System_WindowsProgramming" "default" ]; + resolvedDefaultFeatures = [ "Wdk" "Wdk_Foundation" "Wdk_Storage" "Wdk_Storage_FileSystem" "Wdk_System" "Wdk_System_IO" "Win32" "Win32_Foundation" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Console" "Win32_System_IO" "Win32_System_LibraryLoader" "Win32_System_Memory" "Win32_System_Pipes" "Win32_System_SystemInformation" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_WindowsProgramming" "default" ]; }; "windows-targets" = rec { crateName = "windows-targets"; @@ -14321,9 +14354,9 @@ rec { }; "winnow" = rec { crateName = "winnow"; - version = "1.0.1"; + version = "1.0.3"; edition = "2021"; - sha256 = "1dbji1bwviy08pl74f2qw2m4w9hc4p3vyl3lfj05jdydy59w1nh9"; + sha256 = "1wajycd3krn6h699vydjv7hm0ll5l31p899qzpk59y2is74y34h5"; dependencies = [ { name = "memchr"; @@ -14344,7 +14377,7 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "ascii" "binary" "default" "parser" "std" ]; }; - "wit-bindgen" = rec { + "wit-bindgen 0.51.0" = rec { crateName = "wit-bindgen"; version = "0.51.0"; edition = "2024"; @@ -14371,6 +14404,26 @@ rec { }; resolvedDefaultFeatures = [ "async" "std" ]; }; + "wit-bindgen 0.57.1" = rec { + crateName = "wit-bindgen"; + version = "0.57.1"; + edition = "2024"; + sha256 = "0vjk2jb593ri9k1aq4iqs2si9mrw5q46wxnn78im7hm7hx799gqy"; + libName = "wit_bindgen"; + authors = [ + "Alex Crichton " + ]; + features = { + "async-spawn" = [ "async" "dep:futures" "std" ]; + "bitflags" = [ "dep:bitflags" ]; + "default" = [ "macros" "realloc" "async" "std" "bitflags" "macro-string" ]; + "futures-stream" = [ "async" "dep:futures" ]; + "inter-task-wakeup" = [ "async" ]; + "macro-string" = [ "wit-bindgen-rust-macro?/macro-string" ]; + "macros" = [ "dep:wit-bindgen-rust-macro" ]; + "rustc-dep-of-std" = [ "dep:core" "dep:alloc" ]; + }; + }; "wit-bindgen-core" = rec { crateName = "wit-bindgen-core"; version = "0.51.0"; @@ -14726,9 +14779,9 @@ rec { }; "xml" = rec { crateName = "xml"; - version = "1.2.1"; + version = "1.3.0"; edition = "2021"; - sha256 = "0ak4k990faralbli5a0rb8kvwihccb2rp0r94d4azfy94a6lkamq"; + sha256 = "128s58qhq8whrx90zbw8r5algr7lakgbf7mn05jfk234rbjqavv3"; authors = [ "Vladimir Matveev " "Kornel (https://github.com/kornelski)" @@ -14803,9 +14856,9 @@ rec { }; "zerocopy" = rec { crateName = "zerocopy"; - version = "0.8.48"; + version = "0.8.50"; edition = "2021"; - sha256 = "1sb8plax8jbrsng1jdval7bdhk7hhrx40dz3hwh074k6knzkgm7f"; + sha256 = "1laahnfxs4qyfb1fdf5nbb2qfshi72b1hbi0ffp2zy2m1r7ms1iv"; authors = [ "Joshua Liebow-Feeser " "Jack Wrenn " @@ -14839,9 +14892,9 @@ rec { }; "zerocopy-derive" = rec { crateName = "zerocopy-derive"; - version = "0.8.48"; + version = "0.8.50"; edition = "2021"; - sha256 = "1m5s0g92cxggqc74j83k1priz24k3z93sj5gadppd20p9c4cvqvh"; + sha256 = "0fdnr9qslx1hbn2i9rsvy9s95mychfy2vj90ajsjm2basccinqqb"; procMacro = true; libName = "zerocopy_derive"; authors = [ @@ -14874,11 +14927,11 @@ rec { }; "zerofrom" = rec { crateName = "zerofrom"; - version = "0.1.7"; + version = "0.1.8"; edition = "2021"; - sha256 = "1py40in4rirc9q8w36q67pld0zk8ssg024xhh0cncxgal7ra3yk9"; + sha256 = "0wjjdj7gdmd0iq91gzkxl7dlv0nhkk80l4bmdpzh3a1yh48mmh0f"; authors = [ - "Manish Goregaokar " + "The ICU4X Project Developers" ]; dependencies = [ { diff --git a/crate-hashes.json b/crate-hashes.json index 71fbc1c3..c76bf06c 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#k8s-version@0.1.3": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-certs@0.4.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-operator-derive@0.3.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-operator@0.111.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-shared@0.1.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-telemetry@0.6.3": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-versioned-macros@0.10.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-versioned@0.10.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-webhook@0.9.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.0": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.3": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file From 17eaf7b712bd7d983a18f932c9f8fb79409b432a Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Jun 2026 16:45:56 +0200 Subject: [PATCH 15/39] chore: adapt changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 595701b9..238d3a45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ All notable changes to this project will be documented in this file. - Default `nifi.cluster.flow.election.max.wait.time` to NiFi's upstream value (`5 mins`) instead of the operator's previous `1 mins`. The operator no longer sets this property explicitly; the previous shorter value was left over from a TODO marked as "for testing" and may have caused flow election to settle on incomplete vote sets in cold-start scenarios ([#936]). - Set `nifi.content.repository.archive.max.retention.period` to `3 days` (previously empty, which NiFi interprets as `Long.MAX_VALUE` and effectively disables time-based archive purge). Without a time-based ceiling, the content archive can grow to half the content PVC and accumulate millions of files, which makes the synchronous startup directory scan in `FileSystemRepository.initializeRepository` very slow. Users requiring a longer content-replay window can extend via `configOverrides`. The provenance audit trail is independent of this setting and unaffected ([#936]). - test: Bump vector-aggregator to 0.55.0, replace /graphql call with gRPC call ([#940]). +- BREAKING: Removed product-config machinery. This is a breaking change in terms of configuration. + Users relying on the product-config `properties.yaml` file have to set these properties via the CRD ([#945]). ### Fixed @@ -34,6 +36,7 @@ All notable changes to this project will be documented in this file. [#935]: https://github.com/stackabletech/nifi-operator/pull/935 [#936]: https://github.com/stackabletech/nifi-operator/pull/936 [#940]: https://github.com/stackabletech/nifi-operator/pull/940 +[#945]: https://github.com/stackabletech/nifi-operator/pull/945 ## [26.3.0] - 2026-03-16 From d1472861dfd439cf4fe08ab7a521fc3f50c49b25 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 5 Jun 2026 17:13:16 +0200 Subject: [PATCH 16/39] refactor: consume the config-file writer from stackable-operator Replace the vendored Java-properties writer (rust/operator-binary/src/framework/writer.rs) with stackable_operator::v2::config_file_writer (moved there via operator-rs #1217 on the smooth-operator branch). NiFi's copy was a java-only subset of the canonical hdfs writer; the upstream module's additional to_hadoop_xml simply goes unimported. Drop the now-unused java-properties dependency. The framework module now only contains role_utils. The base dependency tag moves from stackable-operator-0.111.0 to 0.111.1, matching the other operators; cargo only substitutes a [patch] whose package version matches, and the smooth-operator branch carries 0.111.1. No behaviour change; rendered .properties output is byte-identical by construction (same code, new home). Co-Authored-By: Claude Opus 4.8 --- Cargo.lock | 89 ++++-- Cargo.nix | 301 ++++++++++-------- Cargo.toml | 3 +- crate-hashes.json | 4 +- rust/operator-binary/Cargo.toml | 1 - .../src/controller/build/config_map.rs | 2 +- .../src/controller/build/properties.rs | 2 +- .../build/properties/security_properties.rs | 10 +- rust/operator-binary/src/framework.rs | 1 - rust/operator-binary/src/framework/writer.rs | 78 ----- 10 files changed, 239 insertions(+), 252 deletions(-) delete mode 100644 rust/operator-binary/src/framework/writer.rs diff --git a/Cargo.lock b/Cargo.lock index 123bd3ef..10189f5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -771,7 +771,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1572,7 +1572,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "darling", "regex", @@ -1903,9 +1903,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "opentelemetry" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" dependencies = [ "futures-core", "futures-sink", @@ -1917,9 +1917,9 @@ dependencies = [ [[package]] name = "opentelemetry-appender-tracing" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef6a1ac5ca3accf562b8c306fa8483c85f4390f768185ab775f242f7fe8fdcc2" +checksum = "2c0080f0dc1d7c786f467cd85a4e395fcab11ee852004f39a29a18ab7c25d837" dependencies = [ "opentelemetry", "tracing", @@ -1929,9 +1929,9 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" +checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331" dependencies = [ "async-trait", "bytes", @@ -1942,9 +1942,9 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" +checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" dependencies = [ "http", "opentelemetry", @@ -1956,14 +1956,14 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tonic", - "tracing", + "tonic-types", ] [[package]] name = "opentelemetry-proto" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" +checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -1974,21 +1974,22 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e62e29dfe041afb8ed2a6c9737ab57db4907285d999ef8ad3a59092a36bdc846" +checksum = "6ca2f98a0437b427b4b08f19f1caa3c44db885a202bc12cfea13d6c702243d68" [[package]] name = "opentelemetry_sdk" -version = "0.31.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" +checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" dependencies = [ "futures-channel", "futures-executor", "futures-util", "opentelemetry", "percent-encoding", + "portable-atomic", "rand 0.9.4", "thiserror 2.0.18", "tokio", @@ -2281,6 +2282,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.45" @@ -2443,9 +2453,9 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" -version = "0.12.28" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64", "bytes", @@ -2461,9 +2471,6 @@ dependencies = [ "log", "percent-encoding", "pin-project-lite", - "serde", - "serde_json", - "serde_urlencoded", "sync_wrapper", "tokio", "tower", @@ -2983,7 +2990,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "const-oid", "ecdsa", @@ -3015,7 +3022,6 @@ dependencies = [ "fnv", "futures 0.3.32", "indoc", - "java-properties", "pin-project", "rand 0.10.1", "rstest", @@ -3034,7 +3040,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.111.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "base64", "clap", @@ -3046,6 +3052,7 @@ dependencies = [ "futures 0.3.32", "http", "indexmap", + "java-properties", "jiff", "json-patch", "k8s-openapi", @@ -3071,12 +3078,13 @@ dependencies = [ "tracing-subscriber", "url", "uuid", + "xml", ] [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "darling", "proc-macro2", @@ -3086,8 +3094,8 @@ dependencies = [ [[package]] name = "stackable-shared" -version = "0.1.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +version = "0.1.1" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "jiff", "k8s-openapi", @@ -3103,8 +3111,8 @@ dependencies = [ [[package]] name = "stackable-telemetry" -version = "0.6.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +version = "0.6.4" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "axum", "clap", @@ -3128,7 +3136,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "kube", "schemars", @@ -3142,7 +3150,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "convert_case", "convert_case_extras", @@ -3160,7 +3168,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "arc-swap", "async-trait", @@ -3512,6 +3520,17 @@ dependencies = [ "tonic", ] +[[package]] +name = "tonic-types" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab1b02061f83d519bba3caa167f88f261ef05720ab8ebc954ade70de3348e8" +dependencies = [ + "prost", + "prost-types", + "tonic", +] + [[package]] name = "tower" version = "0.5.3" @@ -3623,9 +3642,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.32.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac28f2d093c6c477eaa76b23525478f38de514fa9aeb1285738d4b97a9552fc" +checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" dependencies = [ "js-sys", "opentelemetry", diff --git a/Cargo.nix b/Cargo.nix index 45b7b36e..fe395a1c 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -2409,7 +2409,7 @@ rec { } { name = "windows-sys"; - packageId = "windows-sys 0.52.0"; + packageId = "windows-sys 0.61.2"; target = { target, features }: (target."windows" or false); features = [ "Win32_Foundation" "Win32_System_Diagnostics_Debug" ]; } @@ -5071,7 +5071,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "k8s_version"; @@ -6283,9 +6283,9 @@ rec { }; "opentelemetry" = rec { crateName = "opentelemetry"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "18629xsj4rsyiby9aj511q6wcw6s9m09gx3ymw1yjcvix1mcsjxq"; + sha256 = "10ln14d1jgc8rvw97mblc9blzcgpg1bimim4d170b7ia4mijq55h"; dependencies = [ { name = "futures-core"; @@ -6322,24 +6322,24 @@ rec { ]; features = { "default" = [ "trace" "metrics" "logs" "internal-logs" "futures" ]; + "experimental_metrics_bound_instruments" = [ "metrics" ]; "futures" = [ "futures-core" "futures-sink" "pin-project-lite" ]; "futures-core" = [ "dep:futures-core" ]; "futures-sink" = [ "dep:futures-sink" ]; "internal-logs" = [ "tracing" ]; "pin-project-lite" = [ "dep:pin-project-lite" ]; - "spec_unstable_logs_enabled" = [ "logs" ]; "testing" = [ "trace" ]; "thiserror" = [ "dep:thiserror" ]; "trace" = [ "futures" "thiserror" ]; "tracing" = [ "dep:tracing" ]; }; - resolvedDefaultFeatures = [ "default" "futures" "futures-core" "futures-sink" "internal-logs" "logs" "metrics" "pin-project-lite" "spec_unstable_logs_enabled" "thiserror" "trace" "tracing" ]; + resolvedDefaultFeatures = [ "default" "futures" "futures-core" "futures-sink" "internal-logs" "logs" "metrics" "pin-project-lite" "thiserror" "trace" "tracing" ]; }; "opentelemetry-appender-tracing" = rec { crateName = "opentelemetry-appender-tracing"; - version = "0.31.1"; + version = "0.32.0"; edition = "2021"; - sha256 = "1hnwizzgfhpjfnvml638yy846py8hf2gl1n3p1igbk1srb2ilspg"; + sha256 = "0dyq4myan64sl8wly02jx0gb3jjz7575mn3w8rpphz0xvkq8001c"; libName = "opentelemetry_appender_tracing"; dependencies = [ { @@ -6382,18 +6382,15 @@ rec { ]; features = { "experimental_metadata_attributes" = [ "dep:tracing-log" ]; - "experimental_use_tracing_span_context" = [ "tracing-opentelemetry" ]; "log" = [ "dep:log" ]; - "spec_unstable_logs_enabled" = [ "opentelemetry/spec_unstable_logs_enabled" ]; - "tracing-opentelemetry" = [ "dep:tracing-opentelemetry" ]; }; resolvedDefaultFeatures = [ "default" ]; }; "opentelemetry-http" = rec { crateName = "opentelemetry-http"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "0pc5nw1ds8v8w0nvyall39m92v8m1xl1p3vwvxk6nkhrffdd19np"; + sha256 = "0ca3drvm4fx5nskl7yn42dimy3bg35ppzc85y1p27pz215fh30sn"; libName = "opentelemetry_http"; dependencies = [ { @@ -6429,16 +6426,16 @@ rec { "internal-logs" = [ "opentelemetry/internal-logs" ]; "reqwest" = [ "dep:reqwest" ]; "reqwest-blocking" = [ "dep:reqwest" "reqwest/blocking" ]; - "reqwest-rustls" = [ "dep:reqwest" "reqwest/rustls-tls-native-roots" ]; - "reqwest-rustls-webpki-roots" = [ "dep:reqwest" "reqwest/rustls-tls-webpki-roots" ]; + "reqwest-rustls" = [ "dep:reqwest" "reqwest/default-tls" ]; + "reqwest-rustls-webpki-roots" = [ "dep:reqwest" "reqwest/default-tls" "reqwest/webpki-roots" ]; }; - resolvedDefaultFeatures = [ "internal-logs" "reqwest" "reqwest-blocking" ]; + resolvedDefaultFeatures = [ "reqwest" "reqwest-blocking" ]; }; "opentelemetry-otlp" = rec { crateName = "opentelemetry-otlp"; - version = "0.31.1"; + version = "0.32.0"; edition = "2021"; - sha256 = "07zp0b62b9dajnvvcd6j2ppw5zg7wp4ixka9z6fr3bxrrdmcss8z"; + sha256 = "0d9cys2flpidfxbr6h1103hjc633cax47ihnqgbj0xnicscr4rlr"; libName = "opentelemetry_otlp"; dependencies = [ { @@ -6499,10 +6496,9 @@ rec { usesDefaultFeatures = false; } { - name = "tracing"; - packageId = "tracing"; + name = "tonic-types"; + packageId = "tonic-types"; optional = true; - usesDefaultFeatures = false; } ]; devDependencies = [ @@ -6527,16 +6523,19 @@ rec { ]; features = { "default" = [ "http-proto" "reqwest-blocking-client" "trace" "metrics" "logs" "internal-logs" ]; + "experimental-grpc-retry" = [ "grpc-tonic" "opentelemetry_sdk/experimental_async_runtime" "opentelemetry_sdk/rt-tokio" ]; + "experimental-http-retry" = [ "opentelemetry_sdk/experimental_async_runtime" "opentelemetry_sdk/rt-tokio" "tokio" "httpdate" ]; "flate2" = [ "dep:flate2" ]; - "grpc-tonic" = [ "tonic" "prost" "http" "tokio" "opentelemetry-proto/gen-tonic" ]; + "grpc-tonic" = [ "tonic" "tonic-types" "prost" "http" "tokio" "opentelemetry-proto/gen-tonic" ]; "gzip-http" = [ "flate2" ]; "gzip-tonic" = [ "tonic/gzip" ]; "http" = [ "dep:http" ]; "http-json" = [ "serde_json" "prost" "opentelemetry-http" "opentelemetry-proto/gen-tonic-messages" "opentelemetry-proto/with-serde" "http" "trace" "metrics" ]; "http-proto" = [ "prost" "opentelemetry-http" "opentelemetry-proto/gen-tonic-messages" "http" "trace" "metrics" ]; + "httpdate" = [ "dep:httpdate" ]; "hyper-client" = [ "opentelemetry-http/hyper" ]; "integration-testing" = [ "tonic" "prost" "tokio/full" "trace" "logs" ]; - "internal-logs" = [ "tracing" "opentelemetry_sdk/internal-logs" "opentelemetry-http/internal-logs" ]; + "internal-logs" = [ "opentelemetry_sdk/internal-logs" "opentelemetry/internal-logs" ]; "logs" = [ "opentelemetry/logs" "opentelemetry_sdk/logs" "opentelemetry-proto/logs" ]; "metrics" = [ "opentelemetry/metrics" "opentelemetry_sdk/metrics" "opentelemetry-proto/metrics" ]; "opentelemetry-http" = [ "dep:opentelemetry-http" ]; @@ -6549,27 +6548,27 @@ rec { "serde" = [ "dep:serde" ]; "serde_json" = [ "dep:serde_json" ]; "serialize" = [ "serde" "serde_json" ]; - "tls" = [ "tonic/tls-ring" ]; + "tls" = [ "tls-ring" ]; "tls-aws-lc" = [ "tonic/tls-aws-lc" ]; "tls-provider-agnostic" = [ "tonic/_tls-any" ]; "tls-ring" = [ "tonic/tls-ring" ]; - "tls-roots" = [ "tls" "tonic/tls-native-roots" ]; - "tls-webpki-roots" = [ "tls" "tonic/tls-webpki-roots" ]; + "tls-roots" = [ "tonic/tls-native-roots" ]; + "tls-webpki-roots" = [ "tonic/tls-webpki-roots" ]; "tokio" = [ "dep:tokio" ]; "tonic" = [ "dep:tonic" ]; + "tonic-types" = [ "dep:tonic-types" ]; "trace" = [ "opentelemetry/trace" "opentelemetry_sdk/trace" "opentelemetry-proto/trace" ]; - "tracing" = [ "dep:tracing" ]; "zstd" = [ "dep:zstd" ]; "zstd-http" = [ "zstd" ]; "zstd-tonic" = [ "tonic/zstd" ]; }; - resolvedDefaultFeatures = [ "default" "grpc-tonic" "gzip-tonic" "http" "http-proto" "internal-logs" "logs" "metrics" "opentelemetry-http" "prost" "reqwest" "reqwest-blocking-client" "tokio" "tonic" "trace" "tracing" ]; + resolvedDefaultFeatures = [ "default" "grpc-tonic" "gzip-tonic" "http" "http-proto" "internal-logs" "logs" "metrics" "opentelemetry-http" "prost" "reqwest" "reqwest-blocking-client" "tokio" "tonic" "tonic-types" "trace" ]; }; "opentelemetry-proto" = rec { crateName = "opentelemetry-proto"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "03xkjsjrsm7zkkx5gascqd9bg2z20wymm06l16cyxsp5dpq5s5x7"; + sha256 = "0f5ny4rpnpq6q5q34b8k2q548rf31rpbxkwjqjwzfqxg3yx5imjn"; libName = "opentelemetry_proto"; dependencies = [ { @@ -6613,30 +6612,29 @@ rec { "const-hex" = [ "dep:const-hex" ]; "default" = [ "full" ]; "full" = [ "gen-tonic" "trace" "logs" "metrics" "zpages" "with-serde" "internal-logs" ]; - "gen-tonic" = [ "gen-tonic-messages" "tonic/channel" ]; - "gen-tonic-messages" = [ "tonic" "tonic-prost" "prost" ]; + "gen-tonic" = [ "gen-tonic-messages" "tonic" "tonic-prost" "tonic/channel" ]; + "gen-tonic-messages" = [ "prost" ]; "internal-logs" = [ "opentelemetry/internal-logs" ]; "logs" = [ "opentelemetry/logs" "opentelemetry_sdk/logs" ]; "metrics" = [ "opentelemetry/metrics" "opentelemetry_sdk/metrics" ]; "prost" = [ "dep:prost" ]; "schemars" = [ "dep:schemars" ]; "serde" = [ "dep:serde" ]; - "serde_json" = [ "dep:serde_json" ]; "testing" = [ "opentelemetry/testing" ]; "tonic" = [ "dep:tonic" ]; "tonic-prost" = [ "dep:tonic-prost" ]; "trace" = [ "opentelemetry/trace" "opentelemetry_sdk/trace" ]; "with-schemars" = [ "schemars" ]; - "with-serde" = [ "serde" "const-hex" "base64" "serde_json" ]; + "with-serde" = [ "serde" "const-hex" "base64" ]; "zpages" = [ "trace" ]; }; resolvedDefaultFeatures = [ "gen-tonic" "gen-tonic-messages" "logs" "metrics" "prost" "tonic" "tonic-prost" "trace" ]; }; "opentelemetry-semantic-conventions" = rec { crateName = "opentelemetry-semantic-conventions"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "0in8plv2l2ar7anzi7lrbll0fjfvaymkg5vc5bnvibs1w3gjjbp6"; + sha256 = "0s1x4h1cgmhkxb7i5g02la2vhkf4lg5g26cgn2s2gd1p0j5gk8kc"; libName = "opentelemetry_semantic_conventions"; features = { }; @@ -6644,9 +6642,9 @@ rec { }; "opentelemetry_sdk" = rec { crateName = "opentelemetry_sdk"; - version = "0.31.0"; + version = "0.32.1"; edition = "2021"; - sha256 = "1gbjsggdxfpjbanjvaxa3nq32vfa37i3v13dvx4gsxhrk7sy8jp1"; + sha256 = "1ycl11syranrinhgn4c2hlzhyzyvpa06ryxq5mxgzmf4387ghncv"; dependencies = [ { name = "futures-channel"; @@ -6672,6 +6670,13 @@ rec { packageId = "percent-encoding"; optional = true; } + { + name = "portable-atomic"; + packageId = "portable-atomic"; + usesDefaultFeatures = false; + target = { target, features }: (!("64" == target."has_atomic" or null)); + features = [ "fallback" ]; + } { name = "rand"; packageId = "rand 0.9.4"; @@ -6696,10 +6701,18 @@ rec { optional = true; } ]; + devDependencies = [ + { + name = "tokio"; + packageId = "tokio"; + usesDefaultFeatures = false; + features = [ "macros" "rt-multi-thread" ]; + } + ]; features = { "default" = [ "trace" "metrics" "logs" "internal-logs" ]; "experimental_logs_batch_log_processor_with_async_runtime" = [ "logs" "experimental_async_runtime" ]; - "experimental_logs_concurrent_log_processor" = [ "logs" ]; + "experimental_metrics_bound_instruments" = [ "metrics" "opentelemetry/experimental_metrics_bound_instruments" ]; "experimental_metrics_custom_reader" = [ "metrics" ]; "experimental_metrics_disable_name_validation" = [ "metrics" ]; "experimental_metrics_periodicreader_with_async_runtime" = [ "metrics" "experimental_async_runtime" ]; @@ -6716,15 +6729,14 @@ rec { "rt-tokio-current-thread" = [ "tokio/rt" "tokio/time" "tokio-stream" "experimental_async_runtime" ]; "serde" = [ "dep:serde" ]; "serde_json" = [ "dep:serde_json" ]; - "spec_unstable_logs_enabled" = [ "logs" "opentelemetry/spec_unstable_logs_enabled" ]; "spec_unstable_metrics_views" = [ "metrics" ]; - "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "rt-tokio" "rt-tokio-current-thread" "tokio/macros" "tokio/rt-multi-thread" ]; + "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "tokio/sync" ]; "tokio" = [ "dep:tokio" ]; "tokio-stream" = [ "dep:tokio-stream" ]; "trace" = [ "opentelemetry/trace" "rand" "percent-encoding" ]; "url" = [ "dep:url" ]; }; - resolvedDefaultFeatures = [ "default" "experimental_async_runtime" "internal-logs" "logs" "metrics" "percent-encoding" "rand" "rt-tokio" "spec_unstable_logs_enabled" "tokio" "tokio-stream" "trace" ]; + resolvedDefaultFeatures = [ "default" "experimental_async_runtime" "internal-logs" "logs" "metrics" "percent-encoding" "rand" "rt-tokio" "tokio" "tokio-stream" "trace" ]; }; "ordered-float" = rec { crateName = "ordered-float"; @@ -7233,7 +7245,7 @@ rec { "default" = [ "fallback" ]; "serde" = [ "dep:serde" ]; }; - resolvedDefaultFeatures = [ "require-cas" ]; + resolvedDefaultFeatures = [ "fallback" "require-cas" ]; }; "portable-atomic-util" = rec { crateName = "portable-atomic-util"; @@ -7540,6 +7552,34 @@ rec { ]; }; + "prost-types" = rec { + crateName = "prost-types"; + version = "0.14.3"; + edition = "2021"; + sha256 = "1mrxrciryfgi6a0vmrgyj3g27r9hdhlgwkq71cgv3icbvg5w94c9"; + libName = "prost_types"; + authors = [ + "Dan Burkert " + "Lucio Franco " + "Casper Meijn " + "Tokio Contributors " + ]; + dependencies = [ + { + name = "prost"; + packageId = "prost"; + usesDefaultFeatures = false; + features = [ "derive" ]; + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + "chrono" = [ "dep:chrono" ]; + "default" = [ "std" ]; + "std" = [ "prost/std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "quote" = rec { crateName = "quote"; version = "1.0.45"; @@ -8029,9 +8069,9 @@ rec { }; "reqwest" = rec { crateName = "reqwest"; - version = "0.12.28"; + version = "0.13.4"; edition = "2021"; - sha256 = "0iqidijghgqbzl3bjg5hb4zmigwa4r612bgi0yiq0c90b6jkrpgd"; + sha256 = "1hy1plns9krbh3h1dy2sdjygsfkdcnxm6pbxdi0ya9b5vq8mi711"; authors = [ "Sean McArthur " ]; @@ -8048,7 +8088,7 @@ rec { name = "futures-channel"; packageId = "futures-channel"; optional = true; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "futures-core"; @@ -8068,62 +8108,44 @@ rec { { name = "http-body"; packageId = "http-body"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "http-body-util"; packageId = "http-body-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "hyper"; packageId = "hyper"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "client" ]; } { name = "hyper-util"; packageId = "hyper-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "client" "client-legacy" "client-proxy" "tokio" ]; } { name = "js-sys"; packageId = "js-sys"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "log"; packageId = "log"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "percent-encoding"; packageId = "percent-encoding"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "pin-project-lite"; packageId = "pin-project-lite"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - } - { - name = "serde"; - packageId = "serde"; - } - { - name = "serde_json"; - packageId = "serde_json"; - optional = true; - } - { - name = "serde_json"; - packageId = "serde_json"; - target = { target, features }: ("wasm32" == target."arch" or null); - } - { - name = "serde_urlencoded"; - packageId = "serde_urlencoded"; + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "sync_wrapper"; @@ -8134,27 +8156,27 @@ rec { name = "tokio"; packageId = "tokio"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "net" "time" ]; } { name = "tower"; packageId = "tower"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "retry" "timeout" "util" ]; } { name = "tower-http"; packageId = "tower-http"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "follow-redirect" ]; } { name = "tower-service"; packageId = "tower-service"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "url"; @@ -8163,17 +8185,17 @@ rec { { name = "wasm-bindgen"; packageId = "wasm-bindgen"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "wasm-bindgen-futures"; packageId = "wasm-bindgen-futures"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "web-sys"; packageId = "web-sys"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); features = [ "AbortController" "AbortSignal" "Headers" "Request" "RequestInit" "RequestMode" "Response" "Window" "FormData" "Blob" "BlobPropertyBag" "ServiceWorkerGlobalScope" "RequestCredentials" "File" "ReadableStream" "RequestCache" ]; } ]; @@ -8182,33 +8204,27 @@ rec { name = "futures-util"; packageId = "futures-util"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "std" "alloc" ]; } { name = "hyper"; packageId = "hyper"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "http2" "client" "server" ]; } { name = "hyper-util"; packageId = "hyper-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "http2" "client" "client-legacy" "server-auto" "server-graceful" "tokio" ]; } - { - name = "serde"; - packageId = "serde"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - features = [ "derive" ]; - } { name = "tokio"; packageId = "tokio"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "macros" "rt-multi-thread" ]; } { @@ -8220,40 +8236,37 @@ rec { { name = "wasm-bindgen"; packageId = "wasm-bindgen"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); features = [ "serde-serialize" ]; } ]; features = { + "__native-tls" = [ "dep:hyper-tls" "dep:native-tls-crate" "__tls" "dep:tokio-native-tls" ]; + "__native-tls-alpn" = [ "native-tls-crate?/alpn" "hyper-tls?/alpn" ]; "__rustls" = [ "dep:hyper-rustls" "dep:tokio-rustls" "dep:rustls" "__tls" ]; - "__rustls-ring" = [ "hyper-rustls?/ring" "tokio-rustls?/ring" "rustls?/ring" "quinn?/ring" ]; + "__rustls-aws-lc-rs" = [ "hyper-rustls?/aws-lc-rs" "tokio-rustls?/aws-lc-rs" "rustls?/aws-lc-rs" "quinn?/rustls-aws-lc-rs" ]; "__tls" = [ "dep:rustls-pki-types" "tokio/io-util" ]; "blocking" = [ "dep:futures-channel" "futures-channel?/sink" "dep:futures-util" "futures-util?/io" "futures-util?/sink" "tokio/sync" ]; "brotli" = [ "tower-http/decompression-br" ]; "charset" = [ "dep:encoding_rs" "dep:mime" ]; "cookies" = [ "dep:cookie_crate" "dep:cookie_store" ]; "default" = [ "default-tls" "charset" "http2" "system-proxy" ]; - "default-tls" = [ "dep:hyper-tls" "dep:native-tls-crate" "__tls" "dep:tokio-native-tls" ]; + "default-tls" = [ "rustls" ]; "deflate" = [ "tower-http/decompression-deflate" ]; + "form" = [ "dep:serde" "dep:serde_urlencoded" ]; "gzip" = [ "tower-http/decompression-gzip" ]; - "h2" = [ "dep:h2" ]; "hickory-dns" = [ "dep:hickory-resolver" "dep:once_cell" ]; - "http2" = [ "h2" "hyper/http2" "hyper-util/http2" "hyper-rustls?/http2" ]; - "http3" = [ "rustls-tls-manual-roots" "dep:h3" "dep:h3-quinn" "dep:quinn" "tokio/macros" ]; - "json" = [ "dep:serde_json" ]; - "macos-system-configuration" = [ "system-proxy" ]; + "http2" = [ "dep:h2" "hyper/http2" "hyper-util/http2" "hyper-rustls?/http2" ]; + "http3" = [ "rustls" "dep:h3" "dep:h3-quinn" "dep:quinn" "tokio/macros" ]; + "json" = [ "dep:serde" "dep:serde_json" ]; "multipart" = [ "dep:mime_guess" "dep:futures-util" ]; - "native-tls" = [ "default-tls" ]; - "native-tls-alpn" = [ "native-tls" "native-tls-crate?/alpn" "hyper-tls?/alpn" ]; - "native-tls-vendored" = [ "native-tls" "native-tls-crate?/vendored" ]; - "rustls-tls" = [ "rustls-tls-webpki-roots" ]; - "rustls-tls-manual-roots" = [ "rustls-tls-manual-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-manual-roots-no-provider" = [ "__rustls" ]; - "rustls-tls-native-roots" = [ "rustls-tls-native-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-native-roots-no-provider" = [ "dep:rustls-native-certs" "hyper-rustls?/native-tokio" "__rustls" ]; - "rustls-tls-no-provider" = [ "rustls-tls-manual-roots-no-provider" ]; - "rustls-tls-webpki-roots" = [ "rustls-tls-webpki-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-webpki-roots-no-provider" = [ "dep:webpki-roots" "hyper-rustls?/webpki-tokio" "__rustls" ]; + "native-tls" = [ "__native-tls" "__native-tls-alpn" ]; + "native-tls-no-alpn" = [ "__native-tls" ]; + "native-tls-vendored" = [ "__native-tls" "native-tls-crate?/vendored" "__native-tls-alpn" ]; + "native-tls-vendored-no-alpn" = [ "__native-tls" "native-tls-crate?/vendored" ]; + "query" = [ "dep:serde" "dep:serde_urlencoded" ]; + "rustls" = [ "__rustls-aws-lc-rs" "dep:rustls-platform-verifier" "__rustls" ]; + "rustls-no-provider" = [ "dep:rustls-platform-verifier" "__rustls" ]; "stream" = [ "tokio/fs" "dep:futures-util" "dep:tokio-util" "dep:wasm-streams" ]; "system-proxy" = [ "hyper-util/client-proxy-system" ]; "zstd" = [ "tower-http/decompression-zstd" ]; @@ -9852,7 +9865,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_certs"; @@ -9989,10 +10002,6 @@ rec { name = "indoc"; packageId = "indoc"; } - { - name = "java-properties"; - packageId = "java-properties"; - } { name = "pin-project"; packageId = "pin-project"; @@ -10068,7 +10077,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_operator"; @@ -10119,6 +10128,10 @@ rec { name = "indexmap"; packageId = "indexmap"; } + { + name = "java-properties"; + packageId = "java-properties"; + } { name = "jiff"; packageId = "jiff"; @@ -10234,12 +10247,17 @@ rec { name = "uuid"; packageId = "uuid"; } + { + name = "xml"; + packageId = "xml"; + } ]; features = { "certs" = [ "dep:stackable-certs" ]; + "client-feature-gates" = [ "dep:winnow" ]; "crds" = [ "dep:stackable-versioned" ]; "default" = [ "crds" ]; - "full" = [ "crds" "certs" "time" "webhook" "kube-ws" ]; + "full" = [ "client-feature-gates" "crds" "certs" "time" "webhook" "kube-ws" ]; "kube-ws" = [ "kube/ws" ]; "time" = [ "stackable-shared/time" ]; "webhook" = [ "dep:stackable-webhook" ]; @@ -10253,7 +10271,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; procMacro = true; @@ -10283,12 +10301,12 @@ rec { }; "stackable-shared" = rec { crateName = "stackable-shared"; - version = "0.1.0"; + version = "0.1.1"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_shared"; @@ -10364,12 +10382,12 @@ rec { }; "stackable-telemetry" = rec { crateName = "stackable-telemetry"; - version = "0.6.3"; + version = "0.6.4"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_telemetry"; @@ -10413,7 +10431,7 @@ rec { { name = "opentelemetry_sdk"; packageId = "opentelemetry_sdk"; - features = [ "rt-tokio" "logs" "rt-tokio" "spec_unstable_logs_enabled" ]; + features = [ "rt-tokio" "logs" "rt-tokio" ]; } { name = "pin-project"; @@ -10479,7 +10497,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_versioned"; @@ -10529,7 +10547,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; procMacro = true; @@ -10597,7 +10615,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_webhook"; @@ -11761,6 +11779,33 @@ rec { } ]; + }; + "tonic-types" = rec { + crateName = "tonic-types"; + version = "0.14.6"; + edition = "2024"; + sha256 = "1s286gg71pjajny8xar0azq1w9lgz1ks3jm3pccxb0qz0q11pavk"; + libName = "tonic_types"; + authors = [ + "Lucio Franco " + "Rafael Lemos " + ]; + dependencies = [ + { + name = "prost"; + packageId = "prost"; + } + { + name = "prost-types"; + packageId = "prost-types"; + } + { + name = "tonic"; + packageId = "tonic"; + usesDefaultFeatures = false; + } + ]; + }; "tower" = rec { crateName = "tower"; @@ -12215,9 +12260,9 @@ rec { }; "tracing-opentelemetry" = rec { crateName = "tracing-opentelemetry"; - version = "0.32.1"; + version = "0.33.0"; edition = "2021"; - sha256 = "1z2jjmxbkm1qawlb3bm99x8xwf4g8wjkbcknm9z4fv1w14nqzhhs"; + sha256 = "09nvxy5m7nxmifz4b6szdcyczapp2jcgxcac0jw4ax8klz5n9g5d"; libName = "tracing_opentelemetry"; dependencies = [ { @@ -13955,7 +14000,7 @@ rec { "Win32_Web" = [ "Win32" ]; "Win32_Web_InternetExplorer" = [ "Win32_Web" ]; }; - resolvedDefaultFeatures = [ "Win32" "Win32_Foundation" "Win32_System" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_Threading" "default" ]; + resolvedDefaultFeatures = [ "Win32" "Win32_Foundation" "Win32_System" "Win32_System_Threading" "default" ]; }; "windows-sys 0.61.2" = rec { crateName = "windows-sys"; @@ -14217,7 +14262,7 @@ rec { "Win32_Web" = [ "Win32" ]; "Win32_Web_InternetExplorer" = [ "Win32_Web" ]; }; - resolvedDefaultFeatures = [ "Wdk" "Wdk_Foundation" "Wdk_Storage" "Wdk_Storage_FileSystem" "Wdk_System" "Wdk_System_IO" "Win32" "Win32_Foundation" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Console" "Win32_System_IO" "Win32_System_LibraryLoader" "Win32_System_Memory" "Win32_System_Pipes" "Win32_System_SystemInformation" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_WindowsProgramming" "default" ]; + resolvedDefaultFeatures = [ "Wdk" "Wdk_Foundation" "Wdk_Storage" "Wdk_Storage_FileSystem" "Wdk_System" "Wdk_System_IO" "Win32" "Win32_Foundation" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_IO" "Win32_System_LibraryLoader" "Win32_System_Memory" "Win32_System_Pipes" "Win32_System_SystemInformation" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_WindowsProgramming" "default" ]; }; "windows-targets" = rec { crateName = "windows-targets"; diff --git a/Cargo.toml b/Cargo.toml index ab7f24b3..63b5274d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ edition = "2021" repository = "https://github.com/stackabletech/nifi-operator" [workspace.dependencies] -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.111.0", features = ["webhook"] } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.111.1", features = ["webhook"] } anyhow = "1.0" built = { version = "0.8", features = ["chrono", "git2"] } @@ -19,7 +19,6 @@ const_format = "0.2" fnv = "1.0" futures = { version = "0.3", features = ["compat"] } indoc = "2.0" -java-properties = "2.0" pin-project = "1.1" rand = "0.10" rstest = "0.26" diff --git a/crate-hashes.json b/crate-hashes.json index c76bf06c..cd03561e 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -3,8 +3,8 @@ "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.0": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.3": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 2539a0d4..b52ad2a9 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -17,7 +17,6 @@ const_format.workspace = true fnv.workspace = true futures.workspace = true indoc.workspace = true -java-properties.workspace = true pin-project.workspace = true rand.workspace = true semver.workspace = true diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index c5d4d926..9c08099f 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -53,7 +53,7 @@ pub enum Error { #[snafu(display("failed to serialize JVM security properties for {}", rolegroup))] JvmSecurityProperties { - source: crate::framework::writer::PropertiesWriterError, + source: stackable_operator::v2::config_file_writer::PropertiesWriterError, rolegroup: String, }, diff --git a/rust/operator-binary/src/controller/build/properties.rs b/rust/operator-binary/src/controller/build/properties.rs index 81d8969b..4f9cf3a9 100644 --- a/rust/operator-binary/src/controller/build/properties.rs +++ b/rust/operator-binary/src/controller/build/properties.rs @@ -1,7 +1,7 @@ //! Per-file builders for the NiFi rolegroup ConfigMap. //! //! Each `` module produces the rendered content for one NiFi config file. -//! The shared [`crate::framework::writer`] module serializes `.properties`/`.conf` +//! The shared [`stackable_operator::v2::config_file_writer`] module serializes `.properties`/`.conf` //! key/value maps to the Java-properties on-wire format. use std::collections::BTreeMap; diff --git a/rust/operator-binary/src/controller/build/properties/security_properties.rs b/rust/operator-binary/src/controller/build/properties/security_properties.rs index 97c9304c..8b857eae 100644 --- a/rust/operator-binary/src/controller/build/properties/security_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/security_properties.rs @@ -2,10 +2,14 @@ use std::collections::BTreeMap; +use stackable_operator::v2::config_file_writer::{ + PropertiesWriterError, to_java_properties_string, +}; + use super::ConfigFileName; -use crate::{controller::validate::NifiRoleGroupConfig, framework::writer}; +use crate::controller::validate::NifiRoleGroupConfig; -pub fn build(rg: &NifiRoleGroupConfig) -> Result { +pub fn build(rg: &NifiRoleGroupConfig) -> Result { let mut props: BTreeMap> = BTreeMap::new(); // Defaults previously injected by deploy/config-spec/properties.yaml: props.insert( @@ -19,7 +23,7 @@ pub fn build(rg: &NifiRoleGroupConfig) -> Result(properties: T) -> Result -where - T: Iterator)>, -{ - let mut output = Vec::new(); - write_java_properties(&mut output, properties)?; - String::from_utf8(output).context(FromUtf8Snafu) -} - -/// Writes Java properties to the given writer. A `None` value is written as an -/// empty value (`key=`). -fn write_java_properties<'a, W, T>(writer: W, properties: T) -> Result<(), PropertiesWriterError> -where - W: Write, - T: Iterator)>, -{ - let mut writer = PropertiesWriter::new(writer); - for (k, v) in properties { - let property_value = v.as_deref().unwrap_or_default(); - writer.write(k, property_value).context(PropertiesSnafu)?; - } - writer.flush().context(PropertiesSnafu)?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use std::collections::BTreeMap; - - use super::*; - - fn props(pairs: &[(&str, Option<&str>)]) -> String { - let map: BTreeMap> = pairs - .iter() - .map(|(k, v)| (k.to_string(), v.map(str::to_string))) - .collect(); - to_java_properties_string(map.iter()).unwrap() - } - - #[test] - fn java_properties_renders_key_value() { - assert_eq!(props(&[("a", Some("1")), ("b", Some("2"))]), "a=1\nb=2\n"); - } - - #[test] - fn java_properties_renders_none_as_empty() { - assert_eq!(props(&[("none", None)]), "none=\n"); - } - - #[test] - fn java_properties_escapes_colon_in_value() { - assert_eq!( - props(&[("url", Some("file://this/location/file.abc"))]), - "url=file\\://this/location/file.abc\n" - ); - } -} From 15205384d1cba26efe2e83fc4d81c16b631e6bba Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 8 Jun 2026 15:47:54 +0200 Subject: [PATCH 17/39] deps: remove xml-rs dependency --- Cargo.lock | 32 ++++++++++++++++---------------- Cargo.toml | 1 - 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 10189f5f..6c8910d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -265,9 +265,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "2.12.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84d7ced0ae9557296835c32bf1b1e02b44c746701f898460fb000d7eaa84f00a" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" [[package]] name = "block-buffer" @@ -331,9 +331,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "num-traits", @@ -1135,9 +1135,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes", "itoa", @@ -1770,9 +1770,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.31" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "113b30b4cd05f7c06868fdb2854f66a7b9fece9a48425351cd532e810d74024f" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" [[package]] name = "matchers" @@ -2261,9 +2261,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ "bytes", "prost-derive", @@ -2271,9 +2271,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", "itertools", @@ -2284,9 +2284,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" dependencies = [ "prost", ] @@ -4202,9 +4202,9 @@ checksum = "636f85e5ca6488e96401b61eb7de54f4e44755c988af0f52cf90230c312a1a89" [[package]] name = "yoke" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive", diff --git a/Cargo.toml b/Cargo.toml index 63b5274d..e6424d41 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,6 @@ strum = { version = "0.28", features = ["derive"] } tokio = { version = "1.40", features = ["full"] } tracing = "0.1" url = { version = "2.5.7" } -xml-rs = "1.0" [patch."https://github.com/stackabletech/operator-rs.git"] stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "smooth-operator"} From abde44300f3b3a718563d9ed33d5593bace06e37 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 8 Jun 2026 17:13:25 +0200 Subject: [PATCH 18/39] refactor(1): use NamespaceName and remove optional namespaces --- Cargo.nix | 32 +++++++++---------- rust/operator-binary/src/controller.rs | 17 ++++------ .../src/controller/build/properties.rs | 4 ++- .../src/controller/dereference.rs | 25 +++++++++------ .../src/controller/validate.rs | 21 ++++++------ .../operator-binary/src/operations/upgrade.rs | 15 +++------ .../operator-binary/src/reporting_task/mod.rs | 17 +++++----- rust/operator-binary/src/security/mod.rs | 8 +++-- rust/operator-binary/src/security/oidc.rs | 13 ++++---- .../src/security/sensitive_key.rs | 11 +++---- 10 files changed, 79 insertions(+), 84 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index fe395a1c..b2482245 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -866,9 +866,9 @@ rec { }; "bitflags" = rec { crateName = "bitflags"; - version = "2.12.1"; + version = "2.13.0"; edition = "2021"; - sha256 = "02phhjm7w380zdh8928zf13cfi1bw2qz2ay36ml2jmwmmv8cxmw4"; + sha256 = "1y239gpvl061rfvav7jds8mjs42kmwi39is7yx5d1qw3hvp8nf5l"; authors = [ "The Rust Project Developers" ]; @@ -1047,9 +1047,9 @@ rec { }; "chrono" = rec { crateName = "chrono"; - version = "0.4.44"; + version = "0.4.45"; edition = "2021"; - sha256 = "1c64mk9a235271j5g3v4zrzqqmd43vp9vki7vqfllpqf5rd0fwy6"; + sha256 = "09rkcgk6is2sdhqs9142zv8xqnj8ryx8m9hknllqwyv9wxi9x9qs"; dependencies = [ { name = "iana-time-zone"; @@ -3604,9 +3604,9 @@ rec { }; "http" = rec { crateName = "http"; - version = "1.4.1"; + version = "1.4.2"; edition = "2021"; - sha256 = "1l7k2ia57z3q7q3ka497krzps795kd3fymm2k12lr623y4nldrwb"; + sha256 = "09b4p8fiivkg7wm0b59fyrn1jkm7px298ci7zb9igz6n647gaw39"; authors = [ "Alex Crichton " "Carl Lerche " @@ -5891,9 +5891,9 @@ rec { }; "log" = rec { crateName = "log"; - version = "0.4.31"; + version = "0.4.32"; edition = "2021"; - sha256 = "0kq2fh6q2bjkrm8m6hj8kb7gxfd7cr7qbcpxd1lc1xq5rns30fqi"; + sha256 = "0fmdg0cxig7i4fwf1sw7fmg4d1gdbfzniawcfpwydy1q7320fgwm"; authors = [ "The Rust Project Developers" ]; @@ -7487,9 +7487,9 @@ rec { }; "prost" = rec { crateName = "prost"; - version = "0.14.3"; + version = "0.14.4"; edition = "2021"; - sha256 = "0s057z9nzggzy7x4bbsiar852hg7zb81f4z4phcdb0ig99971snj"; + sha256 = "1qas5v5rap45f43v3ja0jngxrrafrkcwl0iw5a3ld1pz2rscd2jj"; authors = [ "Dan Burkert " "Lucio Franco " @@ -7516,9 +7516,9 @@ rec { }; "prost-derive" = rec { crateName = "prost-derive"; - version = "0.14.3"; + version = "0.14.4"; edition = "2021"; - sha256 = "02zvva6kb0pfvlyc4nac6gd37ncjrs8jq5scxcq4nbqkc8wh5ii7"; + sha256 = "1pqa77d7da5pf6ba3kjj7510m5cynz6902ax01ckvr0pfrgv4w5m"; procMacro = true; libName = "prost_derive"; authors = [ @@ -7554,9 +7554,9 @@ rec { }; "prost-types" = rec { crateName = "prost-types"; - version = "0.14.3"; + version = "0.14.4"; edition = "2021"; - sha256 = "1mrxrciryfgi6a0vmrgyj3g27r9hdhlgwkq71cgv3icbvg5w94c9"; + sha256 = "02ivjvc4cwl5bfgjs3l00hwlrk74z8zlg1xcgx60bww8fvf6fjgr"; libName = "prost_types"; authors = [ "Dan Burkert " @@ -14835,9 +14835,9 @@ rec { }; "yoke" = rec { crateName = "yoke"; - version = "0.8.2"; + version = "0.8.3"; edition = "2021"; - sha256 = "1jprcs7a98a5whvfs6r3jvfh1nnfp6zyijl7y4ywmn88lzywbs5b"; + sha256 = "1xgyj6c2lxj2bp891ynmhws87c6z7yyv2li1v0ss9di40hxf57vh"; authors = [ "Manish Goregaokar " ]; diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index cfb9869b..0dc433e1 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -57,6 +57,7 @@ use stackable_operator::{ statefulset::StatefulSetConditionBuilder, }, utils::{COMMON_BASH_TRAP_FUNCTIONS, cluster_info::KubernetesClusterInfo}, + v2::types::kubernetes::NamespaceName, }; use strum::{EnumDiscriminants, IntoStaticStr}; use tracing::Instrument; @@ -123,9 +124,6 @@ pub enum Error { #[snafu(display("object defines no name"))] ObjectHasNoName, - #[snafu(display("object defines no namespace"))] - ObjectHasNoNamespace, - #[snafu(display("failed to dereference resources"))] Dereference { source: dereference::Error }, @@ -332,7 +330,7 @@ pub async fn reconcile_nifi( let authorization_config = &validated_cluster.cluster_config.authorization; tracing::info!("Checking for sensitive key configuration"); - check_or_generate_sensitive_key(client, nifi) + check_or_generate_sensitive_key(client, nifi, &validated_cluster.namespace) .await .context(SecuritySnafu)?; @@ -352,6 +350,7 @@ pub async fn reconcile_nifi( cluster_version_update_state = upgrade::cluster_version_update_state( nifi, client, + &validated_cluster.namespace, &resolved_product_image.product_version, deployed_version, ) @@ -375,7 +374,7 @@ pub async fn reconcile_nifi( .context(CreateClusterResourcesSnafu)?; if let NifiAuthenticationConfig::Oidc { .. } = authentication_config { - check_or_generate_oidc_admin_password(client, nifi) + check_or_generate_oidc_admin_password(client, nifi, &validated_cluster.namespace) .await .context(SecuritySnafu)?; } @@ -470,6 +469,7 @@ pub async fn reconcile_nifi( nifi, resolved_product_image, &client.kubernetes_cluster_info, + &validated_cluster.namespace, &rolegroup, role, rg, @@ -568,6 +568,7 @@ pub async fn reconcile_nifi( nifi, resolved_product_image, &client.kubernetes_cluster_info, + &validated_cluster.namespace, authentication_config, &rbac_sa.name_any(), ) @@ -648,6 +649,7 @@ async fn build_node_rolegroup_statefulset( nifi: &v1alpha1::NifiCluster, resolved_product_image: &ResolvedProductImage, cluster_info: &KubernetesClusterInfo, + namespace: &NamespaceName, rolegroup_ref: &RoleGroupRef, role: &NifiRoleType, rg: &NifiRoleGroupConfig, @@ -723,11 +725,6 @@ async fn build_node_rolegroup_statefulset( let node_address = format!( "$POD_NAME.{service_name}.{namespace}.svc.{cluster_domain}", service_name = rolegroup_ref.rolegroup_headless_service_name(), - namespace = &nifi - .metadata - .namespace - .as_ref() - .context(ObjectHasNoNamespaceSnafu)?, cluster_domain = cluster_info.cluster_domain, ); diff --git a/rust/operator-binary/src/controller/build/properties.rs b/rust/operator-binary/src/controller/build/properties.rs index 4f9cf3a9..795a5d49 100644 --- a/rust/operator-binary/src/controller/build/properties.rs +++ b/rust/operator-binary/src/controller/build/properties.rs @@ -79,7 +79,7 @@ pub(crate) fn resolved_overrides_for( /// self-contained. #[cfg(test)] pub(crate) mod test_support { - use std::collections::BTreeMap; + use std::{collections::BTreeMap, str::FromStr as _}; use stackable_operator::{ commons::product_image_selection::ResolvedProductImage, @@ -88,6 +88,7 @@ pub(crate) mod test_support { }, kube::ResourceExt as _, kvp::LabelValue, + v2::types::kubernetes::NamespaceName, }; use crate::{ @@ -161,6 +162,7 @@ pub(crate) mod test_support { ValidatedCluster { name: "simple-nifi".to_string(), + namespace: NamespaceName::from_str("default").expect("valid namespace"), image, role_group_configs, git_sync_resources: Default::default(), diff --git a/rust/operator-binary/src/controller/dereference.rs b/rust/operator-binary/src/controller/dereference.rs index 4a394707..a5d31cb6 100644 --- a/rust/operator-binary/src/controller/dereference.rs +++ b/rust/operator-binary/src/controller/dereference.rs @@ -3,8 +3,14 @@ //! Fetches all Kubernetes objects referenced by the NifiCluster spec and returns //! them in [`DereferencedObjects`]. -use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_operator::client::Client; +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + client::Client, + v2::{ + controller_utils::{self, get_namespace}, + types::kubernetes::NamespaceName, + }, +}; use crate::{ crd::v1alpha1, @@ -16,8 +22,8 @@ use crate::{ #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("object defines no namespace"))] - ObjectHasNoNamespace, + #[snafu(display("failed to get the namespace"))] + GetNamespace { source: controller_utils::Error }, #[snafu(display("failed to dereference NiFi authentication classes"))] DereferenceAuthenticationClasses { source: authentication::Error }, @@ -30,6 +36,8 @@ type Result = std::result::Result; /// Kubernetes objects referenced from the [`v1alpha1::NifiCluster`] spec, already fetched. pub struct DereferencedObjects { + /// The namespace of the [`v1alpha1::NifiCluster`], parsed once here and reused everywhere. + pub namespace: NamespaceName, pub authentication_classes: DereferencedAuthenticationClasses, pub authorization: DereferencedAuthorization, } @@ -39,11 +47,7 @@ pub async fn dereference( client: &Client, nifi: &v1alpha1::NifiCluster, ) -> Result { - let namespace = nifi - .metadata - .namespace - .as_deref() - .context(ObjectHasNoNamespaceSnafu)?; + let namespace = get_namespace(nifi).context(GetNamespaceSnafu)?; let authentication_classes = DereferencedAuthenticationClasses::dereference(nifi, client) .await @@ -52,12 +56,13 @@ pub async fn dereference( let authorization = DereferencedAuthorization::dereference( &nifi.spec.cluster_config.authorization, client, - namespace, + namespace.as_ref(), ) .await .context(DereferenceAuthorizationSnafu)?; Ok(DereferencedObjects { + namespace, authentication_classes, authorization, }) diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 7b0270d1..b306acc6 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -13,6 +13,7 @@ use stackable_operator::{ kube::ResourceExt as _, role_utils::JavaCommonConfig, utils::cluster_info::KubernetesClusterInfo, + v2::types::kubernetes::NamespaceName, }; use strum::{EnumDiscriminants, IntoStaticStr}; @@ -45,11 +46,6 @@ pub enum Error { #[snafu(display("invalid NiFi authentication configuration"))] InvalidAuthenticationConfig { source: authentication::Error }, - #[snafu(display("failed to build reporting task service name"))] - ReportingTask { - source: crate::reporting_task::Error, - }, - #[snafu(display("failed to validate config fragment for a rolegroup"))] InvalidConfigFragment { source: stackable_operator::config::fragment::ValidationError, @@ -76,6 +72,8 @@ type Result = std::result::Result; pub struct ValidatedCluster { #[allow(dead_code)] pub name: String, + /// The namespace of the NifiCluster, parsed once in the dereference step and reused everywhere. + pub namespace: NamespaceName, pub image: ResolvedProductImage, pub role_group_configs: BTreeMap>, /// The git-sync resources (volumes, mounts, containers) for each Node rolegroup, @@ -122,7 +120,7 @@ pub fn validate( &dereferenced_objects.authorization, ); - let proxy_hosts = compute_proxy_hosts(nifi, cluster_info)?; + let proxy_hosts = compute_proxy_hosts(nifi, cluster_info, &dereferenced_objects.namespace); let sensitive_properties_algorithm = nifi .spec @@ -140,6 +138,7 @@ pub fn validate( Ok(ValidatedCluster { name: nifi.name_any(), + namespace: dereferenced_objects.namespace.clone(), image, role_group_configs, git_sync_resources, @@ -203,7 +202,8 @@ fn build_git_sync_resources( fn compute_proxy_hosts( nifi: &v1alpha1::NifiCluster, cluster_info: &KubernetesClusterInfo, -) -> Result { + namespace: &NamespaceName, +) -> String { let host_header_check = &nifi.spec.cluster_config.host_header_check; if host_header_check.allow_all { @@ -215,7 +215,7 @@ fn compute_proxy_hosts( "spec.clusterConfig.hostHeaderCheck.additionalAllowedHosts is ignored and only '*' is added to the allow-list." ) } - return Ok("*".to_string()); + return "*".to_string(); } // Address and port are injected from the listener volume during the prepare container @@ -227,8 +227,7 @@ fn compute_proxy_hosts( // Reporting task only exists for NiFi 1.x if nifi.spec.image.product_version().starts_with("1.") { let reporting_task_service_name = - reporting_task::build_reporting_task_fqdn_service_name(nifi, cluster_info) - .context(ReportingTaskSnafu)?; + reporting_task::build_reporting_task_fqdn_service_name(nifi, cluster_info, namespace); proxy_hosts.insert(format!("{reporting_task_service_name}:{HTTPS_PORT}")); } @@ -236,5 +235,5 @@ fn compute_proxy_hosts( let mut proxy_hosts = Vec::from_iter(proxy_hosts); proxy_hosts.sort(); - Ok(proxy_hosts.join(",")) + proxy_hosts.join(",") } diff --git a/rust/operator-binary/src/operations/upgrade.rs b/rust/operator-binary/src/operations/upgrade.rs index 227405ba..53183bd0 100644 --- a/rust/operator-binary/src/operations/upgrade.rs +++ b/rust/operator-binary/src/operations/upgrade.rs @@ -1,20 +1,18 @@ // TODO: This module can be removed once we don't support NiFi 1.x versions anymore // It manages the version upgrade procedure for NiFi versions prior to NiFi 2, since rolling upgrade is not supported there yet -use snafu::{OptionExt, ResultExt, Snafu}; +use snafu::{ResultExt, Snafu}; use stackable_operator::{ client::Client, k8s_openapi::{api::apps::v1::StatefulSet, apimachinery::pkg::apis::meta::v1::LabelSelector}, kvp::Labels, + v2::types::kubernetes::NamespaceName, }; use crate::crd::{APP_NAME, NifiRole, v1alpha1}; #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("object defines no namespace"))] - ObjectHasNoNamespace, - #[snafu(display("failed to fetch deployed StatefulSets"))] FetchStatefulsets { source: stackable_operator::client::Error, @@ -41,15 +39,10 @@ pub enum ClusterVersionUpdateState { pub async fn cluster_version_update_state( nifi: &v1alpha1::NifiCluster, client: &Client, + namespace: &NamespaceName, resolved_version: &String, deployed_version: Option<&String>, ) -> Result { - let namespace = &nifi - .metadata - .namespace - .clone() - .with_context(|| ObjectHasNoNamespaceSnafu {})?; - // Handle full restarts for a version change match deployed_version { Some(deployed_version) => { @@ -66,7 +59,7 @@ pub async fn cluster_version_update_state( // Retrieve the deployed statefulsets to check on the current status of the restart let deployed_statefulsets = client - .list_with_label_selector::(namespace, &selector) + .list_with_label_selector::(namespace.as_ref(), &selector) .await .context(FetchStatefulsetsSnafu)?; diff --git a/rust/operator-binary/src/reporting_task/mod.rs b/rust/operator-binary/src/reporting_task/mod.rs index cdee4c54..d92984d3 100644 --- a/rust/operator-binary/src/reporting_task/mod.rs +++ b/rust/operator-binary/src/reporting_task/mod.rs @@ -46,6 +46,7 @@ use stackable_operator::{ kvp::Labels, shared::time::Duration, utils::cluster_info::KubernetesClusterInfo, + v2::types::kubernetes::NamespaceName, }; use crate::{ @@ -66,9 +67,6 @@ pub enum Error { #[snafu(display("object defines no name"))] ObjectHasNoName, - #[snafu(display("object defines no namespace"))] - ObjectHasNoNamespace, - #[snafu(display("failed to build metadata"))] MetadataBuild { source: stackable_operator::builder::meta::Error, @@ -130,6 +128,7 @@ pub fn build_maybe_reporting_task( nifi: &v1alpha1::NifiCluster, resolved_product_image: &ResolvedProductImage, cluster_info: &KubernetesClusterInfo, + namespace: &NamespaceName, authentication_config: &NifiAuthenticationConfig, sa_name: &str, ) -> Result> { @@ -139,6 +138,7 @@ pub fn build_maybe_reporting_task( nifi, resolved_product_image, cluster_info, + namespace, authentication_config, sa_name, )?, @@ -158,14 +158,12 @@ pub fn build_reporting_task_service_name(nifi_cluster_name: &str) -> String { pub fn build_reporting_task_fqdn_service_name( nifi: &v1alpha1::NifiCluster, cluster_info: &KubernetesClusterInfo, -) -> Result { + namespace: &NamespaceName, +) -> String { let nifi_cluster_name = nifi.name_any(); - let nifi_namespace: &str = &nifi.namespace().context(ObjectHasNoNamespaceSnafu)?; let reporting_task_service_name = build_reporting_task_service_name(&nifi_cluster_name); let cluster_domain = &cluster_info.cluster_domain; - Ok(format!( - "{reporting_task_service_name}.{nifi_namespace}.svc.{cluster_domain}" - )) + format!("{reporting_task_service_name}.{namespace}.svc.{cluster_domain}") } /// Return the name of the first pod belonging to the first role group that contains more than 0 replicas. @@ -268,11 +266,12 @@ fn build_reporting_task_job( nifi: &v1alpha1::NifiCluster, resolved_product_image: &ResolvedProductImage, cluster_info: &KubernetesClusterInfo, + namespace: &NamespaceName, nifi_auth_config: &NifiAuthenticationConfig, sa_name: &str, ) -> Result { let reporting_task_fqdn_service_name = - build_reporting_task_fqdn_service_name(nifi, cluster_info)?; + build_reporting_task_fqdn_service_name(nifi, cluster_info, namespace); let product_version = &resolved_product_image.product_version; let nifi_connect_url = format!("https://{reporting_task_fqdn_service_name}:{HTTPS_PORT}/nifi-api",); diff --git a/rust/operator-binary/src/security/mod.rs b/rust/operator-binary/src/security/mod.rs index 74416edd..61229ce0 100644 --- a/rust/operator-binary/src/security/mod.rs +++ b/rust/operator-binary/src/security/mod.rs @@ -1,7 +1,7 @@ use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::pod::volume::SecretFormat, client::Client, k8s_openapi::api::core::v1::Volume, - shared::time::Duration, + shared::time::Duration, v2::types::kubernetes::NamespaceName, }; use crate::crd::v1alpha1; @@ -29,8 +29,9 @@ pub enum Error { pub async fn check_or_generate_sensitive_key( client: &Client, nifi: &v1alpha1::NifiCluster, + namespace: &NamespaceName, ) -> Result { - sensitive_key::check_or_generate_sensitive_key(client, nifi) + sensitive_key::check_or_generate_sensitive_key(client, nifi, namespace) .await .context(SensitiveKeySnafu) } @@ -38,8 +39,9 @@ pub async fn check_or_generate_sensitive_key( pub async fn check_or_generate_oidc_admin_password( client: &Client, nifi: &v1alpha1::NifiCluster, + namespace: &NamespaceName, ) -> Result { - oidc::check_or_generate_oidc_admin_password(client, nifi) + oidc::check_or_generate_oidc_admin_password(client, nifi, namespace) .await .context(OidcAdminPasswordSnafu) } diff --git a/rust/operator-binary/src/security/oidc.rs b/rust/operator-binary/src/security/oidc.rs index 0d93d942..28be1204 100644 --- a/rust/operator-binary/src/security/oidc.rs +++ b/rust/operator-binary/src/security/oidc.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use rand::{RngExt, distr::Alphanumeric}; -use snafu::{OptionExt, ResultExt, Snafu}; +use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::meta::ObjectMetaBuilder, client::Client, @@ -9,6 +9,7 @@ use stackable_operator::{ crd::authentication::oidc, k8s_openapi::api::core::v1::Secret, kube::{ResourceExt, runtime::reflector::ObjectRef}, + v2::types::kubernetes::NamespaceName, }; use crate::{crd::v1alpha1, security::authentication::STACKABLE_ADMIN_USERNAME}; @@ -17,9 +18,6 @@ type Result = std::result::Result; #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("the NiFi object defines no namespace"))] - ObjectHasNoNamespace, - #[snafu(display("failed to fetch or create OIDC admin password secret"))] OidcAdminPasswordSecret { source: stackable_operator::client::Error, @@ -45,11 +43,14 @@ pub enum Error { pub(crate) async fn check_or_generate_oidc_admin_password( client: &Client, nifi: &v1alpha1::NifiCluster, + namespace: &NamespaceName, ) -> Result { - let namespace: &str = &nifi.namespace().context(ObjectHasNoNamespaceSnafu)?; tracing::debug!("Checking for OIDC admin password configuration"); match client - .get_opt::(&build_oidc_admin_password_secret_name(nifi), namespace) + .get_opt::( + &build_oidc_admin_password_secret_name(nifi), + namespace.as_ref(), + ) .await .context(OidcAdminPasswordSecretSnafu)? { diff --git a/rust/operator-binary/src/security/sensitive_key.rs b/rust/operator-binary/src/security/sensitive_key.rs index 91a79058..e9baafb9 100644 --- a/rust/operator-binary/src/security/sensitive_key.rs +++ b/rust/operator-binary/src/security/sensitive_key.rs @@ -1,10 +1,10 @@ use std::collections::BTreeMap; use rand::{RngExt, distr::Alphanumeric}; -use snafu::{OptionExt, ResultExt, Snafu}; +use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::meta::ObjectMetaBuilder, client::Client, k8s_openapi::api::core::v1::Secret, - kube::ResourceExt, + v2::types::kubernetes::NamespaceName, }; use crate::crd::v1alpha1; @@ -13,9 +13,6 @@ type Result = std::result::Result; #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("object defines no namespace"))] - ObjectHasNoNamespace, - #[snafu(display("failed to check sensitive property key secret"))] SensitiveKeySecret { source: stackable_operator::client::Error, @@ -32,12 +29,12 @@ pub enum Error { pub(crate) async fn check_or_generate_sensitive_key( client: &Client, nifi: &v1alpha1::NifiCluster, + namespace: &NamespaceName, ) -> Result { let sensitive_config = &nifi.spec.cluster_config.sensitive_properties; - let namespace: &str = &nifi.namespace().context(ObjectHasNoNamespaceSnafu)?; match client - .get_opt::(&sensitive_config.key_secret, namespace) + .get_opt::(&sensitive_config.key_secret, namespace.as_ref()) .await .context(SensitiveKeySecretSnafu)? { From c8b03cb3782045670b309d43bdf1a077c2fecce5 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 8 Jun 2026 17:52:57 +0200 Subject: [PATCH 19/39] refactor(2): use uid and metadata. --- rust/operator-binary/src/controller.rs | 31 +-- rust/operator-binary/src/controller/build.rs | 2 + .../src/controller/build/config_map.rs | 60 ++--- .../src/controller/build/git_sync.rs | 38 +++ .../src/controller/build/properties.rs | 29 ++- .../build/properties/authorizers.rs | 15 +- .../build/properties/nifi_properties.rs | 7 +- .../src/controller/build/proxy_hosts.rs | 53 ++++ .../src/controller/validate.rs | 232 ++++++++++-------- .../operator-binary/src/reporting_task/mod.rs | 9 +- .../src/security/authorization.rs | 10 +- 11 files changed, 299 insertions(+), 187 deletions(-) create mode 100644 rust/operator-binary/src/controller/build/git_sync.rs create mode 100644 rust/operator-binary/src/controller/build/proxy_hosts.rs diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 0dc433e1..8fed6117 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -196,8 +196,8 @@ pub enum Error { #[snafu(display("failed to resolve and merge config for role and role group"))] FailedToResolveConfig { source: crate::crd::Error }, - #[snafu(display("missing git-sync resources for rolegroup [{rolegroup}]"))] - MissingGitSyncResources { rolegroup: String }, + #[snafu(display("failed to build git-sync resources"))] + BuildGitSyncResources { source: build::git_sync::Error }, #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] VectorAggregatorConfigMapMissing, @@ -317,13 +317,9 @@ pub async fn reconcile_nifi( .context(DereferenceSnafu)?; // validate (no Kubernetes API calls required) - let validated_cluster = validate::validate( - nifi, - &dereferenced_objects, - &ctx.operator_environment, - &client.kubernetes_cluster_info, - ) - .context(ValidateClusterSnafu)?; + let validated_cluster = + validate::validate(nifi, &dereferenced_objects, &ctx.operator_environment) + .context(ValidateClusterSnafu)?; let resolved_product_image = &validated_cluster.image; let authentication_config = &validated_cluster.cluster_config.authentication; @@ -416,12 +412,10 @@ pub async fn reconcile_nifi( .merged_config(&nifi_role, rolegroup_name) .context(FailedToResolveConfigSnafu)?; - let git_sync_resources = validated_cluster - .git_sync_resources - .get(rolegroup_name) - .context(MissingGitSyncResourcesSnafu { - rolegroup: rolegroup_name.clone(), - })?; + // Computed here for the StatefulSet; the ConfigMap builder computes its own copy. + let git_sync_resources = + build::git_sync::build_git_sync_resources(&validated_cluster, rg) + .context(BuildGitSyncResourcesSnafu)?; let role_group_service_recommended_labels = build_recommended_labels( nifi, @@ -444,14 +438,11 @@ pub async fn reconcile_nifi( let role = nifi.spec.nodes.as_ref().context(NoNodesDefinedSnafu)?; - // The proxy hosts allow-list lets external users access NiFi via addresses we cannot - // predict, so all of them are added to the setting. - // For more information see let rg_configmap = build::config_map::build_rolegroup_config_map( &validated_cluster, &rolegroup, &role_group_service_recommended_labels, - nifi, + &client.kubernetes_cluster_info, ) .context(BuildRoleGroupConfigMapSnafu { rolegroup: rolegroup.clone(), @@ -479,7 +470,7 @@ pub async fn reconcile_nifi( rolling_upgrade_supported, replicas, &rbac_sa.name_any(), - git_sync_resources, + &git_sync_resources, ) .await?; diff --git a/rust/operator-binary/src/controller/build.rs b/rust/operator-binary/src/controller/build.rs index ad7b4079..cae56571 100644 --- a/rust/operator-binary/src/controller/build.rs +++ b/rust/operator-binary/src/controller/build.rs @@ -3,4 +3,6 @@ //! [`ValidatedCluster`]: crate::controller::validate::ValidatedCluster pub mod config_map; +pub mod git_sync; pub mod properties; +pub mod proxy_hosts; diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 9c08099f..dacf4cd0 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -7,13 +7,19 @@ use stackable_operator::{ kvp::ObjectLabels, product_logging::framework::VECTOR_CONFIG_FILE, role_utils::RoleGroupRef, + utils::cluster_info::KubernetesClusterInfo, + v2::builder::meta::ownerreference_from_resource, }; use crate::{ controller::{ - build::properties::{ - ConfigFileName, authorizers, bootstrap_conf, logging, login_identity_providers, - nifi_properties, security_properties, state_management_xml, + build::{ + git_sync, + properties::{ + ConfigFileName, authorizers, bootstrap_conf, logging, login_identity_providers, + nifi_properties, security_properties, state_management_xml, + }, + proxy_hosts, }, validate::ValidatedCluster, }, @@ -22,11 +28,6 @@ use crate::{ #[derive(Debug, Snafu)] pub enum Error { - #[snafu(display("object is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, - }, - #[snafu(display("failed to build metadata"))] MetadataBuild { source: stackable_operator::builder::meta::Error, @@ -62,14 +63,11 @@ pub enum Error { source: crate::security::authentication::Error, }, - #[snafu(display("object has no nodes defined"))] - NoNodesDefined, - #[snafu(display("the cluster has no rolegroup [{role_group}] in role [{role}]"))] MissingRoleGroup { role: String, role_group: String }, - #[snafu(display("missing git-sync resources for rolegroup [{role_group}]"))] - MissingGitSyncResources { role_group: String }, + #[snafu(display("failed to build git-sync resources"))] + BuildGitSyncResources { source: git_sync::Error }, } type Result = std::result::Result; @@ -77,15 +75,13 @@ type Result = std::result::Result; /// Build the rolegroup [`ConfigMap`] configuring the rolegroup based on the /// resolved cluster configuration. /// -/// All NiFi configuration is sourced from `cluster`. The only use of `owner` is for -/// the OwnerReference, `name_and_namespace`, and the raw role spec used for JVM -/// argument merging. `recommended_labels` must be built by the caller (typically via -/// `build_recommended_labels`). +/// All NiFi configuration is sourced from `cluster`. `recommended_labels` must be built by the +/// caller (typically via `build_recommended_labels`). pub fn build_rolegroup_config_map( cluster: &ValidatedCluster, rolegroup: &RoleGroupRef, recommended_labels: &ObjectLabels<'_, v1alpha1::NifiCluster>, - owner: &v1alpha1::NifiCluster, + cluster_info: &KubernetesClusterInfo, ) -> Result { tracing::debug!("building rolegroup ConfigMap"); @@ -99,24 +95,20 @@ pub fn build_rolegroup_config_map( })?; // The raw role spec is only needed for JVM argument merging in `bootstrap_conf`. - let role = owner.spec.nodes.as_ref().context(NoNodesDefinedSnafu)?; + let role = &cluster.nodes; - let git_sync_resources = cluster - .git_sync_resources - .get(&rolegroup.role_group) - .with_context(|| MissingGitSyncResourcesSnafu { - role_group: rolegroup.role_group.clone(), - })?; + let proxy_hosts = proxy_hosts::compute_proxy_hosts(cluster, cluster_info); + let git_sync_resources = + git_sync::build_git_sync_resources(cluster, rg).context(BuildGitSyncResourcesSnafu)?; let mut cm_builder = ConfigMapBuilder::new(); cm_builder .metadata( ObjectMetaBuilder::new() - .name_and_namespace(owner) + .namespace(&cluster.namespace) .name(rolegroup.object_name()) - .ownerreference_from_resource(owner, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) .with_recommended_labels(recommended_labels) .context(MetadataBuildSnafu)? .build(), @@ -133,11 +125,11 @@ pub fn build_rolegroup_config_map( ) .add_data( ConfigFileName::NifiProperties.to_string(), - nifi_properties::build(cluster, rg, git_sync_resources).with_context(|_| { - BuildNifiPropertiesSnafu { + nifi_properties::build(cluster, rg, &proxy_hosts, &git_sync_resources).with_context( + |_| BuildNifiPropertiesSnafu { rolegroup: rolegroup.clone(), - } - })?, + }, + )?, ) .add_data( ConfigFileName::StateManagementXml.to_string(), @@ -150,9 +142,7 @@ pub fn build_rolegroup_config_map( ) .add_data( ConfigFileName::Authorizers.to_string(), - // TODO: authorizers::build currently takes a raw &NifiCluster; once migrated - // to ValidatedCluster this `owner` arg can be removed. - authorizers::build(cluster, owner), + authorizers::build(cluster), ) .add_data( ConfigFileName::SecurityProperties.to_string(), diff --git a/rust/operator-binary/src/controller/build/git_sync.rs b/rust/operator-binary/src/controller/build/git_sync.rs new file mode 100644 index 00000000..0acce6e3 --- /dev/null +++ b/rust/operator-binary/src/controller/build/git_sync.rs @@ -0,0 +1,38 @@ +//! Builds the git-sync resources (volumes, mounts, containers) for a NiFi Node rolegroup. + +use snafu::{ResultExt, Snafu}; +use stackable_operator::crd::git_sync; + +use crate::{ + controller::{ + LOG_VOLUME_NAME, env_vars_from_overrides, + validate::{NifiRoleGroupConfig, ValidatedCluster}, + }, + crd::Container, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("invalid git-sync specification"))] + InvalidGitSyncSpec { source: git_sync::v1alpha2::Error }, +} + +type Result = std::result::Result; + +/// Builds the [`git_sync::v1alpha2::GitSyncResources`] for a single Node rolegroup. The env vars +/// and logging configuration differ per rolegroup, so the resources are computed per rolegroup +/// rather than once for the whole cluster. +pub fn build_git_sync_resources( + cluster: &ValidatedCluster, + rg: &NifiRoleGroupConfig, +) -> Result { + git_sync::v1alpha2::GitSyncResources::new( + &cluster.cluster_config.custom_components_git_sync, + &cluster.image, + &env_vars_from_overrides(&rg.env_overrides), + &[], + LOG_VOLUME_NAME, + &rg.config.logging.for_container(&Container::GitSync), + ) + .context(InvalidGitSyncSpecSnafu) +} diff --git a/rust/operator-binary/src/controller/build/properties.rs b/rust/operator-binary/src/controller/build/properties.rs index 795a5d49..8e3c680c 100644 --- a/rust/operator-binary/src/controller/build/properties.rs +++ b/rust/operator-binary/src/controller/build/properties.rs @@ -88,7 +88,10 @@ pub(crate) mod test_support { }, kube::ResourceExt as _, kvp::LabelValue, - v2::types::kubernetes::NamespaceName, + v2::types::{ + kubernetes::{NamespaceName, Uid}, + operator::ClusterName, + }, }; use crate::{ @@ -160,13 +163,18 @@ pub(crate) mod test_support { pull_secrets: None, }; - ValidatedCluster { - name: "simple-nifi".to_string(), - namespace: NamespaceName::from_str("default").expect("valid namespace"), + let name = ClusterName::from_str("simple-nifi").expect("valid cluster name"); + let namespace = NamespaceName::from_str("default").expect("valid namespace"); + let uid = Uid::from_str("e6ac237d-a6d4-43a1-8135-f36506110912").expect("valid uid"); + + ValidatedCluster::new( + name, + namespace, + uid, image, + nifi.spec.nodes.clone().expect("minimal fixture has nodes"), role_group_configs, - git_sync_resources: Default::default(), - cluster_config: ValidatedClusterConfig { + ValidatedClusterConfig { authentication: NifiAuthenticationConfig::SingleUser { provider: StaticAuthProvider { user_credentials_secret: UserCredentialsSecretRef { @@ -175,11 +183,16 @@ pub(crate) mod test_support { }, }, authorization: ResolvedNifiAuthorizationConfig::SingleUser, - proxy_hosts: "*".to_string(), clustering_backend: v1alpha1::NifiClusteringBackend::Kubernetes {}, sensitive_properties_algorithm: Default::default(), // NifiArgon2AesGcm256 + host_header_check: nifi.spec.cluster_config.host_header_check.clone(), + custom_components_git_sync: nifi + .spec + .cluster_config + .custom_components_git_sync + .clone(), }, - } + ) } /// Return the "default" role-group config from a [`ValidatedCluster`]. diff --git a/rust/operator-binary/src/controller/build/properties/authorizers.rs b/rust/operator-binary/src/controller/build/properties/authorizers.rs index 48e8b563..7f67c661 100644 --- a/rust/operator-binary/src/controller/build/properties/authorizers.rs +++ b/rust/operator-binary/src/controller/build/properties/authorizers.rs @@ -1,29 +1,24 @@ //! Builder for `authorizers.xml`. -use crate::{controller::validate::ValidatedCluster, crd::v1alpha1}; +use crate::controller::validate::ValidatedCluster; -pub fn build(cluster: &ValidatedCluster, nifi: &v1alpha1::NifiCluster) -> String { - // TODO(follow-up PR): narrow get_authorizers_config to resolved fields on ValidatedCluster instead of taking the full NifiCluster. +pub fn build(cluster: &ValidatedCluster) -> String { cluster .cluster_config .authorization - .get_authorizers_config(nifi) + .get_authorizers_config(cluster.name.as_ref()) } #[cfg(test)] mod tests { use super::*; - use crate::controller::build::properties::test_support::{ - MINIMAL_NIFI_YAML, minimal_validated_cluster, - }; + use crate::controller::build::properties::test_support::minimal_validated_cluster; #[test] fn test_build_returns_non_empty_xml_with_authorizers_root() { let cluster = minimal_validated_cluster(); - let nifi: v1alpha1::NifiCluster = - serde_yaml::from_str(MINIMAL_NIFI_YAML).expect("invalid test YAML"); - let xml = build(&cluster, &nifi); + let xml = build(&cluster); assert!(!xml.is_empty(), "authorizers.xml should not be empty"); assert!( diff --git a/rust/operator-binary/src/controller/build/properties/nifi_properties.rs b/rust/operator-binary/src/controller/build/properties/nifi_properties.rs index 04c0ee64..25c2529d 100644 --- a/rust/operator-binary/src/controller/build/properties/nifi_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/nifi_properties.rs @@ -28,10 +28,10 @@ const STORAGE_CONTENT_ARCHIVE_UTILIZATION_FACTOR: f32 = 0.5; pub fn build( cluster: &ValidatedCluster, rg: &NifiRoleGroupConfig, + proxy_hosts: &str, git_sync_resources: &git_sync::v1alpha2::GitSyncResources, ) -> Result { let product_version = &cluster.image.product_version; - let proxy_hosts = &cluster.cluster_config.proxy_hosts; let auth_config = &cluster.cluster_config.authentication; let resource_config = &rg.config.resources; @@ -603,7 +603,7 @@ mod tests { let rg = default_rg(&cluster); let git_sync = empty_git_sync_resources(); - let props = build(&cluster, rg, &git_sync).expect("build should succeed"); + let props = build(&cluster, rg, "*", &git_sync).expect("build should succeed"); // HTTPS port assert!( @@ -690,7 +690,8 @@ mod tests { .insert("default".to_string(), rg.clone()); let git_sync = empty_git_sync_resources(); - let props = build(&cluster, &rg, &git_sync).expect("build with override should succeed"); + let props = + build(&cluster, &rg, "*", &git_sync).expect("build with override should succeed"); assert!( props.contains("some.custom.key=some-custom-value"), diff --git a/rust/operator-binary/src/controller/build/proxy_hosts.rs b/rust/operator-binary/src/controller/build/proxy_hosts.rs new file mode 100644 index 00000000..233ba54e --- /dev/null +++ b/rust/operator-binary/src/controller/build/proxy_hosts.rs @@ -0,0 +1,53 @@ +//! Computes the NiFi proxy hosts allow-list (`nifi.web.proxy.host`). + +use std::collections::HashSet; + +use stackable_operator::utils::cluster_info::KubernetesClusterInfo; + +use crate::{controller::validate::ValidatedCluster, crd::HTTPS_PORT, reporting_task}; + +/// Computes the comma-separated NiFi proxy hosts, or `"*"` if `hostHeaderCheck.allowAll` is set. +/// +/// The proxy hosts allow-list lets external users access NiFi via addresses we cannot predict, +/// so all of them are added to the setting. For more information see +/// +pub fn compute_proxy_hosts( + cluster: &ValidatedCluster, + cluster_info: &KubernetesClusterInfo, +) -> String { + let host_header_check = &cluster.cluster_config.host_header_check; + + if host_header_check.allow_all { + tracing::info!( + "spec.clusterConfig.hostHeaderCheck.allowAll is set to true. All proxy hosts will be allowed." + ); + if !host_header_check.additional_allowed_hosts.is_empty() { + tracing::info!( + "spec.clusterConfig.hostHeaderCheck.additionalAllowedHosts is ignored and only '*' is added to the allow-list." + ) + } + return "*".to_string(); + } + + // Address and port are injected from the listener volume during the prepare container + let mut proxy_hosts = HashSet::from([ + "${env:LISTENER_DEFAULT_ADDRESS}:${env:LISTENER_DEFAULT_PORT_HTTPS}".to_string(), + ]); + proxy_hosts.extend(host_header_check.additional_allowed_hosts.iter().cloned()); + + // Reporting task only exists for NiFi 1.x + if cluster.image.product_version.starts_with("1.") { + let reporting_task_service_name = reporting_task::build_reporting_task_fqdn_service_name( + cluster.name.as_ref(), + &cluster.namespace, + cluster_info, + ); + + proxy_hosts.insert(format!("{reporting_task_service_name}:{HTTPS_PORT}")); + } + + let mut proxy_hosts = Vec::from_iter(proxy_hosts); + proxy_hosts.sort(); + + proxy_hosts.join(",") +} diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index b306acc6..71884ff4 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -3,28 +3,34 @@ //! Synchronously validates inputs that don't require Kubernetes API calls. Produces //! [`ValidatedCluster`], consumed by the rest of `reconcile_nifi`. -use std::collections::{BTreeMap, HashSet}; +use std::collections::BTreeMap; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, commons::product_image_selection::{self, ResolvedProductImage}, crd::git_sync, - kube::ResourceExt as _, + k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta, + kube::{Resource, ResourceExt as _}, role_utils::JavaCommonConfig, - utils::cluster_info::KubernetesClusterInfo, - v2::types::kubernetes::NamespaceName, + v2::{ + HasName, HasUid, + controller_utils::{self, get_cluster_name, get_uid}, + types::{ + kubernetes::{NamespaceName, Uid}, + operator::ClusterName, + }, + }, }; use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ - controller::{LOG_VOLUME_NAME, dereference::DereferencedObjects, env_vars_from_overrides}, + controller::dereference::DereferencedObjects, crd::{ - Container, HTTPS_PORT, NifiConfig, NifiRole, sensitive_properties, + HostHeaderCheckConfig, NifiConfig, NifiRole, NifiRoleType, sensitive_properties, sensitive_properties::NifiSensitiveKeyAlgorithm, v1alpha1, }, framework::role_utils::with_validated_config, - reporting_task, security::{ authentication::{self, NifiAuthenticationConfig}, authorization::ResolvedNifiAuthorizationConfig, @@ -43,6 +49,12 @@ pub enum Error { #[snafu(display("object has no nodes defined"))] NoNodesDefined, + #[snafu(display("failed to get the cluster name"))] + GetClusterName { source: controller_utils::Error }, + + #[snafu(display("failed to get the UID"))] + GetUid { source: controller_utils::Error }, + #[snafu(display("invalid NiFi authentication configuration"))] InvalidAuthenticationConfig { source: authentication::Error }, @@ -53,9 +65,6 @@ pub enum Error { #[snafu(display("invalid sensitive properties algorithm"))] InvalidSensitivePropertiesAlgorithm { source: sensitive_properties::Error }, - - #[snafu(display("invalid git-sync specification"))] - InvalidGitSyncSpec { source: git_sync::v1alpha2::Error }, } pub type NifiRoleGroupConfig = crate::framework::role_utils::RoleGroupConfig< @@ -67,39 +76,122 @@ pub type NifiRoleGroupConfig = crate::framework::role_utils::RoleGroupConfig< type Result = std::result::Result; /// The validated NifiCluster: everything `reconcile_nifi` needs after dereferencing, -/// in fail-safe / resolved form. The raw `NifiCluster` should only be needed for -/// OwnerReferences after this point. +/// in fail-safe / resolved form. This is the single resolved representation of the cluster; +/// downstream builders should source everything from here and never touch the raw `NifiCluster`. pub struct ValidatedCluster { - #[allow(dead_code)] - pub name: String, + /// Synthetic metadata (name, namespace, uid) so `ValidatedCluster` can implement + /// [`Resource`] and be used to build OwnerReferences without the raw `NifiCluster`. + metadata: ObjectMeta, + /// The name of the NifiCluster. + pub name: ClusterName, /// The namespace of the NifiCluster, parsed once in the dereference step and reused everywhere. pub namespace: NamespaceName, + /// The UID of the NifiCluster, used to build OwnerReferences downstream. + pub uid: Uid, + /// The product image. pub image: ResolvedProductImage, - pub role_group_configs: BTreeMap>, - /// The git-sync resources (volumes, mounts, containers) for each Node rolegroup, - /// keyed by rolegroup name. Precomputed here so both the ConfigMap and StatefulSet - /// builders can source them from `ValidatedCluster`. - pub git_sync_resources: BTreeMap, + /// Cluster wide settings. pub cluster_config: ValidatedClusterConfig, + /// The raw Node role spec (`spec.nodes`), needed for JVM argument merging in `bootstrap.conf`. + pub nodes: NifiRoleType, + /// Collected configuration per rolegroup. + pub role_group_configs: BTreeMap>, } +/// The resolved `spec.clusterConfig`. pub struct ValidatedClusterConfig { + /// The cluster authentication settings. pub authentication: NifiAuthenticationConfig, + /// The cluster authorization settings. pub authorization: ResolvedNifiAuthorizationConfig, - /// Comma-separated NiFi proxy hosts, or `"*"` if `hostHeaderCheck.allowAll` is set. - pub proxy_hosts: String, + /// The git-sync specs, resolved into git-sync resources at build time. + pub custom_components_git_sync: Vec, /// The clustering backend (ZooKeeper or Kubernetes), copied from the spec. pub clustering_backend: v1alpha1::NifiClusteringBackend, + /// The host-header-check config, resolved into the proxy hosts allow-list at build time. + pub host_header_check: HostHeaderCheckConfig, /// The validated sensitive properties algorithm. pub sensitive_properties_algorithm: NifiSensitiveKeyAlgorithm, } +impl ValidatedCluster { + /// Builds a [`ValidatedCluster`], deriving the synthetic [`ObjectMeta`] from name, namespace + /// and uid so the struct can implement [`Resource`]. + pub fn new( + name: ClusterName, + namespace: NamespaceName, + uid: Uid, + image: ResolvedProductImage, + nodes: NifiRoleType, + role_group_configs: BTreeMap>, + cluster_config: ValidatedClusterConfig, + ) -> Self { + let metadata = ObjectMeta { + name: Some(name.to_string()), + namespace: Some(namespace.to_string()), + uid: Some(uid.to_string()), + ..ObjectMeta::default() + }; + + Self { + metadata, + name, + namespace, + uid, + image, + nodes, + role_group_configs, + cluster_config, + } + } +} + +impl HasName for ValidatedCluster { + fn to_name(&self) -> String { + self.name.to_string() + } +} + +impl HasUid for ValidatedCluster { + fn to_uid(&self) -> Uid { + self.uid.clone() + } +} + +impl Resource for ValidatedCluster { + type DynamicType = ::DynamicType; + type Scope = ::Scope; + + fn kind(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha1::NifiCluster::kind(dt) + } + + fn group(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha1::NifiCluster::group(dt) + } + + fn version(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha1::NifiCluster::version(dt) + } + + fn plural(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha1::NifiCluster::plural(dt) + } + + fn meta(&self) -> &ObjectMeta { + &self.metadata + } + + fn meta_mut(&mut self) -> &mut ObjectMeta { + &mut self.metadata + } +} + /// Validates the cluster spec and the dereferenced inputs. pub fn validate( nifi: &v1alpha1::NifiCluster, dereferenced_objects: &DereferencedObjects, operator_environment: &OperatorEnvironmentOptions, - cluster_info: &KubernetesClusterInfo, ) -> Result { let image = nifi .spec @@ -120,8 +212,6 @@ pub fn validate( &dereferenced_objects.authorization, ); - let proxy_hosts = compute_proxy_hosts(nifi, cluster_info, &dereferenced_objects.namespace); - let sensitive_properties_algorithm = nifi .spec .cluster_config @@ -133,23 +223,34 @@ pub fn validate( .check_for_nifi_version(&image.product_version) .context(InvalidSensitivePropertiesAlgorithmSnafu)?; + let nodes = nifi + .spec + .nodes + .as_ref() + .context(NoNodesDefinedSnafu)? + .clone(); let role_group_configs = build_role_group_configs(nifi)?; - let git_sync_resources = build_git_sync_resources(nifi, &image, &role_group_configs)?; - Ok(ValidatedCluster { - name: nifi.name_any(), - namespace: dereferenced_objects.namespace.clone(), + let name = get_cluster_name(nifi).context(GetClusterNameSnafu)?; + let namespace = dereferenced_objects.namespace.clone(); + let uid = get_uid(nifi).context(GetUidSnafu)?; + + Ok(ValidatedCluster::new( + name, + namespace, + uid, image, + nodes, role_group_configs, - git_sync_resources, - cluster_config: ValidatedClusterConfig { + ValidatedClusterConfig { authentication: authentication_config, authorization: authorization_config, - proxy_hosts, clustering_backend: nifi.spec.cluster_config.clustering_backend.clone(), sensitive_properties_algorithm, + host_header_check: nifi.spec.cluster_config.host_header_check.clone(), + custom_components_git_sync: nifi.spec.cluster_config.custom_components_git_sync.clone(), }, - }) + )) } fn build_role_group_configs( @@ -170,70 +271,3 @@ fn build_role_group_configs( role_group_configs.insert(NifiRole::Node, groups); Ok(role_group_configs) } - -/// Builds the [`git_sync::v1alpha2::GitSyncResources`] for every Node rolegroup, keyed by -/// rolegroup name. The env vars and logging configuration differ per rolegroup, so the -/// resources are computed per rolegroup rather than once for the whole cluster. -fn build_git_sync_resources( - nifi: &v1alpha1::NifiCluster, - image: &ResolvedProductImage, - role_group_configs: &BTreeMap>, -) -> Result> { - let mut resources = BTreeMap::new(); - - if let Some(groups) = role_group_configs.get(&NifiRole::Node) { - for (rg_name, rg) in groups { - let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new( - &nifi.spec.cluster_config.custom_components_git_sync, - image, - &env_vars_from_overrides(&rg.env_overrides), - &[], - LOG_VOLUME_NAME, - &rg.config.logging.for_container(&Container::GitSync), - ) - .context(InvalidGitSyncSpecSnafu)?; - resources.insert(rg_name.clone(), git_sync_resources); - } - } - - Ok(resources) -} - -fn compute_proxy_hosts( - nifi: &v1alpha1::NifiCluster, - cluster_info: &KubernetesClusterInfo, - namespace: &NamespaceName, -) -> String { - let host_header_check = &nifi.spec.cluster_config.host_header_check; - - if host_header_check.allow_all { - tracing::info!( - "spec.clusterConfig.hostHeaderCheck.allowAll is set to true. All proxy hosts will be allowed." - ); - if !host_header_check.additional_allowed_hosts.is_empty() { - tracing::info!( - "spec.clusterConfig.hostHeaderCheck.additionalAllowedHosts is ignored and only '*' is added to the allow-list." - ) - } - return "*".to_string(); - } - - // Address and port are injected from the listener volume during the prepare container - let mut proxy_hosts = HashSet::from([ - "${env:LISTENER_DEFAULT_ADDRESS}:${env:LISTENER_DEFAULT_PORT_HTTPS}".to_string(), - ]); - proxy_hosts.extend(host_header_check.additional_allowed_hosts.iter().cloned()); - - // Reporting task only exists for NiFi 1.x - if nifi.spec.image.product_version().starts_with("1.") { - let reporting_task_service_name = - reporting_task::build_reporting_task_fqdn_service_name(nifi, cluster_info, namespace); - - proxy_hosts.insert(format!("{reporting_task_service_name}:{HTTPS_PORT}")); - } - - let mut proxy_hosts = Vec::from_iter(proxy_hosts); - proxy_hosts.sort(); - - proxy_hosts.join(",") -} diff --git a/rust/operator-binary/src/reporting_task/mod.rs b/rust/operator-binary/src/reporting_task/mod.rs index d92984d3..4aecdf21 100644 --- a/rust/operator-binary/src/reporting_task/mod.rs +++ b/rust/operator-binary/src/reporting_task/mod.rs @@ -156,12 +156,11 @@ pub fn build_reporting_task_service_name(nifi_cluster_name: &str) -> String { /// Return the FQDN (with namespace, domain) of the reporting task. pub fn build_reporting_task_fqdn_service_name( - nifi: &v1alpha1::NifiCluster, - cluster_info: &KubernetesClusterInfo, + cluster_name: &str, namespace: &NamespaceName, + cluster_info: &KubernetesClusterInfo, ) -> String { - let nifi_cluster_name = nifi.name_any(); - let reporting_task_service_name = build_reporting_task_service_name(&nifi_cluster_name); + let reporting_task_service_name = build_reporting_task_service_name(cluster_name); let cluster_domain = &cluster_info.cluster_domain; format!("{reporting_task_service_name}.{namespace}.svc.{cluster_domain}") } @@ -271,7 +270,7 @@ fn build_reporting_task_job( sa_name: &str, ) -> Result { let reporting_task_fqdn_service_name = - build_reporting_task_fqdn_service_name(nifi, cluster_info, namespace); + build_reporting_task_fqdn_service_name(&nifi.name_any(), namespace, cluster_info); let product_version = &resolved_product_image.product_version; let nifi_connect_url = format!("https://{reporting_task_fqdn_service_name}:{HTTPS_PORT}/nifi-api",); diff --git a/rust/operator-binary/src/security/authorization.rs b/rust/operator-binary/src/security/authorization.rs index cd63bf57..c1d92a9b 100644 --- a/rust/operator-binary/src/security/authorization.rs +++ b/rust/operator-binary/src/security/authorization.rs @@ -7,15 +7,11 @@ use stackable_operator::{ k8s_openapi::api::core::v1::{ ConfigMap, ConfigMapKeySelector, EnvVar, EnvVarSource, Volume, VolumeMount, }, - kube::ResourceExt, }; use crate::{ config::{NIFI_PVC_STORAGE_DIRECTORY, NifiRepository}, - crd::{ - authorization::{NifiAccessPolicyProvider, NifiAuthorization, NifiOpaConfig}, - v1alpha1, - }, + crd::authorization::{NifiAccessPolicyProvider, NifiAuthorization, NifiOpaConfig}, }; const OPA_TLS_VOLUME_NAME: &str = "opa-tls"; @@ -118,7 +114,7 @@ impl ResolvedNifiAuthorizationConfig { } } - pub fn get_authorizers_config(&self, nifi_cluster: &v1alpha1::NifiCluster) -> String { + pub fn get_authorizers_config(&self, cluster_name: &str) -> String { let mut authorizers_xml = indoc! {r#" @@ -133,7 +129,7 @@ impl ResolvedNifiAuthorizationConfig { .. } => { // According to [`OpaConfig::document_url`] we default the stacklet name - let package = package.clone().unwrap_or_else(|| nifi_cluster.name_any()); + let package = package.clone().unwrap_or_else(|| cluster_name.to_owned()); authorizers_xml.push_str(&formatdoc! {r#" authorizer From c593ce95f859d31010f386f9e5468895ed5a4515 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 9 Jun 2026 13:38:12 +0200 Subject: [PATCH 20/39] refactor: use non optional property writers --- Cargo.lock | 51 +++--- Cargo.nix | 55 +++---- extra/crds.yaml | 18 ++- rust/operator-binary/src/config/mod.rs | 2 - .../src/controller/build/properties.rs | 37 +++-- .../build/properties/security_properties.rs | 21 +-- rust/operator-binary/src/crd/mod.rs | 149 +++++++----------- 7 files changed, 153 insertions(+), 180 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6c8910d0..4b6f5acf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1510,13 +1510,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.99" +version = "0.3.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" +checksum = "f2025f20d7a4fa7785846e7b63d10a76d3f1cee98ee5cb79ea59703f95e42162" dependencies = [ "cfg-if", "futures-util", - "once_cell", "wasm-bindgen", ] @@ -1572,7 +1571,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "darling", "regex", @@ -2990,7 +2989,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "const-oid", "ecdsa", @@ -3040,7 +3039,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.111.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "base64", "clap", @@ -3084,7 +3083,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "darling", "proc-macro2", @@ -3095,7 +3094,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "jiff", "k8s-openapi", @@ -3112,7 +3111,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.4" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "axum", "clap", @@ -3136,7 +3135,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "kube", "schemars", @@ -3150,7 +3149,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "convert_case", "convert_case_extras", @@ -3168,7 +3167,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "arc-swap", "async-trait", @@ -3762,9 +3761,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.2" +version = "1.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" dependencies = [ "js-sys", "wasm-bindgen", @@ -3823,9 +3822,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.122" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" +checksum = "a254a4b10c19a76f09a27640e7ffbf9bc30bf67e16a3bf28aaefa4920fe81563" dependencies = [ "cfg-if", "once_cell", @@ -3836,9 +3835,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.72" +version = "0.4.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" +checksum = "54568702fabf5d4849ce2b90fadfa64168a097eaf4b351ce9df8b687a0086aaf" dependencies = [ "js-sys", "wasm-bindgen", @@ -3846,9 +3845,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.122" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" +checksum = "24a40fc75b0ec6f3746ceb10d36f53a93dcd68a93b11b6445983945d79eba0dc" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3856,9 +3855,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.122" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" +checksum = "908f34bd9b9ce3d4caf07b72dfab63d61504d156856c6bd3cd87fa350cf3985b" dependencies = [ "bumpalo", "proc-macro2", @@ -3869,9 +3868,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.122" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" +checksum = "7acbf7616c27b194bbb550bf77ed0c2c3e5b7fd1260a93082b95fb7f47959b92" dependencies = [ "unicode-ident", ] @@ -3912,9 +3911,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.99" +version = "0.3.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" +checksum = "6e0871acf327f283dc6da28a1696cdc64fb355ba9f935d052021fa77f35cce69" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.nix b/Cargo.nix index b2482245..1f22b290 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4858,9 +4858,9 @@ rec { }; "js-sys" = rec { crateName = "js-sys"; - version = "0.3.99"; + version = "0.3.100"; edition = "2021"; - sha256 = "04azrzsz91gr5s3z0ij36lz0kj9ry4lw3jz0mmbiwb251rsc8aql"; + sha256 = "0qi1wjakyw2rx9wwprcfx77g3lvn1b8n6yvfhj2pgym4swh5y0pj"; libName = "js_sys"; authors = [ "The wasm-bindgen Developers" @@ -4877,11 +4877,6 @@ rec { usesDefaultFeatures = false; features = [ "std" ]; } - { - name = "once_cell"; - packageId = "once_cell"; - usesDefaultFeatures = false; - } { name = "wasm-bindgen"; packageId = "wasm-bindgen"; @@ -5071,7 +5066,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "k8s_version"; @@ -9865,7 +9860,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_certs"; @@ -10077,7 +10072,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_operator"; @@ -10271,7 +10266,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; procMacro = true; @@ -10306,7 +10301,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_shared"; @@ -10387,7 +10382,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_telemetry"; @@ -10497,7 +10492,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_versioned"; @@ -10547,7 +10542,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; procMacro = true; @@ -10615,7 +10610,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_webhook"; @@ -12662,9 +12657,9 @@ rec { }; "uuid" = rec { crateName = "uuid"; - version = "1.23.2"; + version = "1.23.3"; edition = "2021"; - sha256 = "1xy942s4z0bi8p3441wvd4ry3hx6ry1c7s6fgrr38462xqybhn6j"; + sha256 = "1drddl03gi12vl1s3l2h371dw39plhn9wappp00v707g7h96nk8l"; authors = [ "Ashley Mannix" "Dylan DPC" @@ -12832,9 +12827,9 @@ rec { }; "wasm-bindgen" = rec { crateName = "wasm-bindgen"; - version = "0.2.122"; + version = "0.2.123"; edition = "2021"; - sha256 = "02flix96brsb2r1i3grnikii302iqpdm337kl3xv5lklz5v4bl1y"; + sha256 = "0qqmx07r597gm8lbz8qngvv0phwvpzzyfh3nl84nz9qr1jqs8m52"; libName = "wasm_bindgen"; authors = [ "The wasm-bindgen Developers" @@ -12883,9 +12878,9 @@ rec { }; "wasm-bindgen-futures" = rec { crateName = "wasm-bindgen-futures"; - version = "0.4.72"; + version = "0.4.73"; edition = "2021"; - sha256 = "03qb24gfr072rk8hb69glfdc8yhqqqq2rhy3j5i0ps8sk79dnwwl"; + sha256 = "1bva12h8gdpqkp753czlxabs0s21lvgzm41brr4lhpdzz818fmjl"; libName = "wasm_bindgen_futures"; authors = [ "The wasm-bindgen Developers" @@ -12911,9 +12906,9 @@ rec { }; "wasm-bindgen-macro" = rec { crateName = "wasm-bindgen-macro"; - version = "0.2.122"; + version = "0.2.123"; edition = "2021"; - sha256 = "1inyl55bvdifx7l60q9wl0ivmw7236jg7jqmcqpxhsx3knq52qci"; + sha256 = "1p50xdwmv543b52bc49vm5lcsgd9adpx647bdisg7ihfbg3hz914"; procMacro = true; libName = "wasm_bindgen_macro"; authors = [ @@ -12935,9 +12930,9 @@ rec { }; "wasm-bindgen-macro-support" = rec { crateName = "wasm-bindgen-macro-support"; - version = "0.2.122"; + version = "0.2.123"; edition = "2021"; - sha256 = "0pjw5kc2mbfz59agk5l21kh4hxzp94rygdvsnr4f3z6b5hv4g419"; + sha256 = "0nwqyc63byl7rp9nnv45av8h85fncfmxywkvy35d9qwwkfyk93wh"; libName = "wasm_bindgen_macro_support"; authors = [ "The wasm-bindgen Developers" @@ -12971,10 +12966,10 @@ rec { }; "wasm-bindgen-shared" = rec { crateName = "wasm-bindgen-shared"; - version = "0.2.122"; + version = "0.2.123"; edition = "2021"; links = "wasm_bindgen"; - sha256 = "0ds4mmfqvxwc5fp33hn0jblf0f6b4lghrd9mpkls66zic4n9p4ls"; + sha256 = "14lvjm3pzywm5c4962i6s5zmngic1knpggshnnxr9c97dihzgjvs"; libName = "wasm_bindgen_shared"; authors = [ "The wasm-bindgen Developers" @@ -13099,9 +13094,9 @@ rec { }; "web-sys" = rec { crateName = "web-sys"; - version = "0.3.99"; + version = "0.3.100"; edition = "2021"; - sha256 = "0dilfvl9jnyhi4skl6cry9wc300r693j0w82jjbq8yy3rx0i8qkd"; + sha256 = "0sffbkrpgyi1402mv4wzp9av6ky6rnb1d2m2dpf87wi7yfn7223f"; libName = "web_sys"; authors = [ "The wasm-bindgen Developers" diff --git a/extra/crds.yaml b/extra/crds.yaml index b7200414..e6ecf94f 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -1266,33 +1266,36 @@ spec: properties: bootstrap.conf: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object nifi.properties: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object security.properties: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object type: object envOverrides: @@ -2137,33 +2140,36 @@ spec: properties: bootstrap.conf: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object nifi.properties: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object security.properties: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object type: object envOverrides: diff --git a/rust/operator-binary/src/config/mod.rs b/rust/operator-binary/src/config/mod.rs index 78e6c47f..b2ade65c 100644 --- a/rust/operator-binary/src/config/mod.rs +++ b/rust/operator-binary/src/config/mod.rs @@ -11,8 +11,6 @@ pub const NIFI_CONFIG_DIRECTORY: &str = "/stackable/nifi/conf"; pub const NIFI_PYTHON_WORKING_DIRECTORY: &str = "/nifi-python-working-directory"; pub const NIFI_PVC_STORAGE_DIRECTORY: &str = "/stackable/data"; -pub const NIFI_BOOTSTRAP_CONF: &str = "bootstrap.conf"; -pub const NIFI_PROPERTIES: &str = "nifi.properties"; pub const JVM_SECURITY_PROPERTIES_FILE: &str = "security.properties"; #[derive(Debug, Display, EnumIter)] diff --git a/rust/operator-binary/src/controller/build/properties.rs b/rust/operator-binary/src/controller/build/properties.rs index 8e3c680c..08d720b9 100644 --- a/rust/operator-binary/src/controller/build/properties.rs +++ b/rust/operator-binary/src/controller/build/properties.rs @@ -4,10 +4,6 @@ //! The shared [`stackable_operator::v2::config_file_writer`] module serializes `.properties`/`.conf` //! key/value maps to the Java-properties on-wire format. -use std::collections::BTreeMap; - -use stackable_operator::config_overrides::KeyValueOverridesProvider; - use crate::controller::validate::NifiRoleGroupConfig; pub mod authorizers; @@ -37,24 +33,27 @@ pub enum ConfigFileName { Logback, } -/// Keep only the set (`Some`) entries of a `key -> optional value` map, as `(key, value)` pairs. -fn defined_entries( - entries: BTreeMap>, -) -> impl Iterator { - entries - .into_iter() - .filter_map(|(key, value)| value.map(|value| (key, value))) -} - -/// Resolve the user overrides for `file` from a rolegroup's config overrides, dropping unset values. +/// Resolve the user overrides for `file` from a rolegroup's config overrides. +/// +/// Keys whose value is unset (`None`, i.e. `key: null` in YAML) are dropped, so only the defined +/// `(key, value)` pairs reach the property writer. pub(crate) fn resolved_overrides_for( rg: &NifiRoleGroupConfig, file: ConfigFileName, -) -> impl Iterator { - defined_entries( - rg.config_overrides - .get_key_value_overrides(&file.to_string()), - ) +) -> impl Iterator + '_ { + let overrides = match file { + ConfigFileName::BootstrapConf => Some(&rg.config_overrides.bootstrap_conf), + ConfigFileName::NifiProperties => Some(&rg.config_overrides.nifi_properties), + ConfigFileName::SecurityProperties => Some(&rg.config_overrides.security_properties), + ConfigFileName::StateManagementXml + | ConfigFileName::LoginIdentityProviders + | ConfigFileName::Authorizers + | ConfigFileName::Logback => None, + }; + overrides + .into_iter() + .flat_map(|o| &o.overrides) + .filter_map(|(key, value)| value.clone().map(|value| (key.clone(), value))) } /// Test helpers for constructing a minimal [`ValidatedCluster`] and related types without diff --git a/rust/operator-binary/src/controller/build/properties/security_properties.rs b/rust/operator-binary/src/controller/build/properties/security_properties.rs index 8b857eae..e2ab7afc 100644 --- a/rust/operator-binary/src/controller/build/properties/security_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/security_properties.rs @@ -10,18 +10,15 @@ use super::ConfigFileName; use crate::controller::validate::NifiRoleGroupConfig; pub fn build(rg: &NifiRoleGroupConfig) -> Result { - let mut props: BTreeMap> = BTreeMap::new(); + let mut props: BTreeMap = BTreeMap::new(); // Defaults previously injected by deploy/config-spec/properties.yaml: - props.insert( - "networkaddress.cache.ttl".to_string(), - Some("30".to_string()), - ); + props.insert("networkaddress.cache.ttl".to_string(), "30".to_string()); props.insert( "networkaddress.cache.negative.ttl".to_string(), - Some("0".to_string()), + "0".to_string(), ); for (k, v) in super::resolved_overrides_for(rg, ConfigFileName::SecurityProperties) { - props.insert(k, Some(v)); + props.insert(k, v); } to_java_properties_string(props.iter()) } @@ -30,7 +27,7 @@ pub fn build(rg: &NifiRoleGroupConfig) -> Result mod tests { use std::collections::BTreeMap; - use stackable_operator::config_overrides::KeyValueConfigOverrides; + use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; use super::*; use crate::{ @@ -44,7 +41,13 @@ mod tests { replicas: 1, config: NifiConfig::default(), config_overrides: NifiConfigOverrides { - security_properties: overrides.map(|o| KeyValueConfigOverrides { overrides: o }), + security_properties: KeyValueConfigOverrides { + overrides: overrides + .unwrap_or_default() + .into_iter() + .map(|(k, v)| (k, Some(v))) + .collect(), + }, ..Default::default() }, env_overrides: BTreeMap::new(), diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 18c2af8b..fce88f34 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -3,8 +3,6 @@ pub mod authorization; pub mod sensitive_properties; pub mod tls; -use std::collections::BTreeMap; - use affinity::get_affinity; use authorization::NifiAuthorization; use sensitive_properties::NifiSensitivePropertiesConfig; @@ -24,7 +22,6 @@ use stackable_operator::{ fragment::{self, Fragment, ValidationError}, merge::Merge, }, - config_overrides::{KeyValueConfigOverrides, KeyValueOverridesProvider}, crd::{authentication::core as auth_core, git_sync}, deep_merger::ObjectOverrides, k8s_openapi::{ @@ -39,12 +36,11 @@ use stackable_operator::{ shared::time::Duration, status::condition::{ClusterCondition, HasStatusCondition}, utils::crds::{raw_object_list_schema, raw_object_schema}, + v2::config_overrides::KeyValueConfigOverrides, versioned::versioned, }; use tls::NifiTls; -use crate::config::{JVM_SECURITY_PROPERTIES_FILE, NIFI_BOOTSTRAP_CONF, NIFI_PROPERTIES}; - pub const APP_NAME: &str = "nifi"; pub const HTTPS_PORT_NAME: &str = "https"; @@ -191,62 +187,17 @@ pub mod versioned { Kubernetes {}, } - #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] + #[derive(Clone, Debug, Default, Deserialize, JsonSchema, Merge, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct NifiConfigOverrides { - #[serde(rename = "bootstrap.conf", skip_serializing_if = "Option::is_none")] - pub bootstrap_conf: Option, - - #[serde(rename = "nifi.properties", skip_serializing_if = "Option::is_none")] - pub nifi_properties: Option, + #[serde(rename = "bootstrap.conf", default)] + pub bootstrap_conf: KeyValueConfigOverrides, - #[serde( - rename = "security.properties", - skip_serializing_if = "Option::is_none" - )] - pub security_properties: Option, - } -} + #[serde(rename = "nifi.properties", default)] + pub nifi_properties: KeyValueConfigOverrides, -impl KeyValueOverridesProvider for v1alpha1::NifiConfigOverrides { - fn get_key_value_overrides(&self, file: &str) -> BTreeMap> { - let overrides = match file { - NIFI_BOOTSTRAP_CONF => &self.bootstrap_conf, - NIFI_PROPERTIES => &self.nifi_properties, - JVM_SECURITY_PROPERTIES_FILE => &self.security_properties, - _ => return BTreeMap::new(), - }; - overrides - .as_ref() - .map(|o| { - o.overrides - .iter() - .map(|(k, v)| (k.clone(), Some(v.clone()))) - .collect() - }) - .unwrap_or_default() - } -} - -impl Merge for v1alpha1::NifiConfigOverrides { - /// Merges per-file overrides: individual key-value pairs from `defaults` are - /// inserted only when the same key is absent from `self`. - fn merge(&mut self, defaults: &Self) { - fn merge_kv( - target: &mut Option, - default: &Option, - ) { - if let Some(default_kv) = default { - let target_kv = target.get_or_insert_with(Default::default); - for (k, v) in &default_kv.overrides { - target_kv.overrides.entry(k.clone()).or_insert(v.clone()); - } - } - } - - merge_kv(&mut self.bootstrap_conf, &defaults.bootstrap_conf); - merge_kv(&mut self.nifi_properties, &defaults.nifi_properties); - merge_kv(&mut self.security_properties, &defaults.security_properties); + #[serde(rename = "security.properties", default)] + pub security_properties: KeyValueConfigOverrides, } } @@ -586,7 +537,7 @@ mod merge_tests { use std::collections::BTreeMap; use stackable_operator::{ - config::merge::Merge as _, config_overrides::KeyValueConfigOverrides, + config::merge::Merge as _, v2::config_overrides::KeyValueConfigOverrides, }; use super::v1alpha1::NifiConfigOverrides; @@ -595,15 +546,15 @@ mod merge_tests { KeyValueConfigOverrides { overrides: pairs .iter() - .map(|(k, v)| (k.to_string(), v.to_string())) + .map(|(k, v)| (k.to_string(), Some(v.to_string()))) .collect::>(), } } fn overrides( - bootstrap: Option, - nifi: Option, - security: Option, + bootstrap: KeyValueConfigOverrides, + nifi: KeyValueConfigOverrides, + security: KeyValueConfigOverrides, ) -> NifiConfigOverrides { NifiConfigOverrides { bootstrap_conf: bootstrap, @@ -614,56 +565,78 @@ mod merge_tests { #[test] fn rolegroup_key_wins_over_role_key() { - let mut rg = overrides(Some(kv(&[("nifi.bootstrap.key", "rg-value")])), None, None); + let mut rg = overrides( + kv(&[("nifi.bootstrap.key", "rg-value")]), + Default::default(), + Default::default(), + ); let role = overrides( - Some(kv(&[("nifi.bootstrap.key", "role-value")])), - None, - None, + kv(&[("nifi.bootstrap.key", "role-value")]), + Default::default(), + Default::default(), ); rg.merge(&role); assert_eq!( - rg.bootstrap_conf.unwrap().overrides["nifi.bootstrap.key"], - "rg-value" + rg.bootstrap_conf.overrides["nifi.bootstrap.key"], + Some("rg-value".to_string()) ); } #[test] fn role_key_fills_gap_absent_from_rolegroup() { - let mut rg = overrides(Some(kv(&[("rg.only.key", "rg-value")])), None, None); + let mut rg = overrides( + kv(&[("rg.only.key", "rg-value")]), + Default::default(), + Default::default(), + ); let role = overrides( - Some(kv(&[ + kv(&[ ("rg.only.key", "role-value"), ("role.only.key", "role-default"), - ])), - None, - None, + ]), + Default::default(), + Default::default(), ); rg.merge(&role); - let result = rg.bootstrap_conf.unwrap(); - assert_eq!(result.overrides["rg.only.key"], "rg-value"); - assert_eq!(result.overrides["role.only.key"], "role-default"); + assert_eq!( + rg.bootstrap_conf.overrides["rg.only.key"], + Some("rg-value".to_string()) + ); + assert_eq!( + rg.bootstrap_conf.overrides["role.only.key"], + Some("role-default".to_string()) + ); } #[test] - fn none_field_adopts_role_values() { - let mut rg = overrides(None, Some(kv(&[("nifi.some.prop", "rg-val")])), None); + fn empty_field_adopts_role_values() { + let mut rg = overrides( + Default::default(), + kv(&[("nifi.some.prop", "rg-val")]), + Default::default(), + ); let role = overrides( - Some(kv(&[("nifi.bootstrap.key", "role-default")])), - Some(kv(&[ + kv(&[("nifi.bootstrap.key", "role-default")]), + kv(&[ ("nifi.some.prop", "role-val"), ("nifi.other.prop", "role-other"), - ])), - None, + ]), + Default::default(), ); rg.merge(&role); assert_eq!( - rg.bootstrap_conf.as_ref().unwrap().overrides["nifi.bootstrap.key"], - "role-default" + rg.bootstrap_conf.overrides["nifi.bootstrap.key"], + Some("role-default".to_string()) + ); + assert_eq!( + rg.nifi_properties.overrides["nifi.some.prop"], + Some("rg-val".to_string()) + ); + assert_eq!( + rg.nifi_properties.overrides["nifi.other.prop"], + Some("role-other".to_string()) ); - let nifi = rg.nifi_properties.as_ref().unwrap(); - assert_eq!(nifi.overrides["nifi.some.prop"], "rg-val"); - assert_eq!(nifi.overrides["nifi.other.prop"], "role-other"); - assert!(rg.security_properties.is_none()); + assert!(rg.security_properties.overrides.is_empty()); } } From 2bcfa15a3014f9aec72eb9a2d5df8c25c440f65f Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 9 Jun 2026 18:05:58 +0200 Subject: [PATCH 21/39] fix: remove error & comment cleanup --- rust/operator-binary/src/config/jvm.rs | 2 +- rust/operator-binary/src/controller.rs | 5 ----- rust/operator-binary/src/crd/mod.rs | 2 +- rust/operator-binary/src/reporting_task/mod.rs | 2 +- rust/operator-binary/src/security/authentication.rs | 4 ++-- 5 files changed, 5 insertions(+), 10 deletions(-) diff --git a/rust/operator-binary/src/config/jvm.rs b/rust/operator-binary/src/config/jvm.rs index 22bb4f9d..4bf19f30 100644 --- a/rust/operator-binary/src/config/jvm.rs +++ b/rust/operator-binary/src/config/jvm.rs @@ -30,7 +30,7 @@ pub enum Error { MergeJvmArgumentOverrides { source: role_utils::Error }, } -/// Create the NiFi bootstrap.conf +/// Build the merged JVM argument overrides used in the NiFi bootstrap.conf pub fn build_merged_jvm_config( merged_config: &NifiConfig, role: &NifiRoleType, diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 8fed6117..afed08e4 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -217,11 +217,6 @@ pub enum Error { source: stackable_operator::commons::rbac::Error, }, - #[snafu(display("Invalid NiFi Authorization Configuration"))] - InvalidNifiAuthorizationConfig { - source: crate::security::authorization::Error, - }, - #[snafu(display("failed to create PodDisruptionBudget"))] FailedToCreatePdb { source: crate::operations::pdb::Error, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index fce88f34..2b8d67db 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -211,7 +211,7 @@ impl HasStatusCondition for v1alpha1::NifiCluster { } impl v1alpha1::NifiCluster { - /// Metadata about a metastore rolegroup + /// Metadata about a Node rolegroup pub fn node_rolegroup_ref(&self, group_name: impl Into) -> RoleGroupRef { RoleGroupRef { cluster: ObjectRef::from_obj(self), diff --git a/rust/operator-binary/src/reporting_task/mod.rs b/rust/operator-binary/src/reporting_task/mod.rs index 4aecdf21..fa2b214a 100644 --- a/rust/operator-binary/src/reporting_task/mod.rs +++ b/rust/operator-binary/src/reporting_task/mod.rs @@ -149,7 +149,7 @@ pub fn build_maybe_reporting_task( } } -/// Return the name of the reporting task. +/// Return the name of the reporting task Service. pub fn build_reporting_task_service_name(nifi_cluster_name: &str) -> String { format!("{nifi_cluster_name}-{REPORTING_TASK_CONTAINER_NAME}") } diff --git a/rust/operator-binary/src/security/authentication.rs b/rust/operator-binary/src/security/authentication.rs index 11d6b6fc..c3c517de 100644 --- a/rust/operator-binary/src/security/authentication.rs +++ b/rust/operator-binary/src/security/authentication.rs @@ -206,8 +206,8 @@ impl NifiAuthenticationConfig { commands } - /// Returns - /// - A list of extra commands for the init container + /// Adds the volumes and volume mounts required by the configured authentication + /// method to the pod and the given container builders. pub fn add_volumes_and_mounts( &self, pod_builder: &mut PodBuilder, From d58382c270907a34aaafa7055b4c61f7091ddf59 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 9 Jun 2026 18:19:39 +0200 Subject: [PATCH 22/39] refactor: improve repo handling --- rust/operator-binary/src/config/mod.rs | 25 +++++++++++ rust/operator-binary/src/controller.rs | 59 ++++++-------------------- 2 files changed, 37 insertions(+), 47 deletions(-) diff --git a/rust/operator-binary/src/config/mod.rs b/rust/operator-binary/src/config/mod.rs index b2ade65c..d7ac693e 100644 --- a/rust/operator-binary/src/config/mod.rs +++ b/rust/operator-binary/src/config/mod.rs @@ -1,6 +1,7 @@ use std::{collections::BTreeMap, fmt::Write}; use snafu::Snafu; +use stackable_operator::k8s_openapi::api::core::v1::VolumeMount; use strum::{Display, EnumIter}; use crate::security::oidc; @@ -29,6 +30,21 @@ pub enum NifiRepository { State, } +/// The repositories that are backed by a [`PersistentVolume`] and therefore need a volume mount +/// in both the prepare and the nifi container. +/// +/// [`NifiRepository::Filebased`] is intentionally excluded: it is only mounted conditionally for +/// file-based authorization (see [`crate::security::authorization`]). +/// +/// [`PersistentVolume`]: stackable_operator::k8s_openapi::api::core::v1::PersistentVolume +pub const PERSISTENT_REPOSITORIES: [NifiRepository; 5] = [ + NifiRepository::Flowfile, + NifiRepository::Database, + NifiRepository::Content, + NifiRepository::Provenance, + NifiRepository::State, +]; + impl NifiRepository { pub fn repository(&self) -> String { format!("{}-repository", self) @@ -37,6 +53,15 @@ impl NifiRepository { pub fn mount_path(&self) -> String { format!("{NIFI_PVC_STORAGE_DIRECTORY}/{}", self) } + + /// The [`VolumeMount`] mounting this repository's volume into a container. + pub fn volume_mount(&self) -> VolumeMount { + VolumeMount { + name: self.repository(), + mount_path: self.mount_path(), + ..VolumeMount::default() + } + } } #[derive(Snafu, Debug)] diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index afed08e4..bb34edc7 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -70,7 +70,10 @@ use validate::NifiRoleGroupConfig; use crate::{ OPERATOR_NAME, - config::{NIFI_CONFIG_DIRECTORY, NIFI_PYTHON_WORKING_DIRECTORY, NifiRepository}, + config::{ + NIFI_CONFIG_DIRECTORY, NIFI_PYTHON_WORKING_DIRECTORY, NifiRepository, + PERSISTENT_REPOSITORIES, + }, crd::{ APP_NAME, BALANCE_PORT, BALANCE_PORT_NAME, Container, HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT, METRICS_PORT_NAME, NifiConfig, NifiNodeRoleConfig, NifiRole, NifiRoleType, @@ -805,29 +808,10 @@ async fn build_node_rolegroup_statefulset( ]) .add_env_vars(env_vars.clone()) .args(vec![prepare_args.join(" && ")]) - .add_volume_mount( - NifiRepository::Flowfile.repository(), - NifiRepository::Flowfile.mount_path(), - ) - .context(AddVolumeMountSnafu)? - .add_volume_mount( - NifiRepository::Database.repository(), - NifiRepository::Database.mount_path(), - ) - .context(AddVolumeMountSnafu)? - .add_volume_mount( - NifiRepository::Content.repository(), - NifiRepository::Content.mount_path(), - ) - .context(AddVolumeMountSnafu)? - .add_volume_mount( - NifiRepository::Provenance.repository(), - NifiRepository::Provenance.mount_path(), - ) - .context(AddVolumeMountSnafu)? - .add_volume_mount( - NifiRepository::State.repository(), - NifiRepository::State.mount_path(), + .add_volume_mounts( + PERSISTENT_REPOSITORIES + .iter() + .map(NifiRepository::volume_mount), ) .context(AddVolumeMountSnafu)? .add_volume_mount("conf", "/conf") @@ -891,29 +875,10 @@ async fn build_node_rolegroup_statefulset( .add_env_vars(env_vars) .add_volume_mount(KEYSTORE_VOLUME_NAME, KEYSTORE_NIFI_CONTAINER_MOUNT) .context(AddVolumeMountSnafu)? - .add_volume_mount( - NifiRepository::Flowfile.repository(), - NifiRepository::Flowfile.mount_path(), - ) - .context(AddVolumeMountSnafu)? - .add_volume_mount( - NifiRepository::Database.repository(), - NifiRepository::Database.mount_path(), - ) - .context(AddVolumeMountSnafu)? - .add_volume_mount( - NifiRepository::Content.repository(), - NifiRepository::Content.mount_path(), - ) - .context(AddVolumeMountSnafu)? - .add_volume_mount( - NifiRepository::Provenance.repository(), - NifiRepository::Provenance.mount_path(), - ) - .context(AddVolumeMountSnafu)? - .add_volume_mount( - NifiRepository::State.repository(), - NifiRepository::State.mount_path(), + .add_volume_mounts( + PERSISTENT_REPOSITORIES + .iter() + .map(NifiRepository::volume_mount), ) .context(AddVolumeMountSnafu)? .add_volume_mount("activeconf", NIFI_CONFIG_DIRECTORY) From 24a157ab069c0120ed5cc0b0b54293a0ca72883a Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 9 Jun 2026 21:02:30 +0200 Subject: [PATCH 23/39] refactor: switch to EnvVarSet --- rust/operator-binary/src/controller.rs | 35 ++------ .../src/controller/build/git_sync.rs | 7 +- .../build/properties/security_properties.rs | 6 +- .../src/controller/validate.rs | 8 +- rust/operator-binary/src/crd/affinity.rs | 14 ++- rust/operator-binary/src/crd/mod.rs | 48 +--------- .../src/framework/role_utils.rs | 87 ++++++++++++++++--- 7 files changed, 106 insertions(+), 99 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index bb34edc7..9b6da330 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -196,9 +196,6 @@ pub enum Error { container_name: String, }, - #[snafu(display("failed to resolve and merge config for role and role group"))] - FailedToResolveConfig { source: crate::crd::Error }, - #[snafu(display("failed to build git-sync resources"))] BuildGitSyncResources { source: build::git_sync::Error }, @@ -406,11 +403,6 @@ pub async fn reconcile_nifi( tracing::debug!("Processing rolegroup {}", rolegroup); - let merged_config = nifi - .merged_config(&nifi_role, rolegroup_name) - .context(FailedToResolveConfigSnafu)?; - - // Computed here for the StatefulSet; the ConfigMap builder computes its own copy. let git_sync_resources = build::git_sync::build_git_sync_resources(&validated_cluster, rg) .context(BuildGitSyncResourcesSnafu)?; @@ -462,7 +454,6 @@ pub async fn reconcile_nifi( &rolegroup, role, rg, - &merged_config, authentication_config, authorization_config, rolling_upgrade_supported, @@ -617,18 +608,6 @@ pub async fn reconcile_nifi( const USERDATA_MOUNTPOINT: &str = "/stackable/userdata"; -/// Build a `Vec` from a plain `BTreeMap` of env overrides. -fn env_vars_from_overrides(env_overrides: &BTreeMap) -> Vec { - env_overrides - .iter() - .map(|(name, value)| EnvVar { - name: name.clone(), - value: Some(value.clone()), - ..EnvVar::default() - }) - .collect() -} - /// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. /// /// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the @@ -642,7 +621,6 @@ async fn build_node_rolegroup_statefulset( rolegroup_ref: &RoleGroupRef, role: &NifiRoleType, rg: &NifiRoleGroupConfig, - merged_config: &NifiConfig, authentication_config: &NifiAuthenticationConfig, authorization_config: &ResolvedNifiAuthorizationConfig, rolling_update_supported: bool, @@ -651,9 +629,12 @@ async fn build_node_rolegroup_statefulset( git_sync_resources: &git_sync::v1alpha2::GitSyncResources, ) -> Result { tracing::debug!("Building statefulset"); - let role_group = role.role_groups.get(&rolegroup_ref.role_group); - let mut env_vars: Vec = env_vars_from_overrides(&rg.env_overrides); + // The validated, merged `NifiConfig` is the single source of truth; the ConfigMap builder + // sources the same `rg.config`. + let merged_config = &rg.config; + + let mut env_vars: Vec = rg.env_overrides.clone().into(); // we need the POD_NAME env var to overwrite `nifi.cluster.node.address` later env_vars.push(EnvVar { @@ -1153,10 +1134,8 @@ async fn build_node_rolegroup_statefulset( ); let mut pod_template = pod_builder.build_template(); - pod_template.merge_from(role.config.pod_overrides.clone()); - if let Some(role_group) = role_group { - pod_template.merge_from(role_group.config.pod_overrides.clone()); - } + // `rg.pod_overrides` is already the role <- rolegroup merge produced by the framework. + pod_template.merge_from(rg.pod_overrides.clone()); Ok(StatefulSet { metadata: ObjectMetaBuilder::new() diff --git a/rust/operator-binary/src/controller/build/git_sync.rs b/rust/operator-binary/src/controller/build/git_sync.rs index 0acce6e3..5b86cf12 100644 --- a/rust/operator-binary/src/controller/build/git_sync.rs +++ b/rust/operator-binary/src/controller/build/git_sync.rs @@ -1,11 +1,11 @@ //! Builds the git-sync resources (volumes, mounts, containers) for a NiFi Node rolegroup. use snafu::{ResultExt, Snafu}; -use stackable_operator::crd::git_sync; +use stackable_operator::{crd::git_sync, k8s_openapi::api::core::v1::EnvVar}; use crate::{ controller::{ - LOG_VOLUME_NAME, env_vars_from_overrides, + LOG_VOLUME_NAME, validate::{NifiRoleGroupConfig, ValidatedCluster}, }, crd::Container, @@ -26,10 +26,11 @@ pub fn build_git_sync_resources( cluster: &ValidatedCluster, rg: &NifiRoleGroupConfig, ) -> Result { + let env_vars: Vec = rg.env_overrides.clone().into(); git_sync::v1alpha2::GitSyncResources::new( &cluster.cluster_config.custom_components_git_sync, &cluster.image, - &env_vars_from_overrides(&rg.env_overrides), + &env_vars, &[], LOG_VOLUME_NAME, &rg.config.logging.for_container(&Container::GitSync), diff --git a/rust/operator-binary/src/controller/build/properties/security_properties.rs b/rust/operator-binary/src/controller/build/properties/security_properties.rs index e2ab7afc..62b097de 100644 --- a/rust/operator-binary/src/controller/build/properties/security_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/security_properties.rs @@ -27,7 +27,9 @@ pub fn build(rg: &NifiRoleGroupConfig) -> Result mod tests { use std::collections::BTreeMap; - use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; + use stackable_operator::v2::{ + builder::pod::container::EnvVarSet, config_overrides::KeyValueConfigOverrides, + }; use super::*; use crate::{ @@ -50,7 +52,7 @@ mod tests { }, ..Default::default() }, - env_overrides: BTreeMap::new(), + env_overrides: EnvVarSet::new(), cli_overrides: BTreeMap::new(), pod_overrides: Default::default(), product_specific_common_config: JavaCommonConfig::default(), diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 71884ff4..923776ac 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -58,9 +58,9 @@ pub enum Error { #[snafu(display("invalid NiFi authentication configuration"))] InvalidAuthenticationConfig { source: authentication::Error }, - #[snafu(display("failed to validate config fragment for a rolegroup"))] - InvalidConfigFragment { - source: stackable_operator::config::fragment::ValidationError, + #[snafu(display("failed to build the config for a rolegroup"))] + BuildRoleGroupConfig { + source: crate::framework::role_utils::Error, }, #[snafu(display("invalid sensitive properties algorithm"))] @@ -263,7 +263,7 @@ fn build_role_group_configs( for (rg_name, rg) in &role.role_groups { let validated_rg = with_validated_config::(rg, role, &default_config) - .context(InvalidConfigFragmentSnafu)?; + .context(BuildRoleGroupConfigSnafu)?; groups.insert(rg_name.clone(), validated_rg); } diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 75f2c2a3..5ac8801e 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -32,7 +32,10 @@ mod tests { }; use super::*; - use crate::crd::v1alpha1; + use crate::{ + crd::{NifiConfig, v1alpha1}, + framework::role_utils::with_validated_config, + }; #[test] fn test_affinity_defaults() { @@ -58,7 +61,14 @@ mod tests { let deserializer = serde_yaml::Deserializer::from_str(input); let nifi: v1alpha1::NifiCluster = serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); - let merged_config = nifi.merged_config(&NifiRole::Node, "default").unwrap(); + + let role = nifi.spec.nodes.as_ref().unwrap(); + let default_config = NifiConfig::default_config("simple-nifi", &NifiRole::Node); + let role_group = role.role_groups.get("default").unwrap(); + let merged_config = + with_validated_config::(role_group, role, &default_config) + .unwrap() + .config; assert_eq!( merged_config.affinity, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 2b8d67db..f4effcbe 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -7,7 +7,6 @@ use affinity::get_affinity; use authorization::NifiAuthorization; use sensitive_properties::NifiSensitivePropertiesConfig; use serde::{Deserialize, Serialize}; -use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ commons::{ affinity::StackableAffinity, @@ -18,17 +17,14 @@ use stackable_operator::{ PvcConfig, PvcConfigFragment, Resources, ResourcesFragment, }, }, - config::{ - fragment::{self, Fragment, ValidationError}, - merge::Merge, - }, + config::{fragment::Fragment, merge::Merge}, crd::{authentication::core as auth_core, git_sync}, deep_merger::ObjectOverrides, k8s_openapi::{ api::core::v1::{PodTemplateSpec, Volume}, apimachinery::pkg::api::resource::Quantity, }, - kube::{CustomResource, ResourceExt, runtime::reflector::ObjectRef}, + kube::{CustomResource, runtime::reflector::ObjectRef}, memory::MemoryQuantity, product_logging::{self, spec::Logging}, role_utils::{GenericRoleConfig, JavaCommonConfig, Role, RoleGroupRef}, @@ -62,15 +58,6 @@ const DEFAULT_NODE_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_ pub type NifiRoleType = Role; -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("the NiFi role [{role}] is missing from spec"))] - MissingNifiRole { role: String }, - - #[snafu(display("fragment validation failure"))] - FragmentValidationFailure { source: ValidationError }, -} - #[versioned( version(name = "v1alpha1"), crates( @@ -230,37 +217,6 @@ impl v1alpha1::NifiCluster { pub fn server_tls_secret_class(&self) -> &str { &self.spec.cluster_config.tls.server_secret_class } - - /// Retrieve and merge resource configs for role and role groups - pub fn merged_config(&self, role: &NifiRole, role_group: &str) -> Result { - // Initialize the result with all default values as baseline - let conf_defaults = NifiConfig::default_config(&self.name_any(), role); - - let role = self.spec.nodes.as_ref().context(MissingNifiRoleSnafu { - role: role.to_string(), - })?; - - // Retrieve role resource config - let mut conf_role = role.config.config.to_owned(); - - // Retrieve rolegroup specific resource config - let mut conf_rolegroup = role - .role_groups - .get(role_group) - .map(|rg| rg.config.config.clone()) - .unwrap_or_default(); - - // Merge more specific configs into default config - // Hierarchy is: - // 1. RoleGroup - // 2. Role - // 3. Default - conf_role.merge(&conf_defaults); - conf_rolegroup.merge(&conf_role); - - tracing::debug!("Merged config: {:?}", conf_rolegroup); - fragment::validate(conf_rolegroup).context(FragmentValidationFailureSnafu) - } } #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] diff --git a/rust/operator-binary/src/framework/role_utils.rs b/rust/operator-binary/src/framework/role_utils.rs index ed5a04b8..d8ca6ea8 100644 --- a/rust/operator-binary/src/framework/role_utils.rs +++ b/rust/operator-binary/src/framework/role_utils.rs @@ -2,7 +2,6 @@ //! `smooth-operator` branch, with simplifications appropriate for nifi-operator. //! //! Differences from upstream: -//! - `env_overrides` is `BTreeMap` instead of `EnvVarSet`. //! - No `cli_overrides_to_vec` helper, `ResourceNames`, or service-account helpers. //! - The `CommonConfig` (a.k.a. `product_specific_common_config`) does NOT need to //! implement `Merge`. Upstream Trino uses `JavaCommonConfig`, which intentionally @@ -15,9 +14,10 @@ //! Replace with `stackable_operator::v2::role_utils::*` once upstream publishes //! the module. -use std::collections::BTreeMap; +use std::{collections::BTreeMap, str::FromStr}; use serde::Serialize; +use snafu::{ResultExt, Snafu}; use stackable_operator::{ config::{ fragment::{self, FromFragment}, @@ -26,19 +26,31 @@ use stackable_operator::{ k8s_openapi::{DeepMerge, api::core::v1::PodTemplateSpec}, role_utils::{Role, RoleGroup}, schemars::JsonSchema, + v2::builder::pod::container::{self, EnvVarName, EnvVarSet}, }; +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to validate the rolegroup config fragment"))] + ValidateConfig { source: fragment::ValidationError }, + + #[snafu(display("environment variable name {name:?} is invalid"))] + ParseEnvVarName { + source: container::Error, + name: String, + }, +} + /// NiFi-friendly view of a validated, merged `RoleGroup`. /// /// Mirrors `stackable_operator::v2::role_utils::RoleGroupConfig` on the -/// `smooth-operator` branch, with `env_overrides: BTreeMap` -/// instead of the upstream `EnvVarSet`. +/// `smooth-operator` branch. #[derive(Clone, Debug, PartialEq)] pub struct RoleGroupConfig { pub replicas: u16, pub config: Config, pub config_overrides: ConfigOverrides, - pub env_overrides: BTreeMap, + pub env_overrides: EnvVarSet, pub cli_overrides: BTreeMap, pub pod_overrides: PodTemplateSpec, pub product_specific_common_config: CommonConfig, @@ -51,7 +63,9 @@ pub struct RoleGroupConfig { /// - `Config` (Fragment): `default_config <- role.config <- rg.config` via `Merge::merge`, /// then validated to `ValidatedConfig` via `FromFragment`. /// - `ConfigOverrides`: `role.config_overrides <- rg.config_overrides` via `Merge::merge`. -/// - `env_overrides` / `cli_overrides`: `extend` (rg keys overwrite role keys). +/// - `env_overrides`: `extend` (rg keys overwrite role keys), then parsed into an `EnvVarSet` +/// (rejecting invalid environment variable names). +/// - `cli_overrides`: `extend` (rg keys overwrite role keys). /// - `pod_overrides`: `DeepMerge::merge_from` (rg overrides role). /// - `product_specific_common_config`: passes through the role-group level value /// (see module docs for rationale). @@ -59,10 +73,7 @@ pub fn with_validated_config, role: &Role, default_config: &Config, -) -> Result< - RoleGroupConfig, - fragment::ValidationError, -> +) -> Result, Error> where ValidatedConfig: FromFragment, CommonConfig: Clone + Default + JsonSchema + Serialize, @@ -70,7 +81,8 @@ where RoleConfig: Default + JsonSchema + Serialize, ConfigOverrides: Clone + Default + JsonSchema + Merge + Serialize, { - let validated_config = validate_config(role_group, role, default_config)?; + let validated_config = + validate_config(role_group, role, default_config).context(ValidateConfigSnafu)?; Ok(RoleGroupConfig { replicas: role_group.replicas.unwrap_or(1), config: validated_config, @@ -90,7 +102,7 @@ where .iter() .map(|(k, v)| (k.clone(), v.clone())) .collect(), - ), + )?, cli_overrides: merged_cli_overrides( role.config.cli_overrides.clone(), role_group.config.cli_overrides.clone(), @@ -131,10 +143,17 @@ where fn merged_env_overrides( role_env_overrides: BTreeMap, role_group_env_overrides: BTreeMap, -) -> BTreeMap { +) -> Result { let mut merged = role_env_overrides; merged.extend(role_group_env_overrides); - merged + + let mut env_overrides = EnvVarSet::new(); + for (name, value) in merged { + let env_var_name = + EnvVarName::from_str(&name).context(ParseEnvVarNameSnafu { name: name.clone() })?; + env_overrides = env_overrides.with_value(&env_var_name, value); + } + Ok(env_overrides) } fn merged_cli_overrides( @@ -154,3 +173,43 @@ fn merged_pod_overrides( merged.merge_from(role_group_pod_overrides); merged } + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use stackable_operator::v2::builder::pod::container::EnvVarName; + + use super::merged_env_overrides; + + #[test] + fn env_overrides_role_group_value_wins_over_role_value() { + let role = BTreeMap::from([ + ("SHARED".to_owned(), "from-role".to_owned()), + ("ROLE_ONLY".to_owned(), "role".to_owned()), + ]); + let role_group = BTreeMap::from([("SHARED".to_owned(), "from-rolegroup".to_owned())]); + + let merged = merged_env_overrides(role, role_group).expect("env overrides should merge"); + + assert_eq!( + merged + .get(&EnvVarName::from_str_unsafe("SHARED")) + .and_then(|env_var| env_var.value.clone()), + Some("from-rolegroup".to_owned()) + ); + assert_eq!( + merged + .get(&EnvVarName::from_str_unsafe("ROLE_ONLY")) + .and_then(|env_var| env_var.value.clone()), + Some("role".to_owned()) + ); + } + + #[test] + fn invalid_env_var_name_is_rejected() { + let role = BTreeMap::from([("INVALID=NAME".to_owned(), "value".to_owned())]); + + assert!(merged_env_overrides(role, BTreeMap::new()).is_err()); + } +} From 085af206ac78e7a422095fe42740f3efcf15ab7e Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 10 Jun 2026 12:10:37 +0200 Subject: [PATCH 24/39] fix: bump stackable-operator for non optional KeyValueConfigOverrides --- Cargo.lock | 34 ++++++------ Cargo.nix | 52 +++++++++---------- crate-hashes.json | 18 +++---- extra/crds.yaml | 42 +++------------ .../src/controller/build/properties.rs | 25 --------- .../build/properties/bootstrap_conf.rs | 5 +- .../build/properties/nifi_properties.rs | 5 +- .../build/properties/security_properties.rs | 11 +--- rust/operator-binary/src/crd/mod.rs | 23 +++----- 9 files changed, 69 insertions(+), 146 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4b6f5acf..4ef5d4e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1571,7 +1571,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "darling", "regex", @@ -2417,9 +2417,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.3" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ "aho-corasick", "memchr", @@ -2440,9 +2440,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "relative-path" @@ -2989,7 +2989,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "const-oid", "ecdsa", @@ -3039,7 +3039,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.111.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "base64", "clap", @@ -3083,7 +3083,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "darling", "proc-macro2", @@ -3094,7 +3094,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "jiff", "k8s-openapi", @@ -3111,7 +3111,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.4" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "axum", "clap", @@ -3135,7 +3135,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "kube", "schemars", @@ -3149,7 +3149,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "convert_case", "convert_case_extras", @@ -3167,7 +3167,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "arc-swap", "async-trait", @@ -4224,18 +4224,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.50" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.50" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.nix b/Cargo.nix index 1f22b290..60fc7565 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -5066,8 +5066,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; - sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "k8s_version"; authors = [ @@ -7914,9 +7914,9 @@ rec { }; "regex" = rec { crateName = "regex"; - version = "1.12.3"; + version = "1.12.4"; edition = "2021"; - sha256 = "0xp2q0x7ybmpa5zlgaz00p8zswcirj9h8nry3rxxsdwi9fhm81z1"; + sha256 = "1fm6si2xpmhwqflabdqsakc0qkq718wx2ljl37nbj75fb5vjnagi"; authors = [ "The Rust Project Developers" "Andrew Gallant " @@ -8033,9 +8033,9 @@ rec { }; "regex-syntax" = rec { crateName = "regex-syntax"; - version = "0.8.10"; + version = "0.8.11"; edition = "2021"; - sha256 = "02jx311ka0daxxc7v45ikzhcl3iydjbbb0mdrpc1xgg8v7c7v2fw"; + sha256 = "1m25h5q2wp976fb9gc3dsc9l99svcvd5cri8lncb51c46ydgzxnn"; libName = "regex_syntax"; authors = [ "The Rust Project Developers" @@ -9860,8 +9860,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; - sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_certs"; authors = [ @@ -10072,8 +10072,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; - sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_operator"; authors = [ @@ -10266,8 +10266,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; - sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -10301,8 +10301,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; - sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_shared"; authors = [ @@ -10382,8 +10382,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; - sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_telemetry"; authors = [ @@ -10492,8 +10492,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; - sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_versioned"; authors = [ @@ -10542,8 +10542,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; - sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -10610,8 +10610,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; - sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_webhook"; authors = [ @@ -14896,9 +14896,9 @@ rec { }; "zerocopy" = rec { crateName = "zerocopy"; - version = "0.8.50"; + version = "0.8.52"; edition = "2021"; - sha256 = "1laahnfxs4qyfb1fdf5nbb2qfshi72b1hbi0ffp2zy2m1r7ms1iv"; + sha256 = "0gv563swc1yn3k8w3wjj07a8q293rkx99nfp3a25vzzmbycj446f"; authors = [ "Joshua Liebow-Feeser " "Jack Wrenn " @@ -14932,9 +14932,9 @@ rec { }; "zerocopy-derive" = rec { crateName = "zerocopy-derive"; - version = "0.8.50"; + version = "0.8.52"; edition = "2021"; - sha256 = "0fdnr9qslx1hbn2i9rsvy9s95mychfy2vj90ajsjm2basccinqqb"; + sha256 = "0c3rhsh4sd9kdym4z55zprybjkydy9y2gvw75d72aapcfa5z7rqs"; procMacro = true; libName = "zerocopy_derive"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index cd03561e..c9a6e6a9 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file diff --git a/extra/crds.yaml b/extra/crds.yaml index e6ecf94f..3d6c1ce0 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -1266,36 +1266,21 @@ spec: properties: bootstrap.conf: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object nifi.properties: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object security.properties: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object type: object envOverrides: @@ -2140,36 +2125,21 @@ spec: properties: bootstrap.conf: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object nifi.properties: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object security.properties: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object type: object envOverrides: diff --git a/rust/operator-binary/src/controller/build/properties.rs b/rust/operator-binary/src/controller/build/properties.rs index 08d720b9..eada5d01 100644 --- a/rust/operator-binary/src/controller/build/properties.rs +++ b/rust/operator-binary/src/controller/build/properties.rs @@ -4,8 +4,6 @@ //! The shared [`stackable_operator::v2::config_file_writer`] module serializes `.properties`/`.conf` //! key/value maps to the Java-properties on-wire format. -use crate::controller::validate::NifiRoleGroupConfig; - pub mod authorizers; pub mod bootstrap_conf; pub mod logging; @@ -33,29 +31,6 @@ pub enum ConfigFileName { Logback, } -/// Resolve the user overrides for `file` from a rolegroup's config overrides. -/// -/// Keys whose value is unset (`None`, i.e. `key: null` in YAML) are dropped, so only the defined -/// `(key, value)` pairs reach the property writer. -pub(crate) fn resolved_overrides_for( - rg: &NifiRoleGroupConfig, - file: ConfigFileName, -) -> impl Iterator + '_ { - let overrides = match file { - ConfigFileName::BootstrapConf => Some(&rg.config_overrides.bootstrap_conf), - ConfigFileName::NifiProperties => Some(&rg.config_overrides.nifi_properties), - ConfigFileName::SecurityProperties => Some(&rg.config_overrides.security_properties), - ConfigFileName::StateManagementXml - | ConfigFileName::LoginIdentityProviders - | ConfigFileName::Authorizers - | ConfigFileName::Logback => None, - }; - overrides - .into_iter() - .flat_map(|o| &o.overrides) - .filter_map(|(key, value)| value.clone().map(|value| (key.clone(), value))) -} - /// Test helpers for constructing a minimal [`ValidatedCluster`] and related types without /// requiring Kubernetes API access. /// diff --git a/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs b/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs index f274db81..050a893e 100644 --- a/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs +++ b/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs @@ -4,7 +4,6 @@ use std::collections::BTreeMap; use snafu::ResultExt; -use super::ConfigFileName; use crate::{ config::{Error, InvalidJVMConfigSnafu, jvm::build_merged_jvm_config}, controller::validate::NifiRoleGroupConfig, @@ -44,9 +43,7 @@ pub fn build( } // configOverrides come last - for (k, v) in super::resolved_overrides_for(rg, ConfigFileName::BootstrapConf) { - bootstrap.insert(k, v); - } + bootstrap.extend(rg.config_overrides.bootstrap_conf.overrides.clone()); Ok(crate::config::format_properties(bootstrap)) } diff --git a/rust/operator-binary/src/controller/build/properties/nifi_properties.rs b/rust/operator-binary/src/controller/build/properties/nifi_properties.rs index 25c2529d..419fa6d2 100644 --- a/rust/operator-binary/src/controller/build/properties/nifi_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/nifi_properties.rs @@ -5,7 +5,6 @@ use std::collections::BTreeMap; use snafu::{ResultExt, ensure}; use stackable_operator::{crd::git_sync, memory::MemoryQuantity}; -use super::ConfigFileName; use crate::{ config::{ CalculateStorageQuotaSnafu, Error, GenerateOidcConfigSnafu, NIFI_PYTHON_WORKING_DIRECTORY, @@ -569,9 +568,7 @@ pub fn build( //########################## // override with config overrides - for (k, v) in super::resolved_overrides_for(rg, ConfigFileName::NifiProperties) { - properties.insert(k, v); - } + properties.extend(rg.config_overrides.nifi_properties.overrides.clone()); Ok(format_properties(properties)) } diff --git a/rust/operator-binary/src/controller/build/properties/security_properties.rs b/rust/operator-binary/src/controller/build/properties/security_properties.rs index 62b097de..a990febb 100644 --- a/rust/operator-binary/src/controller/build/properties/security_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/security_properties.rs @@ -6,7 +6,6 @@ use stackable_operator::v2::config_file_writer::{ PropertiesWriterError, to_java_properties_string, }; -use super::ConfigFileName; use crate::controller::validate::NifiRoleGroupConfig; pub fn build(rg: &NifiRoleGroupConfig) -> Result { @@ -17,9 +16,7 @@ pub fn build(rg: &NifiRoleGroupConfig) -> Result "networkaddress.cache.negative.ttl".to_string(), "0".to_string(), ); - for (k, v) in super::resolved_overrides_for(rg, ConfigFileName::SecurityProperties) { - props.insert(k, v); - } + props.extend(rg.config_overrides.security_properties.overrides.clone()); to_java_properties_string(props.iter()) } @@ -44,11 +41,7 @@ mod tests { config: NifiConfig::default(), config_overrides: NifiConfigOverrides { security_properties: KeyValueConfigOverrides { - overrides: overrides - .unwrap_or_default() - .into_iter() - .map(|(k, v)| (k, Some(v))) - .collect(), + overrides: overrides.unwrap_or_default(), }, ..Default::default() }, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index f4effcbe..9cd7b54e 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -502,7 +502,7 @@ mod merge_tests { KeyValueConfigOverrides { overrides: pairs .iter() - .map(|(k, v)| (k.to_string(), Some(v.to_string()))) + .map(|(k, v)| (k.to_string(), v.to_string())) .collect::>(), } } @@ -534,7 +534,7 @@ mod merge_tests { rg.merge(&role); assert_eq!( rg.bootstrap_conf.overrides["nifi.bootstrap.key"], - Some("rg-value".to_string()) + "rg-value" ); } @@ -554,14 +554,8 @@ mod merge_tests { Default::default(), ); rg.merge(&role); - assert_eq!( - rg.bootstrap_conf.overrides["rg.only.key"], - Some("rg-value".to_string()) - ); - assert_eq!( - rg.bootstrap_conf.overrides["role.only.key"], - Some("role-default".to_string()) - ); + assert_eq!(rg.bootstrap_conf.overrides["rg.only.key"], "rg-value"); + assert_eq!(rg.bootstrap_conf.overrides["role.only.key"], "role-default"); } #[test] @@ -582,15 +576,12 @@ mod merge_tests { rg.merge(&role); assert_eq!( rg.bootstrap_conf.overrides["nifi.bootstrap.key"], - Some("role-default".to_string()) - ); - assert_eq!( - rg.nifi_properties.overrides["nifi.some.prop"], - Some("rg-val".to_string()) + "role-default" ); + assert_eq!(rg.nifi_properties.overrides["nifi.some.prop"], "rg-val"); assert_eq!( rg.nifi_properties.overrides["nifi.other.prop"], - Some("role-other".to_string()) + "role-other" ); assert!(rg.security_properties.overrides.is_empty()); } From 70f457d348029b2dca79be0c18b2d83ce38684df Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 11 Jun 2026 12:35:27 +0200 Subject: [PATCH 25/39] renamed controller, moved validated structs into controller/mod.rs, added validated cluster usage in place of raw cluster in some places --- rust/operator-binary/src/controller/build.rs | 2 +- .../src/controller/build/config_map.rs | 4 +- .../src/controller/build/git_sync.rs | 6 +- .../src/controller/build/properties.rs | 2 +- .../build/properties/authorizers.rs | 2 +- .../properties/login_identity_providers.rs | 2 +- .../build/properties/nifi_properties.rs | 2 +- .../src/controller/build/proxy_hosts.rs | 2 +- rust/operator-binary/src/controller/mod.rs | 147 ++++++++++++++++++ .../src/controller/validate.rs | 135 +--------------- rust/operator-binary/src/main.rs | 9 +- .../src/{controller.rs => nifi_controller.rs} | 27 ++-- rust/operator-binary/src/operations/pdb.rs | 2 +- .../operator-binary/src/reporting_task/mod.rs | 2 +- rust/operator-binary/src/service.rs | 25 +-- 15 files changed, 195 insertions(+), 174 deletions(-) create mode 100644 rust/operator-binary/src/controller/mod.rs rename rust/operator-binary/src/{controller.rs => nifi_controller.rs} (98%) diff --git a/rust/operator-binary/src/controller/build.rs b/rust/operator-binary/src/controller/build.rs index cae56571..1b225f1d 100644 --- a/rust/operator-binary/src/controller/build.rs +++ b/rust/operator-binary/src/controller/build.rs @@ -1,6 +1,6 @@ //! Builders that assemble Kubernetes resources from a [`ValidatedCluster`]. //! -//! [`ValidatedCluster`]: crate::controller::validate::ValidatedCluster +//! [`ValidatedCluster`]: crate::controller::ValidatedCluster pub mod config_map; pub mod git_sync; diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index dacf4cd0..99e5d88f 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -13,6 +13,7 @@ use stackable_operator::{ use crate::{ controller::{ + ValidatedCluster, build::{ git_sync, properties::{ @@ -21,7 +22,6 @@ use crate::{ }, proxy_hosts, }, - validate::ValidatedCluster, }, crd::{NifiRole, v1alpha1}, }; @@ -80,7 +80,7 @@ type Result = std::result::Result; pub fn build_rolegroup_config_map( cluster: &ValidatedCluster, rolegroup: &RoleGroupRef, - recommended_labels: &ObjectLabels<'_, v1alpha1::NifiCluster>, + recommended_labels: &ObjectLabels<'_, ValidatedCluster>, cluster_info: &KubernetesClusterInfo, ) -> Result { tracing::debug!("building rolegroup ConfigMap"); diff --git a/rust/operator-binary/src/controller/build/git_sync.rs b/rust/operator-binary/src/controller/build/git_sync.rs index 5b86cf12..f6c2f8b6 100644 --- a/rust/operator-binary/src/controller/build/git_sync.rs +++ b/rust/operator-binary/src/controller/build/git_sync.rs @@ -4,11 +4,9 @@ use snafu::{ResultExt, Snafu}; use stackable_operator::{crd::git_sync, k8s_openapi::api::core::v1::EnvVar}; use crate::{ - controller::{ - LOG_VOLUME_NAME, - validate::{NifiRoleGroupConfig, ValidatedCluster}, - }, + controller::{ValidatedCluster, validate::NifiRoleGroupConfig}, crd::Container, + nifi_controller::LOG_VOLUME_NAME, }; #[derive(Snafu, Debug)] diff --git a/rust/operator-binary/src/controller/build/properties.rs b/rust/operator-binary/src/controller/build/properties.rs index eada5d01..9f306e5a 100644 --- a/rust/operator-binary/src/controller/build/properties.rs +++ b/rust/operator-binary/src/controller/build/properties.rs @@ -69,7 +69,7 @@ pub(crate) mod test_support { }; use crate::{ - controller::validate::{NifiRoleGroupConfig, ValidatedCluster, ValidatedClusterConfig}, + controller::{ValidatedCluster, ValidatedClusterConfig, validate::NifiRoleGroupConfig}, crd::{NifiConfig, NifiRole, v1alpha1}, framework::role_utils::with_validated_config, security::{ diff --git a/rust/operator-binary/src/controller/build/properties/authorizers.rs b/rust/operator-binary/src/controller/build/properties/authorizers.rs index 7f67c661..ba61afec 100644 --- a/rust/operator-binary/src/controller/build/properties/authorizers.rs +++ b/rust/operator-binary/src/controller/build/properties/authorizers.rs @@ -1,6 +1,6 @@ //! Builder for `authorizers.xml`. -use crate::controller::validate::ValidatedCluster; +use crate::controller::ValidatedCluster; pub fn build(cluster: &ValidatedCluster) -> String { cluster diff --git a/rust/operator-binary/src/controller/build/properties/login_identity_providers.rs b/rust/operator-binary/src/controller/build/properties/login_identity_providers.rs index 3993a67a..94e08d2f 100644 --- a/rust/operator-binary/src/controller/build/properties/login_identity_providers.rs +++ b/rust/operator-binary/src/controller/build/properties/login_identity_providers.rs @@ -1,6 +1,6 @@ //! Builder for `login-identity-providers.xml`. -use crate::controller::validate::ValidatedCluster; +use crate::controller::ValidatedCluster; pub fn build(cluster: &ValidatedCluster) -> Result { cluster diff --git a/rust/operator-binary/src/controller/build/properties/nifi_properties.rs b/rust/operator-binary/src/controller/build/properties/nifi_properties.rs index 419fa6d2..25a8494a 100644 --- a/rust/operator-binary/src/controller/build/properties/nifi_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/nifi_properties.rs @@ -10,7 +10,7 @@ use crate::{ CalculateStorageQuotaSnafu, Error, GenerateOidcConfigSnafu, NIFI_PYTHON_WORKING_DIRECTORY, Nifi1RequiresZookeeperSnafu, NifiRepository, format_properties, }, - controller::validate::{NifiRoleGroupConfig, ValidatedCluster}, + controller::{ValidatedCluster, validate::NifiRoleGroupConfig}, crd::{HTTPS_PORT, v1alpha1}, security::{ authentication::{ diff --git a/rust/operator-binary/src/controller/build/proxy_hosts.rs b/rust/operator-binary/src/controller/build/proxy_hosts.rs index 233ba54e..7f215683 100644 --- a/rust/operator-binary/src/controller/build/proxy_hosts.rs +++ b/rust/operator-binary/src/controller/build/proxy_hosts.rs @@ -4,7 +4,7 @@ use std::collections::HashSet; use stackable_operator::utils::cluster_info::KubernetesClusterInfo; -use crate::{controller::validate::ValidatedCluster, crd::HTTPS_PORT, reporting_task}; +use crate::{controller::ValidatedCluster, crd::HTTPS_PORT, reporting_task}; /// Computes the comma-separated NiFi proxy hosts, or `"*"` if `hostHeaderCheck.allowAll` is set. /// diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs new file mode 100644 index 00000000..96685a38 --- /dev/null +++ b/rust/operator-binary/src/controller/mod.rs @@ -0,0 +1,147 @@ +//! Controller-level vocabulary: the [`ValidatedCluster`] type produced by the +//! [`validate`] step and consumed by the [`build`] steps, plus the +//! `dereference` / `validate` / `build` sub-modules. + +use std::collections::BTreeMap; + +use stackable_operator::{ + commons::product_image_selection::ResolvedProductImage, + crd::git_sync, + k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta, + kube::Resource, + v2::{ + HasName, HasUid, + types::{ + kubernetes::{NamespaceName, Uid}, + operator::ClusterName, + }, + }, +}; + +use crate::{ + crd::{ + HostHeaderCheckConfig, NifiRole, NifiRoleType, + sensitive_properties::NifiSensitiveKeyAlgorithm, v1alpha1, + }, + security::{ + authentication::NifiAuthenticationConfig, authorization::ResolvedNifiAuthorizationConfig, + }, +}; + +pub(crate) mod build; +pub(crate) mod dereference; +pub(crate) mod validate; + +use validate::NifiRoleGroupConfig; + +/// The validated NifiCluster: everything `reconcile_nifi` needs after dereferencing, +/// in fail-safe / resolved form. This is the single resolved representation of the cluster; +/// downstream builders should source everything from here and never touch the raw `NifiCluster`. +pub struct ValidatedCluster { + /// Synthetic metadata (name, namespace, uid) so `ValidatedCluster` can implement + /// [`Resource`] and be used to build OwnerReferences without the raw `NifiCluster`. + metadata: ObjectMeta, + /// The name of the NifiCluster. + pub name: ClusterName, + /// The namespace of the NifiCluster, parsed once in the dereference step and reused everywhere. + pub namespace: NamespaceName, + /// The UID of the NifiCluster, used to build OwnerReferences downstream. + pub uid: Uid, + /// The product image. + pub image: ResolvedProductImage, + /// Cluster wide settings. + pub cluster_config: ValidatedClusterConfig, + /// The raw Node role spec (`spec.nodes`), needed for JVM argument merging in `bootstrap.conf`. + pub nodes: NifiRoleType, + /// Collected configuration per rolegroup. + pub role_group_configs: BTreeMap>, +} + +/// The resolved `spec.clusterConfig`. +pub struct ValidatedClusterConfig { + /// The cluster authentication settings. + pub authentication: NifiAuthenticationConfig, + /// The cluster authorization settings. + pub authorization: ResolvedNifiAuthorizationConfig, + /// The git-sync specs, resolved into git-sync resources at build time. + pub custom_components_git_sync: Vec, + /// The clustering backend (ZooKeeper or Kubernetes), copied from the spec. + pub clustering_backend: v1alpha1::NifiClusteringBackend, + /// The host-header-check config, resolved into the proxy hosts allow-list at build time. + pub host_header_check: HostHeaderCheckConfig, + /// The validated sensitive properties algorithm. + pub sensitive_properties_algorithm: NifiSensitiveKeyAlgorithm, +} + +impl ValidatedCluster { + /// Builds a [`ValidatedCluster`], deriving the synthetic [`ObjectMeta`] from name, namespace + /// and uid so the struct can implement [`Resource`]. + pub fn new( + name: ClusterName, + namespace: NamespaceName, + uid: Uid, + image: ResolvedProductImage, + nodes: NifiRoleType, + role_group_configs: BTreeMap>, + cluster_config: ValidatedClusterConfig, + ) -> Self { + let metadata = ObjectMeta { + name: Some(name.to_string()), + namespace: Some(namespace.to_string()), + uid: Some(uid.to_string()), + ..ObjectMeta::default() + }; + + Self { + metadata, + name, + namespace, + uid, + image, + nodes, + role_group_configs, + cluster_config, + } + } +} + +impl HasName for ValidatedCluster { + fn to_name(&self) -> String { + self.name.to_string() + } +} + +impl HasUid for ValidatedCluster { + fn to_uid(&self) -> Uid { + self.uid.clone() + } +} + +impl Resource for ValidatedCluster { + type DynamicType = ::DynamicType; + type Scope = ::Scope; + + fn kind(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha1::NifiCluster::kind(dt) + } + + fn group(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha1::NifiCluster::group(dt) + } + + fn version(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha1::NifiCluster::version(dt) + } + + fn plural(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha1::NifiCluster::plural(dt) + } + + fn meta(&self) -> &ObjectMeta { + &self.metadata + } + + fn meta_mut(&mut self) -> &mut ObjectMeta { + &mut self.metadata + } +} diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 923776ac..552a2c7d 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -8,28 +8,17 @@ use std::collections::BTreeMap; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, - commons::product_image_selection::{self, ResolvedProductImage}, - crd::git_sync, - k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta, - kube::{Resource, ResourceExt as _}, + commons::product_image_selection, + kube::ResourceExt as _, role_utils::JavaCommonConfig, - v2::{ - HasName, HasUid, - controller_utils::{self, get_cluster_name, get_uid}, - types::{ - kubernetes::{NamespaceName, Uid}, - operator::ClusterName, - }, - }, + v2::controller_utils::{self, get_cluster_name, get_uid}, }; use strum::{EnumDiscriminants, IntoStaticStr}; +use super::{ValidatedCluster, ValidatedClusterConfig}; use crate::{ controller::dereference::DereferencedObjects, - crd::{ - HostHeaderCheckConfig, NifiConfig, NifiRole, NifiRoleType, sensitive_properties, - sensitive_properties::NifiSensitiveKeyAlgorithm, v1alpha1, - }, + crd::{NifiConfig, NifiRole, sensitive_properties, v1alpha1}, framework::role_utils::with_validated_config, security::{ authentication::{self, NifiAuthenticationConfig}, @@ -75,118 +64,6 @@ pub type NifiRoleGroupConfig = crate::framework::role_utils::RoleGroupConfig< type Result = std::result::Result; -/// The validated NifiCluster: everything `reconcile_nifi` needs after dereferencing, -/// in fail-safe / resolved form. This is the single resolved representation of the cluster; -/// downstream builders should source everything from here and never touch the raw `NifiCluster`. -pub struct ValidatedCluster { - /// Synthetic metadata (name, namespace, uid) so `ValidatedCluster` can implement - /// [`Resource`] and be used to build OwnerReferences without the raw `NifiCluster`. - metadata: ObjectMeta, - /// The name of the NifiCluster. - pub name: ClusterName, - /// The namespace of the NifiCluster, parsed once in the dereference step and reused everywhere. - pub namespace: NamespaceName, - /// The UID of the NifiCluster, used to build OwnerReferences downstream. - pub uid: Uid, - /// The product image. - pub image: ResolvedProductImage, - /// Cluster wide settings. - pub cluster_config: ValidatedClusterConfig, - /// The raw Node role spec (`spec.nodes`), needed for JVM argument merging in `bootstrap.conf`. - pub nodes: NifiRoleType, - /// Collected configuration per rolegroup. - pub role_group_configs: BTreeMap>, -} - -/// The resolved `spec.clusterConfig`. -pub struct ValidatedClusterConfig { - /// The cluster authentication settings. - pub authentication: NifiAuthenticationConfig, - /// The cluster authorization settings. - pub authorization: ResolvedNifiAuthorizationConfig, - /// The git-sync specs, resolved into git-sync resources at build time. - pub custom_components_git_sync: Vec, - /// The clustering backend (ZooKeeper or Kubernetes), copied from the spec. - pub clustering_backend: v1alpha1::NifiClusteringBackend, - /// The host-header-check config, resolved into the proxy hosts allow-list at build time. - pub host_header_check: HostHeaderCheckConfig, - /// The validated sensitive properties algorithm. - pub sensitive_properties_algorithm: NifiSensitiveKeyAlgorithm, -} - -impl ValidatedCluster { - /// Builds a [`ValidatedCluster`], deriving the synthetic [`ObjectMeta`] from name, namespace - /// and uid so the struct can implement [`Resource`]. - pub fn new( - name: ClusterName, - namespace: NamespaceName, - uid: Uid, - image: ResolvedProductImage, - nodes: NifiRoleType, - role_group_configs: BTreeMap>, - cluster_config: ValidatedClusterConfig, - ) -> Self { - let metadata = ObjectMeta { - name: Some(name.to_string()), - namespace: Some(namespace.to_string()), - uid: Some(uid.to_string()), - ..ObjectMeta::default() - }; - - Self { - metadata, - name, - namespace, - uid, - image, - nodes, - role_group_configs, - cluster_config, - } - } -} - -impl HasName for ValidatedCluster { - fn to_name(&self) -> String { - self.name.to_string() - } -} - -impl HasUid for ValidatedCluster { - fn to_uid(&self) -> Uid { - self.uid.clone() - } -} - -impl Resource for ValidatedCluster { - type DynamicType = ::DynamicType; - type Scope = ::Scope; - - fn kind(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { - v1alpha1::NifiCluster::kind(dt) - } - - fn group(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { - v1alpha1::NifiCluster::group(dt) - } - - fn version(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { - v1alpha1::NifiCluster::version(dt) - } - - fn plural(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { - v1alpha1::NifiCluster::plural(dt) - } - - fn meta(&self) -> &ObjectMeta { - &self.metadata - } - - fn meta_mut(&mut self) -> &mut ObjectMeta { - &mut self.metadata - } -} - /// Validates the cluster spec and the dereferenced inputs. pub fn validate( nifi: &v1alpha1::NifiCluster, @@ -197,7 +74,7 @@ pub fn validate( .spec .image .resolve( - super::CONTAINER_IMAGE_BASE_NAME, + crate::nifi_controller::CONTAINER_IMAGE_BASE_NAME, &operator_environment.image_repository, crate::built_info::PKG_VERSION, ) diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 77622403..45fc2492 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -33,8 +33,8 @@ use stackable_operator::{ }; use crate::{ - controller::NIFI_FULL_CONTROLLER_NAME, crd::{NifiCluster, NifiClusterVersion, authorization::NifiOpaConfig, v1alpha1}, + nifi_controller::NIFI_FULL_CONTROLLER_NAME, webhooks::conversion::create_webhook_server, }; @@ -43,6 +43,7 @@ mod controller; mod crd; mod framework; mod listener; +mod nifi_controller; mod operations; mod reporting_task; mod security; @@ -172,9 +173,9 @@ async fn main() -> anyhow::Result<()> { ) .graceful_shutdown_on(sigterm_watcher.handle()) .run( - controller::reconcile_nifi, - controller::error_policy, - Arc::new(controller::Ctx { + nifi_controller::reconcile_nifi, + nifi_controller::error_policy, + Arc::new(nifi_controller::Ctx { client: client.clone(), operator_environment, }), diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/nifi_controller.rs similarity index 98% rename from rust/operator-binary/src/controller.rs rename to rust/operator-binary/src/nifi_controller.rs index 9b6da330..fe660e1c 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/nifi_controller.rs @@ -62,18 +62,13 @@ use stackable_operator::{ use strum::{EnumDiscriminants, IntoStaticStr}; use tracing::Instrument; -mod build; -mod dereference; -mod validate; - -use validate::NifiRoleGroupConfig; - use crate::{ OPERATOR_NAME, config::{ NIFI_CONFIG_DIRECTORY, NIFI_PYTHON_WORKING_DIRECTORY, NifiRepository, PERSISTENT_REPOSITORIES, }, + controller::{build, dereference, validate, validate::NifiRoleGroupConfig}, crd::{ APP_NAME, BALANCE_PORT, BALANCE_PORT_NAME, Container, HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT, METRICS_PORT_NAME, NifiConfig, NifiNodeRoleConfig, NifiRole, NifiRoleType, @@ -104,8 +99,8 @@ use crate::{ pub const NIFI_CONTROLLER_NAME: &str = "nificluster"; pub const NIFI_FULL_CONTROLLER_NAME: &str = concatcp!(NIFI_CONTROLLER_NAME, '.', OPERATOR_NAME); -const CONTAINER_IMAGE_BASE_NAME: &str = "nifi"; -const LOG_VOLUME_NAME: &str = "log"; +pub(crate) const CONTAINER_IMAGE_BASE_NAME: &str = "nifi"; +pub(crate) const LOG_VOLUME_NAME: &str = "log"; pub struct Ctx { pub client: Client, @@ -408,7 +403,7 @@ pub async fn reconcile_nifi( .context(BuildGitSyncResourcesSnafu)?; let role_group_service_recommended_labels = build_recommended_labels( - nifi, + &validated_cluster, &resolved_product_image.app_version_label_value, &rolegroup.role, &rolegroup.role_group, @@ -419,9 +414,9 @@ pub async fn reconcile_nifi( .context(LabelBuildSnafu)?; let rg_headless_service = build_rolegroup_headless_service( - nifi, + &validated_cluster, &rolegroup, - role_group_service_recommended_labels.clone(), + &role_group_service_recommended_labels, role_group_service_selector.clone().into(), ) .context(ServiceConfigurationSnafu)?; @@ -464,9 +459,9 @@ pub async fn reconcile_nifi( .await?; let rg_metrics_service = build_rolegroup_metrics_service( - nifi, + &validated_cluster, &rolegroup, - role_group_service_recommended_labels, + &role_group_service_recommended_labels, role_group_service_selector.into(), &resolved_product_image.product_version, ) @@ -1262,12 +1257,12 @@ pub fn error_policy( } } -pub fn build_recommended_labels<'a>( - owner: &'a v1alpha1::NifiCluster, +pub fn build_recommended_labels<'a, T>( + owner: &'a T, app_version: &'a str, role: &'a str, role_group: &'a str, -) -> ObjectLabels<'a, v1alpha1::NifiCluster> { +) -> ObjectLabels<'a, T> { ObjectLabels { owner, app_name: APP_NAME, diff --git a/rust/operator-binary/src/operations/pdb.rs b/rust/operator-binary/src/operations/pdb.rs index 5db207b8..66156889 100644 --- a/rust/operator-binary/src/operations/pdb.rs +++ b/rust/operator-binary/src/operations/pdb.rs @@ -6,8 +6,8 @@ use stackable_operator::{ use crate::{ OPERATOR_NAME, - controller::NIFI_CONTROLLER_NAME, crd::{APP_NAME, NifiRole, v1alpha1}, + nifi_controller::NIFI_CONTROLLER_NAME, }; #[derive(Snafu, Debug)] diff --git a/rust/operator-binary/src/reporting_task/mod.rs b/rust/operator-binary/src/reporting_task/mod.rs index fa2b214a..f6c7a09e 100644 --- a/rust/operator-binary/src/reporting_task/mod.rs +++ b/rust/operator-binary/src/reporting_task/mod.rs @@ -50,8 +50,8 @@ use stackable_operator::{ }; use crate::{ - controller::build_recommended_labels, crd::{APP_NAME, HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT, NifiRole, v1alpha1}, + nifi_controller::build_recommended_labels, security::{ authentication::{NifiAuthenticationConfig, STACKABLE_ADMIN_USERNAME}, build_tls_volume, diff --git a/rust/operator-binary/src/service.rs b/rust/operator-binary/src/service.rs index 9bd4a722..5f43d3ce 100644 --- a/rust/operator-binary/src/service.rs +++ b/rust/operator-binary/src/service.rs @@ -8,7 +8,10 @@ use stackable_operator::{ role_utils::RoleGroupRef, }; -use crate::crd::{HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT, METRICS_PORT_NAME, v1alpha1}; +use crate::{ + controller::ValidatedCluster, + crd::{HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT, METRICS_PORT_NAME, v1alpha1}, +}; #[derive(Snafu, Debug)] pub enum Error { @@ -26,18 +29,18 @@ pub enum Error { /// The rolegroup headless [`Service`] is a service that allows direct access to the instances of a certain rolegroup /// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. pub fn build_rolegroup_headless_service( - nifi: &v1alpha1::NifiCluster, + cluster: &ValidatedCluster, role_group_ref: &RoleGroupRef, - object_labels: ObjectLabels, + object_labels: &ObjectLabels, selector: BTreeMap, ) -> Result { Ok(Service { metadata: ObjectMetaBuilder::new() - .name_and_namespace(nifi) + .name_and_namespace(cluster) .name(role_group_ref.rolegroup_headless_service_name()) - .ownerreference_from_resource(nifi, None, Some(true)) + .ownerreference_from_resource(cluster, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&object_labels) + .with_recommended_labels(object_labels) .context(MetadataBuildSnafu)? .build(), spec: Some(ServiceSpec { @@ -55,19 +58,19 @@ pub fn build_rolegroup_headless_service( /// The rolegroup metrics [`Service`] is a service that exposes metrics and a prometheus scraping label. pub fn build_rolegroup_metrics_service( - nifi: &v1alpha1::NifiCluster, + cluster: &ValidatedCluster, role_group_ref: &RoleGroupRef, - object_labels: ObjectLabels, + object_labels: &ObjectLabels, selector: BTreeMap, product_version: &str, ) -> Result { Ok(Service { metadata: ObjectMetaBuilder::new() - .name_and_namespace(nifi) + .name_and_namespace(cluster) .name(role_group_ref.rolegroup_metrics_service_name()) - .ownerreference_from_resource(nifi, None, Some(true)) + .ownerreference_from_resource(cluster, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&object_labels) + .with_recommended_labels(object_labels) .context(MetadataBuildSnafu)? .with_labels(prometheus_labels()) .with_annotations(prometheus_annotations(product_version)) From e3a7cbf127eeafe91369f8b1425c410d72e34794 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 11 Jun 2026 20:20:21 +0200 Subject: [PATCH 26/39] refactor: use v2 JavaCommonConfig, delete local framework, move jvm.rs to controller/build --- Cargo.lock | 18 +- Cargo.nix | 18 +- rust/operator-binary/src/controller/build.rs | 27 +++ .../src/controller/build/config_map.rs | 20 +- .../src/controller/build/git_sync.rs | 4 +- .../src/{config => controller/build}/jvm.rs | 37 +-- .../src/controller/build/properties.rs | 43 ++-- .../build/properties/bootstrap_conf.rs | 52 ++--- .../build/properties/nifi_properties.rs | 36 +-- .../build/properties/security_properties.rs | 13 +- .../build/properties/state_management_xml.rs | 2 +- rust/operator-binary/src/controller/mod.rs | 41 +++- .../src/controller/validate.rs | 81 ++++--- rust/operator-binary/src/crd/affinity.rs | 18 +- rust/operator-binary/src/crd/constants.rs | 8 + rust/operator-binary/src/crd/mod.rs | 6 +- .../src/{config/mod.rs => crd/storage.rs} | 49 +--- rust/operator-binary/src/framework.rs | 8 - .../src/framework/role_utils.rs | 215 ------------------ rust/operator-binary/src/main.rs | 2 - rust/operator-binary/src/nifi_controller.rs | 13 +- .../src/security/authorization.rs | 7 +- 22 files changed, 260 insertions(+), 458 deletions(-) rename rust/operator-binary/src/{config => controller/build}/jvm.rs (82%) create mode 100644 rust/operator-binary/src/crd/constants.rs rename rust/operator-binary/src/{config/mod.rs => crd/storage.rs} (50%) delete mode 100644 rust/operator-binary/src/framework.rs delete mode 100644 rust/operator-binary/src/framework/role_utils.rs diff --git a/Cargo.lock b/Cargo.lock index 4ef5d4e4..38bf7ecb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1571,7 +1571,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "darling", "regex", @@ -2989,7 +2989,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "const-oid", "ecdsa", @@ -3039,7 +3039,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.111.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "base64", "clap", @@ -3083,7 +3083,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "darling", "proc-macro2", @@ -3094,7 +3094,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "jiff", "k8s-openapi", @@ -3111,7 +3111,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.4" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "axum", "clap", @@ -3135,7 +3135,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "kube", "schemars", @@ -3149,7 +3149,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "convert_case", "convert_case_extras", @@ -3167,7 +3167,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.nix b/Cargo.nix index 60fc7565..85e4e57a 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -5066,7 +5066,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "k8s_version"; @@ -9860,7 +9860,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_certs"; @@ -10072,7 +10072,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_operator"; @@ -10266,7 +10266,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; procMacro = true; @@ -10301,7 +10301,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_shared"; @@ -10382,7 +10382,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_telemetry"; @@ -10492,7 +10492,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_versioned"; @@ -10542,7 +10542,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; procMacro = true; @@ -10610,7 +10610,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_webhook"; diff --git a/rust/operator-binary/src/controller/build.rs b/rust/operator-binary/src/controller/build.rs index 1b225f1d..71b9885c 100644 --- a/rust/operator-binary/src/controller/build.rs +++ b/rust/operator-binary/src/controller/build.rs @@ -2,7 +2,34 @@ //! //! [`ValidatedCluster`]: crate::controller::ValidatedCluster +use snafu::Snafu; + +use crate::{crd::storage::NifiRepository, security::oidc}; + pub mod config_map; pub mod git_sync; +pub mod jvm; pub mod properties; pub mod proxy_hosts; + +/// Errors that can occur while building the NiFi product configuration files. +#[derive(Snafu, Debug)] +#[snafu(visibility(pub(crate)))] +pub enum Error { + #[snafu(display("invalid JVM config"))] + InvalidJVMConfig { source: jvm::Error }, + + #[snafu(display("failed to calculate storage quota for {repo} repository"))] + CalculateStorageQuota { + source: stackable_operator::memory::Error, + repo: NifiRepository, + }, + + #[snafu(display("failed to generate OIDC config"))] + GenerateOidcConfig { source: oidc::Error }, + + #[snafu(display( + "NiFi 1.x requires ZooKeeper (hint: upgrade to NiFi 2.x or set .spec.clusterConfig.zookeeperConfigMapName)" + ))] + Nifi1RequiresZookeeper, +} diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 99e5d88f..e4615e7d 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -35,14 +35,14 @@ pub enum Error { #[snafu(display("failed to build bootstrap.conf"))] BootstrapConfig { - #[snafu(source(from(crate::config::Error, Box::new)))] - source: Box, + #[snafu(source(from(crate::controller::build::Error, Box::new)))] + source: Box, }, #[snafu(display("failed to prepare NiFi configuration for rolegroup {rolegroup}"))] BuildNifiProperties { - #[snafu(source(from(crate::config::Error, Box::new)))] - source: Box, + #[snafu(source(from(crate::controller::build::Error, Box::new)))] + source: Box, rolegroup: RoleGroupRef, }, @@ -94,9 +94,6 @@ pub fn build_rolegroup_config_map( role_group: rolegroup.role_group.clone(), })?; - // The raw role spec is only needed for JVM argument merging in `bootstrap_conf`. - let role = &cluster.nodes; - let proxy_hosts = proxy_hosts::compute_proxy_hosts(cluster, cluster_info); let git_sync_resources = git_sync::build_git_sync_resources(cluster, rg).context(BuildGitSyncResourcesSnafu)?; @@ -115,13 +112,8 @@ pub fn build_rolegroup_config_map( ) .add_data( ConfigFileName::BootstrapConf.to_string(), - bootstrap_conf::build( - rg, - role, - &rolegroup.role_group, - Some(&cluster.cluster_config.authorization), - ) - .context(BootstrapConfigSnafu)?, + bootstrap_conf::build(rg, Some(&cluster.cluster_config.authorization)) + .context(BootstrapConfigSnafu)?, ) .add_data( ConfigFileName::NifiProperties.to_string(), diff --git a/rust/operator-binary/src/controller/build/git_sync.rs b/rust/operator-binary/src/controller/build/git_sync.rs index f6c2f8b6..0117c2d2 100644 --- a/rust/operator-binary/src/controller/build/git_sync.rs +++ b/rust/operator-binary/src/controller/build/git_sync.rs @@ -4,7 +4,7 @@ use snafu::{ResultExt, Snafu}; use stackable_operator::{crd::git_sync, k8s_openapi::api::core::v1::EnvVar}; use crate::{ - controller::{ValidatedCluster, validate::NifiRoleGroupConfig}, + controller::{ValidatedCluster, ValidatedRoleGroupConfig}, crd::Container, nifi_controller::LOG_VOLUME_NAME, }; @@ -22,7 +22,7 @@ type Result = std::result::Result; /// rather than once for the whole cluster. pub fn build_git_sync_resources( cluster: &ValidatedCluster, - rg: &NifiRoleGroupConfig, + rg: &ValidatedRoleGroupConfig, ) -> Result { let env_vars: Vec = rg.env_overrides.clone().into(); git_sync::v1alpha2::GitSyncResources::new( diff --git a/rust/operator-binary/src/config/jvm.rs b/rust/operator-binary/src/controller/build/jvm.rs similarity index 82% rename from rust/operator-binary/src/config/jvm.rs rename to rust/operator-binary/src/controller/build/jvm.rs index 4bf19f30..177bc993 100644 --- a/rust/operator-binary/src/config/jvm.rs +++ b/rust/operator-binary/src/controller/build/jvm.rs @@ -1,12 +1,16 @@ +//! Builder for the NiFi JVM arguments used in `bootstrap.conf`. + use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ memory::{BinaryMultiple, MemoryQuantity}, - role_utils::{self, JvmArgumentOverrides}, + v2::jvm_argument_overrides::JvmArgumentOverrides, }; use crate::{ - config::{JVM_SECURITY_PROPERTIES_FILE, NIFI_CONFIG_DIRECTORY}, - crd::{NifiConfig, NifiRoleType}, + crd::{ + NifiConfig, + constants::{JVM_SECURITY_PROPERTIES_FILE, NIFI_CONFIG_DIRECTORY}, + }, security::{ authentication::{STACKABLE_SERVER_TLS_DIR, STACKABLE_TLS_STORE_PASSWORD}, authorization::ResolvedNifiAuthorizationConfig, @@ -25,18 +29,19 @@ pub enum Error { InvalidMemoryConfig { source: stackable_operator::memory::Error, }, - - #[snafu(display("failed to merge jvm argument overrides"))] - MergeJvmArgumentOverrides { source: role_utils::Error }, } -/// Build the merged JVM argument overrides used in the NiFi bootstrap.conf +/// Build the effective JVM arguments for the NiFi bootstrap.conf. +/// +/// The operator-generated arguments below form the base that the role <- role-group merged +/// user overrides (`merged_jvm_argument_overrides`, produced by +/// [`with_validated_config`](stackable_operator::v2::role_utils::with_validated_config)) are +/// applied on top of. pub fn build_merged_jvm_config( merged_config: &NifiConfig, - role: &NifiRoleType, - role_group: &str, + merged_jvm_argument_overrides: &JvmArgumentOverrides, authorization_config: Option<&ResolvedNifiAuthorizationConfig>, -) -> Result { +) -> Result, Error> { let heap_size = MemoryQuantity::try_from( merged_config .resources @@ -52,7 +57,7 @@ pub fn build_merged_jvm_config( .format_for_java() .context(InvalidMemoryConfigSnafu)?; - let mut jvm_args = vec![ + let mut operator_generated = vec![ // Heap settings format!("-Xmx{java_heap}"), format!("-Xms{java_heap}"), @@ -95,17 +100,15 @@ pub fn build_merged_jvm_config( // OPA Java SDK. if let Some(authz_config) = authorization_config { if authz_config.has_opa_tls() { - jvm_args.push(format!( + operator_generated.push(format!( "-Djavax.net.ssl.trustStore={STACKABLE_SERVER_TLS_DIR}/truststore.p12" )); - jvm_args.push(format!( + operator_generated.push(format!( "-Djavax.net.ssl.trustStorePassword={STACKABLE_TLS_STORE_PASSWORD}" )); - jvm_args.push("-Djavax.net.ssl.trustStoreType=pkcs12".to_owned()); + operator_generated.push("-Djavax.net.ssl.trustStoreType=pkcs12".to_owned()); } } - let operator_generated = JvmArgumentOverrides::new_with_only_additions(jvm_args); - role.get_merged_jvm_argument_overrides(role_group, &operator_generated) - .context(MergeJvmArgumentOverridesSnafu) + Ok(merged_jvm_argument_overrides.apply_to(operator_generated)) } diff --git a/rust/operator-binary/src/controller/build/properties.rs b/rust/operator-binary/src/controller/build/properties.rs index 9f306e5a..d3cb5cec 100644 --- a/rust/operator-binary/src/controller/build/properties.rs +++ b/rust/operator-binary/src/controller/build/properties.rs @@ -4,6 +4,8 @@ //! The shared [`stackable_operator::v2::config_file_writer`] module serializes `.properties`/`.conf` //! key/value maps to the Java-properties on-wire format. +use std::{collections::BTreeMap, fmt::Write}; + pub mod authorizers; pub mod bootstrap_conf; pub mod logging; @@ -12,6 +14,18 @@ pub mod nifi_properties; pub mod security_properties; pub mod state_management_xml; +// TODO: Use crate like https://crates.io/crates/java-properties (currently does not work for Nifi +// because of escapes), to have save handling of escapes etc. +pub(crate) fn format_properties(properties: BTreeMap) -> String { + let mut result = String::new(); + + for (key, value) in properties { + let _ = writeln!(result, "{}={}", key, value); + } + + result +} + /// The names of the files assembled into the NiFi rolegroup ConfigMap. #[derive(Clone, Copy, Debug, strum::Display)] pub enum ConfigFileName { @@ -53,14 +67,13 @@ pub enum ConfigFileName { /// self-contained. #[cfg(test)] pub(crate) mod test_support { - use std::{collections::BTreeMap, str::FromStr as _}; + use std::str::FromStr as _; use stackable_operator::{ commons::product_image_selection::ResolvedProductImage, crd::authentication::r#static::v1alpha1::{ AuthenticationProvider as StaticAuthProvider, UserCredentialsSecretRef, }, - kube::ResourceExt as _, kvp::LabelValue, v2::types::{ kubernetes::{NamespaceName, Uid}, @@ -69,9 +82,11 @@ pub(crate) mod test_support { }; use crate::{ - controller::{ValidatedCluster, ValidatedClusterConfig, validate::NifiRoleGroupConfig}, - crd::{NifiConfig, NifiRole, v1alpha1}, - framework::role_utils::with_validated_config, + controller::{ + ValidatedCluster, ValidatedClusterConfig, ValidatedRoleGroupConfig, + validate::build_role_group_configs, + }, + crd::{NifiRole, v1alpha1}, security::{ authentication::NifiAuthenticationConfig, authorization::ResolvedNifiAuthorizationConfig, @@ -115,19 +130,8 @@ pub(crate) mod test_support { let nifi: v1alpha1::NifiCluster = serde_yaml::from_str(MINIMAL_NIFI_YAML).expect("invalid test YAML"); - let nifi_role = NifiRole::Node; - let role = nifi.spec.nodes.as_ref().unwrap(); - let default_config = NifiConfig::default_config(&nifi.name_any(), &nifi_role); - - let mut role_groups: BTreeMap = BTreeMap::new(); - for (rg_name, rg) in &role.role_groups { - let validated_rg = - with_validated_config::(rg, role, &default_config) - .expect("with_validated_config should succeed for minimal fixture"); - role_groups.insert(rg_name.clone(), validated_rg); - } - let mut role_group_configs = BTreeMap::new(); - role_group_configs.insert(NifiRole::Node, role_groups); + let role_group_configs = build_role_group_configs(&nifi) + .expect("role group configs should merge for minimal fixture"); let image = ResolvedProductImage { product_version: "2.9.0".to_string(), @@ -146,7 +150,6 @@ pub(crate) mod test_support { namespace, uid, image, - nifi.spec.nodes.clone().expect("minimal fixture has nodes"), role_group_configs, ValidatedClusterConfig { authentication: NifiAuthenticationConfig::SingleUser { @@ -170,7 +173,7 @@ pub(crate) mod test_support { } /// Return the "default" role-group config from a [`ValidatedCluster`]. - pub fn default_rg(cluster: &ValidatedCluster) -> &NifiRoleGroupConfig { + pub fn default_rg(cluster: &ValidatedCluster) -> &ValidatedRoleGroupConfig { cluster .role_group_configs .get(&NifiRole::Node) diff --git a/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs b/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs index 050a893e..c91f54d0 100644 --- a/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs +++ b/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs @@ -5,17 +5,16 @@ use std::collections::BTreeMap; use snafu::ResultExt; use crate::{ - config::{Error, InvalidJVMConfigSnafu, jvm::build_merged_jvm_config}, - controller::validate::NifiRoleGroupConfig, - crd::NifiRoleType, + controller::{ + ValidatedRoleGroupConfig, + build::{Error, InvalidJVMConfigSnafu, jvm::build_merged_jvm_config}, + }, operations::graceful_shutdown::graceful_shutdown_config_properties, security::authorization::ResolvedNifiAuthorizationConfig, }; pub fn build( - rg: &NifiRoleGroupConfig, - role: &NifiRoleType, - role_group: &str, + rg: &ValidatedRoleGroupConfig, authorization_config: Option<&ResolvedNifiAuthorizationConfig>, ) -> Result { let mut bootstrap = BTreeMap::new(); @@ -30,50 +29,45 @@ pub fn build( bootstrap.insert("conf.dir".to_string(), "./conf".to_string()); bootstrap.extend(graceful_shutdown_config_properties(&rg.config)); - let merged_jvm_config = - build_merged_jvm_config(&rg.config, role, role_group, authorization_config) - .context(InvalidJVMConfigSnafu)?; + let jvm_args = build_merged_jvm_config( + &rg.config, + &rg.product_specific_common_config.jvm_argument_overrides, + authorization_config, + ) + .context(InvalidJVMConfigSnafu)?; - for (index, argument) in merged_jvm_config - .effective_jvm_config_after_merging() - .iter() - .enumerate() - { + for (index, argument) in jvm_args.iter().enumerate() { bootstrap.insert(format!("java.arg.{}", index + 1), argument.clone()); } // configOverrides come last bootstrap.extend(rg.config_overrides.bootstrap_conf.overrides.clone()); - Ok(crate::config::format_properties(bootstrap)) + Ok(super::format_properties(bootstrap)) } #[cfg(test)] mod tests { use indoc::indoc; - use stackable_operator::kube::ResourceExt as _; use super::*; use crate::{ - crd::{NifiConfig, NifiRole, v1alpha1}, - framework::role_utils::with_validated_config, + controller::validate::build_role_group_configs, + crd::{NifiRole, v1alpha1}, }; fn construct_bootstrap_conf(nifi_cluster: &str) -> String { let nifi: v1alpha1::NifiCluster = serde_yaml::from_str(nifi_cluster).expect("illegal test input"); - let nifi_role = NifiRole::Node; - let role = nifi.spec.nodes.as_ref().unwrap(); - let default_config = NifiConfig::default_config(&nifi.name_any(), &nifi_role); - let rg = with_validated_config::( - role.role_groups.get("default").unwrap(), - role, - &default_config, - ) - .expect("failed to build role group config"); - - build(&rg, role, "default", None).unwrap() + let role_group_configs = + build_role_group_configs(&nifi).expect("failed to build role group configs"); + let rg = role_group_configs + .get(&NifiRole::Node) + .and_then(|groups| groups.get("default")) + .expect("default role group must exist"); + + build(rg, None).unwrap() } #[test] diff --git a/rust/operator-binary/src/controller/build/properties/nifi_properties.rs b/rust/operator-binary/src/controller/build/properties/nifi_properties.rs index 25a8494a..b351eb8e 100644 --- a/rust/operator-binary/src/controller/build/properties/nifi_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/nifi_properties.rs @@ -5,13 +5,17 @@ use std::collections::BTreeMap; use snafu::{ResultExt, ensure}; use stackable_operator::{crd::git_sync, memory::MemoryQuantity}; +use super::format_properties; use crate::{ - config::{ - CalculateStorageQuotaSnafu, Error, GenerateOidcConfigSnafu, NIFI_PYTHON_WORKING_DIRECTORY, - Nifi1RequiresZookeeperSnafu, NifiRepository, format_properties, + controller::{ + ValidatedCluster, ValidatedRoleGroupConfig, + build::{ + CalculateStorageQuotaSnafu, Error, GenerateOidcConfigSnafu, Nifi1RequiresZookeeperSnafu, + }, + }, + crd::{ + HTTPS_PORT, constants::NIFI_PYTHON_WORKING_DIRECTORY, storage::NifiRepository, v1alpha1, }, - controller::{ValidatedCluster, validate::NifiRoleGroupConfig}, - crd::{HTTPS_PORT, v1alpha1}, security::{ authentication::{ NifiAuthenticationConfig, STACKABLE_SERVER_TLS_DIR, STACKABLE_TLS_STORE_PASSWORD, @@ -26,7 +30,7 @@ const STORAGE_CONTENT_ARCHIVE_UTILIZATION_FACTOR: f32 = 0.5; pub fn build( cluster: &ValidatedCluster, - rg: &NifiRoleGroupConfig, + rg: &ValidatedRoleGroupConfig, proxy_hosts: &str, git_sync_resources: &git_sync::v1alpha2::GitSyncResources, ) -> Result { @@ -638,11 +642,9 @@ mod tests { /// Verify that a user configOverride for `nifi.properties` flows through to the output. #[test] fn test_config_override_wins() { - use stackable_operator::kube::ResourceExt as _; - use crate::{ - crd::{NifiConfig, NifiRole, v1alpha1}, - framework::role_utils::with_validated_config, + controller::validate::build_role_group_configs, + crd::{NifiRole, v1alpha1}, }; let yaml = r#" @@ -669,14 +671,12 @@ mod tests { some.custom.key: some-custom-value "#; let nifi: v1alpha1::NifiCluster = serde_yaml::from_str(yaml).expect("invalid test YAML"); - let role = nifi.spec.nodes.as_ref().unwrap(); - let default_config = NifiConfig::default_config(&nifi.name_any(), &NifiRole::Node); - let rg = with_validated_config::( - role.role_groups.get("default").unwrap(), - role, - &default_config, - ) - .expect("with_validated_config should succeed"); + let mut role_group_configs = + build_role_group_configs(&nifi).expect("failed to build role group configs"); + let rg = role_group_configs + .get_mut(&NifiRole::Node) + .and_then(|groups| groups.remove("default")) + .expect("default role group must exist"); // Build a cluster with this rg substituted in let mut cluster = minimal_validated_cluster(); diff --git a/rust/operator-binary/src/controller/build/properties/security_properties.rs b/rust/operator-binary/src/controller/build/properties/security_properties.rs index a990febb..8ea273a8 100644 --- a/rust/operator-binary/src/controller/build/properties/security_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/security_properties.rs @@ -6,9 +6,9 @@ use stackable_operator::v2::config_file_writer::{ PropertiesWriterError, to_java_properties_string, }; -use crate::controller::validate::NifiRoleGroupConfig; +use crate::controller::ValidatedRoleGroupConfig; -pub fn build(rg: &NifiRoleGroupConfig) -> Result { +pub fn build(rg: &ValidatedRoleGroupConfig) -> Result { let mut props: BTreeMap = BTreeMap::new(); // Defaults previously injected by deploy/config-spec/properties.yaml: props.insert("networkaddress.cache.ttl".to_string(), "30".to_string()); @@ -30,13 +30,13 @@ mod tests { use super::*; use crate::{ - controller::validate::NifiRoleGroupConfig, + controller::ValidatedRoleGroupConfig, crd::{NifiConfig, v1alpha1::NifiConfigOverrides}, }; - fn make_rg(overrides: Option>) -> NifiRoleGroupConfig { - use stackable_operator::role_utils::JavaCommonConfig; - NifiRoleGroupConfig { + fn make_rg(overrides: Option>) -> ValidatedRoleGroupConfig { + use stackable_operator::v2::role_utils::JavaCommonConfig; + ValidatedRoleGroupConfig { replicas: 1, config: NifiConfig::default(), config_overrides: NifiConfigOverrides { @@ -46,7 +46,6 @@ mod tests { ..Default::default() }, env_overrides: EnvVarSet::new(), - cli_overrides: BTreeMap::new(), pod_overrides: Default::default(), product_specific_common_config: JavaCommonConfig::default(), } diff --git a/rust/operator-binary/src/controller/build/properties/state_management_xml.rs b/rust/operator-binary/src/controller/build/properties/state_management_xml.rs index 53747c9d..df0d2321 100644 --- a/rust/operator-binary/src/controller/build/properties/state_management_xml.rs +++ b/rust/operator-binary/src/controller/build/properties/state_management_xml.rs @@ -1,6 +1,6 @@ //! Builder for `state-management.xml`. -use crate::{config::NifiRepository, crd::v1alpha1::NifiClusteringBackend}; +use crate::crd::{storage::NifiRepository, v1alpha1::NifiClusteringBackend}; pub fn build(clustering_backend: &NifiClusteringBackend) -> String { // Inert providers are ignored by NiFi itself, but templating still fails if they refer to invalid environment variables, diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index 96685a38..c2804dcd 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -7,10 +7,12 @@ use std::collections::BTreeMap; use stackable_operator::{ commons::product_image_selection::ResolvedProductImage, crd::git_sync, - k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta, + k8s_openapi::{api::core::v1::PodTemplateSpec, apimachinery::pkg::apis::meta::v1::ObjectMeta}, kube::Resource, v2::{ HasName, HasUid, + builder::pod::container::EnvVarSet, + role_utils::JavaCommonConfig, types::{ kubernetes::{NamespaceName, Uid}, operator::ClusterName, @@ -20,7 +22,7 @@ use stackable_operator::{ use crate::{ crd::{ - HostHeaderCheckConfig, NifiRole, NifiRoleType, + HostHeaderCheckConfig, NifiConfig, NifiRole, sensitive_properties::NifiSensitiveKeyAlgorithm, v1alpha1, }, security::{ @@ -32,7 +34,32 @@ pub(crate) mod build; pub(crate) mod dereference; pub(crate) mod validate; -use validate::NifiRoleGroupConfig; +/// A validated, merged (default <- role <- role-group) NiFi rolegroup config. +/// +/// Produced from the result of +/// [`with_validated_config`](stackable_operator::v2::role_utils::with_validated_config) in the +/// [`validate`] step; downstream builders consume this rather than the raw `NifiCluster`. +#[derive(Clone, Debug)] +pub struct ValidatedRoleGroupConfig { + /// The desired number of replicas (defaulted to 1 during validation). + /// + /// The StatefulSet replica count is currently sourced from the raw role-group spec in + /// `nifi_controller` (to keep `replicas: null` semantics during version updates), so this + /// validated value is carried for completeness but not yet read. + #[allow(dead_code)] + pub replicas: u16, + /// The merged and validated rolegroup config. + pub config: NifiConfig, + /// The merged (role <- role group) config-file overrides. + pub config_overrides: v1alpha1::NifiConfigOverrides, + /// The merged (role <- role group) environment variable overrides. + pub env_overrides: EnvVarSet, + /// The merged (role <- role group) pod template overrides. + pub pod_overrides: PodTemplateSpec, + /// The merged (role <- role group) JVM argument overrides, applied on top of the + /// operator-generated JVM arguments when building `bootstrap.conf`. + pub product_specific_common_config: JavaCommonConfig, +} /// The validated NifiCluster: everything `reconcile_nifi` needs after dereferencing, /// in fail-safe / resolved form. This is the single resolved representation of the cluster; @@ -51,10 +78,8 @@ pub struct ValidatedCluster { pub image: ResolvedProductImage, /// Cluster wide settings. pub cluster_config: ValidatedClusterConfig, - /// The raw Node role spec (`spec.nodes`), needed for JVM argument merging in `bootstrap.conf`. - pub nodes: NifiRoleType, /// Collected configuration per rolegroup. - pub role_group_configs: BTreeMap>, + pub role_group_configs: BTreeMap>, } /// The resolved `spec.clusterConfig`. @@ -81,8 +106,7 @@ impl ValidatedCluster { namespace: NamespaceName, uid: Uid, image: ResolvedProductImage, - nodes: NifiRoleType, - role_group_configs: BTreeMap>, + role_group_configs: BTreeMap>, cluster_config: ValidatedClusterConfig, ) -> Self { let metadata = ObjectMeta { @@ -98,7 +122,6 @@ impl ValidatedCluster { namespace, uid, image, - nodes, role_group_configs, cluster_config, } diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 552a2c7d..b1acaaff 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -3,23 +3,27 @@ //! Synchronously validates inputs that don't require Kubernetes API calls. Produces //! [`ValidatedCluster`], consumed by the rest of `reconcile_nifi`. -use std::collections::BTreeMap; +use std::{collections::BTreeMap, str::FromStr as _}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, commons::product_image_selection, + config::fragment, kube::ResourceExt as _, - role_utils::JavaCommonConfig, - v2::controller_utils::{self, get_cluster_name, get_uid}, + role_utils::CommonConfiguration, + v2::{ + builder::pod::container::{self, EnvVarName, EnvVarSet}, + controller_utils::{self, get_cluster_name, get_uid}, + role_utils::with_validated_config, + }, }; use strum::{EnumDiscriminants, IntoStaticStr}; -use super::{ValidatedCluster, ValidatedClusterConfig}; +use super::{ValidatedCluster, ValidatedClusterConfig, ValidatedRoleGroupConfig}; use crate::{ controller::dereference::DereferencedObjects, crd::{NifiConfig, NifiRole, sensitive_properties, v1alpha1}, - framework::role_utils::with_validated_config, security::{ authentication::{self, NifiAuthenticationConfig}, authorization::ResolvedNifiAuthorizationConfig, @@ -47,21 +51,19 @@ pub enum Error { #[snafu(display("invalid NiFi authentication configuration"))] InvalidAuthenticationConfig { source: authentication::Error }, - #[snafu(display("failed to build the config for a rolegroup"))] - BuildRoleGroupConfig { - source: crate::framework::role_utils::Error, + #[snafu(display("failed to validate the rolegroup config fragment"))] + ValidateRoleGroupConfig { source: fragment::ValidationError }, + + #[snafu(display("environment variable name {name:?} is invalid"))] + ParseEnvVarName { + source: container::Error, + name: String, }, #[snafu(display("invalid sensitive properties algorithm"))] InvalidSensitivePropertiesAlgorithm { source: sensitive_properties::Error }, } -pub type NifiRoleGroupConfig = crate::framework::role_utils::RoleGroupConfig< - NifiConfig, - JavaCommonConfig, - v1alpha1::NifiConfigOverrides, ->; - type Result = std::result::Result; /// Validates the cluster spec and the dereferenced inputs. @@ -100,12 +102,6 @@ pub fn validate( .check_for_nifi_version(&image.product_version) .context(InvalidSensitivePropertiesAlgorithmSnafu)?; - let nodes = nifi - .spec - .nodes - .as_ref() - .context(NoNodesDefinedSnafu)? - .clone(); let role_group_configs = build_role_group_configs(nifi)?; let name = get_cluster_name(nifi).context(GetClusterNameSnafu)?; @@ -117,7 +113,6 @@ pub fn validate( namespace, uid, image, - nodes, role_group_configs, ValidatedClusterConfig { authentication: authentication_config, @@ -130,18 +125,48 @@ pub fn validate( )) } -fn build_role_group_configs( +pub(crate) fn build_role_group_configs( nifi: &v1alpha1::NifiCluster, -) -> Result>> { +) -> Result>> { let role = nifi.spec.nodes.as_ref().context(NoNodesDefinedSnafu)?; let default_config = NifiConfig::default_config(&nifi.name_any(), &NifiRole::Node); - let mut groups: BTreeMap = BTreeMap::new(); + let mut groups: BTreeMap = BTreeMap::new(); for (rg_name, rg) in &role.role_groups { - let validated_rg = - with_validated_config::(rg, role, &default_config) - .context(BuildRoleGroupConfigSnafu)?; - groups.insert(rg_name.clone(), validated_rg); + let validated = with_validated_config::(rg, role, &default_config) + .context(ValidateRoleGroupConfigSnafu)?; + + let CommonConfiguration { + config, + config_overrides, + env_overrides, + cli_overrides: _, + pod_overrides, + product_specific_common_config, + } = validated.config; + + // Convert the merged env-override HashMap into an EnvVarSet, validating each name + // eagerly. Keys are unique (HashMap), so insertion order is irrelevant. + let mut env_overrides_set = EnvVarSet::new(); + for (name, value) in env_overrides { + env_overrides_set = env_overrides_set.with_value( + &EnvVarName::from_str(&name) + .context(ParseEnvVarNameSnafu { name: name.clone() })?, + value, + ); + } + + groups.insert( + rg_name.clone(), + ValidatedRoleGroupConfig { + replicas: validated.replicas.unwrap_or(1), + config, + config_overrides, + env_overrides: env_overrides_set, + pod_overrides, + product_specific_common_config, + }, + ); } let mut role_group_configs = BTreeMap::new(); diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 5ac8801e..b6b75db1 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -32,10 +32,7 @@ mod tests { }; use super::*; - use crate::{ - crd::{NifiConfig, v1alpha1}, - framework::role_utils::with_validated_config, - }; + use crate::{controller::validate::build_role_group_configs, crd::v1alpha1}; #[test] fn test_affinity_defaults() { @@ -62,13 +59,12 @@ mod tests { let nifi: v1alpha1::NifiCluster = serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); - let role = nifi.spec.nodes.as_ref().unwrap(); - let default_config = NifiConfig::default_config("simple-nifi", &NifiRole::Node); - let role_group = role.role_groups.get("default").unwrap(); - let merged_config = - with_validated_config::(role_group, role, &default_config) - .unwrap() - .config; + let role_group_configs = build_role_group_configs(&nifi).unwrap(); + let merged_config = &role_group_configs + .get(&NifiRole::Node) + .and_then(|groups| groups.get("default")) + .unwrap() + .config; assert_eq!( merged_config.affinity, diff --git a/rust/operator-binary/src/crd/constants.rs b/rust/operator-binary/src/crd/constants.rs new file mode 100644 index 00000000..b0a650fc --- /dev/null +++ b/rust/operator-binary/src/crd/constants.rs @@ -0,0 +1,8 @@ +//! Product layout constants (filesystem paths and well-known file names) shared across the +//! controller, security and build modules. + +pub const NIFI_CONFIG_DIRECTORY: &str = "/stackable/nifi/conf"; +pub const NIFI_PYTHON_WORKING_DIRECTORY: &str = "/nifi-python-working-directory"; +pub const NIFI_PVC_STORAGE_DIRECTORY: &str = "/stackable/data"; + +pub const JVM_SECURITY_PROPERTIES_FILE: &str = "security.properties"; diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 9cd7b54e..7c4f0522 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,6 +1,8 @@ pub mod affinity; pub mod authorization; +pub mod constants; pub mod sensitive_properties; +pub mod storage; pub mod tls; use affinity::get_affinity; @@ -27,12 +29,12 @@ use stackable_operator::{ kube::{CustomResource, runtime::reflector::ObjectRef}, memory::MemoryQuantity, product_logging::{self, spec::Logging}, - role_utils::{GenericRoleConfig, JavaCommonConfig, Role, RoleGroupRef}, + role_utils::{GenericRoleConfig, Role, RoleGroupRef}, schemars::{self, JsonSchema}, shared::time::Duration, status::condition::{ClusterCondition, HasStatusCondition}, utils::crds::{raw_object_list_schema, raw_object_schema}, - v2::config_overrides::KeyValueConfigOverrides, + v2::{config_overrides::KeyValueConfigOverrides, role_utils::JavaCommonConfig}, versioned::versioned, }; use tls::NifiTls; diff --git a/rust/operator-binary/src/config/mod.rs b/rust/operator-binary/src/crd/storage.rs similarity index 50% rename from rust/operator-binary/src/config/mod.rs rename to rust/operator-binary/src/crd/storage.rs index d7ac693e..1dfae843 100644 --- a/rust/operator-binary/src/config/mod.rs +++ b/rust/operator-binary/src/crd/storage.rs @@ -1,18 +1,9 @@ -use std::{collections::BTreeMap, fmt::Write}; +//! NiFi repository storage layout: which repositories exist and how they map to volumes. -use snafu::Snafu; use stackable_operator::k8s_openapi::api::core::v1::VolumeMount; use strum::{Display, EnumIter}; -use crate::security::oidc; - -pub mod jvm; - -pub const NIFI_CONFIG_DIRECTORY: &str = "/stackable/nifi/conf"; -pub const NIFI_PYTHON_WORKING_DIRECTORY: &str = "/nifi-python-working-directory"; -pub const NIFI_PVC_STORAGE_DIRECTORY: &str = "/stackable/data"; - -pub const JVM_SECURITY_PROPERTIES_FILE: &str = "security.properties"; +use crate::crd::constants::NIFI_PVC_STORAGE_DIRECTORY; #[derive(Debug, Display, EnumIter)] pub enum NifiRepository { @@ -63,39 +54,3 @@ impl NifiRepository { } } } - -#[derive(Snafu, Debug)] -#[snafu(visibility(pub(crate)))] -pub enum Error { - #[snafu(display("invalid memory resource configuration - missing default or value in crd?"))] - MissingMemoryResourceConfig, - - #[snafu(display("invalid JVM config"))] - InvalidJVMConfig { source: jvm::Error }, - - #[snafu(display("failed to calculate storage quota for {repo} repository"))] - CalculateStorageQuota { - source: stackable_operator::memory::Error, - repo: NifiRepository, - }, - - #[snafu(display("failed to generate OIDC config"))] - GenerateOidcConfig { source: oidc::Error }, - - #[snafu(display( - "NiFi 1.x requires ZooKeeper (hint: upgrade to NiFi 2.x or set .spec.clusterConfig.zookeeperConfigMapName)" - ))] - Nifi1RequiresZookeeper, -} - -// TODO: Use crate like https://crates.io/crates/java-properties (currently does not work for Nifi -// because of escapes), to have save handling of escapes etc. -pub(crate) fn format_properties(properties: BTreeMap) -> String { - let mut result = String::new(); - - for (key, value) in properties { - let _ = writeln!(result, "{}={}", key, value); - } - - result -} diff --git a/rust/operator-binary/src/framework.rs b/rust/operator-binary/src/framework.rs deleted file mode 100644 index 0f5717f4..00000000 --- a/rust/operator-binary/src/framework.rs +++ /dev/null @@ -1,8 +0,0 @@ -//! Local additions to `stackable-operator` that are not yet (well) generalized in -//! `stackable_operator::v2::*`. -//! -//! Follow-up: replace these with `stackable_operator::v2::*` imports once upstream -//! reconciles `with_validated_config` (it currently returns a bare `RoleGroup`, and -//! the upstream `RoleGroupConfig` uses `EnvVarSet` rather than a plain map). - -pub mod role_utils; diff --git a/rust/operator-binary/src/framework/role_utils.rs b/rust/operator-binary/src/framework/role_utils.rs deleted file mode 100644 index d8ca6ea8..00000000 --- a/rust/operator-binary/src/framework/role_utils.rs +++ /dev/null @@ -1,215 +0,0 @@ -//! Vendored variant of `stackable_operator::v2::role_utils` from the -//! `smooth-operator` branch, with simplifications appropriate for nifi-operator. -//! -//! Differences from upstream: -//! - No `cli_overrides_to_vec` helper, `ResourceNames`, or service-account helpers. -//! - The `CommonConfig` (a.k.a. `product_specific_common_config`) does NOT need to -//! implement `Merge`. Upstream Trino uses `JavaCommonConfig`, which intentionally -//! does not implement `Merge` because its inner `JvmArgumentOverrides::try_merge` -//! is fallible (regex validation). Merging JVM argument overrides for Trino is -//! handled separately via `Role::get_merged_jvm_argument_overrides`. The -//! `RoleGroupConfig::product_specific_common_config` field here simply carries -//! the role-group level value through. -//! -//! Replace with `stackable_operator::v2::role_utils::*` once upstream publishes -//! the module. - -use std::{collections::BTreeMap, str::FromStr}; - -use serde::Serialize; -use snafu::{ResultExt, Snafu}; -use stackable_operator::{ - config::{ - fragment::{self, FromFragment}, - merge::{Merge, merge}, - }, - k8s_openapi::{DeepMerge, api::core::v1::PodTemplateSpec}, - role_utils::{Role, RoleGroup}, - schemars::JsonSchema, - v2::builder::pod::container::{self, EnvVarName, EnvVarSet}, -}; - -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("failed to validate the rolegroup config fragment"))] - ValidateConfig { source: fragment::ValidationError }, - - #[snafu(display("environment variable name {name:?} is invalid"))] - ParseEnvVarName { - source: container::Error, - name: String, - }, -} - -/// NiFi-friendly view of a validated, merged `RoleGroup`. -/// -/// Mirrors `stackable_operator::v2::role_utils::RoleGroupConfig` on the -/// `smooth-operator` branch. -#[derive(Clone, Debug, PartialEq)] -pub struct RoleGroupConfig { - pub replicas: u16, - pub config: Config, - pub config_overrides: ConfigOverrides, - pub env_overrides: EnvVarSet, - pub cli_overrides: BTreeMap, - pub pod_overrides: PodTemplateSpec, - pub product_specific_common_config: CommonConfig, -} - -/// Merges and validates the `RoleGroup` with the given `role` and `default_config`, -/// returning a `RoleGroupConfig`. -/// -/// Merge order matches `with_validated_config` on `smooth-operator`: -/// - `Config` (Fragment): `default_config <- role.config <- rg.config` via `Merge::merge`, -/// then validated to `ValidatedConfig` via `FromFragment`. -/// - `ConfigOverrides`: `role.config_overrides <- rg.config_overrides` via `Merge::merge`. -/// - `env_overrides`: `extend` (rg keys overwrite role keys), then parsed into an `EnvVarSet` -/// (rejecting invalid environment variable names). -/// - `cli_overrides`: `extend` (rg keys overwrite role keys). -/// - `pod_overrides`: `DeepMerge::merge_from` (rg overrides role). -/// - `product_specific_common_config`: passes through the role-group level value -/// (see module docs for rationale). -pub fn with_validated_config( - role_group: &RoleGroup, - role: &Role, - default_config: &Config, -) -> Result, Error> -where - ValidatedConfig: FromFragment, - CommonConfig: Clone + Default + JsonSchema + Serialize, - Config: Clone + Merge, - RoleConfig: Default + JsonSchema + Serialize, - ConfigOverrides: Clone + Default + JsonSchema + Merge + Serialize, -{ - let validated_config = - validate_config(role_group, role, default_config).context(ValidateConfigSnafu)?; - Ok(RoleGroupConfig { - replicas: role_group.replicas.unwrap_or(1), - config: validated_config, - config_overrides: merged_config_overrides( - &role.config.config_overrides, - role_group.config.config_overrides.clone(), - ), - env_overrides: merged_env_overrides( - role.config - .env_overrides - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(), - role_group - .config - .env_overrides - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(), - )?, - cli_overrides: merged_cli_overrides( - role.config.cli_overrides.clone(), - role_group.config.cli_overrides.clone(), - ), - pod_overrides: merged_pod_overrides( - role.config.pod_overrides.clone(), - role_group.config.pod_overrides.clone(), - ), - product_specific_common_config: role_group.config.product_specific_common_config.clone(), - }) -} - -fn validate_config( - role_group: &RoleGroup, - role: &Role, - default_config: &Config, -) -> Result -where - ValidatedConfig: FromFragment, - CommonConfig: Default + JsonSchema + Serialize, - Config: Clone + Merge, - RoleConfig: Default + JsonSchema + Serialize, - ConfigOverrides: Default + JsonSchema + Serialize, -{ - role_group.validate_config(role, default_config) -} - -fn merged_config_overrides( - role_config_overrides: &ConfigOverrides, - role_group_config_overrides: ConfigOverrides, -) -> ConfigOverrides -where - ConfigOverrides: Merge, -{ - merge(role_group_config_overrides, role_config_overrides) -} - -fn merged_env_overrides( - role_env_overrides: BTreeMap, - role_group_env_overrides: BTreeMap, -) -> Result { - let mut merged = role_env_overrides; - merged.extend(role_group_env_overrides); - - let mut env_overrides = EnvVarSet::new(); - for (name, value) in merged { - let env_var_name = - EnvVarName::from_str(&name).context(ParseEnvVarNameSnafu { name: name.clone() })?; - env_overrides = env_overrides.with_value(&env_var_name, value); - } - Ok(env_overrides) -} - -fn merged_cli_overrides( - role_cli_overrides: BTreeMap, - role_group_cli_overrides: BTreeMap, -) -> BTreeMap { - let mut merged = role_cli_overrides; - merged.extend(role_group_cli_overrides); - merged -} - -fn merged_pod_overrides( - role_pod_overrides: PodTemplateSpec, - role_group_pod_overrides: PodTemplateSpec, -) -> PodTemplateSpec { - let mut merged = role_pod_overrides; - merged.merge_from(role_group_pod_overrides); - merged -} - -#[cfg(test)] -mod tests { - use std::collections::BTreeMap; - - use stackable_operator::v2::builder::pod::container::EnvVarName; - - use super::merged_env_overrides; - - #[test] - fn env_overrides_role_group_value_wins_over_role_value() { - let role = BTreeMap::from([ - ("SHARED".to_owned(), "from-role".to_owned()), - ("ROLE_ONLY".to_owned(), "role".to_owned()), - ]); - let role_group = BTreeMap::from([("SHARED".to_owned(), "from-rolegroup".to_owned())]); - - let merged = merged_env_overrides(role, role_group).expect("env overrides should merge"); - - assert_eq!( - merged - .get(&EnvVarName::from_str_unsafe("SHARED")) - .and_then(|env_var| env_var.value.clone()), - Some("from-rolegroup".to_owned()) - ); - assert_eq!( - merged - .get(&EnvVarName::from_str_unsafe("ROLE_ONLY")) - .and_then(|env_var| env_var.value.clone()), - Some("role".to_owned()) - ); - } - - #[test] - fn invalid_env_var_name_is_rejected() { - let role = BTreeMap::from([("INVALID=NAME".to_owned(), "value".to_owned())]); - - assert!(merged_env_overrides(role, BTreeMap::new()).is_err()); - } -} diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 45fc2492..117be255 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -38,10 +38,8 @@ use crate::{ webhooks::conversion::create_webhook_server, }; -mod config; mod controller; mod crd; -mod framework; mod listener; mod nifi_controller; mod operations; diff --git a/rust/operator-binary/src/nifi_controller.rs b/rust/operator-binary/src/nifi_controller.rs index fe660e1c..fe6de84d 100644 --- a/rust/operator-binary/src/nifi_controller.rs +++ b/rust/operator-binary/src/nifi_controller.rs @@ -64,16 +64,15 @@ use tracing::Instrument; use crate::{ OPERATOR_NAME, - config::{ - NIFI_CONFIG_DIRECTORY, NIFI_PYTHON_WORKING_DIRECTORY, NifiRepository, - PERSISTENT_REPOSITORIES, - }, - controller::{build, dereference, validate, validate::NifiRoleGroupConfig}, + controller::{ValidatedRoleGroupConfig, build, dereference, validate}, crd::{ APP_NAME, BALANCE_PORT, BALANCE_PORT_NAME, Container, HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT, METRICS_PORT_NAME, NifiConfig, NifiNodeRoleConfig, NifiRole, NifiRoleType, NifiStatus, PROTOCOL_PORT, PROTOCOL_PORT_NAME, STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, - authorization::NifiAccessPolicyProvider, v1alpha1, + authorization::NifiAccessPolicyProvider, + constants::{NIFI_CONFIG_DIRECTORY, NIFI_PYTHON_WORKING_DIRECTORY}, + storage::{NifiRepository, PERSISTENT_REPOSITORIES}, + v1alpha1, }, listener::{ LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_group_listener, build_group_listener_pvc, @@ -615,7 +614,7 @@ async fn build_node_rolegroup_statefulset( namespace: &NamespaceName, rolegroup_ref: &RoleGroupRef, role: &NifiRoleType, - rg: &NifiRoleGroupConfig, + rg: &ValidatedRoleGroupConfig, authentication_config: &NifiAuthenticationConfig, authorization_config: &ResolvedNifiAuthorizationConfig, rolling_update_supported: bool, diff --git a/rust/operator-binary/src/security/authorization.rs b/rust/operator-binary/src/security/authorization.rs index c1d92a9b..58ea0878 100644 --- a/rust/operator-binary/src/security/authorization.rs +++ b/rust/operator-binary/src/security/authorization.rs @@ -9,9 +9,10 @@ use stackable_operator::{ }, }; -use crate::{ - config::{NIFI_PVC_STORAGE_DIRECTORY, NifiRepository}, - crd::authorization::{NifiAccessPolicyProvider, NifiAuthorization, NifiOpaConfig}, +use crate::crd::{ + authorization::{NifiAccessPolicyProvider, NifiAuthorization, NifiOpaConfig}, + constants::NIFI_PVC_STORAGE_DIRECTORY, + storage::NifiRepository, }; const OPA_TLS_VOLUME_NAME: &str = "opa-tls"; From 141efef77a5c5268bcd876b0e303e350abcca49e Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 12 Jun 2026 12:06:02 +0200 Subject: [PATCH 27/39] refactor: use v2 owerref, RoleGroupName, labels, ResourceNames --- .../src/controller/build/config_map.rs | 51 +++-- .../src/controller/build/properties.rs | 9 +- .../build/properties/bootstrap_conf.rs | 9 +- .../build/properties/nifi_properties.rs | 9 +- rust/operator-binary/src/controller/mod.rs | 113 +++++++++- .../src/controller/validate.rs | 23 +- rust/operator-binary/src/crd/affinity.rs | 9 +- rust/operator-binary/src/listener.rs | 29 +-- rust/operator-binary/src/nifi_controller.rs | 200 ++++++------------ .../operator-binary/src/reporting_task/mod.rs | 78 +++---- rust/operator-binary/src/service.rs | 84 ++++---- 11 files changed, 321 insertions(+), 293 deletions(-) diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index e4615e7d..86cefe7a 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -4,11 +4,9 @@ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, k8s_openapi::api::core::v1::ConfigMap, - kvp::ObjectLabels, product_logging::framework::VECTOR_CONFIG_FILE, - role_utils::RoleGroupRef, utils::cluster_info::KubernetesClusterInfo, - v2::builder::meta::ownerreference_from_resource, + v2::{builder::meta::ownerreference_from_resource, types::operator::RoleGroupName}, }; use crate::{ @@ -23,16 +21,11 @@ use crate::{ proxy_hosts, }, }, - crd::{NifiRole, v1alpha1}, + crd::NifiRole, }; #[derive(Debug, Snafu)] pub enum Error { - #[snafu(display("failed to build metadata"))] - MetadataBuild { - source: stackable_operator::builder::meta::Error, - }, - #[snafu(display("failed to build bootstrap.conf"))] BootstrapConfig { #[snafu(source(from(crate::controller::build::Error, Box::new)))] @@ -43,19 +36,19 @@ pub enum Error { BuildNifiProperties { #[snafu(source(from(crate::controller::build::Error, Box::new)))] source: Box, - rolegroup: RoleGroupRef, + rolegroup: RoleGroupName, }, #[snafu(display("failed to build ConfigMap for {rolegroup}"))] BuildRoleGroupConfig { source: stackable_operator::builder::configmap::Error, - rolegroup: RoleGroupRef, + rolegroup: RoleGroupName, }, #[snafu(display("failed to serialize JVM security properties for {}", rolegroup))] JvmSecurityProperties { source: stackable_operator::v2::config_file_writer::PropertiesWriterError, - rolegroup: String, + rolegroup: RoleGroupName, }, #[snafu(display("failed to build login-identity-providers configuration"))] @@ -75,23 +68,25 @@ type Result = std::result::Result; /// Build the rolegroup [`ConfigMap`] configuring the rolegroup based on the /// resolved cluster configuration. /// -/// All NiFi configuration is sourced from `cluster`. `recommended_labels` must be built by the -/// caller (typically via `build_recommended_labels`). +/// All NiFi configuration is sourced from `cluster`. +/// +/// `vector_config` is the Vector agent config (`vector.yaml`) built by the caller (where a +/// `RoleGroupRef` is available); it is `None` when the Vector agent is disabled. pub fn build_rolegroup_config_map( cluster: &ValidatedCluster, - rolegroup: &RoleGroupRef, - recommended_labels: &ObjectLabels<'_, ValidatedCluster>, + role_group_name: &RoleGroupName, cluster_info: &KubernetesClusterInfo, + vector_config: Option, ) -> Result { tracing::debug!("building rolegroup ConfigMap"); let rg = cluster .role_group_configs .get(&NifiRole::Node) - .and_then(|groups| groups.get(&rolegroup.role_group)) + .and_then(|groups| groups.get(role_group_name)) .with_context(|| MissingRoleGroupSnafu { role: NifiRole::Node.to_string(), - role_group: rolegroup.role_group.clone(), + role_group: role_group_name.to_string(), })?; let proxy_hosts = proxy_hosts::compute_proxy_hosts(cluster, cluster_info); @@ -103,11 +98,15 @@ pub fn build_rolegroup_config_map( cm_builder .metadata( ObjectMetaBuilder::new() - .namespace(&cluster.namespace) - .name(rolegroup.object_name()) + .name_and_namespace(cluster) + .name( + cluster + .resource_names(role_group_name) + .role_group_config_map() + .to_string(), + ) .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) - .with_recommended_labels(recommended_labels) - .context(MetadataBuildSnafu)? + .with_labels(cluster.recommended_labels(role_group_name)) .build(), ) .add_data( @@ -119,7 +118,7 @@ pub fn build_rolegroup_config_map( ConfigFileName::NifiProperties.to_string(), nifi_properties::build(cluster, rg, &proxy_hosts, &git_sync_resources).with_context( |_| BuildNifiPropertiesSnafu { - rolegroup: rolegroup.clone(), + rolegroup: role_group_name.clone(), }, )?, ) @@ -139,7 +138,7 @@ pub fn build_rolegroup_config_map( .add_data( ConfigFileName::SecurityProperties.to_string(), security_properties::build(rg).with_context(|_| JvmSecurityPropertiesSnafu { - rolegroup: rolegroup.role_group.clone(), + rolegroup: role_group_name.clone(), })?, ); @@ -147,13 +146,13 @@ pub fn build_rolegroup_config_map( cm_builder.add_data(ConfigFileName::Logback.to_string(), logback_config); } - if let Some(vector_config) = logging::build_vector_config(rolegroup, &rg.config.logging) { + if let Some(vector_config) = vector_config { cm_builder.add_data(VECTOR_CONFIG_FILE, vector_config); } cm_builder .build() .with_context(|_| BuildRoleGroupConfigSnafu { - rolegroup: rolegroup.clone(), + rolegroup: role_group_name.clone(), }) } diff --git a/rust/operator-binary/src/controller/build/properties.rs b/rust/operator-binary/src/controller/build/properties.rs index d3cb5cec..f43a1433 100644 --- a/rust/operator-binary/src/controller/build/properties.rs +++ b/rust/operator-binary/src/controller/build/properties.rs @@ -77,7 +77,7 @@ pub(crate) mod test_support { kvp::LabelValue, v2::types::{ kubernetes::{NamespaceName, Uid}, - operator::ClusterName, + operator::{ClusterName, ProductVersion, RoleGroupName}, }, }; @@ -144,12 +144,15 @@ pub(crate) mod test_support { let name = ClusterName::from_str("simple-nifi").expect("valid cluster name"); let namespace = NamespaceName::from_str("default").expect("valid namespace"); let uid = Uid::from_str("e6ac237d-a6d4-43a1-8135-f36506110912").expect("valid uid"); + let product_version = ProductVersion::from_str(&image.app_version_label_value) + .expect("valid product version"); ValidatedCluster::new( name, namespace, uid, image, + product_version, role_group_configs, ValidatedClusterConfig { authentication: NifiAuthenticationConfig::SingleUser { @@ -177,7 +180,9 @@ pub(crate) mod test_support { cluster .role_group_configs .get(&NifiRole::Node) - .and_then(|rgs| rgs.get("default")) + .and_then(|rgs| { + rgs.get(&RoleGroupName::from_str("default").expect("valid role-group name")) + }) .expect("minimal_validated_cluster must contain a 'default' role group") } diff --git a/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs b/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs index c91f54d0..5cb17d99 100644 --- a/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs +++ b/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs @@ -49,6 +49,7 @@ pub fn build( #[cfg(test)] mod tests { use indoc::indoc; + use stackable_operator::v2::types::operator::RoleGroupName; use super::*; use crate::{ @@ -64,7 +65,13 @@ mod tests { build_role_group_configs(&nifi).expect("failed to build role group configs"); let rg = role_group_configs .get(&NifiRole::Node) - .and_then(|groups| groups.get("default")) + .and_then(|groups| { + groups.get( + &"default" + .parse::() + .expect("valid role-group name"), + ) + }) .expect("default role group must exist"); build(rg, None).unwrap() diff --git a/rust/operator-binary/src/controller/build/properties/nifi_properties.rs b/rust/operator-binary/src/controller/build/properties/nifi_properties.rs index b351eb8e..a1e318d4 100644 --- a/rust/operator-binary/src/controller/build/properties/nifi_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/nifi_properties.rs @@ -642,6 +642,8 @@ mod tests { /// Verify that a user configOverride for `nifi.properties` flows through to the output. #[test] fn test_config_override_wins() { + use stackable_operator::v2::types::operator::RoleGroupName; + use crate::{ controller::validate::build_role_group_configs, crd::{NifiRole, v1alpha1}, @@ -673,9 +675,12 @@ mod tests { let nifi: v1alpha1::NifiCluster = serde_yaml::from_str(yaml).expect("invalid test YAML"); let mut role_group_configs = build_role_group_configs(&nifi).expect("failed to build role group configs"); + let default_rg_name = "default" + .parse::() + .expect("valid role-group name"); let rg = role_group_configs .get_mut(&NifiRole::Node) - .and_then(|groups| groups.remove("default")) + .and_then(|groups| groups.remove(&default_rg_name)) .expect("default role group must exist"); // Build a cluster with this rg substituted in @@ -684,7 +689,7 @@ mod tests { .role_group_configs .get_mut(&NifiRole::Node) .unwrap() - .insert("default".to_string(), rg.clone()); + .insert(default_rg_name, rg.clone()); let git_sync = empty_git_sync_resources(); let props = diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index c2804dcd..356f72af 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -2,29 +2,37 @@ //! [`validate`] step and consumed by the [`build`] steps, plus the //! `dereference` / `validate` / `build` sub-modules. -use std::collections::BTreeMap; +use std::{collections::BTreeMap, str::FromStr as _}; use stackable_operator::{ commons::product_image_selection::ResolvedProductImage, crd::git_sync, k8s_openapi::{api::core::v1::PodTemplateSpec, apimachinery::pkg::apis::meta::v1::ObjectMeta}, kube::Resource, + kvp::Labels, v2::{ - HasName, HasUid, + HasName, HasUid, NameIsValidLabelValue, builder::pod::container::EnvVarSet, + kvp::label::{recommended_labels, role_group_selector, role_selector}, + role_group_utils::ResourceNames, role_utils::JavaCommonConfig, types::{ kubernetes::{NamespaceName, Uid}, - operator::ClusterName, + operator::{ + ClusterName, ControllerName, OperatorName, ProductName, ProductVersion, + RoleGroupName, RoleName, + }, }, }, }; use crate::{ + OPERATOR_NAME, crd::{ - HostHeaderCheckConfig, NifiConfig, NifiRole, + APP_NAME, HostHeaderCheckConfig, NifiConfig, NifiRole, sensitive_properties::NifiSensitiveKeyAlgorithm, v1alpha1, }, + nifi_controller::NIFI_CONTROLLER_NAME, security::{ authentication::NifiAuthenticationConfig, authorization::ResolvedNifiAuthorizationConfig, }, @@ -76,10 +84,13 @@ pub struct ValidatedCluster { pub uid: Uid, /// The product image. pub image: ResolvedProductImage, + /// The product version as a type-safe label value, used for the `app.kubernetes.io/version` + /// label on built resources. + pub product_version: ProductVersion, /// Cluster wide settings. pub cluster_config: ValidatedClusterConfig, /// Collected configuration per rolegroup. - pub role_group_configs: BTreeMap>, + pub role_group_configs: BTreeMap>, } /// The resolved `spec.clusterConfig`. @@ -101,12 +112,14 @@ pub struct ValidatedClusterConfig { impl ValidatedCluster { /// Builds a [`ValidatedCluster`], deriving the synthetic [`ObjectMeta`] from name, namespace /// and uid so the struct can implement [`Resource`]. + #[allow(clippy::too_many_arguments)] pub fn new( name: ClusterName, namespace: NamespaceName, uid: Uid, image: ResolvedProductImage, - role_group_configs: BTreeMap>, + product_version: ProductVersion, + role_group_configs: BTreeMap>, cluster_config: ValidatedClusterConfig, ) -> Self { let metadata = ObjectMeta { @@ -122,10 +135,98 @@ impl ValidatedCluster { namespace, uid, image, + product_version, role_group_configs, cluster_config, } } + + /// The single NiFi role name (`node`). + pub fn role_name() -> RoleName { + RoleName::from_str(&NifiRole::Node.to_string()) + .expect("the node role name is a valid role name") + } + + /// Type-safe names for the resources of a given role group. + pub(crate) fn resource_names(&self, role_group_name: &RoleGroupName) -> ResourceNames { + ResourceNames { + cluster_name: self.name.clone(), + role_name: Self::role_name(), + role_group_name: role_group_name.clone(), + } + } + + /// Recommended labels for a role-group resource, using the given product version. + fn recommended_labels_for( + &self, + product_version: &ProductVersion, + role_group_name: &RoleGroupName, + ) -> Labels { + recommended_labels( + self, + &product_name(), + product_version, + &operator_name(), + &controller_name(), + &Self::role_name(), + role_group_name, + ) + } + + /// Recommended labels for a role-group resource. + pub fn recommended_labels(&self, role_group_name: &RoleGroupName) -> Labels { + self.recommended_labels_for(&self.product_version, role_group_name) + } + + /// Recommended labels for resources whose labels must stay stable across version upgrades + /// (e.g. PVC templates, which are immutable once created), using the placeholder version + /// `none` for `app.kubernetes.io/version`. + pub fn recommended_labels_unversioned(&self, role_group_name: &RoleGroupName) -> Labels { + let unversioned = ProductVersion::from_str("none") + .expect("'none' is a valid product version label value"); + self.recommended_labels_for(&unversioned, role_group_name) + } + + /// Selector labels matching the pods of a role group. + pub fn role_group_selector(&self, role_group_name: &RoleGroupName) -> Labels { + role_group_selector(self, &product_name(), &Self::role_name(), role_group_name) + } + + /// Selector labels matching all pods of the (single) NiFi role. + pub fn role_selector(&self) -> Labels { + role_selector(self, &product_name(), &Self::role_name()) + } + + /// Recommended labels for a role-level resource (the per-role [`Listener`]), which has no + /// associated role group. Uses the placeholder role-group `none`, preserving the historical + /// `app.kubernetes.io/role-group: none` label. + pub fn recommended_labels_role_level(&self) -> Labels { + let role_group = + RoleGroupName::from_str("none").expect("'none' is a valid role-group name"); + self.recommended_labels(&role_group) + } +} + +/// The product name (`nifi`) as a type-safe label value. +fn product_name() -> ProductName { + ProductName::from_str(APP_NAME).expect("'nifi' is a valid product name") +} + +/// The operator name as a type-safe label value. +fn operator_name() -> OperatorName { + OperatorName::from_str(OPERATOR_NAME).expect("the operator name is a valid label value") +} + +/// The controller name as a type-safe label value. +fn controller_name() -> ControllerName { + ControllerName::from_str(NIFI_CONTROLLER_NAME) + .expect("the controller name is a valid label value") +} + +impl NameIsValidLabelValue for ValidatedCluster { + fn to_label_value(&self) -> String { + self.name.to_label_value() + } } impl HasName for ValidatedCluster { diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index b1acaaff..e118e83f 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -16,6 +16,7 @@ use stackable_operator::{ builder::pod::container::{self, EnvVarName, EnvVarSet}, controller_utils::{self, get_cluster_name, get_uid}, role_utils::with_validated_config, + types::operator::{ProductVersion, RoleGroupName}, }, }; use strum::{EnumDiscriminants, IntoStaticStr}; @@ -54,6 +55,12 @@ pub enum Error { #[snafu(display("failed to validate the rolegroup config fragment"))] ValidateRoleGroupConfig { source: fragment::ValidationError }, + #[snafu(display("the role-group name {role_group:?} is invalid"))] + ParseRoleGroupName { + source: stackable_operator::v2::macros::attributed_string_type::Error, + role_group: String, + }, + #[snafu(display("environment variable name {name:?} is invalid"))] ParseEnvVarName { source: container::Error, @@ -108,11 +115,17 @@ pub fn validate( let namespace = dereferenced_objects.namespace.clone(); let uid = get_uid(nifi).context(GetUidSnafu)?; + // `app_version_label_value` is constructed to be a valid label value, so it is always a valid + // `ProductVersion`. It is used for the `app.kubernetes.io/version` label on built resources. + let product_version = ProductVersion::from_str(&image.app_version_label_value) + .expect("the app version label value is a valid product version"); + Ok(ValidatedCluster::new( name, namespace, uid, image, + product_version, role_group_configs, ValidatedClusterConfig { authentication: authentication_config, @@ -127,12 +140,16 @@ pub fn validate( pub(crate) fn build_role_group_configs( nifi: &v1alpha1::NifiCluster, -) -> Result>> { +) -> Result>> { let role = nifi.spec.nodes.as_ref().context(NoNodesDefinedSnafu)?; let default_config = NifiConfig::default_config(&nifi.name_any(), &NifiRole::Node); - let mut groups: BTreeMap = BTreeMap::new(); + let mut groups: BTreeMap = BTreeMap::new(); for (rg_name, rg) in &role.role_groups { + let role_group_name = + RoleGroupName::from_str(rg_name).with_context(|_| ParseRoleGroupNameSnafu { + role_group: rg_name.clone(), + })?; let validated = with_validated_config::(rg, role, &default_config) .context(ValidateRoleGroupConfigSnafu)?; @@ -157,7 +174,7 @@ pub(crate) fn build_role_group_configs( } groups.insert( - rg_name.clone(), + role_group_name, ValidatedRoleGroupConfig { replicas: validated.replicas.unwrap_or(1), config, diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index b6b75db1..201c91af 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -29,6 +29,7 @@ mod tests { api::core::v1::{PodAffinityTerm, PodAntiAffinity, WeightedPodAffinityTerm}, apimachinery::pkg::apis::meta::v1::LabelSelector, }, + v2::types::operator::RoleGroupName, }; use super::*; @@ -62,7 +63,13 @@ mod tests { let role_group_configs = build_role_group_configs(&nifi).unwrap(); let merged_config = &role_group_configs .get(&NifiRole::Node) - .and_then(|groups| groups.get("default")) + .and_then(|groups| { + groups.get( + &"default" + .parse::() + .expect("valid role-group name"), + ) + }) .unwrap() .config; diff --git a/rust/operator-binary/src/listener.rs b/rust/operator-binary/src/listener.rs index 27ac8582..c7fe39c8 100644 --- a/rust/operator-binary/src/listener.rs +++ b/rust/operator-binary/src/listener.rs @@ -7,26 +7,20 @@ use stackable_operator::{ crd::listener::v1alpha1::{Listener, ListenerPort, ListenerSpec}, k8s_openapi::api::core::v1::PersistentVolumeClaim, kube::ResourceExt, - kvp::{Labels, ObjectLabels}, + kvp::Labels, + v2::builder::meta::ownerreference_from_resource, }; -use crate::crd::{HTTPS_PORT, HTTPS_PORT_NAME, v1alpha1}; +use crate::{ + controller::ValidatedCluster, + crd::{HTTPS_PORT, HTTPS_PORT_NAME, v1alpha1}, +}; pub const LISTENER_VOLUME_NAME: &str = "listener"; pub const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("listener object is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, - }, - - #[snafu(display("failed to build listener object meta data"))] - BuildObjectMeta { - source: stackable_operator::builder::meta::Error, - }, - #[snafu(display("failed to build listener volume"))] BuildListenerPersistentVolume { source: stackable_operator::builder::pod::volume::ListenerOperatorVolumeSourceBuilderError, @@ -34,19 +28,16 @@ pub enum Error { } pub fn build_group_listener( - nifi: &v1alpha1::NifiCluster, - object_labels: ObjectLabels, + cluster: &ValidatedCluster, listener_class: String, listener_group_name: String, ) -> Result { Ok(Listener { metadata: ObjectMetaBuilder::new() - .name_and_namespace(nifi) + .name_and_namespace(cluster) .name(listener_group_name) - .ownerreference_from_resource(nifi, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&object_labels) - .context(BuildObjectMetaSnafu)? + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(cluster.recommended_labels_role_level()) .build(), spec: ListenerSpec { class_name: Some(listener_class), diff --git a/rust/operator-binary/src/nifi_controller.rs b/rust/operator-binary/src/nifi_controller.rs index fe6de84d..1fbb3f01 100644 --- a/rust/operator-binary/src/nifi_controller.rs +++ b/rust/operator-binary/src/nifi_controller.rs @@ -37,7 +37,6 @@ use stackable_operator::{ core::{DeserializeGuard, error_boundary}, runtime::controller::Action, }, - kvp::{Labels, ObjectLabels}, logging::controller::ReconcilerError, memory::{BinaryMultiple, MemoryQuantity}, product_logging::{ @@ -50,21 +49,21 @@ use stackable_operator::{ CustomContainerLogConfig, }, }, - role_utils::{GenericRoleConfig, RoleGroupRef}, + role_utils::GenericRoleConfig, shared::time::Duration, status::condition::{ compute_conditions, operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, }, utils::{COMMON_BASH_TRAP_FUNCTIONS, cluster_info::KubernetesClusterInfo}, - v2::types::kubernetes::NamespaceName, + v2::{builder::meta::ownerreference_from_resource, types::operator::RoleGroupName}, }; use strum::{EnumDiscriminants, IntoStaticStr}; use tracing::Instrument; use crate::{ OPERATOR_NAME, - controller::{ValidatedRoleGroupConfig, build, dereference, validate}, + controller::{ValidatedCluster, ValidatedRoleGroupConfig, build, dereference, validate}, crd::{ APP_NAME, BALANCE_PORT, BALANCE_PORT_NAME, Container, HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT, METRICS_PORT_NAME, NifiConfig, NifiNodeRoleConfig, NifiRole, NifiRoleType, @@ -145,13 +144,13 @@ pub enum Error { #[snafu(display("failed to apply Service for {}", rolegroup))] ApplyRoleGroupService { source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, + rolegroup: RoleGroupName, }, #[snafu(display("failed to build rolegroup ConfigMap for {}", rolegroup))] BuildRoleGroupConfigMap { source: build::config_map::Error, - rolegroup: RoleGroupRef, + rolegroup: RoleGroupName, }, #[snafu(display("object has no nodes defined"))] @@ -160,13 +159,13 @@ pub enum Error { #[snafu(display("failed to apply ConfigMap for {}", rolegroup))] ApplyRoleGroupConfig { source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, + rolegroup: RoleGroupName, }, #[snafu(display("failed to apply StatefulSet for {}", rolegroup))] ApplyRoleGroupStatefulSet { source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, + rolegroup: RoleGroupName, }, #[snafu(display("failed to apply create ReportingTask service"))] @@ -179,11 +178,6 @@ pub enum Error { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("object is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, - }, - #[snafu(display("illegal container name: [{container_name}]"))] IllegalContainerName { source: stackable_operator::builder::pod::container::Error, @@ -221,22 +215,12 @@ pub enum Error { source: crate::operations::graceful_shutdown::Error, }, - #[snafu(display("failed to build metadata"))] - MetadataBuild { - source: stackable_operator::builder::meta::Error, - }, - #[snafu(display("failed to get required labels"))] GetRequiredLabels { source: stackable_operator::kvp::KeyValuePairError, }, - #[snafu(display("failed to build labels"))] - LabelBuild { - source: stackable_operator::kvp::LabelError, - }, - #[snafu(display("failed to add Authentication Volumes and VolumeMounts"))] AddAuthVolumes { source: crate::security::authentication::Error, @@ -272,9 +256,6 @@ pub enum Error { #[snafu(display("failed to configure listener"))] ListenerConfiguration { source: crate::listener::Error }, - #[snafu(display("failed to configure service"))] - ServiceConfiguration { source: crate::service::Error }, - #[snafu(display("failed to build authorization configuration"))] AuthorizationConfiguration { source: authorization::Error }, } @@ -390,49 +371,38 @@ pub async fn reconcile_nifi( .role_group_configs .get(&nifi_role) .context(NoNodesDefinedSnafu)?; - for (rolegroup_name, rg) in node_role_group_configs.iter() { - let rg_span = tracing::info_span!("rolegroup_span", rolegroup = rolegroup_name.as_str()); + for (role_group_name, rg) in node_role_group_configs.iter() { + let rg_span = tracing::info_span!("rolegroup_span", rolegroup = role_group_name.as_ref()); async { - let rolegroup = nifi.node_rolegroup_ref(rolegroup_name); - - tracing::debug!("Processing rolegroup {}", rolegroup); + tracing::debug!("Processing rolegroup {role_group_name}"); let git_sync_resources = build::git_sync::build_git_sync_resources(&validated_cluster, rg) .context(BuildGitSyncResourcesSnafu)?; - let role_group_service_recommended_labels = build_recommended_labels( - &validated_cluster, - &resolved_product_image.app_version_label_value, - &rolegroup.role, - &rolegroup.role_group, - ); - - let role_group_service_selector = - Labels::role_group_selector(nifi, APP_NAME, &rolegroup.role, &rolegroup.role_group) - .context(LabelBuildSnafu)?; - - let rg_headless_service = build_rolegroup_headless_service( - &validated_cluster, - &rolegroup, - &role_group_service_recommended_labels, - role_group_service_selector.clone().into(), - ) - .context(ServiceConfigurationSnafu)?; + let rg_headless_service = + build_rolegroup_headless_service(&validated_cluster, role_group_name); let role = nifi.spec.nodes.as_ref().context(NoNodesDefinedSnafu)?; + // The Vector agent config is the only remaining consumer of `RoleGroupRef`, which is + // therefore constructed on demand here rather than threaded through the build steps. + let vector_config = build::properties::logging::build_vector_config( + &nifi.node_rolegroup_ref(role_group_name.to_string()), + &rg.config.logging, + ); + let rg_configmap = build::config_map::build_rolegroup_config_map( &validated_cluster, - &rolegroup, - &role_group_service_recommended_labels, + role_group_name, &client.kubernetes_cluster_info, + vector_config, ) .context(BuildRoleGroupConfigMapSnafu { - rolegroup: rolegroup.clone(), + rolegroup: role_group_name.clone(), })?; - let role_group = role.role_groups.get(&rolegroup.role_group); + let role_group = role.role_groups.get(role_group_name.as_ref()); let replicas = if cluster_version_update_state == ClusterVersionUpdateState::UpdateRequested { Some(0) @@ -442,10 +412,10 @@ pub async fn reconcile_nifi( let rg_statefulset = build_node_rolegroup_statefulset( nifi, + &validated_cluster, resolved_product_image, &client.kubernetes_cluster_info, - &validated_cluster.namespace, - &rolegroup, + role_group_name, role, rg, authentication_config, @@ -457,34 +427,28 @@ pub async fn reconcile_nifi( ) .await?; - let rg_metrics_service = build_rolegroup_metrics_service( - &validated_cluster, - &rolegroup, - &role_group_service_recommended_labels, - role_group_service_selector.into(), - &resolved_product_image.product_version, - ) - .context(ServiceConfigurationSnafu)?; + let rg_metrics_service = + build_rolegroup_metrics_service(&validated_cluster, role_group_name); cluster_resources .add(client, rg_metrics_service) .await .with_context(|_| ApplyRoleGroupServiceSnafu { - rolegroup: rolegroup.clone(), + rolegroup: role_group_name.clone(), })?; cluster_resources .add(client, rg_headless_service) .await .with_context(|_| ApplyRoleGroupServiceSnafu { - rolegroup: rolegroup.clone(), + rolegroup: role_group_name.clone(), })?; cluster_resources .add(client, rg_configmap) .await .with_context(|_| ApplyRoleGroupConfigSnafu { - rolegroup: rolegroup.clone(), + rolegroup: role_group_name.clone(), })?; // Note: The StatefulSet needs to be applied after all ConfigMaps and Secrets it mounts @@ -495,7 +459,7 @@ pub async fn reconcile_nifi( .add(client, rg_statefulset) .await .with_context(|_| ApplyRoleGroupStatefulSetSnafu { - rolegroup: rolegroup.clone(), + rolegroup: role_group_name.clone(), })?, ); @@ -518,13 +482,7 @@ pub async fn reconcile_nifi( .context(FailedToCreatePdbSnafu)?; let role_group_listener = build_group_listener( - nifi, - build_recommended_labels( - nifi, - &resolved_product_image.app_version_label_value, - &nifi_role.to_string(), - "none", - ), + &validated_cluster, listener_class.to_owned(), group_listener_name(nifi, &nifi_role.to_string()), ) @@ -540,6 +498,7 @@ pub async fn reconcile_nifi( if nifi.spec.cluster_config.create_reporting_task_job.enabled { if let Some((reporting_task_job, reporting_task_service)) = build_maybe_reporting_task( nifi, + &validated_cluster, resolved_product_image, &client.kubernetes_cluster_info, &validated_cluster.namespace, @@ -609,10 +568,10 @@ const USERDATA_MOUNTPOINT: &str = "/stackable/userdata"; #[allow(clippy::too_many_arguments)] async fn build_node_rolegroup_statefulset( nifi: &v1alpha1::NifiCluster, + cluster: &ValidatedCluster, resolved_product_image: &ResolvedProductImage, cluster_info: &KubernetesClusterInfo, - namespace: &NamespaceName, - rolegroup_ref: &RoleGroupRef, + role_group_name: &RoleGroupName, role: &NifiRoleType, rg: &ValidatedRoleGroupConfig, authentication_config: &NifiAuthenticationConfig, @@ -624,6 +583,9 @@ async fn build_node_rolegroup_statefulset( ) -> Result { tracing::debug!("Building statefulset"); + // Type-safe names for this role group's resources (StatefulSet, ConfigMap, headless Service). + let resource_names = cluster.resource_names(role_group_name); + // The validated, merged `NifiConfig` is the single source of truth; the ConfigMap builder // sources the same `rg.config`. let merged_config = &rg.config; @@ -688,7 +650,8 @@ async fn build_node_rolegroup_statefulset( let node_address = format!( "$POD_NAME.{service_name}.{namespace}.svc.{cluster_domain}", - service_name = rolegroup_ref.rolegroup_headless_service_name(), + service_name = resource_names.headless_service_name(), + namespace = cluster.namespace, cluster_domain = cluster_info.cluster_domain, ); @@ -899,12 +862,7 @@ async fn build_node_rolegroup_statefulset( let mut pod_builder = PodBuilder::new(); - let recommended_object_labels = build_recommended_labels( - nifi, - &resolved_product_image.app_version_label_value, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - ); + let recommended_object_labels = cluster.recommended_labels(role_group_name); add_graceful_shutdown_config(merged_config, &mut pod_builder).context(GracefulShutdownSnafu)?; @@ -981,7 +939,7 @@ async fn build_node_rolegroup_statefulset( .add_volume(Volume { name: "log-config".to_string(), config_map: Some(ConfigMapVolumeSource { - name: rolegroup_ref.object_name(), + name: resource_names.role_group_config_map().to_string(), ..ConfigMapVolumeSource::default() }), ..Volume::default() @@ -1020,13 +978,7 @@ async fn build_node_rolegroup_statefulset( .context(AddAuthVolumesSnafu)?; let metadata = ObjectMetaBuilder::new() - .with_recommended_labels(&build_recommended_labels( - nifi, - &resolved_product_image.app_version_label_value, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - )) - .context(MetadataBuildSnafu)? + .with_labels(recommended_object_labels.clone()) .build(); let requested_secret_lifetime = merged_config @@ -1043,7 +995,7 @@ async fn build_node_rolegroup_statefulset( .add_volume(stackable_operator::k8s_openapi::api::core::v1::Volume { name: "config".to_string(), config_map: Some(ConfigMapVolumeSource { - name: rolegroup_ref.object_name(), + name: resource_names.role_group_config_map().to_string(), ..Default::default() }), ..Default::default() @@ -1052,7 +1004,7 @@ async fn build_node_rolegroup_statefulset( .add_volume(Volume { name: "conf".to_string(), config_map: Some(ConfigMapVolumeSource { - name: rolegroup_ref.object_name(), + name: resource_names.role_group_config_map().to_string(), ..ConfigMapVolumeSource::default() }), ..Volume::default() @@ -1076,7 +1028,7 @@ async fn build_node_rolegroup_statefulset( nifi, KEYSTORE_VOLUME_NAME, [ - rolegroup_ref.rolegroup_metrics_service_name(), + crate::service::metrics_service_name(cluster, role_group_name), build_reporting_task_service_name(&nifi_cluster_name), ], SecretFormat::TlsPkcs12, @@ -1133,31 +1085,20 @@ async fn build_node_rolegroup_statefulset( Ok(StatefulSet { metadata: ObjectMetaBuilder::new() - .name_and_namespace(nifi) - .name(rolegroup_ref.object_name()) - .ownerreference_from_resource(nifi, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&recommended_object_labels) - .context(MetadataBuildSnafu)? + .name_and_namespace(cluster) + .name(resource_names.stateful_set_name().to_string()) + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(recommended_object_labels) .with_label(RESTART_CONTROLLER_ENABLED_LABEL.to_owned()) .build(), spec: Some(StatefulSetSpec { pod_management_policy: Some("Parallel".to_string()), replicas, selector: LabelSelector { - match_labels: Some( - Labels::role_group_selector( - nifi, - APP_NAME, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - ) - .context(LabelBuildSnafu)? - .into(), - ), + match_labels: Some(cluster.role_group_selector(role_group_name).into()), ..LabelSelector::default() }, - service_name: Some(rolegroup_ref.rolegroup_headless_service_name()), + service_name: Some(resource_names.headless_service_name().to_string()), template: pod_template, update_strategy: Some(StatefulSetUpdateStrategy { type_: if rolling_update_supported { @@ -1169,7 +1110,8 @@ async fn build_node_rolegroup_statefulset( }), volume_claim_templates: Some(get_volume_claim_templates( nifi, - rolegroup_ref, + cluster, + role_group_name, merged_config, authorization_config, )?), @@ -1181,7 +1123,8 @@ async fn build_node_rolegroup_statefulset( fn get_volume_claim_templates( nifi: &v1alpha1::NifiCluster, - rolegroup_ref: &RoleGroupRef, + cluster: &ValidatedCluster, + role_group_name: &RoleGroupName, merged_config: &NifiConfig, authorization_config: &ResolvedNifiAuthorizationConfig, ) -> Result> { @@ -1208,22 +1151,16 @@ fn get_volume_claim_templates( ), ]; - // Used for PVC templates that cannot be modified once they are deployed - let unversioned_recommended_labels = Labels::recommended(&build_recommended_labels( - nifi, - // A version value is required, and we do want to use the "recommended" format for the other desired labels - "none", - &rolegroup_ref.role, - &rolegroup_ref.role_group, - )) - .context(LabelBuildSnafu)?; + // Used for PVC templates that cannot be modified once they are deployed, so the version label + // is set to the placeholder `none` to keep the labels stable across version upgrades. + let unversioned_recommended_labels = cluster.recommended_labels_unversioned(role_group_name); // listener endpoints will use persistent volumes // so that load balancers can hard-code the target addresses and // that it is possible to connect to a consistent address pvcs.push( build_group_listener_pvc( - &group_listener_name(nifi, &rolegroup_ref.role), + &group_listener_name(nifi, &NifiRole::Node.to_string()), &unversioned_recommended_labels, ) .context(ListenerConfigurationSnafu)?, @@ -1255,20 +1192,3 @@ pub fn error_policy( _ => Action::requeue(*Duration::from_secs(10)), } } - -pub fn build_recommended_labels<'a, T>( - owner: &'a T, - app_version: &'a str, - role: &'a str, - role_group: &'a str, -) -> ObjectLabels<'a, T> { - ObjectLabels { - owner, - app_name: APP_NAME, - app_version, - operator_name: OPERATOR_NAME, - controller_name: NIFI_CONTROLLER_NAME, - role, - role_group, - } -} diff --git a/rust/operator-binary/src/reporting_task/mod.rs b/rust/operator-binary/src/reporting_task/mod.rs index f6c7a09e..d7a77162 100644 --- a/rust/operator-binary/src/reporting_task/mod.rs +++ b/rust/operator-binary/src/reporting_task/mod.rs @@ -22,7 +22,7 @@ //! Therefore, since the support of NiFi 1.25.0, an additional service for the Reporting Task Job containing a //! random but deterministic NiFi node to ensure the communication with a single node. //! -use std::collections::BTreeMap; +use std::{collections::BTreeMap, str::FromStr as _}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ @@ -43,15 +43,17 @@ use stackable_operator::{ }, }, kube::ResourceExt, - kvp::Labels, shared::time::Duration, utils::cluster_info::KubernetesClusterInfo, - v2::types::kubernetes::NamespaceName, + v2::{ + builder::meta::ownerreference_from_resource, + types::{kubernetes::NamespaceName, operator::RoleGroupName}, + }, }; use crate::{ - crd::{APP_NAME, HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT, NifiRole, v1alpha1}, - nifi_controller::build_recommended_labels, + controller::ValidatedCluster, + crd::{HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT, NifiRole, v1alpha1}, security::{ authentication::{NifiAuthenticationConfig, STACKABLE_ADMIN_USERNAME}, build_tls_volume, @@ -67,32 +69,17 @@ pub enum Error { #[snafu(display("object defines no name"))] ObjectHasNoName, - #[snafu(display("failed to build metadata"))] - MetadataBuild { - source: stackable_operator::builder::meta::Error, - }, - #[snafu(display("illegal container name: [{container_name}]"))] IllegalContainerName { source: stackable_operator::builder::pod::container::Error, container_name: String, }, - #[snafu(display("object is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, - }, - #[snafu(display("failed to add Authentication Volumes and VolumeMounts"))] AddAuthVolumes { source: crate::security::authentication::Error, }, - #[snafu(display("failed to build labels"))] - LabelBuild { - source: stackable_operator::kvp::LabelError, - }, - #[snafu(display("failed to build secret volume"))] SecretVolumeBuildFailure { source: crate::security::Error }, @@ -124,8 +111,10 @@ type Result = std::result::Result; /// /// NiFi 2.x and above automatically server Prometheus metrics via the API, but as of 2024-11-08 /// requires authentication. +#[allow(clippy::too_many_arguments)] pub fn build_maybe_reporting_task( nifi: &v1alpha1::NifiCluster, + cluster: &ValidatedCluster, resolved_product_image: &ResolvedProductImage, cluster_info: &KubernetesClusterInfo, namespace: &NamespaceName, @@ -136,19 +125,26 @@ pub fn build_maybe_reporting_task( Ok(Some(( build_reporting_task_job( nifi, + cluster, resolved_product_image, cluster_info, namespace, authentication_config, sa_name, )?, - build_reporting_task_service(nifi, resolved_product_image)?, + build_reporting_task_service(nifi, cluster)?, ))) } else { Ok(None) } } +/// The placeholder role-group name (`global`) used for the labels of the cluster-global reporting +/// task resources, which are not tied to a specific role group. +fn reporting_task_role_group() -> RoleGroupName { + RoleGroupName::from_str("global").expect("'global' is a valid role-group name") +} + /// Return the name of the reporting task Service. pub fn build_reporting_task_service_name(nifi_cluster_name: &str) -> String { format!("{nifi_cluster_name}-{REPORTING_TASK_CONTAINER_NAME}") @@ -205,13 +201,10 @@ fn get_reporting_task_service_selector_pod(nifi: &v1alpha1::NifiCluster) -> Resu /// Build the internal Reporting Task Service in order to communicate with a single NiFi node. fn build_reporting_task_service( nifi: &v1alpha1::NifiCluster, - resolved_product_image: &ResolvedProductImage, + cluster: &ValidatedCluster, ) -> Result { let nifi_cluster_name = nifi.name_any(); - let role_name = NifiRole::Node.to_string(); - let mut selector: BTreeMap = Labels::role_selector(nifi, APP_NAME, &role_name) - .context(LabelBuildSnafu)? - .into(); + let mut selector: BTreeMap = cluster.role_selector().into(); let service_selector_pod = get_reporting_task_service_selector_pod(nifi)?; selector.insert( @@ -221,17 +214,10 @@ fn build_reporting_task_service( Ok(Service { metadata: ObjectMetaBuilder::new() - .name_and_namespace(nifi) + .name_and_namespace(cluster) .name(build_reporting_task_service_name(&nifi_cluster_name)) - .ownerreference_from_resource(nifi, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&build_recommended_labels( - nifi, - &resolved_product_image.app_version_label_value, - &role_name, - "global", - )) - .context(MetadataBuildSnafu)? + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(cluster.recommended_labels(&reporting_task_role_group())) .build(), spec: Some(ServiceSpec { ports: Some(vec![ServicePort { @@ -261,8 +247,10 @@ fn build_reporting_task_service( /// as well as a public certificate provided by the Stackable /// [`secret-operator`](https://github.com/stackabletech/secret-operator) /// +#[allow(clippy::too_many_arguments)] fn build_reporting_task_job( nifi: &v1alpha1::NifiCluster, + cluster: &ValidatedCluster, resolved_product_image: &ResolvedProductImage, cluster_info: &KubernetesClusterInfo, namespace: &NamespaceName, @@ -333,10 +321,9 @@ fn build_reporting_task_job( let mut pod_template = pb .metadata( ObjectMetaBuilder::new() - .name_and_namespace(nifi) + .name_and_namespace(cluster) .name(job_name.clone()) - .ownerreference_from_resource(nifi, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) .build(), ) .image_pull_secrets_from_product_image(resolved_product_image) @@ -372,17 +359,10 @@ fn build_reporting_task_job( let job = Job { metadata: ObjectMetaBuilder::new() - .name_and_namespace(nifi) + .name_and_namespace(cluster) .name(job_name) - .ownerreference_from_resource(nifi, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&build_recommended_labels( - nifi, - &resolved_product_image.app_version_label_value, - "global", - "global", - )) - .context(MetadataBuildSnafu)? + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(cluster.recommended_labels(&reporting_task_role_group())) .build(), spec: Some(JobSpec { backoff_limit: Some(100), diff --git a/rust/operator-binary/src/service.rs b/rust/operator-binary/src/service.rs index 5f43d3ce..b74cde64 100644 --- a/rust/operator-binary/src/service.rs +++ b/rust/operator-binary/src/service.rs @@ -1,77 +1,58 @@ -use std::collections::BTreeMap; - -use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::meta::ObjectMetaBuilder, k8s_openapi::api::core::v1::{Service, ServicePort, ServiceSpec}, - kvp::{Annotations, Labels, ObjectLabels}, - role_utils::RoleGroupRef, + kvp::{Annotations, Labels}, + v2::{builder::meta::ownerreference_from_resource, types::operator::RoleGroupName}, }; use crate::{ controller::ValidatedCluster, - crd::{HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT, METRICS_PORT_NAME, v1alpha1}, + crd::{HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT, METRICS_PORT_NAME}, }; -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("object is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, - }, - - #[snafu(display("failed to build Metadata"))] - MetadataBuild { - source: stackable_operator::builder::meta::Error, - }, -} - /// The rolegroup headless [`Service`] is a service that allows direct access to the instances of a certain rolegroup /// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. pub fn build_rolegroup_headless_service( cluster: &ValidatedCluster, - role_group_ref: &RoleGroupRef, - object_labels: &ObjectLabels, - selector: BTreeMap, -) -> Result { - Ok(Service { + role_group_name: &RoleGroupName, +) -> Service { + Service { metadata: ObjectMetaBuilder::new() .name_and_namespace(cluster) - .name(role_group_ref.rolegroup_headless_service_name()) - .ownerreference_from_resource(cluster, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(object_labels) - .context(MetadataBuildSnafu)? + .name( + cluster + .resource_names(role_group_name) + .headless_service_name() + .to_string(), + ) + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(cluster.recommended_labels(role_group_name)) .build(), spec: Some(ServiceSpec { // Internal communication does not need to be exposed type_: Some("ClusterIP".to_string()), cluster_ip: Some("None".to_string()), ports: Some(headless_service_ports()), - selector: Some(selector), + selector: Some(cluster.role_group_selector(role_group_name).into()), publish_not_ready_addresses: Some(true), ..ServiceSpec::default() }), status: None, - }) + } } /// The rolegroup metrics [`Service`] is a service that exposes metrics and a prometheus scraping label. pub fn build_rolegroup_metrics_service( cluster: &ValidatedCluster, - role_group_ref: &RoleGroupRef, - object_labels: &ObjectLabels, - selector: BTreeMap, - product_version: &str, -) -> Result { - Ok(Service { + role_group_name: &RoleGroupName, +) -> Service { + let product_version = &cluster.image.product_version; + Service { metadata: ObjectMetaBuilder::new() .name_and_namespace(cluster) - .name(role_group_ref.rolegroup_metrics_service_name()) - .ownerreference_from_resource(cluster, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(object_labels) - .context(MetadataBuildSnafu)? + .name(metrics_service_name(cluster, role_group_name)) + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(cluster.recommended_labels(role_group_name)) .with_labels(prometheus_labels()) .with_annotations(prometheus_annotations(product_version)) .build(), @@ -80,12 +61,27 @@ pub fn build_rolegroup_metrics_service( type_: Some("ClusterIP".to_string()), cluster_ip: Some("None".to_string()), ports: Some(vec![metrics_service_port(product_version)]), - selector: Some(selector), + selector: Some(cluster.role_group_selector(role_group_name).into()), publish_not_ready_addresses: Some(true), ..ServiceSpec::default() }), status: None, - }) + } +} + +/// The name of the rolegroup metrics [`Service`] (`---metrics`). +/// +/// [`ResourceNames`](stackable_operator::v2::role_group_utils::ResourceNames) has no metrics +/// service name, so it is derived from the qualified role-group name (which equals the StatefulSet +/// name) here, preserving the previous `RoleGroupRef::rolegroup_metrics_service_name` output. +pub(crate) fn metrics_service_name( + cluster: &ValidatedCluster, + role_group_name: &RoleGroupName, +) -> String { + format!( + "{qualified}-metrics", + qualified = cluster.resource_names(role_group_name).stateful_set_name() + ) } fn headless_service_ports() -> Vec { From 3f40bae232e632d196075594d7ec5e82ac513e05 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 12 Jun 2026 12:08:24 +0200 Subject: [PATCH 28/39] chore: regenerate --- Cargo.nix | 18 +++++++++--------- crate-hashes.json | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index 85e4e57a..26b6144d 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -5067,7 +5067,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + sha256 = "0s5giiz1zhz644s2fy9bfvd1hybm4smz75ak33fkvh2flvjfqba3"; }; libName = "k8s_version"; authors = [ @@ -9861,7 +9861,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + sha256 = "0s5giiz1zhz644s2fy9bfvd1hybm4smz75ak33fkvh2flvjfqba3"; }; libName = "stackable_certs"; authors = [ @@ -10073,7 +10073,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + sha256 = "0s5giiz1zhz644s2fy9bfvd1hybm4smz75ak33fkvh2flvjfqba3"; }; libName = "stackable_operator"; authors = [ @@ -10267,7 +10267,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + sha256 = "0s5giiz1zhz644s2fy9bfvd1hybm4smz75ak33fkvh2flvjfqba3"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -10302,7 +10302,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + sha256 = "0s5giiz1zhz644s2fy9bfvd1hybm4smz75ak33fkvh2flvjfqba3"; }; libName = "stackable_shared"; authors = [ @@ -10383,7 +10383,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + sha256 = "0s5giiz1zhz644s2fy9bfvd1hybm4smz75ak33fkvh2flvjfqba3"; }; libName = "stackable_telemetry"; authors = [ @@ -10493,7 +10493,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + sha256 = "0s5giiz1zhz644s2fy9bfvd1hybm4smz75ak33fkvh2flvjfqba3"; }; libName = "stackable_versioned"; authors = [ @@ -10543,7 +10543,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + sha256 = "0s5giiz1zhz644s2fy9bfvd1hybm4smz75ak33fkvh2flvjfqba3"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -10611,7 +10611,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + sha256 = "0s5giiz1zhz644s2fy9bfvd1hybm4smz75ak33fkvh2flvjfqba3"; }; libName = "stackable_webhook"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index c9a6e6a9..a658a742 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "0s5giiz1zhz644s2fy9bfvd1hybm4smz75ak33fkvh2flvjfqba3", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "0s5giiz1zhz644s2fy9bfvd1hybm4smz75ak33fkvh2flvjfqba3", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "0s5giiz1zhz644s2fy9bfvd1hybm4smz75ak33fkvh2flvjfqba3", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "0s5giiz1zhz644s2fy9bfvd1hybm4smz75ak33fkvh2flvjfqba3", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "0s5giiz1zhz644s2fy9bfvd1hybm4smz75ak33fkvh2flvjfqba3", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "0s5giiz1zhz644s2fy9bfvd1hybm4smz75ak33fkvh2flvjfqba3", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "0s5giiz1zhz644s2fy9bfvd1hybm4smz75ak33fkvh2flvjfqba3", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "0s5giiz1zhz644s2fy9bfvd1hybm4smz75ak33fkvh2flvjfqba3", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "0s5giiz1zhz644s2fy9bfvd1hybm4smz75ak33fkvh2flvjfqba3", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file From aa21bd7fe3821e6c4cc840a8b957bcdb1d3be655 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 12 Jun 2026 12:49:29 +0200 Subject: [PATCH 29/39] fix(tests): reduce resources and logging rolegroup length (<16) --- tests/templates/kuttl/logging/04-assert.yaml | 4 ++-- .../templates/kuttl/logging/04-install-nifi.yaml.j2 | 4 ++-- .../logging/nifi-vector-aggregator-values.yaml.j2 | 12 ++++++------ tests/templates/kuttl/resources/02-assert.yaml.j2 | 6 +++--- .../kuttl/resources/02-install-nifi.yaml.j2 | 6 +++--- tests/templates/kuttl/resources/03-assert.yaml | 4 ++-- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/templates/kuttl/logging/04-assert.yaml b/tests/templates/kuttl/logging/04-assert.yaml index 00306d69..9a58760b 100644 --- a/tests/templates/kuttl/logging/04-assert.yaml +++ b/tests/templates/kuttl/logging/04-assert.yaml @@ -6,7 +6,7 @@ timeout: 600 apiVersion: apps/v1 kind: StatefulSet metadata: - name: test-nifi-node-automatic-log-config + name: test-nifi-node-automatic-log status: readyReplicas: 1 replicas: 1 @@ -14,7 +14,7 @@ status: apiVersion: apps/v1 kind: StatefulSet metadata: - name: test-nifi-node-custom-log-config + name: test-nifi-node-custom-log status: readyReplicas: 1 replicas: 1 diff --git a/tests/templates/kuttl/logging/04-install-nifi.yaml.j2 b/tests/templates/kuttl/logging/04-install-nifi.yaml.j2 index 3bdb0ab4..2bbfb08e 100644 --- a/tests/templates/kuttl/logging/04-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/logging/04-install-nifi.yaml.j2 @@ -110,7 +110,7 @@ spec: # Quicker startup, and we only have a single node "nifi.cluster.flow.election.max.wait.time": "10 secs" roleGroups: - automatic-log-config: + automatic-log: replicas: 1 config: logging: @@ -151,7 +151,7 @@ spec: - name: prepared-logs configMap: name: prepared-logs - custom-log-config: + custom-log: replicas: 1 config: logging: diff --git a/tests/templates/kuttl/logging/nifi-vector-aggregator-values.yaml.j2 b/tests/templates/kuttl/logging/nifi-vector-aggregator-values.yaml.j2 index 1c4d5f31..e823917e 100644 --- a/tests/templates/kuttl/logging/nifi-vector-aggregator-values.yaml.j2 +++ b/tests/templates/kuttl/logging/nifi-vector-aggregator-values.yaml.j2 @@ -28,37 +28,37 @@ customConfig: type: filter inputs: [validEvents] condition: >- - .pod == "test-nifi-node-automatic-log-config-0" && + .pod == "test-nifi-node-automatic-log-0" && .container == "nifi" filteredAutomaticLogConfigNodeGitSync: type: filter inputs: [validEvents] condition: >- - .pod == "test-nifi-node-automatic-log-config-0" && + .pod == "test-nifi-node-automatic-log-0" && .container == "git-sync-0" filteredAutomaticLogConfigNodeGitSyncInit: type: filter inputs: [validEvents] condition: >- - .pod == "test-nifi-node-automatic-log-config-0" && + .pod == "test-nifi-node-automatic-log-0" && .container == "git-sync-0-init" filteredAutomaticLogConfigNodeVector: type: filter inputs: [validEvents] condition: >- - .pod == "test-nifi-node-automatic-log-config-0" && + .pod == "test-nifi-node-automatic-log-0" && .container == "vector" filteredCustomLogConfigNodeNifi: type: filter inputs: [validEvents] condition: >- - .pod == "test-nifi-node-custom-log-config-0" && + .pod == "test-nifi-node-custom-log-0" && .container == "nifi" filteredCustomLogConfigNodeVector: type: filter inputs: [validEvents] condition: >- - .pod == "test-nifi-node-custom-log-config-0" && + .pod == "test-nifi-node-custom-log-0" && .container == "vector" filteredInvalidEvents: type: filter diff --git a/tests/templates/kuttl/resources/02-assert.yaml.j2 b/tests/templates/kuttl/resources/02-assert.yaml.j2 index 9d882491..21d54fd3 100644 --- a/tests/templates/kuttl/resources/02-assert.yaml.j2 +++ b/tests/templates/kuttl/resources/02-assert.yaml.j2 @@ -7,7 +7,7 @@ timeout: 1800 apiVersion: apps/v1 kind: StatefulSet metadata: - name: test-nifi-node-resources-from-role + name: test-nifi-node-from-role spec: template: spec: @@ -30,7 +30,7 @@ status: apiVersion: apps/v1 kind: StatefulSet metadata: - name: test-nifi-node-resources-from-role-group + name: test-nifi-node-from-role-group spec: template: spec: @@ -53,7 +53,7 @@ status: apiVersion: apps/v1 kind: StatefulSet metadata: - name: test-nifi-node-resources-from-pod-overrides + name: test-nifi-node-pod-overrides spec: template: spec: diff --git a/tests/templates/kuttl/resources/02-install-nifi.yaml.j2 b/tests/templates/kuttl/resources/02-install-nifi.yaml.j2 index 0ed1cc0e..8af93d1d 100644 --- a/tests/templates/kuttl/resources/02-install-nifi.yaml.j2 +++ b/tests/templates/kuttl/resources/02-install-nifi.yaml.j2 @@ -73,9 +73,9 @@ spec: # Quicker startup, and we only have a single node "nifi.cluster.flow.election.max.wait.time": "10 secs" roleGroups: - resources-from-role: + from-role: replicas: 1 - resources-from-role-group: + from-role-group: config: resources: cpu: @@ -84,7 +84,7 @@ spec: memory: limit: 3Gi replicas: 1 - resources-from-pod-overrides: + pod-overrides: podOverrides: spec: containers: diff --git a/tests/templates/kuttl/resources/03-assert.yaml b/tests/templates/kuttl/resources/03-assert.yaml index 7a06b755..a89bfcd3 100644 --- a/tests/templates/kuttl/resources/03-assert.yaml +++ b/tests/templates/kuttl/resources/03-assert.yaml @@ -5,7 +5,7 @@ metadata: name: check-jvm-heap-args timeout: 600 commands: - - script: kubectl get cm -n $NAMESPACE test-nifi-node-resources-from-role -o yaml | grep -E 'java.arg..=-Xmx1638m' | xargs test ! -z + - script: kubectl get cm -n $NAMESPACE test-nifi-node-from-role -o yaml | grep -E 'java.arg..=-Xmx1638m' | xargs test ! -z --- apiVersion: kuttl.dev/v1beta1 kind: TestAssert @@ -13,4 +13,4 @@ metadata: name: check-jvm-heap-args timeout: 600 commands: - - script: kubectl get cm -n $NAMESPACE test-nifi-node-resources-from-role-group -o yaml | grep -E 'java.arg..=-Xms2457m' | xargs test ! -z + - script: kubectl get cm -n $NAMESPACE test-nifi-node-from-role-group -o yaml | grep -E 'java.arg..=-Xms2457m' | xargs test ! -z From 976e914b1d8402255d387f4ff05e75a9a4de40d6 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 13 Jun 2026 21:01:03 +0200 Subject: [PATCH 30/39] refactor: move k8s resources to build/resource --- rust/operator-binary/src/controller/build.rs | 2 + .../build}/graceful_shutdown.rs | 0 .../build/properties/bootstrap_conf.rs | 6 +- .../src/controller/build/proxy_hosts.rs | 5 +- .../build/resource}/listener.rs | 0 .../src/controller/build/resource/mod.rs | 9 + .../build/resource}/pdb.rs | 0 .../build/resource/reporting_task.rs} | 0 .../build/resource}/service.rs | 0 .../controller/build/resource/statefulset.rs | 757 +++++++++++++++++ rust/operator-binary/src/controller/mod.rs | 1 + .../src/{operations => controller}/upgrade.rs | 0 rust/operator-binary/src/main.rs | 4 - rust/operator-binary/src/nifi_controller.rs | 759 +----------------- rust/operator-binary/src/operations/mod.rs | 3 - 15 files changed, 809 insertions(+), 737 deletions(-) rename rust/operator-binary/src/{operations => controller/build}/graceful_shutdown.rs (100%) rename rust/operator-binary/src/{ => controller/build/resource}/listener.rs (100%) create mode 100644 rust/operator-binary/src/controller/build/resource/mod.rs rename rust/operator-binary/src/{operations => controller/build/resource}/pdb.rs (100%) rename rust/operator-binary/src/{reporting_task/mod.rs => controller/build/resource/reporting_task.rs} (100%) rename rust/operator-binary/src/{ => controller/build/resource}/service.rs (100%) create mode 100644 rust/operator-binary/src/controller/build/resource/statefulset.rs rename rust/operator-binary/src/{operations => controller}/upgrade.rs (100%) delete mode 100644 rust/operator-binary/src/operations/mod.rs diff --git a/rust/operator-binary/src/controller/build.rs b/rust/operator-binary/src/controller/build.rs index 71b9885c..10e04bb8 100644 --- a/rust/operator-binary/src/controller/build.rs +++ b/rust/operator-binary/src/controller/build.rs @@ -8,9 +8,11 @@ use crate::{crd::storage::NifiRepository, security::oidc}; pub mod config_map; pub mod git_sync; +pub mod graceful_shutdown; pub mod jvm; pub mod properties; pub mod proxy_hosts; +pub mod resource; /// Errors that can occur while building the NiFi product configuration files. #[derive(Snafu, Debug)] diff --git a/rust/operator-binary/src/operations/graceful_shutdown.rs b/rust/operator-binary/src/controller/build/graceful_shutdown.rs similarity index 100% rename from rust/operator-binary/src/operations/graceful_shutdown.rs rename to rust/operator-binary/src/controller/build/graceful_shutdown.rs diff --git a/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs b/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs index 5cb17d99..841ab67e 100644 --- a/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs +++ b/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs @@ -7,9 +7,11 @@ use snafu::ResultExt; use crate::{ controller::{ ValidatedRoleGroupConfig, - build::{Error, InvalidJVMConfigSnafu, jvm::build_merged_jvm_config}, + build::{ + Error, InvalidJVMConfigSnafu, graceful_shutdown::graceful_shutdown_config_properties, + jvm::build_merged_jvm_config, + }, }, - operations::graceful_shutdown::graceful_shutdown_config_properties, security::authorization::ResolvedNifiAuthorizationConfig, }; diff --git a/rust/operator-binary/src/controller/build/proxy_hosts.rs b/rust/operator-binary/src/controller/build/proxy_hosts.rs index 7f215683..1efb82e9 100644 --- a/rust/operator-binary/src/controller/build/proxy_hosts.rs +++ b/rust/operator-binary/src/controller/build/proxy_hosts.rs @@ -4,7 +4,10 @@ use std::collections::HashSet; use stackable_operator::utils::cluster_info::KubernetesClusterInfo; -use crate::{controller::ValidatedCluster, crd::HTTPS_PORT, reporting_task}; +use crate::{ + controller::{ValidatedCluster, build::resource::reporting_task}, + crd::HTTPS_PORT, +}; /// Computes the comma-separated NiFi proxy hosts, or `"*"` if `hostHeaderCheck.allowAll` is set. /// diff --git a/rust/operator-binary/src/listener.rs b/rust/operator-binary/src/controller/build/resource/listener.rs similarity index 100% rename from rust/operator-binary/src/listener.rs rename to rust/operator-binary/src/controller/build/resource/listener.rs diff --git a/rust/operator-binary/src/controller/build/resource/mod.rs b/rust/operator-binary/src/controller/build/resource/mod.rs new file mode 100644 index 00000000..619ebcaa --- /dev/null +++ b/rust/operator-binary/src/controller/build/resource/mod.rs @@ -0,0 +1,9 @@ +//! Builders that assemble individual Kubernetes resources from a [`ValidatedCluster`]. +//! +//! [`ValidatedCluster`]: crate::controller::ValidatedCluster + +pub mod listener; +pub mod pdb; +pub mod reporting_task; +pub mod service; +pub mod statefulset; diff --git a/rust/operator-binary/src/operations/pdb.rs b/rust/operator-binary/src/controller/build/resource/pdb.rs similarity index 100% rename from rust/operator-binary/src/operations/pdb.rs rename to rust/operator-binary/src/controller/build/resource/pdb.rs diff --git a/rust/operator-binary/src/reporting_task/mod.rs b/rust/operator-binary/src/controller/build/resource/reporting_task.rs similarity index 100% rename from rust/operator-binary/src/reporting_task/mod.rs rename to rust/operator-binary/src/controller/build/resource/reporting_task.rs diff --git a/rust/operator-binary/src/service.rs b/rust/operator-binary/src/controller/build/resource/service.rs similarity index 100% rename from rust/operator-binary/src/service.rs rename to rust/operator-binary/src/controller/build/resource/service.rs diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs new file mode 100644 index 00000000..54b3fd36 --- /dev/null +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -0,0 +1,757 @@ +//! Builds the rolegroup [`StatefulSet`] that runs a NiFi node role group. + +use std::collections::BTreeMap; + +use indoc::formatdoc; +use snafu::{OptionExt, ResultExt, Snafu}; +use stackable_operator::{ + builder::{ + self, + meta::ObjectMetaBuilder, + pod::{ + PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, + security::PodSecurityContextBuilder, volume::SecretFormat, + }, + }, + commons::product_image_selection::ResolvedProductImage, + constants::RESTART_CONTROLLER_ENABLED_LABEL, + crd::{authentication::oidc::v1alpha1::AuthenticationProvider, git_sync}, + k8s_openapi::{ + DeepMerge, + api::{ + apps::v1::{StatefulSet, StatefulSetSpec, StatefulSetUpdateStrategy}, + core::v1::{ + ConfigMapKeySelector, ConfigMapVolumeSource, EmptyDirVolumeSource, EnvVar, + EnvVarSource, ObjectFieldSelector, PersistentVolumeClaim, Probe, + SecretVolumeSource, TCPSocketAction, Volume, + }, + }, + apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, + }, + kube::ResourceExt, + memory::{BinaryMultiple, MemoryQuantity}, + product_logging::{ + self, + framework::{ + LoggingError, create_vector_shutdown_file_command, remove_vector_shutdown_file_command, + }, + spec::{ + ConfigMapLogConfig, ContainerLogConfig, ContainerLogConfigChoice, + CustomContainerLogConfig, + }, + }, + utils::{COMMON_BASH_TRAP_FUNCTIONS, cluster_info::KubernetesClusterInfo}, + v2::{builder::meta::ownerreference_from_resource, types::operator::RoleGroupName}, +}; + +use crate::{ + controller::{ + ValidatedCluster, ValidatedRoleGroupConfig, + build::{ + graceful_shutdown::add_graceful_shutdown_config, + resource::{ + listener::{ + LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_group_listener_pvc, + group_listener_name, + }, + reporting_task::build_reporting_task_service_name, + }, + }, + }, + crd::{ + BALANCE_PORT, BALANCE_PORT_NAME, Container, HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT, + METRICS_PORT_NAME, NifiConfig, NifiRole, NifiRoleType, PROTOCOL_PORT, PROTOCOL_PORT_NAME, + STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, + authorization::NifiAccessPolicyProvider, + constants::{NIFI_CONFIG_DIRECTORY, NIFI_PYTHON_WORKING_DIRECTORY}, + storage::{NifiRepository, PERSISTENT_REPOSITORIES}, + v1alpha1, + }, + nifi_controller::LOG_VOLUME_NAME, + security::{ + authentication::{ + NifiAuthenticationConfig, STACKABLE_SERVER_TLS_DIR, STACKABLE_TLS_STORE_PASSWORD, + }, + authorization::{self, OPA_TLS_MOUNT_PATH, ResolvedNifiAuthorizationConfig}, + build_tls_volume, + tls::{KEYSTORE_NIFI_CONTAINER_MOUNT, KEYSTORE_VOLUME_NAME, TRUSTSTORE_VOLUME_NAME}, + }, +}; + +/// Errors that can occur while building the rolegroup [`StatefulSet`]. +#[derive(Snafu, Debug)] +#[snafu(visibility(pub(crate)))] +pub enum Error { + #[snafu(display("missing secret lifetime"))] + MissingSecretLifetime, + + #[snafu(display("object defines no name"))] + ObjectHasNoName, + + #[snafu(display("illegal container name: [{container_name}]"))] + IllegalContainerName { + source: stackable_operator::builder::pod::container::Error, + container_name: String, + }, + + #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] + VectorAggregatorConfigMapMissing, + + #[snafu(display("failed to add Authentication Volumes and VolumeMounts"))] + AddAuthVolumes { + source: crate::security::authentication::Error, + }, + + #[snafu(display("security failure"))] + Security { source: crate::security::Error }, + + #[snafu(display("failed to configure logging"))] + ConfigureLogging { source: LoggingError }, + + #[snafu(display("failed to add needed volume"))] + AddVolume { source: builder::pod::Error }, + + #[snafu(display("failed to add needed volumeMount"))] + AddVolumeMount { + source: builder::pod::container::Error, + }, + + #[snafu(display("failed to configure graceful shutdown"))] + GracefulShutdown { + source: crate::controller::build::graceful_shutdown::Error, + }, + + #[snafu(display("failed to configure listener"))] + ListenerConfiguration { + source: crate::controller::build::resource::listener::Error, + }, + + #[snafu(display("failed to build authorization configuration"))] + AuthorizationConfiguration { source: authorization::Error }, +} + +type Result = std::result::Result; + +const USERDATA_MOUNTPOINT: &str = "/stackable/userdata"; + +/// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. +/// +/// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the +/// corresponding [`stackable_operator::k8s_openapi::api::core::v1::Service`] (from [`build_rolegroup_headless_service`]). +#[allow(clippy::too_many_arguments)] +pub(crate) async fn build_node_rolegroup_statefulset( + nifi: &v1alpha1::NifiCluster, + cluster: &ValidatedCluster, + resolved_product_image: &ResolvedProductImage, + cluster_info: &KubernetesClusterInfo, + role_group_name: &RoleGroupName, + role: &NifiRoleType, + rg: &ValidatedRoleGroupConfig, + authentication_config: &NifiAuthenticationConfig, + authorization_config: &ResolvedNifiAuthorizationConfig, + rolling_update_supported: bool, + replicas: Option, + service_account_name: &str, + git_sync_resources: &git_sync::v1alpha2::GitSyncResources, +) -> Result { + tracing::debug!("Building statefulset"); + + // Type-safe names for this role group's resources (StatefulSet, ConfigMap, headless Service). + let resource_names = cluster.resource_names(role_group_name); + + // The validated, merged `NifiConfig` is the single source of truth; the ConfigMap builder + // sources the same `rg.config`. + let merged_config = &rg.config; + + let mut env_vars: Vec = rg.env_overrides.clone().into(); + + // we need the POD_NAME env var to overwrite `nifi.cluster.node.address` later + env_vars.push(EnvVar { + name: "POD_NAME".to_string(), + value_from: Some(EnvVarSource { + field_ref: Some(ObjectFieldSelector { + api_version: Some("v1".to_string()), + field_path: "metadata.name".to_string(), + }), + ..EnvVarSource::default() + }), + ..EnvVar::default() + }); + + // Needed for the `containerdebug` process to log it's tracing information to. + env_vars.push(EnvVar { + name: "CONTAINERDEBUG_LOG_DIRECTORY".to_string(), + value: Some(format!("{STACKABLE_LOG_DIR}/containerdebug")), + ..Default::default() + }); + + env_vars.push(EnvVar { + name: "STACKLET_NAME".to_string(), + value: Some(nifi.name_unchecked().to_string()), + ..Default::default() + }); + + match &nifi.spec.cluster_config.clustering_backend { + v1alpha1::NifiClusteringBackend::ZooKeeper { + zookeeper_config_map_name, + } => { + let zookeeper_env_var = |name: &str| EnvVar { + name: name.to_string(), + value_from: Some(EnvVarSource { + config_map_key_ref: Some(ConfigMapKeySelector { + name: zookeeper_config_map_name.to_string(), + key: name.to_string(), + ..ConfigMapKeySelector::default() + }), + ..EnvVarSource::default() + }), + ..EnvVar::default() + }; + env_vars.push(zookeeper_env_var("ZOOKEEPER_HOSTS")); + env_vars.push(zookeeper_env_var("ZOOKEEPER_CHROOT")); + } + v1alpha1::NifiClusteringBackend::Kubernetes {} => {} + } + + if let NifiAuthenticationConfig::Oidc { oidc, .. } = authentication_config { + env_vars.extend(AuthenticationProvider::client_credentials_env_var_mounts( + oidc.client_credentials_secret_ref.clone(), + )); + } + + env_vars.extend(authorization_config.get_env_vars()); + + let node_address = format!( + "$POD_NAME.{service_name}.{namespace}.svc.{cluster_domain}", + service_name = resource_names.headless_service_name(), + namespace = cluster.namespace, + cluster_domain = cluster_info.cluster_domain, + ); + + let sensitive_key_secret = &nifi.spec.cluster_config.sensitive_properties.key_secret; + + let prepare_container_name = Container::Prepare.to_string(); + let mut prepare_args = vec![]; + + if let Some(ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Automatic(log_config)), + }) = merged_config.logging.containers.get(&Container::Prepare) + { + prepare_args.push(product_logging::framework::capture_shell_output( + STACKABLE_LOG_DIR, + &prepare_container_name, + log_config, + )); + } + + // Note(sbernauer): In https://github.com/stackabletech/issues/issues/764 we migrated all usages + // of keytool to our own cert-utils tool. As it uses the same code as secret-operator, it also + // uses RC2. Thus, the keytool usage here LGTM (no alias trickery) and has my nod of approval. + prepare_args.extend(vec![ + // The source directory is a secret-op mount and we do not want to write / add anything in there + // Therefore we import all the contents to a truststore in "writable" empty dirs. + // Keytool is only barking if a password is not set for the destination truststore (which we set) + // and do provide an empty password for the source truststore coming from the secret-operator. + // Using no password will result in a warning. + format!("echo Importing {KEYSTORE_NIFI_CONTAINER_MOUNT}/keystore.p12 to {STACKABLE_SERVER_TLS_DIR}/keystore.p12"), + format!("cp {KEYSTORE_NIFI_CONTAINER_MOUNT}/keystore.p12 {STACKABLE_SERVER_TLS_DIR}/keystore.p12"), + format!("echo Importing {KEYSTORE_NIFI_CONTAINER_MOUNT}/truststore.p12 to {STACKABLE_SERVER_TLS_DIR}/truststore.p12"), + // secret-operator currently encrypts keystores with RC2, which NiFi is unable to read: https://github.com/stackabletech/nifi-operator/pull/510 + // As a workaround, reencrypt the keystore with keytool. + // keytool crashes if the target truststore already exists (covering up the true error + // if the init container fails later on in the script), so delete it first. + format!("test ! -e {STACKABLE_SERVER_TLS_DIR}/truststore.p12 || rm {STACKABLE_SERVER_TLS_DIR}/truststore.p12"), + format!("keytool -importkeystore -srckeystore {KEYSTORE_NIFI_CONTAINER_MOUNT}/truststore.p12 -destkeystore {STACKABLE_SERVER_TLS_DIR}/truststore.p12 -srcstorepass {STACKABLE_TLS_STORE_PASSWORD} -deststorepass {STACKABLE_TLS_STORE_PASSWORD}"), + + "echo Replacing config directory".to_string(), + "cp /conf/* /stackable/nifi/conf".to_string(), + "test -L /stackable/nifi/conf/logback.xml || ln -sf /stackable/log_config/logback.xml /stackable/nifi/conf/logback.xml".to_string(), + format!(r#"export NODE_ADDRESS="{node_address}""#), + ]); + + // This commands needs to go first, as they might set env variables needed by the templating + prepare_args.extend_from_slice( + authentication_config + .get_additional_container_args() + .as_slice(), + ); + + // Add OPA certificate to truststore if OPA TLS is enabled + if authorization_config.has_opa_tls() { + prepare_args.extend(vec![ + "echo Importing OPA CA certificate to truststore".to_string(), + format!("keytool -importcert -file {OPA_TLS_MOUNT_PATH}/ca.crt -keystore {STACKABLE_SERVER_TLS_DIR}/truststore.p12 -storepass {STACKABLE_TLS_STORE_PASSWORD} -alias opa-ca -noprompt"), + ]); + } + + prepare_args.extend(vec![ + "export LISTENER_DEFAULT_ADDRESS=$(cat /stackable/listener/default-address/address)" + .to_string(), + ]); + prepare_args.extend(vec![ + "export LISTENER_DEFAULT_PORT_HTTPS=$(cat /stackable/listener/default-address/ports/https)" + .to_string(), + ]); + + prepare_args.extend(vec![ + "echo Templating config files".to_string(), + "config-utils template /stackable/nifi/conf/nifi.properties".to_string(), + "config-utils template /stackable/nifi/conf/state-management.xml".to_string(), + "config-utils template /stackable/nifi/conf/login-identity-providers.xml".to_string(), + "config-utils template /stackable/nifi/conf/authorizers.xml".to_string(), + "config-utils template /stackable/nifi/conf/security.properties".to_string(), + ]); + + let mut container_prepare = + ContainerBuilder::new(&prepare_container_name).with_context(|_| { + IllegalContainerNameSnafu { + container_name: prepare_container_name.to_string(), + } + })?; + + container_prepare + .image_from_product_image(resolved_product_image) + .command(vec![ + "/bin/bash".to_string(), + "-c".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + ]) + .add_env_vars(env_vars.clone()) + .args(vec![prepare_args.join(" && ")]) + .add_volume_mounts( + PERSISTENT_REPOSITORIES + .iter() + .map(NifiRepository::volume_mount), + ) + .context(AddVolumeMountSnafu)? + .add_volume_mount("conf", "/conf") + .context(AddVolumeMountSnafu)? + .add_volume_mount(KEYSTORE_VOLUME_NAME, KEYSTORE_NIFI_CONTAINER_MOUNT) + .context(AddVolumeMountSnafu)? + .add_volume_mount("activeconf", NIFI_CONFIG_DIRECTORY) + .context(AddVolumeMountSnafu)? + .add_volume_mount("sensitiveproperty", "/stackable/sensitiveproperty") + .context(AddVolumeMountSnafu)? + .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) + .context(AddVolumeMountSnafu)? + .add_volume_mount(TRUSTSTORE_VOLUME_NAME, STACKABLE_SERVER_TLS_DIR) + .context(AddVolumeMountSnafu)? + .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) + .context(AddVolumeMountSnafu)? + .add_volume_mounts(authorization_config.get_volume_mounts()) + .context(AddVolumeMountSnafu)? + .resources( + ResourceRequirementsBuilder::new() + .with_cpu_request("500m") + .with_cpu_limit("2000m") + .with_memory_request("4096Mi") + .with_memory_limit("4096Mi") + .build(), + ); + + let nifi_container_name = Container::Nifi.to_string(); + let mut container_nifi_builder = + ContainerBuilder::new(&nifi_container_name).with_context(|_| { + IllegalContainerNameSnafu { + container_name: nifi_container_name, + } + })?; + + let nifi_args = vec![formatdoc! {" + {COMMON_BASH_TRAP_FUNCTIONS} + {remove_vector_shutdown_file_command} + prepare_signal_handlers + containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & + bin/nifi.sh run & + wait_for_termination $! + {create_vector_shutdown_file_command} + ", + remove_vector_shutdown_file_command = + remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), + create_vector_shutdown_file_command = + create_vector_shutdown_file_command(STACKABLE_LOG_DIR), + }]; + + let container_nifi = container_nifi_builder + .image_from_product_image(resolved_product_image) + .command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]) + .args(nifi_args) + .add_env_vars(env_vars) + .add_volume_mount(KEYSTORE_VOLUME_NAME, KEYSTORE_NIFI_CONTAINER_MOUNT) + .context(AddVolumeMountSnafu)? + .add_volume_mounts( + PERSISTENT_REPOSITORIES + .iter() + .map(NifiRepository::volume_mount), + ) + .context(AddVolumeMountSnafu)? + .add_volume_mount("activeconf", NIFI_CONFIG_DIRECTORY) + .context(AddVolumeMountSnafu)? + .add_volume_mount("log-config", STACKABLE_LOG_CONFIG_DIR) + .context(AddVolumeMountSnafu)? + .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) + .context(AddVolumeMountSnafu)? + .add_volume_mount(TRUSTSTORE_VOLUME_NAME, STACKABLE_SERVER_TLS_DIR) + .context(AddVolumeMountSnafu)? + .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) + .context(AddVolumeMountSnafu)? + .add_volume_mounts(authorization_config.get_volume_mounts()) + .context(AddVolumeMountSnafu)? + .add_container_port(HTTPS_PORT_NAME, HTTPS_PORT.into()) + .add_container_port(PROTOCOL_PORT_NAME, PROTOCOL_PORT.into()) + .add_container_port(BALANCE_PORT_NAME, BALANCE_PORT.into()) + .liveness_probe(Probe { + initial_delay_seconds: Some(10), + period_seconds: Some(10), + tcp_socket: Some(TCPSocketAction { + port: IntOrString::String(HTTPS_PORT_NAME.to_string()), + ..TCPSocketAction::default() + }), + ..Probe::default() + }) + .startup_probe(Probe { + initial_delay_seconds: Some(10), + period_seconds: Some(10), + failure_threshold: Some(20 * 6), + tcp_socket: Some(TCPSocketAction { + port: IntOrString::String(HTTPS_PORT_NAME.to_string()), + ..TCPSocketAction::default() + }), + ..Probe::default() + }) + .resources(merged_config.resources.clone().into()); + + // NiFi 2.x.x offers nifi-api/flow/metrics/prometheus at the HTTPS_PORT, therefore METRICS_PORT is only required for NiFi 1.x.x. + if resolved_product_image.product_version.starts_with("1.") { + container_nifi.add_container_port(METRICS_PORT_NAME, METRICS_PORT.into()); + } + + let mut pod_builder = PodBuilder::new(); + + let recommended_object_labels = cluster.recommended_labels(role_group_name); + + add_graceful_shutdown_config(merged_config, &mut pod_builder).context(GracefulShutdownSnafu)?; + + // Add user configured extra volumes if any are specified + for volume in &nifi.spec.cluster_config.extra_volumes { + // Extract values into vars so we make it impossible to log something other than + // what we actually use to create the mounts - maybe paranoid, but hey .. + let volume_name = &volume.name; + let mount_point = format!("{USERDATA_MOUNTPOINT}/{}", volume.name); + + tracing::info!( + ?volume_name, + ?mount_point, + ?role, + "Adding user specified extra volume", + ); + pod_builder + .add_volume(volume.clone()) + .context(AddVolumeSnafu)?; + container_nifi + .add_volume_mount(volume_name, mount_point) + .context(AddVolumeMountSnafu)?; + } + + let volume_name = "nifi-python-working-directory".to_string(); + pod_builder + .add_empty_dir_volume(&volume_name, None) + .context(AddVolumeSnafu)?; + container_nifi + .add_volume_mount(&volume_name, NIFI_PYTHON_WORKING_DIRECTORY) + .context(AddVolumeMountSnafu)?; + + container_nifi + .add_volume_mounts(git_sync_resources.git_content_volume_mounts.to_owned()) + .context(AddVolumeMountSnafu)?; + + // We want to add nifi container first for easier defaulting into this container + pod_builder.add_container(container_nifi.build()); + // After calling `build()` the ContainerBuilder shouldn't be used anymore, so we drop it + drop(container_nifi_builder); + + for container in git_sync_resources.git_sync_containers.iter().cloned() { + pod_builder.add_container(container); + } + for container in git_sync_resources.git_sync_init_containers.iter().cloned() { + pod_builder.add_init_container(container); + } + pod_builder + .add_volumes(git_sync_resources.git_content_volumes.to_owned()) + .context(AddVolumeSnafu)?; + pod_builder + .add_volumes(git_sync_resources.git_ca_cert_volumes.to_owned()) + .context(AddVolumeSnafu)?; + + if let Some(ContainerLogConfig { + choice: + Some(ContainerLogConfigChoice::Custom(CustomContainerLogConfig { + custom: ConfigMapLogConfig { config_map }, + })), + }) = merged_config.logging.containers.get(&Container::Nifi) + { + pod_builder + .add_volume(Volume { + name: "log-config".to_string(), + config_map: Some(ConfigMapVolumeSource { + name: config_map.clone(), + ..ConfigMapVolumeSource::default() + }), + ..Volume::default() + }) + .context(AddVolumeSnafu)?; + } else { + pod_builder + .add_volume(Volume { + name: "log-config".to_string(), + config_map: Some(ConfigMapVolumeSource { + name: resource_names.role_group_config_map().to_string(), + ..ConfigMapVolumeSource::default() + }), + ..Volume::default() + }) + .context(AddVolumeSnafu)?; + } + + if merged_config.logging.enable_vector_agent { + match &nifi.spec.cluster_config.vector_aggregator_config_map_name { + Some(vector_aggregator_config_map_name) => { + pod_builder.add_container( + product_logging::framework::vector_container( + resolved_product_image, + "config", + LOG_VOLUME_NAME, + merged_config.logging.containers.get(&Container::Vector), + ResourceRequirementsBuilder::new() + .with_cpu_request("250m") + .with_cpu_limit("500m") + .with_memory_request("128Mi") + .with_memory_limit("128Mi") + .build(), + vector_aggregator_config_map_name, + ) + .context(ConfigureLoggingSnafu)?, + ); + } + None => { + VectorAggregatorConfigMapMissingSnafu.fail()?; + } + } + } + + authentication_config + .add_volumes_and_mounts(&mut pod_builder, vec![&mut container_prepare]) + .context(AddAuthVolumesSnafu)?; + + let metadata = ObjectMetaBuilder::new() + .with_labels(recommended_object_labels.clone()) + .build(); + + let requested_secret_lifetime = merged_config + .requested_secret_lifetime + .context(MissingSecretLifetimeSnafu)?; + let nifi_cluster_name = nifi.name_any(); + pod_builder + .metadata(metadata) + .image_pull_secrets_from_product_image(resolved_product_image) + .add_init_container(container_prepare.build()) + .affinity(&merged_config.affinity) + // One volume for the NiFi configuration. A script will later on edit (e.g. nodename) + // and copy the whole content to the /conf folder. + .add_volume(stackable_operator::k8s_openapi::api::core::v1::Volume { + name: "config".to_string(), + config_map: Some(ConfigMapVolumeSource { + name: resource_names.role_group_config_map().to_string(), + ..Default::default() + }), + ..Default::default() + }) + .context(AddVolumeSnafu)? + .add_volume(Volume { + name: "conf".to_string(), + config_map: Some(ConfigMapVolumeSource { + name: resource_names.role_group_config_map().to_string(), + ..ConfigMapVolumeSource::default() + }), + ..Volume::default() + }) + .context(AddVolumeSnafu)? + .add_empty_dir_volume( + LOG_VOLUME_NAME, + // Set volume size to higher than theoretically necessary to avoid running out of disk space as log rotation triggers are only checked by Logback every 5s. + Some( + MemoryQuantity { + value: 500.0, + unit: BinaryMultiple::Mebi, + } + .into(), + ), + ) + .context(AddVolumeSnafu)? + // One volume for the keystore and truststore data configmap + .add_volume( + build_tls_volume( + nifi, + KEYSTORE_VOLUME_NAME, + [ + crate::controller::build::resource::service::metrics_service_name( + cluster, + role_group_name, + ), + build_reporting_task_service_name(&nifi_cluster_name), + ], + SecretFormat::TlsPkcs12, + &requested_secret_lifetime, + Some(LISTENER_VOLUME_NAME), + ) + .context(SecuritySnafu)?, + ) + .context(AddVolumeSnafu)? + .add_empty_dir_volume(TRUSTSTORE_VOLUME_NAME, None) + .context(AddVolumeSnafu)? + .add_volumes( + authorization_config + .get_volumes() + .context(AuthorizationConfigurationSnafu)?, + ) + .context(AddVolumeSnafu)?; + + pod_builder + .add_volume(Volume { + name: "sensitiveproperty".to_string(), + secret: Some(SecretVolumeSource { + secret_name: Some(sensitive_key_secret.to_string()), + ..SecretVolumeSource::default() + }), + ..Volume::default() + }) + .context(AddVolumeSnafu)? + .add_volume(Volume { + empty_dir: Some(EmptyDirVolumeSource { + medium: None, + size_limit: None, + }), + name: "activeconf".to_string(), + ..Volume::default() + }) + .context(AddVolumeSnafu)? + .service_account_name(service_account_name) + .security_context(PodSecurityContextBuilder::new().fs_group(1000).build()); + + let mut labels = BTreeMap::new(); + labels.insert( + "app.kubernetes.io/instance".to_string(), + nifi.metadata + .name + .as_deref() + .with_context(|| ObjectHasNoNameSnafu {})? + .to_string(), + ); + + let mut pod_template = pod_builder.build_template(); + // `rg.pod_overrides` is already the role <- rolegroup merge produced by the framework. + pod_template.merge_from(rg.pod_overrides.clone()); + + Ok(StatefulSet { + metadata: ObjectMetaBuilder::new() + .name_and_namespace(cluster) + .name(resource_names.stateful_set_name().to_string()) + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(recommended_object_labels) + .with_label(RESTART_CONTROLLER_ENABLED_LABEL.to_owned()) + .build(), + spec: Some(StatefulSetSpec { + pod_management_policy: Some("Parallel".to_string()), + replicas, + selector: LabelSelector { + match_labels: Some(cluster.role_group_selector(role_group_name).into()), + ..LabelSelector::default() + }, + service_name: Some(resource_names.headless_service_name().to_string()), + template: pod_template, + update_strategy: Some(StatefulSetUpdateStrategy { + type_: if rolling_update_supported { + Some("RollingUpdate".to_string()) + } else { + Some("OnDelete".to_string()) + }, + ..StatefulSetUpdateStrategy::default() + }), + volume_claim_templates: Some(get_volume_claim_templates( + nifi, + cluster, + role_group_name, + merged_config, + authorization_config, + )?), + ..StatefulSetSpec::default() + }), + status: None, + }) +} + +fn get_volume_claim_templates( + nifi: &v1alpha1::NifiCluster, + cluster: &ValidatedCluster, + role_group_name: &RoleGroupName, + merged_config: &NifiConfig, + authorization_config: &ResolvedNifiAuthorizationConfig, +) -> Result> { + let mut pvcs = vec![ + merged_config.resources.storage.content_repo.build_pvc( + &NifiRepository::Content.repository(), + Some(vec!["ReadWriteOnce"]), + ), + merged_config.resources.storage.database_repo.build_pvc( + &NifiRepository::Database.repository(), + Some(vec!["ReadWriteOnce"]), + ), + merged_config.resources.storage.flowfile_repo.build_pvc( + &NifiRepository::Flowfile.repository(), + Some(vec!["ReadWriteOnce"]), + ), + merged_config.resources.storage.provenance_repo.build_pvc( + &NifiRepository::Provenance.repository(), + Some(vec!["ReadWriteOnce"]), + ), + merged_config.resources.storage.state_repo.build_pvc( + &NifiRepository::State.repository(), + Some(vec!["ReadWriteOnce"]), + ), + ]; + + // Used for PVC templates that cannot be modified once they are deployed, so the version label + // is set to the placeholder `none` to keep the labels stable across version upgrades. + let unversioned_recommended_labels = cluster.recommended_labels_unversioned(role_group_name); + + // listener endpoints will use persistent volumes + // so that load balancers can hard-code the target addresses and + // that it is possible to connect to a consistent address + pvcs.push( + build_group_listener_pvc( + &group_listener_name(nifi, &NifiRole::Node.to_string()), + &unversioned_recommended_labels, + ) + .context(ListenerConfigurationSnafu)?, + ); + + // Add file-based PVC if required + if let ResolvedNifiAuthorizationConfig::Standard { + access_policy_provider: NifiAccessPolicyProvider::FileBased { .. }, + } = authorization_config + { + pvcs.push(merged_config.resources.storage.filebased_repo.build_pvc( + &NifiRepository::Filebased.repository(), + Some(vec!["ReadWriteOnce"]), + )) + } + + Ok(pvcs) +} diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index 356f72af..a464f190 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -40,6 +40,7 @@ use crate::{ pub(crate) mod build; pub(crate) mod dereference; +pub(crate) mod upgrade; pub(crate) mod validate; /// A validated, merged (default <- role <- role-group) NiFi rolegroup config. diff --git a/rust/operator-binary/src/operations/upgrade.rs b/rust/operator-binary/src/controller/upgrade.rs similarity index 100% rename from rust/operator-binary/src/operations/upgrade.rs rename to rust/operator-binary/src/controller/upgrade.rs diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 117be255..fff2944e 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -40,12 +40,8 @@ use crate::{ mod controller; mod crd; -mod listener; mod nifi_controller; -mod operations; -mod reporting_task; mod security; -mod service; mod webhooks; mod built_info { diff --git a/rust/operator-binary/src/nifi_controller.rs b/rust/operator-binary/src/nifi_controller.rs index 1fbb3f01..035b94b7 100644 --- a/rust/operator-binary/src/nifi_controller.rs +++ b/rust/operator-binary/src/nifi_controller.rs @@ -1,97 +1,51 @@ //! Ensures that `Pod`s are configured and running for each [`v1alpha1::NifiCluster`]. -use std::{collections::BTreeMap, sync::Arc}; +use std::sync::Arc; use const_format::concatcp; -use indoc::formatdoc; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ - builder::{ - self, - meta::ObjectMetaBuilder, - pod::{ - PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, - security::PodSecurityContextBuilder, volume::SecretFormat, - }, - }, cli::OperatorEnvironmentOptions, client::Client, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, - commons::{product_image_selection::ResolvedProductImage, rbac::build_rbac_resources}, - constants::RESTART_CONTROLLER_ENABLED_LABEL, - crd::{authentication::oidc::v1alpha1::AuthenticationProvider, git_sync}, - k8s_openapi::{ - DeepMerge, - api::{ - apps::v1::{StatefulSet, StatefulSetSpec, StatefulSetUpdateStrategy}, - core::v1::{ - ConfigMapKeySelector, ConfigMapVolumeSource, EmptyDirVolumeSource, EnvVar, - EnvVarSource, ObjectFieldSelector, PersistentVolumeClaim, Probe, - SecretVolumeSource, TCPSocketAction, Volume, - }, - }, - apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, - }, + commons::rbac::build_rbac_resources, kube::{ Resource, ResourceExt, core::{DeserializeGuard, error_boundary}, runtime::controller::Action, }, logging::controller::ReconcilerError, - memory::{BinaryMultiple, MemoryQuantity}, - product_logging::{ - self, - framework::{ - LoggingError, create_vector_shutdown_file_command, remove_vector_shutdown_file_command, - }, - spec::{ - ConfigMapLogConfig, ContainerLogConfig, ContainerLogConfigChoice, - CustomContainerLogConfig, - }, - }, role_utils::GenericRoleConfig, shared::time::Duration, status::condition::{ compute_conditions, operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, }, - utils::{COMMON_BASH_TRAP_FUNCTIONS, cluster_info::KubernetesClusterInfo}, - v2::{builder::meta::ownerreference_from_resource, types::operator::RoleGroupName}, + v2::types::operator::RoleGroupName, }; use strum::{EnumDiscriminants, IntoStaticStr}; use tracing::Instrument; use crate::{ OPERATOR_NAME, - controller::{ValidatedCluster, ValidatedRoleGroupConfig, build, dereference, validate}, - crd::{ - APP_NAME, BALANCE_PORT, BALANCE_PORT_NAME, Container, HTTPS_PORT, HTTPS_PORT_NAME, - METRICS_PORT, METRICS_PORT_NAME, NifiConfig, NifiNodeRoleConfig, NifiRole, NifiRoleType, - NifiStatus, PROTOCOL_PORT, PROTOCOL_PORT_NAME, STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, - authorization::NifiAccessPolicyProvider, - constants::{NIFI_CONFIG_DIRECTORY, NIFI_PYTHON_WORKING_DIRECTORY}, - storage::{NifiRepository, PERSISTENT_REPOSITORIES}, - v1alpha1, - }, - listener::{ - LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_group_listener, build_group_listener_pvc, - group_listener_name, - }, - operations::{ - graceful_shutdown::add_graceful_shutdown_config, - pdb::add_pdbs, + controller::{ + build, + build::resource::{ + listener::{build_group_listener, group_listener_name}, + pdb::add_pdbs, + reporting_task::build_maybe_reporting_task, + service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, + statefulset::build_node_rolegroup_statefulset, + }, + dereference, upgrade::{self, ClusterVersionUpdateState}, + validate, }, - reporting_task::{build_maybe_reporting_task, build_reporting_task_service_name}, + crd::{APP_NAME, NifiNodeRoleConfig, NifiRole, NifiStatus, v1alpha1}, security::{ - authentication::{ - NifiAuthenticationConfig, STACKABLE_SERVER_TLS_DIR, STACKABLE_TLS_STORE_PASSWORD, - }, - authorization::{self, OPA_TLS_MOUNT_PATH, ResolvedNifiAuthorizationConfig}, - build_tls_volume, check_or_generate_oidc_admin_password, check_or_generate_sensitive_key, - tls::{KEYSTORE_NIFI_CONTAINER_MOUNT, KEYSTORE_VOLUME_NAME, TRUSTSTORE_VOLUME_NAME}, + authentication::NifiAuthenticationConfig, check_or_generate_oidc_admin_password, + check_or_generate_sensitive_key, }, - service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, }; pub const NIFI_CONTROLLER_NAME: &str = "nificluster"; @@ -109,17 +63,11 @@ pub struct Ctx { #[strum_discriminants(derive(IntoStaticStr))] #[allow(clippy::enum_variant_names)] pub enum Error { - #[snafu(display("missing secret lifetime"))] - MissingSecretLifetime, - #[snafu(display("NifiCluster object is invalid"))] InvalidNifiCluster { source: error_boundary::InvalidObject, }, - #[snafu(display("object defines no name"))] - ObjectHasNoName, - #[snafu(display("failed to dereference resources"))] Dereference { source: dereference::Error }, @@ -153,6 +101,12 @@ pub enum Error { rolegroup: RoleGroupName, }, + #[snafu(display("failed to build StatefulSet for {}", rolegroup))] + BuildStatefulSet { + source: crate::controller::build::resource::statefulset::Error, + rolegroup: RoleGroupName, + }, + #[snafu(display("object has no nodes defined"))] NoNodesDefined, @@ -178,18 +132,9 @@ pub enum Error { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("illegal container name: [{container_name}]"))] - IllegalContainerName { - source: stackable_operator::builder::pod::container::Error, - container_name: String, - }, - #[snafu(display("failed to build git-sync resources"))] BuildGitSyncResources { source: build::git_sync::Error }, - #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] - VectorAggregatorConfigMapMissing, - #[snafu(display("failed to patch service account"))] ApplyServiceAccount { source: stackable_operator::cluster_resources::Error, @@ -207,12 +152,7 @@ pub enum Error { #[snafu(display("failed to create PodDisruptionBudget"))] FailedToCreatePdb { - source: crate::operations::pdb::Error, - }, - - #[snafu(display("failed to configure graceful shutdown"))] - GracefulShutdown { - source: crate::operations::graceful_shutdown::Error, + source: crate::controller::build::resource::pdb::Error, }, #[snafu(display("failed to get required labels"))] @@ -221,28 +161,12 @@ pub enum Error { stackable_operator::kvp::KeyValuePairError, }, - #[snafu(display("failed to add Authentication Volumes and VolumeMounts"))] - AddAuthVolumes { - source: crate::security::authentication::Error, - }, - #[snafu(display("security failure"))] Security { source: crate::security::Error }, #[snafu(display("reporting task failure"))] ReportingTask { - source: crate::reporting_task::Error, - }, - - #[snafu(display("failed to configure logging"))] - ConfigureLogging { source: LoggingError }, - - #[snafu(display("failed to add needed volume"))] - AddVolume { source: builder::pod::Error }, - - #[snafu(display("failed to add needed volumeMount"))] - AddVolumeMount { - source: builder::pod::container::Error, + source: crate::controller::build::resource::reporting_task::Error, }, #[snafu(display("Failed to determine the state of the version upgrade procedure"))] @@ -254,10 +178,9 @@ pub enum Error { }, #[snafu(display("failed to configure listener"))] - ListenerConfiguration { source: crate::listener::Error }, - - #[snafu(display("failed to build authorization configuration"))] - AuthorizationConfiguration { source: authorization::Error }, + ListenerConfiguration { + source: crate::controller::build::resource::listener::Error, + }, } type Result = std::result::Result; @@ -425,7 +348,10 @@ pub async fn reconcile_nifi( &rbac_sa.name_any(), &git_sync_resources, ) - .await?; + .await + .with_context(|_| BuildStatefulSetSnafu { + rolegroup: role_group_name.clone(), + })?; let rg_metrics_service = build_rolegroup_metrics_service(&validated_cluster, role_group_name); @@ -559,627 +485,6 @@ pub async fn reconcile_nifi( Ok(Action::await_change()) } -const USERDATA_MOUNTPOINT: &str = "/stackable/userdata"; - -/// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. -/// -/// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the -/// corresponding [`stackable_operator::k8s_openapi::api::core::v1::Service`] (from [`build_rolegroup_headless_service`]). -#[allow(clippy::too_many_arguments)] -async fn build_node_rolegroup_statefulset( - nifi: &v1alpha1::NifiCluster, - cluster: &ValidatedCluster, - resolved_product_image: &ResolvedProductImage, - cluster_info: &KubernetesClusterInfo, - role_group_name: &RoleGroupName, - role: &NifiRoleType, - rg: &ValidatedRoleGroupConfig, - authentication_config: &NifiAuthenticationConfig, - authorization_config: &ResolvedNifiAuthorizationConfig, - rolling_update_supported: bool, - replicas: Option, - service_account_name: &str, - git_sync_resources: &git_sync::v1alpha2::GitSyncResources, -) -> Result { - tracing::debug!("Building statefulset"); - - // Type-safe names for this role group's resources (StatefulSet, ConfigMap, headless Service). - let resource_names = cluster.resource_names(role_group_name); - - // The validated, merged `NifiConfig` is the single source of truth; the ConfigMap builder - // sources the same `rg.config`. - let merged_config = &rg.config; - - let mut env_vars: Vec = rg.env_overrides.clone().into(); - - // we need the POD_NAME env var to overwrite `nifi.cluster.node.address` later - env_vars.push(EnvVar { - name: "POD_NAME".to_string(), - value_from: Some(EnvVarSource { - field_ref: Some(ObjectFieldSelector { - api_version: Some("v1".to_string()), - field_path: "metadata.name".to_string(), - }), - ..EnvVarSource::default() - }), - ..EnvVar::default() - }); - - // Needed for the `containerdebug` process to log it's tracing information to. - env_vars.push(EnvVar { - name: "CONTAINERDEBUG_LOG_DIRECTORY".to_string(), - value: Some(format!("{STACKABLE_LOG_DIR}/containerdebug")), - ..Default::default() - }); - - env_vars.push(EnvVar { - name: "STACKLET_NAME".to_string(), - value: Some(nifi.name_unchecked().to_string()), - ..Default::default() - }); - - match &nifi.spec.cluster_config.clustering_backend { - v1alpha1::NifiClusteringBackend::ZooKeeper { - zookeeper_config_map_name, - } => { - let zookeeper_env_var = |name: &str| EnvVar { - name: name.to_string(), - value_from: Some(EnvVarSource { - config_map_key_ref: Some(ConfigMapKeySelector { - name: zookeeper_config_map_name.to_string(), - key: name.to_string(), - ..ConfigMapKeySelector::default() - }), - ..EnvVarSource::default() - }), - ..EnvVar::default() - }; - env_vars.push(zookeeper_env_var("ZOOKEEPER_HOSTS")); - env_vars.push(zookeeper_env_var("ZOOKEEPER_CHROOT")); - } - v1alpha1::NifiClusteringBackend::Kubernetes {} => {} - } - - if let NifiAuthenticationConfig::Oidc { oidc, .. } = authentication_config { - env_vars.extend(AuthenticationProvider::client_credentials_env_var_mounts( - oidc.client_credentials_secret_ref.clone(), - )); - } - - env_vars.extend(authorization_config.get_env_vars()); - - let node_address = format!( - "$POD_NAME.{service_name}.{namespace}.svc.{cluster_domain}", - service_name = resource_names.headless_service_name(), - namespace = cluster.namespace, - cluster_domain = cluster_info.cluster_domain, - ); - - let sensitive_key_secret = &nifi.spec.cluster_config.sensitive_properties.key_secret; - - let prepare_container_name = Container::Prepare.to_string(); - let mut prepare_args = vec![]; - - if let Some(ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic(log_config)), - }) = merged_config.logging.containers.get(&Container::Prepare) - { - prepare_args.push(product_logging::framework::capture_shell_output( - STACKABLE_LOG_DIR, - &prepare_container_name, - log_config, - )); - } - - // Note(sbernauer): In https://github.com/stackabletech/issues/issues/764 we migrated all usages - // of keytool to our own cert-utils tool. As it uses the same code as secret-operator, it also - // uses RC2. Thus, the keytool usage here LGTM (no alias trickery) and has my nod of approval. - prepare_args.extend(vec![ - // The source directory is a secret-op mount and we do not want to write / add anything in there - // Therefore we import all the contents to a truststore in "writable" empty dirs. - // Keytool is only barking if a password is not set for the destination truststore (which we set) - // and do provide an empty password for the source truststore coming from the secret-operator. - // Using no password will result in a warning. - format!("echo Importing {KEYSTORE_NIFI_CONTAINER_MOUNT}/keystore.p12 to {STACKABLE_SERVER_TLS_DIR}/keystore.p12"), - format!("cp {KEYSTORE_NIFI_CONTAINER_MOUNT}/keystore.p12 {STACKABLE_SERVER_TLS_DIR}/keystore.p12"), - format!("echo Importing {KEYSTORE_NIFI_CONTAINER_MOUNT}/truststore.p12 to {STACKABLE_SERVER_TLS_DIR}/truststore.p12"), - // secret-operator currently encrypts keystores with RC2, which NiFi is unable to read: https://github.com/stackabletech/nifi-operator/pull/510 - // As a workaround, reencrypt the keystore with keytool. - // keytool crashes if the target truststore already exists (covering up the true error - // if the init container fails later on in the script), so delete it first. - format!("test ! -e {STACKABLE_SERVER_TLS_DIR}/truststore.p12 || rm {STACKABLE_SERVER_TLS_DIR}/truststore.p12"), - format!("keytool -importkeystore -srckeystore {KEYSTORE_NIFI_CONTAINER_MOUNT}/truststore.p12 -destkeystore {STACKABLE_SERVER_TLS_DIR}/truststore.p12 -srcstorepass {STACKABLE_TLS_STORE_PASSWORD} -deststorepass {STACKABLE_TLS_STORE_PASSWORD}"), - - "echo Replacing config directory".to_string(), - "cp /conf/* /stackable/nifi/conf".to_string(), - "test -L /stackable/nifi/conf/logback.xml || ln -sf /stackable/log_config/logback.xml /stackable/nifi/conf/logback.xml".to_string(), - format!(r#"export NODE_ADDRESS="{node_address}""#), - ]); - - // This commands needs to go first, as they might set env variables needed by the templating - prepare_args.extend_from_slice( - authentication_config - .get_additional_container_args() - .as_slice(), - ); - - // Add OPA certificate to truststore if OPA TLS is enabled - if authorization_config.has_opa_tls() { - prepare_args.extend(vec![ - "echo Importing OPA CA certificate to truststore".to_string(), - format!("keytool -importcert -file {OPA_TLS_MOUNT_PATH}/ca.crt -keystore {STACKABLE_SERVER_TLS_DIR}/truststore.p12 -storepass {STACKABLE_TLS_STORE_PASSWORD} -alias opa-ca -noprompt"), - ]); - } - - prepare_args.extend(vec![ - "export LISTENER_DEFAULT_ADDRESS=$(cat /stackable/listener/default-address/address)" - .to_string(), - ]); - prepare_args.extend(vec![ - "export LISTENER_DEFAULT_PORT_HTTPS=$(cat /stackable/listener/default-address/ports/https)" - .to_string(), - ]); - - prepare_args.extend(vec![ - "echo Templating config files".to_string(), - "config-utils template /stackable/nifi/conf/nifi.properties".to_string(), - "config-utils template /stackable/nifi/conf/state-management.xml".to_string(), - "config-utils template /stackable/nifi/conf/login-identity-providers.xml".to_string(), - "config-utils template /stackable/nifi/conf/authorizers.xml".to_string(), - "config-utils template /stackable/nifi/conf/security.properties".to_string(), - ]); - - let mut container_prepare = - ContainerBuilder::new(&prepare_container_name).with_context(|_| { - IllegalContainerNameSnafu { - container_name: prepare_container_name.to_string(), - } - })?; - - container_prepare - .image_from_product_image(resolved_product_image) - .command(vec![ - "/bin/bash".to_string(), - "-c".to_string(), - "-euo".to_string(), - "pipefail".to_string(), - ]) - .add_env_vars(env_vars.clone()) - .args(vec![prepare_args.join(" && ")]) - .add_volume_mounts( - PERSISTENT_REPOSITORIES - .iter() - .map(NifiRepository::volume_mount), - ) - .context(AddVolumeMountSnafu)? - .add_volume_mount("conf", "/conf") - .context(AddVolumeMountSnafu)? - .add_volume_mount(KEYSTORE_VOLUME_NAME, KEYSTORE_NIFI_CONTAINER_MOUNT) - .context(AddVolumeMountSnafu)? - .add_volume_mount("activeconf", NIFI_CONFIG_DIRECTORY) - .context(AddVolumeMountSnafu)? - .add_volume_mount("sensitiveproperty", "/stackable/sensitiveproperty") - .context(AddVolumeMountSnafu)? - .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) - .context(AddVolumeMountSnafu)? - .add_volume_mount(TRUSTSTORE_VOLUME_NAME, STACKABLE_SERVER_TLS_DIR) - .context(AddVolumeMountSnafu)? - .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) - .context(AddVolumeMountSnafu)? - .add_volume_mounts(authorization_config.get_volume_mounts()) - .context(AddVolumeMountSnafu)? - .resources( - ResourceRequirementsBuilder::new() - .with_cpu_request("500m") - .with_cpu_limit("2000m") - .with_memory_request("4096Mi") - .with_memory_limit("4096Mi") - .build(), - ); - - let nifi_container_name = Container::Nifi.to_string(); - let mut container_nifi_builder = - ContainerBuilder::new(&nifi_container_name).with_context(|_| { - IllegalContainerNameSnafu { - container_name: nifi_container_name, - } - })?; - - let nifi_args = vec![formatdoc! {" - {COMMON_BASH_TRAP_FUNCTIONS} - {remove_vector_shutdown_file_command} - prepare_signal_handlers - containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & - bin/nifi.sh run & - wait_for_termination $! - {create_vector_shutdown_file_command} - ", - remove_vector_shutdown_file_command = - remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), - create_vector_shutdown_file_command = - create_vector_shutdown_file_command(STACKABLE_LOG_DIR), - }]; - - let container_nifi = container_nifi_builder - .image_from_product_image(resolved_product_image) - .command(vec![ - "/bin/bash".to_string(), - "-x".to_string(), - "-euo".to_string(), - "pipefail".to_string(), - "-c".to_string(), - ]) - .args(nifi_args) - .add_env_vars(env_vars) - .add_volume_mount(KEYSTORE_VOLUME_NAME, KEYSTORE_NIFI_CONTAINER_MOUNT) - .context(AddVolumeMountSnafu)? - .add_volume_mounts( - PERSISTENT_REPOSITORIES - .iter() - .map(NifiRepository::volume_mount), - ) - .context(AddVolumeMountSnafu)? - .add_volume_mount("activeconf", NIFI_CONFIG_DIRECTORY) - .context(AddVolumeMountSnafu)? - .add_volume_mount("log-config", STACKABLE_LOG_CONFIG_DIR) - .context(AddVolumeMountSnafu)? - .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) - .context(AddVolumeMountSnafu)? - .add_volume_mount(TRUSTSTORE_VOLUME_NAME, STACKABLE_SERVER_TLS_DIR) - .context(AddVolumeMountSnafu)? - .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) - .context(AddVolumeMountSnafu)? - .add_volume_mounts(authorization_config.get_volume_mounts()) - .context(AddVolumeMountSnafu)? - .add_container_port(HTTPS_PORT_NAME, HTTPS_PORT.into()) - .add_container_port(PROTOCOL_PORT_NAME, PROTOCOL_PORT.into()) - .add_container_port(BALANCE_PORT_NAME, BALANCE_PORT.into()) - .liveness_probe(Probe { - initial_delay_seconds: Some(10), - period_seconds: Some(10), - tcp_socket: Some(TCPSocketAction { - port: IntOrString::String(HTTPS_PORT_NAME.to_string()), - ..TCPSocketAction::default() - }), - ..Probe::default() - }) - .startup_probe(Probe { - initial_delay_seconds: Some(10), - period_seconds: Some(10), - failure_threshold: Some(20 * 6), - tcp_socket: Some(TCPSocketAction { - port: IntOrString::String(HTTPS_PORT_NAME.to_string()), - ..TCPSocketAction::default() - }), - ..Probe::default() - }) - .resources(merged_config.resources.clone().into()); - - // NiFi 2.x.x offers nifi-api/flow/metrics/prometheus at the HTTPS_PORT, therefore METRICS_PORT is only required for NiFi 1.x.x. - if resolved_product_image.product_version.starts_with("1.") { - container_nifi.add_container_port(METRICS_PORT_NAME, METRICS_PORT.into()); - } - - let mut pod_builder = PodBuilder::new(); - - let recommended_object_labels = cluster.recommended_labels(role_group_name); - - add_graceful_shutdown_config(merged_config, &mut pod_builder).context(GracefulShutdownSnafu)?; - - // Add user configured extra volumes if any are specified - for volume in &nifi.spec.cluster_config.extra_volumes { - // Extract values into vars so we make it impossible to log something other than - // what we actually use to create the mounts - maybe paranoid, but hey .. - let volume_name = &volume.name; - let mount_point = format!("{USERDATA_MOUNTPOINT}/{}", volume.name); - - tracing::info!( - ?volume_name, - ?mount_point, - ?role, - "Adding user specified extra volume", - ); - pod_builder - .add_volume(volume.clone()) - .context(AddVolumeSnafu)?; - container_nifi - .add_volume_mount(volume_name, mount_point) - .context(AddVolumeMountSnafu)?; - } - - let volume_name = "nifi-python-working-directory".to_string(); - pod_builder - .add_empty_dir_volume(&volume_name, None) - .context(AddVolumeSnafu)?; - container_nifi - .add_volume_mount(&volume_name, NIFI_PYTHON_WORKING_DIRECTORY) - .context(AddVolumeMountSnafu)?; - - container_nifi - .add_volume_mounts(git_sync_resources.git_content_volume_mounts.to_owned()) - .context(AddVolumeMountSnafu)?; - - // We want to add nifi container first for easier defaulting into this container - pod_builder.add_container(container_nifi.build()); - // After calling `build()` the ContainerBuilder shouldn't be used anymore, so we drop it - drop(container_nifi_builder); - - for container in git_sync_resources.git_sync_containers.iter().cloned() { - pod_builder.add_container(container); - } - for container in git_sync_resources.git_sync_init_containers.iter().cloned() { - pod_builder.add_init_container(container); - } - pod_builder - .add_volumes(git_sync_resources.git_content_volumes.to_owned()) - .context(AddVolumeSnafu)?; - pod_builder - .add_volumes(git_sync_resources.git_ca_cert_volumes.to_owned()) - .context(AddVolumeSnafu)?; - - if let Some(ContainerLogConfig { - choice: - Some(ContainerLogConfigChoice::Custom(CustomContainerLogConfig { - custom: ConfigMapLogConfig { config_map }, - })), - }) = merged_config.logging.containers.get(&Container::Nifi) - { - pod_builder - .add_volume(Volume { - name: "log-config".to_string(), - config_map: Some(ConfigMapVolumeSource { - name: config_map.clone(), - ..ConfigMapVolumeSource::default() - }), - ..Volume::default() - }) - .context(AddVolumeSnafu)?; - } else { - pod_builder - .add_volume(Volume { - name: "log-config".to_string(), - config_map: Some(ConfigMapVolumeSource { - name: resource_names.role_group_config_map().to_string(), - ..ConfigMapVolumeSource::default() - }), - ..Volume::default() - }) - .context(AddVolumeSnafu)?; - } - - if merged_config.logging.enable_vector_agent { - match &nifi.spec.cluster_config.vector_aggregator_config_map_name { - Some(vector_aggregator_config_map_name) => { - pod_builder.add_container( - product_logging::framework::vector_container( - resolved_product_image, - "config", - LOG_VOLUME_NAME, - merged_config.logging.containers.get(&Container::Vector), - ResourceRequirementsBuilder::new() - .with_cpu_request("250m") - .with_cpu_limit("500m") - .with_memory_request("128Mi") - .with_memory_limit("128Mi") - .build(), - vector_aggregator_config_map_name, - ) - .context(ConfigureLoggingSnafu)?, - ); - } - None => { - VectorAggregatorConfigMapMissingSnafu.fail()?; - } - } - } - - authentication_config - .add_volumes_and_mounts(&mut pod_builder, vec![&mut container_prepare]) - .context(AddAuthVolumesSnafu)?; - - let metadata = ObjectMetaBuilder::new() - .with_labels(recommended_object_labels.clone()) - .build(); - - let requested_secret_lifetime = merged_config - .requested_secret_lifetime - .context(MissingSecretLifetimeSnafu)?; - let nifi_cluster_name = nifi.name_any(); - pod_builder - .metadata(metadata) - .image_pull_secrets_from_product_image(resolved_product_image) - .add_init_container(container_prepare.build()) - .affinity(&merged_config.affinity) - // One volume for the NiFi configuration. A script will later on edit (e.g. nodename) - // and copy the whole content to the /conf folder. - .add_volume(stackable_operator::k8s_openapi::api::core::v1::Volume { - name: "config".to_string(), - config_map: Some(ConfigMapVolumeSource { - name: resource_names.role_group_config_map().to_string(), - ..Default::default() - }), - ..Default::default() - }) - .context(AddVolumeSnafu)? - .add_volume(Volume { - name: "conf".to_string(), - config_map: Some(ConfigMapVolumeSource { - name: resource_names.role_group_config_map().to_string(), - ..ConfigMapVolumeSource::default() - }), - ..Volume::default() - }) - .context(AddVolumeSnafu)? - .add_empty_dir_volume( - LOG_VOLUME_NAME, - // Set volume size to higher than theoretically necessary to avoid running out of disk space as log rotation triggers are only checked by Logback every 5s. - Some( - MemoryQuantity { - value: 500.0, - unit: BinaryMultiple::Mebi, - } - .into(), - ), - ) - .context(AddVolumeSnafu)? - // One volume for the keystore and truststore data configmap - .add_volume( - build_tls_volume( - nifi, - KEYSTORE_VOLUME_NAME, - [ - crate::service::metrics_service_name(cluster, role_group_name), - build_reporting_task_service_name(&nifi_cluster_name), - ], - SecretFormat::TlsPkcs12, - &requested_secret_lifetime, - Some(LISTENER_VOLUME_NAME), - ) - .context(SecuritySnafu)?, - ) - .context(AddVolumeSnafu)? - .add_empty_dir_volume(TRUSTSTORE_VOLUME_NAME, None) - .context(AddVolumeSnafu)? - .add_volumes( - authorization_config - .get_volumes() - .context(AuthorizationConfigurationSnafu)?, - ) - .context(AddVolumeSnafu)?; - - pod_builder - .add_volume(Volume { - name: "sensitiveproperty".to_string(), - secret: Some(SecretVolumeSource { - secret_name: Some(sensitive_key_secret.to_string()), - ..SecretVolumeSource::default() - }), - ..Volume::default() - }) - .context(AddVolumeSnafu)? - .add_volume(Volume { - empty_dir: Some(EmptyDirVolumeSource { - medium: None, - size_limit: None, - }), - name: "activeconf".to_string(), - ..Volume::default() - }) - .context(AddVolumeSnafu)? - .service_account_name(service_account_name) - .security_context(PodSecurityContextBuilder::new().fs_group(1000).build()); - - let mut labels = BTreeMap::new(); - labels.insert( - "app.kubernetes.io/instance".to_string(), - nifi.metadata - .name - .as_deref() - .with_context(|| ObjectHasNoNameSnafu {})? - .to_string(), - ); - - let mut pod_template = pod_builder.build_template(); - // `rg.pod_overrides` is already the role <- rolegroup merge produced by the framework. - pod_template.merge_from(rg.pod_overrides.clone()); - - Ok(StatefulSet { - metadata: ObjectMetaBuilder::new() - .name_and_namespace(cluster) - .name(resource_names.stateful_set_name().to_string()) - .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) - .with_labels(recommended_object_labels) - .with_label(RESTART_CONTROLLER_ENABLED_LABEL.to_owned()) - .build(), - spec: Some(StatefulSetSpec { - pod_management_policy: Some("Parallel".to_string()), - replicas, - selector: LabelSelector { - match_labels: Some(cluster.role_group_selector(role_group_name).into()), - ..LabelSelector::default() - }, - service_name: Some(resource_names.headless_service_name().to_string()), - template: pod_template, - update_strategy: Some(StatefulSetUpdateStrategy { - type_: if rolling_update_supported { - Some("RollingUpdate".to_string()) - } else { - Some("OnDelete".to_string()) - }, - ..StatefulSetUpdateStrategy::default() - }), - volume_claim_templates: Some(get_volume_claim_templates( - nifi, - cluster, - role_group_name, - merged_config, - authorization_config, - )?), - ..StatefulSetSpec::default() - }), - status: None, - }) -} - -fn get_volume_claim_templates( - nifi: &v1alpha1::NifiCluster, - cluster: &ValidatedCluster, - role_group_name: &RoleGroupName, - merged_config: &NifiConfig, - authorization_config: &ResolvedNifiAuthorizationConfig, -) -> Result> { - let mut pvcs = vec![ - merged_config.resources.storage.content_repo.build_pvc( - &NifiRepository::Content.repository(), - Some(vec!["ReadWriteOnce"]), - ), - merged_config.resources.storage.database_repo.build_pvc( - &NifiRepository::Database.repository(), - Some(vec!["ReadWriteOnce"]), - ), - merged_config.resources.storage.flowfile_repo.build_pvc( - &NifiRepository::Flowfile.repository(), - Some(vec!["ReadWriteOnce"]), - ), - merged_config.resources.storage.provenance_repo.build_pvc( - &NifiRepository::Provenance.repository(), - Some(vec!["ReadWriteOnce"]), - ), - merged_config.resources.storage.state_repo.build_pvc( - &NifiRepository::State.repository(), - Some(vec!["ReadWriteOnce"]), - ), - ]; - - // Used for PVC templates that cannot be modified once they are deployed, so the version label - // is set to the placeholder `none` to keep the labels stable across version upgrades. - let unversioned_recommended_labels = cluster.recommended_labels_unversioned(role_group_name); - - // listener endpoints will use persistent volumes - // so that load balancers can hard-code the target addresses and - // that it is possible to connect to a consistent address - pvcs.push( - build_group_listener_pvc( - &group_listener_name(nifi, &NifiRole::Node.to_string()), - &unversioned_recommended_labels, - ) - .context(ListenerConfigurationSnafu)?, - ); - - // Add file-based PVC if required - if let ResolvedNifiAuthorizationConfig::Standard { - access_policy_provider: NifiAccessPolicyProvider::FileBased { .. }, - } = authorization_config - { - pvcs.push(merged_config.resources.storage.filebased_repo.build_pvc( - &NifiRepository::Filebased.repository(), - Some(vec!["ReadWriteOnce"]), - )) - } - - Ok(pvcs) -} - pub fn error_policy( _obj: Arc>, error: &Error, diff --git a/rust/operator-binary/src/operations/mod.rs b/rust/operator-binary/src/operations/mod.rs deleted file mode 100644 index c62006d9..00000000 --- a/rust/operator-binary/src/operations/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod graceful_shutdown; -pub mod pdb; -pub mod upgrade; From 1a96afeefad27e3f1f91b818b08fe4b2dcf7920f Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 13 Jun 2026 21:09:15 +0200 Subject: [PATCH 31/39] fix: cleanup constants --- .../src/controller/build/git_sync.rs | 5 ++- .../controller/build/resource/statefulset.rs | 44 ++++++++++++++----- .../src/controller/validate.rs | 5 ++- rust/operator-binary/src/nifi_controller.rs | 3 -- 4 files changed, 40 insertions(+), 17 deletions(-) diff --git a/rust/operator-binary/src/controller/build/git_sync.rs b/rust/operator-binary/src/controller/build/git_sync.rs index 0117c2d2..2171b53d 100644 --- a/rust/operator-binary/src/controller/build/git_sync.rs +++ b/rust/operator-binary/src/controller/build/git_sync.rs @@ -4,9 +4,10 @@ use snafu::{ResultExt, Snafu}; use stackable_operator::{crd::git_sync, k8s_openapi::api::core::v1::EnvVar}; use crate::{ - controller::{ValidatedCluster, ValidatedRoleGroupConfig}, + controller::{ + ValidatedCluster, ValidatedRoleGroupConfig, build::resource::statefulset::LOG_VOLUME_NAME, + }, crd::Container, - nifi_controller::LOG_VOLUME_NAME, }; #[derive(Snafu, Debug)] diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 54b3fd36..c7bc4d23 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -67,7 +67,6 @@ use crate::{ storage::{NifiRepository, PERSISTENT_REPOSITORIES}, v1alpha1, }, - nifi_controller::LOG_VOLUME_NAME, security::{ authentication::{ NifiAuthenticationConfig, STACKABLE_SERVER_TLS_DIR, STACKABLE_TLS_STORE_PASSWORD, @@ -134,6 +133,26 @@ type Result = std::result::Result; const USERDATA_MOUNTPOINT: &str = "/stackable/userdata"; +/// Volume providing the rendered NiFi config (the `conf` ConfigMap), mounted into the prepare +/// container which templates it into [`ACTIVE_CONFIG_VOLUME_NAME`]. +const CONFIG_VOLUME_NAME: &str = "conf"; +const CONFIG_VOLUME_MOUNT: &str = "/conf"; + +/// `emptyDir` holding the live config templated by the prepare container and shared with the NiFi +/// container. +const ACTIVE_CONFIG_VOLUME_NAME: &str = "activeconf"; + +/// Volume holding the generated sensitive-properties key. +const SENSITIVE_PROPERTY_VOLUME_NAME: &str = "sensitiveproperty"; +const SENSITIVE_PROPERTY_VOLUME_MOUNT: &str = "/stackable/sensitiveproperty"; + +/// Volume providing the log config (logback/log4j) ConfigMap. +const LOG_CONFIG_VOLUME_NAME: &str = "log-config"; + +/// Volume the NiFi logs are written to and shared with the Vector sidecar (also used by the +/// git-sync container, see [`crate::controller::build::git_sync`]). +pub(crate) const LOG_VOLUME_NAME: &str = "log"; + /// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. /// /// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the @@ -325,13 +344,16 @@ pub(crate) async fn build_node_rolegroup_statefulset( .map(NifiRepository::volume_mount), ) .context(AddVolumeMountSnafu)? - .add_volume_mount("conf", "/conf") + .add_volume_mount(CONFIG_VOLUME_NAME, CONFIG_VOLUME_MOUNT) .context(AddVolumeMountSnafu)? .add_volume_mount(KEYSTORE_VOLUME_NAME, KEYSTORE_NIFI_CONTAINER_MOUNT) .context(AddVolumeMountSnafu)? - .add_volume_mount("activeconf", NIFI_CONFIG_DIRECTORY) + .add_volume_mount(ACTIVE_CONFIG_VOLUME_NAME, NIFI_CONFIG_DIRECTORY) .context(AddVolumeMountSnafu)? - .add_volume_mount("sensitiveproperty", "/stackable/sensitiveproperty") + .add_volume_mount( + SENSITIVE_PROPERTY_VOLUME_NAME, + SENSITIVE_PROPERTY_VOLUME_MOUNT, + ) .context(AddVolumeMountSnafu)? .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) .context(AddVolumeMountSnafu)? @@ -392,9 +414,9 @@ pub(crate) async fn build_node_rolegroup_statefulset( .map(NifiRepository::volume_mount), ) .context(AddVolumeMountSnafu)? - .add_volume_mount("activeconf", NIFI_CONFIG_DIRECTORY) + .add_volume_mount(ACTIVE_CONFIG_VOLUME_NAME, NIFI_CONFIG_DIRECTORY) .context(AddVolumeMountSnafu)? - .add_volume_mount("log-config", STACKABLE_LOG_CONFIG_DIR) + .add_volume_mount(LOG_CONFIG_VOLUME_NAME, STACKABLE_LOG_CONFIG_DIR) .context(AddVolumeMountSnafu)? .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) .context(AddVolumeMountSnafu)? @@ -499,7 +521,7 @@ pub(crate) async fn build_node_rolegroup_statefulset( { pod_builder .add_volume(Volume { - name: "log-config".to_string(), + name: LOG_CONFIG_VOLUME_NAME.to_string(), config_map: Some(ConfigMapVolumeSource { name: config_map.clone(), ..ConfigMapVolumeSource::default() @@ -510,7 +532,7 @@ pub(crate) async fn build_node_rolegroup_statefulset( } else { pod_builder .add_volume(Volume { - name: "log-config".to_string(), + name: LOG_CONFIG_VOLUME_NAME.to_string(), config_map: Some(ConfigMapVolumeSource { name: resource_names.role_group_config_map().to_string(), ..ConfigMapVolumeSource::default() @@ -575,7 +597,7 @@ pub(crate) async fn build_node_rolegroup_statefulset( }) .context(AddVolumeSnafu)? .add_volume(Volume { - name: "conf".to_string(), + name: CONFIG_VOLUME_NAME.to_string(), config_map: Some(ConfigMapVolumeSource { name: resource_names.role_group_config_map().to_string(), ..ConfigMapVolumeSource::default() @@ -625,7 +647,7 @@ pub(crate) async fn build_node_rolegroup_statefulset( pod_builder .add_volume(Volume { - name: "sensitiveproperty".to_string(), + name: SENSITIVE_PROPERTY_VOLUME_NAME.to_string(), secret: Some(SecretVolumeSource { secret_name: Some(sensitive_key_secret.to_string()), ..SecretVolumeSource::default() @@ -638,7 +660,7 @@ pub(crate) async fn build_node_rolegroup_statefulset( medium: None, size_limit: None, }), - name: "activeconf".to_string(), + name: ACTIVE_CONFIG_VOLUME_NAME.to_string(), ..Volume::default() }) .context(AddVolumeSnafu)? diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index e118e83f..bd0fc6ba 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -31,6 +31,9 @@ use crate::{ }, }; +/// The base name of the NiFi product image, used to resolve the fully-qualified image reference. +const CONTAINER_IMAGE_BASE_NAME: &str = "nifi"; + #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] #[allow(clippy::enum_variant_names)] @@ -83,7 +86,7 @@ pub fn validate( .spec .image .resolve( - crate::nifi_controller::CONTAINER_IMAGE_BASE_NAME, + CONTAINER_IMAGE_BASE_NAME, &operator_environment.image_repository, crate::built_info::PKG_VERSION, ) diff --git a/rust/operator-binary/src/nifi_controller.rs b/rust/operator-binary/src/nifi_controller.rs index 035b94b7..20c6d6b9 100644 --- a/rust/operator-binary/src/nifi_controller.rs +++ b/rust/operator-binary/src/nifi_controller.rs @@ -51,9 +51,6 @@ use crate::{ pub const NIFI_CONTROLLER_NAME: &str = "nificluster"; pub const NIFI_FULL_CONTROLLER_NAME: &str = concatcp!(NIFI_CONTROLLER_NAME, '.', OPERATOR_NAME); -pub(crate) const CONTAINER_IMAGE_BASE_NAME: &str = "nifi"; -pub(crate) const LOG_VOLUME_NAME: &str = "log"; - pub struct Ctx { pub client: Client, pub operator_environment: OperatorEnvironmentOptions, From f1a83617c3a784fadeb49ec26286745d5eddb846 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 13 Jun 2026 22:03:35 +0200 Subject: [PATCH 32/39] refactor: move to logging v2, add vector tests --- .../src/controller/build/config_map.rs | 17 +- .../src/controller/build/properties.rs | 4 +- .../build/properties/bootstrap_conf.rs | 2 +- .../build/properties/nifi_properties.rs | 2 +- .../{logging.rs => product_logging/mod.rs} | 50 +- .../properties/product_logging/test-vector.sh | 11 + .../product_logging/vector-test.yaml | 161 ++++++ .../properties/product_logging/vector.yaml | 175 +++++++ .../build/properties/security_properties.rs | 1 + .../controller/build/resource/statefulset.rs | 67 ++- rust/operator-binary/src/controller/mod.rs | 5 + .../src/controller/validate.rs | 59 ++- rust/operator-binary/src/crd/affinity.rs | 2 +- rust/operator-binary/src/crd/mod.rs | 13 +- rust/operator-binary/src/nifi_controller.rs | 8 - .../kuttl/smoke_v2/34-assert.yaml.j2 | 48 ++ .../kuttl/smoke_v2/35-assert.yaml.j2 | 474 +----------------- 17 files changed, 555 insertions(+), 544 deletions(-) rename rust/operator-binary/src/controller/build/properties/{logging.rs => product_logging/mod.rs} (67%) create mode 100755 rust/operator-binary/src/controller/build/properties/product_logging/test-vector.sh create mode 100644 rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml create mode 100644 rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 86cefe7a..2856b783 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -15,8 +15,8 @@ use crate::{ build::{ git_sync, properties::{ - ConfigFileName, authorizers, bootstrap_conf, logging, login_identity_providers, - nifi_properties, security_properties, state_management_xml, + ConfigFileName, authorizers, bootstrap_conf, login_identity_providers, + nifi_properties, product_logging, security_properties, state_management_xml, }, proxy_hosts, }, @@ -69,14 +69,10 @@ type Result = std::result::Result; /// resolved cluster configuration. /// /// All NiFi configuration is sourced from `cluster`. -/// -/// `vector_config` is the Vector agent config (`vector.yaml`) built by the caller (where a -/// `RoleGroupRef` is available); it is `None` when the Vector agent is disabled. pub fn build_rolegroup_config_map( cluster: &ValidatedCluster, role_group_name: &RoleGroupName, cluster_info: &KubernetesClusterInfo, - vector_config: Option, ) -> Result { tracing::debug!("building rolegroup ConfigMap"); @@ -142,12 +138,15 @@ pub fn build_rolegroup_config_map( })?, ); - if let Some(logback_config) = logging::build_logback_config(&rg.config.logging) { + if let Some(logback_config) = product_logging::build_logback_config(&rg.config.logging) { cm_builder.add_data(ConfigFileName::Logback.to_string(), logback_config); } - if let Some(vector_config) = vector_config { - cm_builder.add_data(VECTOR_CONFIG_FILE, vector_config); + if rg.config.logging.enable_vector_agent { + cm_builder.add_data( + VECTOR_CONFIG_FILE, + product_logging::vector_config_file_content(), + ); } cm_builder diff --git a/rust/operator-binary/src/controller/build/properties.rs b/rust/operator-binary/src/controller/build/properties.rs index f43a1433..d7ac86a1 100644 --- a/rust/operator-binary/src/controller/build/properties.rs +++ b/rust/operator-binary/src/controller/build/properties.rs @@ -8,9 +8,9 @@ use std::{collections::BTreeMap, fmt::Write}; pub mod authorizers; pub mod bootstrap_conf; -pub mod logging; pub mod login_identity_providers; pub mod nifi_properties; +pub mod product_logging; pub mod security_properties; pub mod state_management_xml; @@ -130,7 +130,7 @@ pub(crate) mod test_support { let nifi: v1alpha1::NifiCluster = serde_yaml::from_str(MINIMAL_NIFI_YAML).expect("invalid test YAML"); - let role_group_configs = build_role_group_configs(&nifi) + let role_group_configs = build_role_group_configs(&nifi, &None) .expect("role group configs should merge for minimal fixture"); let image = ResolvedProductImage { diff --git a/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs b/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs index 841ab67e..5ee352ac 100644 --- a/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs +++ b/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs @@ -64,7 +64,7 @@ mod tests { serde_yaml::from_str(nifi_cluster).expect("illegal test input"); let role_group_configs = - build_role_group_configs(&nifi).expect("failed to build role group configs"); + build_role_group_configs(&nifi, &None).expect("failed to build role group configs"); let rg = role_group_configs .get(&NifiRole::Node) .and_then(|groups| { diff --git a/rust/operator-binary/src/controller/build/properties/nifi_properties.rs b/rust/operator-binary/src/controller/build/properties/nifi_properties.rs index a1e318d4..e7a395df 100644 --- a/rust/operator-binary/src/controller/build/properties/nifi_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/nifi_properties.rs @@ -674,7 +674,7 @@ mod tests { "#; let nifi: v1alpha1::NifiCluster = serde_yaml::from_str(yaml).expect("invalid test YAML"); let mut role_group_configs = - build_role_group_configs(&nifi).expect("failed to build role group configs"); + build_role_group_configs(&nifi, &None).expect("failed to build role group configs"); let default_rg_name = "default" .parse::() .expect("valid role-group name"); diff --git a/rust/operator-binary/src/controller/build/properties/logging.rs b/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs similarity index 67% rename from rust/operator-binary/src/controller/build/properties/logging.rs rename to rust/operator-binary/src/controller/build/properties/product_logging/mod.rs index 32d8086c..a8a38f25 100644 --- a/rust/operator-binary/src/controller/build/properties/logging.rs +++ b/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs @@ -6,13 +6,26 @@ use stackable_operator::{ self, spec::{ContainerLogConfig, ContainerLogConfigChoice, Logging}, }, - role_utils::RoleGroupRef, }; -use crate::crd::{Container, MAX_NIFI_LOG_FILES_SIZE, STACKABLE_LOG_DIR, v1alpha1}; +use crate::crd::{Container, MAX_NIFI_LOG_FILES_SIZE, STACKABLE_LOG_DIR}; pub const NIFI_LOG_FILE: &str = "nifi.log4j.xml"; +/// The Vector agent configuration (`vector.yaml`). +/// +/// Embedded statically and shipped in the rolegroup `ConfigMap`; the per-rolegroup values +/// (namespace, cluster/role/role-group, aggregator address, log levels) are injected as +/// environment variables by the Vector container (see +/// [`stackable_operator::v2::product_logging::framework::vector_container`]) and substituted by +/// Vector at runtime. +const VECTOR_CONFIG: &str = include_str!("vector.yaml"); + +/// Returns the Vector agent config (`vector.yaml`) content. +pub fn vector_config_file_content() -> String { + VECTOR_CONFIG.to_owned() +} + const CONSOLE_CONVERSION_PATTERN: &str = "%date %level [%thread] %logger{40} %msg%n"; // This is required to remove double entries in the nifi.log4j.xml as well as nested // console output like: " ... ... @@ -59,28 +72,15 @@ pub fn build_logback_config(logging: &Logging) -> Option { )) } -/// Renders the Vector agent config (`vector.yaml`). -/// -/// Returns `None` when the Vector agent is disabled for this role group. -pub fn build_vector_config( - rolegroup: &RoleGroupRef, - logging: &Logging, -) -> Option { - if !logging.enable_vector_agent { - return None; - } - - let vector_log_config = if let Some(ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic(log_config)), - }) = logging.containers.get(&Container::Vector) - { - Some(log_config) - } else { - None - }; +#[cfg(test)] +mod tests { + use super::*; - Some(product_logging::framework::create_vector_config( - rolegroup, - vector_log_config, - )) + #[test] + fn test_vector_config_file_content() { + let content = vector_config_file_content(); + assert!(!content.is_empty()); + // NiFi logs via logback's `XMLLayout` (log4j XML), so the `files_log4j` source + assert!(content.contains("files_log4j")); + } } diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/test-vector.sh b/rust/operator-binary/src/controller/build/properties/product_logging/test-vector.sh new file mode 100755 index 00000000..6977f395 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/product_logging/test-vector.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env sh + +DATA_DIR=/stackable/log/_vector-state \ +LOG_DIR=/stackable/log \ +NAMESPACE=default \ +CLUSTER_NAME=nifi \ +ROLE_NAME=node \ +ROLE_GROUP_NAME=default \ +VECTOR_AGGREGATOR_ADDRESS=vector-aggregator \ +VECTOR_FILE_LOG_LEVEL=info \ +vector test vector.yaml vector-test.yaml diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml new file mode 100644 index 00000000..bc0ec026 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml @@ -0,0 +1,161 @@ +# Run tests with `./test-vector.sh` +# +# A downside of these test cases is that they compare the whole event and that the message can +# contain source code positions in vector.yaml, e.g. "function call error for \"parse_xml\" at +# (584:643)". Please adapt the tests if you change VRL code in vector.yaml. +--- +tests: + - name: Test stdout log entry + inputs: + - type: log + insert_at: processed_files_stdout + log_fields: + file: /stackable/log/nifi/nifi.stdout.log + message: Starting Apache NiFi + pod: nifi-node-default-0 + source_type: file + timestamp: 2025-10-02T09:27:28.582Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "nifi", + "container": "nifi", + "file": "nifi.stdout.log", + "level": "INFO", + "logger": "ROOT", + "message": "Starting Apache NiFi", + "namespace": "default", + "pod": "nifi-node-default-0", + "role": "node", + "roleGroup": "default", + "timestamp": "2025-10-02T09:27:28.582Z" + } + + assert_eq!(expected_log_event, .) + - name: Test stderr log entry + inputs: + - type: log + insert_at: processed_files_stderr + log_fields: + file: /stackable/log/nifi/nifi.stderr.log + message: "Exception in thread \"main\"" + pod: nifi-node-default-0 + source_type: file + timestamp: 2025-10-02T09:27:28.582Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "nifi", + "container": "nifi", + "file": "nifi.stderr.log", + "level": "ERROR", + "logger": "ROOT", + "message": "Exception in thread \"main\"", + "namespace": "default", + "pod": "nifi-node-default-0", + "role": "node", + "roleGroup": "default", + "timestamp": "2025-10-02T09:27:28.582Z" + } + + assert_eq!(expected_log_event, .) + - name: Test log4j XML log entry without throwable + inputs: + - type: log + insert_at: processed_files_log4j + log_fields: + file: /stackable/log/nifi/nifi.log4j.xml + message: > + NiFi has + started + pod: nifi-node-default-0 + source_type: file + timestamp: 2025-10-02T09:27:29.473487331Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "nifi", + "container": "nifi", + "file": "nifi.log4j.xml", + "level": "INFO", + "logger": "org.apache.nifi.web.server.JettyServer", + "message": "NiFi has started", + "namespace": "default", + "pod": "nifi-node-default-0", + "role": "node", + "roleGroup": "default", + "timestamp": t'2024-01-01T00:00:00Z' + } + + assert_eq!(expected_log_event, .) + - name: Test Vector internal logs + inputs: + - type: log + insert_at: filtered_logs_vector + log_fields: + arch: x86_64 + message: Vector has started. + metadata: + kind: event + level: INFO + module_path: vector::internal_events::process + target: vector + pid: 14 + pod: nifi-node-default-0 + source_type: internal_logs + timestamp: 2025-10-02T09:46:14.479381097Z + version: 0.49.0 + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "arch": "x86_64", + "cluster": "nifi", + "container": "vector", + "level": "INFO", + "logger": "vector::internal_events::process", + "message": "Vector has started.", + "namespace": "default", + "pod": "nifi-node-default-0", + "role": "node", + "roleGroup": "default", + "timestamp": "2025-10-02T09:46:14.479381097Z", + "version": "0.49.0" + } + + assert_eq!(expected_log_event, .) + - name: Test Vector internal log level filtering - INFO passes + inputs: + - type: log + insert_at: filtered_logs_vector + log_fields: + metadata: + level: INFO + outputs: + - extract_from: filtered_logs_vector + conditions: + - type: vrl + source: | + assert_eq!("INFO", .metadata.level) + - name: Test Vector internal log level filtering - DEBUG dropped + inputs: + - type: log + insert_at: filtered_logs_vector + log_fields: + metadata: + level: DEBUG + no_outputs_from: + - filtered_logs_vector diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml new file mode 100644 index 00000000..cae311b7 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml @@ -0,0 +1,175 @@ +--- +data_dir: ${DATA_DIR} + +log_schema: + host_key: pod + +sources: + # Reads the internal Vector logs + vector: + type: internal_logs + + files_stdout: + type: file + include: + - ${LOG_DIR}/*/*.stdout.log + + files_stderr: + type: file + include: + - ${LOG_DIR}/*/*.stderr.log + + # NiFi logs via logback's `XMLLayout`, which emits log4j `` XML. + files_log4j: + type: file + include: + - ${LOG_DIR}/*/*.log4j.xml + line_delimiter: "\r\n" + multiline: + mode: halt_before + start_pattern: ^" + raw_message + "" + parsed_event, err = parse_xml(wrapped_xml_event) + if err != null { + error = "XML not parsable: " + err + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + root = object!(parsed_event.root) + if !is_object(root.event) { + error = "Parsed event contains no \"event\" tag." + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + if keys(root) != ["event"] { + .errors = push(.errors, "Parsed event contains multiple tags: " + join!(keys(root), ", ")) + } + event = object!(root.event) + + epoch_milliseconds, err = to_int(event.@timestamp) + if err == null && epoch_milliseconds != 0 { + converted_timestamp, err = from_unix_timestamp(epoch_milliseconds, "milliseconds") + if err == null { + .timestamp = converted_timestamp + } else { + .errors = push(.errors, "Time not parsable, using current time instead: " + err) + } + } else { + .errors = push(.errors, "Timestamp not found, using current time instead.") + } + + .logger, err = string(event.@logger) + if err != null || is_empty(.logger) { + .errors = push(.errors, "Logger not found.") + } + + level, err = string(event.@level) + if err != null { + .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") + } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], level) { + .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") + } else { + .level = level + } + + message, err = string(event.message) + if err != null || is_empty(message) { + .errors = push(.errors, "Message not found.") + } + throwable = string(event.throwable) ?? "" + .message = join!(compact([message, throwable]), "\n") + } + } + + # Extends the processed files with the fields "container" and "file" + extended_logs_files: + inputs: + - processed_files_* + type: remap + source: | + del(.source_type) + if .errors == [] { + del(.errors) + } + . |= parse_regex!(.file, r'^${LOG_DIR}/(?P.*?)/(?P.*?)$') + + # Filters the logs of the Vector agent according to the defined log level + filtered_logs_vector: + inputs: + - vector + type: filter + condition: > + (.metadata.level == "TRACE" && "${VECTOR_FILE_LOG_LEVEL}" == "trace") || + (.metadata.level == "DEBUG" && includes(["trace", "debug"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "INFO" && includes(["trace", "debug", "info"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "WARN" && includes(["trace", "debug", "info", "warn"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "ERROR" && includes(["trace", "debug", "info", "warn", "error"], "${VECTOR_FILE_LOG_LEVEL}")) + + # Aligns the logs of the Vector agent with the common format + extended_logs_vector: + inputs: + - filtered_logs_vector + type: remap + source: | + .container = "vector" + .level = .metadata.level + .logger = .metadata.module_path + if exists(.file) { .processed_file = del(.file) } + del(.metadata) + del(.pid) + del(.source_type) + + # Add the fields "namespace", "cluster", "role" and "roleGroup" to all logs + extended_logs: + inputs: + - extended_logs_* + type: remap + source: | + .namespace = "${NAMESPACE}" + .cluster = "${CLUSTER_NAME}" + .role = "${ROLE_NAME}" + .roleGroup = "${ROLE_GROUP_NAME}" + +sinks: + # Forward the logs to the Vector aggregator + aggregator: + inputs: + - extended_logs + type: vector + address: ${VECTOR_AGGREGATOR_ADDRESS} diff --git a/rust/operator-binary/src/controller/build/properties/security_properties.rs b/rust/operator-binary/src/controller/build/properties/security_properties.rs index 8ea273a8..45fc983b 100644 --- a/rust/operator-binary/src/controller/build/properties/security_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/security_properties.rs @@ -48,6 +48,7 @@ mod tests { env_overrides: EnvVarSet::new(), pod_overrides: Default::default(), product_specific_common_config: JavaCommonConfig::default(), + vector_container: None, } } diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index c7bc4d23..2ffb75ac 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -1,6 +1,6 @@ //! Builds the rolegroup [`StatefulSet`] that runs a NiFi node role group. -use std::collections::BTreeMap; +use std::{collections::BTreeMap, str::FromStr}; use indoc::formatdoc; use snafu::{OptionExt, ResultExt, Snafu}; @@ -32,16 +32,21 @@ use stackable_operator::{ memory::{BinaryMultiple, MemoryQuantity}, product_logging::{ self, - framework::{ - LoggingError, create_vector_shutdown_file_command, remove_vector_shutdown_file_command, - }, + framework::{create_vector_shutdown_file_command, remove_vector_shutdown_file_command}, spec::{ ConfigMapLogConfig, ContainerLogConfig, ContainerLogConfigChoice, CustomContainerLogConfig, }, }, utils::{COMMON_BASH_TRAP_FUNCTIONS, cluster_info::KubernetesClusterInfo}, - v2::{builder::meta::ownerreference_from_resource, types::operator::RoleGroupName}, + v2::{ + builder::{meta::ownerreference_from_resource, pod::container::EnvVarSet}, + product_logging::framework::vector_container, + types::{ + kubernetes::{ContainerName, VolumeName}, + operator::RoleGroupName, + }, + }, }; use crate::{ @@ -93,9 +98,6 @@ pub enum Error { container_name: String, }, - #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] - VectorAggregatorConfigMapMissing, - #[snafu(display("failed to add Authentication Volumes and VolumeMounts"))] AddAuthVolumes { source: crate::security::authentication::Error, @@ -104,9 +106,6 @@ pub enum Error { #[snafu(display("security failure"))] Security { source: crate::security::Error }, - #[snafu(display("failed to configure logging"))] - ConfigureLogging { source: LoggingError }, - #[snafu(display("failed to add needed volume"))] AddVolume { source: builder::pod::Error }, @@ -153,6 +152,14 @@ const LOG_CONFIG_VOLUME_NAME: &str = "log-config"; /// git-sync container, see [`crate::controller::build::git_sync`]). pub(crate) const LOG_VOLUME_NAME: &str = "log"; +stackable_operator::constant!(VECTOR_CONTAINER_NAME: ContainerName = "vector"); + +// Typed `VolumeName`s for the Vector container's log-config and log volumes. They reuse the +// existing rolegroup-`ConfigMap` "config" volume (which carries `vector.yaml`) and the "log" +// empty-dir, both already added to the pod by `build_node_rolegroup_statefulset`. +stackable_operator::constant!(VECTOR_LOG_CONFIG_VOLUME_NAME: VolumeName = "config"); +stackable_operator::constant!(VECTOR_LOG_VOLUME_NAME: VolumeName = "log"); + /// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. /// /// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the @@ -542,30 +549,20 @@ pub(crate) async fn build_node_rolegroup_statefulset( .context(AddVolumeSnafu)?; } - if merged_config.logging.enable_vector_agent { - match &nifi.spec.cluster_config.vector_aggregator_config_map_name { - Some(vector_aggregator_config_map_name) => { - pod_builder.add_container( - product_logging::framework::vector_container( - resolved_product_image, - "config", - LOG_VOLUME_NAME, - merged_config.logging.containers.get(&Container::Vector), - ResourceRequirementsBuilder::new() - .with_cpu_request("250m") - .with_cpu_limit("500m") - .with_memory_request("128Mi") - .with_memory_limit("128Mi") - .build(), - vector_aggregator_config_map_name, - ) - .context(ConfigureLoggingSnafu)?, - ); - } - None => { - VectorAggregatorConfigMapMissingSnafu.fail()?; - } - } + // The Vector logging config was validated up-front in the `validate` step. The static + // `vector.yaml` is shipped in the rolegroup `ConfigMap`; the per-rolegroup values (namespace, + // cluster/role/role-group, aggregator address, log levels) are injected as environment + // variables here and substituted by Vector at runtime. + if let Some(vector_log_config) = &rg.vector_container { + pod_builder.add_container(vector_container( + &VECTOR_CONTAINER_NAME, + resolved_product_image, + vector_log_config, + &resource_names, + &VECTOR_LOG_CONFIG_VOLUME_NAME, + &VECTOR_LOG_VOLUME_NAME, + EnvVarSet::new(), + )); } authentication_config diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index a464f190..e81623f1 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -14,6 +14,7 @@ use stackable_operator::{ HasName, HasUid, NameIsValidLabelValue, builder::pod::container::EnvVarSet, kvp::label::{recommended_labels, role_group_selector, role_selector}, + product_logging::framework::VectorContainerLogConfig, role_group_utils::ResourceNames, role_utils::JavaCommonConfig, types::{ @@ -68,6 +69,10 @@ pub struct ValidatedRoleGroupConfig { /// The merged (role <- role group) JVM argument overrides, applied on top of the /// operator-generated JVM arguments when building `bootstrap.conf`. pub product_specific_common_config: JavaCommonConfig, + /// The validated Vector container logging config (log config choice + aggregator discovery + /// ConfigMap name), validated up-front in the [`validate`] step. `None` when the Vector agent + /// is disabled for this role group. + pub vector_container: Option, } /// The validated NifiCluster: everything `reconcile_nifi` needs after dereferencing, diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index bd0fc6ba..57346e2c 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -15,8 +15,14 @@ use stackable_operator::{ v2::{ builder::pod::container::{self, EnvVarName, EnvVarSet}, controller_utils::{self, get_cluster_name, get_uid}, + product_logging::framework::{ + VectorContainerLogConfig, validate_logging_configuration_for_container, + }, role_utils::with_validated_config, - types::operator::{ProductVersion, RoleGroupName}, + types::{ + kubernetes::ConfigMapName, + operator::{ProductVersion, RoleGroupName}, + }, }, }; use strum::{EnumDiscriminants, IntoStaticStr}; @@ -24,7 +30,7 @@ use strum::{EnumDiscriminants, IntoStaticStr}; use super::{ValidatedCluster, ValidatedClusterConfig, ValidatedRoleGroupConfig}; use crate::{ controller::dereference::DereferencedObjects, - crd::{NifiConfig, NifiRole, sensitive_properties, v1alpha1}, + crd::{Container, NifiConfig, NifiRole, sensitive_properties, v1alpha1}, security::{ authentication::{self, NifiAuthenticationConfig}, authorization::ResolvedNifiAuthorizationConfig, @@ -72,6 +78,21 @@ pub enum Error { #[snafu(display("invalid sensitive properties algorithm"))] InvalidSensitivePropertiesAlgorithm { source: sensitive_properties::Error }, + + #[snafu(display("invalid Vector aggregator discovery ConfigMap name"))] + ParseVectorAggregatorConfigMapName { + source: stackable_operator::v2::macros::attributed_string_type::Error, + }, + + #[snafu(display( + "the Vector aggregator discovery ConfigMap name is required when the Vector agent is enabled" + ))] + MissingVectorAggregatorConfigMapName, + + #[snafu(display("failed to validate logging configuration"))] + ValidateLoggingConfig { + source: stackable_operator::v2::product_logging::framework::Error, + }, } type Result = std::result::Result; @@ -112,7 +133,18 @@ pub fn validate( .check_for_nifi_version(&image.product_version) .context(InvalidSensitivePropertiesAlgorithmSnafu)?; - let role_group_configs = build_role_group_configs(nifi)?; + // The Vector aggregator discovery ConfigMap name is validated here so an invalid name fails + // up-front. It is only required when the Vector agent is enabled for a role group. + let vector_aggregator_config_map_name = nifi + .spec + .cluster_config + .vector_aggregator_config_map_name + .as_deref() + .map(ConfigMapName::from_str) + .transpose() + .context(ParseVectorAggregatorConfigMapNameSnafu)?; + + let role_group_configs = build_role_group_configs(nifi, &vector_aggregator_config_map_name)?; let name = get_cluster_name(nifi).context(GetClusterNameSnafu)?; let namespace = dereferenced_objects.namespace.clone(); @@ -143,6 +175,7 @@ pub fn validate( pub(crate) fn build_role_group_configs( nifi: &v1alpha1::NifiCluster, + vector_aggregator_config_map_name: &Option, ) -> Result>> { let role = nifi.spec.nodes.as_ref().context(NoNodesDefinedSnafu)?; let default_config = NifiConfig::default_config(&nifi.name_any(), &NifiRole::Node); @@ -176,6 +209,25 @@ pub(crate) fn build_role_group_configs( ); } + // Validate the Vector container logging config up-front (mirroring the opensearch- and + // hive-operators) so an invalid log ConfigMap name, or a missing aggregator discovery + // ConfigMap name, fails before any resources are built. + let vector_container = if config.logging.enable_vector_agent { + let vector_aggregator_config_map_name = vector_aggregator_config_map_name + .clone() + .context(MissingVectorAggregatorConfigMapNameSnafu)?; + Some(VectorContainerLogConfig { + log_config: validate_logging_configuration_for_container( + &config.logging, + &Container::Vector, + ) + .context(ValidateLoggingConfigSnafu)?, + vector_aggregator_config_map_name, + }) + } else { + None + }; + groups.insert( role_group_name, ValidatedRoleGroupConfig { @@ -185,6 +237,7 @@ pub(crate) fn build_role_group_configs( env_overrides: env_overrides_set, pod_overrides, product_specific_common_config, + vector_container, }, ); } diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 201c91af..06e1fa3b 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -60,7 +60,7 @@ mod tests { let nifi: v1alpha1::NifiCluster = serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); - let role_group_configs = build_role_group_configs(&nifi).unwrap(); + let role_group_configs = build_role_group_configs(&nifi, &None).unwrap(); let merged_config = &role_group_configs .get(&NifiRole::Node) .and_then(|groups| { diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 7c4f0522..af09a386 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -26,10 +26,10 @@ use stackable_operator::{ api::core::v1::{PodTemplateSpec, Volume}, apimachinery::pkg::api::resource::Quantity, }, - kube::{CustomResource, runtime::reflector::ObjectRef}, + kube::CustomResource, memory::MemoryQuantity, product_logging::{self, spec::Logging}, - role_utils::{GenericRoleConfig, Role, RoleGroupRef}, + role_utils::{GenericRoleConfig, Role}, schemars::{self, JsonSchema}, shared::time::Duration, status::condition::{ClusterCondition, HasStatusCondition}, @@ -200,15 +200,6 @@ impl HasStatusCondition for v1alpha1::NifiCluster { } impl v1alpha1::NifiCluster { - /// Metadata about a Node rolegroup - pub fn node_rolegroup_ref(&self, group_name: impl Into) -> RoleGroupRef { - RoleGroupRef { - cluster: ObjectRef::from_obj(self), - role: NifiRole::Node.to_string(), - role_group: group_name.into(), - } - } - pub fn role_config(&self, role: &NifiRole) -> Option<&NifiNodeRoleConfig> { match role { NifiRole::Node => self.spec.nodes.as_ref().map(|n| &n.role_config), diff --git a/rust/operator-binary/src/nifi_controller.rs b/rust/operator-binary/src/nifi_controller.rs index 20c6d6b9..864cfa78 100644 --- a/rust/operator-binary/src/nifi_controller.rs +++ b/rust/operator-binary/src/nifi_controller.rs @@ -305,18 +305,10 @@ pub async fn reconcile_nifi( let role = nifi.spec.nodes.as_ref().context(NoNodesDefinedSnafu)?; - // The Vector agent config is the only remaining consumer of `RoleGroupRef`, which is - // therefore constructed on demand here rather than threaded through the build steps. - let vector_config = build::properties::logging::build_vector_config( - &nifi.node_rolegroup_ref(role_group_name.to_string()), - &rg.config.logging, - ); - let rg_configmap = build::config_map::build_rolegroup_config_map( &validated_cluster, role_group_name, &client.kubernetes_cluster_info, - vector_config, ) .context(BuildRoleGroupConfigMapSnafu { rolegroup: role_group_name.clone(), diff --git a/tests/templates/kuttl/smoke_v2/34-assert.yaml.j2 b/tests/templates/kuttl/smoke_v2/34-assert.yaml.j2 index 83a20c2e..c7fa1be4 100644 --- a/tests/templates/kuttl/smoke_v2/34-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke_v2/34-assert.yaml.j2 @@ -49,6 +49,54 @@ spec: app.kubernetes.io/role-group: default stackable.tech/vendor: Stackable spec: + containers: + - name: nifi +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + # The Vector sidecar is configured via environment variables injected by the v2 + # `vector_container` helper; `vector.yaml` references these and they are substituted by + # Vector at runtime. + - name: vector + env: + - name: CLUSTER_NAME + value: nifi + - name: DATA_DIR + value: /stackable/log/_vector-state + - name: LOG_DIR + value: /stackable/log + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ROLE_GROUP_NAME + value: default + - name: ROLE_NAME + value: node + - name: VECTOR_AGGREGATOR_ADDRESS + valueFrom: + configMapKeyRef: + key: ADDRESS + name: vector-aggregator-discovery + - name: VECTOR_CONFIG_YAML + value: /stackable/config/vector.yaml + - name: VECTOR_FILE_LOG_LEVEL + value: info + - name: VECTOR_LOG + value: info + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 250m + memory: 128Mi + volumeMounts: + - mountPath: /stackable/config/vector.yaml + name: config + readOnly: true + subPath: vector.yaml + - mountPath: /stackable/log + name: log +{% endif %} serviceAccount: nifi-serviceaccount serviceAccountName: nifi-serviceaccount terminationGracePeriodSeconds: 300 diff --git a/tests/templates/kuttl/smoke_v2/35-assert.yaml.j2 b/tests/templates/kuttl/smoke_v2/35-assert.yaml.j2 index 88d3b24b..b29058e9 100644 --- a/tests/templates/kuttl/smoke_v2/35-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke_v2/35-assert.yaml.j2 @@ -19,7 +19,7 @@ kind: TestAssert timeout: 60 commands: - script: | - expected=$(cat <<'YAMLEOF' | envsubst '$NAMESPACE' | yq -o=json + expected=$(cat <<'YAMLEOF' | yq -o=json authorizers.xml: | @@ -265,29 +265,32 @@ commands: {% if lookup('env', 'VECTOR_AGGREGATOR') %} vector.yaml: | - data_dir: /stackable/log/_vector-state + --- + data_dir: ${DATA_DIR} log_schema: host_key: pod sources: + # Reads the internal Vector logs vector: type: internal_logs files_stdout: type: file include: - - /stackable/log/*/*.stdout.log + - ${LOG_DIR}/*/*.stdout.log files_stderr: type: file include: - - /stackable/log/*/*.stderr.log + - ${LOG_DIR}/*/*.stderr.log + # NiFi logs via logback's `XMLLayout`, which emits log4j `` XML. files_log4j: type: file include: - - /stackable/log/*/*.log4j.xml + - ${LOG_DIR}/*/*.log4j.xml line_delimiter: "\r\n" multiline: mode: halt_before @@ -295,181 +298,7 @@ commands: condition_pattern: ^ |_index, value| { - stacktrace = stacktrace + " " - class = string(value.@class) ?? "" - method = string(value.@method) ?? "" - if !is_empty(class) && !is_empty(method) { - stacktrace = stacktrace + "at " + class + "." + method - } - file = string(value.@file) ?? "" - line = string(value.@line) ?? "" - if !is_empty(file) && !is_empty(line) { - stacktrace = stacktrace + "(" + file + ":" + line + ")" - } - exact = to_bool(value.@exact) ?? false - location = string(value.@location) ?? "" - version = string(value.@version) ?? "" - if !is_empty(location) && !is_empty(version) { - stacktrace = stacktrace + " " - if !exact { - stacktrace = stacktrace + "~" - } - stacktrace = stacktrace + "[" + location + ":" + version + "]" - } - stacktrace = stacktrace + "\n" - } - if stacktrace != "" { - exception = exception + "\n" + stacktrace - } - } - - message, err = string(event.Message) - if err != null || is_empty(message) { - message = null - .errors = push(.errors, "Message not found.") - } - .message = join!(compact([message, exception]), "\n") - } - } - - processed_files_py: - inputs: - - files_py - type: remap - source: | - raw_message = string!(.message) - - .timestamp = now() - .logger = "" - .level = "INFO" - .message = "" - .errors = [] - - parsed_event, err = parse_json(raw_message) - if err != null { - error = "JSON not parsable: " + err - .errors = push(.errors, error) - log(error, level: "warn") - .message = raw_message - } else if !is_object(parsed_event) { - error = "Parsed event is not a JSON object." - .errors = push(.errors, error) - log(error, level: "warn") - .message = raw_message - } else { - event = object!(parsed_event) - - asctime, err = string(event.asctime) - if err == null { - parsed_timestamp, err = parse_timestamp(asctime, "%F %T,%3f") - if err == null { - .timestamp = parsed_timestamp - } else { - .errors = push(.errors, "Timestamp not parsable, using current time instead: "+ err) - } - } else { - .errors = push(.errors, "Timestamp not found, using current time instead.") - } - - .logger, err = string(event.name) - if err != null || is_empty(.logger) { - .errors = push(.errors, "Logger not found.") - } - - level, err = string(event.levelname) - if err != null { - .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") - } else if level == "DEBUG" { - .level = "DEBUG" - } else if level == "INFO" { - .level = "INFO" - } else if level == "WARNING" { - .level = "WARN" - } else if level == "ERROR" { - .level = "ERROR" - } else if level == "CRITICAL" { - .level = "FATAL" - } else { - .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") - } - - .message, err = string(event.message) - if err != null || is_empty(.message) { - .errors = push(.errors, "Message not found.") - } - } - - processed_files_airlift: - inputs: - - files_airlift - type: remap - source: | - raw_message = string!(.message) - - .timestamp = now() - .logger = "" - .level = "INFO" - .message = "" - .errors = [] - - parsed_event, err = parse_json(raw_message) - if err != null { - error = "JSON not parsable: " + err - .errors = push(.errors, error) - log(error, level: "warn") - .message = raw_message - } else if !is_object(parsed_event) { - error = "Parsed event is not a JSON object." - .errors = push(.errors, error) - log(error, level: "warn") - .message = raw_message - } else { - event = object!(parsed_event) - - timestamp_string, err = string(event.timestamp) - if err == null { - parsed_timestamp, err = parse_timestamp(timestamp_string, "%Y-%m-%dT%H:%M:%S.%fZ") - if err == null { - .timestamp = parsed_timestamp - } else { - .errors = push(.errors, "Timestamp not parsable, using current time instead: " + err) - } - } else { - .errors = push(.errors, "Timestamp not found, using current time instead.") - } - - .logger, err = string(event.logger) - if err != null || is_empty(.logger) { - .errors = push(.errors, "Logger not found.") - } - - level, err = string(event.level) - if err != null { - .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") - } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], level) { - .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") - } else { - .level = level - } - - .thread = string(parsed_event.thread) ?? null - - .message, err = string(event.message) - if err != null || is_empty(.message) { - .errors = push(.errors, "Message not found.") - } - stacktrace = string(event.stackTrace) ?? "" - .message = join!(compact([.message, stacktrace]), "\n\n") - } - + # Extends the processed files with the fields "container" and "file" extended_logs_files: inputs: - processed_files_* @@ -825,14 +394,21 @@ commands: if .errors == [] { del(.errors) } - . |= parse_regex!(.file, r'^/stackable/log/(?P.*?)/(?P.*?)$') + . |= parse_regex!(.file, r'^${LOG_DIR}/(?P.*?)/(?P.*?)$') + # Filters the logs of the Vector agent according to the defined log level filtered_logs_vector: inputs: - vector type: filter - condition: '!includes(["TRACE", "DEBUG"], .metadata.level)' - + condition: > + (.metadata.level == "TRACE" && "${VECTOR_FILE_LOG_LEVEL}" == "trace") || + (.metadata.level == "DEBUG" && includes(["trace", "debug"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "INFO" && includes(["trace", "debug", "info"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "WARN" && includes(["trace", "debug", "info", "warn"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "ERROR" && includes(["trace", "debug", "info", "warn", "error"], "${VECTOR_FILE_LOG_LEVEL}")) + + # Aligns the logs of the Vector agent with the common format extended_logs_vector: inputs: - filtered_logs_vector @@ -846,22 +422,24 @@ commands: del(.pid) del(.source_type) + # Add the fields "namespace", "cluster", "role" and "roleGroup" to all logs extended_logs: inputs: - extended_logs_* type: remap source: | - .namespace = "$NAMESPACE" - .cluster = "nifi" - .role = "node" - .roleGroup = "default" + .namespace = "${NAMESPACE}" + .cluster = "${CLUSTER_NAME}" + .role = "${ROLE_NAME}" + .roleGroup = "${ROLE_GROUP_NAME}" sinks: + # Forward the logs to the Vector aggregator aggregator: inputs: - extended_logs type: vector - address: $VECTOR_AGGREGATOR_ADDRESS + address: ${VECTOR_AGGREGATOR_ADDRESS} {% endif %} YAMLEOF ) From fb8acfe0af2570e839621e194dda9d936961d368 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 13 Jun 2026 22:21:55 +0200 Subject: [PATCH 33/39] refactor: move to v2 types, clusterresources, pdbs, container builders --- .../src/controller/build/resource/pdb.rs | 57 ++++++------------- .../build/resource/reporting_task.rs | 28 ++++----- .../controller/build/resource/statefulset.rs | 34 ++++------- rust/operator-binary/src/controller/mod.rs | 6 +- rust/operator-binary/src/nifi_controller.rs | 45 +++++++-------- 5 files changed, 65 insertions(+), 105 deletions(-) diff --git a/rust/operator-binary/src/controller/build/resource/pdb.rs b/rust/operator-binary/src/controller/build/resource/pdb.rs index 66156889..f11eb102 100644 --- a/rust/operator-binary/src/controller/build/resource/pdb.rs +++ b/rust/operator-binary/src/controller/build/resource/pdb.rs @@ -1,61 +1,36 @@ -use snafu::{ResultExt, Snafu}; use stackable_operator::{ - builder::pdb::PodDisruptionBudgetBuilder, client::Client, cluster_resources::ClusterResources, - commons::pdb::PdbConfig, kube::ResourceExt, + commons::pdb::PdbConfig, k8s_openapi::api::policy::v1::PodDisruptionBudget, + v2::builder::pdb::pod_disruption_budget_builder_with_role, }; use crate::{ - OPERATOR_NAME, - crd::{APP_NAME, NifiRole, v1alpha1}, - nifi_controller::NIFI_CONTROLLER_NAME, + controller::{ValidatedCluster, controller_name, operator_name, product_name}, + crd::NifiRole, }; -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("Cannot create PodDisruptionBudget for role [{role}]"))] - CreatePdb { - source: stackable_operator::builder::pdb::Error, - role: String, - }, - #[snafu(display("Cannot apply PodDisruptionBudget [{name}]"))] - ApplyPdb { - source: stackable_operator::cluster_resources::Error, - name: String, - }, -} - -pub async fn add_pdbs( +/// Builds the [`PodDisruptionBudget`] for the given `role`, or `None` if PDBs are disabled. +pub fn build_pdb( pdb: &PdbConfig, - nifi: &v1alpha1::NifiCluster, + cluster: &ValidatedCluster, role: &NifiRole, - client: &Client, - cluster_resources: &mut ClusterResources<'_>, -) -> Result<(), Error> { +) -> Option { if !pdb.enabled { - return Ok(()); + return None; } let max_unavailable = pdb.max_unavailable.unwrap_or(match role { NifiRole::Node => max_unavailable_nodes(), }); - let pdb = PodDisruptionBudgetBuilder::new_with_role( - nifi, - APP_NAME, - &role.to_string(), - OPERATOR_NAME, - NIFI_CONTROLLER_NAME, + let pdb = pod_disruption_budget_builder_with_role( + cluster, + &product_name(), + &ValidatedCluster::role_name(), + &operator_name(), + &controller_name(), ) - .with_context(|_| CreatePdbSnafu { - role: role.to_string(), - })? .with_max_unavailable(max_unavailable) .build(); - let pdb_name = pdb.name_any(); - cluster_resources - .add(client, pdb) - .await - .with_context(|_| ApplyPdbSnafu { name: pdb_name })?; - Ok(()) + Some(pdb) } fn max_unavailable_nodes() -> u16 { diff --git a/rust/operator-binary/src/controller/build/resource/reporting_task.rs b/rust/operator-binary/src/controller/build/resource/reporting_task.rs index d7a77162..251f5359 100644 --- a/rust/operator-binary/src/controller/build/resource/reporting_task.rs +++ b/rust/operator-binary/src/controller/build/resource/reporting_task.rs @@ -30,7 +30,7 @@ use stackable_operator::{ self, meta::ObjectMetaBuilder, pod::{ - PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, + PodBuilder, resources::ResourceRequirementsBuilder, security::PodSecurityContextBuilder, volume::SecretFormat, }, }, @@ -46,8 +46,11 @@ use stackable_operator::{ shared::time::Duration, utils::cluster_info::KubernetesClusterInfo, v2::{ - builder::meta::ownerreference_from_resource, - types::{kubernetes::NamespaceName, operator::RoleGroupName}, + builder::{meta::ownerreference_from_resource, pod::container::new_container_builder}, + types::{ + kubernetes::{ContainerName, NamespaceName}, + operator::RoleGroupName, + }, }, }; @@ -62,19 +65,13 @@ use crate::{ const REPORTING_TASK_CERT_VOLUME_NAME: &str = "tls"; const REPORTING_TASK_CERT_VOLUME_MOUNT: &str = "/stackable/cert"; -const REPORTING_TASK_CONTAINER_NAME: &str = "reporting-task"; +stackable_operator::constant!(REPORTING_TASK_CONTAINER_NAME: ContainerName = "reporting-task"); #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("object defines no name"))] ObjectHasNoName, - #[snafu(display("illegal container name: [{container_name}]"))] - IllegalContainerName { - source: stackable_operator::builder::pod::container::Error, - container_name: String, - }, - #[snafu(display("failed to add Authentication Volumes and VolumeMounts"))] AddAuthVolumes { source: crate::security::authentication::Error, @@ -147,7 +144,10 @@ fn reporting_task_role_group() -> RoleGroupName { /// Return the name of the reporting task Service. pub fn build_reporting_task_service_name(nifi_cluster_name: &str) -> String { - format!("{nifi_cluster_name}-{REPORTING_TASK_CONTAINER_NAME}") + format!( + "{nifi_cluster_name}-{container}", + container = &*REPORTING_TASK_CONTAINER_NAME + ) } /// Return the FQDN (with namespace, domain) of the reporting task. @@ -284,11 +284,7 @@ fn build_reporting_task_job( format!("-m {METRICS_PORT}"), format!("-c {REPORTING_TASK_CERT_VOLUME_MOUNT}/ca.crt"), ]; - let mut cb = ContainerBuilder::new(REPORTING_TASK_CONTAINER_NAME).with_context(|_| { - IllegalContainerNameSnafu { - container_name: REPORTING_TASK_CONTAINER_NAME.to_string(), - } - })?; + let mut cb = new_container_builder(&REPORTING_TASK_CONTAINER_NAME); cb.image_from_product_image(resolved_product_image) .command(vec!["sh".to_string(), "-c".to_string()]) .args(vec![args.join(" ")]) diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 2ffb75ac..c41aa28a 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -9,7 +9,7 @@ use stackable_operator::{ self, meta::ObjectMetaBuilder, pod::{ - PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, + PodBuilder, resources::ResourceRequirementsBuilder, security::PodSecurityContextBuilder, volume::SecretFormat, }, }, @@ -40,7 +40,10 @@ use stackable_operator::{ }, utils::{COMMON_BASH_TRAP_FUNCTIONS, cluster_info::KubernetesClusterInfo}, v2::{ - builder::{meta::ownerreference_from_resource, pod::container::EnvVarSet}, + builder::{ + meta::ownerreference_from_resource, + pod::container::{EnvVarSet, new_container_builder}, + }, product_logging::framework::vector_container, types::{ kubernetes::{ContainerName, VolumeName}, @@ -92,12 +95,6 @@ pub enum Error { #[snafu(display("object defines no name"))] ObjectHasNoName, - #[snafu(display("illegal container name: [{container_name}]"))] - IllegalContainerName { - source: stackable_operator::builder::pod::container::Error, - container_name: String, - }, - #[snafu(display("failed to add Authentication Volumes and VolumeMounts"))] AddAuthVolumes { source: crate::security::authentication::Error, @@ -152,6 +149,10 @@ const LOG_CONFIG_VOLUME_NAME: &str = "log-config"; /// git-sync container, see [`crate::controller::build::git_sync`]). pub(crate) const LOG_VOLUME_NAME: &str = "log"; +// Container names. These must match the corresponding (kebab-cased) `crate::crd::Container` +// variants, which key the per-container logging config. +stackable_operator::constant!(PREPARE_CONTAINER_NAME: ContainerName = "prepare"); +stackable_operator::constant!(NIFI_CONTAINER_NAME: ContainerName = "nifi"); stackable_operator::constant!(VECTOR_CONTAINER_NAME: ContainerName = "vector"); // Typed `VolumeName`s for the Vector container's log-config and log volumes. They reuse the @@ -256,7 +257,7 @@ pub(crate) async fn build_node_rolegroup_statefulset( let sensitive_key_secret = &nifi.spec.cluster_config.sensitive_properties.key_secret; - let prepare_container_name = Container::Prepare.to_string(); + let prepare_container_name = PREPARE_CONTAINER_NAME.to_string(); let mut prepare_args = vec![]; if let Some(ContainerLogConfig { @@ -328,12 +329,7 @@ pub(crate) async fn build_node_rolegroup_statefulset( "config-utils template /stackable/nifi/conf/security.properties".to_string(), ]); - let mut container_prepare = - ContainerBuilder::new(&prepare_container_name).with_context(|_| { - IllegalContainerNameSnafu { - container_name: prepare_container_name.to_string(), - } - })?; + let mut container_prepare = new_container_builder(&PREPARE_CONTAINER_NAME); container_prepare .image_from_product_image(resolved_product_image) @@ -379,13 +375,7 @@ pub(crate) async fn build_node_rolegroup_statefulset( .build(), ); - let nifi_container_name = Container::Nifi.to_string(); - let mut container_nifi_builder = - ContainerBuilder::new(&nifi_container_name).with_context(|_| { - IllegalContainerNameSnafu { - container_name: nifi_container_name, - } - })?; + let mut container_nifi_builder = new_container_builder(&NIFI_CONTAINER_NAME); let nifi_args = vec![formatdoc! {" {COMMON_BASH_TRAP_FUNCTIONS} diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index e81623f1..1b608201 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -214,17 +214,17 @@ impl ValidatedCluster { } /// The product name (`nifi`) as a type-safe label value. -fn product_name() -> ProductName { +pub(crate) fn product_name() -> ProductName { ProductName::from_str(APP_NAME).expect("'nifi' is a valid product name") } /// The operator name as a type-safe label value. -fn operator_name() -> OperatorName { +pub(crate) fn operator_name() -> OperatorName { OperatorName::from_str(OPERATOR_NAME).expect("the operator name is a valid label value") } /// The controller name as a type-safe label value. -fn controller_name() -> ControllerName { +pub(crate) fn controller_name() -> ControllerName { ControllerName::from_str(NIFI_CONTROLLER_NAME) .expect("the controller name is a valid label value") } diff --git a/rust/operator-binary/src/nifi_controller.rs b/rust/operator-binary/src/nifi_controller.rs index 864cfa78..9246c67a 100644 --- a/rust/operator-binary/src/nifi_controller.rs +++ b/rust/operator-binary/src/nifi_controller.rs @@ -7,10 +7,10 @@ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, client::Client, - cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, + cluster_resources::ClusterResourceApplyStrategy, commons::rbac::build_rbac_resources, kube::{ - Resource, ResourceExt, + ResourceExt, core::{DeserializeGuard, error_boundary}, runtime::controller::Action, }, @@ -21,7 +21,7 @@ use stackable_operator::{ compute_conditions, operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, }, - v2::types::operator::RoleGroupName, + v2::{cluster_resources::cluster_resources_new, types::operator::RoleGroupName}, }; use strum::{EnumDiscriminants, IntoStaticStr}; use tracing::Instrument; @@ -32,12 +32,12 @@ use crate::{ build, build::resource::{ listener::{build_group_listener, group_listener_name}, - pdb::add_pdbs, + pdb::build_pdb, reporting_task::build_maybe_reporting_task, service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, statefulset::build_node_rolegroup_statefulset, }, - dereference, + controller_name, dereference, operator_name, product_name, upgrade::{self, ClusterVersionUpdateState}, validate, }, @@ -71,11 +71,6 @@ pub enum Error { #[snafu(display("failed to validate cluster"))] ValidateCluster { source: validate::Error }, - #[snafu(display("failed to create cluster resources"))] - CreateClusterResources { - source: stackable_operator::cluster_resources::Error, - }, - #[snafu(display("failed to delete orphaned resources"))] DeleteOrphanedResources { source: stackable_operator::cluster_resources::Error, @@ -147,9 +142,9 @@ pub enum Error { source: stackable_operator::commons::rbac::Error, }, - #[snafu(display("failed to create PodDisruptionBudget"))] - FailedToCreatePdb { - source: crate::controller::build::resource::pdb::Error, + #[snafu(display("failed to apply PodDisruptionBudget"))] + ApplyPdb { + source: stackable_operator::cluster_resources::Error, }, #[snafu(display("failed to get required labels"))] @@ -249,15 +244,16 @@ pub async fn reconcile_nifi( } // end todo - let mut cluster_resources = ClusterResources::new( - APP_NAME, - OPERATOR_NAME, - NIFI_CONTROLLER_NAME, - &nifi.object_ref(&()), + let mut cluster_resources = cluster_resources_new( + &product_name(), + &operator_name(), + &controller_name(), + &validated_cluster.name, + &validated_cluster.namespace, + &validated_cluster.uid, ClusterResourceApplyStrategy::from(&nifi.spec.cluster_operation), &nifi.spec.object_overrides, - ) - .context(CreateClusterResourcesSnafu)?; + ); if let NifiAuthenticationConfig::Oidc { .. } = authentication_config { check_or_generate_oidc_admin_password(client, nifi, &validated_cluster.namespace) @@ -392,9 +388,12 @@ pub async fn reconcile_nifi( listener_class, }) = role_config { - add_pdbs(pdb, nifi, &nifi_role, client, &mut cluster_resources) - .await - .context(FailedToCreatePdbSnafu)?; + if let Some(pdb) = build_pdb(pdb, &validated_cluster, &nifi_role) { + cluster_resources + .add(client, pdb) + .await + .context(ApplyPdbSnafu)?; + } let role_group_listener = build_group_listener( &validated_cluster, From 1c5a145d0a550eb3d4e3a8998f3b47304400807b Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 13 Jun 2026 22:38:37 +0200 Subject: [PATCH 34/39] chore: add prettyassert dependency --- Cargo.lock | 23 +++++++++++++ Cargo.nix | 57 +++++++++++++++++++++++++++++++++ Cargo.toml | 1 + rust/operator-binary/Cargo.toml | 1 + 4 files changed, 82 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 38bf7ecb..97ef6735 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -619,6 +619,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.10.7" @@ -2205,6 +2211,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "prettyplease" version = "0.2.37" @@ -3022,6 +3038,7 @@ dependencies = [ "futures 0.3.32", "indoc", "pin-project", + "pretty_assertions", "rand 0.10.1", "rstest", "semver", @@ -4199,6 +4216,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "636f85e5ca6488e96401b61eb7de54f4e44755c988af0f52cf90230c312a1a89" +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "yoke" version = "0.8.3" diff --git a/Cargo.nix b/Cargo.nix index 26b6144d..656b4b35 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -1931,6 +1931,16 @@ rec { }; resolvedDefaultFeatures = [ "default" "from" ]; }; + "diff" = rec { + crateName = "diff"; + version = "0.1.13"; + edition = "2015"; + sha256 = "1j0nzjxci2zqx63hdcihkp0a4dkdmzxd7my4m7zk6cjyfy34j9an"; + authors = [ + "Utkarsh Kukreti " + ]; + + }; "digest" = rec { crateName = "digest"; version = "0.10.7"; @@ -7323,6 +7333,31 @@ rec { }; resolvedDefaultFeatures = [ "simd" "std" ]; }; + "pretty_assertions" = rec { + crateName = "pretty_assertions"; + version = "1.4.1"; + edition = "2018"; + sha256 = "0v8iq35ca4rw3rza5is3wjxwsf88303ivys07anc5yviybi31q9s"; + authors = [ + "Colin Kiegel " + "Florent Fayolle " + "Tom Milligan " + ]; + dependencies = [ + { + name = "diff"; + packageId = "diff"; + } + { + name = "yansi"; + packageId = "yansi"; + } + ]; + features = { + "default" = [ "std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "prettyplease" = rec { crateName = "prettyplease"; version = "0.2.37"; @@ -10054,6 +10089,10 @@ rec { } ]; devDependencies = [ + { + name = "pretty_assertions"; + packageId = "pretty_assertions"; + } { name = "rstest"; packageId = "rstest"; @@ -14828,6 +14867,24 @@ rec { ]; }; + "yansi" = rec { + crateName = "yansi"; + version = "1.0.1"; + edition = "2021"; + sha256 = "0jdh55jyv0dpd38ij4qh60zglbw9aa8wafqai6m0wa7xaxk3mrfg"; + authors = [ + "Sergio Benitez " + ]; + features = { + "default" = [ "std" ]; + "detect-env" = [ "std" ]; + "detect-tty" = [ "is-terminal" "std" ]; + "hyperlink" = [ "std" ]; + "is-terminal" = [ "dep:is-terminal" ]; + "std" = [ "alloc" ]; + }; + resolvedDefaultFeatures = [ "alloc" "default" "std" ]; + }; "yoke" = rec { crateName = "yoke"; version = "0.8.3"; diff --git a/Cargo.toml b/Cargo.toml index e6424d41..5a3b1cfb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ fnv = "1.0" futures = { version = "0.3", features = ["compat"] } indoc = "2.0" pin-project = "1.1" +pretty_assertions = "1.4" rand = "0.10" rstest = "0.26" semver = "1.0" diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index b52ad2a9..555e0a47 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -29,6 +29,7 @@ tracing.workspace = true url.workspace = true [dev-dependencies] +pretty_assertions.workspace = true rstest.workspace = true serde_yaml.workspace = true From 8ce6f51992aa34ba6954eae473bc3bd2b5bc0ef1 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 13 Jun 2026 22:39:48 +0200 Subject: [PATCH 35/39] fix: add tests & cleanup --- .../src/controller/build/resource/pdb.rs | 51 +++++++++ .../src/controller/build/resource/service.rs | 44 ++++++++ .../controller/build/resource/statefulset.rs | 46 ++++---- .../src/controller/validate.rs | 104 ++++++++++++++++++ 4 files changed, 226 insertions(+), 19 deletions(-) diff --git a/rust/operator-binary/src/controller/build/resource/pdb.rs b/rust/operator-binary/src/controller/build/resource/pdb.rs index f11eb102..9c8be14a 100644 --- a/rust/operator-binary/src/controller/build/resource/pdb.rs +++ b/rust/operator-binary/src/controller/build/resource/pdb.rs @@ -36,3 +36,54 @@ pub fn build_pdb( fn max_unavailable_nodes() -> u16 { 1 } + +#[cfg(test)] +mod tests { + use pretty_assertions::assert_eq; + use stackable_operator::{ + commons::pdb::PdbConfig, k8s_openapi::apimachinery::pkg::util::intstr::IntOrString, + }; + + use super::*; + use crate::controller::build::properties::test_support::minimal_validated_cluster; + + #[test] + fn build_pdb_returns_none_when_disabled() { + let cluster = minimal_validated_cluster(); + let pdb = PdbConfig { + enabled: false, + max_unavailable: None, + }; + assert!(build_pdb(&pdb, &cluster, &NifiRole::Node).is_none()); + } + + #[test] + fn build_pdb_uses_explicit_max_unavailable() { + let cluster = minimal_validated_cluster(); + let pdb = PdbConfig { + enabled: true, + max_unavailable: Some(2), + }; + + let spec = build_pdb(&pdb, &cluster, &NifiRole::Node) + .expect("an enabled PDB must be built") + .spec + .expect("the PDB must have a spec"); + assert_eq!(Some(IntOrString::Int(2)), spec.max_unavailable); + } + + #[test] + fn build_pdb_defaults_max_unavailable_to_one() { + let cluster = minimal_validated_cluster(); + let pdb = PdbConfig { + enabled: true, + max_unavailable: None, + }; + + let spec = build_pdb(&pdb, &cluster, &NifiRole::Node) + .expect("an enabled PDB must be built") + .spec + .expect("the PDB must have a spec"); + assert_eq!(Some(IntOrString::Int(1)), spec.max_unavailable); + } +} diff --git a/rust/operator-binary/src/controller/build/resource/service.rs b/rust/operator-binary/src/controller/build/resource/service.rs index b74cde64..f0eeac88 100644 --- a/rust/operator-binary/src/controller/build/resource/service.rs +++ b/rust/operator-binary/src/controller/build/resource/service.rs @@ -139,3 +139,47 @@ fn prometheus_annotations(product_version: &str) -> Annotations { ]) .expect("should be valid annotations") } + +#[cfg(test)] +mod tests { + use std::str::FromStr as _; + + use pretty_assertions::assert_eq; + use rstest::rstest; + + use super::*; + use crate::controller::build::properties::test_support::minimal_validated_cluster; + + #[rstest] + // NiFi 1.x exposes metrics on a dedicated JMX-exporter port ... + #[case("1.28.1", METRICS_PORT_NAME, METRICS_PORT)] + // ... while NiFi 2.x serves them on the HTTPS port. + #[case("2.9.0", HTTPS_PORT_NAME, HTTPS_PORT)] + fn metrics_service_port_depends_on_version( + #[case] product_version: &str, + #[case] expected_name: &str, + #[case] expected_port: u16, + ) { + let port = metrics_service_port(product_version); + assert_eq!(Some(expected_name.to_string()), port.name); + assert_eq!(i32::from(expected_port), port.port); + } + + #[test] + fn headless_service_is_cluster_ip_none_with_https_port() { + let cluster = minimal_validated_cluster(); + let rg = RoleGroupName::from_str("default").expect("valid role-group name"); + + let spec = build_rolegroup_headless_service(&cluster, &rg) + .spec + .expect("headless service must have a spec"); + + assert_eq!(Some("ClusterIP".to_string()), spec.type_); + assert_eq!(Some("None".to_string()), spec.cluster_ip); + + let ports = spec.ports.expect("headless service must expose ports"); + assert_eq!(1, ports.len()); + assert_eq!(Some(HTTPS_PORT_NAME.to_string()), ports[0].name); + assert_eq!(i32::from(HTTPS_PORT), ports[0].port); + } +} diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index c41aa28a..bb6bb2c3 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -57,6 +57,7 @@ use crate::{ ValidatedCluster, ValidatedRoleGroupConfig, build::{ graceful_shutdown::add_graceful_shutdown_config, + properties::ConfigFileName, resource::{ listener::{ LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_group_listener_pvc, @@ -291,8 +292,11 @@ pub(crate) async fn build_node_rolegroup_statefulset( format!("keytool -importkeystore -srckeystore {KEYSTORE_NIFI_CONTAINER_MOUNT}/truststore.p12 -destkeystore {STACKABLE_SERVER_TLS_DIR}/truststore.p12 -srcstorepass {STACKABLE_TLS_STORE_PASSWORD} -deststorepass {STACKABLE_TLS_STORE_PASSWORD}"), "echo Replacing config directory".to_string(), - "cp /conf/* /stackable/nifi/conf".to_string(), - "test -L /stackable/nifi/conf/logback.xml || ln -sf /stackable/log_config/logback.xml /stackable/nifi/conf/logback.xml".to_string(), + format!("cp {CONFIG_VOLUME_MOUNT}/* {NIFI_CONFIG_DIRECTORY}"), + format!( + "test -L {NIFI_CONFIG_DIRECTORY}/{logback} || ln -sf {STACKABLE_LOG_CONFIG_DIR}/{logback} {NIFI_CONFIG_DIRECTORY}/{logback}", + logback = ConfigFileName::Logback + ), format!(r#"export NODE_ADDRESS="{node_address}""#), ]); @@ -311,23 +315,27 @@ pub(crate) async fn build_node_rolegroup_statefulset( ]); } - prepare_args.extend(vec![ - "export LISTENER_DEFAULT_ADDRESS=$(cat /stackable/listener/default-address/address)" - .to_string(), - ]); - prepare_args.extend(vec![ - "export LISTENER_DEFAULT_PORT_HTTPS=$(cat /stackable/listener/default-address/ports/https)" - .to_string(), - ]); - - prepare_args.extend(vec![ - "echo Templating config files".to_string(), - "config-utils template /stackable/nifi/conf/nifi.properties".to_string(), - "config-utils template /stackable/nifi/conf/state-management.xml".to_string(), - "config-utils template /stackable/nifi/conf/login-identity-providers.xml".to_string(), - "config-utils template /stackable/nifi/conf/authorizers.xml".to_string(), - "config-utils template /stackable/nifi/conf/security.properties".to_string(), - ]); + prepare_args.push(format!( + "export LISTENER_DEFAULT_ADDRESS=$(cat {LISTENER_VOLUME_DIR}/default-address/address)" + )); + prepare_args.push(format!( + "export LISTENER_DEFAULT_PORT_HTTPS=$(cat {LISTENER_VOLUME_DIR}/default-address/ports/https)" + )); + + // Template the config files that contain `${env:...}`/`${file:...}` placeholders, in a fixed + // order. Sourced from the `ConfigFileName` enum so the file names stay in sync with the + // ConfigMap builder; `bootstrap.conf` and `logback.xml` are intentionally not templated. + prepare_args.push("echo Templating config files".to_string()); + prepare_args.extend( + [ + ConfigFileName::NifiProperties, + ConfigFileName::StateManagementXml, + ConfigFileName::LoginIdentityProviders, + ConfigFileName::Authorizers, + ConfigFileName::SecurityProperties, + ] + .map(|file| format!("config-utils template {NIFI_CONFIG_DIRECTORY}/{file}")), + ); let mut container_prepare = new_container_builder(&PREPARE_CONTAINER_NAME); diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 57346e2c..63205465 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -246,3 +246,107 @@ pub(crate) fn build_role_group_configs( role_group_configs.insert(NifiRole::Node, groups); Ok(role_group_configs) } + +#[cfg(test)] +mod tests { + use pretty_assertions::assert_eq; + use stackable_operator::v2::types::kubernetes::ConfigMapName; + + use super::*; + + /// A NiFi cluster with the Vector agent enabled at the Node role level. + const NIFI_VECTOR_ENABLED_YAML: &str = r#" + apiVersion: nifi.stackable.tech/v1alpha1 + kind: NifiCluster + metadata: + name: simple-nifi + namespace: default + spec: + image: + productVersion: 2.9.0 + clusterConfig: + authentication: + - authenticationClass: nifi-admin-credentials-simple + sensitiveProperties: + keySecret: simple-nifi-sensitive-property-key + autoGenerate: true + nodes: + config: + logging: + enableVectorAgent: true + roleGroups: + default: + replicas: 1 + "#; + + /// A minimal NiFi cluster with the Vector agent disabled (the default). + const NIFI_VECTOR_DISABLED_YAML: &str = r#" + apiVersion: nifi.stackable.tech/v1alpha1 + kind: NifiCluster + metadata: + name: simple-nifi + namespace: default + spec: + image: + productVersion: 2.9.0 + clusterConfig: + authentication: + - authenticationClass: nifi-admin-credentials-simple + sensitiveProperties: + keySecret: simple-nifi-sensitive-property-key + autoGenerate: true + nodes: + roleGroups: + default: + replicas: 1 + "#; + + fn default_rg( + configs: &BTreeMap>, + ) -> &ValidatedRoleGroupConfig { + configs[&NifiRole::Node] + .get(&RoleGroupName::from_str("default").expect("valid role-group name")) + .expect("the 'default' role group must exist") + } + + #[test] + fn vector_container_is_validated_when_agent_enabled() { + let nifi: v1alpha1::NifiCluster = + serde_yaml::from_str(NIFI_VECTOR_ENABLED_YAML).expect("invalid test YAML"); + let aggregator = Some(ConfigMapName::from_str("nifi-vector-aggregator-discovery").unwrap()); + + let configs = build_role_group_configs(&nifi, &aggregator) + .expect("role group configs should validate"); + + let vector = default_rg(&configs) + .vector_container + .as_ref() + .expect("the Vector container config should be present when the agent is enabled"); + assert_eq!( + "nifi-vector-aggregator-discovery", + vector.vector_aggregator_config_map_name.to_string() + ); + } + + #[test] + fn vector_agent_enabled_without_aggregator_name_fails() { + let nifi: v1alpha1::NifiCluster = + serde_yaml::from_str(NIFI_VECTOR_ENABLED_YAML).expect("invalid test YAML"); + + let error = build_role_group_configs(&nifi, &None) + .expect_err("a missing aggregator ConfigMap name must fail when Vector is enabled"); + assert!(matches!(error, Error::MissingVectorAggregatorConfigMapName)); + } + + #[test] + fn no_vector_container_when_agent_disabled() { + let nifi: v1alpha1::NifiCluster = + serde_yaml::from_str(NIFI_VECTOR_DISABLED_YAML).expect("invalid test YAML"); + + // The aggregator name is not required when the Vector agent is disabled. + let configs = + build_role_group_configs(&nifi, &None).expect("role group configs should validate"); + + assert!(default_rg(&configs).vector_container.is_none()); + } +} From 034d8d89859b69e71c333f8f3142810bddea3b83 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 13 Jun 2026 23:07:11 +0200 Subject: [PATCH 36/39] refactor: remove remaining raw nifi cluster references --- .../src/controller/build/properties.rs | 14 +++ .../src/controller/build/resource/listener.rs | 7 +- .../build/resource/reporting_task.rs | 97 +++++++------------ .../controller/build/resource/statefulset.rs | 46 +++------ rust/operator-binary/src/controller/mod.rs | 20 +++- .../src/controller/validate.rs | 14 +++ rust/operator-binary/src/nifi_controller.rs | 11 +-- rust/operator-binary/src/security/mod.rs | 4 +- rust/operator-binary/src/security/tls.rs | 6 +- 9 files changed, 102 insertions(+), 117 deletions(-) diff --git a/rust/operator-binary/src/controller/build/properties.rs b/rust/operator-binary/src/controller/build/properties.rs index d7ac86a1..4a9eb719 100644 --- a/rust/operator-binary/src/controller/build/properties.rs +++ b/rust/operator-binary/src/controller/build/properties.rs @@ -165,6 +165,20 @@ pub(crate) mod test_support { authorization: ResolvedNifiAuthorizationConfig::SingleUser, clustering_backend: v1alpha1::NifiClusteringBackend::Kubernetes {}, sensitive_properties_algorithm: Default::default(), // NifiArgon2AesGcm256 + sensitive_key_secret: nifi + .spec + .cluster_config + .sensitive_properties + .key_secret + .clone(), + server_tls_secret_class: nifi.server_tls_secret_class().to_string(), + extra_volumes: nifi.spec.cluster_config.extra_volumes.clone(), + reporting_task_pod_overrides: nifi + .spec + .cluster_config + .create_reporting_task_job + .pod_overrides + .clone(), host_header_check: nifi.spec.cluster_config.host_header_check.clone(), custom_components_git_sync: nifi .spec diff --git a/rust/operator-binary/src/controller/build/resource/listener.rs b/rust/operator-binary/src/controller/build/resource/listener.rs index c7fe39c8..ef7f835b 100644 --- a/rust/operator-binary/src/controller/build/resource/listener.rs +++ b/rust/operator-binary/src/controller/build/resource/listener.rs @@ -6,14 +6,13 @@ use stackable_operator::{ }, crd::listener::v1alpha1::{Listener, ListenerPort, ListenerSpec}, k8s_openapi::api::core::v1::PersistentVolumeClaim, - kube::ResourceExt, kvp::Labels, v2::builder::meta::ownerreference_from_resource, }; use crate::{ controller::ValidatedCluster, - crd::{HTTPS_PORT, HTTPS_PORT_NAME, v1alpha1}, + crd::{HTTPS_PORT, HTTPS_PORT_NAME}, }; pub const LISTENER_VOLUME_NAME: &str = "listener"; @@ -64,6 +63,6 @@ pub fn build_group_listener_pvc( .context(BuildListenerPersistentVolumeSnafu) } -pub fn group_listener_name(nifi: &v1alpha1::NifiCluster, role_name: &String) -> String { - format!("{cluster_name}-{role_name}", cluster_name = nifi.name_any(),) +pub fn group_listener_name(cluster: &ValidatedCluster, role_name: &String) -> String { + format!("{cluster_name}-{role_name}", cluster_name = cluster.name) } diff --git a/rust/operator-binary/src/controller/build/resource/reporting_task.rs b/rust/operator-binary/src/controller/build/resource/reporting_task.rs index 251f5359..ffb83cdd 100644 --- a/rust/operator-binary/src/controller/build/resource/reporting_task.rs +++ b/rust/operator-binary/src/controller/build/resource/reporting_task.rs @@ -34,7 +34,6 @@ use stackable_operator::{ security::PodSecurityContextBuilder, volume::SecretFormat, }, }, - commons::product_image_selection::ResolvedProductImage, k8s_openapi::{ DeepMerge, api::{ @@ -42,7 +41,6 @@ use stackable_operator::{ core::v1::{Service, ServicePort, ServiceSpec}, }, }, - kube::ResourceExt, shared::time::Duration, utils::cluster_info::KubernetesClusterInfo, v2::{ @@ -56,11 +54,8 @@ use stackable_operator::{ use crate::{ controller::ValidatedCluster, - crd::{HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT, NifiRole, v1alpha1}, - security::{ - authentication::{NifiAuthenticationConfig, STACKABLE_ADMIN_USERNAME}, - build_tls_volume, - }, + crd::{HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT, NifiRole}, + security::{authentication::STACKABLE_ADMIN_USERNAME, build_tls_volume}, }; const REPORTING_TASK_CERT_VOLUME_NAME: &str = "tls"; @@ -108,28 +103,15 @@ type Result = std::result::Result; /// /// NiFi 2.x and above automatically server Prometheus metrics via the API, but as of 2024-11-08 /// requires authentication. -#[allow(clippy::too_many_arguments)] pub fn build_maybe_reporting_task( - nifi: &v1alpha1::NifiCluster, cluster: &ValidatedCluster, - resolved_product_image: &ResolvedProductImage, cluster_info: &KubernetesClusterInfo, - namespace: &NamespaceName, - authentication_config: &NifiAuthenticationConfig, sa_name: &str, ) -> Result> { - if resolved_product_image.product_version.starts_with("1.") { + if cluster.image.product_version.starts_with("1.") { Ok(Some(( - build_reporting_task_job( - nifi, - cluster, - resolved_product_image, - cluster_info, - namespace, - authentication_config, - sa_name, - )?, - build_reporting_task_service(nifi, cluster)?, + build_reporting_task_job(cluster, cluster_info, sa_name)?, + build_reporting_task_service(cluster)?, ))) } else { Ok(None) @@ -162,51 +144,48 @@ pub fn build_reporting_task_fqdn_service_name( } /// Return the name of the first pod belonging to the first role group that contains more than 0 replicas. -/// If no replicas are set in any rolegroup (e.g. HPA, see ) +/// If no role group has replicas set (e.g. HPA, see ) /// return the first rolegroup just in case. /// This is required to only select a single node in the Reporting Task Service. -fn get_reporting_task_service_selector_pod(nifi: &v1alpha1::NifiCluster) -> Result { - let cluster_name = nifi.name_any(); +/// +/// Note: the validated replicas default to `1` (see [`ValidatedRoleGroupConfig`]), so an +/// HPA-managed role group (raw `replicas: null`) is treated as having a single replica here. +fn get_reporting_task_service_selector_pod(cluster: &ValidatedCluster) -> Result { let node_name = NifiRole::Node.to_string(); - // sort the rolegroups to avoid random sorting and therefore unnecessary reconciles - let sorted_role_groups = nifi - .spec - .nodes - .iter() - .flat_map(|role| &role.role_groups) - .collect::>(); + // The role groups are already sorted by name (`BTreeMap`), avoiding random ordering and + // therefore unnecessary reconciles. + let role_groups = cluster + .role_group_configs + .get(&NifiRole::Node) + .context(FailedBuildReportingTaskServiceSnafu)?; let mut selector_role_group = None; - for (role_group_name, role_group) in sorted_role_groups { - // just pick the first rolegroup in case no replicas are set + for (role_group_name, role_group) in role_groups { + // just pick the first rolegroup in case none has replicas set if selector_role_group.is_none() { selector_role_group = Some(role_group_name); } - if let Some(replicas) = role_group.replicas { - if replicas > 0 { - selector_role_group = Some(role_group_name); - break; - } + if role_group.replicas > 0 { + selector_role_group = Some(role_group_name); + break; } } Ok(format!( "{cluster_name}-{node_name}-{role_group_name}-0", + cluster_name = cluster.name, role_group_name = selector_role_group.context(FailedBuildReportingTaskServiceSnafu)? )) } /// Build the internal Reporting Task Service in order to communicate with a single NiFi node. -fn build_reporting_task_service( - nifi: &v1alpha1::NifiCluster, - cluster: &ValidatedCluster, -) -> Result { - let nifi_cluster_name = nifi.name_any(); +fn build_reporting_task_service(cluster: &ValidatedCluster) -> Result { + let nifi_cluster_name = cluster.name.to_string(); let mut selector: BTreeMap = cluster.role_selector().into(); - let service_selector_pod = get_reporting_task_service_selector_pod(nifi)?; + let service_selector_pod = get_reporting_task_service_selector_pod(cluster)?; selector.insert( "statefulset.kubernetes.io/pod-name".to_string(), service_selector_pod, @@ -247,18 +226,18 @@ fn build_reporting_task_service( /// as well as a public certificate provided by the Stackable /// [`secret-operator`](https://github.com/stackabletech/secret-operator) /// -#[allow(clippy::too_many_arguments)] fn build_reporting_task_job( - nifi: &v1alpha1::NifiCluster, cluster: &ValidatedCluster, - resolved_product_image: &ResolvedProductImage, cluster_info: &KubernetesClusterInfo, - namespace: &NamespaceName, - nifi_auth_config: &NifiAuthenticationConfig, sa_name: &str, ) -> Result { - let reporting_task_fqdn_service_name = - build_reporting_task_fqdn_service_name(&nifi.name_any(), namespace, cluster_info); + let resolved_product_image = &cluster.image; + let nifi_auth_config = &cluster.cluster_config.authentication; + let reporting_task_fqdn_service_name = build_reporting_task_fqdn_service_name( + cluster.name.as_ref(), + &cluster.namespace, + cluster_info, + ); let product_version = &resolved_product_image.product_version; let nifi_connect_url = format!("https://{reporting_task_fqdn_service_name}:{HTTPS_PORT}/nifi-api",); @@ -305,7 +284,7 @@ fn build_reporting_task_job( let job_name = format!( "{}-create-reporting-task-{}", - nifi.name_any(), + cluster.name, product_version.replace('.', "-").to_ascii_lowercase() ); @@ -329,7 +308,7 @@ fn build_reporting_task_job( .add_container(cb.build()) .add_volume( build_tls_volume( - nifi, + &cluster.cluster_config.server_tls_secret_class, REPORTING_TASK_CERT_VOLUME_NAME, Vec::::new(), SecretFormat::TlsPem, @@ -345,13 +324,7 @@ fn build_reporting_task_job( .context(AddVolumeSnafu)? .build_template(); - pod_template.merge_from( - nifi.spec - .cluster_config - .create_reporting_task_job - .pod_overrides - .clone(), - ); + pod_template.merge_from(cluster.cluster_config.reporting_task_pod_overrides.clone()); let job = Job { metadata: ObjectMetaBuilder::new() diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index bb6bb2c3..1a76720a 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -1,6 +1,6 @@ //! Builds the rolegroup [`StatefulSet`] that runs a NiFi node role group. -use std::{collections::BTreeMap, str::FromStr}; +use std::str::FromStr; use indoc::formatdoc; use snafu::{OptionExt, ResultExt, Snafu}; @@ -13,7 +13,6 @@ use stackable_operator::{ security::PodSecurityContextBuilder, volume::SecretFormat, }, }, - commons::product_image_selection::ResolvedProductImage, constants::RESTART_CONTROLLER_ENABLED_LABEL, crd::{authentication::oidc::v1alpha1::AuthenticationProvider, git_sync}, k8s_openapi::{ @@ -28,7 +27,6 @@ use stackable_operator::{ }, apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, }, - kube::ResourceExt, memory::{BinaryMultiple, MemoryQuantity}, product_logging::{ self, @@ -93,9 +91,6 @@ pub enum Error { #[snafu(display("missing secret lifetime"))] MissingSecretLifetime, - #[snafu(display("object defines no name"))] - ObjectHasNoName, - #[snafu(display("failed to add Authentication Volumes and VolumeMounts"))] AddAuthVolumes { source: crate::security::authentication::Error, @@ -168,15 +163,11 @@ stackable_operator::constant!(VECTOR_LOG_VOLUME_NAME: VolumeName = "log"); /// corresponding [`stackable_operator::k8s_openapi::api::core::v1::Service`] (from [`build_rolegroup_headless_service`]). #[allow(clippy::too_many_arguments)] pub(crate) async fn build_node_rolegroup_statefulset( - nifi: &v1alpha1::NifiCluster, cluster: &ValidatedCluster, - resolved_product_image: &ResolvedProductImage, cluster_info: &KubernetesClusterInfo, role_group_name: &RoleGroupName, role: &NifiRoleType, rg: &ValidatedRoleGroupConfig, - authentication_config: &NifiAuthenticationConfig, - authorization_config: &ResolvedNifiAuthorizationConfig, rolling_update_supported: bool, replicas: Option, service_account_name: &str, @@ -184,6 +175,12 @@ pub(crate) async fn build_node_rolegroup_statefulset( ) -> Result { tracing::debug!("Building statefulset"); + // Everything cluster-global is sourced from the `ValidatedCluster`; the raw `NifiCluster` is + // never touched here. + let resolved_product_image = &cluster.image; + let authentication_config = &cluster.cluster_config.authentication; + let authorization_config = &cluster.cluster_config.authorization; + // Type-safe names for this role group's resources (StatefulSet, ConfigMap, headless Service). let resource_names = cluster.resource_names(role_group_name); @@ -215,11 +212,11 @@ pub(crate) async fn build_node_rolegroup_statefulset( env_vars.push(EnvVar { name: "STACKLET_NAME".to_string(), - value: Some(nifi.name_unchecked().to_string()), + value: Some(cluster.name.to_string()), ..Default::default() }); - match &nifi.spec.cluster_config.clustering_backend { + match &cluster.cluster_config.clustering_backend { v1alpha1::NifiClusteringBackend::ZooKeeper { zookeeper_config_map_name, } => { @@ -256,7 +253,7 @@ pub(crate) async fn build_node_rolegroup_statefulset( cluster_domain = cluster_info.cluster_domain, ); - let sensitive_key_secret = &nifi.spec.cluster_config.sensitive_properties.key_secret; + let sensitive_key_secret = &cluster.cluster_config.sensitive_key_secret; let prepare_container_name = PREPARE_CONTAINER_NAME.to_string(); let mut prepare_args = vec![]; @@ -467,7 +464,7 @@ pub(crate) async fn build_node_rolegroup_statefulset( add_graceful_shutdown_config(merged_config, &mut pod_builder).context(GracefulShutdownSnafu)?; // Add user configured extra volumes if any are specified - for volume in &nifi.spec.cluster_config.extra_volumes { + for volume in &cluster.cluster_config.extra_volumes { // Extract values into vars so we make it impossible to log something other than // what we actually use to create the mounts - maybe paranoid, but hey .. let volume_name = &volume.name; @@ -574,7 +571,7 @@ pub(crate) async fn build_node_rolegroup_statefulset( let requested_secret_lifetime = merged_config .requested_secret_lifetime .context(MissingSecretLifetimeSnafu)?; - let nifi_cluster_name = nifi.name_any(); + let nifi_cluster_name = cluster.name.to_string(); pod_builder .metadata(metadata) .image_pull_secrets_from_product_image(resolved_product_image) @@ -615,7 +612,7 @@ pub(crate) async fn build_node_rolegroup_statefulset( // One volume for the keystore and truststore data configmap .add_volume( build_tls_volume( - nifi, + &cluster.cluster_config.server_tls_secret_class, KEYSTORE_VOLUME_NAME, [ crate::controller::build::resource::service::metrics_service_name( @@ -662,16 +659,6 @@ pub(crate) async fn build_node_rolegroup_statefulset( .service_account_name(service_account_name) .security_context(PodSecurityContextBuilder::new().fs_group(1000).build()); - let mut labels = BTreeMap::new(); - labels.insert( - "app.kubernetes.io/instance".to_string(), - nifi.metadata - .name - .as_deref() - .with_context(|| ObjectHasNoNameSnafu {})? - .to_string(), - ); - let mut pod_template = pod_builder.build_template(); // `rg.pod_overrides` is already the role <- rolegroup merge produced by the framework. pod_template.merge_from(rg.pod_overrides.clone()); @@ -702,11 +689,9 @@ pub(crate) async fn build_node_rolegroup_statefulset( ..StatefulSetUpdateStrategy::default() }), volume_claim_templates: Some(get_volume_claim_templates( - nifi, cluster, role_group_name, merged_config, - authorization_config, )?), ..StatefulSetSpec::default() }), @@ -715,12 +700,11 @@ pub(crate) async fn build_node_rolegroup_statefulset( } fn get_volume_claim_templates( - nifi: &v1alpha1::NifiCluster, cluster: &ValidatedCluster, role_group_name: &RoleGroupName, merged_config: &NifiConfig, - authorization_config: &ResolvedNifiAuthorizationConfig, ) -> Result> { + let authorization_config = &cluster.cluster_config.authorization; let mut pvcs = vec![ merged_config.resources.storage.content_repo.build_pvc( &NifiRepository::Content.repository(), @@ -753,7 +737,7 @@ fn get_volume_claim_templates( // that it is possible to connect to a consistent address pvcs.push( build_group_listener_pvc( - &group_listener_name(nifi, &NifiRole::Node.to_string()), + &group_listener_name(cluster, &NifiRole::Node.to_string()), &unversioned_recommended_labels, ) .context(ListenerConfigurationSnafu)?, diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index 1b608201..5a987287 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -7,7 +7,10 @@ use std::{collections::BTreeMap, str::FromStr as _}; use stackable_operator::{ commons::product_image_selection::ResolvedProductImage, crd::git_sync, - k8s_openapi::{api::core::v1::PodTemplateSpec, apimachinery::pkg::apis::meta::v1::ObjectMeta}, + k8s_openapi::{ + api::core::v1::{PodTemplateSpec, Volume}, + apimachinery::pkg::apis::meta::v1::ObjectMeta, + }, kube::Resource, kvp::Labels, v2::{ @@ -53,10 +56,9 @@ pub(crate) mod validate; pub struct ValidatedRoleGroupConfig { /// The desired number of replicas (defaulted to 1 during validation). /// - /// The StatefulSet replica count is currently sourced from the raw role-group spec in - /// `nifi_controller` (to keep `replicas: null` semantics during version updates), so this - /// validated value is carried for completeness but not yet read. - #[allow(dead_code)] + /// The StatefulSet replica count is sourced from the raw role-group spec in `nifi_controller` + /// (to keep `replicas: null` semantics during version updates); this validated value is used to + /// pick the single node targeted by the (NiFi 1.x-only) reporting-task Service. pub replicas: u16, /// The merged and validated rolegroup config. pub config: NifiConfig, @@ -113,6 +115,14 @@ pub struct ValidatedClusterConfig { pub host_header_check: HostHeaderCheckConfig, /// The validated sensitive properties algorithm. pub sensitive_properties_algorithm: NifiSensitiveKeyAlgorithm, + /// The name of the Secret holding the sensitive-properties key, mounted into the NiFi Pods. + pub sensitive_key_secret: String, + /// The SecretClass providing the server TLS certificates. + pub server_tls_secret_class: String, + /// User-provided extra volumes, mounted into every container under `/stackable/userdata/`. + pub extra_volumes: Vec, + /// Pod overrides for the (NiFi 1.x-only) create-reporting-task Job. + pub reporting_task_pod_overrides: PodTemplateSpec, } impl ValidatedCluster { diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 63205465..e390e096 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -167,6 +167,20 @@ pub fn validate( authorization: authorization_config, clustering_backend: nifi.spec.cluster_config.clustering_backend.clone(), sensitive_properties_algorithm, + sensitive_key_secret: nifi + .spec + .cluster_config + .sensitive_properties + .key_secret + .clone(), + server_tls_secret_class: nifi.server_tls_secret_class().to_string(), + extra_volumes: nifi.spec.cluster_config.extra_volumes.clone(), + reporting_task_pod_overrides: nifi + .spec + .cluster_config + .create_reporting_task_job + .pod_overrides + .clone(), host_header_check: nifi.spec.cluster_config.host_header_check.clone(), custom_components_git_sync: nifi.spec.cluster_config.custom_components_git_sync.clone(), }, diff --git a/rust/operator-binary/src/nifi_controller.rs b/rust/operator-binary/src/nifi_controller.rs index 9246c67a..57f74c82 100644 --- a/rust/operator-binary/src/nifi_controller.rs +++ b/rust/operator-binary/src/nifi_controller.rs @@ -208,7 +208,6 @@ pub async fn reconcile_nifi( let resolved_product_image = &validated_cluster.image; let authentication_config = &validated_cluster.cluster_config.authentication; - let authorization_config = &validated_cluster.cluster_config.authorization; tracing::info!("Checking for sensitive key configuration"); check_or_generate_sensitive_key(client, nifi, &validated_cluster.namespace) @@ -319,15 +318,11 @@ pub async fn reconcile_nifi( }; let rg_statefulset = build_node_rolegroup_statefulset( - nifi, &validated_cluster, - resolved_product_image, &client.kubernetes_cluster_info, role_group_name, role, rg, - authentication_config, - authorization_config, rolling_upgrade_supported, replicas, &rbac_sa.name_any(), @@ -398,7 +393,7 @@ pub async fn reconcile_nifi( let role_group_listener = build_group_listener( &validated_cluster, listener_class.to_owned(), - group_listener_name(nifi, &nifi_role.to_string()), + group_listener_name(&validated_cluster, &nifi_role.to_string()), ) .context(ListenerConfigurationSnafu)?; @@ -411,12 +406,8 @@ pub async fn reconcile_nifi( // Only add the reporting task in case it is enabled. if nifi.spec.cluster_config.create_reporting_task_job.enabled { if let Some((reporting_task_job, reporting_task_service)) = build_maybe_reporting_task( - nifi, &validated_cluster, - resolved_product_image, &client.kubernetes_cluster_info, - &validated_cluster.namespace, - authentication_config, &rbac_sa.name_any(), ) .context(ReportingTaskSnafu)? diff --git a/rust/operator-binary/src/security/mod.rs b/rust/operator-binary/src/security/mod.rs index 61229ce0..60f312cd 100644 --- a/rust/operator-binary/src/security/mod.rs +++ b/rust/operator-binary/src/security/mod.rs @@ -47,7 +47,7 @@ pub async fn check_or_generate_oidc_admin_password( } pub fn build_tls_volume( - nifi: &v1alpha1::NifiCluster, + server_tls_secret_class: &str, volume_name: &str, service_scopes: impl IntoIterator>, secret_format: SecretFormat, @@ -55,7 +55,7 @@ pub fn build_tls_volume( listener_scope: Option<&str>, ) -> Result { tls::build_tls_volume( - nifi, + server_tls_secret_class, volume_name, service_scopes, secret_format, diff --git a/rust/operator-binary/src/security/tls.rs b/rust/operator-binary/src/security/tls.rs index 196b7a66..dd1baa36 100644 --- a/rust/operator-binary/src/security/tls.rs +++ b/rust/operator-binary/src/security/tls.rs @@ -6,7 +6,7 @@ use stackable_operator::{ shared::time::Duration, }; -use crate::{crd::v1alpha1, security::authentication::STACKABLE_TLS_STORE_PASSWORD}; +use crate::security::authentication::STACKABLE_TLS_STORE_PASSWORD; pub const KEYSTORE_VOLUME_NAME: &str = "keystore"; pub const KEYSTORE_NIFI_CONTAINER_MOUNT: &str = "/stackable/keystore"; @@ -23,7 +23,7 @@ pub enum Error { } pub(crate) fn build_tls_volume( - nifi: &v1alpha1::NifiCluster, + server_tls_secret_class: &str, volume_name: &str, service_scopes: impl IntoIterator>, secret_format: SecretFormat, @@ -31,7 +31,7 @@ pub(crate) fn build_tls_volume( listener_scope: Option<&str>, ) -> Result { let mut secret_volume_source_builder = SecretOperatorVolumeSourceBuilder::new( - nifi.server_tls_secret_class(), + server_tls_secret_class, // NiFi serves its own TLS endpoints, so the Pod needs both the public // certificate and the private key. SecretClassVolumeProvisionParts::PublicPrivate, From 5475664b65a95a8999e79e183b2cfb943b0d79f3 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 13 Jun 2026 23:17:43 +0200 Subject: [PATCH 37/39] refactor: move gitsync components to ValidatedRoleGroupConfig --- .../src/controller/build/config_map.rs | 14 ++--- .../src/controller/build/git_sync.rs | 29 +++++++---- .../src/controller/build/properties.rs | 17 ++----- .../build/properties/bootstrap_conf.rs | 5 +- .../build/properties/nifi_properties.rs | 30 ++++++----- .../build/properties/security_properties.rs | 1 + .../controller/build/resource/statefulset.rs | 4 +- rust/operator-binary/src/controller/mod.rs | 8 +-- .../src/controller/validate.rs | 51 +++++++++++++++---- rust/operator-binary/src/crd/affinity.rs | 8 ++- rust/operator-binary/src/nifi_controller.rs | 8 --- 11 files changed, 100 insertions(+), 75 deletions(-) diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 2856b783..85056a55 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -13,7 +13,6 @@ use crate::{ controller::{ ValidatedCluster, build::{ - git_sync, properties::{ ConfigFileName, authorizers, bootstrap_conf, login_identity_providers, nifi_properties, product_logging, security_properties, state_management_xml, @@ -58,9 +57,6 @@ pub enum Error { #[snafu(display("the cluster has no rolegroup [{role_group}] in role [{role}]"))] MissingRoleGroup { role: String, role_group: String }, - - #[snafu(display("failed to build git-sync resources"))] - BuildGitSyncResources { source: git_sync::Error }, } type Result = std::result::Result; @@ -86,8 +82,6 @@ pub fn build_rolegroup_config_map( })?; let proxy_hosts = proxy_hosts::compute_proxy_hosts(cluster, cluster_info); - let git_sync_resources = - git_sync::build_git_sync_resources(cluster, rg).context(BuildGitSyncResourcesSnafu)?; let mut cm_builder = ConfigMapBuilder::new(); @@ -112,11 +106,11 @@ pub fn build_rolegroup_config_map( ) .add_data( ConfigFileName::NifiProperties.to_string(), - nifi_properties::build(cluster, rg, &proxy_hosts, &git_sync_resources).with_context( - |_| BuildNifiPropertiesSnafu { + nifi_properties::build(cluster, rg, &proxy_hosts).with_context(|_| { + BuildNifiPropertiesSnafu { rolegroup: role_group_name.clone(), - }, - )?, + } + })?, ) .add_data( ConfigFileName::StateManagementXml.to_string(), diff --git a/rust/operator-binary/src/controller/build/git_sync.rs b/rust/operator-binary/src/controller/build/git_sync.rs index 2171b53d..ee781617 100644 --- a/rust/operator-binary/src/controller/build/git_sync.rs +++ b/rust/operator-binary/src/controller/build/git_sync.rs @@ -1,13 +1,14 @@ //! Builds the git-sync resources (volumes, mounts, containers) for a NiFi Node rolegroup. use snafu::{ResultExt, Snafu}; -use stackable_operator::{crd::git_sync, k8s_openapi::api::core::v1::EnvVar}; +use stackable_operator::{ + commons::product_image_selection::ResolvedProductImage, crd::git_sync, + k8s_openapi::api::core::v1::EnvVar, v2::builder::pod::container::EnvVarSet, +}; use crate::{ - controller::{ - ValidatedCluster, ValidatedRoleGroupConfig, build::resource::statefulset::LOG_VOLUME_NAME, - }, - crd::Container, + controller::build::resource::statefulset::LOG_VOLUME_NAME, + crd::{Container, NifiConfig}, }; #[derive(Snafu, Debug)] @@ -21,18 +22,24 @@ type Result = std::result::Result; /// Builds the [`git_sync::v1alpha2::GitSyncResources`] for a single Node rolegroup. The env vars /// and logging configuration differ per rolegroup, so the resources are computed per rolegroup /// rather than once for the whole cluster. +/// +/// Called from the [`validate`](crate::controller::validate) step; the result is stored on the +/// [`ValidatedRoleGroupConfig`](crate::controller::ValidatedRoleGroupConfig) and consumed by the +/// downstream builders. pub fn build_git_sync_resources( - cluster: &ValidatedCluster, - rg: &ValidatedRoleGroupConfig, + custom_components_git_sync: &[git_sync::v1alpha2::GitSync], + image: &ResolvedProductImage, + config: &NifiConfig, + env_overrides: &EnvVarSet, ) -> Result { - let env_vars: Vec = rg.env_overrides.clone().into(); + let env_vars: Vec = env_overrides.clone().into(); git_sync::v1alpha2::GitSyncResources::new( - &cluster.cluster_config.custom_components_git_sync, - &cluster.image, + custom_components_git_sync, + image, &env_vars, &[], LOG_VOLUME_NAME, - &rg.config.logging.for_container(&Container::GitSync), + &config.logging.for_container(&Container::GitSync), ) .context(InvalidGitSyncSpecSnafu) } diff --git a/rust/operator-binary/src/controller/build/properties.rs b/rust/operator-binary/src/controller/build/properties.rs index 4a9eb719..cdbe6e71 100644 --- a/rust/operator-binary/src/controller/build/properties.rs +++ b/rust/operator-binary/src/controller/build/properties.rs @@ -130,9 +130,6 @@ pub(crate) mod test_support { let nifi: v1alpha1::NifiCluster = serde_yaml::from_str(MINIMAL_NIFI_YAML).expect("invalid test YAML"); - let role_group_configs = build_role_group_configs(&nifi, &None) - .expect("role group configs should merge for minimal fixture"); - let image = ResolvedProductImage { product_version: "2.9.0".to_string(), app_version_label_value: "2.9.0".parse::().unwrap(), @@ -141,6 +138,9 @@ pub(crate) mod test_support { pull_secrets: None, }; + let role_group_configs = build_role_group_configs(&nifi, &image, &None) + .expect("role group configs should merge for minimal fixture"); + let name = ClusterName::from_str("simple-nifi").expect("valid cluster name"); let namespace = NamespaceName::from_str("default").expect("valid namespace"); let uid = Uid::from_str("e6ac237d-a6d4-43a1-8135-f36506110912").expect("valid uid"); @@ -180,11 +180,6 @@ pub(crate) mod test_support { .pod_overrides .clone(), host_header_check: nifi.spec.cluster_config.host_header_check.clone(), - custom_components_git_sync: nifi - .spec - .cluster_config - .custom_components_git_sync - .clone(), }, ) } @@ -199,10 +194,4 @@ pub(crate) mod test_support { }) .expect("minimal_validated_cluster must contain a 'default' role group") } - - /// Build an empty [`GitSyncResources`] (no git-sync configured). - pub fn empty_git_sync_resources() - -> stackable_operator::crd::git_sync::v1alpha2::GitSyncResources { - stackable_operator::crd::git_sync::v1alpha2::GitSyncResources::default() - } } diff --git a/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs b/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs index 5ee352ac..150cea4e 100644 --- a/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs +++ b/rust/operator-binary/src/controller/build/properties/bootstrap_conf.rs @@ -55,7 +55,7 @@ mod tests { use super::*; use crate::{ - controller::validate::build_role_group_configs, + controller::validate::{build_role_group_configs, test_resolved_product_image}, crd::{NifiRole, v1alpha1}, }; @@ -64,7 +64,8 @@ mod tests { serde_yaml::from_str(nifi_cluster).expect("illegal test input"); let role_group_configs = - build_role_group_configs(&nifi, &None).expect("failed to build role group configs"); + build_role_group_configs(&nifi, &test_resolved_product_image(), &None) + .expect("failed to build role group configs"); let rg = role_group_configs .get(&NifiRole::Node) .and_then(|groups| { diff --git a/rust/operator-binary/src/controller/build/properties/nifi_properties.rs b/rust/operator-binary/src/controller/build/properties/nifi_properties.rs index e7a395df..32e5ece3 100644 --- a/rust/operator-binary/src/controller/build/properties/nifi_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/nifi_properties.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use snafu::{ResultExt, ensure}; -use stackable_operator::{crd::git_sync, memory::MemoryQuantity}; +use stackable_operator::memory::MemoryQuantity; use super::format_properties; use crate::{ @@ -32,8 +32,8 @@ pub fn build( cluster: &ValidatedCluster, rg: &ValidatedRoleGroupConfig, proxy_hosts: &str, - git_sync_resources: &git_sync::v1alpha2::GitSyncResources, ) -> Result { + let git_sync_resources = &rg.git_sync_resources; let product_version = &cluster.image.product_version; let auth_config = &cluster.cluster_config.authentication; let resource_config = &rg.config.resources; @@ -590,9 +590,7 @@ fn storage_quantity_to_nifi(quantity: MemoryQuantity) -> String { mod tests { use super::*; use crate::{ - controller::build::properties::test_support::{ - default_rg, empty_git_sync_resources, minimal_validated_cluster, - }, + controller::build::properties::test_support::{default_rg, minimal_validated_cluster}, crd::HTTPS_PORT, }; @@ -602,9 +600,8 @@ mod tests { fn test_stable_keys_present() { let cluster = minimal_validated_cluster(); let rg = default_rg(&cluster); - let git_sync = empty_git_sync_resources(); - let props = build(&cluster, rg, "*", &git_sync).expect("build should succeed"); + let props = build(&cluster, rg, "*").expect("build should succeed"); // HTTPS port assert!( @@ -645,7 +642,7 @@ mod tests { use stackable_operator::v2::types::operator::RoleGroupName; use crate::{ - controller::validate::build_role_group_configs, + controller::validate::{build_role_group_configs, test_resolved_product_image}, crd::{NifiRole, v1alpha1}, }; @@ -674,7 +671,8 @@ mod tests { "#; let nifi: v1alpha1::NifiCluster = serde_yaml::from_str(yaml).expect("invalid test YAML"); let mut role_group_configs = - build_role_group_configs(&nifi, &None).expect("failed to build role group configs"); + build_role_group_configs(&nifi, &test_resolved_product_image(), &None) + .expect("failed to build role group configs"); let default_rg_name = "default" .parse::() .expect("valid role-group name"); @@ -683,17 +681,21 @@ mod tests { .and_then(|groups| groups.remove(&default_rg_name)) .expect("default role group must exist"); - // Build a cluster with this rg substituted in + // Build a cluster with this rg substituted in, then borrow it back for the build call + // (`ValidatedRoleGroupConfig` is not `Clone`). let mut cluster = minimal_validated_cluster(); cluster .role_group_configs .get_mut(&NifiRole::Node) .unwrap() - .insert(default_rg_name, rg.clone()); + .insert(default_rg_name.clone(), rg); + let rg = cluster + .role_group_configs + .get(&NifiRole::Node) + .and_then(|groups| groups.get(&default_rg_name)) + .expect("default role group must exist"); - let git_sync = empty_git_sync_resources(); - let props = - build(&cluster, &rg, "*", &git_sync).expect("build with override should succeed"); + let props = build(&cluster, rg, "*").expect("build with override should succeed"); assert!( props.contains("some.custom.key=some-custom-value"), diff --git a/rust/operator-binary/src/controller/build/properties/security_properties.rs b/rust/operator-binary/src/controller/build/properties/security_properties.rs index 45fc983b..10f4771a 100644 --- a/rust/operator-binary/src/controller/build/properties/security_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/security_properties.rs @@ -49,6 +49,7 @@ mod tests { pod_overrides: Default::default(), product_specific_common_config: JavaCommonConfig::default(), vector_container: None, + git_sync_resources: Default::default(), } } diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 1a76720a..e6c0ba41 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -14,7 +14,7 @@ use stackable_operator::{ }, }, constants::RESTART_CONTROLLER_ENABLED_LABEL, - crd::{authentication::oidc::v1alpha1::AuthenticationProvider, git_sync}, + crd::authentication::oidc::v1alpha1::AuthenticationProvider, k8s_openapi::{ DeepMerge, api::{ @@ -171,7 +171,6 @@ pub(crate) async fn build_node_rolegroup_statefulset( rolling_update_supported: bool, replicas: Option, service_account_name: &str, - git_sync_resources: &git_sync::v1alpha2::GitSyncResources, ) -> Result { tracing::debug!("Building statefulset"); @@ -180,6 +179,7 @@ pub(crate) async fn build_node_rolegroup_statefulset( let resolved_product_image = &cluster.image; let authentication_config = &cluster.cluster_config.authentication; let authorization_config = &cluster.cluster_config.authorization; + let git_sync_resources = &rg.git_sync_resources; // Type-safe names for this role group's resources (StatefulSet, ConfigMap, headless Service). let resource_names = cluster.resource_names(role_group_name); diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index 5a987287..aaaf569d 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -52,7 +52,6 @@ pub(crate) mod validate; /// Produced from the result of /// [`with_validated_config`](stackable_operator::v2::role_utils::with_validated_config) in the /// [`validate`] step; downstream builders consume this rather than the raw `NifiCluster`. -#[derive(Clone, Debug)] pub struct ValidatedRoleGroupConfig { /// The desired number of replicas (defaulted to 1 during validation). /// @@ -75,6 +74,11 @@ pub struct ValidatedRoleGroupConfig { /// ConfigMap name), validated up-front in the [`validate`] step. `None` when the Vector agent /// is disabled for this role group. pub vector_container: Option, + /// The git-sync resources (containers, volumes, mounts) for this role group, resolved from the + /// cluster's `customComponentsGitSync` specs up-front in the [`validate`] step. The env vars and + /// logging config differ per role group, so these are computed per role group. Consumed by both + /// the StatefulSet builder and the `nifi.properties` builder. + pub git_sync_resources: git_sync::v1alpha2::GitSyncResources, } /// The validated NifiCluster: everything `reconcile_nifi` needs after dereferencing, @@ -107,8 +111,6 @@ pub struct ValidatedClusterConfig { pub authentication: NifiAuthenticationConfig, /// The cluster authorization settings. pub authorization: ResolvedNifiAuthorizationConfig, - /// The git-sync specs, resolved into git-sync resources at build time. - pub custom_components_git_sync: Vec, /// The clustering backend (ZooKeeper or Kubernetes), copied from the spec. pub clustering_backend: v1alpha1::NifiClusteringBackend, /// The host-header-check config, resolved into the proxy hosts allow-list at build time. diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index e390e096..421459ce 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -29,7 +29,7 @@ use strum::{EnumDiscriminants, IntoStaticStr}; use super::{ValidatedCluster, ValidatedClusterConfig, ValidatedRoleGroupConfig}; use crate::{ - controller::dereference::DereferencedObjects, + controller::{build::git_sync::build_git_sync_resources, dereference::DereferencedObjects}, crd::{Container, NifiConfig, NifiRole, sensitive_properties, v1alpha1}, security::{ authentication::{self, NifiAuthenticationConfig}, @@ -79,6 +79,11 @@ pub enum Error { #[snafu(display("invalid sensitive properties algorithm"))] InvalidSensitivePropertiesAlgorithm { source: sensitive_properties::Error }, + #[snafu(display("failed to build git-sync resources"))] + BuildGitSyncResources { + source: crate::controller::build::git_sync::Error, + }, + #[snafu(display("invalid Vector aggregator discovery ConfigMap name"))] ParseVectorAggregatorConfigMapName { source: stackable_operator::v2::macros::attributed_string_type::Error, @@ -144,7 +149,8 @@ pub fn validate( .transpose() .context(ParseVectorAggregatorConfigMapNameSnafu)?; - let role_group_configs = build_role_group_configs(nifi, &vector_aggregator_config_map_name)?; + let role_group_configs = + build_role_group_configs(nifi, &image, &vector_aggregator_config_map_name)?; let name = get_cluster_name(nifi).context(GetClusterNameSnafu)?; let namespace = dereferenced_objects.namespace.clone(); @@ -182,13 +188,13 @@ pub fn validate( .pod_overrides .clone(), host_header_check: nifi.spec.cluster_config.host_header_check.clone(), - custom_components_git_sync: nifi.spec.cluster_config.custom_components_git_sync.clone(), }, )) } pub(crate) fn build_role_group_configs( nifi: &v1alpha1::NifiCluster, + image: &product_image_selection::ResolvedProductImage, vector_aggregator_config_map_name: &Option, ) -> Result>> { let role = nifi.spec.nodes.as_ref().context(NoNodesDefinedSnafu)?; @@ -242,6 +248,16 @@ pub(crate) fn build_role_group_configs( None }; + // The git-sync resources depend on this role group's env-var overrides and logging config, + // so they are resolved (and validated) per role group up-front rather than at build time. + let git_sync_resources = build_git_sync_resources( + &nifi.spec.cluster_config.custom_components_git_sync, + image, + &config, + &env_overrides_set, + ) + .context(BuildGitSyncResourcesSnafu)?; + groups.insert( role_group_name, ValidatedRoleGroupConfig { @@ -252,6 +268,7 @@ pub(crate) fn build_role_group_configs( pod_overrides, product_specific_common_config, vector_container, + git_sync_resources, }, ); } @@ -261,6 +278,18 @@ pub(crate) fn build_role_group_configs( Ok(role_group_configs) } +/// A minimal resolved product image (NiFi 2.9.0) for tests that need to build role-group configs. +#[cfg(test)] +pub(crate) fn test_resolved_product_image() -> product_image_selection::ResolvedProductImage { + product_image_selection::ResolvedProductImage { + product_version: "2.9.0".to_string(), + app_version_label_value: "2.9.0".parse().expect("valid label value"), + image: "oci.stackable.tech/sdp/nifi:2.9.0-stackable0.0.0-dev".to_string(), + image_pull_policy: "IfNotPresent".to_string(), + pull_secrets: None, + } +} + #[cfg(test)] mod tests { use pretty_assertions::assert_eq; @@ -329,7 +358,7 @@ mod tests { serde_yaml::from_str(NIFI_VECTOR_ENABLED_YAML).expect("invalid test YAML"); let aggregator = Some(ConfigMapName::from_str("nifi-vector-aggregator-discovery").unwrap()); - let configs = build_role_group_configs(&nifi, &aggregator) + let configs = build_role_group_configs(&nifi, &test_resolved_product_image(), &aggregator) .expect("role group configs should validate"); let vector = default_rg(&configs) @@ -347,9 +376,13 @@ mod tests { let nifi: v1alpha1::NifiCluster = serde_yaml::from_str(NIFI_VECTOR_ENABLED_YAML).expect("invalid test YAML"); - let error = build_role_group_configs(&nifi, &None) - .expect_err("a missing aggregator ConfigMap name must fail when Vector is enabled"); - assert!(matches!(error, Error::MissingVectorAggregatorConfigMapName)); + // `ValidatedRoleGroupConfig` is not `Debug`, so match on the result rather than using + // `expect_err` (which would require the `Ok` value to be `Debug`). + let result = build_role_group_configs(&nifi, &test_resolved_product_image(), &None); + assert!(matches!( + result, + Err(Error::MissingVectorAggregatorConfigMapName) + )); } #[test] @@ -358,8 +391,8 @@ mod tests { serde_yaml::from_str(NIFI_VECTOR_DISABLED_YAML).expect("invalid test YAML"); // The aggregator name is not required when the Vector agent is disabled. - let configs = - build_role_group_configs(&nifi, &None).expect("role group configs should validate"); + let configs = build_role_group_configs(&nifi, &test_resolved_product_image(), &None) + .expect("role group configs should validate"); assert!(default_rg(&configs).vector_container.is_none()); } diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 06e1fa3b..d8b50050 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -33,7 +33,10 @@ mod tests { }; use super::*; - use crate::{controller::validate::build_role_group_configs, crd::v1alpha1}; + use crate::{ + controller::validate::{build_role_group_configs, test_resolved_product_image}, + crd::v1alpha1, + }; #[test] fn test_affinity_defaults() { @@ -60,7 +63,8 @@ mod tests { let nifi: v1alpha1::NifiCluster = serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); - let role_group_configs = build_role_group_configs(&nifi, &None).unwrap(); + let role_group_configs = + build_role_group_configs(&nifi, &test_resolved_product_image(), &None).unwrap(); let merged_config = &role_group_configs .get(&NifiRole::Node) .and_then(|groups| { diff --git a/rust/operator-binary/src/nifi_controller.rs b/rust/operator-binary/src/nifi_controller.rs index 57f74c82..f1008fcc 100644 --- a/rust/operator-binary/src/nifi_controller.rs +++ b/rust/operator-binary/src/nifi_controller.rs @@ -124,9 +124,6 @@ pub enum Error { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("failed to build git-sync resources"))] - BuildGitSyncResources { source: build::git_sync::Error }, - #[snafu(display("failed to patch service account"))] ApplyServiceAccount { source: stackable_operator::cluster_resources::Error, @@ -291,10 +288,6 @@ pub async fn reconcile_nifi( async { tracing::debug!("Processing rolegroup {role_group_name}"); - let git_sync_resources = - build::git_sync::build_git_sync_resources(&validated_cluster, rg) - .context(BuildGitSyncResourcesSnafu)?; - let rg_headless_service = build_rolegroup_headless_service(&validated_cluster, role_group_name); @@ -326,7 +319,6 @@ pub async fn reconcile_nifi( rolling_upgrade_supported, replicas, &rbac_sa.name_any(), - &git_sync_resources, ) .await .with_context(|_| BuildStatefulSetSnafu { From b264e5c47674306843e5f7f31f9dd46269fe57c7 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 13 Jun 2026 23:28:29 +0200 Subject: [PATCH 38/39] refactor: provide name to ValidatedRoleGroupConfig --- .../src/controller/build/config_map.rs | 43 ++++++------------- .../build/properties/security_properties.rs | 7 ++- .../controller/build/resource/statefulset.rs | 29 +++++-------- rust/operator-binary/src/controller/mod.rs | 4 ++ .../src/controller/validate.rs | 3 +- rust/operator-binary/src/nifi_controller.rs | 3 +- 6 files changed, 37 insertions(+), 52 deletions(-) diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 85056a55..6aa49fac 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -1,6 +1,6 @@ //! Build per-rolegroup `ConfigMap` for the NiFi cluster. -use snafu::{OptionExt, ResultExt, Snafu}; +use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, k8s_openapi::api::core::v1::ConfigMap, @@ -9,18 +9,15 @@ use stackable_operator::{ v2::{builder::meta::ownerreference_from_resource, types::operator::RoleGroupName}, }; -use crate::{ - controller::{ - ValidatedCluster, - build::{ - properties::{ - ConfigFileName, authorizers, bootstrap_conf, login_identity_providers, - nifi_properties, product_logging, security_properties, state_management_xml, - }, - proxy_hosts, +use crate::controller::{ + ValidatedCluster, ValidatedRoleGroupConfig, + build::{ + properties::{ + ConfigFileName, authorizers, bootstrap_conf, login_identity_providers, nifi_properties, + product_logging, security_properties, state_management_xml, }, + proxy_hosts, }, - crd::NifiRole, }; #[derive(Debug, Snafu)] @@ -54,9 +51,6 @@ pub enum Error { InvalidNifiAuthenticationConfig { source: crate::security::authentication::Error, }, - - #[snafu(display("the cluster has no rolegroup [{role_group}] in role [{role}]"))] - MissingRoleGroup { role: String, role_group: String }, } type Result = std::result::Result; @@ -67,20 +61,11 @@ type Result = std::result::Result; /// All NiFi configuration is sourced from `cluster`. pub fn build_rolegroup_config_map( cluster: &ValidatedCluster, - role_group_name: &RoleGroupName, + rg: &ValidatedRoleGroupConfig, cluster_info: &KubernetesClusterInfo, ) -> Result { tracing::debug!("building rolegroup ConfigMap"); - let rg = cluster - .role_group_configs - .get(&NifiRole::Node) - .and_then(|groups| groups.get(role_group_name)) - .with_context(|| MissingRoleGroupSnafu { - role: NifiRole::Node.to_string(), - role_group: role_group_name.to_string(), - })?; - let proxy_hosts = proxy_hosts::compute_proxy_hosts(cluster, cluster_info); let mut cm_builder = ConfigMapBuilder::new(); @@ -91,12 +76,12 @@ pub fn build_rolegroup_config_map( .name_and_namespace(cluster) .name( cluster - .resource_names(role_group_name) + .resource_names(&rg.name) .role_group_config_map() .to_string(), ) .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) - .with_labels(cluster.recommended_labels(role_group_name)) + .with_labels(cluster.recommended_labels(&rg.name)) .build(), ) .add_data( @@ -108,7 +93,7 @@ pub fn build_rolegroup_config_map( ConfigFileName::NifiProperties.to_string(), nifi_properties::build(cluster, rg, &proxy_hosts).with_context(|_| { BuildNifiPropertiesSnafu { - rolegroup: role_group_name.clone(), + rolegroup: rg.name.clone(), } })?, ) @@ -128,7 +113,7 @@ pub fn build_rolegroup_config_map( .add_data( ConfigFileName::SecurityProperties.to_string(), security_properties::build(rg).with_context(|_| JvmSecurityPropertiesSnafu { - rolegroup: role_group_name.clone(), + rolegroup: rg.name.clone(), })?, ); @@ -146,6 +131,6 @@ pub fn build_rolegroup_config_map( cm_builder .build() .with_context(|_| BuildRoleGroupConfigSnafu { - rolegroup: role_group_name.clone(), + rolegroup: rg.name.clone(), }) } diff --git a/rust/operator-binary/src/controller/build/properties/security_properties.rs b/rust/operator-binary/src/controller/build/properties/security_properties.rs index 10f4771a..401291d1 100644 --- a/rust/operator-binary/src/controller/build/properties/security_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/security_properties.rs @@ -35,8 +35,13 @@ mod tests { }; fn make_rg(overrides: Option>) -> ValidatedRoleGroupConfig { - use stackable_operator::v2::role_utils::JavaCommonConfig; + use std::str::FromStr as _; + + use stackable_operator::v2::{ + role_utils::JavaCommonConfig, types::operator::RoleGroupName, + }; ValidatedRoleGroupConfig { + name: RoleGroupName::from_str("default").expect("valid role-group name"), replicas: 1, config: NifiConfig::default(), config_overrides: NifiConfigOverrides { diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index e6c0ba41..7a42ed85 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -43,10 +43,7 @@ use stackable_operator::{ pod::container::{EnvVarSet, new_container_builder}, }, product_logging::framework::vector_container, - types::{ - kubernetes::{ContainerName, VolumeName}, - operator::RoleGroupName, - }, + types::kubernetes::{ContainerName, VolumeName}, }, }; @@ -67,7 +64,7 @@ use crate::{ }, crd::{ BALANCE_PORT, BALANCE_PORT_NAME, Container, HTTPS_PORT, HTTPS_PORT_NAME, METRICS_PORT, - METRICS_PORT_NAME, NifiConfig, NifiRole, NifiRoleType, PROTOCOL_PORT, PROTOCOL_PORT_NAME, + METRICS_PORT_NAME, NifiRole, NifiRoleType, PROTOCOL_PORT, PROTOCOL_PORT_NAME, STACKABLE_LOG_CONFIG_DIR, STACKABLE_LOG_DIR, authorization::NifiAccessPolicyProvider, constants::{NIFI_CONFIG_DIRECTORY, NIFI_PYTHON_WORKING_DIRECTORY}, @@ -165,7 +162,6 @@ stackable_operator::constant!(VECTOR_LOG_VOLUME_NAME: VolumeName = "log"); pub(crate) async fn build_node_rolegroup_statefulset( cluster: &ValidatedCluster, cluster_info: &KubernetesClusterInfo, - role_group_name: &RoleGroupName, role: &NifiRoleType, rg: &ValidatedRoleGroupConfig, rolling_update_supported: bool, @@ -182,7 +178,7 @@ pub(crate) async fn build_node_rolegroup_statefulset( let git_sync_resources = &rg.git_sync_resources; // Type-safe names for this role group's resources (StatefulSet, ConfigMap, headless Service). - let resource_names = cluster.resource_names(role_group_name); + let resource_names = cluster.resource_names(&rg.name); // The validated, merged `NifiConfig` is the single source of truth; the ConfigMap builder // sources the same `rg.config`. @@ -459,7 +455,7 @@ pub(crate) async fn build_node_rolegroup_statefulset( let mut pod_builder = PodBuilder::new(); - let recommended_object_labels = cluster.recommended_labels(role_group_name); + let recommended_object_labels = cluster.recommended_labels(&rg.name); add_graceful_shutdown_config(merged_config, &mut pod_builder).context(GracefulShutdownSnafu)?; @@ -616,8 +612,7 @@ pub(crate) async fn build_node_rolegroup_statefulset( KEYSTORE_VOLUME_NAME, [ crate::controller::build::resource::service::metrics_service_name( - cluster, - role_group_name, + cluster, &rg.name, ), build_reporting_task_service_name(&nifi_cluster_name), ], @@ -675,7 +670,7 @@ pub(crate) async fn build_node_rolegroup_statefulset( pod_management_policy: Some("Parallel".to_string()), replicas, selector: LabelSelector { - match_labels: Some(cluster.role_group_selector(role_group_name).into()), + match_labels: Some(cluster.role_group_selector(&rg.name).into()), ..LabelSelector::default() }, service_name: Some(resource_names.headless_service_name().to_string()), @@ -688,11 +683,7 @@ pub(crate) async fn build_node_rolegroup_statefulset( }, ..StatefulSetUpdateStrategy::default() }), - volume_claim_templates: Some(get_volume_claim_templates( - cluster, - role_group_name, - merged_config, - )?), + volume_claim_templates: Some(get_volume_claim_templates(cluster, rg)?), ..StatefulSetSpec::default() }), status: None, @@ -701,9 +692,9 @@ pub(crate) async fn build_node_rolegroup_statefulset( fn get_volume_claim_templates( cluster: &ValidatedCluster, - role_group_name: &RoleGroupName, - merged_config: &NifiConfig, + rg: &ValidatedRoleGroupConfig, ) -> Result> { + let merged_config = &rg.config; let authorization_config = &cluster.cluster_config.authorization; let mut pvcs = vec![ merged_config.resources.storage.content_repo.build_pvc( @@ -730,7 +721,7 @@ fn get_volume_claim_templates( // Used for PVC templates that cannot be modified once they are deployed, so the version label // is set to the placeholder `none` to keep the labels stable across version upgrades. - let unversioned_recommended_labels = cluster.recommended_labels_unversioned(role_group_name); + let unversioned_recommended_labels = cluster.recommended_labels_unversioned(&rg.name); // listener endpoints will use persistent volumes // so that load balancers can hard-code the target addresses and diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index aaaf569d..e95f49f0 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -53,6 +53,10 @@ pub(crate) mod validate; /// [`with_validated_config`](stackable_operator::v2::role_utils::with_validated_config) in the /// [`validate`] step; downstream builders consume this rather than the raw `NifiCluster`. pub struct ValidatedRoleGroupConfig { + /// The role-group name (the key under which this config is stored in + /// [`ValidatedCluster::role_group_configs`]). Carried here so builders that consume the config + /// don't also need the name threaded through as a separate parameter. + pub name: RoleGroupName, /// The desired number of replicas (defaulted to 1 during validation). /// /// The StatefulSet replica count is sourced from the raw role-group spec in `nifi_controller` diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 421459ce..028cf7fa 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -259,8 +259,9 @@ pub(crate) fn build_role_group_configs( .context(BuildGitSyncResourcesSnafu)?; groups.insert( - role_group_name, + role_group_name.clone(), ValidatedRoleGroupConfig { + name: role_group_name, replicas: validated.replicas.unwrap_or(1), config, config_overrides, diff --git a/rust/operator-binary/src/nifi_controller.rs b/rust/operator-binary/src/nifi_controller.rs index f1008fcc..6957d104 100644 --- a/rust/operator-binary/src/nifi_controller.rs +++ b/rust/operator-binary/src/nifi_controller.rs @@ -295,7 +295,7 @@ pub async fn reconcile_nifi( let rg_configmap = build::config_map::build_rolegroup_config_map( &validated_cluster, - role_group_name, + rg, &client.kubernetes_cluster_info, ) .context(BuildRoleGroupConfigMapSnafu { @@ -313,7 +313,6 @@ pub async fn reconcile_nifi( let rg_statefulset = build_node_rolegroup_statefulset( &validated_cluster, &client.kubernetes_cluster_info, - role_group_name, role, rg, rolling_upgrade_supported, From 4717dd081036b0399a4f05b27baf65e689deecf3 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 14 Jun 2026 15:32:45 +0200 Subject: [PATCH 39/39] refactor: remove raw nifi cluster from upgrade --- .../operator-binary/src/controller/upgrade.rs | 24 ++++++------------- rust/operator-binary/src/nifi_controller.rs | 13 ++++------ 2 files changed, 11 insertions(+), 26 deletions(-) diff --git a/rust/operator-binary/src/controller/upgrade.rs b/rust/operator-binary/src/controller/upgrade.rs index 53183bd0..787e1228 100644 --- a/rust/operator-binary/src/controller/upgrade.rs +++ b/rust/operator-binary/src/controller/upgrade.rs @@ -5,11 +5,9 @@ use snafu::{ResultExt, Snafu}; use stackable_operator::{ client::Client, k8s_openapi::{api::apps::v1::StatefulSet, apimachinery::pkg::apis::meta::v1::LabelSelector}, - kvp::Labels, - v2::types::kubernetes::NamespaceName, }; -use crate::crd::{APP_NAME, NifiRole, v1alpha1}; +use super::ValidatedCluster; #[derive(Snafu, Debug)] pub enum Error { @@ -17,11 +15,6 @@ pub enum Error { FetchStatefulsets { source: stackable_operator::client::Error, }, - - #[snafu(display("failed to build labels"))] - LabelBuild { - source: stackable_operator::kvp::LabelError, - }, } type Result = std::result::Result; @@ -37,12 +30,13 @@ pub enum ClusterVersionUpdateState { } pub async fn cluster_version_update_state( - nifi: &v1alpha1::NifiCluster, + cluster: &ValidatedCluster, client: &Client, - namespace: &NamespaceName, - resolved_version: &String, deployed_version: Option<&String>, ) -> Result { + // The version we want to converge to, i.e. the resolved product image version. + let resolved_version = &cluster.image.product_version; + // Handle full restarts for a version change match deployed_version { Some(deployed_version) => { @@ -50,16 +44,12 @@ pub async fn cluster_version_update_state( // Check if statefulsets are already scaled to zero, if not - requeue let selector = LabelSelector { match_expressions: None, - match_labels: Some( - Labels::role_selector(nifi, APP_NAME, &NifiRole::Node.to_string()) - .context(LabelBuildSnafu)? - .into(), - ), + match_labels: Some(cluster.role_selector().into()), }; // Retrieve the deployed statefulsets to check on the current status of the restart let deployed_statefulsets = client - .list_with_label_selector::(namespace.as_ref(), &selector) + .list_with_label_selector::(cluster.namespace.as_ref(), &selector) .await .context(FetchStatefulsetsSnafu)?; diff --git a/rust/operator-binary/src/nifi_controller.rs b/rust/operator-binary/src/nifi_controller.rs index 6957d104..59a25cf7 100644 --- a/rust/operator-binary/src/nifi_controller.rs +++ b/rust/operator-binary/src/nifi_controller.rs @@ -224,15 +224,10 @@ pub async fn reconcile_nifi( && deployed_version.is_some_and(|v| v.starts_with("2.")); if !rolling_upgrade_supported { - cluster_version_update_state = upgrade::cluster_version_update_state( - nifi, - client, - &validated_cluster.namespace, - &resolved_product_image.product_version, - deployed_version, - ) - .await - .context(ClusterVersionUpdateStateSnafu)?; + cluster_version_update_state = + upgrade::cluster_version_update_state(&validated_cluster, client, deployed_version) + .await + .context(ClusterVersionUpdateStateSnafu)?; if cluster_version_update_state == ClusterVersionUpdateState::UpdateInProgress { return Ok(Action::await_change());