From c4c298f24c9f31fe7865917e0986dee452b5a15c Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Jun 2026 14:28:24 +0200 Subject: [PATCH 01/56] chore: switch to smooth-operator branch and vendor the config writer Activate the operator-rs smooth-operator patch (matching trino/hdfs) and add a vendored Hadoop XML / Java-properties writer copied from hdfs-operator, so the operator no longer needs product-config for rendering. Reroute discovery and the controller to the vendored writer. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 643 +++++++++---------- Cargo.toml | 6 +- rust/operator-binary/Cargo.toml | 2 + rust/operator-binary/src/config/mod.rs | 1 + rust/operator-binary/src/config/writer.rs | 145 +++++ rust/operator-binary/src/discovery.rs | 2 +- rust/operator-binary/src/hbase_controller.rs | 15 +- 7 files changed, 456 insertions(+), 358 deletions(-) create mode 100644 rust/operator-binary/src/config/writer.rs diff --git a/Cargo.lock b/Cargo.lock index 3d1d136e..18141326 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,9 +47,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -62,15 +62,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] @@ -103,9 +103,9 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "arc-swap" -version = "1.8.2" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f3647c145568cec02c42054e07bdf9a5a698e15b466fb2341bfc393cd24aa5" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" dependencies = [ "rustversion", ] @@ -163,15 +163,15 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "axum" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ "axum-core", "bytes", @@ -265,9 +265,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "2.11.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +checksum = "84d7ced0ae9557296835c32bf1b1e02b44c746701f898460fb000d7eaa84f00a" [[package]] name = "block-buffer" @@ -280,9 +280,9 @@ dependencies = [ [[package]] name = "built" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4ad8f11f288f48ca24471bbd51ac257aaeaaa07adae295591266b792902ae64" +checksum = "5c0e531d93d39c34eef561e929e8a7f86d77a5af08aac4f6d6e39976c51858e9" dependencies = [ "chrono", "git2", @@ -290,9 +290,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.20.2" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytes" @@ -302,9 +302,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "cc" -version = "1.2.56" +version = "1.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" dependencies = [ "find-msvc-tools", "jobserver", @@ -331,9 +331,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.60" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", "clap_derive", @@ -341,9 +341,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.60" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstream", "anstyle", @@ -353,9 +353,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.55" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ "heck", "proc-macro2", @@ -365,15 +365,15 @@ dependencies = [ [[package]] name = "clap_lex" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "colorchoice" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "concurrent-queue" @@ -392,11 +392,12 @@ checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "const_format" -version = "0.2.35" +version = "0.2.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7faa7469a93a566e9ccc1c73fe783b4a65c274c5ace346038dca9c39fe0030ad" +checksum = "4481a617ad9a412be3b97c5d403fef8ed023103368908b9c50af598ff467cc1e" dependencies = [ "const_format_proc_macros", + "konst", ] [[package]] @@ -612,9 +613,9 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", @@ -675,9 +676,9 @@ dependencies = [ [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "elliptic-curve" @@ -787,9 +788,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.3.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "ff" @@ -923,9 +924,9 @@ checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-timer" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" [[package]] name = "futures-util" @@ -982,15 +983,14 @@ dependencies = [ [[package]] name = "git2" -version = "0.20.4" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" +checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" dependencies = [ "bitflags", "libc", "libgit2-sys", "log", - "url", ] [[package]] @@ -1024,9 +1024,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" dependencies = [ "atomic-waker", "bytes", @@ -1052,6 +1052,12 @@ dependencies = [ "foldhash", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "heck" version = "0.5.0" @@ -1080,9 +1086,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" dependencies = [ "bytes", "itoa", @@ -1131,9 +1137,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hyper" -version = "1.8.1" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" dependencies = [ "atomic-waker", "bytes", @@ -1146,7 +1152,6 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "pin-utils", "smallvec", "tokio", "want", @@ -1154,9 +1159,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.7" +version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ "http", "hyper", @@ -1164,7 +1169,6 @@ dependencies = [ "log", "rustls", "rustls-native-certs", - "rustls-pki-types", "tokio", "tokio-rustls", "tower-service", @@ -1232,12 +1236,13 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", + "utf8_iter", "yoke", "zerofrom", "zerovec", @@ -1245,9 +1250,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", @@ -1258,9 +1263,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -1272,15 +1277,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ "icu_collections", "icu_locale_core", @@ -1292,15 +1297,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", @@ -1330,9 +1335,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", @@ -1340,12 +1345,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.17.1", ] [[package]] @@ -1359,19 +1364,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.11.0" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" - -[[package]] -name = "iri-string" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" -dependencies = [ - "memchr", - "serde", -] +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "is_terminal_polyfill" @@ -1390,9 +1385,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "java-properties" @@ -1407,9 +1402,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.21" +version = "0.2.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3e3d65f018c6ae946ab16e80944b97096ed73c35b221d1c478a6c81d8f57940" +checksum = "4603d3033e49e2b0e31229fcab20a5d40089c607d975cd9c80551dc69eed9102" dependencies = [ "jiff-static", "jiff-tzdb-platform", @@ -1417,14 +1412,14 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.61.2", + "windows-link", ] [[package]] name = "jiff-static" -version = "0.2.21" +version = "0.2.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17c2b211d863c7fde02cbea8a3c1a439b98e109286554f2860bdded7ff83818" +checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47" dependencies = [ "proc-macro2", "quote", @@ -1433,9 +1428,9 @@ dependencies = [ [[package]] name = "jiff-tzdb" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68971ebff725b9e2ca27a601c5eb38a4c5d64422c4cbab0c535f248087eda5c2" +checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076" [[package]] name = "jiff-tzdb-platform" @@ -1458,24 +1453,27 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.90" +version = "0.3.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14dc6f6450b3f6d4ed5b16327f38fed626d375a886159ca555bd7822c0c3a5a6" +checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] [[package]] name = "json-patch" -version = "4.1.0" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f300e415e2134745ef75f04562dd0145405c2f7fd92065db029ac4b16b57fe90" +checksum = "7421438de105a0827e44fadd05377727847d717c80ce29a229f85fd04c427b72" dependencies = [ "jsonptr", + "schemars", "serde", "serde_json", - "thiserror 1.0.69", + "thiserror 2.0.18", ] [[package]] @@ -1503,9 +1501,9 @@ dependencies = [ [[package]] name = "k8s-openapi" -version = "0.27.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05a6d6f3611ad1d21732adbd7a2e921f598af6c92d71ae6e2620da4b67ee1f0d" +checksum = "51b326f5219dd55872a72c1b6ddd1b830b8334996c667449c29391d657d78d5e" dependencies = [ "base64", "jiff", @@ -1517,13 +1515,28 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "darling", "regex", - "snafu 0.9.0", + "snafu 0.9.1", +] + +[[package]] +name = "konst" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128133ed7824fcd73d6e7b17957c5eb7bacb885649bd8c69708b2331a10bcefb" +dependencies = [ + "konst_macro_rules", ] +[[package]] +name = "konst_macro_rules" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" + [[package]] name = "kube" version = "3.1.0" @@ -1617,7 +1630,7 @@ dependencies = [ "backon", "educe", "futures 0.3.32", - "hashbrown", + "hashbrown 0.16.1", "hostname", "json-patch", "k8s-openapi", @@ -1643,15 +1656,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.182" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libgit2-sys" -version = "0.18.3+1.9.2" +version = "0.18.5+1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487" +checksum = "005d6ae6eac1912906073e069f7db60b1fa98e052a68227824afe3e3a1c59ca2" dependencies = [ "cc", "libc", @@ -1667,9 +1680,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libz-sys" -version = "1.1.24" +version = "1.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4735e9cbde5aac84a5ce588f6b23a90b9b0b528f6c5a8db8a4aff300463a0839" +checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" dependencies = [ "cc", "libc", @@ -1679,9 +1692,9 @@ dependencies = [ [[package]] name = "litemap" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "lock_api" @@ -1694,9 +1707,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "113b30b4cd05f7c06868fdb2854f66a7b9fece9a48425351cd532e810d74024f" [[package]] name = "matchers" @@ -1715,9 +1728,9 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" [[package]] name = "mime" @@ -1737,9 +1750,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.1" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi", @@ -1766,16 +1779,16 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand 0.8.5", + "rand 0.8.6", "smallvec", "zeroize", ] [[package]] name = "num-conv" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" @@ -1809,9 +1822,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "once_cell_polyfill" @@ -1866,9 +1879,9 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.31.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2366db2dca4d2ad033cad11e6ee42844fd727007af5ad04a1730f4cb8163bf" +checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" dependencies = [ "http", "opentelemetry", @@ -1913,7 +1926,7 @@ dependencies = [ "futures-util", "opentelemetry", "percent-encoding", - "rand 0.9.2", + "rand 0.9.4", "thiserror 2.0.18", "tokio", "tokio-stream", @@ -2039,18 +2052,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.10" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.10" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", @@ -2059,15 +2072,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" - -[[package]] -name = "pin-utils" -version = "0.1.0" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pkcs1" @@ -2092,9 +2099,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "portable-atomic" @@ -2104,18 +2111,18 @@ checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "portable-atomic-util" -version = "0.2.5" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a9db96d7fa8782dd8c15ce32ffe8680bbd1e978a43bf51a34d39483540495f5" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" dependencies = [ "portable-atomic", ] [[package]] name = "potential_utf" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] @@ -2146,9 +2153,9 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "3.4.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ "toml_edit", ] @@ -2203,9 +2210,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.44" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -2218,9 +2225,9 @@ checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "rand_chacha 0.3.1", "rand_core 0.6.4", @@ -2228,9 +2235,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -2457,9 +2464,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.37" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "log", "once_cell", @@ -2472,9 +2479,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -2484,9 +2491,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ "zeroize", ] @@ -2516,9 +2523,9 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] @@ -2603,9 +2610,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" @@ -2660,9 +2667,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -2746,9 +2753,9 @@ checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f" [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "signal-hook-registry" @@ -2772,9 +2779,9 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" [[package]] name = "slab" @@ -2809,11 +2816,11 @@ dependencies = [ [[package]] name = "snafu" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1d4bced6a69f90b2056c03dcff2c4737f98d6fb9e0853493996e1d253ca29c6" +checksum = "d1a012328be2e3f5d5f6f3218147ca02588cea4cb865e876849ab6debcf36522" dependencies = [ - "snafu-derive 0.9.0", + "snafu-derive 0.9.1", ] [[package]] @@ -2841,9 +2848,9 @@ dependencies = [ [[package]] name = "snafu-derive" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" +checksum = "5f103c50866b8743da9429b8a581d81a27c2d3a9c4ac7df8f8571c1dd7896eda" dependencies = [ "heck", "proc-macro2", @@ -2853,12 +2860,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.2" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -2886,19 +2893,19 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "const-oid", "ecdsa", "k8s-openapi", "kube", "p256", - "rand 0.9.2", + "rand 0.9.4", "rand_core 0.6.4", "rsa", "sha2", "signature", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-shared", "tokio", "tokio-rustls", @@ -2918,23 +2925,25 @@ dependencies = [ "fnv", "futures 0.3.32", "indoc", + "java-properties", "product-config", "rstest", "serde", "serde_json", "serde_yaml", "shell-escape", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-operator", "strum", "tokio", "tracing", + "xml", ] [[package]] name = "stackable-operator" version = "0.111.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "base64", "clap", @@ -2951,14 +2960,14 @@ dependencies = [ "k8s-openapi", "kube", "product-config", - "rand 0.9.2", + "rand 0.9.4", "regex", "schemars", "semver", "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-operator-derive", "stackable-shared", "stackable-telemetry", @@ -2970,12 +2979,13 @@ dependencies = [ "tracing-appender", "tracing-subscriber", "url", + "uuid", ] [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "darling", "proc-macro2", @@ -2986,7 +2996,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "jiff", "k8s-openapi", @@ -2995,7 +3005,7 @@ dependencies = [ "semver", "serde", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "strum", "time", ] @@ -3003,7 +3013,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "axum", "clap", @@ -3014,7 +3024,7 @@ dependencies = [ "opentelemetry-semantic-conventions", "opentelemetry_sdk", "pin-project", - "snafu 0.9.0", + "snafu 0.9.1", "strum", "tokio", "tower", @@ -3027,21 +3037,21 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "kube", "schemars", "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-versioned-macros", ] [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "convert_case", "convert_case_extras", @@ -3059,7 +3069,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#7a5f0c3fbcd091340214a23f0607fcd4b4fcc152" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "arc-swap", "async-trait", @@ -3072,10 +3082,10 @@ dependencies = [ "kube", "opentelemetry", "opentelemetry-semantic-conventions", - "rand 0.9.2", + "rand 0.9.4", "serde", "serde_json", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-certs", "stackable-shared", "stackable-telemetry", @@ -3121,6 +3131,12 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + [[package]] name = "syn" version = "1.0.109" @@ -3245,9 +3261,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", "zerovec", @@ -3276,9 +3292,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.49.0" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -3293,9 +3309,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", @@ -3339,18 +3355,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.7.5+spec-1.1.0" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" dependencies = [ "serde_core", ] [[package]] name = "toml_edit" -version = "0.23.10+spec-1.0.0" +version = "0.25.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269" +checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" dependencies = [ "indexmap", "toml_datetime", @@ -3360,18 +3376,18 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.0.9+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ "winnow", ] [[package]] name = "tonic" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "base64", @@ -3396,9 +3412,9 @@ dependencies = [ [[package]] name = "tonic-prost" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" dependencies = [ "bytes", "prost", @@ -3426,9 +3442,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "base64", "bitflags", @@ -3436,13 +3452,13 @@ dependencies = [ "futures-util", "http", "http-body", - "iri-string", "mime", "pin-project-lite", "tower", "tower-layer", "tower-service", "tracing", + "url", ] [[package]] @@ -3471,11 +3487,12 @@ dependencies = [ [[package]] name = "tracing-appender" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" dependencies = [ "crossbeam-channel", + "symlink", "thiserror 2.0.18", "time", "tracing-subscriber", @@ -3541,9 +3558,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.22" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ "matchers", "nu-ansi-term", @@ -3568,9 +3585,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "ucd-trie" @@ -3586,9 +3603,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-segmentation" -version = "1.12.0" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] name = "unicode-xid" @@ -3633,6 +3650,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -3668,18 +3695,18 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.2+wasi-0.2.9" +version = "1.0.3+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" dependencies = [ "wit-bindgen", ] [[package]] name = "wasm-bindgen" -version = "0.2.113" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60722a937f594b7fde9adb894d7c092fc1bb6612897c46368d18e7a20208eff2" +checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" dependencies = [ "cfg-if", "once_cell", @@ -3690,23 +3717,19 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.63" +version = "0.4.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a89f4650b770e4521aa6573724e2aed4704372151bd0de9d16a3bbabb87441a" +checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" dependencies = [ - "cfg-if", - "futures-util", "js-sys", - "once_cell", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.113" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac8c6395094b6b91c4af293f4c79371c163f9a6f56184d2c9a85f5a95f3950" +checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3714,9 +3737,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.113" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3fabce6159dc20728033842636887e4877688ae94382766e00b180abac9d60" +checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" dependencies = [ "bumpalo", "proc-macro2", @@ -3727,18 +3750,18 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.113" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de0e091bdb824da87dc01d967388880d017a0a9bc4f3bdc0d86ee9f9336e3bb5" +checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" -version = "0.3.90" +version = "0.3.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "705eceb4ce901230f8625bd1d665128056ccbe4b7408faa625eec1ba80f59a97" +checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" dependencies = [ "js-sys", "wasm-bindgen", @@ -3819,16 +3842,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", + "windows-targets", ] [[package]] @@ -3846,31 +3860,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -3879,116 +3876,68 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" -version = "0.7.14" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" dependencies = [ "memchr", ] [[package]] name = "wit-bindgen" -version = "0.51.0" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "x509-cert" @@ -4006,15 +3955,15 @@ dependencies = [ [[package]] name = "xml" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8aa498d22c9bbaf482329839bc5620c46be275a19a812e9a22a2b07529a642a" +checksum = "636f85e5ca6488e96401b61eb7de54f4e44755c988af0f52cf90230c312a1a89" [[package]] name = "yoke" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -4023,9 +3972,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", @@ -4035,18 +3984,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.40" +version = "0.8.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5" +checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.40" +version = "0.8.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953" +checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" dependencies = [ "proc-macro2", "quote", @@ -4055,18 +4004,18 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", @@ -4096,9 +4045,9 @@ dependencies = [ [[package]] name = "zerotrie" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", @@ -4107,9 +4056,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ "yoke", "zerofrom", @@ -4118,9 +4067,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 0375c93e..c2e49e69 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ const_format = "0.2" fnv = "1.0" futures = { version = "0.3", features = ["compat"] } indoc = "2.0" +java-properties = "2.0" rstest = "0.26" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" @@ -29,7 +30,8 @@ snafu = "0.9" strum = { version = "0.28", features = ["derive"] } tokio = { version = "1.40", features = ["full"] } tracing = "0.1" +xml = "1.3" -# [patch."https://github.com/stackabletech/operator-rs.git"] -# stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } +[patch."https://github.com/stackabletech/operator-rs.git"] +stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "smooth-operator" } # stackable-operator = { path = "../operator-rs/crates/stackable-operator" } diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 3472b246..336c050d 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -18,6 +18,7 @@ const_format.workspace = true fnv.workspace = true futures.workspace = true indoc.workspace = true +java-properties.workspace = true serde.workspace = true serde_json.workspace = true shell-escape.workspace = true @@ -25,6 +26,7 @@ snafu.workspace = true strum.workspace = true tokio.workspace = true tracing.workspace = true +xml.workspace = true [build-dependencies] built.workspace = true diff --git a/rust/operator-binary/src/config/mod.rs b/rust/operator-binary/src/config/mod.rs index 271c6d99..cb7d7920 100644 --- a/rust/operator-binary/src/config/mod.rs +++ b/rust/operator-binary/src/config/mod.rs @@ -1 +1,2 @@ pub mod jvm; +pub mod writer; diff --git a/rust/operator-binary/src/config/writer.rs b/rust/operator-binary/src/config/writer.rs new file mode 100644 index 00000000..08250f9f --- /dev/null +++ b/rust/operator-binary/src/config/writer.rs @@ -0,0 +1,145 @@ +//! Writers for Hadoop XML config files and Java `.properties` files. +//! +//! Vendored from the `product-config` crate's `writer` module so the operator no +//! longer depends on `product-config` for rendering. + +use std::io::Write; + +use java_properties::{PropertiesError, PropertiesWriter}; +use snafu::{ResultExt, Snafu}; +use xml::escape::escape_str_attribute; + +#[derive(Debug, Snafu)] +pub enum PropertiesWriterError { + #[snafu(display("failed to create properties file"))] + Properties { source: PropertiesError }, + + #[snafu(display("failed to convert properties file byte array to UTF-8"))] + FromUtf8 { source: std::string::FromUtf8Error }, +} + +/// Creates a common Java properties file string in the format: +/// `property_1=value_1\nproperty_2=value_2\n`. +pub fn to_java_properties_string<'a, T>(properties: T) -> Result +where + T: Iterator)>, +{ + let mut output = Vec::new(); + write_java_properties(&mut output, properties)?; + String::from_utf8(output).context(FromUtf8Snafu) +} + +/// Writes Java properties to the given writer. A `None` value is written as an +/// empty value (`key=`). +fn write_java_properties<'a, W, T>(writer: W, properties: T) -> Result<(), PropertiesWriterError> +where + W: Write, + T: Iterator)>, +{ + let mut writer = PropertiesWriter::new(writer); + for (k, v) in properties { + let property_value = v.as_deref().unwrap_or_default(); + writer.write(k, property_value).context(PropertiesSnafu)?; + } + writer.flush().context(PropertiesSnafu)?; + Ok(()) +} + +/// Converts properties into a Hadoop configuration XML, including the wrapping +/// `...` elements. Properties with a `None` value +/// are skipped. Keys and values are XML-escaped. +pub fn to_hadoop_xml<'a, T>(properties: T) -> String +where + T: Iterator)>, +{ + let mut snippet = String::new(); + for (k, v) in properties { + let escaped_value = match v { + Some(value) => escape_str_attribute(value), + None => continue, + }; + let escaped_key = escape_str_attribute(k); + snippet.push_str(&format!( + " \n {escaped_key}\n {escaped_value}\n \n" + )); + } + format!("\n\n{snippet}") +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use super::*; + + fn xml(pairs: &[(&str, Option<&str>)]) -> String { + let map: BTreeMap> = pairs + .iter() + .map(|(k, v)| (k.to_string(), v.map(str::to_string))) + .collect(); + to_hadoop_xml(map.iter()) + } + + fn props(pairs: &[(&str, Option<&str>)]) -> String { + let map: BTreeMap> = pairs + .iter() + .map(|(k, v)| (k.to_string(), v.map(str::to_string))) + .collect(); + to_java_properties_string(map.iter()).unwrap() + } + + #[test] + fn hadoop_xml_wraps_empty_configuration() { + assert_eq!( + xml(&[]), + "\n\n" + ); + } + + #[test] + fn hadoop_xml_renders_single_property() { + assert_eq!( + xml(&[("fs.defaultFS", Some("hdfs://hdfs/"))]), + "\n\n \ + \n fs.defaultFS\n \ + hdfs://hdfs/\n \n" + ); + } + + #[test] + fn hadoop_xml_skips_none_values() { + assert_eq!( + xml(&[("kept", Some("1")), ("dropped", None)]), + "\n\n \ + \n kept\n \ + 1\n \n" + ); + } + + #[test] + fn hadoop_xml_escapes_special_characters() { + let rendered = xml(&[("k", Some("&b"))]); + assert!( + rendered.contains("<a>&b"), + "{rendered}" + ); + } + + #[test] + fn java_properties_renders_key_value() { + assert_eq!(props(&[("a", Some("1")), ("b", Some("2"))]), "a=1\nb=2\n"); + } + + #[test] + fn java_properties_renders_none_as_empty() { + assert_eq!(props(&[("none", None)]), "none=\n"); + } + + #[test] + fn java_properties_escapes_colon_in_value() { + assert_eq!( + props(&[("url", Some("file://this/location/file.abc"))]), + "url=file\\://this/location/file.abc\n" + ); + } +} diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index 543204fc..4023085b 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use product_config::writer::to_hadoop_xml; +use crate::config::writer::to_hadoop_xml; use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 87de79be..23000822 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -8,11 +8,7 @@ use std::{ use const_format::concatcp; use indoc::formatdoc; -use product_config::{ - ProductConfigManager, - types::PropertyNameKind, - writer::{PropertiesWriterError, to_hadoop_xml, to_java_properties_string}, -}; +use product_config::{ProductConfigManager, types::PropertyNameKind}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{ @@ -66,9 +62,12 @@ use strum::{EnumDiscriminants, IntoStaticStr, ParseError}; use crate::{ OPERATOR_NAME, - config::jvm::{ - construct_global_jvm_args, construct_hbase_heapsize_env, - construct_role_specific_non_heap_jvm_args, + config::{ + jvm::{ + construct_global_jvm_args, construct_hbase_heapsize_env, + construct_role_specific_non_heap_jvm_args, + }, + writer::{PropertiesWriterError, to_hadoop_xml, to_java_properties_string}, }, crd::{ APP_NAME, AnyServiceConfig, Container, HBASE_ENV_SH, HBASE_MASTER_PORT, From 7304fd9d5b4eb90f8fc8d3e0421bfeedc4b02526 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Jun 2026 14:58:19 +0200 Subject: [PATCH 02/56] refactor: reshape ValidatedCluster toward the v2 framework Add name (ClusterName) and a ValidatedClusterConfig, rename role_groups to role_group_configs, and add merged config_overrides/env_overrides per role group. The product_config_properties map is kept temporarily so the existing config-map path still compiles. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/controller/validate.rs | 136 +++++++++++++++++- rust/operator-binary/src/crd/mod.rs | 72 ++++------ rust/operator-binary/src/hbase_controller.rs | 42 ++++-- 3 files changed, 189 insertions(+), 61 deletions(-) diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 67abc63a..36d7620e 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -4,15 +4,19 @@ use product_config::ProductConfigManager; use snafu::{ResultExt, Snafu}; use stackable_operator::{ commons::product_image_selection::{self}, + config::merge::Merge, + kube::ResourceExt, product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, role_utils::GenericRoleConfig, + v2::types::operator::ClusterName, }; use crate::{ controller::dereference::DereferencedObjects, crd::{HbaseRole, v1alpha1}, hbase_controller::{ - CONTAINER_IMAGE_BASE_NAME, ValidatedCluster, ValidatedRoleConfig, ValidatedRoleGroupConfig, + CONTAINER_IMAGE_BASE_NAME, ValidatedCluster, ValidatedClusterConfig, ValidatedRoleConfig, + ValidatedRoleGroupConfig, }, }; @@ -23,6 +27,11 @@ pub enum Error { source: product_image_selection::Error, }, + #[snafu(display("invalid cluster name"))] + InvalidClusterName { + source: stackable_operator::v2::macros::attributed_string_type::Error, + }, + #[snafu(display("invalid role properties"))] RoleProperties { source: crate::crd::Error }, @@ -104,6 +113,8 @@ pub fn validate_cluster( rolegroup_name.clone(), ValidatedRoleGroupConfig { merged_config, + config_overrides: merged_config_overrides(hbase, &hbase_role, rolegroup_name), + env_overrides: merged_env_overrides(hbase, &hbase_role, rolegroup_name), product_config_properties: rolegroup_config.clone(), }, ); @@ -113,10 +124,127 @@ pub fn validate_cluster( } Ok(ValidatedCluster { + name: ClusterName::from_str(&hbase.name_any()).context(InvalidClusterNameSnafu)?, image: resolved_product_image, - role_groups, + cluster_config: ValidatedClusterConfig { + zookeeper_connection_information: dereferenced_objects + .zookeeper_connection_information, + hbase_opa_config: dereferenced_objects.hbase_opa_config, + kerberos_enabled: hbase.has_kerberos_enabled(), + }, + role_group_configs: role_groups, role_configs, - zookeeper_connection_information: dereferenced_objects.zookeeper_connection_information, - hbase_opa_config: dereferenced_objects.hbase_opa_config, }) } + +/// Merge role-level then role-group-level `configOverrides` (role group wins). +fn merged_config_overrides( + hbase: &v1alpha1::HbaseCluster, + role: &HbaseRole, + role_group: &str, +) -> v1alpha1::HbaseConfigOverrides { + let (role_overrides, role_group_overrides) = match role { + HbaseRole::Master => ( + hbase + .spec + .masters + .as_ref() + .map(|r| r.config.config_overrides.clone()), + hbase + .spec + .masters + .as_ref() + .and_then(|r| r.role_groups.get(role_group)) + .map(|rg| rg.config.config_overrides.clone()), + ), + HbaseRole::RegionServer => ( + hbase + .spec + .region_servers + .as_ref() + .map(|r| r.config.config_overrides.clone()), + hbase + .spec + .region_servers + .as_ref() + .and_then(|r| r.role_groups.get(role_group)) + .map(|rg| rg.config.config_overrides.clone()), + ), + HbaseRole::RestServer => ( + hbase + .spec + .rest_servers + .as_ref() + .map(|r| r.config.config_overrides.clone()), + hbase + .spec + .rest_servers + .as_ref() + .and_then(|r| r.role_groups.get(role_group)) + .map(|rg| rg.config.config_overrides.clone()), + ), + }; + + let role_overrides = role_overrides.unwrap_or_default(); + let mut merged = role_group_overrides.unwrap_or_default(); + merged.merge(&role_overrides); + merged +} + +/// Merge role-level then role-group-level `envOverrides` (role group wins). +fn merged_env_overrides( + hbase: &v1alpha1::HbaseCluster, + role: &HbaseRole, + role_group: &str, +) -> BTreeMap { + let (role_overrides, role_group_overrides) = match role { + HbaseRole::Master => ( + hbase + .spec + .masters + .as_ref() + .map(|r| r.config.env_overrides.clone()), + hbase + .spec + .masters + .as_ref() + .and_then(|r| r.role_groups.get(role_group)) + .map(|rg| rg.config.env_overrides.clone()), + ), + HbaseRole::RegionServer => ( + hbase + .spec + .region_servers + .as_ref() + .map(|r| r.config.env_overrides.clone()), + hbase + .spec + .region_servers + .as_ref() + .and_then(|r| r.role_groups.get(role_group)) + .map(|rg| rg.config.env_overrides.clone()), + ), + HbaseRole::RestServer => ( + hbase + .spec + .rest_servers + .as_ref() + .map(|r| r.config.env_overrides.clone()), + hbase + .spec + .rest_servers + .as_ref() + .and_then(|r| r.role_groups.get(role_group)) + .map(|rg| rg.config.env_overrides.clone()), + ), + }; + + let mut env_overrides = BTreeMap::new(); + if let Some(role_overrides) = role_overrides { + env_overrides.extend(role_overrides); + } + if let Some(role_group_overrides) = role_group_overrides { + env_overrides.extend(role_group_overrides); + } + env_overrides +} diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 80b0707a..a04386d3 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -23,7 +23,7 @@ use stackable_operator::{ fragment::{self, Fragment, ValidationError}, merge::{Atomic, Merge}, }, - config_overrides::{KeyValueConfigOverrides, KeyValueOverridesProvider}, + config_overrides::KeyValueOverridesProvider, deep_merger::ObjectOverrides, k8s_openapi::{ DeepMerge, @@ -38,6 +38,7 @@ use stackable_operator::{ schemars::{self, JsonSchema}, shared::time::Duration, status::condition::{ClusterCondition, HasStatusCondition}, + v2::config_overrides::KeyValueConfigOverrides, versioned::versioned, }; use strum::{Display, EnumIter, EnumString}; @@ -233,43 +234,23 @@ pub mod versioned { pub authorization: Option, } - #[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] + #[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, Merge, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct HbaseConfigOverrides { - #[serde( - default, - rename = "hbase-site.xml", - skip_serializing_if = "Option::is_none" - )] - pub hbase_site_xml: Option, - - #[serde( - default, - rename = "hbase-env.sh", - skip_serializing_if = "Option::is_none" - )] - pub hbase_env_sh: Option, - - #[serde( - default, - rename = "ssl-server.xml", - skip_serializing_if = "Option::is_none" - )] - pub ssl_server_xml: Option, - - #[serde( - default, - rename = "ssl-client.xml", - skip_serializing_if = "Option::is_none" - )] - pub ssl_client_xml: Option, - - #[serde( - default, - rename = "security.properties", - skip_serializing_if = "Option::is_none" - )] - pub security_properties: Option, + #[serde(default, rename = "hbase-site.xml")] + pub hbase_site_xml: KeyValueConfigOverrides, + + #[serde(default, rename = "hbase-env.sh")] + pub hbase_env_sh: KeyValueConfigOverrides, + + #[serde(default, rename = "ssl-server.xml")] + pub ssl_server_xml: KeyValueConfigOverrides, + + #[serde(default, rename = "ssl-client.xml")] + pub ssl_client_xml: KeyValueConfigOverrides, + + #[serde(default, rename = "security.properties")] + pub security_properties: KeyValueConfigOverrides, } } @@ -584,17 +565,14 @@ impl v1alpha1::HbaseCluster { impl KeyValueOverridesProvider for v1alpha1::HbaseConfigOverrides { fn get_key_value_overrides(&self, file: &str) -> BTreeMap> { - let field = match file { - HBASE_SITE_XML => self.hbase_site_xml.as_ref(), - HBASE_ENV_SH => self.hbase_env_sh.as_ref(), - SSL_SERVER_XML => self.ssl_server_xml.as_ref(), - SSL_CLIENT_XML => self.ssl_client_xml.as_ref(), - JVM_SECURITY_PROPERTIES_FILE => self.security_properties.as_ref(), - _ => None, - }; - field - .map(KeyValueConfigOverrides::as_product_config_overrides) - .unwrap_or_default() + match file { + HBASE_SITE_XML => self.hbase_site_xml.overrides.clone(), + HBASE_ENV_SH => self.hbase_env_sh.overrides.clone(), + SSL_SERVER_XML => self.ssl_server_xml.overrides.clone(), + SSL_CLIENT_XML => self.ssl_client_xml.overrides.clone(), + JVM_SECURITY_PROPERTIES_FILE => self.security_properties.overrides.clone(), + _ => BTreeMap::new(), + } } } diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 23000822..c2331b12 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -57,6 +57,7 @@ use stackable_operator::{ statefulset::StatefulSetConditionBuilder, }, utils::cluster_info::KubernetesClusterInfo, + v2::types::operator::ClusterName, }; use strum::{EnumDiscriminants, IntoStaticStr, ParseError}; @@ -109,17 +110,28 @@ pub struct Ctx { pub operator_environment: OperatorEnvironmentOptions, } -/// The validated cluster: proves that product-config validation and config merging -/// succeeded for every role and role group before any resources are created. -/// Placed in the controller so that subsequent steps that reference this struct -/// only depend on the controller. +/// The validated cluster: proves that config merging and validation succeeded for +/// every role and role group before any resources are created. #[derive(Clone, Debug)] pub struct ValidatedCluster { + /// The logical (and Kubernetes object) name of the cluster. + // Populated now; consumed by the new build path in a later commit. + #[allow(dead_code)] + pub name: ClusterName, pub image: ResolvedProductImage, - pub role_groups: BTreeMap>, + pub cluster_config: ValidatedClusterConfig, + pub role_group_configs: BTreeMap>, pub role_configs: BTreeMap, +} + +/// Cluster-wide settings resolved once during validation. +#[derive(Clone, Debug)] +pub struct ValidatedClusterConfig { pub zookeeper_connection_information: ZookeeperConnectionInformation, pub hbase_opa_config: Option, + // Populated now; consumed by the new build path in a later commit. + #[allow(dead_code)] + pub kerberos_enabled: bool, } /// Per-role configuration extracted during validation. @@ -128,10 +140,20 @@ pub struct ValidatedRoleConfig { pub pdb: stackable_operator::commons::pdb::PdbConfig, } -/// Per-rolegroup configuration: the merged CRD config plus the product-config properties. +/// Per-rolegroup configuration: the merged CRD config plus the merged +/// (role <- role group) `configOverrides` and `envOverrides`. +/// +/// `product_config_properties` is retained temporarily so the existing controller +/// path keeps compiling; it is removed in a later commit. #[derive(Clone, Debug)] pub struct ValidatedRoleGroupConfig { pub merged_config: AnyServiceConfig, + // Populated now; consumed by the new build path in a later commit. + #[allow(dead_code)] + pub config_overrides: v1alpha1::HbaseConfigOverrides, + // Populated now; consumed by the new build path in a later commit. + #[allow(dead_code)] + pub env_overrides: BTreeMap, pub product_config_properties: HashMap>, } @@ -363,7 +385,7 @@ pub async fn reconcile_hbase( let mut ss_cond_builder = StatefulSetConditionBuilder::default(); - for (hbase_role, role_group_configs) in &validated.role_groups { + for (hbase_role, role_group_configs) in &validated.role_group_configs { for (rolegroup_name, validated_rg_config) in role_group_configs { let rolegroup = hbase.server_rolegroup_ref(hbase_role.to_string(), rolegroup_name); @@ -379,10 +401,10 @@ pub async fn reconcile_hbase( &client.kubernetes_cluster_info, &rolegroup, &validated_rg_config.product_config_properties, - &validated.zookeeper_connection_information, + &validated.cluster_config.zookeeper_connection_information, &validated_rg_config.merged_config, &validated.image, - validated.hbase_opa_config.as_ref(), + validated.cluster_config.hbase_opa_config.as_ref(), )?; let rg_statefulset = build_rolegroup_statefulset( hbase, @@ -443,7 +465,7 @@ pub async fn reconcile_hbase( let discovery_cm = build_discovery_configmap( hbase, &client.kubernetes_cluster_info, - &validated.zookeeper_connection_information, + &validated.cluster_config.zookeeper_connection_information, &validated.image, ) .context(BuildDiscoveryConfigMapSnafu)?; From 8f116c06b481b6860d9473fc48a7ccda9d3bca0d Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Jun 2026 15:40:39 +0200 Subject: [PATCH 03/56] feat: add controller/build config-map module without product-config Add per-file builders (hbase-site, hbase-env, ssl-server, ssl-client, security.properties) and a config_map orchestrator that render from the validated cluster, plus unit tests. Defaults previously sourced from properties.yaml (cluster distributed, role-specific DNS TTLs, HBASE_MANAGES_ZK) are now in code. Not yet wired into reconcile. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/controller/build/config_map.rs | 155 +++++++++++ .../src/controller/build/mod.rs | 2 + .../controller/build/properties/hbase_env.rs | 146 ++++++++++ .../controller/build/properties/hbase_site.rs | 254 ++++++++++++++++++ .../src/controller/build/properties/mod.rs | 118 ++++++++ .../build/properties/security_properties.rs | 86 ++++++ .../controller/build/properties/ssl_client.rs | 47 ++++ .../controller/build/properties/ssl_server.rs | 47 ++++ rust/operator-binary/src/controller/mod.rs | 1 + .../src/controller/validate.rs | 1 - rust/operator-binary/src/crd/mod.rs | 10 + rust/operator-binary/src/hbase_controller.rs | 5 - 12 files changed, 866 insertions(+), 6 deletions(-) create mode 100644 rust/operator-binary/src/controller/build/config_map.rs create mode 100644 rust/operator-binary/src/controller/build/mod.rs create mode 100644 rust/operator-binary/src/controller/build/properties/hbase_env.rs create mode 100644 rust/operator-binary/src/controller/build/properties/hbase_site.rs create mode 100644 rust/operator-binary/src/controller/build/properties/mod.rs create mode 100644 rust/operator-binary/src/controller/build/properties/security_properties.rs create mode 100644 rust/operator-binary/src/controller/build/properties/ssl_client.rs create mode 100644 rust/operator-binary/src/controller/build/properties/ssl_server.rs diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs new file mode 100644 index 00000000..c5586691 --- /dev/null +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -0,0 +1,155 @@ +//! Build the per-rolegroup `ConfigMap` for the HbaseCluster. + +use snafu::{OptionExt, ResultExt, Snafu}; +use stackable_operator::{ + builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, + k8s_openapi::api::core::v1::ConfigMap, + role_utils::RoleGroupRef, + utils::cluster_info::KubernetesClusterInfo, +}; + +use crate::{ + config::writer::PropertiesWriterError, + controller::build::properties::{ + ConfigFileName, hbase_env, hbase_site, security_properties, ssl_client, ssl_server, + }, + crd::{HbaseRole, v1alpha1}, + hbase_controller::{ValidatedCluster, build_recommended_labels}, + product_logging::extend_role_group_config_map, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("the validated cluster has no role group {role_group:?} for role {role:?}"))] + MissingRoleGroup { role: String, role_group: String }, + + #[snafu(display("failed to build hbase-site.xml"))] + BuildHbaseSite { source: hbase_site::Error }, + + #[snafu(display("failed to build hbase-env.sh"))] + BuildHbaseEnv { source: hbase_env::Error }, + + #[snafu(display("failed to serialize {} for {role_group}", ConfigFileName::Security))] + JvmSecurityProperties { + source: PropertiesWriterError, + role_group: String, + }, + + #[snafu(display("object is missing metadata to build owner reference"))] + ObjectMissingMetadataForOwnerRef { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to build object meta data"))] + ObjectMeta { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to add the logging configuration to the ConfigMap {cm_name:?}"))] + InvalidLoggingConfig { + source: crate::product_logging::Error, + cm_name: String, + }, + + #[snafu(display("cannot build config map for role {role:?} and role group {role_group:?}"))] + Assemble { + source: stackable_operator::builder::configmap::Error, + role: String, + role_group: String, + }, +} + +type Result = std::result::Result; + +#[allow(dead_code)] +pub fn build_rolegroup_config_map( + hbase: &v1alpha1::HbaseCluster, + cluster: &ValidatedCluster, + role: &HbaseRole, + cluster_info: &KubernetesClusterInfo, + rolegroup_ref: &RoleGroupRef, +) -> Result { + tracing::info!("Setting up ConfigMap for {:?}", rolegroup_ref); + + let rg = cluster + .role_group_configs + .get(role) + .and_then(|groups| groups.get(&rolegroup_ref.role_group)) + .with_context(|| MissingRoleGroupSnafu { + role: rolegroup_ref.role.clone(), + role_group: rolegroup_ref.role_group.clone(), + })?; + + let cluster_config = &cluster.cluster_config; + let overrides = &rg.config_overrides; + + let hbase_site_xml = hbase_site::build( + hbase, + role, + cluster_info, + &rg.merged_config, + cluster_config + .zookeeper_connection_information + .as_hbase_settings(), + cluster_config.hbase_opa_config.as_ref(), + overrides.hbase_site_xml.clone(), + ) + .context(BuildHbaseSiteSnafu)?; + + let hbase_env_sh = hbase_env::build( + hbase, + &rg.merged_config, + role, + &rolegroup_ref.role_group, + overrides.hbase_env_sh.clone(), + ) + .context(BuildHbaseEnvSnafu)?; + + let ssl_server_xml = ssl_server::build(hbase, overrides.ssl_server_xml.clone()); + let ssl_client_xml = ssl_client::build(hbase, overrides.ssl_client_xml.clone()); + + let security_properties = security_properties::build(role, overrides.security_properties.clone()) + .with_context(|_| JvmSecurityPropertiesSnafu { + role_group: rolegroup_ref.role_group.clone(), + })?; + + let cm_metadata = ObjectMetaBuilder::new() + .name_and_namespace(hbase) + .name(rolegroup_ref.object_name()) + .ownerreference_from_resource(hbase, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(&build_recommended_labels( + hbase, + &cluster.image.app_version_label_value, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + )) + .context(ObjectMetaSnafu)? + .build(); + + let mut builder = ConfigMapBuilder::new(); + builder + .metadata(cm_metadata) + .add_data(ConfigFileName::HbaseSite.to_string(), hbase_site_xml) + .add_data(ConfigFileName::HbaseEnv.to_string(), hbase_env_sh) + .add_data(ConfigFileName::Security.to_string(), security_properties); + + // HBase does not like empty config files: + // Caused by: com.ctc.wstx.exc.WstxEOFException: Unexpected EOF in prolog at [row,col,system-id]: [1,0,"file:/stackable/conf/ssl-server.xml"] + if !ssl_server_xml.is_empty() { + builder.add_data(ConfigFileName::SslServer.to_string(), ssl_server_xml); + } + if !ssl_client_xml.is_empty() { + builder.add_data(ConfigFileName::SslClient.to_string(), ssl_client_xml); + } + + extend_role_group_config_map(rolegroup_ref, rg.merged_config.logging(), &mut builder) + .context(InvalidLoggingConfigSnafu { + cm_name: rolegroup_ref.object_name(), + })?; + + builder.build().with_context(|_| AssembleSnafu { + role: rolegroup_ref.role.clone(), + role_group: rolegroup_ref.role_group.clone(), + }) +} diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs new file mode 100644 index 00000000..933a20b9 --- /dev/null +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -0,0 +1,2 @@ +pub mod config_map; +pub mod properties; diff --git a/rust/operator-binary/src/controller/build/properties/hbase_env.rs b/rust/operator-binary/src/controller/build/properties/hbase_env.rs new file mode 100644 index 00000000..a3fbe0c2 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/hbase_env.rs @@ -0,0 +1,146 @@ +//! Builds the `hbase-env.sh` config file: JVM heap/options env vars + overrides. + +use std::{collections::BTreeMap, fmt::Write}; + +use snafu::{ResultExt, Snafu}; +use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; + +use crate::{ + config::jvm::{ + self, construct_global_jvm_args, construct_hbase_heapsize_env, + construct_role_specific_non_heap_jvm_args, + }, + controller::build::properties::resolved_overrides, + crd::{AnyServiceConfig, HbaseRole, v1alpha1}, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to construct the HBASE_HEAPSIZE env variable"))] + ConstructHbaseHeapsizeEnv { source: jvm::Error }, + + #[snafu(display("failed to construct the JVM arguments"))] + ConstructJvmArgument { source: jvm::Error }, +} + +/// Renders `hbase-env.sh` as `export VAR="VALUE"` lines. +pub fn build( + hbase: &v1alpha1::HbaseCluster, + merged_config: &AnyServiceConfig, + role: &HbaseRole, + role_group: &str, + overrides: KeyValueConfigOverrides, +) -> Result { + let mut env: BTreeMap = BTreeMap::new(); + + env.insert("HBASE_MANAGES_ZK".to_string(), "false".to_string()); + env.insert( + "HBASE_HEAPSIZE".to_string(), + construct_hbase_heapsize_env(merged_config).context(ConstructHbaseHeapsizeEnvSnafu)?, + ); + env.insert( + "HBASE_OPTS".to_string(), + construct_global_jvm_args(hbase.has_kerberos_enabled()), + ); + + let role_specific_non_heap_jvm_args = + construct_role_specific_non_heap_jvm_args(hbase, role, role_group) + .context(ConstructJvmArgumentSnafu)?; + match role { + HbaseRole::Master => { + env.insert( + "HBASE_MASTER_OPTS".to_string(), + role_specific_non_heap_jvm_args, + ); + } + HbaseRole::RegionServer => { + env.insert( + "HBASE_REGIONSERVER_OPTS".to_string(), + role_specific_non_heap_jvm_args, + ); + } + HbaseRole::RestServer => { + env.insert( + "HBASE_REST_OPTS".to_string(), + role_specific_non_heap_jvm_args, + ); + } + } + + // configOverride come last + env.extend(resolved_overrides(overrides)); + + Ok(env + .iter() + .fold(String::new(), |mut output, (variable, value)| { + let _ = writeln!(output, "export {variable}=\"{value}\""); + output + })) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::controller::build::properties::test_support::{config_overrides, minimal_hbase}; + + fn master_merged_config(hbase: &v1alpha1::HbaseCluster) -> AnyServiceConfig { + hbase + .merged_config(&HbaseRole::Master, "default", "simple-hdfs") + .expect("merged config for the minimal master group") + } + + fn region_server_merged_config(hbase: &v1alpha1::HbaseCluster) -> AnyServiceConfig { + hbase + .merged_config(&HbaseRole::RegionServer, "default", "simple-hdfs") + .expect("merged config for the minimal region server group") + } + + #[test] + fn renders_operator_defaults() { + let hbase = minimal_hbase(); + let merged = master_merged_config(&hbase); + let env = build( + &hbase, + &merged, + &HbaseRole::Master, + "default", + config_overrides(&[]), + ) + .unwrap(); + assert!(env.contains("export HBASE_MANAGES_ZK=\"false\""), "{env}"); + assert!(env.contains("export HBASE_MASTER_OPTS="), "{env}"); + } + + #[test] + fn renders_region_server_opts() { + let hbase = minimal_hbase(); + let merged = region_server_merged_config(&hbase); + let env = build( + &hbase, + &merged, + &HbaseRole::RegionServer, + "default", + config_overrides(&[]), + ) + .unwrap(); + assert!(env.contains("export HBASE_REGIONSERVER_OPTS="), "{env}"); + } + + #[test] + fn user_override_appears() { + let hbase = minimal_hbase(); + let merged = master_merged_config(&hbase); + let env = build( + &hbase, + &merged, + &HbaseRole::Master, + "default", + config_overrides(&[("CUSTOM_VAR", "custom_value")]), + ) + .unwrap(); + assert!( + env.contains("export CUSTOM_VAR=\"custom_value\""), + "{env}" + ); + } +} diff --git a/rust/operator-binary/src/controller/build/properties/hbase_site.rs b/rust/operator-binary/src/controller/build/properties/hbase_site.rs new file mode 100644 index 00000000..f97a1b2d --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/hbase_site.rs @@ -0,0 +1,254 @@ +//! Builds the `hbase-site.xml` config file: operator defaults, ZooKeeper wiring, +//! kerberos/OPA security config, role-specific bind settings, with user +//! `configOverrides` applied last. + +use std::collections::BTreeMap; + +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + utils::cluster_info::KubernetesClusterInfo, v2::config_overrides::KeyValueConfigOverrides, +}; + +use crate::{ + config::writer::to_hadoop_xml, + controller::build::properties::resolved_overrides, + crd::{ + AnyServiceConfig, HBASE_CLUSTER_DISTRIBUTED, HBASE_MASTER_PORT, HBASE_MASTER_UI_PORT, + HBASE_REGIONSERVER_PORT, HBASE_REGIONSERVER_UI_PORT, HBASE_ROOTDIR, HbaseRole, v1alpha1, + }, + kerberos::{self, kerberos_config_properties}, + security::opa::HbaseOpaConfig, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to add the kerberos configuration"))] + AddKerberosConfig { source: kerberos::Error }, +} + +/// Renders `hbase-site.xml`. +#[allow(clippy::too_many_arguments)] +pub fn build( + hbase: &v1alpha1::HbaseCluster, + role: &HbaseRole, + cluster_info: &KubernetesClusterInfo, + merged_config: &AnyServiceConfig, + zookeeper_config: BTreeMap, + opa_config: Option<&HbaseOpaConfig>, + overrides: KeyValueConfigOverrides, +) -> Result { + let mut config: BTreeMap = BTreeMap::new(); + + // Defaults previously injected by product-config's `compute_files`. + config.insert(HBASE_CLUSTER_DISTRIBUTED.to_string(), "true".to_string()); + if let Some(rootdir) = merged_config.hbase_rootdir() { + config.insert(HBASE_ROOTDIR.to_string(), rootdir); + } + + config.extend(zookeeper_config); + config.extend(kerberos_config_properties(hbase, cluster_info).context(AddKerberosConfigSnafu)?); + config.extend(opa_config.map_or(vec![], |config| config.hbase_site_config())); + + // Set flag to override default behaviour, which is that the + // RPC client should bind the client address (forcing outgoing + // RPC traffic to happen from the same network interface that + // the RPC server is bound on). + config.insert( + "hbase.client.rpc.bind.address".to_string(), + "false".to_string(), + ); + + match role { + HbaseRole::Master => { + config.insert( + "hbase.master.ipc.address".to_string(), + "0.0.0.0".to_string(), + ); + config.insert( + "hbase.master.ipc.port".to_string(), + HBASE_MASTER_PORT.to_string(), + ); + config.insert( + "hbase.master.hostname".to_string(), + "${env:HBASE_SERVICE_HOST}".to_string(), + ); + config.insert( + "hbase.master.port".to_string(), + "${env:HBASE_SERVICE_PORT}".to_string(), + ); + config.insert( + "hbase.master.info.port".to_string(), + "${env:HBASE_INFO_PORT}".to_string(), + ); + config.insert( + "hbase.master.bound.info.port".to_string(), + HBASE_MASTER_UI_PORT.to_string(), + ); + } + HbaseRole::RegionServer => { + config.insert( + "hbase.regionserver.ipc.address".to_string(), + "0.0.0.0".to_string(), + ); + config.insert( + "hbase.regionserver.ipc.port".to_string(), + HBASE_REGIONSERVER_PORT.to_string(), + ); + config.insert( + "hbase.unsafe.regionserver.hostname".to_string(), + "${env:HBASE_SERVICE_HOST}".to_string(), + ); + config.insert( + "hbase.regionserver.port".to_string(), + "${env:HBASE_SERVICE_PORT}".to_string(), + ); + config.insert( + "hbase.regionserver.info.port".to_string(), + "${env:HBASE_INFO_PORT}".to_string(), + ); + config.insert( + "hbase.regionserver.bound.info.port".to_string(), + HBASE_REGIONSERVER_UI_PORT.to_string(), + ); + } + HbaseRole::RestServer => { + config.insert( + // N.B. a custom tag, so as not to interfere with HBase internals. + // The other roles use a patch to correctly resolve host/port. + "hbase.rest.endpoint".to_string(), + "${env:HBASE_SERVICE_HOST}:${env:HBASE_SERVICE_PORT}".to_string(), + ); + } + }; + + // configOverride come last + config.extend(resolved_overrides(overrides)); + + Ok(to_hadoop_xml( + config + .into_iter() + .map(|(k, v)| (k, Some(v))) + .collect::>() + .iter(), + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::controller::build::properties::test_support::{ + cluster_info, config_overrides, minimal_hbase, + }; + + fn master_merged_config(hbase: &v1alpha1::HbaseCluster) -> AnyServiceConfig { + hbase + .merged_config(&HbaseRole::Master, "default", "simple-hdfs") + .expect("merged config for the minimal master group") + } + + fn region_server_merged_config(hbase: &v1alpha1::HbaseCluster) -> AnyServiceConfig { + hbase + .merged_config(&HbaseRole::RegionServer, "default", "simple-hdfs") + .expect("merged config for the minimal region server group") + } + + fn rest_server_merged_config(hbase: &v1alpha1::HbaseCluster) -> AnyServiceConfig { + hbase + .merged_config(&HbaseRole::RestServer, "default", "simple-hdfs") + .expect("merged config for the minimal rest server group") + } + + #[test] + fn renders_operator_defaults() { + let hbase = minimal_hbase(); + let merged = master_merged_config(&hbase); + let xml = build( + &hbase, + &HbaseRole::Master, + &cluster_info(), + &merged, + BTreeMap::new(), + None, + config_overrides(&[]), + ) + .unwrap(); + assert!( + xml.contains("hbase.cluster.distributed\n true"), + "{xml}" + ); + assert!( + xml.contains("hbase.master.ipc.address\n 0.0.0.0"), + "{xml}" + ); + } + + #[test] + fn renders_region_server_bind_settings() { + let hbase = minimal_hbase(); + let merged = region_server_merged_config(&hbase); + let xml = build( + &hbase, + &HbaseRole::RegionServer, + &cluster_info(), + &merged, + BTreeMap::new(), + None, + config_overrides(&[]), + ) + .unwrap(); + assert!( + xml.contains( + "hbase.regionserver.ipc.address\n 0.0.0.0" + ), + "{xml}" + ); + assert!( + xml.contains( + "hbase.unsafe.regionserver.hostname\n ${env:HBASE_SERVICE_HOST}" + ), + "{xml}" + ); + } + + #[test] + fn renders_rest_server_endpoint() { + let hbase = minimal_hbase(); + let merged = rest_server_merged_config(&hbase); + let xml = build( + &hbase, + &HbaseRole::RestServer, + &cluster_info(), + &merged, + BTreeMap::new(), + None, + config_overrides(&[]), + ) + .unwrap(); + assert!( + xml.contains( + "hbase.rest.endpoint\n ${env:HBASE_SERVICE_HOST}:${env:HBASE_SERVICE_PORT}" + ), + "{xml}" + ); + } + + #[test] + fn user_override_wins() { + let hbase = minimal_hbase(); + let merged = master_merged_config(&hbase); + let xml = build( + &hbase, + &HbaseRole::Master, + &cluster_info(), + &merged, + BTreeMap::new(), + None, + config_overrides(&[("hbase.cluster.distributed", "false")]), + ) + .unwrap(); + assert!( + xml.contains("hbase.cluster.distributed\n false"), + "{xml}" + ); + } +} diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs new file mode 100644 index 00000000..5fb54649 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -0,0 +1,118 @@ +//! Per-file builders for the HBase config files assembled into the rolegroup +//! `ConfigMap`. Each `.rs` module produces the rendered content for one +//! config file; the shared [`crate::config::writer`] module serializes maps to +//! the Hadoop-XML / Java-properties on-wire format. + +use std::collections::BTreeMap; + +use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; + +pub mod hbase_env; +pub mod hbase_site; +pub mod security_properties; +pub mod ssl_client; +pub mod ssl_server; + +/// Keep only the set (`Some`) entries of a `key -> optional value` map, as `(key, value)` pairs. +fn defined_entries( + entries: BTreeMap>, +) -> impl Iterator { + entries + .into_iter() + .filter_map(|(key, value)| value.map(|value| (key, value))) +} + +/// Resolve user-provided [`KeyValueConfigOverrides`] into the key/value pairs to merge +/// into a config file, dropping entries whose value is unset (`None`). +fn resolved_overrides( + overrides: KeyValueConfigOverrides, +) -> impl Iterator { + defined_entries(overrides.overrides) +} + +/// The names of the HBase config files assembled into the rolegroup `ConfigMap`. +#[derive(Clone, Copy, Debug, strum::Display)] +pub enum ConfigFileName { + #[strum(serialize = "hbase-site.xml")] + HbaseSite, + #[strum(serialize = "hbase-env.sh")] + HbaseEnv, + #[strum(serialize = "ssl-server.xml")] + SslServer, + #[strum(serialize = "ssl-client.xml")] + SslClient, + #[strum(serialize = "security.properties")] + Security, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn file_names_match_the_hbase_on_disk_names() { + assert_eq!(ConfigFileName::HbaseSite.to_string(), "hbase-site.xml"); + assert_eq!(ConfigFileName::HbaseEnv.to_string(), "hbase-env.sh"); + assert_eq!(ConfigFileName::SslServer.to_string(), "ssl-server.xml"); + assert_eq!(ConfigFileName::SslClient.to_string(), "ssl-client.xml"); + assert_eq!(ConfigFileName::Security.to_string(), "security.properties"); + } +} + +#[cfg(test)] +pub(crate) mod test_support { + use stackable_operator::{ + commons::networking::DomainName, utils::cluster_info::KubernetesClusterInfo, + v2::config_overrides::KeyValueConfigOverrides, + }; + + use crate::crd::v1alpha1; + + /// Builds a [`KeyValueConfigOverrides`] from `(key, value)` pairs for tests. + pub fn config_overrides(pairs: &[(&str, &str)]) -> KeyValueConfigOverrides { + KeyValueConfigOverrides { + overrides: pairs + .iter() + .map(|(k, v)| (k.to_string(), Some(v.to_string()))) + .collect(), + } + } + + /// A minimal three-role HbaseCluster used to drive the per-file builder tests. + pub const MINIMAL_HBASE_YAML: &str = r#" +--- +apiVersion: hbase.stackable.tech/v1alpha1 +kind: HbaseCluster +metadata: + name: hbase + namespace: default +spec: + image: + productVersion: 2.6.3 + clusterConfig: + hdfsConfigMapName: simple-hdfs + zookeeperConfigMapName: simple-znode + masters: + roleGroups: + default: + replicas: 1 + regionServers: + roleGroups: + default: + replicas: 1 + restServers: + roleGroups: + default: + replicas: 1 +"#; + + pub fn minimal_hbase() -> v1alpha1::HbaseCluster { + serde_yaml::from_str(MINIMAL_HBASE_YAML).expect("invalid test HbaseCluster YAML") + } + + pub fn cluster_info() -> KubernetesClusterInfo { + KubernetesClusterInfo { + cluster_domain: DomainName::try_from("cluster.local").unwrap(), + } + } +} diff --git a/rust/operator-binary/src/controller/build/properties/security_properties.rs b/rust/operator-binary/src/controller/build/properties/security_properties.rs new file mode 100644 index 00000000..421f925f --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/security_properties.rs @@ -0,0 +1,86 @@ +//! Builds the `security.properties` (JVM security) config file. +//! +//! The operator injects role-specific JVM DNS cache TTLs (preserving the +//! behaviour previously defined in `properties.yaml`). User `configOverrides` +//! are applied on top. + +use std::collections::BTreeMap; + +use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; + +use crate::{ + config::writer::{PropertiesWriterError, to_java_properties_string}, + controller::build::properties::resolved_overrides, + crd::HbaseRole, +}; + +/// Renders `security.properties`: role-specific DNS cache TTLs plus user overrides. +pub fn build( + role: &HbaseRole, + overrides: KeyValueConfigOverrides, +) -> Result { + // Role-specific positive DNS cache TTLs. Caching forever (the JVM default for + // successful lookups) breaks failover when a peer's IP changes, so cap the + // positive cache and disable the negative cache. + let positive_ttl = match role { + HbaseRole::Master => "5", + HbaseRole::RegionServer => "10", + HbaseRole::RestServer => "30", + }; + + let mut config: BTreeMap> = BTreeMap::from([ + ( + "networkaddress.cache.ttl".to_string(), + Some(positive_ttl.to_string()), + ), + ( + "networkaddress.cache.negative.ttl".to_string(), + Some("0".to_string()), + ), + ]); + // Overrides applied last so users win. + config.extend(resolved_overrides(overrides).map(|(key, value)| (key, Some(value)))); + to_java_properties_string(config.iter()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::controller::build::properties::test_support::config_overrides; + + #[test] + fn injects_master_dns_cache_ttl() { + assert_eq!( + build(&HbaseRole::Master, config_overrides(&[])).unwrap(), + "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=5\n" + ); + } + + #[test] + fn injects_regionserver_dns_cache_ttl() { + assert_eq!( + build(&HbaseRole::RegionServer, config_overrides(&[])).unwrap(), + "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=10\n" + ); + } + + #[test] + fn injects_restserver_dns_cache_ttl() { + assert_eq!( + build(&HbaseRole::RestServer, config_overrides(&[])).unwrap(), + "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=30\n" + ); + } + + #[test] + fn user_overrides_win_over_injected_defaults() { + assert_eq!( + build( + &HbaseRole::Master, + config_overrides(&[("networkaddress.cache.ttl", "60")]) + ) + .unwrap(), + "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=60\n" + ); + } +} diff --git a/rust/operator-binary/src/controller/build/properties/ssl_client.rs b/rust/operator-binary/src/controller/build/properties/ssl_client.rs new file mode 100644 index 00000000..fdfac797 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/ssl_client.rs @@ -0,0 +1,47 @@ +//! Builds the `ssl-client.xml` config file: kerberos/TLS client settings + overrides. +use std::collections::BTreeMap; + +use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; + +use crate::{ + config::writer::to_hadoop_xml, controller::build::properties::resolved_overrides, + crd::v1alpha1, kerberos::kerberos_ssl_client_settings, +}; + +/// Renders `ssl-client.xml`. Returns "" (HBase rejects empty XML files) when empty. +pub fn build(hbase: &v1alpha1::HbaseCluster, overrides: KeyValueConfigOverrides) -> String { + let mut config: BTreeMap> = BTreeMap::new(); + config.extend( + kerberos_ssl_client_settings(hbase) + .into_iter() + .map(|(k, v)| (k, Some(v))), + ); + config.extend(resolved_overrides(overrides).map(|(k, v)| (k, Some(v)))); + if config.is_empty() { + return String::new(); + } + to_hadoop_xml(config.iter()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::controller::build::properties::test_support::{config_overrides, minimal_hbase}; + + #[test] + fn non_kerberos_without_overrides_renders_empty_string() { + assert_eq!(build(&minimal_hbase(), config_overrides(&[])), ""); + } + + #[test] + fn user_override_appears_in_xml() { + let xml = build( + &minimal_hbase(), + config_overrides(&[("ssl.client.keystore.type", "jks")]), + ); + assert!( + xml.contains("ssl.client.keystore.type\n jks"), + "{xml}" + ); + } +} diff --git a/rust/operator-binary/src/controller/build/properties/ssl_server.rs b/rust/operator-binary/src/controller/build/properties/ssl_server.rs new file mode 100644 index 00000000..f2d6a91c --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/ssl_server.rs @@ -0,0 +1,47 @@ +//! Builds the `ssl-server.xml` config file: kerberos/TLS server settings + overrides. +use std::collections::BTreeMap; + +use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; + +use crate::{ + config::writer::to_hadoop_xml, controller::build::properties::resolved_overrides, + crd::v1alpha1, kerberos::kerberos_ssl_server_settings, +}; + +/// Renders `ssl-server.xml`. Returns "" (HBase rejects empty XML files) when empty. +pub fn build(hbase: &v1alpha1::HbaseCluster, overrides: KeyValueConfigOverrides) -> String { + let mut config: BTreeMap> = BTreeMap::new(); + config.extend( + kerberos_ssl_server_settings(hbase) + .into_iter() + .map(|(k, v)| (k, Some(v))), + ); + config.extend(resolved_overrides(overrides).map(|(k, v)| (k, Some(v)))); + if config.is_empty() { + return String::new(); + } + to_hadoop_xml(config.iter()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::controller::build::properties::test_support::{config_overrides, minimal_hbase}; + + #[test] + fn non_kerberos_without_overrides_renders_empty_string() { + assert_eq!(build(&minimal_hbase(), config_overrides(&[])), ""); + } + + #[test] + fn user_override_appears_in_xml() { + let xml = build( + &minimal_hbase(), + config_overrides(&[("ssl.server.keystore.type", "jks")]), + ); + assert!( + xml.contains("ssl.server.keystore.type\n jks"), + "{xml}" + ); + } +} diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index 1b261dfe..a1196d5d 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -1,2 +1,3 @@ +pub mod build; pub mod dereference; pub mod validate; diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 36d7620e..d2aa20b4 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -130,7 +130,6 @@ pub fn validate_cluster( zookeeper_connection_information: dereferenced_objects .zookeeper_connection_information, hbase_opa_config: dereferenced_objects.hbase_opa_config, - kerberos_enabled: hbase.has_kerberos_enabled(), }, role_group_configs: role_groups, role_configs, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index a04386d3..13144611 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1289,6 +1289,16 @@ impl AnyServiceConfig { } } + /// The configured `hbase.rootdir`, if any. Previously injected into + /// `hbase-site.xml` via product-config's `compute_files`. + pub fn hbase_rootdir(&self) -> Option { + match self { + AnyServiceConfig::Master(config) => config.hbase_rootdir.clone(), + AnyServiceConfig::RegionServer(config) => config.hbase_rootdir.clone(), + AnyServiceConfig::RestServer(config) => config.hbase_rootdir.clone(), + } + } + /// Returns command line arguments to pass on to the region mover tool. /// The following arguments are excluded because they are already part of the /// hbase-entrypoint.sh script. diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index c2331b12..3710541a 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -129,9 +129,6 @@ pub struct ValidatedCluster { pub struct ValidatedClusterConfig { pub zookeeper_connection_information: ZookeeperConnectionInformation, pub hbase_opa_config: Option, - // Populated now; consumed by the new build path in a later commit. - #[allow(dead_code)] - pub kerberos_enabled: bool, } /// Per-role configuration extracted during validation. @@ -148,8 +145,6 @@ pub struct ValidatedRoleConfig { #[derive(Clone, Debug)] pub struct ValidatedRoleGroupConfig { pub merged_config: AnyServiceConfig, - // Populated now; consumed by the new build path in a later commit. - #[allow(dead_code)] pub config_overrides: v1alpha1::HbaseConfigOverrides, // Populated now; consumed by the new build path in a later commit. #[allow(dead_code)] From a793c48e26c8dbb85c9022d0470d67d1167ab096 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Jun 2026 16:01:55 +0200 Subject: [PATCH 04/56] refactor: build rolegroup ConfigMap from the validated cluster Wire reconcile to the new controller/build config_map orchestrator, build container env from validated env_overrides instead of the product-config Env map, and delete the old PropertyNameKind-based config-map and hbase-env.sh builders. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/controller/build/config_map.rs | 1 - rust/operator-binary/src/crd/mod.rs | 29 +- rust/operator-binary/src/hbase_controller.rs | 353 ++---------------- 3 files changed, 37 insertions(+), 346 deletions(-) diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index c5586691..741d23a0 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -61,7 +61,6 @@ pub enum Error { type Result = std::result::Result; -#[allow(dead_code)] pub fn build_rolegroup_config_map( hbase: &v1alpha1::HbaseCluster, cluster: &ValidatedCluster, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 13144611..4b1a6328 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -576,20 +576,15 @@ impl KeyValueOverridesProvider for v1alpha1::HbaseConfigOverrides { } } -pub fn merged_env(rolegroup_config: Option<&BTreeMap>) -> Vec { - let merged_env: Vec = if let Some(rolegroup_config) = rolegroup_config { - rolegroup_config - .iter() - .map(|(env_name, env_value)| EnvVar { - name: env_name.clone(), - value: Some(env_value.to_owned()), - value_from: None, - }) - .collect() - } else { - vec![] - }; - merged_env +pub fn merged_env(rolegroup_config: &BTreeMap) -> Vec { + rolegroup_config + .iter() + .map(|(env_name, env_value)| EnvVar { + name: env_name.clone(), + value: Some(env_value.to_owned()), + value_from: None, + }) + .collect() } #[derive( @@ -1443,8 +1438,10 @@ spec: .unwrap() .get("default") .unwrap() - .get(&PropertyNameKind::Env); - let merged_env = merged_env(rolegroup_config); + .get(&PropertyNameKind::Env) + .cloned() + .unwrap_or_default(); + let merged_env = merged_env(&rolegroup_config); let env_map: BTreeMap<&str, Option> = merged_env .iter() diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 3710541a..90fe9c8e 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -2,7 +2,6 @@ use std::{ collections::{BTreeMap, HashMap}, - fmt::Write, sync::Arc, }; @@ -13,7 +12,6 @@ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{ self, - configmap::ConfigMapBuilder, meta::ObjectMetaBuilder, pod::{ PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, @@ -28,8 +26,8 @@ use stackable_operator::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec}, core::v1::{ - ConfigMap, ConfigMapVolumeSource, ContainerPort, EnvVar, Probe, Service, - ServiceAccount, ServicePort, ServiceSpec, TCPSocketAction, Volume, + ConfigMapVolumeSource, ContainerPort, EnvVar, Probe, Service, ServiceAccount, + ServicePort, ServiceSpec, TCPSocketAction, Volume, }, }, apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, @@ -56,35 +54,20 @@ use stackable_operator::{ compute_conditions, operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, }, - utils::cluster_info::KubernetesClusterInfo, v2::types::operator::ClusterName, }; use strum::{EnumDiscriminants, IntoStaticStr, ParseError}; use crate::{ OPERATOR_NAME, - config::{ - jvm::{ - construct_global_jvm_args, construct_hbase_heapsize_env, - construct_role_specific_non_heap_jvm_args, - }, - writer::{PropertiesWriterError, to_hadoop_xml, to_java_properties_string}, - }, crd::{ - APP_NAME, AnyServiceConfig, Container, HBASE_ENV_SH, HBASE_MASTER_PORT, - HBASE_MASTER_UI_PORT, HBASE_REGIONSERVER_PORT, HBASE_REGIONSERVER_UI_PORT, HBASE_SITE_XML, - HbaseClusterStatus, HbaseRole, JVM_SECURITY_PROPERTIES_FILE, LISTENER_VOLUME_DIR, - LISTENER_VOLUME_NAME, SSL_CLIENT_XML, SSL_SERVER_XML, merged_env, v1alpha1, + APP_NAME, AnyServiceConfig, CONFIG_DIR_NAME, Container, HbaseClusterStatus, HbaseRole, + LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, merged_env, v1alpha1, }, discovery::build_discovery_configmap, - kerberos::{ - self, add_kerberos_pod_config, kerberos_config_properties, kerberos_ssl_client_settings, - kerberos_ssl_server_settings, - }, + kerberos::{self, add_kerberos_pod_config}, operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, - product_logging::{ - CONTAINERDEBUG_LOG_DIRECTORY, STACKABLE_LOG_DIR, extend_role_group_config_map, - }, + product_logging::{CONTAINERDEBUG_LOG_DIRECTORY, STACKABLE_LOG_DIR}, security::opa::HbaseOpaConfig, zookeeper::ZookeeperConnectionInformation, }; @@ -146,9 +129,9 @@ pub struct ValidatedRoleConfig { pub struct ValidatedRoleGroupConfig { pub merged_config: AnyServiceConfig, pub config_overrides: v1alpha1::HbaseConfigOverrides, - // Populated now; consumed by the new build path in a later commit. - #[allow(dead_code)] pub env_overrides: BTreeMap, + // Retained until the product-config path is fully removed in a later commit. + #[allow(dead_code)] pub product_config_properties: HashMap>, } @@ -182,10 +165,9 @@ pub enum Error { #[snafu(display("failed to build discovery configmap"))] BuildDiscoveryConfigMap { source: super::discovery::Error }, - #[snafu(display("failed to build ConfigMap for {}", rolegroup))] - BuildRoleGroupConfig { - source: stackable_operator::builder::configmap::Error, - rolegroup: RoleGroupRef, + #[snafu(display("failed to build rolegroup ConfigMap"))] + BuildRolegroupConfigMap { + source: crate::controller::build::config_map::Error, }, #[snafu(display("failed to apply ConfigMap for {}", rolegroup))] @@ -224,12 +206,6 @@ pub enum Error { #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] VectorAggregatorConfigMapMissing, - #[snafu(display("failed to add the logging configuration to the ConfigMap [{cm_name}]"))] - InvalidLoggingConfig { - source: crate::product_logging::Error, - cm_name: String, - }, - #[snafu(display("failed to add kerberos config"))] AddKerberosConfig { source: kerberos::Error }, @@ -243,15 +219,6 @@ pub enum Error { source: stackable_operator::commons::rbac::Error, }, - #[snafu(display( - "failed to serialize [{JVM_SECURITY_PROPERTIES_FILE}] for {}", - rolegroup - ))] - SerializeJvmSecurity { - source: PropertiesWriterError, - rolegroup: RoleGroupRef, - }, - #[snafu(display("failed to create PodDisruptionBudget"))] FailedToCreatePdb { source: crate::operations::pdb::Error, @@ -289,12 +256,6 @@ pub enum Error { source: error_boundary::InvalidObject, }, - #[snafu(display("failed to construct HBASE_HEAPSIZE env variable"))] - ConstructHbaseHeapsizeEnv { source: crate::config::jvm::Error }, - - #[snafu(display("failed to construct JVM arguments"))] - ConstructJvmArgument { source: crate::config::jvm::Error }, - #[snafu(display("failed to build Labels"))] LabelBuild { source: stackable_operator::kvp::LabelError, @@ -390,23 +351,19 @@ pub async fn reconcile_hbase( let rg_metrics_service = build_rolegroup_metrics_service(hbase, hbase_role, &rolegroup, &validated.image)?; - let rg_configmap = build_rolegroup_config_map( + let rg_configmap = crate::controller::build::config_map::build_rolegroup_config_map( hbase, + &validated, hbase_role, &client.kubernetes_cluster_info, &rolegroup, - &validated_rg_config.product_config_properties, - &validated.cluster_config.zookeeper_connection_information, - &validated_rg_config.merged_config, - &validated.image, - validated.cluster_config.hbase_opa_config.as_ref(), - )?; + ) + .context(BuildRolegroupConfigMapSnafu)?; let rg_statefulset = build_rolegroup_statefulset( hbase, hbase_role, &rolegroup, - &validated_rg_config.product_config_properties, - &validated_rg_config.merged_config, + validated_rg_config, &validated.image, &rbac_sa, )?; @@ -488,216 +445,6 @@ pub async fn reconcile_hbase( Ok(Action::await_change()) } -/// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator -#[allow(clippy::too_many_arguments)] -fn build_rolegroup_config_map( - hbase: &v1alpha1::HbaseCluster, - hbase_role: &HbaseRole, - cluster_info: &KubernetesClusterInfo, - rolegroup: &RoleGroupRef, - rolegroup_config: &HashMap>, - zookeeper_connection_information: &ZookeeperConnectionInformation, - merged_config: &AnyServiceConfig, - resolved_product_image: &ResolvedProductImage, - hbase_opa_config: Option<&HbaseOpaConfig>, -) -> Result { - let mut hbase_site_xml = String::new(); - let mut hbase_env_sh = String::new(); - let mut ssl_server_xml = String::new(); - let mut ssl_client_xml = String::new(); - - for (property_name_kind, config) in rolegroup_config { - match property_name_kind { - PropertyNameKind::File(file_name) if file_name == HBASE_SITE_XML => { - let mut hbase_site_config = BTreeMap::new(); - hbase_site_config.extend(zookeeper_connection_information.as_hbase_settings()); - hbase_site_config.extend( - kerberos_config_properties(hbase, cluster_info) - .context(AddKerberosConfigSnafu)?, - ); - hbase_site_config - .extend(hbase_opa_config.map_or(vec![], |config| config.hbase_site_config())); - - // Set flag to override default behaviour, which is that the - // RPC client should bind the client address (forcing outgoing - // RPC traffic to happen from the same network interface that - // the RPC server is bound on). - hbase_site_config.insert( - "hbase.client.rpc.bind.address".to_string(), - "false".to_string(), - ); - - match hbase_role { - HbaseRole::Master => { - hbase_site_config.insert( - "hbase.master.ipc.address".to_string(), - "0.0.0.0".to_string(), - ); - hbase_site_config.insert( - "hbase.master.ipc.port".to_string(), - HBASE_MASTER_PORT.to_string(), - ); - hbase_site_config.insert( - "hbase.master.hostname".to_string(), - "${env:HBASE_SERVICE_HOST}".to_string(), - ); - hbase_site_config.insert( - "hbase.master.port".to_string(), - "${env:HBASE_SERVICE_PORT}".to_string(), - ); - hbase_site_config.insert( - "hbase.master.info.port".to_string(), - "${env:HBASE_INFO_PORT}".to_string(), - ); - hbase_site_config.insert( - "hbase.master.bound.info.port".to_string(), - HBASE_MASTER_UI_PORT.to_string(), - ); - } - HbaseRole::RegionServer => { - hbase_site_config.insert( - "hbase.regionserver.ipc.address".to_string(), - "0.0.0.0".to_string(), - ); - hbase_site_config.insert( - "hbase.regionserver.ipc.port".to_string(), - HBASE_REGIONSERVER_PORT.to_string(), - ); - hbase_site_config.insert( - "hbase.unsafe.regionserver.hostname".to_string(), - "${env:HBASE_SERVICE_HOST}".to_string(), - ); - hbase_site_config.insert( - "hbase.regionserver.port".to_string(), - "${env:HBASE_SERVICE_PORT}".to_string(), - ); - hbase_site_config.insert( - "hbase.regionserver.info.port".to_string(), - "${env:HBASE_INFO_PORT}".to_string(), - ); - hbase_site_config.insert( - "hbase.regionserver.bound.info.port".to_string(), - HBASE_REGIONSERVER_UI_PORT.to_string(), - ); - } - HbaseRole::RestServer => { - hbase_site_config.insert( - // N.B. a custom tag, so as not to interfere with HBase internals. - // The other roles use a patch to correctly resolve host/port. - "hbase.rest.endpoint".to_string(), - "${env:HBASE_SERVICE_HOST}:${env:HBASE_SERVICE_PORT}".to_string(), - ); - } - }; - - // configOverride come last - hbase_site_config.extend(config.clone()); - hbase_site_xml = to_hadoop_xml( - hbase_site_config - .into_iter() - .map(|(k, v)| (k, Some(v))) - .collect::>() - .iter(), - ); - } - PropertyNameKind::File(file_name) if file_name == HBASE_ENV_SH => { - let mut hbase_env_config = - build_hbase_env_sh(hbase, merged_config, hbase_role, &rolegroup.role_group)?; - - // configOverride come last - hbase_env_config.extend(config.clone()); - hbase_env_sh = write_hbase_env_sh(hbase_env_config.iter()); - } - PropertyNameKind::File(file_name) if file_name == SSL_SERVER_XML => { - let mut config_opts = BTreeMap::new(); - config_opts.extend(kerberos_ssl_server_settings(hbase)); - - // configOverride come last - config_opts.extend(config.clone()); - ssl_server_xml = to_hadoop_xml( - config_opts - .into_iter() - .map(|(k, v)| (k, Some(v))) - .collect::>() - .iter(), - ); - } - PropertyNameKind::File(file_name) if file_name == SSL_CLIENT_XML => { - let mut config_opts = BTreeMap::new(); - config_opts.extend(kerberos_ssl_client_settings(hbase)); - - // configOverride come last - config_opts.extend(config.clone()); - ssl_client_xml = to_hadoop_xml( - config_opts - .into_iter() - .map(|(k, v)| (k, Some(v))) - .collect::>() - .iter(), - ); - } - _ => {} - } - } - - let jvm_sec_props: BTreeMap> = rolegroup_config - .get(&PropertyNameKind::File( - JVM_SECURITY_PROPERTIES_FILE.to_string(), - )) - .cloned() - .unwrap_or_default() - .into_iter() - .map(|(k, v)| (k, Some(v))) - .collect(); - let jvm_sec_props = to_java_properties_string(jvm_sec_props.iter()).with_context(|_| { - SerializeJvmSecuritySnafu { - rolegroup: rolegroup.clone(), - } - })?; - - let mut builder = ConfigMapBuilder::new(); - - let cm_metadata = ObjectMetaBuilder::new() - .name_and_namespace(hbase) - .name(rolegroup.object_name()) - .ownerreference_from_resource(hbase, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&build_recommended_labels( - hbase, - &resolved_product_image.app_version_label_value, - &rolegroup.role, - &rolegroup.role_group, - )) - .context(ObjectMetaSnafu)? - .build(); - - builder - .metadata(cm_metadata) - .add_data(HBASE_SITE_XML, hbase_site_xml) - .add_data(HBASE_ENV_SH, hbase_env_sh) - .add_data(JVM_SECURITY_PROPERTIES_FILE, jvm_sec_props); - - // HBase does not like empty config files: - // Caused by: com.ctc.wstx.exc.WstxEOFException: Unexpected EOF in prolog at [row,col,system-id]: [1,0,"file:/stackable/conf/ssl-server.xml"] - if !ssl_server_xml.is_empty() { - builder.add_data(SSL_SERVER_XML, ssl_server_xml); - } - if !ssl_client_xml.is_empty() { - builder.add_data(SSL_CLIENT_XML, ssl_client_xml); - } - - extend_role_group_config_map(rolegroup, merged_config.logging(), &mut builder).context( - InvalidLoggingConfigSnafu { - cm_name: rolegroup.object_name(), - }, - )?; - - builder.build().map_err(|e| Error::BuildRoleGroupConfig { - source: e, - rolegroup: rolegroup.clone(), - }) -} - /// The rolegroup [`Service`] is a headless service that allows direct access to the instances of a certain rolegroup /// /// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. @@ -833,11 +580,11 @@ fn build_rolegroup_statefulset( hbase: &v1alpha1::HbaseCluster, hbase_role: &HbaseRole, rolegroup_ref: &RoleGroupRef, - rolegroup_config: &HashMap>, - merged_config: &AnyServiceConfig, + validated_rg_config: &ValidatedRoleGroupConfig, resolved_product_image: &ResolvedProductImage, service_account: &ServiceAccount, ) -> Result { + let merged_config = &validated_rg_config.merged_config; let hbase_version = &resolved_product_image.app_version_label_value; let ports = hbase_role @@ -879,7 +626,12 @@ fn build_rolegroup_statefulset( ..probe_template }; - let mut merged_env = merged_env(rolegroup_config.get(&PropertyNameKind::Env)); + let mut env_map: BTreeMap = BTreeMap::from([ + ("HBASE_CONF_DIR".to_string(), CONFIG_DIR_NAME.to_string()), + ("HADOOP_CONF_DIR".to_string(), CONFIG_DIR_NAME.to_string()), + ]); + env_map.extend(validated_rg_config.env_overrides.clone()); + let mut merged_env = merged_env(&env_map); // This env var is set for all roles to avoid bash's "unbound variable" errors merged_env.extend([ EnvVar { @@ -1125,16 +877,6 @@ fn command() -> Vec { ] } -fn write_hbase_env_sh<'a, T>(properties: T) -> String -where - T: Iterator, -{ - properties.fold(String::new(), |mut output, (variable, value)| { - let _ = writeln!(output, "export {variable}=\"{value}\""); - output - }) -} - pub fn error_policy( _obj: Arc>, error: &Error, @@ -1164,53 +906,6 @@ pub fn build_recommended_labels<'a>( } } -/// The content of the HBase `hbase-env.sh` file. -fn build_hbase_env_sh( - hbase: &v1alpha1::HbaseCluster, - merged_config: &AnyServiceConfig, - hbase_role: &HbaseRole, - role_group: &str, -) -> Result, Error> { - let mut result = BTreeMap::new(); - - result.insert("HBASE_MANAGES_ZK".to_string(), "false".to_string()); - - result.insert( - "HBASE_HEAPSIZE".to_owned(), - construct_hbase_heapsize_env(merged_config).context(ConstructHbaseHeapsizeEnvSnafu)?, - ); - result.insert( - "HBASE_OPTS".to_owned(), - construct_global_jvm_args(hbase.has_kerberos_enabled()), - ); - let role_specific_non_heap_jvm_args = - construct_role_specific_non_heap_jvm_args(hbase, hbase_role, role_group) - .context(ConstructJvmArgumentSnafu)?; - - match hbase_role { - HbaseRole::Master => { - result.insert( - "HBASE_MASTER_OPTS".to_string(), - role_specific_non_heap_jvm_args, - ); - } - HbaseRole::RegionServer => { - result.insert( - "HBASE_REGIONSERVER_OPTS".to_string(), - role_specific_non_heap_jvm_args, - ); - } - HbaseRole::RestServer => { - result.insert( - "HBASE_REST_OPTS".to_string(), - role_specific_non_heap_jvm_args, - ); - } - } - - Ok(result) -} - #[cfg(test)] mod test { use rstest::rstest; From a106a7dfcfe40300734780b7b368d662ebccad98 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Jun 2026 16:39:41 +0200 Subject: [PATCH 05/56] refactor: validate the HbaseCluster without product-config Drop ProductConfigManager / transform_all_roles_to_config / validate_all_roles_and_groups_config from the validate step, remove the Configuration trait impls and build_role_properties from the CRD, delete the now-unused product_config_properties field and the Ctx.product_config field, and stop loading the product config in main.rs. The product-config crate is now unreferenced in source. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/controller/validate.rs | 157 ++++++--- rust/operator-binary/src/crd/mod.rs | 326 +----------------- rust/operator-binary/src/hbase_controller.rs | 14 +- rust/operator-binary/src/main.rs | 8 +- 4 files changed, 118 insertions(+), 387 deletions(-) diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index d2aa20b4..720d9c2d 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -1,15 +1,14 @@ use std::{collections::BTreeMap, str::FromStr}; -use product_config::ProductConfigManager; use snafu::{ResultExt, Snafu}; use stackable_operator::{ commons::product_image_selection::{self}, config::merge::Merge, kube::ResourceExt, - product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, role_utils::GenericRoleConfig, v2::types::operator::ClusterName, }; +use strum::IntoEnumIterator; use crate::{ controller::dereference::DereferencedObjects, @@ -32,24 +31,8 @@ pub enum Error { source: stackable_operator::v2::macros::attributed_string_type::Error, }, - #[snafu(display("invalid role properties"))] - RoleProperties { source: crate::crd::Error }, - - #[snafu(display("failed to generate product config"))] - GenerateProductConfig { - source: stackable_operator::product_config_utils::Error, - }, - - #[snafu(display("invalid product config"))] - InvalidProductConfig { - source: stackable_operator::product_config_utils::Error, - }, - - #[snafu(display("could not parse Hbase role [{role}]"))] - UnidentifiedHbaseRole { - source: strum::ParseError, - role: String, - }, + #[snafu(display("the HbaseCluster has no {role} role defined"))] + MissingRequiredRole { role: String }, #[snafu(display("failed to resolve and merge config for role and role group"))] FailedToResolveConfig { source: crate::crd::Error }, @@ -58,7 +41,6 @@ pub enum Error { pub fn validate_cluster( hbase: &v1alpha1::HbaseCluster, image_repository: &str, - product_config_manager: &ProductConfigManager, dereferenced_objects: DereferencedObjects, ) -> Result { let resolved_product_image = hbase @@ -71,24 +53,25 @@ pub fn validate_cluster( ) .context(ResolveProductImageSnafu)?; - let roles = hbase.build_role_properties().context(RolePropertiesSnafu)?; - - let validated_config = validate_all_roles_and_groups_config( - &resolved_product_image.product_version, - &transform_all_roles_to_config(hbase, &roles).context(GenerateProductConfigSnafu)?, - product_config_manager, - false, - false, - ) - .context(InvalidProductConfigSnafu)?; - let mut role_groups = BTreeMap::new(); let mut role_configs = BTreeMap::new(); - for (role_name, group_config) in validated_config.iter() { - let hbase_role = HbaseRole::from_str(role_name).context(UnidentifiedHbaseRoleSnafu { - role: role_name.to_string(), - })?; + for hbase_role in HbaseRole::iter() { + let role_group_names = role_group_names(hbase, &hbase_role); + + // masters and region servers are required (preserves the old build_role_properties check); + // rest servers are optional. + if role_group_names.is_empty() { + match hbase_role { + HbaseRole::Master | HbaseRole::RegionServer => { + return MissingRequiredRoleSnafu { + role: hbase_role.to_string(), + } + .fail(); + } + HbaseRole::RestServer => continue, + } + } if let Some(GenericRoleConfig { pod_disruption_budget: pdb, @@ -98,13 +81,11 @@ pub fn validate_cluster( } let mut group_configs = BTreeMap::new(); - for (rolegroup_name, rolegroup_config) in group_config.iter() { - let rolegroup = hbase.server_rolegroup_ref(role_name, rolegroup_name); - + for rolegroup_name in role_group_names { let merged_config = hbase .merged_config( &hbase_role, - &rolegroup.role_group, + &rolegroup_name, &hbase.spec.cluster_config.hdfs_config_map_name, ) .context(FailedToResolveConfigSnafu)?; @@ -113,9 +94,8 @@ pub fn validate_cluster( rolegroup_name.clone(), ValidatedRoleGroupConfig { merged_config, - config_overrides: merged_config_overrides(hbase, &hbase_role, rolegroup_name), - env_overrides: merged_env_overrides(hbase, &hbase_role, rolegroup_name), - product_config_properties: rolegroup_config.clone(), + config_overrides: merged_config_overrides(hbase, &hbase_role, &rolegroup_name), + env_overrides: merged_env_overrides(hbase, &hbase_role, &rolegroup_name), }, ); } @@ -136,6 +116,28 @@ pub fn validate_cluster( }) } +/// The names of the role groups defined for `role` in the spec. +fn role_group_names(hbase: &v1alpha1::HbaseCluster, role: &HbaseRole) -> Vec { + match role { + HbaseRole::Master => hbase + .spec + .masters + .as_ref() + .map(|r| r.role_groups.keys().cloned().collect()), + HbaseRole::RegionServer => hbase + .spec + .region_servers + .as_ref() + .map(|r| r.role_groups.keys().cloned().collect()), + HbaseRole::RestServer => hbase + .spec + .rest_servers + .as_ref() + .map(|r| r.role_groups.keys().cloned().collect()), + } + .unwrap_or_default() +} + /// Merge role-level then role-group-level `configOverrides` (role group wins). fn merged_config_overrides( hbase: &v1alpha1::HbaseCluster, @@ -247,3 +249,72 @@ fn merged_env_overrides( } env_overrides } + +#[cfg(test)] +mod tests { + use indoc::indoc; + + use super::*; + + #[test] + fn test_env_overrides() { + let input = indoc! {r#" +--- +apiVersion: hbase.stackable.tech/v1alpha1 +kind: HbaseCluster +metadata: + name: test-hbase +spec: + image: + productVersion: 2.6.4 + clusterConfig: + hdfsConfigMapName: test-hdfs + zookeeperConfigMapName: test-znode + masters: + envOverrides: + TEST_VAR_FROM_MASTER: MASTER + TEST_VAR: MASTER + config: + logging: + enableVectorAgent: False + roleGroups: + default: + replicas: 1 + envOverrides: + TEST_VAR_FROM_MRG: MASTER + TEST_VAR: MASTER_RG + regionServers: + config: + logging: + enableVectorAgent: False + regionMover: + runBeforeShutdown: false + roleGroups: + default: + replicas: 1 + restServers: + config: + logging: + enableVectorAgent: False + roleGroups: + default: + replicas: 1 + "#}; + + let deserializer = serde_yaml::Deserializer::from_str(input); + let hbase: v1alpha1::HbaseCluster = + serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); + + let env_overrides = merged_env_overrides(&hbase, &HbaseRole::Master, "default"); + + assert_eq!(env_overrides.get("TEST_VAR"), Some(&"MASTER_RG".to_string())); + assert_eq!( + env_overrides.get("TEST_VAR_FROM_MASTER"), + Some(&"MASTER".to_string()) + ); + assert_eq!( + env_overrides.get("TEST_VAR_FROM_MRG"), + Some(&"MASTER".to_string()) + ); + } +} diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 4b1a6328..937d24c8 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,6 +1,5 @@ -use std::collections::{BTreeMap, HashMap}; +use std::collections::BTreeMap; -use product_config::types::PropertyNameKind; use security::AuthenticationConfig; use serde::{Deserialize, Serialize}; use shell_escape::escape; @@ -23,7 +22,6 @@ use stackable_operator::{ fragment::{self, Fragment, ValidationError}, merge::{Atomic, Merge}, }, - config_overrides::KeyValueOverridesProvider, deep_merger::ObjectOverrides, k8s_openapi::{ DeepMerge, @@ -32,7 +30,6 @@ use stackable_operator::{ }, kube::{CustomResource, ResourceExt, runtime::reflector::ObjectRef}, kvp::Labels, - product_config_utils::Configuration, product_logging::{self, spec::Logging}, role_utils::{GenericRoleConfig, JavaCommonConfig, Role, RoleGroupRef}, schemars::{self, JsonSchema}, @@ -61,10 +58,7 @@ pub const TLS_STORE_PASSWORD: &str = "changeit"; pub const JVM_SECURITY_PROPERTIES_FILE: &str = "security.properties"; -pub const HBASE_ENV_SH: &str = "hbase-env.sh"; pub const HBASE_SITE_XML: &str = "hbase-site.xml"; -pub const SSL_SERVER_XML: &str = "ssl-server.xml"; -pub const SSL_CLIENT_XML: &str = "ssl-client.xml"; pub const HBASE_CLUSTER_DISTRIBUTED: &str = "hbase.cluster.distributed"; pub const HBASE_ROOTDIR: &str = "hbase.rootdir"; @@ -122,12 +116,6 @@ pub enum Error { #[snafu(display("fragment validation failure"))] FragmentValidationFailure { source: ValidationError }, - #[snafu(display("object defines no master role"))] - NoMasterRole, - - #[snafu(display("object defines no regionserver role"))] - NoRegionServerRole, - #[snafu(display("incompatible merge types"))] IncompatibleMergeTypes, @@ -370,67 +358,6 @@ impl v1alpha1::HbaseCluster { }) } - // The result type is only defined once, there is no value in extracting it into a type definition. - #[allow(clippy::type_complexity)] - pub fn build_role_properties( - &self, - ) -> Result< - HashMap< - String, - ( - Vec, - Role< - impl Configuration, - v1alpha1::HbaseConfigOverrides, - GenericRoleConfig, - JavaCommonConfig, - >, - ), - >, - Error, - > { - let config_types = vec![ - PropertyNameKind::Env, - PropertyNameKind::File(HBASE_ENV_SH.to_string()), - PropertyNameKind::File(HBASE_SITE_XML.to_string()), - PropertyNameKind::File(SSL_SERVER_XML.to_string()), - PropertyNameKind::File(SSL_CLIENT_XML.to_string()), - PropertyNameKind::File(JVM_SECURITY_PROPERTIES_FILE.to_string()), - ]; - - let mut roles = HashMap::from([( - HbaseRole::Master.to_string(), - ( - config_types.to_owned(), - self.spec - .masters - .clone() - .context(NoMasterRoleSnafu)? - .erase(), - ), - )]); - roles.insert( - HbaseRole::RegionServer.to_string(), - ( - config_types.to_owned(), - self.spec - .region_servers - .clone() - .context(NoRegionServerRoleSnafu)? - .erase(), - ), - ); - - if let Some(rest_servers) = self.spec.rest_servers.as_ref() { - roles.insert( - HbaseRole::RestServer.to_string(), - (config_types, rest_servers.to_owned().erase()), - ); - } - - Ok(roles) - } - pub fn merge_pod_overrides( &self, pod_template: &mut PodTemplateSpec, @@ -563,19 +490,6 @@ impl v1alpha1::HbaseCluster { } } -impl KeyValueOverridesProvider for v1alpha1::HbaseConfigOverrides { - fn get_key_value_overrides(&self, file: &str) -> BTreeMap> { - match file { - HBASE_SITE_XML => self.hbase_site_xml.overrides.clone(), - HBASE_ENV_SH => self.hbase_env_sh.overrides.clone(), - SSL_SERVER_XML => self.ssl_server_xml.overrides.clone(), - SSL_CLIENT_XML => self.ssl_client_xml.overrides.clone(), - JVM_SECURITY_PROPERTIES_FILE => self.security_properties.overrides.clone(), - _ => BTreeMap::new(), - } - } -} - pub fn merged_env(rolegroup_config: &BTreeMap) -> Vec { rolegroup_config .iter() @@ -1004,73 +918,6 @@ pub struct HbaseConfig { pub listener_class: String, } -impl Configuration for HbaseConfigFragment { - type Configurable = v1alpha1::HbaseCluster; - - fn compute_env( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - // Maps env var name to env var object. This allows env_overrides to work - // as expected (i.e. users can override the env var value). - let mut vars: BTreeMap> = BTreeMap::new(); - - vars.insert( - "HBASE_CONF_DIR".to_string(), - Some(CONFIG_DIR_NAME.to_string()), - ); - // required by phoenix (for cases where Kerberos is enabled): see https://issues.apache.org/jira/browse/PHOENIX-2369 - vars.insert( - "HADOOP_CONF_DIR".to_string(), - Some(CONFIG_DIR_NAME.to_string()), - ); - Ok(vars) - } - - fn compute_cli( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - Ok(BTreeMap::new()) - } - - fn compute_files( - &self, - _resource: &Self::Configurable, - _role_name: &str, - file: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - let mut result = BTreeMap::new(); - - match file { - HBASE_ENV_SH => { - // The contents of this file cannot be built entirely here because we don't have - // access to the clusterConfig or product version. - // These are needed to set up Kerberos. - // To avoid fragmentation of the code needed to build this file, we moved the - // implementation to the hbase_controller::build_hbase_env_sh() function. - } - HBASE_SITE_XML => { - result.insert( - HBASE_CLUSTER_DISTRIBUTED.to_string(), - Some("true".to_string()), - ); - result.insert(HBASE_ROOTDIR.to_string(), self.hbase_rootdir.clone()); - } - _ => {} - } - - result.retain(|_, maybe_value| maybe_value.is_some()); - - Ok(result) - } -} - #[derive(Fragment, Clone, Debug, JsonSchema, PartialEq, Serialize, Deserialize)] #[fragment_attrs( derive( @@ -1156,71 +1003,6 @@ pub struct RegionServerConfig { pub listener_class: String, } -impl Configuration for RegionServerConfigFragment { - type Configurable = v1alpha1::HbaseCluster; - - fn compute_env( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - let mut vars: BTreeMap> = BTreeMap::new(); - - vars.insert( - "HBASE_CONF_DIR".to_string(), - Some(CONFIG_DIR_NAME.to_string()), - ); - // required by phoenix (for cases where Kerberos is enabled): see https://issues.apache.org/jira/browse/PHOENIX-2369 - vars.insert( - "HADOOP_CONF_DIR".to_string(), - Some(CONFIG_DIR_NAME.to_string()), - ); - Ok(vars) - } - - fn compute_cli( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - Ok(BTreeMap::new()) - } - - fn compute_files( - &self, - _resource: &Self::Configurable, - _role_name: &str, - file: &str, - ) -> Result>, stackable_operator::product_config_utils::Error> - { - let mut result = BTreeMap::new(); - - match file { - HBASE_ENV_SH => { - // The contents of this file cannot be built entirely here because we don't have - // access to the clusterConfig or product version. - // These are needed to set up Kerberos. - // To avoid fragmentation of the code needed to build this file, we moved the - // implementation to the hbase_controller::build_hbase_env_sh() function. - } - HBASE_SITE_XML => { - result.insert( - HBASE_CLUSTER_DISTRIBUTED.to_string(), - Some("true".to_string()), - ); - result.insert(HBASE_ROOTDIR.to_string(), self.hbase_rootdir.clone()); - } - _ => {} - } - - result.retain(|_, maybe_value| maybe_value.is_some()); - - Ok(result) - } -} - #[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct HbaseClusterStatus { @@ -1352,116 +1134,12 @@ impl AnyServiceConfig { #[cfg(test)] mod tests { - use std::collections::{BTreeMap, HashMap}; - use indoc::indoc; - use product_config::{ProductConfigManager, types::PropertyNameKind}; use rstest::rstest; - use stackable_operator::{ - product_config_utils::{ - transform_all_roles_to_config, validate_all_roles_and_groups_config, - }, - versioned::test_utils::RoundtripTestData, - }; + use stackable_operator::versioned::test_utils::RoundtripTestData; use super::*; - #[test] - pub fn test_env_overrides() { - let input = indoc! {r#" ---- -apiVersion: hbase.stackable.tech/v1alpha1 -kind: HbaseCluster -metadata: - name: test-hbase -spec: - image: - productVersion: 2.6.4 - clusterConfig: - hdfsConfigMapName: test-hdfs - zookeeperConfigMapName: test-znode - masters: - envOverrides: - TEST_VAR_FROM_MASTER: MASTER - TEST_VAR: MASTER - config: - logging: - enableVectorAgent: False - roleGroups: - default: - replicas: 1 - envOverrides: - TEST_VAR_FROM_MRG: MASTER - TEST_VAR: MASTER_RG - regionServers: - config: - logging: - enableVectorAgent: False - regionMover: - runBeforeShutdown: false - roleGroups: - default: - replicas: 1 - restServers: - config: - logging: - enableVectorAgent: False - roleGroups: - default: - replicas: 1 - "#}; - - let deserializer = serde_yaml::Deserializer::from_str(input); - let hbase: v1alpha1::HbaseCluster = - serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); - - let roles = HashMap::from([( - HbaseRole::Master.to_string(), - ( - vec![PropertyNameKind::Env], - hbase.spec.masters.clone().unwrap(), - ), - )]); - - let validated_config = validate_all_roles_and_groups_config( - "2.6.4", - &transform_all_roles_to_config(&hbase, &roles).unwrap(), - &ProductConfigManager::from_yaml_file("../../deploy/config-spec/properties.yaml") - .unwrap(), - false, - false, - ) - .unwrap(); - - let rolegroup_config = validated_config - .get(&HbaseRole::Master.to_string()) - .unwrap() - .get("default") - .unwrap() - .get(&PropertyNameKind::Env) - .cloned() - .unwrap_or_default(); - let merged_env = merged_env(&rolegroup_config); - - let env_map: BTreeMap<&str, Option> = merged_env - .iter() - .map(|env_var| (env_var.name.as_str(), env_var.value.clone())) - .collect(); - - assert_eq!( - Some(&Some("MASTER_RG".to_string())), - env_map.get("TEST_VAR") - ); - assert_eq!( - Some(&Some("MASTER".to_string())), - env_map.get("TEST_VAR_FROM_MASTER") - ); - assert_eq!( - Some(&Some("MASTER".to_string())), - env_map.get("TEST_VAR_FROM_MRG") - ); - } - #[rstest] #[case("default", false, 1, vec![])] #[case("groupRegionMover", true, 5, vec!["--some".to_string(), "extra".to_string()])] diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 90fe9c8e..7955e31d 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -1,13 +1,9 @@ //! Ensures that `Pod`s are configured and running for each [`v1alpha1::HbaseCluster`] -use std::{ - collections::{BTreeMap, HashMap}, - sync::Arc, -}; +use std::{collections::BTreeMap, sync::Arc}; use const_format::concatcp; use indoc::formatdoc; -use product_config::{ProductConfigManager, types::PropertyNameKind}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{ @@ -89,7 +85,6 @@ pub const CONTAINER_IMAGE_BASE_NAME: &str = "hbase"; pub struct Ctx { pub client: stackable_operator::client::Client, - pub product_config: ProductConfigManager, pub operator_environment: OperatorEnvironmentOptions, } @@ -122,17 +117,11 @@ pub struct ValidatedRoleConfig { /// Per-rolegroup configuration: the merged CRD config plus the merged /// (role <- role group) `configOverrides` and `envOverrides`. -/// -/// `product_config_properties` is retained temporarily so the existing controller -/// path keeps compiling; it is removed in a later commit. #[derive(Clone, Debug)] pub struct ValidatedRoleGroupConfig { pub merged_config: AnyServiceConfig, pub config_overrides: v1alpha1::HbaseConfigOverrides, pub env_overrides: BTreeMap, - // Retained until the product-config path is fully removed in a later commit. - #[allow(dead_code)] - pub product_config_properties: HashMap>, } #[derive(Snafu, Debug, EnumDiscriminants)] @@ -307,7 +296,6 @@ pub async fn reconcile_hbase( let validated = crate::controller::validate::validate_cluster( hbase, &ctx.operator_environment.image_repository, - &ctx.product_config, dereferenced_objects, ) .context(ValidateSnafu)?; diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 917c95fd..6a97d872 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -73,7 +73,7 @@ async fn main() -> anyhow::Result<()> { Command::Run(RunArguments { operator_environment, watch_namespace, - product_config, + product_config: _, maintenance, common, }) => { @@ -120,11 +120,6 @@ async fn main() -> anyhow::Result<()> { .run(sigterm_watcher.handle()) .map_err(|err| anyhow!(err).context("failed to run webhook server")); - let product_config = product_config.load(&[ - "deploy/config-spec/properties.yaml", - "/etc/stackable/hbase-operator/config-spec/properties.yaml", - ])?; - let event_recorder = Arc::new(Recorder::new( client.as_kube_client(), Reporter { @@ -165,7 +160,6 @@ async fn main() -> anyhow::Result<()> { Arc::new(hbase_controller::Ctx { client: client.clone(), operator_environment, - product_config, }), ) .for_each_concurrent( From 47a854b0bf05ff986fe3674c7c9ae45a1864df8f Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Jun 2026 17:19:32 +0200 Subject: [PATCH 06/56] chore: remove the product-config dependency Empty config-spec/properties.yaml (mirroring trino/opensearch/hdfs) and drop the product-config crate from both Cargo manifests. The operator now renders all config from typed, validated inputs; product-config remains only as a transitive dependency of stackable-operator. Also remove the stale product-config CLI/env-var docs. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 1 + Cargo.lock | 1 - Cargo.toml | 1 - deploy/config-spec/properties.yaml | 268 +----------------- .../reference/commandline-parameters.adoc | 13 - .../reference/environment-variables.adoc | 26 -- rust/operator-binary/Cargo.toml | 1 - 7 files changed, 2 insertions(+), 309 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7e5e974..0942934b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Previously, arbitrary file names were silently accepted and ignored ([#751]). - Bump `stackable-operator` to 0.111.1 and snafu to 0.9 ([#751], [#752]). - Internal operator refactoring: introduce dereference() and validate() steps in the reconciler ([#757]). +- Removed the `product-config` dependency; configuration is now rendered from typed, validated inputs ([#757]). - test: Bump vector-aggregator to 0.55.0, replace /graphql call with gRPC call ([#762]). [#745]: https://github.com/stackabletech/hbase-operator/pull/745 diff --git a/Cargo.lock b/Cargo.lock index 18141326..814820ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2926,7 +2926,6 @@ dependencies = [ "futures 0.3.32", "indoc", "java-properties", - "product-config", "rstest", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index c2e49e69..ed454cd1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,6 @@ edition = "2021" repository = "https://github.com/stackabletech/hbase-operator" [workspace.dependencies] -product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.8.0" } stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.111.1", features = ["crds", "webhook"] } anyhow = "1.0" diff --git a/deploy/config-spec/properties.yaml b/deploy/config-spec/properties.yaml index c3840065..9bd8c3b2 100644 --- a/deploy/config-spec/properties.yaml +++ b/deploy/config-spec/properties.yaml @@ -2,270 +2,4 @@ version: 0.1.0 spec: units: [] - -properties: - - ################################################################################################# - # security.properties - ################################################################################################# - - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "5" - roles: - - name: "master" - required: true - asOfVersion: "0.0.0" - comment: "master - TTL for successfully resolved domain names." - description: "master - TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "10" - roles: - - name: "regionserver" - required: true - asOfVersion: "0.0.0" - comment: "regionserver - TTL for successfully resolved domain names." - description: "regionserver - TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "restserver" - required: true - asOfVersion: "0.0.0" - comment: "restserver - TTL for successfully resolved domain names." - description: "restserver - TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "master" - required: true - asOfVersion: "0.0.0" - comment: "master - TTL for domain names that cannot be resolved." - description: "master - TTL for domain names that cannot be resolved." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "regionserver" - required: true - asOfVersion: "0.0.0" - comment: "regionserver - TTL for domain names that cannot be resolved." - description: "regionserver - TTL for domain names that cannot be resolved." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "restserver" - required: true - asOfVersion: "0.0.0" - comment: "restserver - TTL for domain names that cannot be resolved." - description: "restserver - TTL for domain names that cannot be resolved." - - - ################################################################################################# - # hbase-env.sh - ################################################################################################# - - - property: &hbaseManagesZk - propertyNames: - - name: "HBASE_MANAGES_ZK" - kind: - type: "file" - file: "hbase-env.sh" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - recommendedValues: - - fromVersion: "0.0.0" - value: "false" - roles: - - name: "master" - required: false - - name: "regionserver" - required: false - - name: "restserver" - required: false - asOfVersion: "0.0.0" - description: "This variable tells HBase whether to start/stop the ZooKeeper ensemble servers as part of HBase start/stop." - - property: &hbaseOpts - propertyNames: - - name: "HBASE_OPTS" - kind: - type: "file" - file: "hbase-env.sh" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "" - recommendedValues: - - fromVersion: "0.0.0" - value: "" - roles: - - name: "master" - required: false - - name: "regionserver" - required: false - - name: "restserver" - required: false - asOfVersion: "0.0.0" - description: "This variable allows to pass VM options to HBase." - - ################################################################################################# - # hbase-site.xml - ################################################################################################# - - - property: &hbaseClusterDistributed - propertyNames: - - name: "hbase.cluster.distributed" - kind: - type: "file" - file: "hbase-site.xml" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "false" - recommendedValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "master" - required: false - - name: "regionserver" - required: false - - name: "restserver" - required: false - asOfVersion: "0.0.0" - description: "The mode the cluster will be in. Possible values are false for standalone mode and true for distributed mode. If false, startup will run all HBase and ZooKeeper daemons together in the one JVM." - - - property: &hbaseRootdir - propertyNames: - - name: "hbase.rootdir" - kind: - type: "file" - file: "hbase-site.xml" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "/hbase" - roles: - - name: "master" - required: true - - name: "regionserver" - required: true # this can be false (only required for master) - kept for compatibility (avoid restarts) - - name: "restserver" - required: true # this can be false (only required for master) - kept for compatibility (avoid restarts) - asOfVersion: "0.0.0" - description: "The directory shared by region servers and into which HBase persists. The URL should be 'fully-qualified' to include the filesystem scheme. For example, to specify the HDFS directory '/hbase' where the HDFS instance's namenode is running at namenode.example.org on port 9000, set this value to: hdfs://namenode.example.org:9000/hbase. By default, we write to whatever ${hbase.tmp.dir} is set too -- usually /tmp -- so change this configuration or else all data will be lost on machine restart." - - - property: &hbaseZookeeperQuorum - propertyNames: - - name: "hbase.zookeeper.quorum" - kind: - type: "file" - file: "hbase-site.xml" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "127.0.0.1" - roles: - - name: "master" - required: false - - name: "regionserver" - required: false - - name: "restserver" - required: false - asOfVersion: "0.0.0" - description: "Comma separated list of servers in the ZooKeeper ensemble (This config. should have been named hbase.zookeeper.ensemble). For example, \"host1.mydomain.com,host2.mydomain.com,host3.mydomain.com\". By default this is set to localhost for local and pseudo-distributed modes of operation. For a fully-distributed setup, this should be set to a full list of ZooKeeper ensemble servers. If HBASE_MANAGES_ZK is set in hbase-env.sh this is the list of servers which hbase will start/stop ZooKeeper on as part of cluster start/stop. Client-side, we will take this list of ensemble members and put it together with the hbase.zookeeper.property.clientPort config. and pass it into zookeeper constructor as the connectString parameter." - - ################################################################################################# - # hdfs-site.xml - ################################################################################################# - - - property: &hdfsConfig - propertyNames: - - name: "content" - kind: - type: "file" - file: "hbase-site.xml" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "" - roles: - - name: "master" - required: false - - name: "regionserver" - required: false - - name: "restserver" - required: false - asOfVersion: "0.0.0" - description: "The HDFS configuration file" +properties: [] diff --git a/docs/modules/hbase/pages/reference/commandline-parameters.adoc b/docs/modules/hbase/pages/reference/commandline-parameters.adoc index 8f28dd89..7f15684e 100644 --- a/docs/modules/hbase/pages/reference/commandline-parameters.adoc +++ b/docs/modules/hbase/pages/reference/commandline-parameters.adoc @@ -2,19 +2,6 @@ This operator accepts the following command line parameters: -== product-config - -*Default value*: `/etc/stackable/hbase-operator/config-spec/properties.yaml` - -*Required*: false - -*Multiple values:* false - -[source] ----- -stackable-hbase-operator run --product-config /foo/bar/properties.yaml ----- - == watch-namespace *Default value*: All namespaces diff --git a/docs/modules/hbase/pages/reference/environment-variables.adoc b/docs/modules/hbase/pages/reference/environment-variables.adoc index a6003bce..ab123938 100644 --- a/docs/modules/hbase/pages/reference/environment-variables.adoc +++ b/docs/modules/hbase/pages/reference/environment-variables.adoc @@ -33,32 +33,6 @@ docker run \ oci.stackable.tech/sdp/hbase-operator:0.0.0-dev ---- -== PRODUCT_CONFIG - -*Default value*: `/etc/stackable/hbase-operator/config-spec/properties.yaml` - -*Required*: false - -*Multiple values*: false - -[source] ----- -export PRODUCT_CONFIG=/foo/bar/properties.yaml -stackable-hbase-operator run ----- - -or via docker: - ----- -docker run \ - --name hbase-operator \ - --network host \ - --env KUBECONFIG=/home/stackable/.kube/config \ - --env PRODUCT_CONFIG=/my/product/config.yaml \ - --mount type=bind,source="$HOME/.kube/config",target="/home/stackable/.kube/config" \ - oci.stackable.tech/sdp/hbase-operator:0.0.0-dev ----- - == WATCH_NAMESPACE *Default value*: All namespaces diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 336c050d..35e88f0e 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -9,7 +9,6 @@ repository.workspace = true publish = false [dependencies] -product-config.workspace = true stackable-operator.workspace = true anyhow.workspace = true From 0146d35db028b8af00124ff77fd4106911010ee8 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Jun 2026 19:16:18 +0200 Subject: [PATCH 07/56] refactor: decouple config-map builders from HbaseCluster Pre-resolve kerberos/TLS settings + kerberos flag (ValidatedClusterConfig) and the role-specific non-heap JVM args (ValidatedRoleGroupConfig) during validate, so the hbase-site/hbase-env/ssl-server/ssl-client builders render purely from the validated cluster. The config_map orchestrator still uses HbaseCluster only for the ConfigMap owner reference/metadata (to be decoupled in a follow-up). Rename validated -> validated_cluster. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/controller/build/config_map.rs | 27 +++++----- .../controller/build/properties/hbase_env.rs | 51 +++++++----------- .../controller/build/properties/hbase_site.rs | 54 +++++++------------ .../src/controller/build/properties/mod.rs | 11 +--- .../controller/build/properties/ssl_client.rs | 36 ++++++++----- .../controller/build/properties/ssl_server.rs | 36 ++++++++----- .../src/controller/validate.rs | 27 ++++++++++ rust/operator-binary/src/hbase_controller.rs | 48 +++++++++-------- 8 files changed, 149 insertions(+), 141 deletions(-) diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 741d23a0..796650fc 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -5,7 +5,6 @@ use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, k8s_openapi::api::core::v1::ConfigMap, role_utils::RoleGroupRef, - utils::cluster_info::KubernetesClusterInfo, }; use crate::{ @@ -23,9 +22,6 @@ pub enum Error { #[snafu(display("the validated cluster has no role group {role_group:?} for role {role:?}"))] MissingRoleGroup { role: String, role_group: String }, - #[snafu(display("failed to build hbase-site.xml"))] - BuildHbaseSite { source: hbase_site::Error }, - #[snafu(display("failed to build hbase-env.sh"))] BuildHbaseEnv { source: hbase_env::Error }, @@ -62,10 +58,11 @@ pub enum Error { type Result = std::result::Result; pub fn build_rolegroup_config_map( + // `hbase` is retained only for the ConfigMap ObjectMeta / owner reference; the rendered + // content comes entirely from `cluster`. To be decoupled in a follow-up. hbase: &v1alpha1::HbaseCluster, cluster: &ValidatedCluster, role: &HbaseRole, - cluster_info: &KubernetesClusterInfo, rolegroup_ref: &RoleGroupRef, ) -> Result { tracing::info!("Setting up ConfigMap for {:?}", rolegroup_ref); @@ -83,29 +80,33 @@ pub fn build_rolegroup_config_map( let overrides = &rg.config_overrides; let hbase_site_xml = hbase_site::build( - hbase, role, - cluster_info, &rg.merged_config, cluster_config .zookeeper_connection_information .as_hbase_settings(), + cluster_config.hbase_site_kerberos_config.clone(), cluster_config.hbase_opa_config.as_ref(), overrides.hbase_site_xml.clone(), - ) - .context(BuildHbaseSiteSnafu)?; + ); let hbase_env_sh = hbase_env::build( - hbase, &rg.merged_config, role, - &rolegroup_ref.role_group, + cluster_config.kerberos_enabled, + rg.non_heap_jvm_args.clone(), overrides.hbase_env_sh.clone(), ) .context(BuildHbaseEnvSnafu)?; - let ssl_server_xml = ssl_server::build(hbase, overrides.ssl_server_xml.clone()); - let ssl_client_xml = ssl_client::build(hbase, overrides.ssl_client_xml.clone()); + let ssl_server_xml = ssl_server::build( + cluster_config.ssl_server_settings.clone(), + overrides.ssl_server_xml.clone(), + ); + let ssl_client_xml = ssl_client::build( + cluster_config.ssl_client_settings.clone(), + overrides.ssl_client_xml.clone(), + ); let security_properties = security_properties::build(role, overrides.security_properties.clone()) .with_context(|_| JvmSecurityPropertiesSnafu { diff --git a/rust/operator-binary/src/controller/build/properties/hbase_env.rs b/rust/operator-binary/src/controller/build/properties/hbase_env.rs index a3fbe0c2..42fc905e 100644 --- a/rust/operator-binary/src/controller/build/properties/hbase_env.rs +++ b/rust/operator-binary/src/controller/build/properties/hbase_env.rs @@ -6,29 +6,23 @@ use snafu::{ResultExt, Snafu}; use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; use crate::{ - config::jvm::{ - self, construct_global_jvm_args, construct_hbase_heapsize_env, - construct_role_specific_non_heap_jvm_args, - }, + config::jvm::{self, construct_global_jvm_args, construct_hbase_heapsize_env}, controller::build::properties::resolved_overrides, - crd::{AnyServiceConfig, HbaseRole, v1alpha1}, + crd::{AnyServiceConfig, HbaseRole}, }; #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("failed to construct the HBASE_HEAPSIZE env variable"))] ConstructHbaseHeapsizeEnv { source: jvm::Error }, - - #[snafu(display("failed to construct the JVM arguments"))] - ConstructJvmArgument { source: jvm::Error }, } /// Renders `hbase-env.sh` as `export VAR="VALUE"` lines. pub fn build( - hbase: &v1alpha1::HbaseCluster, merged_config: &AnyServiceConfig, role: &HbaseRole, - role_group: &str, + kerberos_enabled: bool, + non_heap_jvm_args: String, overrides: KeyValueConfigOverrides, ) -> Result { let mut env: BTreeMap = BTreeMap::new(); @@ -40,30 +34,18 @@ pub fn build( ); env.insert( "HBASE_OPTS".to_string(), - construct_global_jvm_args(hbase.has_kerberos_enabled()), + construct_global_jvm_args(kerberos_enabled), ); - let role_specific_non_heap_jvm_args = - construct_role_specific_non_heap_jvm_args(hbase, role, role_group) - .context(ConstructJvmArgumentSnafu)?; match role { HbaseRole::Master => { - env.insert( - "HBASE_MASTER_OPTS".to_string(), - role_specific_non_heap_jvm_args, - ); + env.insert("HBASE_MASTER_OPTS".to_string(), non_heap_jvm_args); } HbaseRole::RegionServer => { - env.insert( - "HBASE_REGIONSERVER_OPTS".to_string(), - role_specific_non_heap_jvm_args, - ); + env.insert("HBASE_REGIONSERVER_OPTS".to_string(), non_heap_jvm_args); } HbaseRole::RestServer => { - env.insert( - "HBASE_REST_OPTS".to_string(), - role_specific_non_heap_jvm_args, - ); + env.insert("HBASE_REST_OPTS".to_string(), non_heap_jvm_args); } } @@ -81,7 +63,10 @@ pub fn build( #[cfg(test)] mod tests { use super::*; - use crate::controller::build::properties::test_support::{config_overrides, minimal_hbase}; + use crate::{ + controller::build::properties::test_support::{config_overrides, minimal_hbase}, + crd::v1alpha1, + }; fn master_merged_config(hbase: &v1alpha1::HbaseCluster) -> AnyServiceConfig { hbase @@ -100,10 +85,10 @@ mod tests { let hbase = minimal_hbase(); let merged = master_merged_config(&hbase); let env = build( - &hbase, &merged, &HbaseRole::Master, - "default", + false, + "-Xtest".to_string(), config_overrides(&[]), ) .unwrap(); @@ -116,10 +101,10 @@ mod tests { let hbase = minimal_hbase(); let merged = region_server_merged_config(&hbase); let env = build( - &hbase, &merged, &HbaseRole::RegionServer, - "default", + false, + "-Xtest".to_string(), config_overrides(&[]), ) .unwrap(); @@ -131,10 +116,10 @@ mod tests { let hbase = minimal_hbase(); let merged = master_merged_config(&hbase); let env = build( - &hbase, &merged, &HbaseRole::Master, - "default", + false, + "-Xtest".to_string(), config_overrides(&[("CUSTOM_VAR", "custom_value")]), ) .unwrap(); diff --git a/rust/operator-binary/src/controller/build/properties/hbase_site.rs b/rust/operator-binary/src/controller/build/properties/hbase_site.rs index f97a1b2d..4bae593c 100644 --- a/rust/operator-binary/src/controller/build/properties/hbase_site.rs +++ b/rust/operator-binary/src/controller/build/properties/hbase_site.rs @@ -4,39 +4,28 @@ use std::collections::BTreeMap; -use snafu::{ResultExt, Snafu}; -use stackable_operator::{ - utils::cluster_info::KubernetesClusterInfo, v2::config_overrides::KeyValueConfigOverrides, -}; +use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; use crate::{ config::writer::to_hadoop_xml, controller::build::properties::resolved_overrides, crd::{ AnyServiceConfig, HBASE_CLUSTER_DISTRIBUTED, HBASE_MASTER_PORT, HBASE_MASTER_UI_PORT, - HBASE_REGIONSERVER_PORT, HBASE_REGIONSERVER_UI_PORT, HBASE_ROOTDIR, HbaseRole, v1alpha1, + HBASE_REGIONSERVER_PORT, HBASE_REGIONSERVER_UI_PORT, HBASE_ROOTDIR, HbaseRole, }, - kerberos::{self, kerberos_config_properties}, security::opa::HbaseOpaConfig, }; -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("failed to add the kerberos configuration"))] - AddKerberosConfig { source: kerberos::Error }, -} - /// Renders `hbase-site.xml`. #[allow(clippy::too_many_arguments)] pub fn build( - hbase: &v1alpha1::HbaseCluster, role: &HbaseRole, - cluster_info: &KubernetesClusterInfo, merged_config: &AnyServiceConfig, zookeeper_config: BTreeMap, + kerberos_config: BTreeMap, opa_config: Option<&HbaseOpaConfig>, overrides: KeyValueConfigOverrides, -) -> Result { +) -> String { let mut config: BTreeMap = BTreeMap::new(); // Defaults previously injected by product-config's `compute_files`. @@ -46,7 +35,7 @@ pub fn build( } config.extend(zookeeper_config); - config.extend(kerberos_config_properties(hbase, cluster_info).context(AddKerberosConfigSnafu)?); + config.extend(kerberos_config); config.extend(opa_config.map_or(vec![], |config| config.hbase_site_config())); // Set flag to override default behaviour, which is that the @@ -124,20 +113,21 @@ pub fn build( // configOverride come last config.extend(resolved_overrides(overrides)); - Ok(to_hadoop_xml( + to_hadoop_xml( config .into_iter() .map(|(k, v)| (k, Some(v))) .collect::>() .iter(), - )) + ) } #[cfg(test)] mod tests { use super::*; - use crate::controller::build::properties::test_support::{ - cluster_info, config_overrides, minimal_hbase, + use crate::{ + controller::build::properties::test_support::{config_overrides, minimal_hbase}, + crd::v1alpha1, }; fn master_merged_config(hbase: &v1alpha1::HbaseCluster) -> AnyServiceConfig { @@ -163,15 +153,13 @@ mod tests { let hbase = minimal_hbase(); let merged = master_merged_config(&hbase); let xml = build( - &hbase, &HbaseRole::Master, - &cluster_info(), &merged, BTreeMap::new(), + BTreeMap::new(), None, config_overrides(&[]), - ) - .unwrap(); + ); assert!( xml.contains("hbase.cluster.distributed\n true"), "{xml}" @@ -187,15 +175,13 @@ mod tests { let hbase = minimal_hbase(); let merged = region_server_merged_config(&hbase); let xml = build( - &hbase, &HbaseRole::RegionServer, - &cluster_info(), &merged, BTreeMap::new(), + BTreeMap::new(), None, config_overrides(&[]), - ) - .unwrap(); + ); assert!( xml.contains( "hbase.regionserver.ipc.address\n 0.0.0.0" @@ -215,15 +201,13 @@ mod tests { let hbase = minimal_hbase(); let merged = rest_server_merged_config(&hbase); let xml = build( - &hbase, &HbaseRole::RestServer, - &cluster_info(), &merged, BTreeMap::new(), + BTreeMap::new(), None, config_overrides(&[]), - ) - .unwrap(); + ); assert!( xml.contains( "hbase.rest.endpoint\n ${env:HBASE_SERVICE_HOST}:${env:HBASE_SERVICE_PORT}" @@ -237,15 +221,13 @@ mod tests { let hbase = minimal_hbase(); let merged = master_merged_config(&hbase); let xml = build( - &hbase, &HbaseRole::Master, - &cluster_info(), &merged, BTreeMap::new(), + BTreeMap::new(), None, config_overrides(&[("hbase.cluster.distributed", "false")]), - ) - .unwrap(); + ); assert!( xml.contains("hbase.cluster.distributed\n false"), "{xml}" diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index 5fb54649..31c7af24 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -61,10 +61,7 @@ mod tests { #[cfg(test)] pub(crate) mod test_support { - use stackable_operator::{ - commons::networking::DomainName, utils::cluster_info::KubernetesClusterInfo, - v2::config_overrides::KeyValueConfigOverrides, - }; + use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; use crate::crd::v1alpha1; @@ -109,10 +106,4 @@ spec: pub fn minimal_hbase() -> v1alpha1::HbaseCluster { serde_yaml::from_str(MINIMAL_HBASE_YAML).expect("invalid test HbaseCluster YAML") } - - pub fn cluster_info() -> KubernetesClusterInfo { - KubernetesClusterInfo { - cluster_domain: DomainName::try_from("cluster.local").unwrap(), - } - } } diff --git a/rust/operator-binary/src/controller/build/properties/ssl_client.rs b/rust/operator-binary/src/controller/build/properties/ssl_client.rs index fdfac797..c6db3f50 100644 --- a/rust/operator-binary/src/controller/build/properties/ssl_client.rs +++ b/rust/operator-binary/src/controller/build/properties/ssl_client.rs @@ -3,19 +3,12 @@ use std::collections::BTreeMap; use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; -use crate::{ - config::writer::to_hadoop_xml, controller::build::properties::resolved_overrides, - crd::v1alpha1, kerberos::kerberos_ssl_client_settings, -}; +use crate::{config::writer::to_hadoop_xml, controller::build::properties::resolved_overrides}; /// Renders `ssl-client.xml`. Returns "" (HBase rejects empty XML files) when empty. -pub fn build(hbase: &v1alpha1::HbaseCluster, overrides: KeyValueConfigOverrides) -> String { +pub fn build(settings: BTreeMap, overrides: KeyValueConfigOverrides) -> String { let mut config: BTreeMap> = BTreeMap::new(); - config.extend( - kerberos_ssl_client_settings(hbase) - .into_iter() - .map(|(k, v)| (k, Some(v))), - ); + config.extend(settings.into_iter().map(|(k, v)| (k, Some(v)))); config.extend(resolved_overrides(overrides).map(|(k, v)| (k, Some(v)))); if config.is_empty() { return String::new(); @@ -26,17 +19,32 @@ pub fn build(hbase: &v1alpha1::HbaseCluster, overrides: KeyValueConfigOverrides) #[cfg(test)] mod tests { use super::*; - use crate::controller::build::properties::test_support::{config_overrides, minimal_hbase}; + use crate::controller::build::properties::test_support::config_overrides; #[test] - fn non_kerberos_without_overrides_renders_empty_string() { - assert_eq!(build(&minimal_hbase(), config_overrides(&[])), ""); + fn empty_settings_without_overrides_renders_empty_string() { + assert_eq!(build(BTreeMap::new(), config_overrides(&[])), ""); + } + + #[test] + fn settings_appear_in_xml() { + let xml = build( + BTreeMap::from([( + "ssl.client.truststore.type".to_string(), + "pkcs12".to_string(), + )]), + config_overrides(&[]), + ); + assert!( + xml.contains("ssl.client.truststore.type\n pkcs12"), + "{xml}" + ); } #[test] fn user_override_appears_in_xml() { let xml = build( - &minimal_hbase(), + BTreeMap::new(), config_overrides(&[("ssl.client.keystore.type", "jks")]), ); assert!( diff --git a/rust/operator-binary/src/controller/build/properties/ssl_server.rs b/rust/operator-binary/src/controller/build/properties/ssl_server.rs index f2d6a91c..36da52ea 100644 --- a/rust/operator-binary/src/controller/build/properties/ssl_server.rs +++ b/rust/operator-binary/src/controller/build/properties/ssl_server.rs @@ -3,19 +3,12 @@ use std::collections::BTreeMap; use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; -use crate::{ - config::writer::to_hadoop_xml, controller::build::properties::resolved_overrides, - crd::v1alpha1, kerberos::kerberos_ssl_server_settings, -}; +use crate::{config::writer::to_hadoop_xml, controller::build::properties::resolved_overrides}; /// Renders `ssl-server.xml`. Returns "" (HBase rejects empty XML files) when empty. -pub fn build(hbase: &v1alpha1::HbaseCluster, overrides: KeyValueConfigOverrides) -> String { +pub fn build(settings: BTreeMap, overrides: KeyValueConfigOverrides) -> String { let mut config: BTreeMap> = BTreeMap::new(); - config.extend( - kerberos_ssl_server_settings(hbase) - .into_iter() - .map(|(k, v)| (k, Some(v))), - ); + config.extend(settings.into_iter().map(|(k, v)| (k, Some(v)))); config.extend(resolved_overrides(overrides).map(|(k, v)| (k, Some(v)))); if config.is_empty() { return String::new(); @@ -26,17 +19,32 @@ pub fn build(hbase: &v1alpha1::HbaseCluster, overrides: KeyValueConfigOverrides) #[cfg(test)] mod tests { use super::*; - use crate::controller::build::properties::test_support::{config_overrides, minimal_hbase}; + use crate::controller::build::properties::test_support::config_overrides; #[test] - fn non_kerberos_without_overrides_renders_empty_string() { - assert_eq!(build(&minimal_hbase(), config_overrides(&[])), ""); + fn empty_settings_without_overrides_renders_empty_string() { + assert_eq!(build(BTreeMap::new(), config_overrides(&[])), ""); + } + + #[test] + fn settings_appear_in_xml() { + let xml = build( + BTreeMap::from([( + "ssl.server.keystore.type".to_string(), + "pkcs12".to_string(), + )]), + config_overrides(&[]), + ); + assert!( + xml.contains("ssl.server.keystore.type\n pkcs12"), + "{xml}" + ); } #[test] fn user_override_appears_in_xml() { let xml = build( - &minimal_hbase(), + BTreeMap::new(), config_overrides(&[("ssl.server.keystore.type", "jks")]), ); assert!( diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 720d9c2d..bdc610da 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -6,17 +6,22 @@ use stackable_operator::{ config::merge::Merge, kube::ResourceExt, role_utils::GenericRoleConfig, + utils::cluster_info::KubernetesClusterInfo, v2::types::operator::ClusterName, }; use strum::IntoEnumIterator; use crate::{ + config::jvm::construct_role_specific_non_heap_jvm_args, controller::dereference::DereferencedObjects, crd::{HbaseRole, v1alpha1}, hbase_controller::{ CONTAINER_IMAGE_BASE_NAME, ValidatedCluster, ValidatedClusterConfig, ValidatedRoleConfig, ValidatedRoleGroupConfig, }, + kerberos::{ + self, kerberos_config_properties, kerberos_ssl_client_settings, kerberos_ssl_server_settings, + }, }; #[derive(Snafu, Debug)] @@ -36,11 +41,18 @@ pub enum Error { #[snafu(display("failed to resolve and merge config for role and role group"))] FailedToResolveConfig { source: crate::crd::Error }, + + #[snafu(display("failed to resolve kerberos config"))] + AddKerberosConfig { source: kerberos::Error }, + + #[snafu(display("failed to construct role-specific JVM arguments"))] + ConstructJvmArgument { source: crate::config::jvm::Error }, } pub fn validate_cluster( hbase: &v1alpha1::HbaseCluster, image_repository: &str, + cluster_info: &KubernetesClusterInfo, dereferenced_objects: DereferencedObjects, ) -> Result { let resolved_product_image = hbase @@ -96,6 +108,12 @@ pub fn validate_cluster( merged_config, config_overrides: merged_config_overrides(hbase, &hbase_role, &rolegroup_name), env_overrides: merged_env_overrides(hbase, &hbase_role, &rolegroup_name), + non_heap_jvm_args: construct_role_specific_non_heap_jvm_args( + hbase, + &hbase_role, + &rolegroup_name, + ) + .context(ConstructJvmArgumentSnafu)?, }, ); } @@ -103,6 +121,11 @@ pub fn validate_cluster( role_groups.insert(hbase_role, group_configs); } + let hbase_site_kerberos_config = + kerberos_config_properties(hbase, cluster_info).context(AddKerberosConfigSnafu)?; + let ssl_server_settings = kerberos_ssl_server_settings(hbase); + let ssl_client_settings = kerberos_ssl_client_settings(hbase); + Ok(ValidatedCluster { name: ClusterName::from_str(&hbase.name_any()).context(InvalidClusterNameSnafu)?, image: resolved_product_image, @@ -110,6 +133,10 @@ pub fn validate_cluster( zookeeper_connection_information: dereferenced_objects .zookeeper_connection_information, hbase_opa_config: dereferenced_objects.hbase_opa_config, + kerberos_enabled: hbase.has_kerberos_enabled(), + hbase_site_kerberos_config, + ssl_server_settings, + ssl_client_settings, }, role_group_configs: role_groups, role_configs, diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 7955e31d..41df54be 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -52,7 +52,7 @@ use stackable_operator::{ }, v2::types::operator::ClusterName, }; -use strum::{EnumDiscriminants, IntoStaticStr, ParseError}; +use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ OPERATOR_NAME, @@ -107,6 +107,13 @@ pub struct ValidatedCluster { pub struct ValidatedClusterConfig { pub zookeeper_connection_information: ZookeeperConnectionInformation, pub hbase_opa_config: Option, + pub kerberos_enabled: bool, + /// Pre-resolved kerberos properties for hbase-site.xml (empty when kerberos is disabled). + pub hbase_site_kerberos_config: BTreeMap, + /// Pre-resolved ssl-server.xml settings (empty when HTTPS is disabled). + pub ssl_server_settings: BTreeMap, + /// Pre-resolved ssl-client.xml settings (empty when HTTPS is disabled). + pub ssl_client_settings: BTreeMap, } /// Per-role configuration extracted during validation. @@ -122,6 +129,8 @@ pub struct ValidatedRoleGroupConfig { pub merged_config: AnyServiceConfig, pub config_overrides: v1alpha1::HbaseConfigOverrides, pub env_overrides: BTreeMap, + /// Pre-resolved role-specific non-heap JVM args (operator-generated + role/role-group overrides). + pub non_heap_jvm_args: String, } #[derive(Snafu, Debug, EnumDiscriminants)] @@ -176,12 +185,6 @@ pub enum Error { source: stackable_operator::builder::meta::Error, }, - #[snafu(display("no configmap_name for {cm_name} discovery is configured"))] - MissingConfigMap { - source: stackable_operator::builder::meta::Error, - cm_name: String, - }, - #[snafu(display("failed to patch service account"))] ApplyServiceAccount { source: stackable_operator::cluster_resources::Error, @@ -226,9 +229,6 @@ pub enum Error { source: stackable_operator::builder::meta::Error, }, - #[snafu(display("unknown role [{role}]"))] - UnknownHbaseRole { source: ParseError, role: String }, - #[snafu(display("failed to configure logging"))] ConfigureLogging { source: LoggingError }, @@ -293,9 +293,10 @@ pub async fn reconcile_hbase( .await .context(DereferenceSnafu)?; - let validated = crate::controller::validate::validate_cluster( + let validated_cluster = crate::controller::validate::validate_cluster( hbase, &ctx.operator_environment.image_repository, + &client.kubernetes_cluster_info, dereferenced_objects, ) .context(ValidateSnafu)?; @@ -329,21 +330,24 @@ pub async fn reconcile_hbase( let mut ss_cond_builder = StatefulSetConditionBuilder::default(); - for (hbase_role, role_group_configs) in &validated.role_group_configs { + for (hbase_role, role_group_configs) in &validated_cluster.role_group_configs { for (rolegroup_name, validated_rg_config) in role_group_configs { let rolegroup = hbase.server_rolegroup_ref(hbase_role.to_string(), rolegroup_name); let rg_service = - build_rolegroup_service(hbase, hbase_role, &rolegroup, &validated.image)?; + build_rolegroup_service(hbase, hbase_role, &rolegroup, &validated_cluster.image)?; - let rg_metrics_service = - build_rolegroup_metrics_service(hbase, hbase_role, &rolegroup, &validated.image)?; + let rg_metrics_service = build_rolegroup_metrics_service( + hbase, + hbase_role, + &rolegroup, + &validated_cluster.image, + )?; let rg_configmap = crate::controller::build::config_map::build_rolegroup_config_map( hbase, - &validated, + &validated_cluster, hbase_role, - &client.kubernetes_cluster_info, &rolegroup, ) .context(BuildRolegroupConfigMapSnafu)?; @@ -352,7 +356,7 @@ pub async fn reconcile_hbase( hbase_role, &rolegroup, validated_rg_config, - &validated.image, + &validated_cluster.image, &rbac_sa, )?; cluster_resources @@ -387,7 +391,7 @@ pub async fn reconcile_hbase( ); } - if let Some(role_config) = validated.role_configs.get(hbase_role) { + if let Some(role_config) = validated_cluster.role_configs.get(hbase_role) { add_pdbs( &role_config.pdb, hbase, @@ -405,8 +409,10 @@ pub async fn reconcile_hbase( let discovery_cm = build_discovery_configmap( hbase, &client.kubernetes_cluster_info, - &validated.cluster_config.zookeeper_connection_information, - &validated.image, + &validated_cluster + .cluster_config + .zookeeper_connection_information, + &validated_cluster.image, ) .context(BuildDiscoveryConfigMapSnafu)?; cluster_resources From 965c2126abfcd2fd5329517a49a0ae9354c8da20 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Jun 2026 19:28:31 +0200 Subject: [PATCH 08/56] chore: regenerate --- Cargo.nix | 1396 +++++++---------- crate-hashes.json | 18 +- .../hbase-operator/configs/properties.yaml | 268 +--- extra/crds.yaml | 90 +- 4 files changed, 633 insertions(+), 1139 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index fdd7ec68..55fc2419 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -216,9 +216,9 @@ rec { }; "anstream" = rec { crateName = "anstream"; - version = "0.6.21"; + version = "1.0.0"; edition = "2021"; - sha256 = "0jjgixms4qjj58dzr846h2s29p8w7ynwr9b9x6246m1pwy0v5ma3"; + sha256 = "13d2bj0xfg012s4rmq44zc8zgy1q8k9yp7yhvfnarscnmwpj2jl2"; dependencies = [ { name = "anstyle"; @@ -261,9 +261,9 @@ rec { }; "anstyle" = rec { crateName = "anstyle"; - version = "1.0.13"; + version = "1.0.14"; edition = "2021"; - sha256 = "0y2ynjqajpny6q0amvfzzgw0gfw3l47z85km4gvx87vg02lcr4ji"; + sha256 = "0030szmgj51fxkic1hpakxxgappxzwm6m154a3gfml83lq63l2wl"; features = { "default" = [ "std" ]; }; @@ -271,9 +271,9 @@ rec { }; "anstyle-parse" = rec { crateName = "anstyle-parse"; - version = "0.2.7"; + version = "1.0.0"; edition = "2021"; - sha256 = "1hhmkkfr95d462b3zf6yl2vfzdqfy5726ya572wwg8ha9y148xjf"; + sha256 = "03hkv2690s0crssbnmfkr76kw1k7ah2i6s5amdy9yca2n8w7zkjj"; libName = "anstyle_parse"; dependencies = [ { @@ -345,9 +345,9 @@ rec { }; "arc-swap" = rec { crateName = "arc-swap"; - version = "1.8.2"; + version = "1.9.1"; edition = "2018"; - sha256 = "19aas8y3kz0v6jr6yijvw6cad9grpl3lw1a25k0cws2m2iy69wzr"; + sha256 = "01xjlahcya8igdalxmda375lnlhjqwjz0cdqhy0bc1jkyzb1yfka"; libName = "arc_swap"; authors = [ "Michal 'vorner' Vaner " @@ -489,9 +489,9 @@ rec { }; "autocfg" = rec { crateName = "autocfg"; - version = "1.5.0"; + version = "1.5.1"; edition = "2015"; - sha256 = "1s77f98id9l4af4alklmzq46f21c980v13z2r1pcxx6bqgw0d1n0"; + sha256 = "0lqasy5i30flcgih1b50kvsk6z32g09r1q4ql7q81pj6228jy0zj"; authors = [ "Josh Stone " ]; @@ -499,9 +499,9 @@ rec { }; "axum" = rec { crateName = "axum"; - version = "0.8.8"; + version = "0.8.9"; edition = "2021"; - sha256 = "1f4p0m04mgwpn8b40i9r5mgqxk6w11sv4yri6xfqk305nhyayllb"; + sha256 = "146df5x8dhczm1sp939gr3839220wl6rxc1k65bzc450z72ridii"; dependencies = [ { name = "axum-core"; @@ -866,9 +866,9 @@ rec { }; "bitflags" = rec { crateName = "bitflags"; - version = "2.11.0"; + version = "2.12.1"; edition = "2021"; - sha256 = "1bwjibwry5nfwsfm9kjg2dqx5n5nja9xymwbfl6svnn8jsz6ff44"; + sha256 = "02phhjm7w380zdh8928zf13cfi1bw2qz2ay36ml2jmwmmv8cxmw4"; authors = [ "The Rust Project Developers" ]; @@ -898,9 +898,9 @@ rec { }; "built" = rec { crateName = "built"; - version = "0.8.0"; - edition = "2021"; - sha256 = "0r5f08lpjsr6j5ajkbmd0ymfmajpq8ddbfvi8ji8rx48y88qzbgl"; + version = "0.8.1"; + edition = "2024"; + sha256 = "1saq332pd6g3svvc9ah8myjpfvgqlzl2ksb1ypp3976kjcfm63jw"; authors = [ "Lukas Lueg " ]; @@ -924,15 +924,16 @@ rec { "chrono" = [ "dep:chrono" ]; "dependency-tree" = [ "cargo-lock/dependency-tree" ]; "git2" = [ "dep:git2" ]; + "gix" = [ "dep:gix" ]; "semver" = [ "dep:semver" ]; }; resolvedDefaultFeatures = [ "chrono" "git2" ]; }; "bumpalo" = rec { crateName = "bumpalo"; - version = "3.20.2"; + version = "3.20.3"; edition = "2021"; - sha256 = "1jrgxlff76k9glam0akhwpil2fr1w32gbjdf5hpipc7ld2c7h82x"; + sha256 = "0jc6va3nwcqikm7chnpdv1s87my3gs2j7g1sc7g3k91brg3arxbj"; authors = [ "Nick Fitzgerald " ]; @@ -961,9 +962,9 @@ rec { }; "cc" = rec { crateName = "cc"; - version = "1.2.56"; + version = "1.2.63"; edition = "2018"; - sha256 = "1chvh9g2izhqad7vzy4cc7xpdljdvqpsr6x6hv1hmyqv3mlkbgxf"; + sha256 = "0zy2bqc4nvj6bv2cipx4h4bn65wf1zqf1fw1hsh64mmvg1hh2vjm"; authors = [ "Alex Crichton " ]; @@ -1060,10 +1061,10 @@ rec { }; "clap" = rec { crateName = "clap"; - version = "4.5.60"; - edition = "2021"; + version = "4.6.1"; + edition = "2024"; crateBin = []; - sha256 = "02h3nzznssjgp815nnbzk0r62y2iw03kdli75c233kirld6z75r7"; + sha256 = "0lcf88l7vlg796rrqr7wipbbmfa5sgsgx4211b7xmxxv8dz13nqx"; dependencies = [ { name = "clap_builder"; @@ -1102,9 +1103,9 @@ rec { }; "clap_builder" = rec { crateName = "clap_builder"; - version = "4.5.60"; - edition = "2021"; - sha256 = "0xk8mdizvmmn6w5ij5cwhy5pbgyac4w9pfvl6nqmjl7a5hql38i4"; + version = "4.6.0"; + edition = "2024"; + sha256 = "17q6np22yxhh5y5v53y4l31ps3hlaz45mvz2n2nicr7n3c056jki"; dependencies = [ { name = "anstream"; @@ -1141,9 +1142,9 @@ rec { }; "clap_derive" = rec { crateName = "clap_derive"; - version = "4.5.55"; - edition = "2021"; - sha256 = "1r949xis3jmhzh387smd70vc8a3b9734ck3g5ahg59a63bd969x9"; + version = "4.6.1"; + edition = "2024"; + sha256 = "1acpz49hi00iv9jkapixjzcv7s51x8qkfaqscjm36rqgf428dkpj"; procMacro = true; dependencies = [ { @@ -1173,16 +1174,16 @@ rec { }; "clap_lex" = rec { crateName = "clap_lex"; - version = "1.0.0"; - edition = "2021"; - sha256 = "0c8888qi1l9sayqlv666h8s0yxn2qc6jr88v1zagk43mpjjjx0is"; + version = "1.1.0"; + edition = "2024"; + sha256 = "1ycqkpygnlqnndghhcxjb44lzl0nmgsia64x9581030yifxs7m68"; }; "colorchoice" = rec { crateName = "colorchoice"; - version = "1.0.4"; + version = "1.0.5"; edition = "2021"; - sha256 = "0x8ymkz1xr77rcj1cfanhf416pc4v681gmkc9dzb3jqja7f62nxh"; + sha256 = "0w75k89hw39p0mnnhlrwr23q50rza1yjki44qvh2mgrnj065a1qx"; }; "concurrent-queue" = rec { @@ -1226,9 +1227,9 @@ rec { }; "const_format" = rec { crateName = "const_format"; - version = "0.2.35"; + version = "0.2.36"; edition = "2021"; - sha256 = "1b9h03z3k76ail1ldqxcqmsc4raa7dwgwwqwrjf6wmism5lp9akz"; + sha256 = "07ncczs8yndga2f8p4386c827l4fxwzl0pbwp7ijnhcsmlbsd0a4"; authors = [ "rodrimati1992 " ]; @@ -1237,6 +1238,12 @@ rec { name = "const_format_proc_macros"; packageId = "const_format_proc_macros"; } + { + name = "konst"; + packageId = "konst"; + usesDefaultFeatures = false; + features = [ "rust_1_64" ]; + } ]; features = { "__debug" = [ "const_format_proc_macros/debug" ]; @@ -1250,10 +1257,9 @@ rec { "constant_time_as_str" = [ "fmt" ]; "derive" = [ "fmt" "const_format_proc_macros/derive" ]; "fmt" = [ "rust_1_83" ]; - "konst" = [ "dep:konst" ]; "more_str_macros" = [ "rust_1_64" ]; "nightly_const_generics" = [ "const_generics" ]; - "rust_1_64" = [ "rust_1_51" "konst" "konst/rust_1_64" ]; + "rust_1_64" = [ "rust_1_51" ]; "rust_1_83" = [ "rust_1_64" ]; }; resolvedDefaultFeatures = [ "default" ]; @@ -1901,9 +1907,9 @@ rec { }; "displaydoc" = rec { crateName = "displaydoc"; - version = "0.2.5"; + version = "0.2.6"; edition = "2021"; - sha256 = "1q0alair462j21iiqwrr21iabkfnb13d6x5w95lkdg21q2xrqdlp"; + sha256 = "0kyxwfbdmagd8afzb2pzja7wj8dhah7smxdsgw00iq8pa2jhmiqs"; procMacro = true; authors = [ "Jane Lusby " @@ -2103,12 +2109,9 @@ rec { }; "either" = rec { crateName = "either"; - version = "1.15.0"; + version = "1.16.0"; edition = "2021"; - sha256 = "069p1fknsmzn9llaizh77kip0pqmcwpdsykv2x30xpjyija5gis8"; - authors = [ - "bluss" - ]; + sha256 = "17k7jfbdz7k440h6lws9baz8p9zlxgb41sig3w81h80nwzsjyqli"; features = { "default" = [ "std" ]; "serde" = [ "dep:serde" ]; @@ -2452,9 +2455,9 @@ rec { }; "fastrand" = rec { crateName = "fastrand"; - version = "2.3.0"; + version = "2.4.1"; edition = "2018"; - sha256 = "1ghiahsw1jd68df895cy5h3gzwk30hndidn3b682zmshpgmrx41p"; + sha256 = "1mnqxxnxvd69ma9mczabpbbsgwlhd6l78yv3vd681453a9s247wz"; authors = [ "Stjepan Glavina " ]; @@ -2825,9 +2828,9 @@ rec { }; "futures-timer" = rec { crateName = "futures-timer"; - version = "3.0.3"; + version = "3.0.4"; edition = "2018"; - sha256 = "094vw8k37djpbwv74bwf2qb7n6v6ghif4myss6smd6hgyajb127j"; + sha256 = "0s39in8ivw7g4d37pf31q02y44zd1hpfkd1pgra2slcqibdzlhxg"; libName = "futures_timer"; authors = [ "Alex Crichton " @@ -3085,9 +3088,9 @@ rec { }; "git2" = rec { crateName = "git2"; - version = "0.20.4"; - edition = "2018"; - sha256 = "0azykjpk3j6s354z23jkyq3r3pbmlw9ha1zsxkw5cnnpi1h2b23v"; + version = "0.21.0"; + edition = "2021"; + sha256 = "0bmqga9vlyx5sdlr0i28z0362s89xv9i4qcv20vvx9j54y9vzpfx"; authors = [ "Josh Triplett " "Alex Crichton " @@ -3109,17 +3112,14 @@ rec { name = "log"; packageId = "log"; } - { - name = "url"; - packageId = "url"; - } ]; features = { - "default" = [ "ssh" "https" ]; - "https" = [ "libgit2-sys/https" "openssl-sys" "openssl-probe" ]; + "cred" = [ "dep:url" ]; + "https" = [ "libgit2-sys/https" "openssl-sys" "openssl-probe" "cred" ]; "openssl-probe" = [ "dep:openssl-probe" ]; "openssl-sys" = [ "dep:openssl-sys" ]; - "ssh" = [ "libgit2-sys/ssh" ]; + "ssh" = [ "libgit2-sys/ssh" "cred" ]; + "unstable-sha256" = [ "libgit2-sys/unstable-sha256" ]; "vendored-libgit2" = [ "libgit2-sys/vendored" ]; "vendored-openssl" = [ "openssl-sys/vendored" "libgit2-sys/vendored-openssl" ]; "zlib-ng-compat" = [ "libgit2-sys/zlib-ng-compat" ]; @@ -3209,9 +3209,9 @@ rec { }; "h2" = rec { crateName = "h2"; - version = "0.4.13"; + version = "0.4.14"; edition = "2021"; - sha256 = "0m6w5gg0n0m1m5915bxrv8n4rlazhx5icknkslz719jhh4xdli1g"; + sha256 = "0cw7jk7kn2vn6f8w8ssh6gis1mljnfjxd606gvi4sjpyjayfy7qp"; authors = [ "Carl Lerche " "Sean McArthur " @@ -3279,7 +3279,7 @@ rec { features = { }; }; - "hashbrown" = rec { + "hashbrown 0.16.1" = rec { crateName = "hashbrown"; version = "0.16.1"; edition = "2021"; @@ -3322,6 +3322,24 @@ rec { }; resolvedDefaultFeatures = [ "allocator-api2" "default" "default-hasher" "equivalent" "inline-more" "raw-entry" ]; }; + "hashbrown 0.17.1" = rec { + crateName = "hashbrown"; + version = "0.17.1"; + edition = "2024"; + sha256 = "0jmqz7i4yl6cm7rbn0i2ffkfrmwi6xkmzkaldr2v8bcsx2v0jngd"; + features = { + "alloc" = [ "dep:alloc" ]; + "allocator-api2" = [ "dep:allocator-api2" ]; + "core" = [ "dep:core" ]; + "default" = [ "default-hasher" "inline-more" "allocator-api2" "equivalent" "raw-entry" ]; + "default-hasher" = [ "dep:foldhash" ]; + "equivalent" = [ "dep:equivalent" ]; + "nightly" = [ "foldhash?/nightly" "bumpalo/allocator_api" ]; + "rayon" = [ "dep:rayon" ]; + "rustc-dep-of-std" = [ "nightly" "core" "alloc" "rustc-internal-api" ]; + "serde" = [ "dep:serde_core" "dep:serde" ]; + }; + }; "heck" = rec { crateName = "heck"; version = "0.5.0"; @@ -3383,9 +3401,9 @@ rec { }; "http" = rec { crateName = "http"; - version = "1.4.0"; + version = "1.4.1"; edition = "2021"; - sha256 = "06iind4cwsj1d6q8c2xgq8i2wka4ps74kmws24gsi1bzdlw2mfp3"; + sha256 = "1l7k2ia57z3q7q3ka497krzps795kd3fymm2k12lr623y4nldrwb"; authors = [ "Alex Crichton " "Carl Lerche " @@ -3502,9 +3520,9 @@ rec { }; "hyper" = rec { crateName = "hyper"; - version = "1.8.1"; + version = "1.10.1"; edition = "2021"; - sha256 = "04cxr8j5y86bhxxlyqb8xkxjskpajk7cxwfzzk4v3my3a3rd9cia"; + sha256 = "1624nwrh1ci34psqcl3q8q266kha8kd6fmqjj14qck49l59iqa2m"; authors = [ "Sean McArthur " ]; @@ -3561,11 +3579,6 @@ rec { packageId = "pin-project-lite"; optional = true; } - { - name = "pin-utils"; - packageId = "pin-utils"; - optional = true; - } { name = "smallvec"; packageId = "smallvec"; @@ -3603,7 +3616,7 @@ rec { "client" = [ "dep:want" "dep:pin-project-lite" "dep:smallvec" ]; "ffi" = [ "dep:http-body-util" "dep:futures-util" ]; "full" = [ "client" "http1" "http2" "server" ]; - "http1" = [ "dep:atomic-waker" "dep:futures-channel" "dep:futures-core" "dep:httparse" "dep:itoa" "dep:pin-utils" ]; + "http1" = [ "dep:atomic-waker" "dep:futures-channel" "dep:futures-core" "dep:httparse" "dep:itoa" ]; "http2" = [ "dep:futures-channel" "dep:futures-core" "dep:h2" ]; "server" = [ "dep:httpdate" "dep:pin-project-lite" "dep:smallvec" ]; "tracing" = [ "dep:tracing" ]; @@ -3612,9 +3625,9 @@ rec { }; "hyper-rustls" = rec { crateName = "hyper-rustls"; - version = "0.27.7"; + version = "0.27.9"; edition = "2021"; - sha256 = "0n6g8998szbzhnvcs1b7ibn745grxiqmlpg53xz206v826v3xjg3"; + sha256 = "03vfnsm873wsp1dk0q85nxvk7w6syp8c2m5bcdjcyfgg4786ijik"; libName = "hyper_rustls"; dependencies = [ { @@ -3647,11 +3660,6 @@ rec { packageId = "rustls-native-certs"; optional = true; } - { - name = "rustls-pki-types"; - packageId = "rustls-pki-types"; - rename = "pki-types"; - } { name = "tokio"; packageId = "tokio"; @@ -3951,9 +3959,9 @@ rec { }; "icu_collections" = rec { crateName = "icu_collections"; - version = "2.1.1"; + version = "2.2.0"; edition = "2021"; - sha256 = "0hsblchsdl64q21qwrs4hvc2672jrf466zivbj1bwyv606bn8ssc"; + sha256 = "070r7xd0pynm0hnc1v2jzlbxka6wf50f81wybf9xg0y82v6x3119"; authors = [ "The ICU4X Project Developers" ]; @@ -3969,6 +3977,11 @@ rec { usesDefaultFeatures = false; features = [ "zerovec" ]; } + { + name = "utf8_iter"; + packageId = "utf8_iter"; + usesDefaultFeatures = false; + } { name = "yoke"; packageId = "yoke"; @@ -3996,9 +4009,9 @@ rec { }; "icu_locale_core" = rec { crateName = "icu_locale_core"; - version = "2.1.1"; + version = "2.2.0"; edition = "2021"; - sha256 = "1djvdc2f5ylmp1ymzv4gcnmq1s4hqfim9nxlcm173lsd01hpifpd"; + sha256 = "0a9cmin5w1x3bg941dlmgszn33qgq428k7qiqn5did72ndi9n8cj"; authors = [ "The ICU4X Project Developers" ]; @@ -4030,6 +4043,14 @@ rec { usesDefaultFeatures = false; } ]; + devDependencies = [ + { + name = "litemap"; + packageId = "litemap"; + usesDefaultFeatures = false; + features = [ "testing" ]; + } + ]; features = { "alloc" = [ "litemap/alloc" "tinystr/alloc" "writeable/alloc" "serde?/alloc" ]; "databake" = [ "dep:databake" "alloc" ]; @@ -4040,9 +4061,9 @@ rec { }; "icu_normalizer" = rec { crateName = "icu_normalizer"; - version = "2.1.1"; + version = "2.2.0"; edition = "2021"; - sha256 = "16dmn5596la2qm0r3vih0bzjfi0vx9a20yqjha6r1y3vnql8hv2z"; + sha256 = "1d7krxr0xpc4x9635k1100a24nh0nrc59n65j6yk6gbfkplmwvn5"; authors = [ "The ICU4X Project Developers" ]; @@ -4084,6 +4105,7 @@ rec { "compiled_data" = [ "dep:icu_normalizer_data" "icu_properties?/compiled_data" "icu_provider/baked" ]; "datagen" = [ "serde" "dep:databake" "icu_properties" "icu_collections/databake" "zerovec/databake" "icu_properties?/datagen" "icu_provider/export" ]; "default" = [ "compiled_data" "utf8_iter" "utf16_iter" ]; + "harfbuzz_traits" = [ "dep:harfbuzz-traits" ]; "icu_properties" = [ "dep:icu_properties" ]; "serde" = [ "dep:serde" "icu_collections/serde" "zerovec/serde" "icu_properties?/serde" "icu_provider/serde" ]; "utf16_iter" = [ "dep:utf16_iter" "dep:write16" ]; @@ -4093,9 +4115,9 @@ rec { }; "icu_normalizer_data" = rec { crateName = "icu_normalizer_data"; - version = "2.1.1"; + version = "2.2.0"; edition = "2021"; - sha256 = "02jnzizg6q75m41l6c13xc7nkc5q8yr1b728dcgfhpzw076wrvbs"; + sha256 = "0f5d5d5fhhr9937m2z6z38fzh6agf14z24kwlr6lyczafypf0fys"; authors = [ "The ICU4X Project Developers" ]; @@ -4103,9 +4125,9 @@ rec { }; "icu_properties" = rec { crateName = "icu_properties"; - version = "2.1.2"; + version = "2.2.0"; edition = "2021"; - sha256 = "1v3lbmhhi7i6jgw51ikjb1p50qh5rb67grlkdnkc63l7zq1gq2q2"; + sha256 = "1pkh3s837808cbwxvfagwc28cvwrz2d9h5rl02jwrhm51ryvdqxy"; authors = [ "The ICU4X Project Developers" ]; @@ -4150,6 +4172,7 @@ rec { "compiled_data" = [ "dep:icu_properties_data" "icu_provider/baked" ]; "datagen" = [ "serde" "dep:databake" "zerovec/databake" "icu_collections/databake" "icu_locale_core/databake" "zerotrie/databake" "icu_provider/export" ]; "default" = [ "compiled_data" ]; + "harfbuzz_traits" = [ "dep:harfbuzz-traits" ]; "serde" = [ "dep:serde" "icu_locale_core/serde" "zerovec/serde" "icu_collections/serde" "icu_provider/serde" "zerotrie/serde" ]; "unicode_bidi" = [ "dep:unicode-bidi" ]; }; @@ -4157,9 +4180,9 @@ rec { }; "icu_properties_data" = rec { crateName = "icu_properties_data"; - version = "2.1.2"; + version = "2.2.0"; edition = "2021"; - sha256 = "1bvpkh939rgzrjfdb7hz47v4wijngk0snmcgrnpwc9fpz162jv31"; + sha256 = "052awny0qwkbcbpd5jg2cd7vl5ry26pq4hz1nfsgf10c3qhbnawf"; authors = [ "The ICU4X Project Developers" ]; @@ -4167,9 +4190,9 @@ rec { }; "icu_provider" = rec { crateName = "icu_provider"; - version = "2.1.1"; + version = "2.2.0"; edition = "2021"; - sha256 = "0576b7dizgyhpfa74kacv86y4g1p7v5ffd6c56kf1q82rvq2r5l5"; + sha256 = "08dl8pxbwr8zsz4c5vphqb7xw0hykkznwi4rw7bk6pwb3krlr70k"; authors = [ "The ICU4X Project Developers" ]; @@ -4270,9 +4293,9 @@ rec { }; "idna_adapter" = rec { crateName = "idna_adapter"; - version = "1.2.1"; - edition = "2021"; - sha256 = "0i0339pxig6mv786nkqcxnwqa87v4m94b2653f6k3aj0jmhfkjis"; + version = "1.2.2"; + edition = "2024"; + sha256 = "0557p76l8hj35r9zn1yv7c6x1c0qbrsffmg80n0yy8361ly3fs6b"; authors = [ "The rust-url developers" ]; @@ -4295,9 +4318,9 @@ rec { }; "indexmap" = rec { crateName = "indexmap"; - version = "2.13.0"; - edition = "2021"; - sha256 = "05qh5c4h2hrnyypphxpwflk45syqbzvqsvvyxg43mp576w2ff53p"; + version = "2.14.0"; + edition = "2024"; + sha256 = "1na9z6f0d5pkjr1lgsni470v98gv2r7c41j8w48skr089x2yjrnl"; dependencies = [ { name = "equivalent"; @@ -4306,7 +4329,7 @@ rec { } { name = "hashbrown"; - packageId = "hashbrown"; + packageId = "hashbrown 0.17.1"; usesDefaultFeatures = false; } ]; @@ -4346,55 +4369,24 @@ rec { }; "ipnet" = rec { crateName = "ipnet"; - version = "2.11.0"; + version = "2.12.0"; edition = "2018"; - sha256 = "0c5i9sfi2asai28m8xp48k5gvwkqrg5ffpi767py6mzsrswv17s6"; + sha256 = "1qpq2y0asyv0jppw7zww9y96fpnpinwap8a0phhqqgyy3znnz3yr"; authors = [ "Kris Price " ]; features = { "default" = [ "std" ]; - "heapless" = [ "dep:heapless" ]; - "json" = [ "serde" "schemars" ]; - "schemars" = [ "dep:schemars" ]; - "ser_as_str" = [ "heapless" ]; + "heapless" = [ "dep:heapless" "serde" ]; + "json" = [ "schemars08" "serde" ]; + "schemars" = [ "schemars08" ]; + "schemars08" = [ "dep:schemars08" ]; + "schemars1" = [ "dep:schemars1" ]; + "ser_as_str" = [ "dep:heapless" ]; "serde" = [ "dep:serde" ]; }; resolvedDefaultFeatures = [ "default" "std" ]; }; - "iri-string" = rec { - crateName = "iri-string"; - version = "0.7.10"; - edition = "2021"; - sha256 = "06kk3a5jz576p7vrpf7zz9jv3lrgcyp7pczcblcxdnryg3q3h4y9"; - libName = "iri_string"; - authors = [ - "YOSHIOKA Takuma " - ]; - dependencies = [ - { - name = "memchr"; - packageId = "memchr"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "serde"; - packageId = "serde"; - optional = true; - usesDefaultFeatures = false; - features = [ "derive" ]; - } - ]; - features = { - "alloc" = [ "serde?/alloc" ]; - "default" = [ "std" ]; - "memchr" = [ "dep:memchr" ]; - "serde" = [ "dep:serde" ]; - "std" = [ "alloc" "memchr?/std" "serde?/std" ]; - }; - resolvedDefaultFeatures = [ "alloc" "default" "std" ]; - }; "is_terminal_polyfill" = rec { crateName = "is_terminal_polyfill"; version = "1.70.2"; @@ -4427,9 +4419,9 @@ rec { }; "itoa" = rec { crateName = "itoa"; - version = "1.0.17"; + version = "1.0.18"; edition = "2021"; - sha256 = "1lh93xydrdn1g9x547bd05g0d3hra7pd1k4jfd2z1pl1h5hwdv4j"; + sha256 = "10jnd1vpfkb8kj38rlkn2a6k02afvj3qmw054dfpzagrpl6achlg"; authors = [ "David Tolnay " ]; @@ -4464,9 +4456,9 @@ rec { }; "jiff" = rec { crateName = "jiff"; - version = "0.2.21"; + version = "0.2.28"; edition = "2021"; - sha256 = "0h3rypc82v4a8wf1s8jvqdryv5khp520ks0nmd3fjslc05gxdqxk"; + sha256 = "00lixngcc7amh2fcsxfr0z38j06lllhapz192biv1qj97q1x60s6"; authors = [ "Andrew Gallant " ]; @@ -4512,12 +4504,10 @@ rec { usesDefaultFeatures = false; } { - name = "windows-sys"; - packageId = "windows-sys 0.61.2"; + name = "windows-link"; + packageId = "windows-link"; optional = true; - usesDefaultFeatures = false; target = { target, features }: (target."windows" or false); - features = [ "Win32_Foundation" "Win32_System_Time" ]; } ]; devDependencies = [ @@ -4536,7 +4526,7 @@ rec { "static-tz" = [ "dep:jiff-static" ]; "std" = [ "alloc" "log?/std" "serde_core?/std" ]; "tz-fat" = [ "jiff-static?/tz-fat" ]; - "tz-system" = [ "std" "dep:windows-sys" ]; + "tz-system" = [ "std" "dep:windows-link" ]; "tzdb-bundle-always" = [ "dep:jiff-tzdb" "alloc" ]; "tzdb-bundle-platform" = [ "dep:jiff-tzdb-platform" "alloc" ]; "tzdb-concatenated" = [ "std" ]; @@ -4546,9 +4536,9 @@ rec { }; "jiff-static" = rec { crateName = "jiff-static"; - version = "0.2.21"; + version = "0.2.28"; edition = "2021"; - sha256 = "061qz1zyvp8bhvr58r9817hri6s338y8msnb0bg7yg463lhjnz51"; + sha256 = "0irbhfh2f4i9w5l53jcmh6ssnhdd92wfy76978chgwnxilvk4bbq"; procMacro = true; libName = "jiff_static"; authors = [ @@ -4575,9 +4565,9 @@ rec { }; "jiff-tzdb" = rec { crateName = "jiff-tzdb"; - version = "0.1.5"; + version = "0.1.6"; edition = "2021"; - sha256 = "1hm5xn3q092zac6apjy4492ddid473mwa0d64z5f5f95yyzix5v8"; + sha256 = "0xihzlnnyk0xnrzpq4xcyjdcmy8xc3ychzb9ayjkh4vgha2fy069"; libName = "jiff_tzdb"; libPath = "lib.rs"; authors = [ @@ -4628,14 +4618,25 @@ rec { }; "js-sys" = rec { crateName = "js-sys"; - version = "0.3.90"; + version = "0.3.99"; edition = "2021"; - sha256 = "19m5qg024y5xanjrq5c6m1sx69nnzqw7ychnbgnx9xmka1j6zp0l"; + sha256 = "04azrzsz91gr5s3z0ij36lz0kj9ry4lw3jz0mmbiwb251rsc8aql"; libName = "js_sys"; authors = [ "The wasm-bindgen Developers" ]; dependencies = [ + { + name = "cfg-if"; + packageId = "cfg-if"; + } + { + name = "futures-util"; + packageId = "futures-util"; + optional = true; + usesDefaultFeatures = false; + features = [ "std" ]; + } { name = "once_cell"; packageId = "once_cell"; @@ -4649,15 +4650,16 @@ rec { ]; features = { "default" = [ "std" "unsafe-eval" ]; - "std" = [ "wasm-bindgen/std" ]; + "futures-core-03-stream" = [ "dep:futures-util" "dep:futures-core" ]; + "std" = [ "wasm-bindgen/std" "dep:futures-util" ]; }; resolvedDefaultFeatures = [ "default" "std" "unsafe-eval" ]; }; "json-patch" = rec { crateName = "json-patch"; - version = "4.1.0"; + version = "4.2.0"; edition = "2021"; - sha256 = "147yaxmv3i4s0bdna86rgwpmqh2507fn4ighfpplaiqkw8ay807k"; + sha256 = "0wkv896d0pzq56i2kkl0giqpv117fwvhbpgs8iz85805w66l68bl"; libName = "json_patch"; authors = [ "Ivan Dubrov " @@ -4667,6 +4669,11 @@ rec { name = "jsonptr"; packageId = "jsonptr"; } + { + name = "schemars"; + packageId = "schemars"; + optional = true; + } { name = "serde"; packageId = "serde"; @@ -4678,10 +4685,14 @@ rec { } { name = "thiserror"; - packageId = "thiserror 1.0.69"; + packageId = "thiserror 2.0.18"; } ]; devDependencies = [ + { + name = "schemars"; + packageId = "schemars"; + } { name = "serde_json"; packageId = "serde_json"; @@ -4693,7 +4704,7 @@ rec { "schemars" = [ "dep:schemars" ]; "utoipa" = [ "dep:utoipa" ]; }; - resolvedDefaultFeatures = [ "default" "diff" ]; + resolvedDefaultFeatures = [ "default" "diff" "schemars" ]; }; "jsonpath-rust" = rec { crateName = "jsonpath-rust"; @@ -4766,10 +4777,10 @@ rec { }; "k8s-openapi" = rec { crateName = "k8s-openapi"; - version = "0.27.0"; + version = "0.27.1"; edition = "2021"; - links = "k8s-openapi-0.27.0"; - sha256 = "038zxrklpni04rpaww9dr7v8ln8zj8p7mgdd68bx5l8sc7rxd9h5"; + links = "k8s-openapi-0.27.1"; + sha256 = "0pldsxbxd4ckq94p8rkck4s862w33gfns6rclxr5imcx47sjdcsi"; libName = "k8s_openapi"; authors = [ "Arnav Singh " @@ -4819,8 +4830,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "k8s_version"; @@ -4839,7 +4850,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } ]; features = { @@ -4848,6 +4859,53 @@ rec { }; resolvedDefaultFeatures = [ "darling" ]; }; + "konst" = rec { + crateName = "konst"; + version = "0.2.20"; + edition = "2018"; + sha256 = "1yyf1fhk28wbf1lqrga9as4cpfmpbry9a5vvdqyxgz14g3nk708j"; + authors = [ + "rodrimati1992 " + ]; + dependencies = [ + { + name = "konst_macro_rules"; + packageId = "konst_macro_rules"; + } + ]; + features = { + "__ui" = [ "__test" "trybuild" "rust_latest_stable" ]; + "const_generics" = [ "rust_1_51" ]; + "constant_time_slice" = [ "rust_latest_stable" ]; + "default" = [ "cmp" "parsing" ]; + "deref_raw_in_fn" = [ "rust_1_56" ]; + "konst_proc_macros" = [ "dep:konst_proc_macros" ]; + "mut_refs" = [ "rust_latest_stable" "konst_macro_rules/mut_refs" ]; + "nightly_mut_refs" = [ "mut_refs" "konst_macro_rules/nightly_mut_refs" ]; + "parsing" = [ "parsing_no_proc" "konst_proc_macros" ]; + "rust_1_51" = [ "konst_macro_rules/rust_1_51" ]; + "rust_1_55" = [ "rust_1_51" "konst_macro_rules/rust_1_55" ]; + "rust_1_56" = [ "rust_1_55" "konst_macro_rules/rust_1_56" ]; + "rust_1_57" = [ "rust_1_56" "konst_macro_rules/rust_1_57" ]; + "rust_1_61" = [ "rust_1_57" "konst_macro_rules/rust_1_61" ]; + "rust_1_64" = [ "rust_1_61" ]; + "rust_latest_stable" = [ "rust_1_64" ]; + "trybuild" = [ "dep:trybuild" ]; + }; + resolvedDefaultFeatures = [ "rust_1_51" "rust_1_55" "rust_1_56" "rust_1_57" "rust_1_61" "rust_1_64" ]; + }; + "konst_macro_rules" = rec { + crateName = "konst_macro_rules"; + version = "0.2.19"; + edition = "2018"; + sha256 = "0dswja0dqcww4x3fwjnirc0azv2n6cazn8yv0kddksd8awzkz4x4"; + authors = [ + "rodrimati1992 " + ]; + features = { + }; + resolvedDefaultFeatures = [ "rust_1_51" "rust_1_55" "rust_1_56" "rust_1_57" "rust_1_61" ]; + }; "kube" = rec { crateName = "kube"; version = "3.1.0"; @@ -5322,7 +5380,7 @@ rec { } { name = "hashbrown"; - packageId = "hashbrown"; + packageId = "hashbrown 0.16.1"; } { name = "hostname"; @@ -5424,9 +5482,9 @@ rec { }; "libc" = rec { crateName = "libc"; - version = "0.2.182"; + version = "0.2.186"; edition = "2021"; - sha256 = "04k1w1mq9f4cxv520dbr5xw1i7xkbc9fcrvaggyjy25jdkdvl038"; + sha256 = "0rnyhzjyqq9x56skkllbjzzzwym3r61lq3l4hqj64v71gw0r3av8"; authors = [ "The Rust Project Developers" ]; @@ -5440,10 +5498,10 @@ rec { }; "libgit2-sys" = rec { crateName = "libgit2-sys"; - version = "0.18.3+1.9.2"; + version = "0.18.5+1.9.4"; edition = "2021"; links = "git2"; - sha256 = "11rlbyihj3k35mnkxxz4yvsnlx33a4r9srl66c5vp08pp72arcy9"; + sha256 = "18lwqnhy7qxg4iw24s1a0n7aj7qbnryry1iy0w32k4f1xbk6lp80"; libName = "libgit2_sys"; libPath = "lib.rs"; authors = [ @@ -5501,10 +5559,10 @@ rec { }; "libz-sys" = rec { crateName = "libz-sys"; - version = "1.1.24"; + version = "1.1.29"; edition = "2018"; links = "z"; - sha256 = "0f8879301wxgljw8snkcix90p6qbm4inp3sqrsjq9b2svv5yjda7"; + sha256 = "1n98kqya7a7a0cxf5n5z3g13rj7a1vqxynk2xc7bja1qfxbrdg45"; libName = "libz_sys"; authors = [ "Alex Crichton " @@ -5543,9 +5601,9 @@ rec { }; "litemap" = rec { crateName = "litemap"; - version = "0.8.1"; + version = "0.8.2"; edition = "2021"; - sha256 = "0xsy8pfp9s802rsj1bq2ys2kbk1g36w5dr3gkfip7gphb5x60wv3"; + sha256 = "1w7628bc7wwcxc4n4s5kw0610xk06710nh2hn5kwwk2wa91z9nlj"; authors = [ "The ICU4X Project Developers" ]; @@ -5581,9 +5639,9 @@ rec { }; "log" = rec { crateName = "log"; - version = "0.4.29"; + version = "0.4.31"; edition = "2021"; - sha256 = "15q8j9c8g5zpkcw0hnd6cf2z7fxqnvsjh3rw5mv5q10r83i34l2y"; + sha256 = "0kq2fh6q2bjkrm8m6hj8kb7gxfd7cr7qbcpxd1lc1xq5rns30fqi"; authors = [ "The Rust Project Developers" ]; @@ -5637,9 +5695,9 @@ rec { }; "memchr" = rec { crateName = "memchr"; - version = "2.8.0"; + version = "2.8.1"; edition = "2021"; - sha256 = "0y9zzxcqxvdqg6wyag7vc3h0blhdn7hkq164bxyx2vph8zs5ijpq"; + sha256 = "1n448jx01h5z2xknj6x2dhxgr8s8fb717cf6vfqj5lmhkpj7m53b"; authors = [ "Andrew Gallant " "bluss" @@ -5700,9 +5758,9 @@ rec { }; "mio" = rec { crateName = "mio"; - version = "1.1.1"; + version = "1.2.1"; edition = "2021"; - sha256 = "1z2phpalqbdgihrcjp8y09l3kgq6309jnhnr6h11l9s7mnqcm6x6"; + sha256 = "1nkggmrlnjs93w8rja4lvjj4aml1xqahgimv1h0p7d373kvhmg82"; authors = [ "Carl Lerche " "Thomas de Zeeuw " @@ -5712,17 +5770,7 @@ rec { { name = "libc"; packageId = "libc"; - target = { target, features }: ("hermit" == target."os" or null); - } - { - name = "libc"; - packageId = "libc"; - target = { target, features }: ("wasi" == target."os" or null); - } - { - name = "libc"; - packageId = "libc"; - target = { target, features }: (target."unix" or false); + target = { target, features }: ((target."unix" or false) || ("hermit" == target."os" or null) || ("wasi" == target."os" or null)); } { name = "wasi"; @@ -5811,7 +5859,7 @@ rec { } { name = "rand"; - packageId = "rand 0.8.5"; + packageId = "rand 0.8.6"; optional = true; usesDefaultFeatures = false; } @@ -5830,7 +5878,7 @@ rec { devDependencies = [ { name = "rand"; - packageId = "rand 0.8.5"; + packageId = "rand 0.8.6"; features = [ "small_rng" ]; } ]; @@ -5848,9 +5896,9 @@ rec { }; "num-conv" = rec { crateName = "num-conv"; - version = "0.2.0"; + version = "0.2.2"; edition = "2021"; - sha256 = "0l4hj7lp8zbb9am4j3p7vlcv47y9bbazinvnxx9zjhiwkibyr5yg"; + sha256 = "0hg4f9bwmy7cwpxdkm165dmkfc8jhkkayci234jsmi5ssb33j5sj"; libName = "num_conv"; authors = [ "Jacob Pratt " @@ -5944,9 +5992,9 @@ rec { }; "once_cell" = rec { crateName = "once_cell"; - version = "1.21.3"; + version = "1.21.4"; edition = "2021"; - sha256 = "0b9x77lb9f1j6nqgf5aka4s2qj0nly176bpbrv6f9iakk5ff3xa2"; + sha256 = "0l1v676wf71kjg2khch4dphwh1jp3291ffiymr2mvy1kxd5kwz4z"; authors = [ "Aleksey Kladov " ]; @@ -6136,9 +6184,9 @@ rec { }; "opentelemetry-otlp" = rec { crateName = "opentelemetry-otlp"; - version = "0.31.0"; + version = "0.31.1"; edition = "2021"; - sha256 = "1gv3h75z8c0p9b85mbq7f1rgsi18wip1xlfa6g82lkfa5pdnc8vs"; + sha256 = "07zp0b62b9dajnvvcd6j2ppw5zg7wp4ixka9z6fr3bxrrdmcss8z"; libName = "opentelemetry_otlp"; dependencies = [ { @@ -6250,6 +6298,9 @@ rec { "serde_json" = [ "dep:serde_json" ]; "serialize" = [ "serde" "serde_json" ]; "tls" = [ "tonic/tls-ring" ]; + "tls-aws-lc" = [ "tonic/tls-aws-lc" ]; + "tls-provider-agnostic" = [ "tonic/_tls-any" ]; + "tls-ring" = [ "tonic/tls-ring" ]; "tls-roots" = [ "tls" "tonic/tls-native-roots" ]; "tls-webpki-roots" = [ "tls" "tonic/tls-webpki-roots" ]; "tokio" = [ "dep:tokio" ]; @@ -6371,7 +6422,7 @@ rec { } { name = "rand"; - packageId = "rand 0.9.2"; + packageId = "rand 0.9.4"; optional = true; usesDefaultFeatures = false; features = [ "std" "std_rng" "small_rng" "os_rng" "thread_rng" ]; @@ -6795,9 +6846,9 @@ rec { }; "pin-project" = rec { crateName = "pin-project"; - version = "1.1.10"; + version = "1.1.13"; edition = "2021"; - sha256 = "12kadbnfm1f43cyadw9gsbyln1cy7vj764wz5c8wxaiza3filzv7"; + sha256 = "09091qp946lpmjz4yp0xil1r5v4hgc91fi19dg5csayhdqrv4ri4"; libName = "pin_project"; dependencies = [ { @@ -6809,9 +6860,9 @@ rec { }; "pin-project-internal" = rec { crateName = "pin-project-internal"; - version = "1.1.10"; + version = "1.1.13"; edition = "2021"; - sha256 = "0qgqzfl0f4lzaz7yl5llhbg97g68r15kljzihaw9wm64z17qx4bf"; + sha256 = "12rzlh07i1sdgrvzj6wgkka5bjqyvbfsl8knq6qi7g16m7q9aqy9"; procMacro = true; libName = "pin_project_internal"; dependencies = [ @@ -6834,22 +6885,11 @@ rec { }; "pin-project-lite" = rec { crateName = "pin-project-lite"; - version = "0.2.16"; + version = "0.2.17"; edition = "2018"; - sha256 = "16wzc7z7dfkf9bmjin22f5282783f6mdksnr0nv0j5ym5f9gyg1v"; + sha256 = "1kfmwvs271si96zay4mm8887v5khw0c27jc9srw1a75ykvgj54x8"; libName = "pin_project_lite"; - }; - "pin-utils" = rec { - crateName = "pin-utils"; - version = "0.1.0"; - edition = "2018"; - sha256 = "117ir7vslsl2z1a7qzhws4pd01cg2d3338c47swjyvqv2n60v1wb"; - libName = "pin_utils"; - authors = [ - "Josef Brandl " - ]; - }; "pkcs1" = rec { crateName = "pkcs1"; @@ -6921,9 +6961,9 @@ rec { }; "pkg-config" = rec { crateName = "pkg-config"; - version = "0.3.32"; + version = "0.3.33"; edition = "2018"; - sha256 = "0k4h3gnzs94sjb2ix6jyksacs52cf1fanpwsmlhjnwrdnp8dppby"; + sha256 = "17jnqmcbxsnwhg9gjf0nh6dj5k0x3hgwi3mb9krjnmfa9v435w8r"; libName = "pkg_config"; authors = [ "Alex Crichton " @@ -6945,9 +6985,9 @@ rec { }; "portable-atomic-util" = rec { crateName = "portable-atomic-util"; - version = "0.2.5"; + version = "0.2.7"; edition = "2018"; - sha256 = "1xcm0ia8756k6hdgafx4g3lx3fw0hvz2zqswq7c2sy58gxnvk7bs"; + sha256 = "0616j0fhy6y71hyxg3n86f6hng0fmsc269s3wp4gl8ww4p8hd8f2"; libName = "portable_atomic_util"; dependencies = [ { @@ -6958,15 +6998,16 @@ rec { } ]; features = { + "serde" = [ "dep:serde" ]; "std" = [ "alloc" ]; }; resolvedDefaultFeatures = [ "alloc" ]; }; "potential_utf" = rec { crateName = "potential_utf"; - version = "0.1.4"; + version = "0.1.5"; edition = "2021"; - sha256 = "0xxg0pkfpq299wvwln409z4fk80rbv55phh3f1jhjajy5x1ljfdp"; + sha256 = "0r0518fr32xbkgzqap509s3r60cr0iancsg9j1jgf37cyz7b20q1"; authors = [ "The ICU4X Project Developers" ]; @@ -7048,9 +7089,9 @@ rec { }; "proc-macro-crate" = rec { crateName = "proc-macro-crate"; - version = "3.4.0"; + version = "3.5.0"; edition = "2021"; - sha256 = "10v9qi51n4phn1lrj5r94kjq7yhci9jrkqnn6wpan05yjsgb3711"; + sha256 = "0kv1g1d1zjwxlgcaba2qlshzyy32j03xic8rskqlcr5mnblsfyz6"; libName = "proc_macro_crate"; authors = [ "Bastian Köcher " @@ -7210,9 +7251,9 @@ rec { }; "quote" = rec { crateName = "quote"; - version = "1.0.44"; + version = "1.0.45"; edition = "2021"; - sha256 = "1r7c7hxl66vz3q9qizgjhy77pdrrypqgk4ghc7260xvvfb7ypci1"; + sha256 = "095rb5rg7pbnwdp6v8w5jw93wndwyijgci1b5lw8j1h5cscn3wj1"; authors = [ "David Tolnay " ]; @@ -7241,11 +7282,11 @@ rec { "rustc-dep-of-std" = [ "core" ]; }; }; - "rand 0.8.5" = rec { + "rand 0.8.6" = rec { crateName = "rand"; - version = "0.8.5"; + version = "0.8.6"; edition = "2018"; - sha256 = "013l6931nn7gkc23jz5mm3qdhf93jjf0fg64nz2lp4i51qd8vbrl"; + sha256 = "12kd4rljn86m00rcaz4c1rcya4mb4gk5ig6i8xq00a8wjgxfr82w"; authors = [ "The Rand Project Developers" "The Rust Project Developers" @@ -7267,22 +7308,19 @@ rec { "default" = [ "std" "std_rng" ]; "getrandom" = [ "rand_core/getrandom" ]; "libc" = [ "dep:libc" ]; - "log" = [ "dep:log" ]; - "packed_simd" = [ "dep:packed_simd" ]; "rand_chacha" = [ "dep:rand_chacha" ]; "serde" = [ "dep:serde" ]; "serde1" = [ "serde" "rand_core/serde1" ]; - "simd_support" = [ "packed_simd" ]; "std" = [ "rand_core/std" "rand_chacha/std" "alloc" "getrandom" "libc" ]; "std_rng" = [ "rand_chacha" ]; }; resolvedDefaultFeatures = [ "rand_chacha" "std_rng" ]; }; - "rand 0.9.2" = rec { + "rand 0.9.4" = rec { crateName = "rand"; - version = "0.9.2"; + version = "0.9.4"; edition = "2021"; - sha256 = "1lah73ainvrgl7brcxx0pwhpnqa3sm3qaj672034jz8i0q7pgckd"; + sha256 = "1sknbxgs6nfg0nxdd7689lwbyr2i4vaswchrv4b34z8vpc3azia4"; authors = [ "The Rand Project Developers" "The Rust Project Developers" @@ -7302,7 +7340,6 @@ rec { ]; features = { "default" = [ "std" "std_rng" "os_rng" "small_rng" "thread_rng" ]; - "log" = [ "dep:log" ]; "os_rng" = [ "rand_core/os_rng" ]; "serde" = [ "dep:serde" "rand_core/serde" ]; "std" = [ "rand_core/std" "rand_chacha?/std" "alloc" ]; @@ -8177,9 +8214,9 @@ rec { }; "rustls" = rec { crateName = "rustls"; - version = "0.23.37"; + version = "0.23.40"; edition = "2021"; - sha256 = "193k5h0wcih6ghvkrxyzwncivr1bd3a8yw3lzp13pzfcbz5jb03m"; + sha256 = "12qnv3ag4wrw7aj8jng74kgrilpjm2b1rfcjaac8h691frccv1pg"; dependencies = [ { name = "log"; @@ -8246,9 +8283,9 @@ rec { }; "rustls-native-certs" = rec { crateName = "rustls-native-certs"; - version = "0.8.3"; + version = "0.8.4"; edition = "2021"; - sha256 = "0qrajg2n90bcr3bcq6j95gjm7a9lirfkkdmjj32419dyyzan0931"; + sha256 = "0kgazl8zc1sv63qg179bz96ilzh56lzfa5k92ji7d265f4kibdfs"; libName = "rustls_native_certs"; dependencies = [ { @@ -8277,9 +8314,9 @@ rec { }; "rustls-pki-types" = rec { crateName = "rustls-pki-types"; - version = "1.14.0"; + version = "1.14.1"; edition = "2021"; - sha256 = "1p9zsgslvwzzkzhm6bqicffqndr4jpx67992b0vl0pi21a5hy15y"; + sha256 = "1a9pr54y0f3qr97bxpd3ahjldq0gqdld0h799xbnwdzbwxx1k9rh"; libName = "rustls_pki_types"; dependencies = [ { @@ -8358,9 +8395,9 @@ rec { }; "schannel" = rec { crateName = "schannel"; - version = "0.1.28"; + version = "0.1.29"; edition = "2018"; - sha256 = "1qb6s5gyxfz2inz753a4z3mc1d266mwvz0c5w7ppd3h44swq27c9"; + sha256 = "0ffrzz5vf2s3gnzvphgb5gg8fqifvryl07qcf7q3x1scj3jbghci"; authors = [ "Steven Fackler " "Steffen Butzer " @@ -8656,9 +8693,9 @@ rec { }; "semver" = rec { crateName = "semver"; - version = "1.0.27"; - edition = "2018"; - sha256 = "1qmi3akfrnqc2hfkdgcxhld5bv961wbk8my3ascv5068mc5fnryp"; + version = "1.0.28"; + edition = "2021"; + sha256 = "1kaimrpy876bcgi8bfj0qqfxk77zm9iz2zhn1hp9hj685z854y4a"; authors = [ "David Tolnay " ]; @@ -8815,9 +8852,9 @@ rec { }; "serde_json" = rec { crateName = "serde_json"; - version = "1.0.149"; + version = "1.0.150"; edition = "2021"; - sha256 = "11jdx4vilzrjjd1dpgy67x5lgzr0laplz30dhv75lnf5ffa07z43"; + sha256 = "1ffgfhy9kndjnrz8lmy95pr758p2zk8dxv6yi99x0vkkni24w0g8"; authors = [ "Erick Tryzelaar " "David Tolnay " @@ -9069,9 +9106,9 @@ rec { }; "shlex" = rec { crateName = "shlex"; - version = "1.3.0"; - edition = "2015"; - sha256 = "0r1y6bv26c1scpxvhg2cabimrmwgbp4p3wy6syj9n0c4s3q2znhg"; + version = "2.0.1"; + edition = "2018"; + sha256 = "1fjsll1cd7d2bcpdij9kd6w62rpbc7qqzvydvs021vsmr1cxvypq"; authors = [ "comex " "Fenhl " @@ -9139,9 +9176,9 @@ rec { }; "simd-adler32" = rec { crateName = "simd-adler32"; - version = "0.3.8"; + version = "0.3.9"; edition = "2018"; - sha256 = "18lx2gdgislabbvlgw5q3j5ssrr77v8kmkrxaanp3liimp2sc873"; + sha256 = "0532ysdwcvzyp2bwpk8qz0hijplcdwpssr5gy5r7qwqqy5z5qgbh"; libName = "simd_adler32"; authors = [ "Marvin Countryman " @@ -9250,29 +9287,25 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "rust_1_61" "rust_1_65" "std" ]; }; - "snafu 0.9.0" = rec { + "snafu 0.9.1" = rec { crateName = "snafu"; - version = "0.9.0"; + version = "0.9.1"; edition = "2018"; - sha256 = "1ii9r99x5qcn754m624yzgb9hzvkqkrcygf0aqh0pyb9dbnvrm6i"; + sha256 = "08k5yfydxdlshivfhrdq9km8qn02r93q28gkyvazbqz2icr1586i"; authors = [ "Jake Goulding " ]; dependencies = [ { name = "snafu-derive"; - packageId = "snafu-derive 0.9.0"; + packageId = "snafu-derive 0.9.1"; } ]; features = { - "backtrace" = [ "dep:backtrace" ]; - "backtraces-impl-backtrace-crate" = [ "backtrace" ]; + "backtraces-impl-backtrace-crate" = [ "dep:backtrace" ]; "default" = [ "std" "rust_1_81" ]; - "futures" = [ "futures-core-crate" "pin-project" ]; - "futures-core-crate" = [ "dep:futures-core-crate" ]; - "futures-crate" = [ "dep:futures-crate" ]; - "internal-dev-dependencies" = [ "futures-crate" ]; - "pin-project" = [ "dep:pin-project" ]; + "futures" = [ "dep:futures-core" "dep:pin-project" ]; + "internal-dev-dependencies" = [ "dep:futures" ]; "std" = [ "alloc" ]; "unstable-provider-api" = [ "snafu-derive/unstable-provider-api" ]; }; @@ -9340,11 +9373,11 @@ rec { }; resolvedDefaultFeatures = [ "rust_1_61" ]; }; - "snafu-derive 0.9.0" = rec { + "snafu-derive 0.9.1" = rec { crateName = "snafu-derive"; - version = "0.9.0"; + version = "0.9.1"; edition = "2018"; - sha256 = "0h0x61kyj4fvilcr2nj02l85shw1ika64vq9brf2gyna662ln9al"; + sha256 = "1nkfi7bis72pz3w7vb64m79w49qsv20sbf19jkd471vbhr83q42z"; procMacro = true; libName = "snafu_derive"; authors = [ @@ -9370,7 +9403,7 @@ rec { name = "syn"; packageId = "syn 2.0.117"; usesDefaultFeatures = false; - features = [ "clone-impls" "derive" "full" "parsing" "printing" "proc-macro" ]; + features = [ "clone-impls" "derive" "full" "parsing" "printing" "proc-macro" "visit-mut" ]; } ]; features = { @@ -9378,9 +9411,9 @@ rec { }; "socket2" = rec { crateName = "socket2"; - version = "0.6.2"; + version = "0.6.4"; edition = "2021"; - sha256 = "1q073zkvz96h216mfz6niqk2kjqrgqv2va6zj34qh84zv4xamx46"; + sha256 = "0ldyp5rhba15spwxj1n94xh7sjks1398c3vwpwkxkd1087nwzlaj"; authors = [ "Alex Crichton " "Thomas de Zeeuw " @@ -9389,11 +9422,11 @@ rec { { name = "libc"; packageId = "libc"; - target = { target, features }: (target."unix" or false); + target = { target, features }: ((target."unix" or false) || ("wasi" == target."os" or null)); } { name = "windows-sys"; - packageId = "windows-sys 0.60.2"; + packageId = "windows-sys 0.61.2"; target = { target, features }: (target."windows" or false); features = [ "Win32_Foundation" "Win32_Networking_WinSock" "Win32_System_IO" "Win32_System_Threading" "Win32_System_WindowsProgramming" ]; } @@ -9478,8 +9511,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_certs"; @@ -9516,7 +9549,7 @@ rec { } { name = "rand"; - packageId = "rand 0.9.2"; + packageId = "rand 0.9.4"; } { name = "rand_core"; @@ -9538,7 +9571,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-shared"; @@ -9617,8 +9650,8 @@ rec { packageId = "indoc"; } { - name = "product-config"; - packageId = "product-config"; + name = "java-properties"; + packageId = "java-properties"; } { name = "serde"; @@ -9635,7 +9668,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-operator"; @@ -9656,6 +9689,10 @@ rec { name = "tracing"; packageId = "tracing"; } + { + name = "xml"; + packageId = "xml"; + } ]; buildDependencies = [ { @@ -9682,8 +9719,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_operator"; @@ -9741,6 +9778,7 @@ rec { { name = "json-patch"; packageId = "json-patch"; + features = [ "schemars" ]; } { name = "k8s-openapi"; @@ -9760,7 +9798,7 @@ rec { } { name = "rand"; - packageId = "rand 0.9.2"; + packageId = "rand 0.9.4"; } { name = "regex"; @@ -9790,7 +9828,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-operator-derive"; @@ -9844,6 +9882,10 @@ rec { packageId = "url"; features = [ "serde" ]; } + { + name = "uuid"; + packageId = "uuid"; + } ]; features = { "certs" = [ "dep:stackable-certs" ]; @@ -9862,8 +9904,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; procMacro = true; @@ -9897,8 +9939,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_shared"; @@ -9943,7 +9985,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "strum"; @@ -9978,8 +10020,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_telemetry"; @@ -10031,7 +10073,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "strum"; @@ -10088,8 +10130,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_versioned"; @@ -10123,7 +10165,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-versioned-macros"; @@ -10138,8 +10180,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; procMacro = true; @@ -10206,8 +10248,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "7a5f0c3fbcd091340214a23f0607fcd4b4fcc152"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; }; libName = "stackable_webhook"; @@ -10267,7 +10309,7 @@ rec { } { name = "rand"; - packageId = "rand 0.9.2"; + packageId = "rand 0.9.4"; } { name = "serde"; @@ -10280,7 +10322,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-certs"; @@ -10410,6 +10452,16 @@ rec { }; resolvedDefaultFeatures = [ "i128" ]; }; + "symlink" = rec { + crateName = "symlink"; + version = "0.1.0"; + edition = "2015"; + sha256 = "02h1i0b81mxb4vns4xrvrfibpcvs7jqqav8p3yilwik8cv73r5x7"; + authors = [ + "Chris Morgan " + ]; + + }; "syn 1.0.109" = rec { crateName = "syn"; version = "1.0.109"; @@ -10750,9 +10802,9 @@ rec { }; "tinystr" = rec { crateName = "tinystr"; - version = "0.8.2"; + version = "0.8.3"; edition = "2021"; - sha256 = "0sa8z88axdsf088hgw5p4xcyi6g3w3sgbb6qdp81bph9bk2fkls2"; + sha256 = "0vfr8x285w6zsqhna0a9jyhylwiafb2kc8pj2qaqaahw48236cn8"; authors = [ "The ICU4X Project Developers" ]; @@ -10842,9 +10894,9 @@ rec { }; "tokio" = rec { crateName = "tokio"; - version = "1.49.0"; + version = "1.52.3"; edition = "2021"; - sha256 = "11ix3pl03s0bp71q3wddrbf8xr0cpn47d7fzr6m42r3kswy918kj"; + sha256 = "1zpzazypkg61sw91na1m85x5s4rsjym335fwwhwm1hcs70dz1iwg"; authors = [ "Tokio Contributors " ]; @@ -10860,6 +10912,12 @@ rec { optional = true; target = { target, features }: ((target."tokio_unstable" or false) && ("linux" == target."os" or null)); } + { + name = "libc"; + packageId = "libc"; + optional = true; + target = { target, features }: ("wasi" == target."os" or null); + } { name = "libc"; packageId = "libc"; @@ -10899,7 +10957,7 @@ rec { name = "socket2"; packageId = "socket2"; optional = true; - target = { target, features }: (!(builtins.elem "wasm" target."family")); + target = { target, features }: ((!(builtins.elem "wasm" target."family")) || (("wasi" == target."os" or null) && (!("p1" == target."env" or null)))); features = [ "all" ]; } { @@ -10957,9 +11015,9 @@ rec { }; "tokio-macros" = rec { crateName = "tokio-macros"; - version = "2.6.0"; + version = "2.7.0"; edition = "2021"; - sha256 = "19czvgliginbzyhhfbmj77wazqn2y8g27y2nirfajdlm41bphh5g"; + sha256 = "15m4f37mdafs0gg36sh0rskm1i768lb7zmp8bw67kaxr3avnqniq"; procMacro = true; libName = "tokio_macros"; authors = [ @@ -11127,9 +11185,9 @@ rec { }; "toml_datetime" = rec { crateName = "toml_datetime"; - version = "0.7.5+spec-1.1.0"; - edition = "2021"; - sha256 = "0iqkgvgsxmszpai53dbip7sf2igic39s4dby29dbqf1h9bnwzqcj"; + version = "1.1.1+spec-1.1.0"; + edition = "2024"; + sha256 = "1mws2mkkf46l7inn77azhm0vdwxngv9vsbhbl0ah33p2c9gzcr9i"; dependencies = [ { name = "serde_core"; @@ -11148,9 +11206,9 @@ rec { }; "toml_edit" = rec { crateName = "toml_edit"; - version = "0.23.10+spec-1.0.0"; - edition = "2021"; - sha256 = "0saj5c676j8a3sqaj9akkp09wambg8aflji4zblwwa70azvvkj44"; + version = "0.25.12+spec-1.1.0"; + edition = "2024"; + sha256 = "1mx5paq837rjw7w51zprrjynk1vaig9yzxfqz9ac79jmd7f3w5fj"; dependencies = [ { name = "indexmap"; @@ -11183,9 +11241,9 @@ rec { }; "toml_parser" = rec { crateName = "toml_parser"; - version = "1.0.9+spec-1.1.0"; - edition = "2021"; - sha256 = "1i54qpvvcppy8ybdn9gssas81vfzq0kmgkcnxzhyf8w9w0al8bbh"; + version = "1.1.2+spec-1.1.0"; + edition = "2024"; + sha256 = "09kmzc55a0j21whm290wlf5a8b18a0qc87a1s8sncrckc6wfkax2"; dependencies = [ { name = "winnow"; @@ -11203,9 +11261,9 @@ rec { }; "tonic" = rec { crateName = "tonic"; - version = "0.14.5"; - edition = "2021"; - sha256 = "1v4k7aa28m7722gz9qak2jiy7lis1ycm4fdmq63iip4m0qdcdizy"; + version = "0.14.6"; + edition = "2024"; + sha256 = "1vs5ci6z6b9xhfsnx4s8qx6bqi1zzcrxncjp71147a0gqwc5aamc"; authors = [ "Lucio Franco " ]; @@ -11332,9 +11390,9 @@ rec { }; "tonic-prost" = rec { crateName = "tonic-prost"; - version = "0.14.5"; - edition = "2021"; - sha256 = "02fkg2bv87q0yds2wz3w0s7i1x6qcgbrl00dy6ipajdapfh7clx5"; + version = "0.14.6"; + edition = "2024"; + sha256 = "184y40nf0iyzc5rg32ivgd88snv68sqy1kchynn55r1vhml9z12h"; libName = "tonic_prost"; authors = [ "Lucio Franco " @@ -11476,9 +11534,9 @@ rec { }; "tower-http" = rec { crateName = "tower-http"; - version = "0.6.8"; + version = "0.6.11"; edition = "2018"; - sha256 = "1y514jwzbyrmrkbaajpwmss4rg0mak82k16d6588w9ncaffmbrnl"; + sha256 = "0h08wjgs3hwnq11iwwzlmnabn1h4cl0fzd48svaccvqffkiggz2c"; libName = "tower_http"; authors = [ "Tower Maintainers " @@ -11512,11 +11570,6 @@ rec { packageId = "http-body"; optional = true; } - { - name = "iri-string"; - packageId = "iri-string"; - optional = true; - } { name = "mime"; packageId = "mime"; @@ -11546,6 +11599,11 @@ rec { optional = true; usesDefaultFeatures = false; } + { + name = "url"; + packageId = "url"; + optional = true; + } ]; devDependencies = [ { @@ -11567,35 +11625,33 @@ rec { } ]; features = { - "async-compression" = [ "dep:async-compression" ]; "auth" = [ "base64" "validate-request" ]; "base64" = [ "dep:base64" ]; "catch-panic" = [ "tracing" "futures-util/std" "dep:http-body" "dep:http-body-util" ]; - "compression-br" = [ "async-compression/brotli" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; - "compression-deflate" = [ "async-compression/zlib" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; + "compression-br" = [ "dep:async-compression" "async-compression?/brotli" "futures-core" "dep:http-body" "tokio-util" "dep:tokio" ]; + "compression-deflate" = [ "dep:async-compression" "async-compression?/zlib" "futures-core" "dep:http-body" "tokio-util" "dep:tokio" ]; "compression-full" = [ "compression-br" "compression-deflate" "compression-gzip" "compression-zstd" ]; - "compression-gzip" = [ "async-compression/gzip" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; - "compression-zstd" = [ "async-compression/zstd" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; - "decompression-br" = [ "async-compression/brotli" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; - "decompression-deflate" = [ "async-compression/zlib" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; + "compression-gzip" = [ "dep:async-compression" "async-compression?/gzip" "futures-core" "dep:http-body" "tokio-util" "dep:tokio" ]; + "compression-zstd" = [ "dep:async-compression" "async-compression?/zstd" "futures-core" "dep:http-body" "tokio-util" "dep:tokio" ]; + "decompression-br" = [ "dep:async-compression" "async-compression?/brotli" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "dep:tokio" ]; + "decompression-deflate" = [ "dep:async-compression" "async-compression?/zlib" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "dep:tokio" ]; "decompression-full" = [ "decompression-br" "decompression-deflate" "decompression-gzip" "decompression-zstd" ]; - "decompression-gzip" = [ "async-compression/gzip" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; - "decompression-zstd" = [ "async-compression/zstd" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; - "follow-redirect" = [ "futures-util" "dep:http-body" "iri-string" "tower/util" ]; - "fs" = [ "futures-core" "futures-util" "dep:http-body" "dep:http-body-util" "tokio/fs" "tokio-util/io" "tokio/io-util" "dep:http-range-header" "mime_guess" "mime" "percent-encoding" "httpdate" "set-status" "futures-util/alloc" "tracing" ]; - "full" = [ "add-extension" "auth" "catch-panic" "compression-full" "cors" "decompression-full" "follow-redirect" "fs" "limit" "map-request-body" "map-response-body" "metrics" "normalize-path" "propagate-header" "redirect" "request-id" "sensitive-headers" "set-header" "set-status" "timeout" "trace" "util" "validate-request" ]; + "decompression-gzip" = [ "dep:async-compression" "async-compression?/gzip" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "dep:tokio" ]; + "decompression-zstd" = [ "dep:async-compression" "async-compression?/zstd" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "dep:tokio" ]; + "follow-redirect" = [ "futures-util" "dep:http-body" "dep:url" "tower/util" ]; + "fs" = [ "dep:tokio" "tokio?/fs" "tokio?/io-util" "futures-core" "futures-util" "dep:http-body" "dep:http-body-util" "tokio-util/io" "dep:http-range-header" "mime_guess" "mime" "percent-encoding" "httpdate" "set-status" "futures-util/alloc" ]; + "full" = [ "add-extension" "auth" "catch-panic" "compression-full" "cors" "decompression-full" "follow-redirect" "fs" "limit" "map-request-body" "map-response-body" "metrics" "normalize-path" "on-early-drop" "propagate-header" "redirect" "request-id" "sensitive-headers" "set-header" "set-status" "timeout" "trace" "util" "validate-request" ]; "futures-core" = [ "dep:futures-core" ]; "futures-util" = [ "dep:futures-util" ]; "httpdate" = [ "dep:httpdate" ]; - "iri-string" = [ "dep:iri-string" ]; "limit" = [ "dep:http-body" "dep:http-body-util" ]; - "metrics" = [ "dep:http-body" "tokio/time" ]; + "metrics" = [ "dep:http-body" "dep:tokio" "tokio?/time" ]; "mime" = [ "dep:mime" ]; "mime_guess" = [ "dep:mime_guess" ]; + "on-early-drop" = [ "dep:http-body" ]; "percent-encoding" = [ "dep:percent-encoding" ]; "request-id" = [ "uuid" ]; - "timeout" = [ "dep:http-body" "tokio/time" ]; - "tokio" = [ "dep:tokio" ]; + "timeout" = [ "dep:http-body" "dep:tokio" "tokio?/time" ]; "tokio-util" = [ "dep:tokio-util" ]; "tower" = [ "dep:tower" ]; "trace" = [ "dep:http-body" "tracing" ]; @@ -11604,7 +11660,7 @@ rec { "uuid" = [ "dep:uuid" ]; "validate-request" = [ "mime" ]; }; - resolvedDefaultFeatures = [ "auth" "base64" "default" "follow-redirect" "futures-util" "iri-string" "map-response-body" "mime" "tower" "trace" "tracing" "util" "validate-request" ]; + resolvedDefaultFeatures = [ "auth" "base64" "default" "follow-redirect" "futures-util" "map-response-body" "mime" "tower" "trace" "tracing" "util" "validate-request" ]; }; "tower-layer" = rec { crateName = "tower-layer"; @@ -11677,9 +11733,9 @@ rec { }; "tracing-appender" = rec { crateName = "tracing-appender"; - version = "0.2.4"; + version = "0.2.5"; edition = "2018"; - sha256 = "1bxf7xvsr89glbq174cx0b9pinaacbhlmc85y1ssniv2rq5lhvbq"; + sha256 = "0g4a6q5s3wafid5lqw1ljzvh1nhk3a4zmb627fxv96dr7qcqc1h5"; libName = "tracing_appender"; authors = [ "Zeki Sherif " @@ -11690,6 +11746,10 @@ rec { name = "crossbeam-channel"; packageId = "crossbeam-channel"; } + { + name = "symlink"; + packageId = "symlink"; + } { name = "thiserror"; packageId = "thiserror 2.0.18"; @@ -11911,9 +11971,9 @@ rec { }; "tracing-subscriber" = rec { crateName = "tracing-subscriber"; - version = "0.3.22"; + version = "0.3.23"; edition = "2018"; - sha256 = "07hz575a0p1c2i4xw3gs3hkrykhndnkbfhyqdwjhvayx4ww18c1g"; + sha256 = "06fkr0qhggvrs861d7f74pn3i3a10h5jsp4n70jj9ys5b675fzyb"; libName = "tracing_subscriber"; authors = [ "Eliza Weisman " @@ -12044,13 +12104,9 @@ rec { }; "typenum" = rec { crateName = "typenum"; - version = "1.19.0"; + version = "1.20.1"; edition = "2018"; - sha256 = "1fw2mpbn2vmqan56j1b3fbpcdg80mz26fm53fs16bq5xcq84hban"; - authors = [ - "Paho Lurie-Gregg " - "Andre Bogus " - ]; + sha256 = "086s9ly0906kw5yw41249fba97w5zfxf03pyfwdkffvcprqfixdn"; features = { "scale-info" = [ "dep:scale-info" ]; "scale_info" = [ "scale-info/derive" ]; @@ -12083,9 +12139,9 @@ rec { }; "unicode-segmentation" = rec { crateName = "unicode-segmentation"; - version = "1.12.0"; + version = "1.13.3"; edition = "2018"; - sha256 = "14qla2jfx74yyb9ds3d2mpwpa4l4lzb9z57c6d2ba511458z5k7n"; + sha256 = "1a47zaq83p386r3baq4m018xd5q4q0grdg56i1x042dzn71x7xf6"; libName = "unicode_segmentation"; authors = [ "kwantam " @@ -12211,6 +12267,66 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; + "uuid" = rec { + crateName = "uuid"; + version = "1.23.2"; + edition = "2021"; + sha256 = "1xy942s4z0bi8p3441wvd4ry3hx6ry1c7s6fgrr38462xqybhn6j"; + authors = [ + "Ashley Mannix" + "Dylan DPC" + "Hunar Roop Kahlon" + ]; + dependencies = [ + { + name = "js-sys"; + packageId = "js-sys"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)) && (builtins.elem "atomics" targetFeatures)); + } + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); + } + ]; + devDependencies = [ + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + "atomic" = [ "dep:atomic" ]; + "borsh" = [ "dep:borsh" "dep:borsh-derive" ]; + "bytemuck" = [ "dep:bytemuck" ]; + "default" = [ "std" ]; + "fast-rng" = [ "rng" "dep:rand" ]; + "js" = [ "dep:wasm-bindgen" "dep:js-sys" ]; + "md5" = [ "dep:md-5" ]; + "rng" = [ "dep:getrandom" ]; + "rng-getrandom" = [ "rng" "dep:getrandom" "uuid-rng-internal-lib" "uuid-rng-internal-lib/getrandom" ]; + "rng-rand" = [ "rng" "dep:rand" "uuid-rng-internal-lib" "uuid-rng-internal-lib/rand" ]; + "serde" = [ "dep:serde_core" ]; + "sha1" = [ "dep:sha1_smol" ]; + "slog" = [ "dep:slog" ]; + "std" = [ "wasm-bindgen?/std" "js-sys?/std" ]; + "uuid-rng-internal-lib" = [ "dep:uuid-rng-internal-lib" ]; + "v1" = [ "atomic" ]; + "v3" = [ "md5" ]; + "v4" = [ "rng" ]; + "v5" = [ "sha1" ]; + "v6" = [ "atomic" ]; + "v7" = [ "rng" ]; + "zerocopy" = [ "dep:zerocopy" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "valuable" = rec { crateName = "valuable"; version = "0.1.1"; @@ -12278,9 +12394,9 @@ rec { }; "wasip2" = rec { crateName = "wasip2"; - version = "1.0.2+wasi-0.2.9"; + version = "1.0.3+wasi-0.2.9"; edition = "2021"; - sha256 = "1xdw7v08jpfjdg94sp4lbdgzwa587m5ifpz6fpdnkh02kwizj5wm"; + sha256 = "1mi3w855dz99xzjqc4aa8c9q5b6z1y5c963pkk4cvmr6vdr4c1i0"; dependencies = [ { name = "wit-bindgen"; @@ -12298,9 +12414,9 @@ rec { }; "wasm-bindgen" = rec { crateName = "wasm-bindgen"; - version = "0.2.113"; + version = "0.2.122"; edition = "2021"; - sha256 = "1wpg101a5rqqilv4cz4929kbph9g15y4v2fvkbg7yjsrgy9jlwk0"; + sha256 = "02flix96brsb2r1i3grnikii302iqpdm337kl3xv5lklz5v4bl1y"; libName = "wasm_bindgen"; authors = [ "The wasm-bindgen Developers" @@ -12349,62 +12465,37 @@ rec { }; "wasm-bindgen-futures" = rec { crateName = "wasm-bindgen-futures"; - version = "0.4.63"; + version = "0.4.72"; edition = "2021"; - sha256 = "06j4hyxvlfvas7lhvgai44vh8izd59774wv5m8hla3kp1djz92ca"; + sha256 = "03qb24gfr072rk8hb69glfdc8yhqqqq2rhy3j5i0ps8sk79dnwwl"; libName = "wasm_bindgen_futures"; authors = [ "The wasm-bindgen Developers" ]; dependencies = [ - { - name = "cfg-if"; - packageId = "cfg-if"; - } - { - name = "futures-util"; - packageId = "futures-util"; - optional = true; - usesDefaultFeatures = false; - features = [ "std" ]; - } { name = "js-sys"; packageId = "js-sys"; usesDefaultFeatures = false; } - { - name = "once_cell"; - packageId = "once_cell"; - usesDefaultFeatures = false; - } { name = "wasm-bindgen"; packageId = "wasm-bindgen"; usesDefaultFeatures = false; } - { - name = "web-sys"; - packageId = "web-sys"; - usesDefaultFeatures = false; - target = { target, features }: (builtins.elem "atomics" targetFeatures); - features = [ "MessageEvent" "Worker" ]; - } ]; features = { "default" = [ "std" ]; - "futures-core" = [ "dep:futures-core" ]; - "futures-core-03-stream" = [ "futures-core" ]; - "futures-util" = [ "dep:futures-util" ]; - "std" = [ "wasm-bindgen/std" "js-sys/std" "web-sys/std" "futures-util" ]; + "futures-core-03-stream" = [ "js-sys/futures-core-03-stream" ]; + "std" = [ "wasm-bindgen/std" "js-sys/std" ]; }; - resolvedDefaultFeatures = [ "default" "futures-util" "std" ]; + resolvedDefaultFeatures = [ "default" "std" ]; }; "wasm-bindgen-macro" = rec { crateName = "wasm-bindgen-macro"; - version = "0.2.113"; + version = "0.2.122"; edition = "2021"; - sha256 = "0l1rbylzb1cs5i6ihmkgk8zic71pg563yadgqj8nnjq9jmiqrb0g"; + sha256 = "1inyl55bvdifx7l60q9wl0ivmw7236jg7jqmcqpxhsx3knq52qci"; procMacro = true; libName = "wasm_bindgen_macro"; authors = [ @@ -12426,9 +12517,9 @@ rec { }; "wasm-bindgen-macro-support" = rec { crateName = "wasm-bindgen-macro-support"; - version = "0.2.113"; + version = "0.2.122"; edition = "2021"; - sha256 = "0q4xmjmq1c80drv84hz9i9l7fj3yi0v2d11kh1r21p2rc77angxb"; + sha256 = "0pjw5kc2mbfz59agk5l21kh4hxzp94rygdvsnr4f3z6b5hv4g419"; libName = "wasm_bindgen_macro_support"; authors = [ "The wasm-bindgen Developers" @@ -12462,10 +12553,10 @@ rec { }; "wasm-bindgen-shared" = rec { crateName = "wasm-bindgen-shared"; - version = "0.2.113"; + version = "0.2.122"; edition = "2021"; links = "wasm_bindgen"; - sha256 = "1d9vdqrzksbfv30bvwy4kc57l08di24775hxq1yshkc2vcdhj3ny"; + sha256 = "0ds4mmfqvxwc5fp33hn0jblf0f6b4lghrd9mpkls66zic4n9p4ls"; libName = "wasm_bindgen_shared"; authors = [ "The wasm-bindgen Developers" @@ -12480,9 +12571,9 @@ rec { }; "web-sys" = rec { crateName = "web-sys"; - version = "0.3.90"; + version = "0.3.99"; edition = "2021"; - sha256 = "15wsyn0bmhgf4nkgl23l9fzcqml029jxdlavcbw304lhrsscwpkh"; + sha256 = "0dilfvl9jnyhi4skl6cry9wc300r693j0w82jjbq8yy3rx0i8qkd"; libName = "web_sys"; authors = [ "The wasm-bindgen Developers" @@ -12566,6 +12657,7 @@ rec { "CssStyleSheet" = [ "StyleSheet" ]; "CssSupportsRule" = [ "CssConditionRule" "CssGroupingRule" "CssRule" ]; "CssTransition" = [ "Animation" "EventTarget" ]; + "CssViewTransitionRule" = [ "CssRule" ]; "CustomEvent" = [ "Event" ]; "DedicatedWorkerGlobalScope" = [ "EventTarget" "WorkerGlobalScope" ]; "DelayNode" = [ "AudioNode" "EventTarget" ]; @@ -12972,7 +13064,7 @@ rec { "default" = [ "std" ]; "std" = [ "wasm-bindgen/std" "js-sys/std" ]; }; - resolvedDefaultFeatures = [ "AbortController" "AbortSignal" "Blob" "BlobPropertyBag" "Event" "EventTarget" "File" "FormData" "Headers" "MessageEvent" "ReadableStream" "Request" "RequestCache" "RequestCredentials" "RequestInit" "RequestMode" "Response" "ServiceWorkerGlobalScope" "Window" "Worker" "WorkerGlobalScope" "default" "std" ]; + resolvedDefaultFeatures = [ "AbortController" "AbortSignal" "Blob" "BlobPropertyBag" "EventTarget" "File" "FormData" "Headers" "ReadableStream" "Request" "RequestCache" "RequestCredentials" "RequestInit" "RequestMode" "Response" "ServiceWorkerGlobalScope" "Window" "WorkerGlobalScope" "default" "std" ]; }; "web-time" = rec { crateName = "web-time"; @@ -13146,7 +13238,7 @@ rec { dependencies = [ { name = "windows-targets"; - packageId = "windows-targets 0.52.6"; + packageId = "windows-targets"; } ]; features = { @@ -13382,271 +13474,6 @@ rec { }; resolvedDefaultFeatures = [ "Win32" "Win32_Foundation" "Win32_System" "Win32_System_Threading" "default" ]; }; - "windows-sys 0.60.2" = rec { - crateName = "windows-sys"; - version = "0.60.2"; - edition = "2021"; - sha256 = "1jrbc615ihqnhjhxplr2kw7rasrskv9wj3lr80hgfd42sbj01xgj"; - libName = "windows_sys"; - authors = [ - "Microsoft" - ]; - dependencies = [ - { - name = "windows-targets"; - packageId = "windows-targets 0.53.5"; - usesDefaultFeatures = false; - } - ]; - features = { - "Wdk" = [ "Win32_Foundation" ]; - "Wdk_Devices" = [ "Wdk" ]; - "Wdk_Devices_Bluetooth" = [ "Wdk_Devices" ]; - "Wdk_Devices_HumanInterfaceDevice" = [ "Wdk_Devices" ]; - "Wdk_Foundation" = [ "Wdk" ]; - "Wdk_Graphics" = [ "Wdk" ]; - "Wdk_Graphics_Direct3D" = [ "Wdk_Graphics" ]; - "Wdk_NetworkManagement" = [ "Wdk" ]; - "Wdk_NetworkManagement_Ndis" = [ "Wdk_NetworkManagement" ]; - "Wdk_NetworkManagement_WindowsFilteringPlatform" = [ "Wdk_NetworkManagement" ]; - "Wdk_Storage" = [ "Wdk" ]; - "Wdk_Storage_FileSystem" = [ "Wdk_Storage" ]; - "Wdk_Storage_FileSystem_Minifilters" = [ "Wdk_Storage_FileSystem" ]; - "Wdk_System" = [ "Wdk" ]; - "Wdk_System_IO" = [ "Wdk_System" ]; - "Wdk_System_Memory" = [ "Wdk_System" ]; - "Wdk_System_OfflineRegistry" = [ "Wdk_System" ]; - "Wdk_System_Registry" = [ "Wdk_System" ]; - "Wdk_System_SystemInformation" = [ "Wdk_System" ]; - "Wdk_System_SystemServices" = [ "Wdk_System" ]; - "Wdk_System_Threading" = [ "Wdk_System" ]; - "Win32" = [ "Win32_Foundation" ]; - "Win32_Data" = [ "Win32" ]; - "Win32_Data_HtmlHelp" = [ "Win32_Data" ]; - "Win32_Data_RightsManagement" = [ "Win32_Data" ]; - "Win32_Devices" = [ "Win32" ]; - "Win32_Devices_AllJoyn" = [ "Win32_Devices" ]; - "Win32_Devices_Beep" = [ "Win32_Devices" ]; - "Win32_Devices_BiometricFramework" = [ "Win32_Devices" ]; - "Win32_Devices_Bluetooth" = [ "Win32_Devices" ]; - "Win32_Devices_Cdrom" = [ "Win32_Devices" ]; - "Win32_Devices_Communication" = [ "Win32_Devices" ]; - "Win32_Devices_DeviceAndDriverInstallation" = [ "Win32_Devices" ]; - "Win32_Devices_DeviceQuery" = [ "Win32_Devices" ]; - "Win32_Devices_Display" = [ "Win32_Devices" ]; - "Win32_Devices_Dvd" = [ "Win32_Devices" ]; - "Win32_Devices_Enumeration" = [ "Win32_Devices" ]; - "Win32_Devices_Enumeration_Pnp" = [ "Win32_Devices_Enumeration" ]; - "Win32_Devices_Fax" = [ "Win32_Devices" ]; - "Win32_Devices_HumanInterfaceDevice" = [ "Win32_Devices" ]; - "Win32_Devices_Nfc" = [ "Win32_Devices" ]; - "Win32_Devices_Nfp" = [ "Win32_Devices" ]; - "Win32_Devices_PortableDevices" = [ "Win32_Devices" ]; - "Win32_Devices_Properties" = [ "Win32_Devices" ]; - "Win32_Devices_Pwm" = [ "Win32_Devices" ]; - "Win32_Devices_Sensors" = [ "Win32_Devices" ]; - "Win32_Devices_SerialCommunication" = [ "Win32_Devices" ]; - "Win32_Devices_Tapi" = [ "Win32_Devices" ]; - "Win32_Devices_Usb" = [ "Win32_Devices" ]; - "Win32_Devices_WebServicesOnDevices" = [ "Win32_Devices" ]; - "Win32_Foundation" = [ "Win32" ]; - "Win32_Gaming" = [ "Win32" ]; - "Win32_Globalization" = [ "Win32" ]; - "Win32_Graphics" = [ "Win32" ]; - "Win32_Graphics_Dwm" = [ "Win32_Graphics" ]; - "Win32_Graphics_Gdi" = [ "Win32_Graphics" ]; - "Win32_Graphics_GdiPlus" = [ "Win32_Graphics" ]; - "Win32_Graphics_Hlsl" = [ "Win32_Graphics" ]; - "Win32_Graphics_OpenGL" = [ "Win32_Graphics" ]; - "Win32_Graphics_Printing" = [ "Win32_Graphics" ]; - "Win32_Graphics_Printing_PrintTicket" = [ "Win32_Graphics_Printing" ]; - "Win32_Management" = [ "Win32" ]; - "Win32_Management_MobileDeviceManagementRegistration" = [ "Win32_Management" ]; - "Win32_Media" = [ "Win32" ]; - "Win32_Media_Audio" = [ "Win32_Media" ]; - "Win32_Media_DxMediaObjects" = [ "Win32_Media" ]; - "Win32_Media_KernelStreaming" = [ "Win32_Media" ]; - "Win32_Media_Multimedia" = [ "Win32_Media" ]; - "Win32_Media_Streaming" = [ "Win32_Media" ]; - "Win32_Media_WindowsMediaFormat" = [ "Win32_Media" ]; - "Win32_NetworkManagement" = [ "Win32" ]; - "Win32_NetworkManagement_Dhcp" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_Dns" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_InternetConnectionWizard" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_IpHelper" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_Multicast" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_Ndis" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_NetBios" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_NetManagement" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_NetShell" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_NetworkDiagnosticsFramework" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_P2P" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_QoS" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_Rras" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_Snmp" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_WNet" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_WebDav" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_WiFi" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_WindowsConnectionManager" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_WindowsFilteringPlatform" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_WindowsFirewall" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_WindowsNetworkVirtualization" = [ "Win32_NetworkManagement" ]; - "Win32_Networking" = [ "Win32" ]; - "Win32_Networking_ActiveDirectory" = [ "Win32_Networking" ]; - "Win32_Networking_Clustering" = [ "Win32_Networking" ]; - "Win32_Networking_HttpServer" = [ "Win32_Networking" ]; - "Win32_Networking_Ldap" = [ "Win32_Networking" ]; - "Win32_Networking_WebSocket" = [ "Win32_Networking" ]; - "Win32_Networking_WinHttp" = [ "Win32_Networking" ]; - "Win32_Networking_WinInet" = [ "Win32_Networking" ]; - "Win32_Networking_WinSock" = [ "Win32_Networking" ]; - "Win32_Networking_WindowsWebServices" = [ "Win32_Networking" ]; - "Win32_Security" = [ "Win32" ]; - "Win32_Security_AppLocker" = [ "Win32_Security" ]; - "Win32_Security_Authentication" = [ "Win32_Security" ]; - "Win32_Security_Authentication_Identity" = [ "Win32_Security_Authentication" ]; - "Win32_Security_Authorization" = [ "Win32_Security" ]; - "Win32_Security_Credentials" = [ "Win32_Security" ]; - "Win32_Security_Cryptography" = [ "Win32_Security" ]; - "Win32_Security_Cryptography_Catalog" = [ "Win32_Security_Cryptography" ]; - "Win32_Security_Cryptography_Certificates" = [ "Win32_Security_Cryptography" ]; - "Win32_Security_Cryptography_Sip" = [ "Win32_Security_Cryptography" ]; - "Win32_Security_Cryptography_UI" = [ "Win32_Security_Cryptography" ]; - "Win32_Security_DiagnosticDataQuery" = [ "Win32_Security" ]; - "Win32_Security_DirectoryServices" = [ "Win32_Security" ]; - "Win32_Security_EnterpriseData" = [ "Win32_Security" ]; - "Win32_Security_ExtensibleAuthenticationProtocol" = [ "Win32_Security" ]; - "Win32_Security_Isolation" = [ "Win32_Security" ]; - "Win32_Security_LicenseProtection" = [ "Win32_Security" ]; - "Win32_Security_NetworkAccessProtection" = [ "Win32_Security" ]; - "Win32_Security_WinTrust" = [ "Win32_Security" ]; - "Win32_Security_WinWlx" = [ "Win32_Security" ]; - "Win32_Storage" = [ "Win32" ]; - "Win32_Storage_Cabinets" = [ "Win32_Storage" ]; - "Win32_Storage_CloudFilters" = [ "Win32_Storage" ]; - "Win32_Storage_Compression" = [ "Win32_Storage" ]; - "Win32_Storage_DistributedFileSystem" = [ "Win32_Storage" ]; - "Win32_Storage_FileHistory" = [ "Win32_Storage" ]; - "Win32_Storage_FileSystem" = [ "Win32_Storage" ]; - "Win32_Storage_Imapi" = [ "Win32_Storage" ]; - "Win32_Storage_IndexServer" = [ "Win32_Storage" ]; - "Win32_Storage_InstallableFileSystems" = [ "Win32_Storage" ]; - "Win32_Storage_IscsiDisc" = [ "Win32_Storage" ]; - "Win32_Storage_Jet" = [ "Win32_Storage" ]; - "Win32_Storage_Nvme" = [ "Win32_Storage" ]; - "Win32_Storage_OfflineFiles" = [ "Win32_Storage" ]; - "Win32_Storage_OperationRecorder" = [ "Win32_Storage" ]; - "Win32_Storage_Packaging" = [ "Win32_Storage" ]; - "Win32_Storage_Packaging_Appx" = [ "Win32_Storage_Packaging" ]; - "Win32_Storage_ProjectedFileSystem" = [ "Win32_Storage" ]; - "Win32_Storage_StructuredStorage" = [ "Win32_Storage" ]; - "Win32_Storage_Vhd" = [ "Win32_Storage" ]; - "Win32_Storage_Xps" = [ "Win32_Storage" ]; - "Win32_System" = [ "Win32" ]; - "Win32_System_AddressBook" = [ "Win32_System" ]; - "Win32_System_Antimalware" = [ "Win32_System" ]; - "Win32_System_ApplicationInstallationAndServicing" = [ "Win32_System" ]; - "Win32_System_ApplicationVerifier" = [ "Win32_System" ]; - "Win32_System_ClrHosting" = [ "Win32_System" ]; - "Win32_System_Com" = [ "Win32_System" ]; - "Win32_System_Com_Marshal" = [ "Win32_System_Com" ]; - "Win32_System_Com_StructuredStorage" = [ "Win32_System_Com" ]; - "Win32_System_Com_Urlmon" = [ "Win32_System_Com" ]; - "Win32_System_ComponentServices" = [ "Win32_System" ]; - "Win32_System_Console" = [ "Win32_System" ]; - "Win32_System_CorrelationVector" = [ "Win32_System" ]; - "Win32_System_DataExchange" = [ "Win32_System" ]; - "Win32_System_DeploymentServices" = [ "Win32_System" ]; - "Win32_System_DeveloperLicensing" = [ "Win32_System" ]; - "Win32_System_Diagnostics" = [ "Win32_System" ]; - "Win32_System_Diagnostics_Ceip" = [ "Win32_System_Diagnostics" ]; - "Win32_System_Diagnostics_Debug" = [ "Win32_System_Diagnostics" ]; - "Win32_System_Diagnostics_Debug_Extensions" = [ "Win32_System_Diagnostics_Debug" ]; - "Win32_System_Diagnostics_Etw" = [ "Win32_System_Diagnostics" ]; - "Win32_System_Diagnostics_ProcessSnapshotting" = [ "Win32_System_Diagnostics" ]; - "Win32_System_Diagnostics_ToolHelp" = [ "Win32_System_Diagnostics" ]; - "Win32_System_Diagnostics_TraceLogging" = [ "Win32_System_Diagnostics" ]; - "Win32_System_DistributedTransactionCoordinator" = [ "Win32_System" ]; - "Win32_System_Environment" = [ "Win32_System" ]; - "Win32_System_ErrorReporting" = [ "Win32_System" ]; - "Win32_System_EventCollector" = [ "Win32_System" ]; - "Win32_System_EventLog" = [ "Win32_System" ]; - "Win32_System_EventNotificationService" = [ "Win32_System" ]; - "Win32_System_GroupPolicy" = [ "Win32_System" ]; - "Win32_System_HostCompute" = [ "Win32_System" ]; - "Win32_System_HostComputeNetwork" = [ "Win32_System" ]; - "Win32_System_HostComputeSystem" = [ "Win32_System" ]; - "Win32_System_Hypervisor" = [ "Win32_System" ]; - "Win32_System_IO" = [ "Win32_System" ]; - "Win32_System_Iis" = [ "Win32_System" ]; - "Win32_System_Ioctl" = [ "Win32_System" ]; - "Win32_System_JobObjects" = [ "Win32_System" ]; - "Win32_System_Js" = [ "Win32_System" ]; - "Win32_System_Kernel" = [ "Win32_System" ]; - "Win32_System_LibraryLoader" = [ "Win32_System" ]; - "Win32_System_Mailslots" = [ "Win32_System" ]; - "Win32_System_Mapi" = [ "Win32_System" ]; - "Win32_System_Memory" = [ "Win32_System" ]; - "Win32_System_Memory_NonVolatile" = [ "Win32_System_Memory" ]; - "Win32_System_MessageQueuing" = [ "Win32_System" ]; - "Win32_System_MixedReality" = [ "Win32_System" ]; - "Win32_System_Ole" = [ "Win32_System" ]; - "Win32_System_PasswordManagement" = [ "Win32_System" ]; - "Win32_System_Performance" = [ "Win32_System" ]; - "Win32_System_Performance_HardwareCounterProfiling" = [ "Win32_System_Performance" ]; - "Win32_System_Pipes" = [ "Win32_System" ]; - "Win32_System_Power" = [ "Win32_System" ]; - "Win32_System_ProcessStatus" = [ "Win32_System" ]; - "Win32_System_Recovery" = [ "Win32_System" ]; - "Win32_System_Registry" = [ "Win32_System" ]; - "Win32_System_RemoteDesktop" = [ "Win32_System" ]; - "Win32_System_RemoteManagement" = [ "Win32_System" ]; - "Win32_System_RestartManager" = [ "Win32_System" ]; - "Win32_System_Restore" = [ "Win32_System" ]; - "Win32_System_Rpc" = [ "Win32_System" ]; - "Win32_System_Search" = [ "Win32_System" ]; - "Win32_System_Search_Common" = [ "Win32_System_Search" ]; - "Win32_System_SecurityCenter" = [ "Win32_System" ]; - "Win32_System_Services" = [ "Win32_System" ]; - "Win32_System_SetupAndMigration" = [ "Win32_System" ]; - "Win32_System_Shutdown" = [ "Win32_System" ]; - "Win32_System_StationsAndDesktops" = [ "Win32_System" ]; - "Win32_System_SubsystemForLinux" = [ "Win32_System" ]; - "Win32_System_SystemInformation" = [ "Win32_System" ]; - "Win32_System_SystemServices" = [ "Win32_System" ]; - "Win32_System_Threading" = [ "Win32_System" ]; - "Win32_System_Time" = [ "Win32_System" ]; - "Win32_System_TpmBaseServices" = [ "Win32_System" ]; - "Win32_System_UserAccessLogging" = [ "Win32_System" ]; - "Win32_System_Variant" = [ "Win32_System" ]; - "Win32_System_VirtualDosMachines" = [ "Win32_System" ]; - "Win32_System_WindowsProgramming" = [ "Win32_System" ]; - "Win32_System_Wmi" = [ "Win32_System" ]; - "Win32_UI" = [ "Win32" ]; - "Win32_UI_Accessibility" = [ "Win32_UI" ]; - "Win32_UI_ColorSystem" = [ "Win32_UI" ]; - "Win32_UI_Controls" = [ "Win32_UI" ]; - "Win32_UI_Controls_Dialogs" = [ "Win32_UI_Controls" ]; - "Win32_UI_HiDpi" = [ "Win32_UI" ]; - "Win32_UI_Input" = [ "Win32_UI" ]; - "Win32_UI_Input_Ime" = [ "Win32_UI_Input" ]; - "Win32_UI_Input_KeyboardAndMouse" = [ "Win32_UI_Input" ]; - "Win32_UI_Input_Pointer" = [ "Win32_UI_Input" ]; - "Win32_UI_Input_Touch" = [ "Win32_UI_Input" ]; - "Win32_UI_Input_XboxController" = [ "Win32_UI_Input" ]; - "Win32_UI_InteractionContext" = [ "Win32_UI" ]; - "Win32_UI_Magnification" = [ "Win32_UI" ]; - "Win32_UI_Shell" = [ "Win32_UI" ]; - "Win32_UI_Shell_Common" = [ "Win32_UI_Shell" ]; - "Win32_UI_Shell_PropertiesSystem" = [ "Win32_UI_Shell" ]; - "Win32_UI_TabletPC" = [ "Win32_UI" ]; - "Win32_UI_TextServices" = [ "Win32_UI" ]; - "Win32_UI_WindowsAndMessaging" = [ "Win32_UI" ]; - "Win32_Web" = [ "Win32" ]; - "Win32_Web_InternetExplorer" = [ "Win32_Web" ]; - }; - resolvedDefaultFeatures = [ "Win32" "Win32_Foundation" "Win32_Networking" "Win32_Networking_WinSock" "Win32_System" "Win32_System_IO" "Win32_System_Threading" "Win32_System_WindowsProgramming" "default" ]; - }; "windows-sys 0.61.2" = rec { crateName = "windows-sys"; version = "0.61.2"; @@ -13907,9 +13734,9 @@ rec { "Win32_Web" = [ "Win32" ]; "Win32_Web_InternetExplorer" = [ "Win32_Web" ]; }; - resolvedDefaultFeatures = [ "Wdk" "Wdk_Foundation" "Wdk_Storage" "Wdk_Storage_FileSystem" "Wdk_System" "Wdk_System_IO" "Win32" "Win32_Foundation" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_IO" "Win32_System_LibraryLoader" "Win32_System_Memory" "Win32_System_Pipes" "Win32_System_SystemInformation" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_Time" "Win32_System_WindowsProgramming" "default" ]; + resolvedDefaultFeatures = [ "Wdk" "Wdk_Foundation" "Wdk_Storage" "Wdk_Storage_FileSystem" "Wdk_System" "Wdk_System_IO" "Win32" "Win32_Foundation" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_IO" "Win32_System_LibraryLoader" "Win32_System_Memory" "Win32_System_Pipes" "Win32_System_SystemInformation" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_WindowsProgramming" "default" ]; }; - "windows-targets 0.52.6" = rec { + "windows-targets" = rec { crateName = "windows-targets"; version = "0.52.6"; edition = "2021"; @@ -13921,104 +13748,48 @@ rec { dependencies = [ { name = "windows_aarch64_gnullvm"; - packageId = "windows_aarch64_gnullvm 0.52.6"; + packageId = "windows_aarch64_gnullvm"; target = { target, features }: (target.name == "aarch64-pc-windows-gnullvm"); } { name = "windows_aarch64_msvc"; - packageId = "windows_aarch64_msvc 0.52.6"; + packageId = "windows_aarch64_msvc"; target = { target, features }: (("aarch64" == target."arch" or null) && ("msvc" == target."env" or null) && (!(target."windows_raw_dylib" or false))); } { name = "windows_i686_gnu"; - packageId = "windows_i686_gnu 0.52.6"; + packageId = "windows_i686_gnu"; target = { target, features }: (("x86" == target."arch" or null) && ("gnu" == target."env" or null) && (!("llvm" == target."abi" or null)) && (!(target."windows_raw_dylib" or false))); } { name = "windows_i686_gnullvm"; - packageId = "windows_i686_gnullvm 0.52.6"; + packageId = "windows_i686_gnullvm"; target = { target, features }: (target.name == "i686-pc-windows-gnullvm"); } { name = "windows_i686_msvc"; - packageId = "windows_i686_msvc 0.52.6"; + packageId = "windows_i686_msvc"; target = { target, features }: (("x86" == target."arch" or null) && ("msvc" == target."env" or null) && (!(target."windows_raw_dylib" or false))); } { name = "windows_x86_64_gnu"; - packageId = "windows_x86_64_gnu 0.52.6"; + packageId = "windows_x86_64_gnu"; target = { target, features }: (("x86_64" == target."arch" or null) && ("gnu" == target."env" or null) && (!("llvm" == target."abi" or null)) && (!(target."windows_raw_dylib" or false))); } { name = "windows_x86_64_gnullvm"; - packageId = "windows_x86_64_gnullvm 0.52.6"; + packageId = "windows_x86_64_gnullvm"; target = { target, features }: (target.name == "x86_64-pc-windows-gnullvm"); } { name = "windows_x86_64_msvc"; - packageId = "windows_x86_64_msvc 0.52.6"; + packageId = "windows_x86_64_msvc"; target = { target, features }: ((("x86_64" == target."arch" or null) || ("arm64ec" == target."arch" or null)) && ("msvc" == target."env" or null) && (!(target."windows_raw_dylib" or false))); } ]; }; - "windows-targets 0.53.5" = rec { - crateName = "windows-targets"; - version = "0.53.5"; - edition = "2021"; - sha256 = "1wv9j2gv3l6wj3gkw5j1kr6ymb5q6dfc42yvydjhv3mqa7szjia9"; - libName = "windows_targets"; - dependencies = [ - { - name = "windows-link"; - packageId = "windows-link"; - usesDefaultFeatures = false; - target = { target, features }: (target."windows_raw_dylib" or false); - } - { - name = "windows_aarch64_gnullvm"; - packageId = "windows_aarch64_gnullvm 0.53.1"; - target = { target, features }: (target.name == "aarch64-pc-windows-gnullvm"); - } - { - name = "windows_aarch64_msvc"; - packageId = "windows_aarch64_msvc 0.53.1"; - target = { target, features }: (("aarch64" == target."arch" or null) && ("msvc" == target."env" or null) && (!(target."windows_raw_dylib" or false))); - } - { - name = "windows_i686_gnu"; - packageId = "windows_i686_gnu 0.53.1"; - target = { target, features }: (("x86" == target."arch" or null) && ("gnu" == target."env" or null) && (!("llvm" == target."abi" or null)) && (!(target."windows_raw_dylib" or false))); - } - { - name = "windows_i686_gnullvm"; - packageId = "windows_i686_gnullvm 0.53.1"; - target = { target, features }: (target.name == "i686-pc-windows-gnullvm"); - } - { - name = "windows_i686_msvc"; - packageId = "windows_i686_msvc 0.53.1"; - target = { target, features }: (("x86" == target."arch" or null) && ("msvc" == target."env" or null) && (!(target."windows_raw_dylib" or false))); - } - { - name = "windows_x86_64_gnu"; - packageId = "windows_x86_64_gnu 0.53.1"; - target = { target, features }: (("x86_64" == target."arch" or null) && ("gnu" == target."env" or null) && (!("llvm" == target."abi" or null)) && (!(target."windows_raw_dylib" or false))); - } - { - name = "windows_x86_64_gnullvm"; - packageId = "windows_x86_64_gnullvm 0.53.1"; - target = { target, features }: (target.name == "x86_64-pc-windows-gnullvm"); - } - { - name = "windows_x86_64_msvc"; - packageId = "windows_x86_64_msvc 0.53.1"; - target = { target, features }: ((("x86_64" == target."arch" or null) || ("arm64ec" == target."arch" or null)) && ("msvc" == target."env" or null) && (!(target."windows_raw_dylib" or false))); - } - ]; - - }; - "windows_aarch64_gnullvm 0.52.6" = rec { + "windows_aarch64_gnullvm" = rec { crateName = "windows_aarch64_gnullvm"; version = "0.52.6"; edition = "2021"; @@ -14028,14 +13799,7 @@ rec { ]; }; - "windows_aarch64_gnullvm 0.53.1" = rec { - crateName = "windows_aarch64_gnullvm"; - version = "0.53.1"; - edition = "2021"; - sha256 = "0lqvdm510mka9w26vmga7hbkmrw9glzc90l4gya5qbxlm1pl3n59"; - - }; - "windows_aarch64_msvc 0.52.6" = rec { + "windows_aarch64_msvc" = rec { crateName = "windows_aarch64_msvc"; version = "0.52.6"; edition = "2021"; @@ -14045,14 +13809,7 @@ rec { ]; }; - "windows_aarch64_msvc 0.53.1" = rec { - crateName = "windows_aarch64_msvc"; - version = "0.53.1"; - edition = "2021"; - sha256 = "01jh2adlwx043rji888b22whx4bm8alrk3khjpik5xn20kl85mxr"; - - }; - "windows_i686_gnu 0.52.6" = rec { + "windows_i686_gnu" = rec { crateName = "windows_i686_gnu"; version = "0.52.6"; edition = "2021"; @@ -14062,14 +13819,7 @@ rec { ]; }; - "windows_i686_gnu 0.53.1" = rec { - crateName = "windows_i686_gnu"; - version = "0.53.1"; - edition = "2021"; - sha256 = "18wkcm82ldyg4figcsidzwbg1pqd49jpm98crfz0j7nqd6h6s3ln"; - - }; - "windows_i686_gnullvm 0.52.6" = rec { + "windows_i686_gnullvm" = rec { crateName = "windows_i686_gnullvm"; version = "0.52.6"; edition = "2021"; @@ -14079,14 +13829,7 @@ rec { ]; }; - "windows_i686_gnullvm 0.53.1" = rec { - crateName = "windows_i686_gnullvm"; - version = "0.53.1"; - edition = "2021"; - sha256 = "030qaxqc4salz6l4immfb6sykc6gmhyir9wzn2w8mxj8038mjwzs"; - - }; - "windows_i686_msvc 0.52.6" = rec { + "windows_i686_msvc" = rec { crateName = "windows_i686_msvc"; version = "0.52.6"; edition = "2021"; @@ -14096,14 +13839,7 @@ rec { ]; }; - "windows_i686_msvc 0.53.1" = rec { - crateName = "windows_i686_msvc"; - version = "0.53.1"; - edition = "2021"; - sha256 = "1hi6scw3mn2pbdl30ji5i4y8vvspb9b66l98kkz350pig58wfyhy"; - - }; - "windows_x86_64_gnu 0.52.6" = rec { + "windows_x86_64_gnu" = rec { crateName = "windows_x86_64_gnu"; version = "0.52.6"; edition = "2021"; @@ -14113,14 +13849,7 @@ rec { ]; }; - "windows_x86_64_gnu 0.53.1" = rec { - crateName = "windows_x86_64_gnu"; - version = "0.53.1"; - edition = "2021"; - sha256 = "16d4yiysmfdlsrghndr97y57gh3kljkwhfdbcs05m1jasz6l4f4w"; - - }; - "windows_x86_64_gnullvm 0.52.6" = rec { + "windows_x86_64_gnullvm" = rec { crateName = "windows_x86_64_gnullvm"; version = "0.52.6"; edition = "2021"; @@ -14130,14 +13859,7 @@ rec { ]; }; - "windows_x86_64_gnullvm 0.53.1" = rec { - crateName = "windows_x86_64_gnullvm"; - version = "0.53.1"; - edition = "2021"; - sha256 = "1qbspgv4g3q0vygkg8rnql5c6z3caqv38japiynyivh75ng1gyhg"; - - }; - "windows_x86_64_msvc 0.52.6" = rec { + "windows_x86_64_msvc" = rec { crateName = "windows_x86_64_msvc"; version = "0.52.6"; edition = "2021"; @@ -14146,19 +13868,12 @@ rec { "Microsoft" ]; - }; - "windows_x86_64_msvc 0.53.1" = rec { - crateName = "windows_x86_64_msvc"; - version = "0.53.1"; - edition = "2021"; - sha256 = "0l6npq76vlq4ksn4bwsncpr8508mk0gmznm6wnhjg95d19gzzfyn"; - }; "winnow" = rec { crateName = "winnow"; - version = "0.7.14"; + version = "1.0.3"; edition = "2021"; - sha256 = "0a88ahjqhyn2ln1yplq2xsigm09kxqkdkkk2c2mfxkbzszln8lss"; + sha256 = "1wajycd3krn6h699vydjv7hm0ll5l31p899qzpk59y2is74y34h5"; dependencies = [ { name = "memchr"; @@ -14168,38 +13883,42 @@ rec { } ]; features = { + "ascii" = [ "parser" ]; + "binary" = [ "parser" ]; "debug" = [ "std" "dep:anstream" "dep:anstyle" "dep:is_terminal_polyfill" "dep:terminal_size" ]; - "default" = [ "std" ]; + "default" = [ "std" "ascii" "binary" ]; "simd" = [ "dep:memchr" ]; "std" = [ "alloc" "memchr?/std" ]; - "unstable-doc" = [ "alloc" "std" "simd" "unstable-recover" ]; + "unstable-doc" = [ "alloc" "std" "ascii" "binary" "simd" "unstable-recover" ]; + "unstable-recover" = [ "parser" ]; }; - resolvedDefaultFeatures = [ "alloc" "default" "std" ]; + resolvedDefaultFeatures = [ "alloc" "ascii" "binary" "default" "parser" "std" ]; }; "wit-bindgen" = rec { crateName = "wit-bindgen"; - version = "0.51.0"; + version = "0.57.1"; edition = "2024"; - sha256 = "19fazgch8sq5cvjv3ynhhfh5d5x08jq2pkw8jfb05vbcyqcr496p"; + sha256 = "0vjk2jb593ri9k1aq4iqs2si9mrw5q46wxnn78im7hm7hx799gqy"; libName = "wit_bindgen"; authors = [ "Alex Crichton " ]; features = { - "async" = [ "std" "wit-bindgen-rust-macro?/async" ]; - "async-spawn" = [ "async" "dep:futures" ]; + "async-spawn" = [ "async" "dep:futures" "std" ]; "bitflags" = [ "dep:bitflags" ]; - "default" = [ "macros" "realloc" "async" "std" "bitflags" ]; + "default" = [ "macros" "realloc" "async" "std" "bitflags" "macro-string" ]; + "futures-stream" = [ "async" "dep:futures" ]; "inter-task-wakeup" = [ "async" ]; + "macro-string" = [ "wit-bindgen-rust-macro?/macro-string" ]; "macros" = [ "dep:wit-bindgen-rust-macro" ]; "rustc-dep-of-std" = [ "dep:core" "dep:alloc" ]; }; }; "writeable" = rec { crateName = "writeable"; - version = "0.6.2"; + version = "0.6.3"; edition = "2021"; - sha256 = "1fg08y97n6vk7l0rnjggw3xyrii6dcqg54wqaxldrlk98zdy1pcy"; + sha256 = "1i54d13h9bpap2hf13xcry1s4lxh7ap3923g8f3c0grd7c9fbyhz"; authors = [ "The ICU4X Project Developers" ]; @@ -14266,9 +13985,9 @@ rec { }; "xml" = rec { crateName = "xml"; - version = "1.2.1"; + version = "1.3.0"; edition = "2021"; - sha256 = "0ak4k990faralbli5a0rb8kvwihccb2rp0r94d4azfy94a6lkamq"; + sha256 = "128s58qhq8whrx90zbw8r5algr7lakgbf7mn05jfk234rbjqavv3"; authors = [ "Vladimir Matveev " "Kornel (https://github.com/kornelski)" @@ -14277,9 +13996,9 @@ rec { }; "yoke" = rec { crateName = "yoke"; - version = "0.8.1"; + version = "0.8.2"; edition = "2021"; - sha256 = "0m29dm0bf5iakxgma0bj6dbmc3b8qi9b1vaw9sa76kdqmz3fbmkj"; + sha256 = "1jprcs7a98a5whvfs6r3jvfh1nnfp6zyijl7y4ywmn88lzywbs5b"; authors = [ "Manish Goregaokar " ]; @@ -14312,9 +14031,9 @@ rec { }; "yoke-derive" = rec { crateName = "yoke-derive"; - version = "0.8.1"; + version = "0.8.2"; edition = "2021"; - sha256 = "0pbyja133jnng4mrhimzdq4a0y26421g734ybgz8wsgbfhl0andn"; + sha256 = "13l5y5sz4lqm7rmyakjbh6vwgikxiql51xfff9hq2j485hk4r16y"; procMacro = true; libName = "yoke_derive"; authors = [ @@ -14343,9 +14062,9 @@ rec { }; "zerocopy" = rec { crateName = "zerocopy"; - version = "0.8.40"; + version = "0.8.50"; edition = "2021"; - sha256 = "1r9j2mlb54q1l9pgall3mk0gg6cprhdncvbbgsgxnxmmj3jcd2d7"; + sha256 = "1laahnfxs4qyfb1fdf5nbb2qfshi72b1hbi0ffp2zy2m1r7ms1iv"; authors = [ "Joshua Liebow-Feeser " "Jack Wrenn " @@ -14379,9 +14098,9 @@ rec { }; "zerocopy-derive" = rec { crateName = "zerocopy-derive"; - version = "0.8.40"; + version = "0.8.50"; edition = "2021"; - sha256 = "0lsrhg5nvf0c40z644a014l2nrvh7xw0ff3i9744k9vif2d4hp7n"; + sha256 = "0fdnr9qslx1hbn2i9rsvy9s95mychfy2vj90ajsjm2basccinqqb"; procMacro = true; libName = "zerocopy_derive"; authors = [ @@ -14414,11 +14133,11 @@ rec { }; "zerofrom" = rec { crateName = "zerofrom"; - version = "0.1.6"; + version = "0.1.8"; edition = "2021"; - sha256 = "19dyky67zkjichsb7ykhv0aqws3q0jfvzww76l66c19y6gh45k2h"; + sha256 = "0wjjdj7gdmd0iq91gzkxl7dlv0nhkk80l4bmdpzh3a1yh48mmh0f"; authors = [ - "Manish Goregaokar " + "The ICU4X Project Developers" ]; dependencies = [ { @@ -14436,9 +14155,9 @@ rec { }; "zerofrom-derive" = rec { crateName = "zerofrom-derive"; - version = "0.1.6"; + version = "0.1.7"; edition = "2021"; - sha256 = "00l5niw7c1b0lf1vhvajpjmcnbdp2vn96jg4nmkhq2db0rp5s7np"; + sha256 = "18c4wsnznhdxx6m80piil1lbyszdiwsshgjrybqcm4b6qic22lqi"; procMacro = true; libName = "zerofrom_derive"; authors = [ @@ -14517,9 +14236,9 @@ rec { }; "zerotrie" = rec { crateName = "zerotrie"; - version = "0.2.3"; + version = "0.2.4"; edition = "2021"; - sha256 = "0lbqznlqazmrwwzslw0ci7p3pqxykrbfhq29npj0gmb2amxc2n9a"; + sha256 = "1gr0pkcn3qsr6in6iixqyp0vbzwf2j1jzyvh7yl2yydh3p9m548g"; authors = [ "The ICU4X Project Developers" ]; @@ -14544,7 +14263,9 @@ rec { } ]; features = { + "alloc" = [ "zerovec?/alloc" ]; "databake" = [ "dep:databake" "zerovec?/databake" ]; + "dense" = [ "dep:zerovec" ]; "litemap" = [ "dep:litemap" "alloc" ]; "serde" = [ "dep:serde_core" "dep:litemap" "alloc" "litemap/serde" "zerovec?/serde" ]; "yoke" = [ "dep:yoke" ]; @@ -14555,9 +14276,9 @@ rec { }; "zerovec" = rec { crateName = "zerovec"; - version = "0.11.5"; + version = "0.11.6"; edition = "2021"; - sha256 = "00m0p47k2g9mkv505hky5xh3r6ps7v8qc0dy4pspg542jj972a3c"; + sha256 = "0fdjsy6b31q9i0d73sl7xjd12xadbwi45lkpfgqnmasrqg5i3ych"; authors = [ "The ICU4X Project Developers" ]; @@ -14580,11 +14301,20 @@ rec { usesDefaultFeatures = false; } ]; + devDependencies = [ + { + name = "yoke"; + packageId = "yoke"; + usesDefaultFeatures = false; + features = [ "derive" ]; + } + ]; features = { "alloc" = [ "serde?/alloc" ]; "databake" = [ "dep:databake" ]; "derive" = [ "dep:zerovec-derive" ]; "hashmap" = [ "dep:twox-hash" "alloc" ]; + "schemars" = [ "dep:schemars" "alloc" ]; "serde" = [ "dep:serde" ]; "yoke" = [ "dep:yoke" ]; }; @@ -14592,9 +14322,9 @@ rec { }; "zerovec-derive" = rec { crateName = "zerovec-derive"; - version = "0.11.2"; + version = "0.11.3"; edition = "2021"; - sha256 = "1wsig4h5j7a1scd5hrlnragnazjny9qjc44hancb6p6a76ay7p7a"; + sha256 = "0m85qj92mmfvhjra6ziqky5b1p4kcmp5069k7kfadp5hr8jw8pb2"; procMacro = true; libName = "zerovec_derive"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index 86f2b840..5564a89e 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#k8s-version@0.1.3": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-certs@0.4.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-operator-derive@0.3.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-operator@0.111.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-shared@0.1.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-telemetry@0.6.3": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-versioned-macros@0.10.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-versioned@0.10.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.1#stackable-webhook@0.9.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.3": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file diff --git a/deploy/helm/hbase-operator/configs/properties.yaml b/deploy/helm/hbase-operator/configs/properties.yaml index c3840065..9bd8c3b2 100644 --- a/deploy/helm/hbase-operator/configs/properties.yaml +++ b/deploy/helm/hbase-operator/configs/properties.yaml @@ -2,270 +2,4 @@ version: 0.1.0 spec: units: [] - -properties: - - ################################################################################################# - # security.properties - ################################################################################################# - - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "5" - roles: - - name: "master" - required: true - asOfVersion: "0.0.0" - comment: "master - TTL for successfully resolved domain names." - description: "master - TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "10" - roles: - - name: "regionserver" - required: true - asOfVersion: "0.0.0" - comment: "regionserver - TTL for successfully resolved domain names." - description: "regionserver - TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "restserver" - required: true - asOfVersion: "0.0.0" - comment: "restserver - TTL for successfully resolved domain names." - description: "restserver - TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "master" - required: true - asOfVersion: "0.0.0" - comment: "master - TTL for domain names that cannot be resolved." - description: "master - TTL for domain names that cannot be resolved." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "regionserver" - required: true - asOfVersion: "0.0.0" - comment: "regionserver - TTL for domain names that cannot be resolved." - description: "regionserver - TTL for domain names that cannot be resolved." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "restserver" - required: true - asOfVersion: "0.0.0" - comment: "restserver - TTL for domain names that cannot be resolved." - description: "restserver - TTL for domain names that cannot be resolved." - - - ################################################################################################# - # hbase-env.sh - ################################################################################################# - - - property: &hbaseManagesZk - propertyNames: - - name: "HBASE_MANAGES_ZK" - kind: - type: "file" - file: "hbase-env.sh" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - recommendedValues: - - fromVersion: "0.0.0" - value: "false" - roles: - - name: "master" - required: false - - name: "regionserver" - required: false - - name: "restserver" - required: false - asOfVersion: "0.0.0" - description: "This variable tells HBase whether to start/stop the ZooKeeper ensemble servers as part of HBase start/stop." - - property: &hbaseOpts - propertyNames: - - name: "HBASE_OPTS" - kind: - type: "file" - file: "hbase-env.sh" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "" - recommendedValues: - - fromVersion: "0.0.0" - value: "" - roles: - - name: "master" - required: false - - name: "regionserver" - required: false - - name: "restserver" - required: false - asOfVersion: "0.0.0" - description: "This variable allows to pass VM options to HBase." - - ################################################################################################# - # hbase-site.xml - ################################################################################################# - - - property: &hbaseClusterDistributed - propertyNames: - - name: "hbase.cluster.distributed" - kind: - type: "file" - file: "hbase-site.xml" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "false" - recommendedValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "master" - required: false - - name: "regionserver" - required: false - - name: "restserver" - required: false - asOfVersion: "0.0.0" - description: "The mode the cluster will be in. Possible values are false for standalone mode and true for distributed mode. If false, startup will run all HBase and ZooKeeper daemons together in the one JVM." - - - property: &hbaseRootdir - propertyNames: - - name: "hbase.rootdir" - kind: - type: "file" - file: "hbase-site.xml" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "/hbase" - roles: - - name: "master" - required: true - - name: "regionserver" - required: true # this can be false (only required for master) - kept for compatibility (avoid restarts) - - name: "restserver" - required: true # this can be false (only required for master) - kept for compatibility (avoid restarts) - asOfVersion: "0.0.0" - description: "The directory shared by region servers and into which HBase persists. The URL should be 'fully-qualified' to include the filesystem scheme. For example, to specify the HDFS directory '/hbase' where the HDFS instance's namenode is running at namenode.example.org on port 9000, set this value to: hdfs://namenode.example.org:9000/hbase. By default, we write to whatever ${hbase.tmp.dir} is set too -- usually /tmp -- so change this configuration or else all data will be lost on machine restart." - - - property: &hbaseZookeeperQuorum - propertyNames: - - name: "hbase.zookeeper.quorum" - kind: - type: "file" - file: "hbase-site.xml" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "127.0.0.1" - roles: - - name: "master" - required: false - - name: "regionserver" - required: false - - name: "restserver" - required: false - asOfVersion: "0.0.0" - description: "Comma separated list of servers in the ZooKeeper ensemble (This config. should have been named hbase.zookeeper.ensemble). For example, \"host1.mydomain.com,host2.mydomain.com,host3.mydomain.com\". By default this is set to localhost for local and pseudo-distributed modes of operation. For a fully-distributed setup, this should be set to a full list of ZooKeeper ensemble servers. If HBASE_MANAGES_ZK is set in hbase-env.sh this is the list of servers which hbase will start/stop ZooKeeper on as part of cluster start/stop. Client-side, we will take this list of ensemble members and put it together with the hbase.zookeeper.property.clientPort config. and pass it into zookeeper constructor as the connectString parameter." - - ################################################################################################# - # hdfs-site.xml - ################################################################################################# - - - property: &hdfsConfig - propertyNames: - - name: "content" - kind: - type: "file" - file: "hbase-site.xml" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "" - roles: - - name: "master" - required: false - - name: "regionserver" - required: false - - name: "restserver" - required: false - asOfVersion: "0.0.0" - description: "The HDFS configuration file" +properties: [] diff --git a/extra/crds.yaml b/extra/crds.yaml index fc3180bb..80a33e12 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -494,53 +494,58 @@ spec: properties: hbase-env.sh: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object hbase-site.xml: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object security.properties: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object ssl-client.xml: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object ssl-server.xml: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object type: object envOverrides: @@ -934,53 +939,58 @@ spec: properties: hbase-env.sh: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object hbase-site.xml: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object security.properties: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object ssl-client.xml: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object ssl-server.xml: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object type: object envOverrides: @@ -1397,53 +1407,58 @@ spec: properties: hbase-env.sh: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object hbase-site.xml: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object security.properties: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object ssl-client.xml: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object ssl-server.xml: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object type: object envOverrides: @@ -1873,53 +1888,58 @@ spec: properties: hbase-env.sh: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object hbase-site.xml: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object security.properties: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object ssl-client.xml: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object ssl-server.xml: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object type: object envOverrides: @@ -2286,53 +2306,58 @@ spec: properties: hbase-env.sh: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object hbase-site.xml: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object security.properties: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object ssl-client.xml: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object ssl-server.xml: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object type: object envOverrides: @@ -2726,53 +2751,58 @@ spec: properties: hbase-env.sh: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object hbase-site.xml: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object security.properties: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object ssl-client.xml: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object ssl-server.xml: additionalProperties: + nullable: true type: string + default: {} description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. This is backwards-compatible with the existing flat key-value YAML format used by `HashMap`. - nullable: true type: object type: object envOverrides: From 90dfbcdd908877aa60259f71eee0316c13e48c6f Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 3 Jun 2026 19:28:49 +0200 Subject: [PATCH 09/56] fix: change paramter order & fmt --- .../src/controller/build/config_map.rs | 29 ++++++++++--------- .../controller/build/properties/hbase_env.rs | 5 +--- .../controller/build/properties/hbase_site.rs | 4 +-- .../controller/build/properties/ssl_server.rs | 5 +--- .../src/controller/validate.rs | 11 ++++--- rust/operator-binary/src/discovery.rs | 2 +- rust/operator-binary/src/hbase_controller.rs | 2 +- 7 files changed, 28 insertions(+), 30 deletions(-) diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 796650fc..3d9103d9 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -58,12 +58,12 @@ pub enum Error { type Result = std::result::Result; pub fn build_rolegroup_config_map( - // `hbase` is retained only for the ConfigMap ObjectMeta / owner reference; the rendered - // content comes entirely from `cluster`. To be decoupled in a follow-up. - hbase: &v1alpha1::HbaseCluster, cluster: &ValidatedCluster, role: &HbaseRole, rolegroup_ref: &RoleGroupRef, + // `owner` is retained only for the ConfigMap ObjectMeta / owner reference; the rendered + // content comes entirely from `cluster`. To be decoupled in a follow-up. + owner_ref: &v1alpha1::HbaseCluster, ) -> Result { tracing::info!("Setting up ConfigMap for {:?}", rolegroup_ref); @@ -108,18 +108,20 @@ pub fn build_rolegroup_config_map( overrides.ssl_client_xml.clone(), ); - let security_properties = security_properties::build(role, overrides.security_properties.clone()) - .with_context(|_| JvmSecurityPropertiesSnafu { - role_group: rolegroup_ref.role_group.clone(), - })?; + let security_properties = + security_properties::build(role, overrides.security_properties.clone()).with_context( + |_| JvmSecurityPropertiesSnafu { + role_group: rolegroup_ref.role_group.clone(), + }, + )?; let cm_metadata = ObjectMetaBuilder::new() - .name_and_namespace(hbase) + .name_and_namespace(owner_ref) .name(rolegroup_ref.object_name()) - .ownerreference_from_resource(hbase, None, Some(true)) + .ownerreference_from_resource(owner_ref, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(&build_recommended_labels( - hbase, + owner_ref, &cluster.image.app_version_label_value, &rolegroup_ref.role, &rolegroup_ref.role_group, @@ -143,10 +145,11 @@ pub fn build_rolegroup_config_map( builder.add_data(ConfigFileName::SslClient.to_string(), ssl_client_xml); } - extend_role_group_config_map(rolegroup_ref, rg.merged_config.logging(), &mut builder) - .context(InvalidLoggingConfigSnafu { + extend_role_group_config_map(rolegroup_ref, rg.merged_config.logging(), &mut builder).context( + InvalidLoggingConfigSnafu { cm_name: rolegroup_ref.object_name(), - })?; + }, + )?; builder.build().with_context(|_| AssembleSnafu { role: rolegroup_ref.role.clone(), diff --git a/rust/operator-binary/src/controller/build/properties/hbase_env.rs b/rust/operator-binary/src/controller/build/properties/hbase_env.rs index 42fc905e..bed4d356 100644 --- a/rust/operator-binary/src/controller/build/properties/hbase_env.rs +++ b/rust/operator-binary/src/controller/build/properties/hbase_env.rs @@ -123,9 +123,6 @@ mod tests { config_overrides(&[("CUSTOM_VAR", "custom_value")]), ) .unwrap(); - assert!( - env.contains("export CUSTOM_VAR=\"custom_value\""), - "{env}" - ); + assert!(env.contains("export CUSTOM_VAR=\"custom_value\""), "{env}"); } } diff --git a/rust/operator-binary/src/controller/build/properties/hbase_site.rs b/rust/operator-binary/src/controller/build/properties/hbase_site.rs index 4bae593c..39e16420 100644 --- a/rust/operator-binary/src/controller/build/properties/hbase_site.rs +++ b/rust/operator-binary/src/controller/build/properties/hbase_site.rs @@ -183,9 +183,7 @@ mod tests { config_overrides(&[]), ); assert!( - xml.contains( - "hbase.regionserver.ipc.address\n 0.0.0.0" - ), + xml.contains("hbase.regionserver.ipc.address\n 0.0.0.0"), "{xml}" ); assert!( diff --git a/rust/operator-binary/src/controller/build/properties/ssl_server.rs b/rust/operator-binary/src/controller/build/properties/ssl_server.rs index 36da52ea..bd3bd911 100644 --- a/rust/operator-binary/src/controller/build/properties/ssl_server.rs +++ b/rust/operator-binary/src/controller/build/properties/ssl_server.rs @@ -29,10 +29,7 @@ mod tests { #[test] fn settings_appear_in_xml() { let xml = build( - BTreeMap::from([( - "ssl.server.keystore.type".to_string(), - "pkcs12".to_string(), - )]), + BTreeMap::from([("ssl.server.keystore.type".to_string(), "pkcs12".to_string())]), config_overrides(&[]), ); assert!( diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index bdc610da..c44e31b3 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -20,7 +20,8 @@ use crate::{ ValidatedRoleGroupConfig, }, kerberos::{ - self, kerberos_config_properties, kerberos_ssl_client_settings, kerberos_ssl_server_settings, + self, kerberos_config_properties, kerberos_ssl_client_settings, + kerberos_ssl_server_settings, }, }; @@ -130,8 +131,7 @@ pub fn validate_cluster( name: ClusterName::from_str(&hbase.name_any()).context(InvalidClusterNameSnafu)?, image: resolved_product_image, cluster_config: ValidatedClusterConfig { - zookeeper_connection_information: dereferenced_objects - .zookeeper_connection_information, + zookeeper_connection_information: dereferenced_objects.zookeeper_connection_information, hbase_opa_config: dereferenced_objects.hbase_opa_config, kerberos_enabled: hbase.has_kerberos_enabled(), hbase_site_kerberos_config, @@ -334,7 +334,10 @@ spec: let env_overrides = merged_env_overrides(&hbase, &HbaseRole::Master, "default"); - assert_eq!(env_overrides.get("TEST_VAR"), Some(&"MASTER_RG".to_string())); + assert_eq!( + env_overrides.get("TEST_VAR"), + Some(&"MASTER_RG".to_string()) + ); assert_eq!( env_overrides.get("TEST_VAR_FROM_MASTER"), Some(&"MASTER".to_string()) diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index 4023085b..73ac145b 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -1,6 +1,5 @@ use std::collections::BTreeMap; -use crate::config::writer::to_hadoop_xml; use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, @@ -11,6 +10,7 @@ use stackable_operator::{ }; use crate::{ + config::writer::to_hadoop_xml, crd::{HBASE_SITE_XML, HbaseRole, v1alpha1}, hbase_controller::build_recommended_labels, kerberos::{self, kerberos_discovery_config_properties}, diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 41df54be..3492a078 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -345,10 +345,10 @@ pub async fn reconcile_hbase( )?; let rg_configmap = crate::controller::build::config_map::build_rolegroup_config_map( - hbase, &validated_cluster, hbase_role, &rolegroup, + hbase, ) .context(BuildRolegroupConfigMapSnafu)?; let rg_statefulset = build_rolegroup_statefulset( From 86fc9ae475afb6ed95e9c076a023526056aa3d98 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Jun 2026 12:36:39 +0200 Subject: [PATCH 10/56] fix: use builder pattern for discovery configmap --- .../src/{ => controller/build}/discovery.rs | 48 +++++++++---------- .../src/controller/build/mod.rs | 1 + .../src/controller/validate.rs | 7 ++- rust/operator-binary/src/crd/mod.rs | 2 - rust/operator-binary/src/hbase_controller.rs | 20 ++++---- rust/operator-binary/src/main.rs | 1 - 6 files changed, 38 insertions(+), 41 deletions(-) rename rust/operator-binary/src/{ => controller/build}/discovery.rs (60%) diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/controller/build/discovery.rs similarity index 60% rename from rust/operator-binary/src/discovery.rs rename to rust/operator-binary/src/controller/build/discovery.rs index 73ac145b..bf746e25 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/controller/build/discovery.rs @@ -1,20 +1,19 @@ +//! Build the discovery `ConfigMap` for the HbaseCluster. + use std::collections::BTreeMap; use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, - commons::product_image_selection::ResolvedProductImage, k8s_openapi::api::core::v1::ConfigMap, kube::runtime::reflector::ObjectRef, - utils::cluster_info::KubernetesClusterInfo, }; use crate::{ config::writer::to_hadoop_xml, - crd::{HBASE_SITE_XML, HbaseRole, v1alpha1}, - hbase_controller::build_recommended_labels, - kerberos::{self, kerberos_discovery_config_properties}, - zookeeper::ZookeeperConnectionInformation, + controller::build::properties::ConfigFileName, + crd::{HbaseRole, v1alpha1}, + hbase_controller::{ValidatedCluster, build_recommended_labels}, }; type Result = std::result::Result; @@ -36,35 +35,34 @@ pub enum Error { ObjectMeta { source: stackable_operator::builder::meta::Error, }, - - #[snafu(display("failed to add Kerberos discovery"))] - AddKerberosDiscovery { source: kerberos::Error }, } /// Creates a discovery config map containing the `hbase-site.xml` for clients. -pub fn build_discovery_configmap( - hbase: &v1alpha1::HbaseCluster, - cluster_info: &KubernetesClusterInfo, - zookeeper_connection_information: &ZookeeperConnectionInformation, - resolved_product_image: &ResolvedProductImage, +/// +/// The rendered content comes entirely from `cluster`; `owner_ref` is retained only for the +/// ConfigMap ObjectMeta / owner reference. +pub fn build_discovery_config_map( + cluster: &ValidatedCluster, + owner_ref: &v1alpha1::HbaseCluster, ) -> Result { - let mut hbase_site = zookeeper_connection_information.as_hbase_settings(); - hbase_site.extend( - kerberos_discovery_config_properties(hbase, cluster_info) - .context(AddKerberosDiscoverySnafu)?, - ); + let cluster_config = &cluster.cluster_config; + + let mut hbase_site = cluster_config + .zookeeper_connection_information + .as_hbase_settings(); + hbase_site.extend(cluster_config.discovery_kerberos_config.clone()); ConfigMapBuilder::new() .metadata( ObjectMetaBuilder::new() - .name_and_namespace(hbase) - .ownerreference_from_resource(hbase, None, Some(true)) + .name_and_namespace(owner_ref) + .ownerreference_from_resource(owner_ref, None, Some(true)) .with_context(|_| ObjectMissingMetadataForOwnerRefSnafu { - hbase: ObjectRef::from_obj(hbase), + hbase: ObjectRef::from_obj(owner_ref), })? .with_recommended_labels(&build_recommended_labels( - hbase, - &resolved_product_image.app_version_label_value, + owner_ref, + &cluster.image.app_version_label_value, &HbaseRole::RegionServer.to_string(), "discovery", )) @@ -72,7 +70,7 @@ pub fn build_discovery_configmap( .build(), ) .add_data( - HBASE_SITE_XML, + ConfigFileName::HbaseSite.to_string(), to_hadoop_xml( hbase_site .into_iter() diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs index 933a20b9..4a7bba72 100644 --- a/rust/operator-binary/src/controller/build/mod.rs +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -1,2 +1,3 @@ pub mod config_map; +pub mod discovery; pub mod properties; diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index c44e31b3..6d5fdd9a 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -20,8 +20,8 @@ use crate::{ ValidatedRoleGroupConfig, }, kerberos::{ - self, kerberos_config_properties, kerberos_ssl_client_settings, - kerberos_ssl_server_settings, + self, kerberos_config_properties, kerberos_discovery_config_properties, + kerberos_ssl_client_settings, kerberos_ssl_server_settings, }, }; @@ -124,6 +124,8 @@ pub fn validate_cluster( let hbase_site_kerberos_config = kerberos_config_properties(hbase, cluster_info).context(AddKerberosConfigSnafu)?; + let discovery_kerberos_config = kerberos_discovery_config_properties(hbase, cluster_info) + .context(AddKerberosConfigSnafu)?; let ssl_server_settings = kerberos_ssl_server_settings(hbase); let ssl_client_settings = kerberos_ssl_client_settings(hbase); @@ -135,6 +137,7 @@ pub fn validate_cluster( hbase_opa_config: dereferenced_objects.hbase_opa_config, kerberos_enabled: hbase.has_kerberos_enabled(), hbase_site_kerberos_config, + discovery_kerberos_config, ssl_server_settings, ssl_client_settings, }, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 937d24c8..564544d9 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -58,8 +58,6 @@ pub const TLS_STORE_PASSWORD: &str = "changeit"; pub const JVM_SECURITY_PROPERTIES_FILE: &str = "security.properties"; -pub const HBASE_SITE_XML: &str = "hbase-site.xml"; - pub const HBASE_CLUSTER_DISTRIBUTED: &str = "hbase.cluster.distributed"; pub const HBASE_ROOTDIR: &str = "hbase.rootdir"; diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 3492a078..c1282dbe 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -56,11 +56,11 @@ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ OPERATOR_NAME, + controller::build::discovery::build_discovery_config_map, crd::{ APP_NAME, AnyServiceConfig, CONFIG_DIR_NAME, Container, HbaseClusterStatus, HbaseRole, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, merged_env, v1alpha1, }, - discovery::build_discovery_configmap, kerberos::{self, add_kerberos_pod_config}, operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, product_logging::{CONTAINERDEBUG_LOG_DIRECTORY, STACKABLE_LOG_DIR}, @@ -110,6 +110,9 @@ pub struct ValidatedClusterConfig { pub kerberos_enabled: bool, /// Pre-resolved kerberos properties for hbase-site.xml (empty when kerberos is disabled). pub hbase_site_kerberos_config: BTreeMap, + /// Pre-resolved kerberos properties for the discovery `hbase-site.xml` exposed to clients + /// (empty when kerberos is disabled). + pub discovery_kerberos_config: BTreeMap, /// Pre-resolved ssl-server.xml settings (empty when HTTPS is disabled). pub ssl_server_settings: BTreeMap, /// Pre-resolved ssl-client.xml settings (empty when HTTPS is disabled). @@ -161,7 +164,9 @@ pub enum Error { }, #[snafu(display("failed to build discovery configmap"))] - BuildDiscoveryConfigMap { source: super::discovery::Error }, + BuildDiscoveryConfigMap { + source: crate::controller::build::discovery::Error, + }, #[snafu(display("failed to build rolegroup ConfigMap"))] BuildRolegroupConfigMap { @@ -406,15 +411,8 @@ pub async fn reconcile_hbase( // Discovery CM will fail to build until the rest of the cluster has been deployed, so do it last // so that failure won't inhibit the rest of the cluster from booting up. - let discovery_cm = build_discovery_configmap( - hbase, - &client.kubernetes_cluster_info, - &validated_cluster - .cluster_config - .zookeeper_connection_information, - &validated_cluster.image, - ) - .context(BuildDiscoveryConfigMapSnafu)?; + let discovery_cm = build_discovery_config_map(&validated_cluster, hbase) + .context(BuildDiscoveryConfigMapSnafu)?; cluster_resources .add(client, discovery_cm) .await diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 6a97d872..97cfaba3 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -40,7 +40,6 @@ use crate::{ mod config; mod controller; mod crd; -mod discovery; mod hbase_controller; mod kerberos; mod operations; From 6740e26286ece64dfa707b09770856a618fc9d77 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Jun 2026 12:45:09 +0200 Subject: [PATCH 11/56] chore: rebuild hashes --- Cargo.nix | 18 +++++++++--------- crate-hashes.json | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index 55fc2419..cd85e73e 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4832,7 +4832,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "k8s_version"; authors = [ @@ -9513,7 +9513,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_certs"; authors = [ @@ -9721,7 +9721,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_operator"; authors = [ @@ -9906,7 +9906,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -9941,7 +9941,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_shared"; authors = [ @@ -10022,7 +10022,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_telemetry"; authors = [ @@ -10132,7 +10132,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_versioned"; authors = [ @@ -10182,7 +10182,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -10250,7 +10250,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; - sha256 = "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk"; + sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_webhook"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index 5564a89e..c76bf06c 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.3": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "0lj969rjbxairjglrnaq0xhabvdrq5nd6wl1i0y9pr50nhh7zvgk", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.0": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.3": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file From 1cd040198c9ba3cca6010d91f9562c437508b73b Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Jun 2026 12:45:28 +0200 Subject: [PATCH 12/56] fix: remove result from product logging --- .../src/controller/build/config_map.rs | 12 +------ rust/operator-binary/src/product_logging.rs | 31 +------------------ 2 files changed, 2 insertions(+), 41 deletions(-) diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 3d9103d9..17bb3e94 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -41,12 +41,6 @@ pub enum Error { source: stackable_operator::builder::meta::Error, }, - #[snafu(display("failed to add the logging configuration to the ConfigMap {cm_name:?}"))] - InvalidLoggingConfig { - source: crate::product_logging::Error, - cm_name: String, - }, - #[snafu(display("cannot build config map for role {role:?} and role group {role_group:?}"))] Assemble { source: stackable_operator::builder::configmap::Error, @@ -145,11 +139,7 @@ pub fn build_rolegroup_config_map( builder.add_data(ConfigFileName::SslClient.to_string(), ssl_client_xml); } - extend_role_group_config_map(rolegroup_ref, rg.merged_config.logging(), &mut builder).context( - InvalidLoggingConfigSnafu { - cm_name: rolegroup_ref.object_name(), - }, - )?; + extend_role_group_config_map(rolegroup_ref, rg.merged_config.logging(), &mut builder); builder.build().with_context(|_| AssembleSnafu { role: rolegroup_ref.role.clone(), diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs index 61db3245..0ba89a87 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/product_logging.rs @@ -1,4 +1,3 @@ -use snafu::Snafu; use stackable_operator::{ builder::configmap::ConfigMapBuilder, memory::BinaryMultiple, @@ -16,32 +15,6 @@ use crate::{ hbase_controller::MAX_HBASE_LOG_FILES_SIZE, }; -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("object has no namespace"))] - ObjectHasNoNamespace, - - #[snafu(display("failed to retrieve the ConfigMap [{cm_name}]"))] - ConfigMapNotFound { - source: stackable_operator::client::Error, - cm_name: String, - }, - - #[snafu(display("failed to retrieve the entry [{entry}] for ConfigMap [{cm_name}]"))] - MissingConfigMapEntry { - entry: &'static str, - cm_name: String, - }, - - #[snafu(display("crd validation failure"))] - CrdValidationFailure { source: crate::crd::Error }, - - #[snafu(display("vectorAggregatorConfigMapName must be set"))] - MissingVectorAggregatorAddress, -} - -type Result = std::result::Result; - const CONSOLE_CONVERSION_PATTERN: &str = "%d{ISO8601} %-5p [%t] %c{2}: %.1000m%n"; const HBASE_LOG4J2_FILE: &str = "hbase.log4j2.xml"; pub const LOG4J2_CONFIG_FILE: &str = "log4j2.properties"; @@ -54,7 +27,7 @@ pub fn extend_role_group_config_map( rolegroup: &RoleGroupRef, logging: &Logging, cm_builder: &mut ConfigMapBuilder, -) -> Result<()> { +) { if let Some(ContainerLogConfig { choice: Some(ContainerLogConfigChoice::Automatic(log_config)), }) = logging.containers.get(&Container::Hbase) @@ -77,8 +50,6 @@ pub fn extend_role_group_config_map( product_logging::framework::create_vector_config(rolegroup, vector_log_config), ); } - - Ok(()) } fn log4j_config(log_config: &AutomaticContainerLogConfig) -> String { From 1027e54ac965a911646ae841d8fd21e18ffd7ce5 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Jun 2026 12:52:40 +0200 Subject: [PATCH 13/56] fix: move product logging to build step --- rust/operator-binary/src/controller/build/config_map.rs | 4 ++-- .../build/properties/logging.rs} | 9 +++------ .../src/controller/build/properties/mod.rs | 4 ++++ rust/operator-binary/src/hbase_controller.rs | 5 ++++- rust/operator-binary/src/main.rs | 1 - 5 files changed, 13 insertions(+), 10 deletions(-) rename rust/operator-binary/src/{product_logging.rs => controller/build/properties/logging.rs} (82%) diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 17bb3e94..34b4476b 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -10,11 +10,11 @@ use stackable_operator::{ use crate::{ config::writer::PropertiesWriterError, controller::build::properties::{ - ConfigFileName, hbase_env, hbase_site, security_properties, ssl_client, ssl_server, + ConfigFileName, hbase_env, hbase_site, logging::extend_role_group_config_map, + security_properties, ssl_client, ssl_server, }, crd::{HbaseRole, v1alpha1}, hbase_controller::{ValidatedCluster, build_recommended_labels}, - product_logging::extend_role_group_config_map, }; #[derive(Snafu, Debug)] diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/controller/build/properties/logging.rs similarity index 82% rename from rust/operator-binary/src/product_logging.rs rename to rust/operator-binary/src/controller/build/properties/logging.rs index 0ba89a87..9a2d5828 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/controller/build/properties/logging.rs @@ -11,16 +11,13 @@ use stackable_operator::{ }; use crate::{ + controller::build::properties::ConfigFileName, crd::{Container, v1alpha1}, - hbase_controller::MAX_HBASE_LOG_FILES_SIZE, + hbase_controller::{MAX_HBASE_LOG_FILES_SIZE, STACKABLE_LOG_DIR}, }; const CONSOLE_CONVERSION_PATTERN: &str = "%d{ISO8601} %-5p [%t] %c{2}: %.1000m%n"; const HBASE_LOG4J2_FILE: &str = "hbase.log4j2.xml"; -pub const LOG4J2_CONFIG_FILE: &str = "log4j2.properties"; -pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; -pub static CONTAINERDEBUG_LOG_DIRECTORY: std::sync::LazyLock = - std::sync::LazyLock::new(|| format!("{STACKABLE_LOG_DIR}/containerdebug")); /// Extend the role group ConfigMap with logging and Vector configurations pub fn extend_role_group_config_map( @@ -32,7 +29,7 @@ pub fn extend_role_group_config_map( choice: Some(ContainerLogConfigChoice::Automatic(log_config)), }) = logging.containers.get(&Container::Hbase) { - cm_builder.add_data(LOG4J2_CONFIG_FILE, log4j_config(log_config)); + cm_builder.add_data(ConfigFileName::Log4j2.to_string(), log4j_config(log_config)); } let vector_log_config = if let Some(ContainerLogConfig { diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index 31c7af24..db919037 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -9,6 +9,7 @@ use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; pub mod hbase_env; pub mod hbase_site; +pub mod logging; pub mod security_properties; pub mod ssl_client; pub mod ssl_server; @@ -43,6 +44,8 @@ pub enum ConfigFileName { SslClient, #[strum(serialize = "security.properties")] Security, + #[strum(serialize = "log4j2.properties")] + Log4j2, } #[cfg(test)] @@ -56,6 +59,7 @@ mod tests { assert_eq!(ConfigFileName::SslServer.to_string(), "ssl-server.xml"); assert_eq!(ConfigFileName::SslClient.to_string(), "ssl-client.xml"); assert_eq!(ConfigFileName::Security.to_string(), "security.properties"); + assert_eq!(ConfigFileName::Log4j2.to_string(), "log4j2.properties"); } } diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index c1282dbe..b6c5c9bd 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -63,7 +63,6 @@ use crate::{ }, kerberos::{self, add_kerberos_pod_config}, operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, - product_logging::{CONTAINERDEBUG_LOG_DIRECTORY, STACKABLE_LOG_DIR}, security::opa::HbaseOpaConfig, zookeeper::ZookeeperConnectionInformation, }; @@ -75,6 +74,10 @@ pub const MAX_HBASE_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { unit: BinaryMultiple::Mebi, }; +pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; +pub static CONTAINERDEBUG_LOG_DIRECTORY: std::sync::LazyLock = + std::sync::LazyLock::new(|| format!("{STACKABLE_LOG_DIR}/containerdebug")); + // These constants are hard coded in hbase-entrypoint.sh // You need to change them there too. const HDFS_DISCOVERY_TMP_DIR: &str = "/stackable/tmp/hdfs"; diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 97cfaba3..2bd815b8 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -43,7 +43,6 @@ mod crd; mod hbase_controller; mod kerberos; mod operations; -mod product_logging; mod security; mod webhooks; mod zookeeper; From 0fe526b11439f1a2243dc9b4111934fe0841e083 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Jun 2026 13:01:55 +0200 Subject: [PATCH 14/56] fix: move constants --- .../controller/build/properties/logging.rs | 9 +++++-- .../src/controller/validate.rs | 5 ++-- rust/operator-binary/src/hbase_controller.rs | 27 +++++++++---------- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/rust/operator-binary/src/controller/build/properties/logging.rs b/rust/operator-binary/src/controller/build/properties/logging.rs index 9a2d5828..93195534 100644 --- a/rust/operator-binary/src/controller/build/properties/logging.rs +++ b/rust/operator-binary/src/controller/build/properties/logging.rs @@ -1,6 +1,6 @@ use stackable_operator::{ builder::configmap::ConfigMapBuilder, - memory::BinaryMultiple, + memory::{BinaryMultiple, MemoryQuantity}, product_logging::{ self, spec::{ @@ -13,7 +13,12 @@ use stackable_operator::{ use crate::{ controller::build::properties::ConfigFileName, crd::{Container, v1alpha1}, - hbase_controller::{MAX_HBASE_LOG_FILES_SIZE, STACKABLE_LOG_DIR}, +}; + +pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; +pub const MAX_HBASE_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { + value: 10.0, + unit: BinaryMultiple::Mebi, }; const CONSOLE_CONVERSION_PATTERN: &str = "%d{ISO8601} %-5p [%t] %c{2}: %.1000m%n"; diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 6d5fdd9a..b4a8cba2 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -16,8 +16,7 @@ use crate::{ controller::dereference::DereferencedObjects, crd::{HbaseRole, v1alpha1}, hbase_controller::{ - CONTAINER_IMAGE_BASE_NAME, ValidatedCluster, ValidatedClusterConfig, ValidatedRoleConfig, - ValidatedRoleGroupConfig, + ValidatedCluster, ValidatedClusterConfig, ValidatedRoleConfig, ValidatedRoleGroupConfig, }, kerberos::{ self, kerberos_config_properties, kerberos_discovery_config_properties, @@ -25,6 +24,8 @@ use crate::{ }, }; +const CONTAINER_IMAGE_BASE_NAME: &str = "hbase"; + #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("failed to resolve product image"))] diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index b6c5c9bd..c57f3f52 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -35,7 +35,6 @@ use stackable_operator::{ }, kvp::{Annotations, Label, LabelError, Labels, ObjectLabels}, logging::controller::ReconcilerError, - memory::{BinaryMultiple, MemoryQuantity}, product_logging::{ self, framework::LoggingError, @@ -56,7 +55,10 @@ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ OPERATOR_NAME, - controller::build::discovery::build_discovery_config_map, + controller::build::{ + discovery::build_discovery_config_map, + properties::logging::{MAX_HBASE_LOG_FILES_SIZE, STACKABLE_LOG_DIR}, + }, crd::{ APP_NAME, AnyServiceConfig, CONFIG_DIR_NAME, Container, HbaseClusterStatus, HbaseRole, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, merged_env, v1alpha1, @@ -69,12 +71,7 @@ use crate::{ pub const HBASE_CONTROLLER_NAME: &str = "hbasecluster"; pub const FULL_HBASE_CONTROLLER_NAME: &str = concatcp!(HBASE_CONTROLLER_NAME, '.', OPERATOR_NAME); -pub const MAX_HBASE_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { - value: 10.0, - unit: BinaryMultiple::Mebi, -}; -pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; pub static CONTAINERDEBUG_LOG_DIRECTORY: std::sync::LazyLock = std::sync::LazyLock::new(|| format!("{STACKABLE_LOG_DIR}/containerdebug")); @@ -84,8 +81,6 @@ const HDFS_DISCOVERY_TMP_DIR: &str = "/stackable/tmp/hdfs"; const HBASE_CONFIG_TMP_DIR: &str = "/stackable/tmp/hbase"; const HBASE_LOG_CONFIG_TMP_DIR: &str = "/stackable/tmp/log_config"; -pub const CONTAINER_IMAGE_BASE_NAME: &str = "hbase"; - pub struct Ctx { pub client: stackable_operator::client::Client, pub operator_environment: OperatorEnvironmentOptions, @@ -108,18 +103,20 @@ pub struct ValidatedCluster { /// Cluster-wide settings resolved once during validation. #[derive(Clone, Debug)] pub struct ValidatedClusterConfig { - pub zookeeper_connection_information: ZookeeperConnectionInformation, + // Pre-resolved OPA connection configuration. pub hbase_opa_config: Option, pub kerberos_enabled: bool, - /// Pre-resolved kerberos properties for hbase-site.xml (empty when kerberos is disabled). + // Pre-resolved kerberos properties for hbase-site.xml (empty when kerberos is disabled). pub hbase_site_kerberos_config: BTreeMap, - /// Pre-resolved kerberos properties for the discovery `hbase-site.xml` exposed to clients - /// (empty when kerberos is disabled). + // Pre-resolved kerberos properties for the discovery `hbase-site.xml` exposed to clients + // (empty when kerberos is disabled). pub discovery_kerberos_config: BTreeMap, - /// Pre-resolved ssl-server.xml settings (empty when HTTPS is disabled). + // Pre-resolved ssl-server.xml settings (empty when HTTPS is disabled). pub ssl_server_settings: BTreeMap, - /// Pre-resolved ssl-client.xml settings (empty when HTTPS is disabled). + // Pre-resolved ssl-client.xml settings (empty when HTTPS is disabled). pub ssl_client_settings: BTreeMap, + // Pre-resolved zookeeper connection settings. + pub zookeeper_connection_information: ZookeeperConnectionInformation, } /// Per-role configuration extracted during validation. From f69955951421eade4f310b145012ad6a7d8ae7e8 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Jun 2026 13:09:25 +0200 Subject: [PATCH 15/56] refactor: consolidate logging to build step --- .../src/controller/build/config_map.rs | 13 ++++- .../controller/build/properties/logging.rs | 58 ++++++++++--------- 2 files changed, 41 insertions(+), 30 deletions(-) diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 34b4476b..374a650e 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -4,14 +4,14 @@ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, k8s_openapi::api::core::v1::ConfigMap, + product_logging::framework::VECTOR_CONFIG_FILE, role_utils::RoleGroupRef, }; use crate::{ config::writer::PropertiesWriterError, controller::build::properties::{ - ConfigFileName, hbase_env, hbase_site, logging::extend_role_group_config_map, - security_properties, ssl_client, ssl_server, + ConfigFileName, hbase_env, hbase_site, logging, security_properties, ssl_client, ssl_server, }, crd::{HbaseRole, v1alpha1}, hbase_controller::{ValidatedCluster, build_recommended_labels}, @@ -139,7 +139,14 @@ pub fn build_rolegroup_config_map( builder.add_data(ConfigFileName::SslClient.to_string(), ssl_client_xml); } - extend_role_group_config_map(rolegroup_ref, rg.merged_config.logging(), &mut builder); + if let Some(log4j2_properties) = logging::build_log4j2(rg.merged_config.logging()) { + builder.add_data(ConfigFileName::Log4j2.to_string(), log4j2_properties); + } + if let Some(vector_config) = + logging::build_vector_config(rolegroup_ref, rg.merged_config.logging()) + { + builder.add_data(VECTOR_CONFIG_FILE, vector_config); + } builder.build().with_context(|_| AssembleSnafu { role: rolegroup_ref.role.clone(), diff --git a/rust/operator-binary/src/controller/build/properties/logging.rs b/rust/operator-binary/src/controller/build/properties/logging.rs index 93195534..5e105d24 100644 --- a/rust/operator-binary/src/controller/build/properties/logging.rs +++ b/rust/operator-binary/src/controller/build/properties/logging.rs @@ -1,5 +1,4 @@ use stackable_operator::{ - builder::configmap::ConfigMapBuilder, memory::{BinaryMultiple, MemoryQuantity}, product_logging::{ self, @@ -10,10 +9,7 @@ use stackable_operator::{ role_utils::RoleGroupRef, }; -use crate::{ - controller::build::properties::ConfigFileName, - crd::{Container, v1alpha1}, -}; +use crate::crd::{Container, v1alpha1}; pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; pub const MAX_HBASE_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { @@ -24,34 +20,42 @@ pub const MAX_HBASE_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { const CONSOLE_CONVERSION_PATTERN: &str = "%d{ISO8601} %-5p [%t] %c{2}: %.1000m%n"; const HBASE_LOG4J2_FILE: &str = "hbase.log4j2.xml"; -/// Extend the role group ConfigMap with logging and Vector configurations -pub fn extend_role_group_config_map( +/// Renders `log4j2.properties` for the HBase container. +/// +/// Returns `None` when the HBase container does not use the operator's automatic logging +/// configuration (e.g. a custom log ConfigMap is referenced instead), in which case no +/// `log4j2.properties` should be added to the rolegroup `ConfigMap`. +pub fn build_log4j2(logging: &Logging) -> Option { + match logging.containers.get(&Container::Hbase) { + Some(ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Automatic(log_config)), + }) => Some(log4j_config(log_config)), + _ => None, + } +} + +/// Renders the Vector agent config (`vector.yaml`). +/// +/// Returns `None` when the Vector agent is disabled for this role group. +pub fn build_vector_config( rolegroup: &RoleGroupRef, logging: &Logging, - cm_builder: &mut ConfigMapBuilder, -) { - if let Some(ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic(log_config)), - }) = logging.containers.get(&Container::Hbase) - { - cm_builder.add_data(ConfigFileName::Log4j2.to_string(), log4j_config(log_config)); +) -> Option { + if !logging.enable_vector_agent { + return None; } - let vector_log_config = if let Some(ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic(log_config)), - }) = logging.containers.get(&Container::Vector) - { - Some(log_config) - } else { - None + let vector_log_config = match logging.containers.get(&Container::Vector) { + Some(ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Automatic(log_config)), + }) => Some(log_config), + _ => None, }; - if logging.enable_vector_agent { - cm_builder.add_data( - product_logging::framework::VECTOR_CONFIG_FILE, - product_logging::framework::create_vector_config(rolegroup, vector_log_config), - ); - } + Some(product_logging::framework::create_vector_config( + rolegroup, + vector_log_config, + )) } fn log4j_config(log_config: &AutomaticContainerLogConfig) -> String { From 70b20e3a80181ca82d0bd0196dd0201de8867ad3 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Jun 2026 13:11:35 +0200 Subject: [PATCH 16/56] chore: adapted changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0942934b..a6c906d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,12 +16,15 @@ - Internal operator refactoring: introduce dereference() and validate() steps in the reconciler ([#757]). - Removed the `product-config` dependency; configuration is now rendered from typed, validated inputs ([#757]). - test: Bump vector-aggregator to 0.55.0, replace /graphql call with gRPC call ([#762]). +- BREAKING: Removed product-config machinery. This is a breaking change in terms of configuration. + Users relying on the product-config `properties.yaml` file have to set these properties via the CRD ([#765]). [#745]: https://github.com/stackabletech/hbase-operator/pull/745 [#751]: https://github.com/stackabletech/hbase-operator/pull/751 [#752]: https://github.com/stackabletech/hbase-operator/pull/752 [#757]: https://github.com/stackabletech/hbase-operator/pull/757 [#762]: https://github.com/stackabletech/hbase-operator/pull/762 +[#765]: https://github.com/stackabletech/hbase-operator/pull/765 ## [26.3.0] - 2026-03-16 From aec271a1975ccc216800fc96c330078d0b568c5d Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 4 Jun 2026 15:42:28 +0200 Subject: [PATCH 17/56] chore: replace json_pp with jq in getting started scripts --- .../hbase/examples/getting_started/getting_started.sh | 6 +++--- .../hbase/examples/getting_started/getting_started.sh.j2 | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/modules/hbase/examples/getting_started/getting_started.sh b/docs/modules/hbase/examples/getting_started/getting_started.sh index 319e36c8..580abc75 100755 --- a/docs/modules/hbase/examples/getting_started/getting_started.sh +++ b/docs/modules/hbase/examples/getting_started/getting_started.sh @@ -135,7 +135,7 @@ version() { echo "Check cluster status..." # tag::cluster-status[] kubectl exec -n default simple-hbase-restserver-default-0 \ --- curl -s -XGET -H "Accept: application/json" "http://simple-hbase-restserver-default-headless:8080/status/cluster" | json_pp +-- curl -s -XGET -H "Accept: application/json" "http://simple-hbase-restserver-default-headless:8080/status/cluster" | jq # end::cluster-status[] echo "Check table via REST API..." @@ -148,13 +148,13 @@ kubectl exec -n default simple-hbase-restserver-default-0 \ # tag::get-table[] kubectl exec -n default simple-hbase-restserver-default-0 \ --- curl -s -XGET -H "Accept: application/json" "http://simple-hbase-restserver-default-headless:8080/users/schema" | json_pp +-- curl -s -XGET -H "Accept: application/json" "http://simple-hbase-restserver-default-headless:8080/users/schema" | jq # end::get-table[] get_all() { # tag::get-tables[] kubectl exec -n default simple-hbase-restserver-default-0 \ - -- curl -s -XGET -H "Accept: application/json" "http://simple-hbase-restserver-default-headless:8080/" | json_pp + -- curl -s -XGET -H "Accept: application/json" "http://simple-hbase-restserver-default-headless:8080/" | jq # end::get-tables[] } diff --git a/docs/modules/hbase/examples/getting_started/getting_started.sh.j2 b/docs/modules/hbase/examples/getting_started/getting_started.sh.j2 index fefca39d..6852176c 100755 --- a/docs/modules/hbase/examples/getting_started/getting_started.sh.j2 +++ b/docs/modules/hbase/examples/getting_started/getting_started.sh.j2 @@ -135,7 +135,7 @@ version() { echo "Check cluster status..." # tag::cluster-status[] kubectl exec -n default simple-hbase-restserver-default-0 \ --- curl -s -XGET -H "Accept: application/json" "http://simple-hbase-restserver-default-headless:8080/status/cluster" | json_pp +-- curl -s -XGET -H "Accept: application/json" "http://simple-hbase-restserver-default-headless:8080/status/cluster" | jq # end::cluster-status[] echo "Check table via REST API..." @@ -148,13 +148,13 @@ kubectl exec -n default simple-hbase-restserver-default-0 \ # tag::get-table[] kubectl exec -n default simple-hbase-restserver-default-0 \ --- curl -s -XGET -H "Accept: application/json" "http://simple-hbase-restserver-default-headless:8080/users/schema" | json_pp +-- curl -s -XGET -H "Accept: application/json" "http://simple-hbase-restserver-default-headless:8080/users/schema" | jq # end::get-table[] get_all() { # tag::get-tables[] kubectl exec -n default simple-hbase-restserver-default-0 \ - -- curl -s -XGET -H "Accept: application/json" "http://simple-hbase-restserver-default-headless:8080/" | json_pp + -- curl -s -XGET -H "Accept: application/json" "http://simple-hbase-restserver-default-headless:8080/" | jq # end::get-tables[] } From 8b8c10447f1b3796b547ccd2c6cd297b759ad2be Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 4 Jun 2026 19:28:19 +0200 Subject: [PATCH 18/56] add snapshot tests and make them succeed --- extra/crds.yaml | 4 + .../controller/build/properties/hbase_site.rs | 4 +- .../controller/build/properties/ssl_client.rs | 10 +- .../controller/build/properties/ssl_server.rs | 10 +- rust/operator-binary/src/crd/mod.rs | 26 +++-- tests/templates/kuttl/smoke/32-assert.yaml | 40 +++++++ .../32_cm-test-hbase-master-default.yaml.j2 | 105 ++++++++++++++++++ ...cm-test-hbase-regionserver-default.yaml.j2 | 105 ++++++++++++++++++ ...2_cm-test-hbase-restserver-default.yaml.j2 | 85 ++++++++++++++ 9 files changed, 358 insertions(+), 31 deletions(-) create mode 100644 tests/templates/kuttl/smoke/32-assert.yaml create mode 100644 tests/templates/kuttl/smoke/32_cm-test-hbase-master-default.yaml.j2 create mode 100644 tests/templates/kuttl/smoke/32_cm-test-hbase-regionserver-default.yaml.j2 create mode 100644 tests/templates/kuttl/smoke/32_cm-test-hbase-restserver-default.yaml.j2 diff --git a/extra/crds.yaml b/extra/crds.yaml index 80a33e12..13eb43b5 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -242,6 +242,7 @@ spec: nullable: true type: string hbaseRootdir: + description: Root directory for Hbase on the filesystem (usually a path in HDFS). Default is `/hbase`. nullable: true type: string listenerClass: @@ -687,6 +688,7 @@ spec: nullable: true type: string hbaseRootdir: + description: Root directory for Hbase on the filesystem (usually a path in HDFS). Default is `/hbase`. nullable: true type: string listenerClass: @@ -2054,6 +2056,7 @@ spec: nullable: true type: string hbaseRootdir: + description: Root directory for Hbase on the filesystem (usually a path in HDFS). Default is `/hbase`. nullable: true type: string listenerClass: @@ -2499,6 +2502,7 @@ spec: nullable: true type: string hbaseRootdir: + description: Root directory for Hbase on the filesystem (usually a path in HDFS). Default is `/hbase`. nullable: true type: string listenerClass: diff --git a/rust/operator-binary/src/controller/build/properties/hbase_site.rs b/rust/operator-binary/src/controller/build/properties/hbase_site.rs index 39e16420..3d7eea78 100644 --- a/rust/operator-binary/src/controller/build/properties/hbase_site.rs +++ b/rust/operator-binary/src/controller/build/properties/hbase_site.rs @@ -30,9 +30,7 @@ pub fn build( // Defaults previously injected by product-config's `compute_files`. config.insert(HBASE_CLUSTER_DISTRIBUTED.to_string(), "true".to_string()); - if let Some(rootdir) = merged_config.hbase_rootdir() { - config.insert(HBASE_ROOTDIR.to_string(), rootdir); - } + config.insert(HBASE_ROOTDIR.to_string(), merged_config.hbase_rootdir()); config.extend(zookeeper_config); config.extend(kerberos_config); diff --git a/rust/operator-binary/src/controller/build/properties/ssl_client.rs b/rust/operator-binary/src/controller/build/properties/ssl_client.rs index c6db3f50..83d0039d 100644 --- a/rust/operator-binary/src/controller/build/properties/ssl_client.rs +++ b/rust/operator-binary/src/controller/build/properties/ssl_client.rs @@ -5,14 +5,11 @@ use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; use crate::{config::writer::to_hadoop_xml, controller::build::properties::resolved_overrides}; -/// Renders `ssl-client.xml`. Returns "" (HBase rejects empty XML files) when empty. +/// Renders `ssl-client.xml`. pub fn build(settings: BTreeMap, overrides: KeyValueConfigOverrides) -> String { let mut config: BTreeMap> = BTreeMap::new(); config.extend(settings.into_iter().map(|(k, v)| (k, Some(v)))); config.extend(resolved_overrides(overrides).map(|(k, v)| (k, Some(v)))); - if config.is_empty() { - return String::new(); - } to_hadoop_xml(config.iter()) } @@ -21,11 +18,6 @@ mod tests { use super::*; use crate::controller::build::properties::test_support::config_overrides; - #[test] - fn empty_settings_without_overrides_renders_empty_string() { - assert_eq!(build(BTreeMap::new(), config_overrides(&[])), ""); - } - #[test] fn settings_appear_in_xml() { let xml = build( diff --git a/rust/operator-binary/src/controller/build/properties/ssl_server.rs b/rust/operator-binary/src/controller/build/properties/ssl_server.rs index bd3bd911..2ae17e9b 100644 --- a/rust/operator-binary/src/controller/build/properties/ssl_server.rs +++ b/rust/operator-binary/src/controller/build/properties/ssl_server.rs @@ -5,14 +5,11 @@ use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; use crate::{config::writer::to_hadoop_xml, controller::build::properties::resolved_overrides}; -/// Renders `ssl-server.xml`. Returns "" (HBase rejects empty XML files) when empty. +/// Renders `ssl-server.xml`. pub fn build(settings: BTreeMap, overrides: KeyValueConfigOverrides) -> String { let mut config: BTreeMap> = BTreeMap::new(); config.extend(settings.into_iter().map(|(k, v)| (k, Some(v)))); config.extend(resolved_overrides(overrides).map(|(k, v)| (k, Some(v)))); - if config.is_empty() { - return String::new(); - } to_hadoop_xml(config.iter()) } @@ -21,11 +18,6 @@ mod tests { use super::*; use crate::controller::build::properties::test_support::config_overrides; - #[test] - fn empty_settings_without_overrides_renders_empty_string() { - assert_eq!(build(BTreeMap::new(), config_overrides(&[])), ""); - } - #[test] fn settings_appear_in_xml() { let xml = build( diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 564544d9..1d1a5036 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -60,6 +60,7 @@ pub const JVM_SECURITY_PROPERTIES_FILE: &str = "security.properties"; pub const HBASE_CLUSTER_DISTRIBUTED: &str = "hbase.cluster.distributed"; pub const HBASE_ROOTDIR: &str = "hbase.rootdir"; +const DEFAULT_HBASE_ROOTDIR: &str = "/hbase"; const HBASE_UI_PORT_NAME_HTTP: &str = "ui-http"; const HBASE_UI_PORT_NAME_HTTPS: &str = "ui-https"; @@ -87,6 +88,10 @@ const DEFAULT_REGION_MOVER_DELTA_TO_SHUTDOWN: Duration = Duration::from_minutes_ const DEFAULT_LISTENER_CLASS: &str = "cluster-internal"; +fn default_hbase_rootdir() -> String { + DEFAULT_HBASE_ROOTDIR.to_string() +} + pub type MasterRoleType = Role; @@ -593,7 +598,7 @@ impl HbaseRole { }; HbaseConfigFragment { - hbase_rootdir: None, + hbase_rootdir: Some(default_hbase_rootdir()), resources, logging: product_logging::spec::default_logging(), affinity: get_affinity(cluster_name, self, hdfs_discovery_cm_name), @@ -796,7 +801,7 @@ impl AnyConfigFragment { match role { HbaseRole::RegionServer => { AnyConfigFragment::RegionServer(RegionServerConfigFragment { - hbase_rootdir: None, + hbase_rootdir: Some(default_hbase_rootdir()), resources: default_resources(role), logging: product_logging::spec::default_logging(), affinity: get_affinity(cluster_name, role, hdfs_discovery_cm_name), @@ -814,7 +819,7 @@ impl AnyConfigFragment { }) } HbaseRole::RestServer => AnyConfigFragment::RestServer(HbaseConfigFragment { - hbase_rootdir: None, + hbase_rootdir: Some(default_hbase_rootdir()), resources: default_resources(role), logging: product_logging::spec::default_logging(), affinity: get_affinity(cluster_name, role, hdfs_discovery_cm_name), @@ -825,7 +830,7 @@ impl AnyConfigFragment { listener_class: Some(DEFAULT_LISTENER_CLASS.to_string()), }), HbaseRole::Master => AnyConfigFragment::Master(HbaseConfigFragment { - hbase_rootdir: None, + hbase_rootdir: Some(default_hbase_rootdir()), resources: default_resources(role), logging: product_logging::spec::default_logging(), affinity: get_affinity(cluster_name, role, hdfs_discovery_cm_name), @@ -891,8 +896,9 @@ pub enum Container { serde(rename_all = "camelCase") )] pub struct HbaseConfig { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub hbase_rootdir: Option, + /// Root directory for Hbase on the filesystem (usually a path in HDFS). Default is `/hbase`. + #[serde(default = "default_hbase_rootdir")] + pub hbase_rootdir: String, #[fragment_attrs(serde(default))] pub resources: Resources, @@ -971,8 +977,8 @@ impl Atomic for RegionMoverExtraCliOpts {} serde(rename_all = "camelCase") )] pub struct RegionServerConfig { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub hbase_rootdir: Option, + #[serde(default = "default_hbase_rootdir")] + pub hbase_rootdir: String, #[fragment_attrs(serde(default))] pub resources: Resources, #[fragment_attrs(serde(default))] @@ -1064,9 +1070,9 @@ impl AnyServiceConfig { } } - /// The configured `hbase.rootdir`, if any. Previously injected into + /// The configured `hbase.rootdir`. Previously injected into /// `hbase-site.xml` via product-config's `compute_files`. - pub fn hbase_rootdir(&self) -> Option { + pub fn hbase_rootdir(&self) -> String { match self { AnyServiceConfig::Master(config) => config.hbase_rootdir.clone(), AnyServiceConfig::RegionServer(config) => config.hbase_rootdir.clone(), diff --git a/tests/templates/kuttl/smoke/32-assert.yaml b/tests/templates/kuttl/smoke/32-assert.yaml new file mode 100644 index 00000000..230ffd91 --- /dev/null +++ b/tests/templates/kuttl/smoke/32-assert.yaml @@ -0,0 +1,40 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 60 +commands: + # + # ConfigMap data snapshot: test-hbase-master-default + # + - script: | + export ZNODE_PATH=$(kubectl -n $NAMESPACE get zookeeperznode test-znode -o jsonpath='{.status.znodePath}') + envsubst '$NAMESPACE $ZNODE_PATH' < 32_cm-test-hbase-master-default.yaml > /tmp/$NAMESPACE-expected.yaml + kubectl -n $NAMESPACE get cm test-hbase-master-default -o yaml | yq '.data' > /tmp/$NAMESPACE-actual.yaml + if ! diff /tmp/$NAMESPACE-expected.yaml /tmp/$NAMESPACE-actual.yaml; then + echo "ERROR: ConfigMap test-hbase-master-default data drifted from snapshot." + exit 1 + fi + + # + # ConfigMap data snapshot: test-hbase-regionserver-default + # + - script: | + export ZNODE_PATH=$(kubectl -n $NAMESPACE get zookeeperznode test-znode -o jsonpath='{.status.znodePath}') + envsubst '$NAMESPACE $ZNODE_PATH' < 32_cm-test-hbase-regionserver-default.yaml > /tmp/$NAMESPACE-expected.yaml + kubectl -n $NAMESPACE get cm test-hbase-regionserver-default -o yaml | yq '.data' > /tmp/$NAMESPACE-actual.yaml + if ! diff /tmp/$NAMESPACE-expected.yaml /tmp/$NAMESPACE-actual.yaml; then + echo "ERROR: ConfigMap test-hbase-regionserver-default data drifted from snapshot." + exit 1 + fi + + # + # ConfigMap data snapshot: test-hbase-restserver-default + # + - script: | + export ZNODE_PATH=$(kubectl -n $NAMESPACE get zookeeperznode test-znode -o jsonpath='{.status.znodePath}') + envsubst '$NAMESPACE $ZNODE_PATH' < 32_cm-test-hbase-restserver-default.yaml > /tmp/$NAMESPACE-expected.yaml + kubectl -n $NAMESPACE get cm test-hbase-restserver-default -o yaml | yq '.data' > /tmp/$NAMESPACE-actual.yaml + if ! diff /tmp/$NAMESPACE-expected.yaml /tmp/$NAMESPACE-actual.yaml; then + echo "ERROR: ConfigMap test-hbase-restserver-default data drifted from snapshot." + exit 1 + fi diff --git a/tests/templates/kuttl/smoke/32_cm-test-hbase-master-default.yaml.j2 b/tests/templates/kuttl/smoke/32_cm-test-hbase-master-default.yaml.j2 new file mode 100644 index 00000000..adadeee8 --- /dev/null +++ b/tests/templates/kuttl/smoke/32_cm-test-hbase-master-default.yaml.j2 @@ -0,0 +1,105 @@ +hbase-env.sh: | + export HBASE_HEAPSIZE="819m" + export HBASE_MANAGES_ZK="false" + export HBASE_MASTER_OPTS="-Djava.security.properties=/stackable/conf/security.properties" + export HBASE_OPTS="" +hbase-site.xml: |- + + + + hbase.client.rpc.bind.address + false + + + hbase.cluster.distributed + true + + + hbase.master.bound.info.port + 16010 + + + hbase.master.hostname + ${env:HBASE_SERVICE_HOST} + + + hbase.master.info.port + ${env:HBASE_INFO_PORT} + + + hbase.master.ipc.address + 0.0.0.0 + + + hbase.master.ipc.port + 16000 + + + hbase.master.port + ${env:HBASE_SERVICE_PORT} + + + hbase.regionserver.wal.codec + org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec + + + hbase.rootdir + /hbase + + + hbase.zookeeper.property.clientPort + 2282 + + + hbase.zookeeper.quorum + test-zk-server.$NAMESPACE.svc.cluster.local:2282 + + + phoenix.log.saltBuckets + 2 + + + zookeeper.znode.parent + $ZNODE_PATH/hbase + + +log4j2.properties: |- + appenders = FILE, CONSOLE + + appender.CONSOLE.type = Console + appender.CONSOLE.name = CONSOLE + appender.CONSOLE.target = SYSTEM_ERR + appender.CONSOLE.layout.type = PatternLayout + appender.CONSOLE.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n + appender.CONSOLE.filter.threshold.type = ThresholdFilter + appender.CONSOLE.filter.threshold.level = INFO + + appender.FILE.type = RollingFile + appender.FILE.name = FILE + appender.FILE.fileName = /stackable/log/hbase/hbase.log4j2.xml + appender.FILE.filePattern = /stackable/log/hbase/hbase.log4j2.xml.%i + appender.FILE.layout.type = XMLLayout + appender.FILE.policies.type = Policies + appender.FILE.policies.size.type = SizeBasedTriggeringPolicy + appender.FILE.policies.size.size = 5MB + appender.FILE.strategy.type = DefaultRolloverStrategy + appender.FILE.strategy.max = 1 + appender.FILE.filter.threshold.type = ThresholdFilter + appender.FILE.filter.threshold.level = INFO + + + rootLogger.level=INFO + rootLogger.appenderRefs = CONSOLE, FILE + rootLogger.appenderRef.CONSOLE.ref = CONSOLE + rootLogger.appenderRef.FILE.ref = FILE +security.properties: | + networkaddress.cache.negative.ttl=0 + networkaddress.cache.ttl=5 +ssl-client.xml: |- + + + +ssl-server.xml: |- + + + diff --git a/tests/templates/kuttl/smoke/32_cm-test-hbase-regionserver-default.yaml.j2 b/tests/templates/kuttl/smoke/32_cm-test-hbase-regionserver-default.yaml.j2 new file mode 100644 index 00000000..5b44d0ca --- /dev/null +++ b/tests/templates/kuttl/smoke/32_cm-test-hbase-regionserver-default.yaml.j2 @@ -0,0 +1,105 @@ +hbase-env.sh: | + export HBASE_HEAPSIZE="819m" + export HBASE_MANAGES_ZK="false" + export HBASE_OPTS="" + export HBASE_REGIONSERVER_OPTS="-Djava.security.properties=/stackable/conf/security.properties" +hbase-site.xml: |- + + + + hbase.client.rpc.bind.address + false + + + hbase.cluster.distributed + true + + + hbase.regionserver.bound.info.port + 16030 + + + hbase.regionserver.info.port + ${env:HBASE_INFO_PORT} + + + hbase.regionserver.ipc.address + 0.0.0.0 + + + hbase.regionserver.ipc.port + 16020 + + + hbase.regionserver.port + ${env:HBASE_SERVICE_PORT} + + + hbase.regionserver.wal.codec + org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec + + + hbase.rootdir + /hbase + + + hbase.unsafe.regionserver.hostname + ${env:HBASE_SERVICE_HOST} + + + hbase.zookeeper.property.clientPort + 2282 + + + hbase.zookeeper.quorum + test-zk-server.$NAMESPACE.svc.cluster.local:2282 + + + phoenix.log.saltBuckets + 2 + + + zookeeper.znode.parent + $ZNODE_PATH/hbase + + +log4j2.properties: |- + appenders = FILE, CONSOLE + + appender.CONSOLE.type = Console + appender.CONSOLE.name = CONSOLE + appender.CONSOLE.target = SYSTEM_ERR + appender.CONSOLE.layout.type = PatternLayout + appender.CONSOLE.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n + appender.CONSOLE.filter.threshold.type = ThresholdFilter + appender.CONSOLE.filter.threshold.level = INFO + + appender.FILE.type = RollingFile + appender.FILE.name = FILE + appender.FILE.fileName = /stackable/log/hbase/hbase.log4j2.xml + appender.FILE.filePattern = /stackable/log/hbase/hbase.log4j2.xml.%i + appender.FILE.layout.type = XMLLayout + appender.FILE.policies.type = Policies + appender.FILE.policies.size.type = SizeBasedTriggeringPolicy + appender.FILE.policies.size.size = 5MB + appender.FILE.strategy.type = DefaultRolloverStrategy + appender.FILE.strategy.max = 1 + appender.FILE.filter.threshold.type = ThresholdFilter + appender.FILE.filter.threshold.level = INFO + + + rootLogger.level=INFO + rootLogger.appenderRefs = CONSOLE, FILE + rootLogger.appenderRef.CONSOLE.ref = CONSOLE + rootLogger.appenderRef.FILE.ref = FILE +security.properties: | + networkaddress.cache.negative.ttl=0 + networkaddress.cache.ttl=10 +ssl-client.xml: |- + + + +ssl-server.xml: |- + + + diff --git a/tests/templates/kuttl/smoke/32_cm-test-hbase-restserver-default.yaml.j2 b/tests/templates/kuttl/smoke/32_cm-test-hbase-restserver-default.yaml.j2 new file mode 100644 index 00000000..c38c12b1 --- /dev/null +++ b/tests/templates/kuttl/smoke/32_cm-test-hbase-restserver-default.yaml.j2 @@ -0,0 +1,85 @@ +hbase-env.sh: | + export HBASE_HEAPSIZE="819m" + export HBASE_MANAGES_ZK="false" + export HBASE_OPTS="" + export HBASE_REST_OPTS="-Djava.security.properties=/stackable/conf/security.properties" +hbase-site.xml: |- + + + + hbase.client.rpc.bind.address + false + + + hbase.cluster.distributed + true + + + hbase.regionserver.wal.codec + org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec + + + hbase.rest.endpoint + ${env:HBASE_SERVICE_HOST}:${env:HBASE_SERVICE_PORT} + + + hbase.rootdir + /hbase + + + hbase.zookeeper.property.clientPort + 2282 + + + hbase.zookeeper.quorum + test-zk-server.$NAMESPACE.svc.cluster.local:2282 + + + phoenix.log.saltBuckets + 2 + + + zookeeper.znode.parent + $ZNODE_PATH/hbase + + +log4j2.properties: |- + appenders = FILE, CONSOLE + + appender.CONSOLE.type = Console + appender.CONSOLE.name = CONSOLE + appender.CONSOLE.target = SYSTEM_ERR + appender.CONSOLE.layout.type = PatternLayout + appender.CONSOLE.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n + appender.CONSOLE.filter.threshold.type = ThresholdFilter + appender.CONSOLE.filter.threshold.level = INFO + + appender.FILE.type = RollingFile + appender.FILE.name = FILE + appender.FILE.fileName = /stackable/log/hbase/hbase.log4j2.xml + appender.FILE.filePattern = /stackable/log/hbase/hbase.log4j2.xml.%i + appender.FILE.layout.type = XMLLayout + appender.FILE.policies.type = Policies + appender.FILE.policies.size.type = SizeBasedTriggeringPolicy + appender.FILE.policies.size.size = 5MB + appender.FILE.strategy.type = DefaultRolloverStrategy + appender.FILE.strategy.max = 1 + appender.FILE.filter.threshold.type = ThresholdFilter + appender.FILE.filter.threshold.level = INFO + + + rootLogger.level=INFO + rootLogger.appenderRefs = CONSOLE, FILE + rootLogger.appenderRef.CONSOLE.ref = CONSOLE + rootLogger.appenderRef.FILE.ref = FILE +security.properties: | + networkaddress.cache.negative.ttl=0 + networkaddress.cache.ttl=30 +ssl-client.xml: |- + + + +ssl-server.xml: |- + + + From 1523e25c87f69b594bcd026ba2563ab783f94462 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 5 Jun 2026 16:08:11 +0200 Subject: [PATCH 19/56] refactor: consume the config-file writer from stackable-operator Replace the vendored java-properties/Hadoop-XML writer (rust/operator-binary/src/config/writer.rs) with stackable_operator::v2::config_file_writer, which hosts the same code (moved there verbatim via operator-rs #1217 on the smooth-operator branch; hbase's copy was byte-identical to hdfs's, the canonical source). Drop the now-unused java-properties and xml dependencies. No behaviour change; rendered output is byte-identical by construction (same code, new home). Co-Authored-By: Claude Opus 4.8 --- Cargo.lock | 88 ++++-- Cargo.nix | 299 ++++++++++-------- Cargo.toml | 2 - crate-hashes.json | 4 +- rust/operator-binary/Cargo.toml | 2 - rust/operator-binary/src/config/mod.rs | 1 - rust/operator-binary/src/config/writer.rs | 145 --------- .../src/controller/build/config_map.rs | 2 +- .../src/controller/build/discovery.rs | 2 +- .../controller/build/properties/hbase_site.rs | 5 +- .../src/controller/build/properties/mod.rs | 4 +- .../build/properties/security_properties.rs | 11 +- .../controller/build/properties/ssl_client.rs | 6 +- .../controller/build/properties/ssl_server.rs | 6 +- 14 files changed, 245 insertions(+), 332 deletions(-) delete mode 100644 rust/operator-binary/src/config/writer.rs diff --git a/Cargo.lock b/Cargo.lock index 814820ac..4b9e0936 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1515,7 +1515,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "darling", "regex", @@ -1840,9 +1840,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "opentelemetry" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" dependencies = [ "futures-core", "futures-sink", @@ -1854,9 +1854,9 @@ dependencies = [ [[package]] name = "opentelemetry-appender-tracing" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef6a1ac5ca3accf562b8c306fa8483c85f4390f768185ab775f242f7fe8fdcc2" +checksum = "2c0080f0dc1d7c786f467cd85a4e395fcab11ee852004f39a29a18ab7c25d837" dependencies = [ "opentelemetry", "tracing", @@ -1866,9 +1866,9 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" +checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331" dependencies = [ "async-trait", "bytes", @@ -1879,9 +1879,9 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" +checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" dependencies = [ "http", "opentelemetry", @@ -1893,14 +1893,14 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tonic", - "tracing", + "tonic-types", ] [[package]] name = "opentelemetry-proto" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" +checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -1911,21 +1911,22 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e62e29dfe041afb8ed2a6c9737ab57db4907285d999ef8ad3a59092a36bdc846" +checksum = "6ca2f98a0437b427b4b08f19f1caa3c44db885a202bc12cfea13d6c702243d68" [[package]] name = "opentelemetry_sdk" -version = "0.31.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" +checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" dependencies = [ "futures-channel", "futures-executor", "futures-util", "opentelemetry", "percent-encoding", + "portable-atomic", "rand 0.9.4", "thiserror 2.0.18", "tokio", @@ -2208,6 +2209,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.45" @@ -2347,9 +2357,9 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" -version = "0.12.28" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64", "bytes", @@ -2365,9 +2375,6 @@ dependencies = [ "log", "percent-encoding", "pin-project-lite", - "serde", - "serde_json", - "serde_urlencoded", "sync_wrapper", "tokio", "tower", @@ -2893,7 +2900,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "const-oid", "ecdsa", @@ -2925,7 +2932,6 @@ dependencies = [ "fnv", "futures 0.3.32", "indoc", - "java-properties", "rstest", "serde", "serde_json", @@ -2936,13 +2942,12 @@ dependencies = [ "strum", "tokio", "tracing", - "xml", ] [[package]] name = "stackable-operator" version = "0.111.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "base64", "clap", @@ -2954,6 +2959,7 @@ dependencies = [ "futures 0.3.32", "http", "indexmap", + "java-properties", "jiff", "json-patch", "k8s-openapi", @@ -2979,12 +2985,13 @@ dependencies = [ "tracing-subscriber", "url", "uuid", + "xml", ] [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "darling", "proc-macro2", @@ -2994,8 +3001,8 @@ dependencies = [ [[package]] name = "stackable-shared" -version = "0.1.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +version = "0.1.1" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "jiff", "k8s-openapi", @@ -3011,8 +3018,8 @@ dependencies = [ [[package]] name = "stackable-telemetry" -version = "0.6.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +version = "0.6.4" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "axum", "clap", @@ -3036,7 +3043,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "kube", "schemars", @@ -3050,7 +3057,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "convert_case", "convert_case_extras", @@ -3068,7 +3075,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "arc-swap", "async-trait", @@ -3420,6 +3427,17 @@ dependencies = [ "tonic", ] +[[package]] +name = "tonic-types" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab1b02061f83d519bba3caa167f88f261ef05720ab8ebc954ade70de3348e8" +dependencies = [ + "prost", + "prost-types", + "tonic", +] + [[package]] name = "tower" version = "0.5.3" @@ -3531,9 +3549,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.32.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac28f2d093c6c477eaa76b23525478f38de514fa9aeb1285738d4b97a9552fc" +checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" dependencies = [ "js-sys", "opentelemetry", diff --git a/Cargo.nix b/Cargo.nix index cd85e73e..5fac81a9 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4831,7 +4831,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "k8s_version"; @@ -6031,9 +6031,9 @@ rec { }; "opentelemetry" = rec { crateName = "opentelemetry"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "18629xsj4rsyiby9aj511q6wcw6s9m09gx3ymw1yjcvix1mcsjxq"; + sha256 = "10ln14d1jgc8rvw97mblc9blzcgpg1bimim4d170b7ia4mijq55h"; dependencies = [ { name = "futures-core"; @@ -6070,24 +6070,24 @@ rec { ]; features = { "default" = [ "trace" "metrics" "logs" "internal-logs" "futures" ]; + "experimental_metrics_bound_instruments" = [ "metrics" ]; "futures" = [ "futures-core" "futures-sink" "pin-project-lite" ]; "futures-core" = [ "dep:futures-core" ]; "futures-sink" = [ "dep:futures-sink" ]; "internal-logs" = [ "tracing" ]; "pin-project-lite" = [ "dep:pin-project-lite" ]; - "spec_unstable_logs_enabled" = [ "logs" ]; "testing" = [ "trace" ]; "thiserror" = [ "dep:thiserror" ]; "trace" = [ "futures" "thiserror" ]; "tracing" = [ "dep:tracing" ]; }; - resolvedDefaultFeatures = [ "default" "futures" "futures-core" "futures-sink" "internal-logs" "logs" "metrics" "pin-project-lite" "spec_unstable_logs_enabled" "thiserror" "trace" "tracing" ]; + resolvedDefaultFeatures = [ "default" "futures" "futures-core" "futures-sink" "internal-logs" "logs" "metrics" "pin-project-lite" "thiserror" "trace" "tracing" ]; }; "opentelemetry-appender-tracing" = rec { crateName = "opentelemetry-appender-tracing"; - version = "0.31.1"; + version = "0.32.0"; edition = "2021"; - sha256 = "1hnwizzgfhpjfnvml638yy846py8hf2gl1n3p1igbk1srb2ilspg"; + sha256 = "0dyq4myan64sl8wly02jx0gb3jjz7575mn3w8rpphz0xvkq8001c"; libName = "opentelemetry_appender_tracing"; dependencies = [ { @@ -6130,18 +6130,15 @@ rec { ]; features = { "experimental_metadata_attributes" = [ "dep:tracing-log" ]; - "experimental_use_tracing_span_context" = [ "tracing-opentelemetry" ]; "log" = [ "dep:log" ]; - "spec_unstable_logs_enabled" = [ "opentelemetry/spec_unstable_logs_enabled" ]; - "tracing-opentelemetry" = [ "dep:tracing-opentelemetry" ]; }; resolvedDefaultFeatures = [ "default" ]; }; "opentelemetry-http" = rec { crateName = "opentelemetry-http"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "0pc5nw1ds8v8w0nvyall39m92v8m1xl1p3vwvxk6nkhrffdd19np"; + sha256 = "0ca3drvm4fx5nskl7yn42dimy3bg35ppzc85y1p27pz215fh30sn"; libName = "opentelemetry_http"; dependencies = [ { @@ -6177,16 +6174,16 @@ rec { "internal-logs" = [ "opentelemetry/internal-logs" ]; "reqwest" = [ "dep:reqwest" ]; "reqwest-blocking" = [ "dep:reqwest" "reqwest/blocking" ]; - "reqwest-rustls" = [ "dep:reqwest" "reqwest/rustls-tls-native-roots" ]; - "reqwest-rustls-webpki-roots" = [ "dep:reqwest" "reqwest/rustls-tls-webpki-roots" ]; + "reqwest-rustls" = [ "dep:reqwest" "reqwest/default-tls" ]; + "reqwest-rustls-webpki-roots" = [ "dep:reqwest" "reqwest/default-tls" "reqwest/webpki-roots" ]; }; - resolvedDefaultFeatures = [ "internal-logs" "reqwest" "reqwest-blocking" ]; + resolvedDefaultFeatures = [ "reqwest" "reqwest-blocking" ]; }; "opentelemetry-otlp" = rec { crateName = "opentelemetry-otlp"; - version = "0.31.1"; + version = "0.32.0"; edition = "2021"; - sha256 = "07zp0b62b9dajnvvcd6j2ppw5zg7wp4ixka9z6fr3bxrrdmcss8z"; + sha256 = "0d9cys2flpidfxbr6h1103hjc633cax47ihnqgbj0xnicscr4rlr"; libName = "opentelemetry_otlp"; dependencies = [ { @@ -6247,10 +6244,9 @@ rec { usesDefaultFeatures = false; } { - name = "tracing"; - packageId = "tracing"; + name = "tonic-types"; + packageId = "tonic-types"; optional = true; - usesDefaultFeatures = false; } ]; devDependencies = [ @@ -6275,16 +6271,19 @@ rec { ]; features = { "default" = [ "http-proto" "reqwest-blocking-client" "trace" "metrics" "logs" "internal-logs" ]; + "experimental-grpc-retry" = [ "grpc-tonic" "opentelemetry_sdk/experimental_async_runtime" "opentelemetry_sdk/rt-tokio" ]; + "experimental-http-retry" = [ "opentelemetry_sdk/experimental_async_runtime" "opentelemetry_sdk/rt-tokio" "tokio" "httpdate" ]; "flate2" = [ "dep:flate2" ]; - "grpc-tonic" = [ "tonic" "prost" "http" "tokio" "opentelemetry-proto/gen-tonic" ]; + "grpc-tonic" = [ "tonic" "tonic-types" "prost" "http" "tokio" "opentelemetry-proto/gen-tonic" ]; "gzip-http" = [ "flate2" ]; "gzip-tonic" = [ "tonic/gzip" ]; "http" = [ "dep:http" ]; "http-json" = [ "serde_json" "prost" "opentelemetry-http" "opentelemetry-proto/gen-tonic-messages" "opentelemetry-proto/with-serde" "http" "trace" "metrics" ]; "http-proto" = [ "prost" "opentelemetry-http" "opentelemetry-proto/gen-tonic-messages" "http" "trace" "metrics" ]; + "httpdate" = [ "dep:httpdate" ]; "hyper-client" = [ "opentelemetry-http/hyper" ]; "integration-testing" = [ "tonic" "prost" "tokio/full" "trace" "logs" ]; - "internal-logs" = [ "tracing" "opentelemetry_sdk/internal-logs" "opentelemetry-http/internal-logs" ]; + "internal-logs" = [ "opentelemetry_sdk/internal-logs" "opentelemetry/internal-logs" ]; "logs" = [ "opentelemetry/logs" "opentelemetry_sdk/logs" "opentelemetry-proto/logs" ]; "metrics" = [ "opentelemetry/metrics" "opentelemetry_sdk/metrics" "opentelemetry-proto/metrics" ]; "opentelemetry-http" = [ "dep:opentelemetry-http" ]; @@ -6297,27 +6296,27 @@ rec { "serde" = [ "dep:serde" ]; "serde_json" = [ "dep:serde_json" ]; "serialize" = [ "serde" "serde_json" ]; - "tls" = [ "tonic/tls-ring" ]; + "tls" = [ "tls-ring" ]; "tls-aws-lc" = [ "tonic/tls-aws-lc" ]; "tls-provider-agnostic" = [ "tonic/_tls-any" ]; "tls-ring" = [ "tonic/tls-ring" ]; - "tls-roots" = [ "tls" "tonic/tls-native-roots" ]; - "tls-webpki-roots" = [ "tls" "tonic/tls-webpki-roots" ]; + "tls-roots" = [ "tonic/tls-native-roots" ]; + "tls-webpki-roots" = [ "tonic/tls-webpki-roots" ]; "tokio" = [ "dep:tokio" ]; "tonic" = [ "dep:tonic" ]; + "tonic-types" = [ "dep:tonic-types" ]; "trace" = [ "opentelemetry/trace" "opentelemetry_sdk/trace" "opentelemetry-proto/trace" ]; - "tracing" = [ "dep:tracing" ]; "zstd" = [ "dep:zstd" ]; "zstd-http" = [ "zstd" ]; "zstd-tonic" = [ "tonic/zstd" ]; }; - resolvedDefaultFeatures = [ "default" "grpc-tonic" "gzip-tonic" "http" "http-proto" "internal-logs" "logs" "metrics" "opentelemetry-http" "prost" "reqwest" "reqwest-blocking-client" "tokio" "tonic" "trace" "tracing" ]; + resolvedDefaultFeatures = [ "default" "grpc-tonic" "gzip-tonic" "http" "http-proto" "internal-logs" "logs" "metrics" "opentelemetry-http" "prost" "reqwest" "reqwest-blocking-client" "tokio" "tonic" "tonic-types" "trace" ]; }; "opentelemetry-proto" = rec { crateName = "opentelemetry-proto"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "03xkjsjrsm7zkkx5gascqd9bg2z20wymm06l16cyxsp5dpq5s5x7"; + sha256 = "0f5ny4rpnpq6q5q34b8k2q548rf31rpbxkwjqjwzfqxg3yx5imjn"; libName = "opentelemetry_proto"; dependencies = [ { @@ -6361,30 +6360,29 @@ rec { "const-hex" = [ "dep:const-hex" ]; "default" = [ "full" ]; "full" = [ "gen-tonic" "trace" "logs" "metrics" "zpages" "with-serde" "internal-logs" ]; - "gen-tonic" = [ "gen-tonic-messages" "tonic/channel" ]; - "gen-tonic-messages" = [ "tonic" "tonic-prost" "prost" ]; + "gen-tonic" = [ "gen-tonic-messages" "tonic" "tonic-prost" "tonic/channel" ]; + "gen-tonic-messages" = [ "prost" ]; "internal-logs" = [ "opentelemetry/internal-logs" ]; "logs" = [ "opentelemetry/logs" "opentelemetry_sdk/logs" ]; "metrics" = [ "opentelemetry/metrics" "opentelemetry_sdk/metrics" ]; "prost" = [ "dep:prost" ]; "schemars" = [ "dep:schemars" ]; "serde" = [ "dep:serde" ]; - "serde_json" = [ "dep:serde_json" ]; "testing" = [ "opentelemetry/testing" ]; "tonic" = [ "dep:tonic" ]; "tonic-prost" = [ "dep:tonic-prost" ]; "trace" = [ "opentelemetry/trace" "opentelemetry_sdk/trace" ]; "with-schemars" = [ "schemars" ]; - "with-serde" = [ "serde" "const-hex" "base64" "serde_json" ]; + "with-serde" = [ "serde" "const-hex" "base64" ]; "zpages" = [ "trace" ]; }; resolvedDefaultFeatures = [ "gen-tonic" "gen-tonic-messages" "logs" "metrics" "prost" "tonic" "tonic-prost" "trace" ]; }; "opentelemetry-semantic-conventions" = rec { crateName = "opentelemetry-semantic-conventions"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "0in8plv2l2ar7anzi7lrbll0fjfvaymkg5vc5bnvibs1w3gjjbp6"; + sha256 = "0s1x4h1cgmhkxb7i5g02la2vhkf4lg5g26cgn2s2gd1p0j5gk8kc"; libName = "opentelemetry_semantic_conventions"; features = { }; @@ -6392,9 +6390,9 @@ rec { }; "opentelemetry_sdk" = rec { crateName = "opentelemetry_sdk"; - version = "0.31.0"; + version = "0.32.1"; edition = "2021"; - sha256 = "1gbjsggdxfpjbanjvaxa3nq32vfa37i3v13dvx4gsxhrk7sy8jp1"; + sha256 = "1ycl11syranrinhgn4c2hlzhyzyvpa06ryxq5mxgzmf4387ghncv"; dependencies = [ { name = "futures-channel"; @@ -6420,6 +6418,13 @@ rec { packageId = "percent-encoding"; optional = true; } + { + name = "portable-atomic"; + packageId = "portable-atomic"; + usesDefaultFeatures = false; + target = { target, features }: (!("64" == target."has_atomic" or null)); + features = [ "fallback" ]; + } { name = "rand"; packageId = "rand 0.9.4"; @@ -6444,10 +6449,18 @@ rec { optional = true; } ]; + devDependencies = [ + { + name = "tokio"; + packageId = "tokio"; + usesDefaultFeatures = false; + features = [ "macros" "rt-multi-thread" ]; + } + ]; features = { "default" = [ "trace" "metrics" "logs" "internal-logs" ]; "experimental_logs_batch_log_processor_with_async_runtime" = [ "logs" "experimental_async_runtime" ]; - "experimental_logs_concurrent_log_processor" = [ "logs" ]; + "experimental_metrics_bound_instruments" = [ "metrics" "opentelemetry/experimental_metrics_bound_instruments" ]; "experimental_metrics_custom_reader" = [ "metrics" ]; "experimental_metrics_disable_name_validation" = [ "metrics" ]; "experimental_metrics_periodicreader_with_async_runtime" = [ "metrics" "experimental_async_runtime" ]; @@ -6464,15 +6477,14 @@ rec { "rt-tokio-current-thread" = [ "tokio/rt" "tokio/time" "tokio-stream" "experimental_async_runtime" ]; "serde" = [ "dep:serde" ]; "serde_json" = [ "dep:serde_json" ]; - "spec_unstable_logs_enabled" = [ "logs" "opentelemetry/spec_unstable_logs_enabled" ]; "spec_unstable_metrics_views" = [ "metrics" ]; - "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "rt-tokio" "rt-tokio-current-thread" "tokio/macros" "tokio/rt-multi-thread" ]; + "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "tokio/sync" ]; "tokio" = [ "dep:tokio" ]; "tokio-stream" = [ "dep:tokio-stream" ]; "trace" = [ "opentelemetry/trace" "rand" "percent-encoding" ]; "url" = [ "dep:url" ]; }; - resolvedDefaultFeatures = [ "default" "experimental_async_runtime" "internal-logs" "logs" "metrics" "percent-encoding" "rand" "rt-tokio" "spec_unstable_logs_enabled" "tokio" "tokio-stream" "trace" ]; + resolvedDefaultFeatures = [ "default" "experimental_async_runtime" "internal-logs" "logs" "metrics" "percent-encoding" "rand" "rt-tokio" "tokio" "tokio-stream" "trace" ]; }; "ordered-float" = rec { crateName = "ordered-float"; @@ -6981,7 +6993,7 @@ rec { "default" = [ "fallback" ]; "serde" = [ "dep:serde" ]; }; - resolvedDefaultFeatures = [ "require-cas" ]; + resolvedDefaultFeatures = [ "fallback" "require-cas" ]; }; "portable-atomic-util" = rec { crateName = "portable-atomic-util"; @@ -7249,6 +7261,34 @@ rec { ]; }; + "prost-types" = rec { + crateName = "prost-types"; + version = "0.14.3"; + edition = "2021"; + sha256 = "1mrxrciryfgi6a0vmrgyj3g27r9hdhlgwkq71cgv3icbvg5w94c9"; + libName = "prost_types"; + authors = [ + "Dan Burkert " + "Lucio Franco " + "Casper Meijn " + "Tokio Contributors " + ]; + dependencies = [ + { + name = "prost"; + packageId = "prost"; + usesDefaultFeatures = false; + features = [ "derive" ]; + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + "chrono" = [ "dep:chrono" ]; + "default" = [ "std" ]; + "std" = [ "prost/std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "quote" = rec { crateName = "quote"; version = "1.0.45"; @@ -7678,9 +7718,9 @@ rec { }; "reqwest" = rec { crateName = "reqwest"; - version = "0.12.28"; + version = "0.13.4"; edition = "2021"; - sha256 = "0iqidijghgqbzl3bjg5hb4zmigwa4r612bgi0yiq0c90b6jkrpgd"; + sha256 = "1hy1plns9krbh3h1dy2sdjygsfkdcnxm6pbxdi0ya9b5vq8mi711"; authors = [ "Sean McArthur " ]; @@ -7697,7 +7737,7 @@ rec { name = "futures-channel"; packageId = "futures-channel"; optional = true; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "futures-core"; @@ -7717,62 +7757,44 @@ rec { { name = "http-body"; packageId = "http-body"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "http-body-util"; packageId = "http-body-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "hyper"; packageId = "hyper"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "client" ]; } { name = "hyper-util"; packageId = "hyper-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "client" "client-legacy" "client-proxy" "tokio" ]; } { name = "js-sys"; packageId = "js-sys"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "log"; packageId = "log"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "percent-encoding"; packageId = "percent-encoding"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "pin-project-lite"; packageId = "pin-project-lite"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - } - { - name = "serde"; - packageId = "serde"; - } - { - name = "serde_json"; - packageId = "serde_json"; - optional = true; - } - { - name = "serde_json"; - packageId = "serde_json"; - target = { target, features }: ("wasm32" == target."arch" or null); - } - { - name = "serde_urlencoded"; - packageId = "serde_urlencoded"; + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "sync_wrapper"; @@ -7783,27 +7805,27 @@ rec { name = "tokio"; packageId = "tokio"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "net" "time" ]; } { name = "tower"; packageId = "tower"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "retry" "timeout" "util" ]; } { name = "tower-http"; packageId = "tower-http"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "follow-redirect" ]; } { name = "tower-service"; packageId = "tower-service"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "url"; @@ -7812,17 +7834,17 @@ rec { { name = "wasm-bindgen"; packageId = "wasm-bindgen"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "wasm-bindgen-futures"; packageId = "wasm-bindgen-futures"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "web-sys"; packageId = "web-sys"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); features = [ "AbortController" "AbortSignal" "Headers" "Request" "RequestInit" "RequestMode" "Response" "Window" "FormData" "Blob" "BlobPropertyBag" "ServiceWorkerGlobalScope" "RequestCredentials" "File" "ReadableStream" "RequestCache" ]; } ]; @@ -7831,33 +7853,27 @@ rec { name = "futures-util"; packageId = "futures-util"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "std" "alloc" ]; } { name = "hyper"; packageId = "hyper"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "http2" "client" "server" ]; } { name = "hyper-util"; packageId = "hyper-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "http2" "client" "client-legacy" "server-auto" "server-graceful" "tokio" ]; } - { - name = "serde"; - packageId = "serde"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - features = [ "derive" ]; - } { name = "tokio"; packageId = "tokio"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "macros" "rt-multi-thread" ]; } { @@ -7869,40 +7885,37 @@ rec { { name = "wasm-bindgen"; packageId = "wasm-bindgen"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); features = [ "serde-serialize" ]; } ]; features = { + "__native-tls" = [ "dep:hyper-tls" "dep:native-tls-crate" "__tls" "dep:tokio-native-tls" ]; + "__native-tls-alpn" = [ "native-tls-crate?/alpn" "hyper-tls?/alpn" ]; "__rustls" = [ "dep:hyper-rustls" "dep:tokio-rustls" "dep:rustls" "__tls" ]; - "__rustls-ring" = [ "hyper-rustls?/ring" "tokio-rustls?/ring" "rustls?/ring" "quinn?/ring" ]; + "__rustls-aws-lc-rs" = [ "hyper-rustls?/aws-lc-rs" "tokio-rustls?/aws-lc-rs" "rustls?/aws-lc-rs" "quinn?/rustls-aws-lc-rs" ]; "__tls" = [ "dep:rustls-pki-types" "tokio/io-util" ]; "blocking" = [ "dep:futures-channel" "futures-channel?/sink" "dep:futures-util" "futures-util?/io" "futures-util?/sink" "tokio/sync" ]; "brotli" = [ "tower-http/decompression-br" ]; "charset" = [ "dep:encoding_rs" "dep:mime" ]; "cookies" = [ "dep:cookie_crate" "dep:cookie_store" ]; "default" = [ "default-tls" "charset" "http2" "system-proxy" ]; - "default-tls" = [ "dep:hyper-tls" "dep:native-tls-crate" "__tls" "dep:tokio-native-tls" ]; + "default-tls" = [ "rustls" ]; "deflate" = [ "tower-http/decompression-deflate" ]; + "form" = [ "dep:serde" "dep:serde_urlencoded" ]; "gzip" = [ "tower-http/decompression-gzip" ]; - "h2" = [ "dep:h2" ]; "hickory-dns" = [ "dep:hickory-resolver" "dep:once_cell" ]; - "http2" = [ "h2" "hyper/http2" "hyper-util/http2" "hyper-rustls?/http2" ]; - "http3" = [ "rustls-tls-manual-roots" "dep:h3" "dep:h3-quinn" "dep:quinn" "tokio/macros" ]; - "json" = [ "dep:serde_json" ]; - "macos-system-configuration" = [ "system-proxy" ]; + "http2" = [ "dep:h2" "hyper/http2" "hyper-util/http2" "hyper-rustls?/http2" ]; + "http3" = [ "rustls" "dep:h3" "dep:h3-quinn" "dep:quinn" "tokio/macros" ]; + "json" = [ "dep:serde" "dep:serde_json" ]; "multipart" = [ "dep:mime_guess" "dep:futures-util" ]; - "native-tls" = [ "default-tls" ]; - "native-tls-alpn" = [ "native-tls" "native-tls-crate?/alpn" "hyper-tls?/alpn" ]; - "native-tls-vendored" = [ "native-tls" "native-tls-crate?/vendored" ]; - "rustls-tls" = [ "rustls-tls-webpki-roots" ]; - "rustls-tls-manual-roots" = [ "rustls-tls-manual-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-manual-roots-no-provider" = [ "__rustls" ]; - "rustls-tls-native-roots" = [ "rustls-tls-native-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-native-roots-no-provider" = [ "dep:rustls-native-certs" "hyper-rustls?/native-tokio" "__rustls" ]; - "rustls-tls-no-provider" = [ "rustls-tls-manual-roots-no-provider" ]; - "rustls-tls-webpki-roots" = [ "rustls-tls-webpki-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-webpki-roots-no-provider" = [ "dep:webpki-roots" "hyper-rustls?/webpki-tokio" "__rustls" ]; + "native-tls" = [ "__native-tls" "__native-tls-alpn" ]; + "native-tls-no-alpn" = [ "__native-tls" ]; + "native-tls-vendored" = [ "__native-tls" "native-tls-crate?/vendored" "__native-tls-alpn" ]; + "native-tls-vendored-no-alpn" = [ "__native-tls" "native-tls-crate?/vendored" ]; + "query" = [ "dep:serde" "dep:serde_urlencoded" ]; + "rustls" = [ "__rustls-aws-lc-rs" "dep:rustls-platform-verifier" "__rustls" ]; + "rustls-no-provider" = [ "dep:rustls-platform-verifier" "__rustls" ]; "stream" = [ "tokio/fs" "dep:futures-util" "dep:tokio-util" "dep:wasm-streams" ]; "system-proxy" = [ "hyper-util/client-proxy-system" ]; "zstd" = [ "tower-http/decompression-zstd" ]; @@ -9512,7 +9525,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_certs"; @@ -9649,10 +9662,6 @@ rec { name = "indoc"; packageId = "indoc"; } - { - name = "java-properties"; - packageId = "java-properties"; - } { name = "serde"; packageId = "serde"; @@ -9689,10 +9698,6 @@ rec { name = "tracing"; packageId = "tracing"; } - { - name = "xml"; - packageId = "xml"; - } ]; buildDependencies = [ { @@ -9720,7 +9725,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_operator"; @@ -9771,6 +9776,10 @@ rec { name = "indexmap"; packageId = "indexmap"; } + { + name = "java-properties"; + packageId = "java-properties"; + } { name = "jiff"; packageId = "jiff"; @@ -9886,12 +9895,17 @@ rec { name = "uuid"; packageId = "uuid"; } + { + name = "xml"; + packageId = "xml"; + } ]; features = { "certs" = [ "dep:stackable-certs" ]; + "client-feature-gates" = [ "dep:winnow" ]; "crds" = [ "dep:stackable-versioned" ]; "default" = [ "crds" ]; - "full" = [ "crds" "certs" "time" "webhook" "kube-ws" ]; + "full" = [ "client-feature-gates" "crds" "certs" "time" "webhook" "kube-ws" ]; "kube-ws" = [ "kube/ws" ]; "time" = [ "stackable-shared/time" ]; "webhook" = [ "dep:stackable-webhook" ]; @@ -9905,7 +9919,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; procMacro = true; @@ -9935,12 +9949,12 @@ rec { }; "stackable-shared" = rec { crateName = "stackable-shared"; - version = "0.1.0"; + version = "0.1.1"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_shared"; @@ -10016,12 +10030,12 @@ rec { }; "stackable-telemetry" = rec { crateName = "stackable-telemetry"; - version = "0.6.3"; + version = "0.6.4"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_telemetry"; @@ -10065,7 +10079,7 @@ rec { { name = "opentelemetry_sdk"; packageId = "opentelemetry_sdk"; - features = [ "rt-tokio" "logs" "rt-tokio" "spec_unstable_logs_enabled" ]; + features = [ "rt-tokio" "logs" "rt-tokio" ]; } { name = "pin-project"; @@ -10131,7 +10145,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_versioned"; @@ -10181,7 +10195,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; procMacro = true; @@ -10249,7 +10263,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_webhook"; @@ -11413,6 +11427,33 @@ rec { } ]; + }; + "tonic-types" = rec { + crateName = "tonic-types"; + version = "0.14.6"; + edition = "2024"; + sha256 = "1s286gg71pjajny8xar0azq1w9lgz1ks3jm3pccxb0qz0q11pavk"; + libName = "tonic_types"; + authors = [ + "Lucio Franco " + "Rafael Lemos " + ]; + dependencies = [ + { + name = "prost"; + packageId = "prost"; + } + { + name = "prost-types"; + packageId = "prost-types"; + } + { + name = "tonic"; + packageId = "tonic"; + usesDefaultFeatures = false; + } + ]; + }; "tower" = rec { crateName = "tower"; @@ -11867,9 +11908,9 @@ rec { }; "tracing-opentelemetry" = rec { crateName = "tracing-opentelemetry"; - version = "0.32.1"; + version = "0.33.0"; edition = "2021"; - sha256 = "1z2jjmxbkm1qawlb3bm99x8xwf4g8wjkbcknm9z4fv1w14nqzhhs"; + sha256 = "09nvxy5m7nxmifz4b6szdcyczapp2jcgxcac0jw4ax8klz5n9g5d"; libName = "tracing_opentelemetry"; dependencies = [ { diff --git a/Cargo.toml b/Cargo.toml index ed454cd1..77b61fc8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,6 @@ const_format = "0.2" fnv = "1.0" futures = { version = "0.3", features = ["compat"] } indoc = "2.0" -java-properties = "2.0" rstest = "0.26" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" @@ -29,7 +28,6 @@ snafu = "0.9" strum = { version = "0.28", features = ["derive"] } tokio = { version = "1.40", features = ["full"] } tracing = "0.1" -xml = "1.3" [patch."https://github.com/stackabletech/operator-rs.git"] stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "smooth-operator" } diff --git a/crate-hashes.json b/crate-hashes.json index c76bf06c..cd03561e 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -3,8 +3,8 @@ "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.0": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.3": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 35e88f0e..902fedb0 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -17,7 +17,6 @@ const_format.workspace = true fnv.workspace = true futures.workspace = true indoc.workspace = true -java-properties.workspace = true serde.workspace = true serde_json.workspace = true shell-escape.workspace = true @@ -25,7 +24,6 @@ snafu.workspace = true strum.workspace = true tokio.workspace = true tracing.workspace = true -xml.workspace = true [build-dependencies] built.workspace = true diff --git a/rust/operator-binary/src/config/mod.rs b/rust/operator-binary/src/config/mod.rs index cb7d7920..271c6d99 100644 --- a/rust/operator-binary/src/config/mod.rs +++ b/rust/operator-binary/src/config/mod.rs @@ -1,2 +1 @@ pub mod jvm; -pub mod writer; diff --git a/rust/operator-binary/src/config/writer.rs b/rust/operator-binary/src/config/writer.rs deleted file mode 100644 index 08250f9f..00000000 --- a/rust/operator-binary/src/config/writer.rs +++ /dev/null @@ -1,145 +0,0 @@ -//! Writers for Hadoop XML config files and Java `.properties` files. -//! -//! Vendored from the `product-config` crate's `writer` module so the operator no -//! longer depends on `product-config` for rendering. - -use std::io::Write; - -use java_properties::{PropertiesError, PropertiesWriter}; -use snafu::{ResultExt, Snafu}; -use xml::escape::escape_str_attribute; - -#[derive(Debug, Snafu)] -pub enum PropertiesWriterError { - #[snafu(display("failed to create properties file"))] - Properties { source: PropertiesError }, - - #[snafu(display("failed to convert properties file byte array to UTF-8"))] - FromUtf8 { source: std::string::FromUtf8Error }, -} - -/// Creates a common Java properties file string in the format: -/// `property_1=value_1\nproperty_2=value_2\n`. -pub fn to_java_properties_string<'a, T>(properties: T) -> Result -where - T: Iterator)>, -{ - let mut output = Vec::new(); - write_java_properties(&mut output, properties)?; - String::from_utf8(output).context(FromUtf8Snafu) -} - -/// Writes Java properties to the given writer. A `None` value is written as an -/// empty value (`key=`). -fn write_java_properties<'a, W, T>(writer: W, properties: T) -> Result<(), PropertiesWriterError> -where - W: Write, - T: Iterator)>, -{ - let mut writer = PropertiesWriter::new(writer); - for (k, v) in properties { - let property_value = v.as_deref().unwrap_or_default(); - writer.write(k, property_value).context(PropertiesSnafu)?; - } - writer.flush().context(PropertiesSnafu)?; - Ok(()) -} - -/// Converts properties into a Hadoop configuration XML, including the wrapping -/// `...` elements. Properties with a `None` value -/// are skipped. Keys and values are XML-escaped. -pub fn to_hadoop_xml<'a, T>(properties: T) -> String -where - T: Iterator)>, -{ - let mut snippet = String::new(); - for (k, v) in properties { - let escaped_value = match v { - Some(value) => escape_str_attribute(value), - None => continue, - }; - let escaped_key = escape_str_attribute(k); - snippet.push_str(&format!( - " \n {escaped_key}\n {escaped_value}\n \n" - )); - } - format!("\n\n{snippet}") -} - -#[cfg(test)] -mod tests { - use std::collections::BTreeMap; - - use super::*; - - fn xml(pairs: &[(&str, Option<&str>)]) -> String { - let map: BTreeMap> = pairs - .iter() - .map(|(k, v)| (k.to_string(), v.map(str::to_string))) - .collect(); - to_hadoop_xml(map.iter()) - } - - fn props(pairs: &[(&str, Option<&str>)]) -> String { - let map: BTreeMap> = pairs - .iter() - .map(|(k, v)| (k.to_string(), v.map(str::to_string))) - .collect(); - to_java_properties_string(map.iter()).unwrap() - } - - #[test] - fn hadoop_xml_wraps_empty_configuration() { - assert_eq!( - xml(&[]), - "\n\n" - ); - } - - #[test] - fn hadoop_xml_renders_single_property() { - assert_eq!( - xml(&[("fs.defaultFS", Some("hdfs://hdfs/"))]), - "\n\n \ - \n fs.defaultFS\n \ - hdfs://hdfs/\n \n" - ); - } - - #[test] - fn hadoop_xml_skips_none_values() { - assert_eq!( - xml(&[("kept", Some("1")), ("dropped", None)]), - "\n\n \ - \n kept\n \ - 1\n \n" - ); - } - - #[test] - fn hadoop_xml_escapes_special_characters() { - let rendered = xml(&[("k", Some("&b"))]); - assert!( - rendered.contains("<a>&b"), - "{rendered}" - ); - } - - #[test] - fn java_properties_renders_key_value() { - assert_eq!(props(&[("a", Some("1")), ("b", Some("2"))]), "a=1\nb=2\n"); - } - - #[test] - fn java_properties_renders_none_as_empty() { - assert_eq!(props(&[("none", None)]), "none=\n"); - } - - #[test] - fn java_properties_escapes_colon_in_value() { - assert_eq!( - props(&[("url", Some("file://this/location/file.abc"))]), - "url=file\\://this/location/file.abc\n" - ); - } -} diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 374a650e..312cda19 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -6,10 +6,10 @@ use stackable_operator::{ k8s_openapi::api::core::v1::ConfigMap, product_logging::framework::VECTOR_CONFIG_FILE, role_utils::RoleGroupRef, + v2::config_file_writer::PropertiesWriterError, }; use crate::{ - config::writer::PropertiesWriterError, controller::build::properties::{ ConfigFileName, hbase_env, hbase_site, logging, security_properties, ssl_client, ssl_server, }, diff --git a/rust/operator-binary/src/controller/build/discovery.rs b/rust/operator-binary/src/controller/build/discovery.rs index bf746e25..9e669d7f 100644 --- a/rust/operator-binary/src/controller/build/discovery.rs +++ b/rust/operator-binary/src/controller/build/discovery.rs @@ -7,10 +7,10 @@ use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, k8s_openapi::api::core::v1::ConfigMap, kube::runtime::reflector::ObjectRef, + v2::config_file_writer::to_hadoop_xml, }; use crate::{ - config::writer::to_hadoop_xml, controller::build::properties::ConfigFileName, crd::{HbaseRole, v1alpha1}, hbase_controller::{ValidatedCluster, build_recommended_labels}, diff --git a/rust/operator-binary/src/controller/build/properties/hbase_site.rs b/rust/operator-binary/src/controller/build/properties/hbase_site.rs index 3d7eea78..2d010911 100644 --- a/rust/operator-binary/src/controller/build/properties/hbase_site.rs +++ b/rust/operator-binary/src/controller/build/properties/hbase_site.rs @@ -4,10 +4,11 @@ use std::collections::BTreeMap; -use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; +use stackable_operator::v2::{ + config_file_writer::to_hadoop_xml, config_overrides::KeyValueConfigOverrides, +}; use crate::{ - config::writer::to_hadoop_xml, controller::build::properties::resolved_overrides, crd::{ AnyServiceConfig, HBASE_CLUSTER_DISTRIBUTED, HBASE_MASTER_PORT, HBASE_MASTER_UI_PORT, diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index db919037..c4cf5bb8 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -1,7 +1,7 @@ //! Per-file builders for the HBase config files assembled into the rolegroup //! `ConfigMap`. Each `.rs` module produces the rendered content for one -//! config file; the shared [`crate::config::writer`] module serializes maps to -//! the Hadoop-XML / Java-properties on-wire format. +//! config file; the shared [`stackable_operator::v2::config_file_writer`] +//! module serializes maps to the Hadoop-XML / Java-properties on-wire format. use std::collections::BTreeMap; diff --git a/rust/operator-binary/src/controller/build/properties/security_properties.rs b/rust/operator-binary/src/controller/build/properties/security_properties.rs index 421f925f..35ab38b4 100644 --- a/rust/operator-binary/src/controller/build/properties/security_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/security_properties.rs @@ -6,14 +6,13 @@ use std::collections::BTreeMap; -use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; - -use crate::{ - config::writer::{PropertiesWriterError, to_java_properties_string}, - controller::build::properties::resolved_overrides, - crd::HbaseRole, +use stackable_operator::v2::{ + config_file_writer::{PropertiesWriterError, to_java_properties_string}, + config_overrides::KeyValueConfigOverrides, }; +use crate::{controller::build::properties::resolved_overrides, crd::HbaseRole}; + /// Renders `security.properties`: role-specific DNS cache TTLs plus user overrides. pub fn build( role: &HbaseRole, diff --git a/rust/operator-binary/src/controller/build/properties/ssl_client.rs b/rust/operator-binary/src/controller/build/properties/ssl_client.rs index 83d0039d..3fc8093f 100644 --- a/rust/operator-binary/src/controller/build/properties/ssl_client.rs +++ b/rust/operator-binary/src/controller/build/properties/ssl_client.rs @@ -1,9 +1,11 @@ //! Builds the `ssl-client.xml` config file: kerberos/TLS client settings + overrides. use std::collections::BTreeMap; -use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; +use stackable_operator::v2::{ + config_file_writer::to_hadoop_xml, config_overrides::KeyValueConfigOverrides, +}; -use crate::{config::writer::to_hadoop_xml, controller::build::properties::resolved_overrides}; +use crate::controller::build::properties::resolved_overrides; /// Renders `ssl-client.xml`. pub fn build(settings: BTreeMap, overrides: KeyValueConfigOverrides) -> String { diff --git a/rust/operator-binary/src/controller/build/properties/ssl_server.rs b/rust/operator-binary/src/controller/build/properties/ssl_server.rs index 2ae17e9b..edf70aac 100644 --- a/rust/operator-binary/src/controller/build/properties/ssl_server.rs +++ b/rust/operator-binary/src/controller/build/properties/ssl_server.rs @@ -1,9 +1,11 @@ //! Builds the `ssl-server.xml` config file: kerberos/TLS server settings + overrides. use std::collections::BTreeMap; -use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; +use stackable_operator::v2::{ + config_file_writer::to_hadoop_xml, config_overrides::KeyValueConfigOverrides, +}; -use crate::{config::writer::to_hadoop_xml, controller::build::properties::resolved_overrides}; +use crate::controller::build::properties::resolved_overrides; /// Renders `ssl-server.xml`. pub fn build(settings: BTreeMap, overrides: KeyValueConfigOverrides) -> String { From 36cc1da5fe2d2c52648364bfc8a07ebe5cc7df9a Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 8 Jun 2026 19:51:22 +0200 Subject: [PATCH 20/56] refactor: add uid, namespace to ValidatedCluster --- .../src/controller/build/config_map.rs | 17 +-- .../src/controller/build/discovery.rs | 28 +---- .../src/controller/validate.rs | 29 +++-- rust/operator-binary/src/hbase_controller.rs | 112 ++++++++++++++++-- 4 files changed, 129 insertions(+), 57 deletions(-) diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 312cda19..199f991f 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -6,7 +6,7 @@ use stackable_operator::{ k8s_openapi::api::core::v1::ConfigMap, product_logging::framework::VECTOR_CONFIG_FILE, role_utils::RoleGroupRef, - v2::config_file_writer::PropertiesWriterError, + v2::{builder::meta::ownerreference_from_resource, config_file_writer::PropertiesWriterError}, }; use crate::{ @@ -31,11 +31,6 @@ pub enum Error { role_group: String, }, - #[snafu(display("object is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, - }, - #[snafu(display("failed to build object meta data"))] ObjectMeta { source: stackable_operator::builder::meta::Error, @@ -55,9 +50,6 @@ pub fn build_rolegroup_config_map( cluster: &ValidatedCluster, role: &HbaseRole, rolegroup_ref: &RoleGroupRef, - // `owner` is retained only for the ConfigMap ObjectMeta / owner reference; the rendered - // content comes entirely from `cluster`. To be decoupled in a follow-up. - owner_ref: &v1alpha1::HbaseCluster, ) -> Result { tracing::info!("Setting up ConfigMap for {:?}", rolegroup_ref); @@ -110,12 +102,11 @@ pub fn build_rolegroup_config_map( )?; let cm_metadata = ObjectMetaBuilder::new() - .name_and_namespace(owner_ref) + .name_and_namespace(cluster) .name(rolegroup_ref.object_name()) - .ownerreference_from_resource(owner_ref, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) .with_recommended_labels(&build_recommended_labels( - owner_ref, + cluster, &cluster.image.app_version_label_value, &rolegroup_ref.role, &rolegroup_ref.role_group, diff --git a/rust/operator-binary/src/controller/build/discovery.rs b/rust/operator-binary/src/controller/build/discovery.rs index 9e669d7f..99dace99 100644 --- a/rust/operator-binary/src/controller/build/discovery.rs +++ b/rust/operator-binary/src/controller/build/discovery.rs @@ -6,13 +6,12 @@ use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, k8s_openapi::api::core::v1::ConfigMap, - kube::runtime::reflector::ObjectRef, - v2::config_file_writer::to_hadoop_xml, + v2::{builder::meta::ownerreference_from_resource, config_file_writer::to_hadoop_xml}, }; use crate::{ controller::build::properties::ConfigFileName, - crd::{HbaseRole, v1alpha1}, + crd::HbaseRole, hbase_controller::{ValidatedCluster, build_recommended_labels}, }; @@ -20,12 +19,6 @@ type Result = std::result::Result; #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("object {hbase} is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, - hbase: ObjectRef, - }, - #[snafu(display("failed to build ConfigMap"))] BuildConfigMap { source: stackable_operator::builder::configmap::Error, @@ -38,13 +31,7 @@ pub enum Error { } /// Creates a discovery config map containing the `hbase-site.xml` for clients. -/// -/// The rendered content comes entirely from `cluster`; `owner_ref` is retained only for the -/// ConfigMap ObjectMeta / owner reference. -pub fn build_discovery_config_map( - cluster: &ValidatedCluster, - owner_ref: &v1alpha1::HbaseCluster, -) -> Result { +pub fn build_discovery_config_map(cluster: &ValidatedCluster) -> Result { let cluster_config = &cluster.cluster_config; let mut hbase_site = cluster_config @@ -55,13 +42,10 @@ pub fn build_discovery_config_map( ConfigMapBuilder::new() .metadata( ObjectMetaBuilder::new() - .name_and_namespace(owner_ref) - .ownerreference_from_resource(owner_ref, None, Some(true)) - .with_context(|_| ObjectMissingMetadataForOwnerRefSnafu { - hbase: ObjectRef::from_obj(owner_ref), - })? + .name_and_namespace(cluster) + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) .with_recommended_labels(&build_recommended_labels( - owner_ref, + cluster, &cluster.image.app_version_label_value, &HbaseRole::RegionServer.to_string(), "discovery", diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index b4a8cba2..792acd00 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -1,13 +1,12 @@ -use std::{collections::BTreeMap, str::FromStr}; +use std::collections::BTreeMap; use snafu::{ResultExt, Snafu}; use stackable_operator::{ commons::product_image_selection::{self}, config::merge::Merge, - kube::ResourceExt, role_utils::GenericRoleConfig, utils::cluster_info::KubernetesClusterInfo, - v2::types::operator::ClusterName, + v2::controller_utils::{get_cluster_name, get_namespace, get_uid}, }; use strum::IntoEnumIterator; @@ -33,9 +32,9 @@ pub enum Error { source: product_image_selection::Error, }, - #[snafu(display("invalid cluster name"))] - InvalidClusterName { - source: stackable_operator::v2::macros::attributed_string_type::Error, + #[snafu(display("failed to determine the cluster identity (name, namespace and uid)"))] + GetClusterIdentity { + source: stackable_operator::v2::controller_utils::Error, }, #[snafu(display("the HbaseCluster has no {role} role defined"))] @@ -130,10 +129,16 @@ pub fn validate_cluster( let ssl_server_settings = kerberos_ssl_server_settings(hbase); let ssl_client_settings = kerberos_ssl_client_settings(hbase); - Ok(ValidatedCluster { - name: ClusterName::from_str(&hbase.name_any()).context(InvalidClusterNameSnafu)?, - image: resolved_product_image, - cluster_config: ValidatedClusterConfig { + let name = get_cluster_name(hbase).context(GetClusterIdentitySnafu)?; + let namespace = get_namespace(hbase).context(GetClusterIdentitySnafu)?; + let uid = get_uid(hbase).context(GetClusterIdentitySnafu)?; + + Ok(ValidatedCluster::new( + name, + namespace, + uid, + resolved_product_image, + ValidatedClusterConfig { zookeeper_connection_information: dereferenced_objects.zookeeper_connection_information, hbase_opa_config: dereferenced_objects.hbase_opa_config, kerberos_enabled: hbase.has_kerberos_enabled(), @@ -142,9 +147,9 @@ pub fn validate_cluster( ssl_server_settings, ssl_client_settings, }, - role_group_configs: role_groups, + role_groups, role_configs, - }) + )) } /// The names of the role groups defined for `role` in the spec. diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index c57f3f52..d76d8768 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -26,7 +26,10 @@ use stackable_operator::{ ServicePort, ServiceSpec, TCPSocketAction, Volume, }, }, - apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, + apimachinery::pkg::{ + apis::meta::v1::{LabelSelector, ObjectMeta}, + util::intstr::IntOrString, + }, }, kube::{ Resource, ResourceExt, @@ -49,7 +52,13 @@ use stackable_operator::{ compute_conditions, operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, }, - v2::types::operator::ClusterName, + v2::{ + HasName, HasUid, NameIsValidLabelValue, + types::{ + kubernetes::{NamespaceName, Uid}, + operator::ClusterName, + }, + }, }; use strum::{EnumDiscriminants, IntoStaticStr}; @@ -90,16 +99,100 @@ pub struct Ctx { /// every role and role group before any resources are created. #[derive(Clone, Debug)] pub struct ValidatedCluster { + /// Backs the [`Resource`] implementation (provides `meta()`/`name_any()`) so the build + /// functions can derive `ObjectMeta`, owner references and labels without the full + /// `HbaseCluster` object. Holds only name, namespace and uid. + metadata: ObjectMeta, /// The logical (and Kubernetes object) name of the cluster. - // Populated now; consumed by the new build path in a later commit. - #[allow(dead_code)] pub name: ClusterName, + /// The namespace the cluster lives in. Part of the cluster identity; currently consumed via + /// the [`Resource`] metadata (`name_and_namespace`) rather than read directly. + #[allow(dead_code)] + pub namespace: NamespaceName, + /// The UID of the `HbaseCluster` object, used to build owner references. + pub uid: Uid, pub image: ResolvedProductImage, pub cluster_config: ValidatedClusterConfig, pub role_group_configs: BTreeMap>, pub role_configs: BTreeMap, } +impl ValidatedCluster { + #[allow(clippy::too_many_arguments)] + pub fn new( + name: ClusterName, + namespace: NamespaceName, + uid: Uid, + image: ResolvedProductImage, + cluster_config: ValidatedClusterConfig, + role_group_configs: BTreeMap>, + role_configs: BTreeMap, + ) -> Self { + Self { + metadata: ObjectMeta { + name: Some(name.to_string()), + namespace: Some(namespace.to_string()), + uid: Some(uid.to_string()), + ..ObjectMeta::default() + }, + name, + namespace, + uid, + image, + cluster_config, + role_group_configs, + role_configs, + } + } +} + +impl Resource for ValidatedCluster { + type DynamicType = ::DynamicType; + type Scope = ::Scope; + + fn group(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha1::HbaseCluster::group(dt) + } + + fn version(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha1::HbaseCluster::version(dt) + } + + fn kind(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha1::HbaseCluster::kind(dt) + } + + fn plural(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha1::HbaseCluster::plural(dt) + } + + fn meta(&self) -> &ObjectMeta { + &self.metadata + } + + fn meta_mut(&mut self) -> &mut ObjectMeta { + &mut self.metadata + } +} + +impl HasName for ValidatedCluster { + fn to_name(&self) -> String { + self.name.to_string() + } +} + +impl HasUid for ValidatedCluster { + fn to_uid(&self) -> Uid { + self.uid.clone() + } +} + +impl NameIsValidLabelValue for ValidatedCluster { + fn to_label_value(&self) -> String { + self.name.to_label_value() + } +} + /// Cluster-wide settings resolved once during validation. #[derive(Clone, Debug)] pub struct ValidatedClusterConfig { @@ -353,7 +446,6 @@ pub async fn reconcile_hbase( &validated_cluster, hbase_role, &rolegroup, - hbase, ) .context(BuildRolegroupConfigMapSnafu)?; let rg_statefulset = build_rolegroup_statefulset( @@ -411,8 +503,8 @@ pub async fn reconcile_hbase( // Discovery CM will fail to build until the rest of the cluster has been deployed, so do it last // so that failure won't inhibit the rest of the cluster from booting up. - let discovery_cm = build_discovery_config_map(&validated_cluster, hbase) - .context(BuildDiscoveryConfigMapSnafu)?; + let discovery_cm = + build_discovery_config_map(&validated_cluster).context(BuildDiscoveryConfigMapSnafu)?; cluster_resources .add(client, discovery_cm) .await @@ -881,12 +973,12 @@ pub fn error_policy( } } -pub fn build_recommended_labels<'a>( - owner: &'a v1alpha1::HbaseCluster, +pub fn build_recommended_labels<'a, R>( + owner: &'a R, app_version: &'a str, role: &'a str, role_group: &'a str, -) -> ObjectLabels<'a, v1alpha1::HbaseCluster> { +) -> ObjectLabels<'a, R> { ObjectLabels { owner, app_name: APP_NAME, From 634449b82b5f0dad3233fc9e2ae74f258347b632 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 9 Jun 2026 12:14:26 +0200 Subject: [PATCH 21/56] refactor: use non optional property writers --- Cargo.lock | 83 +++++++++--------- Cargo.nix | 87 +++++++++---------- .../src/controller/build/discovery.rs | 10 +-- .../controller/build/properties/hbase_site.rs | 8 +- .../build/properties/security_properties.rs | 8 +- .../controller/build/properties/ssl_client.rs | 6 +- .../controller/build/properties/ssl_server.rs | 6 +- 7 files changed, 94 insertions(+), 114 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4b9e0936..15c16088 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -265,9 +265,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "2.12.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84d7ced0ae9557296835c32bf1b1e02b44c746701f898460fb000d7eaa84f00a" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" [[package]] name = "block-buffer" @@ -320,9 +320,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "num-traits", @@ -1086,9 +1086,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes", "itoa", @@ -1453,13 +1453,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.99" +version = "0.3.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" +checksum = "f2025f20d7a4fa7785846e7b63d10a76d3f1cee98ee5cb79ea59703f95e42162" dependencies = [ "cfg-if", "futures-util", - "once_cell", "wasm-bindgen", ] @@ -1515,7 +1514,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "darling", "regex", @@ -1707,9 +1706,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.31" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "113b30b4cd05f7c06868fdb2854f66a7b9fece9a48425351cd532e810d74024f" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" [[package]] name = "matchers" @@ -2188,9 +2187,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ "bytes", "prost-derive", @@ -2198,9 +2197,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", "itertools", @@ -2211,9 +2210,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" dependencies = [ "prost", ] @@ -2900,7 +2899,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "const-oid", "ecdsa", @@ -2947,7 +2946,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.111.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "base64", "clap", @@ -2991,7 +2990,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "darling", "proc-macro2", @@ -3002,7 +3001,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "jiff", "k8s-openapi", @@ -3019,7 +3018,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.4" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "axum", "clap", @@ -3043,7 +3042,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "kube", "schemars", @@ -3057,7 +3056,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "convert_case", "convert_case_extras", @@ -3075,7 +3074,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "arc-swap", "async-trait", @@ -3669,9 +3668,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.2" +version = "1.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" dependencies = [ "js-sys", "wasm-bindgen", @@ -3721,9 +3720,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.122" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" +checksum = "a254a4b10c19a76f09a27640e7ffbf9bc30bf67e16a3bf28aaefa4920fe81563" dependencies = [ "cfg-if", "once_cell", @@ -3734,9 +3733,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.72" +version = "0.4.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" +checksum = "54568702fabf5d4849ce2b90fadfa64168a097eaf4b351ce9df8b687a0086aaf" dependencies = [ "js-sys", "wasm-bindgen", @@ -3744,9 +3743,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.122" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" +checksum = "24a40fc75b0ec6f3746ceb10d36f53a93dcd68a93b11b6445983945d79eba0dc" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3754,9 +3753,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.122" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" +checksum = "908f34bd9b9ce3d4caf07b72dfab63d61504d156856c6bd3cd87fa350cf3985b" dependencies = [ "bumpalo", "proc-macro2", @@ -3767,18 +3766,18 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.122" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" +checksum = "7acbf7616c27b194bbb550bf77ed0c2c3e5b7fd1260a93082b95fb7f47959b92" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" -version = "0.3.99" +version = "0.3.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" +checksum = "6e0871acf327f283dc6da28a1696cdc64fb355ba9f935d052021fa77f35cce69" dependencies = [ "js-sys", "wasm-bindgen", @@ -3978,9 +3977,9 @@ checksum = "636f85e5ca6488e96401b61eb7de54f4e44755c988af0f52cf90230c312a1a89" [[package]] name = "yoke" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive", diff --git a/Cargo.nix b/Cargo.nix index 5fac81a9..cfa748a7 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -866,9 +866,9 @@ rec { }; "bitflags" = rec { crateName = "bitflags"; - version = "2.12.1"; + version = "2.13.0"; edition = "2021"; - sha256 = "02phhjm7w380zdh8928zf13cfi1bw2qz2ay36ml2jmwmmv8cxmw4"; + sha256 = "1y239gpvl061rfvav7jds8mjs42kmwi39is7yx5d1qw3hvp8nf5l"; authors = [ "The Rust Project Developers" ]; @@ -1012,9 +1012,9 @@ rec { }; "chrono" = rec { crateName = "chrono"; - version = "0.4.44"; + version = "0.4.45"; edition = "2021"; - sha256 = "1c64mk9a235271j5g3v4zrzqqmd43vp9vki7vqfllpqf5rd0fwy6"; + sha256 = "09rkcgk6is2sdhqs9142zv8xqnj8ryx8m9hknllqwyv9wxi9x9qs"; dependencies = [ { name = "iana-time-zone"; @@ -3401,9 +3401,9 @@ rec { }; "http" = rec { crateName = "http"; - version = "1.4.1"; + version = "1.4.2"; edition = "2021"; - sha256 = "1l7k2ia57z3q7q3ka497krzps795kd3fymm2k12lr623y4nldrwb"; + sha256 = "09b4p8fiivkg7wm0b59fyrn1jkm7px298ci7zb9igz6n647gaw39"; authors = [ "Alex Crichton " "Carl Lerche " @@ -4618,9 +4618,9 @@ rec { }; "js-sys" = rec { crateName = "js-sys"; - version = "0.3.99"; + version = "0.3.100"; edition = "2021"; - sha256 = "04azrzsz91gr5s3z0ij36lz0kj9ry4lw3jz0mmbiwb251rsc8aql"; + sha256 = "0qi1wjakyw2rx9wwprcfx77g3lvn1b8n6yvfhj2pgym4swh5y0pj"; libName = "js_sys"; authors = [ "The wasm-bindgen Developers" @@ -4637,11 +4637,6 @@ rec { usesDefaultFeatures = false; features = [ "std" ]; } - { - name = "once_cell"; - packageId = "once_cell"; - usesDefaultFeatures = false; - } { name = "wasm-bindgen"; packageId = "wasm-bindgen"; @@ -4831,7 +4826,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "k8s_version"; @@ -5639,9 +5634,9 @@ rec { }; "log" = rec { crateName = "log"; - version = "0.4.31"; + version = "0.4.32"; edition = "2021"; - sha256 = "0kq2fh6q2bjkrm8m6hj8kb7gxfd7cr7qbcpxd1lc1xq5rns30fqi"; + sha256 = "0fmdg0cxig7i4fwf1sw7fmg4d1gdbfzniawcfpwydy1q7320fgwm"; authors = [ "The Rust Project Developers" ]; @@ -7196,9 +7191,9 @@ rec { }; "prost" = rec { crateName = "prost"; - version = "0.14.3"; + version = "0.14.4"; edition = "2021"; - sha256 = "0s057z9nzggzy7x4bbsiar852hg7zb81f4z4phcdb0ig99971snj"; + sha256 = "1qas5v5rap45f43v3ja0jngxrrafrkcwl0iw5a3ld1pz2rscd2jj"; authors = [ "Dan Burkert " "Lucio Franco " @@ -7225,9 +7220,9 @@ rec { }; "prost-derive" = rec { crateName = "prost-derive"; - version = "0.14.3"; + version = "0.14.4"; edition = "2021"; - sha256 = "02zvva6kb0pfvlyc4nac6gd37ncjrs8jq5scxcq4nbqkc8wh5ii7"; + sha256 = "1pqa77d7da5pf6ba3kjj7510m5cynz6902ax01ckvr0pfrgv4w5m"; procMacro = true; libName = "prost_derive"; authors = [ @@ -7263,9 +7258,9 @@ rec { }; "prost-types" = rec { crateName = "prost-types"; - version = "0.14.3"; + version = "0.14.4"; edition = "2021"; - sha256 = "1mrxrciryfgi6a0vmrgyj3g27r9hdhlgwkq71cgv3icbvg5w94c9"; + sha256 = "02ivjvc4cwl5bfgjs3l00hwlrk74z8zlg1xcgx60bww8fvf6fjgr"; libName = "prost_types"; authors = [ "Dan Burkert " @@ -9525,7 +9520,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_certs"; @@ -9725,7 +9720,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_operator"; @@ -9919,7 +9914,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; procMacro = true; @@ -9954,7 +9949,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_shared"; @@ -10035,7 +10030,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_telemetry"; @@ -10145,7 +10140,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_versioned"; @@ -10195,7 +10190,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; procMacro = true; @@ -10263,7 +10258,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_webhook"; @@ -12310,9 +12305,9 @@ rec { }; "uuid" = rec { crateName = "uuid"; - version = "1.23.2"; + version = "1.23.3"; edition = "2021"; - sha256 = "1xy942s4z0bi8p3441wvd4ry3hx6ry1c7s6fgrr38462xqybhn6j"; + sha256 = "1drddl03gi12vl1s3l2h371dw39plhn9wappp00v707g7h96nk8l"; authors = [ "Ashley Mannix" "Dylan DPC" @@ -12455,9 +12450,9 @@ rec { }; "wasm-bindgen" = rec { crateName = "wasm-bindgen"; - version = "0.2.122"; + version = "0.2.123"; edition = "2021"; - sha256 = "02flix96brsb2r1i3grnikii302iqpdm337kl3xv5lklz5v4bl1y"; + sha256 = "0qqmx07r597gm8lbz8qngvv0phwvpzzyfh3nl84nz9qr1jqs8m52"; libName = "wasm_bindgen"; authors = [ "The wasm-bindgen Developers" @@ -12506,9 +12501,9 @@ rec { }; "wasm-bindgen-futures" = rec { crateName = "wasm-bindgen-futures"; - version = "0.4.72"; + version = "0.4.73"; edition = "2021"; - sha256 = "03qb24gfr072rk8hb69glfdc8yhqqqq2rhy3j5i0ps8sk79dnwwl"; + sha256 = "1bva12h8gdpqkp753czlxabs0s21lvgzm41brr4lhpdzz818fmjl"; libName = "wasm_bindgen_futures"; authors = [ "The wasm-bindgen Developers" @@ -12534,9 +12529,9 @@ rec { }; "wasm-bindgen-macro" = rec { crateName = "wasm-bindgen-macro"; - version = "0.2.122"; + version = "0.2.123"; edition = "2021"; - sha256 = "1inyl55bvdifx7l60q9wl0ivmw7236jg7jqmcqpxhsx3knq52qci"; + sha256 = "1p50xdwmv543b52bc49vm5lcsgd9adpx647bdisg7ihfbg3hz914"; procMacro = true; libName = "wasm_bindgen_macro"; authors = [ @@ -12558,9 +12553,9 @@ rec { }; "wasm-bindgen-macro-support" = rec { crateName = "wasm-bindgen-macro-support"; - version = "0.2.122"; + version = "0.2.123"; edition = "2021"; - sha256 = "0pjw5kc2mbfz59agk5l21kh4hxzp94rygdvsnr4f3z6b5hv4g419"; + sha256 = "0nwqyc63byl7rp9nnv45av8h85fncfmxywkvy35d9qwwkfyk93wh"; libName = "wasm_bindgen_macro_support"; authors = [ "The wasm-bindgen Developers" @@ -12594,10 +12589,10 @@ rec { }; "wasm-bindgen-shared" = rec { crateName = "wasm-bindgen-shared"; - version = "0.2.122"; + version = "0.2.123"; edition = "2021"; links = "wasm_bindgen"; - sha256 = "0ds4mmfqvxwc5fp33hn0jblf0f6b4lghrd9mpkls66zic4n9p4ls"; + sha256 = "14lvjm3pzywm5c4962i6s5zmngic1knpggshnnxr9c97dihzgjvs"; libName = "wasm_bindgen_shared"; authors = [ "The wasm-bindgen Developers" @@ -12612,9 +12607,9 @@ rec { }; "web-sys" = rec { crateName = "web-sys"; - version = "0.3.99"; + version = "0.3.100"; edition = "2021"; - sha256 = "0dilfvl9jnyhi4skl6cry9wc300r693j0w82jjbq8yy3rx0i8qkd"; + sha256 = "0sffbkrpgyi1402mv4wzp9av6ky6rnb1d2m2dpf87wi7yfn7223f"; libName = "web_sys"; authors = [ "The wasm-bindgen Developers" @@ -14037,9 +14032,9 @@ rec { }; "yoke" = rec { crateName = "yoke"; - version = "0.8.2"; + version = "0.8.3"; edition = "2021"; - sha256 = "1jprcs7a98a5whvfs6r3jvfh1nnfp6zyijl7y4ywmn88lzywbs5b"; + sha256 = "1xgyj6c2lxj2bp891ynmhws87c6z7yyv2li1v0ss9di40hxf57vh"; authors = [ "Manish Goregaokar " ]; diff --git a/rust/operator-binary/src/controller/build/discovery.rs b/rust/operator-binary/src/controller/build/discovery.rs index 99dace99..f7409471 100644 --- a/rust/operator-binary/src/controller/build/discovery.rs +++ b/rust/operator-binary/src/controller/build/discovery.rs @@ -1,7 +1,5 @@ //! Build the discovery `ConfigMap` for the HbaseCluster. -use std::collections::BTreeMap; - use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, @@ -55,13 +53,7 @@ pub fn build_discovery_config_map(cluster: &ValidatedCluster) -> Result>() - .iter(), - ), + to_hadoop_xml(hbase_site.iter()), ) .build() .context(BuildConfigMapSnafu) diff --git a/rust/operator-binary/src/controller/build/properties/hbase_site.rs b/rust/operator-binary/src/controller/build/properties/hbase_site.rs index 2d010911..200de3d4 100644 --- a/rust/operator-binary/src/controller/build/properties/hbase_site.rs +++ b/rust/operator-binary/src/controller/build/properties/hbase_site.rs @@ -112,13 +112,7 @@ pub fn build( // configOverride come last config.extend(resolved_overrides(overrides)); - to_hadoop_xml( - config - .into_iter() - .map(|(k, v)| (k, Some(v))) - .collect::>() - .iter(), - ) + to_hadoop_xml(config.iter()) } #[cfg(test)] diff --git a/rust/operator-binary/src/controller/build/properties/security_properties.rs b/rust/operator-binary/src/controller/build/properties/security_properties.rs index 35ab38b4..69246812 100644 --- a/rust/operator-binary/src/controller/build/properties/security_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/security_properties.rs @@ -27,18 +27,18 @@ pub fn build( HbaseRole::RestServer => "30", }; - let mut config: BTreeMap> = BTreeMap::from([ + let mut config: BTreeMap = BTreeMap::from([ ( "networkaddress.cache.ttl".to_string(), - Some(positive_ttl.to_string()), + positive_ttl.to_string(), ), ( "networkaddress.cache.negative.ttl".to_string(), - Some("0".to_string()), + "0".to_string(), ), ]); // Overrides applied last so users win. - config.extend(resolved_overrides(overrides).map(|(key, value)| (key, Some(value)))); + config.extend(resolved_overrides(overrides)); to_java_properties_string(config.iter()) } diff --git a/rust/operator-binary/src/controller/build/properties/ssl_client.rs b/rust/operator-binary/src/controller/build/properties/ssl_client.rs index 3fc8093f..7cc7afbc 100644 --- a/rust/operator-binary/src/controller/build/properties/ssl_client.rs +++ b/rust/operator-binary/src/controller/build/properties/ssl_client.rs @@ -9,9 +9,9 @@ use crate::controller::build::properties::resolved_overrides; /// Renders `ssl-client.xml`. pub fn build(settings: BTreeMap, overrides: KeyValueConfigOverrides) -> String { - let mut config: BTreeMap> = BTreeMap::new(); - config.extend(settings.into_iter().map(|(k, v)| (k, Some(v)))); - config.extend(resolved_overrides(overrides).map(|(k, v)| (k, Some(v)))); + let mut config: BTreeMap = BTreeMap::new(); + config.extend(settings); + config.extend(resolved_overrides(overrides)); to_hadoop_xml(config.iter()) } diff --git a/rust/operator-binary/src/controller/build/properties/ssl_server.rs b/rust/operator-binary/src/controller/build/properties/ssl_server.rs index edf70aac..b14917d3 100644 --- a/rust/operator-binary/src/controller/build/properties/ssl_server.rs +++ b/rust/operator-binary/src/controller/build/properties/ssl_server.rs @@ -9,9 +9,9 @@ use crate::controller::build::properties::resolved_overrides; /// Renders `ssl-server.xml`. pub fn build(settings: BTreeMap, overrides: KeyValueConfigOverrides) -> String { - let mut config: BTreeMap> = BTreeMap::new(); - config.extend(settings.into_iter().map(|(k, v)| (k, Some(v)))); - config.extend(resolved_overrides(overrides).map(|(k, v)| (k, Some(v)))); + let mut config: BTreeMap = BTreeMap::new(); + config.extend(settings); + config.extend(resolved_overrides(overrides)); to_hadoop_xml(config.iter()) } From 424a3a4de032263f59ff09e17aff162159b68261 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 9 Jun 2026 19:45:16 +0200 Subject: [PATCH 22/56] fix: cleanup, move constants, fix comments --- .../controller/build/properties/hbase_site.rs | 10 +- .../src/controller/build/properties/mod.rs | 14 +- .../controller/build/properties/ssl_client.rs | 11 +- .../controller/build/properties/ssl_server.rs | 11 +- .../src/controller/validate.rs | 3 +- rust/operator-binary/src/crd/mod.rs | 81 +---------- rust/operator-binary/src/hbase_controller.rs | 5 +- rust/operator-binary/src/kerberos.rs | 126 +++++++----------- rust/operator-binary/src/main.rs | 4 +- rust/operator-binary/src/operations/pdb.rs | 3 +- rust/operator-binary/src/zookeeper.rs | 2 +- 11 files changed, 80 insertions(+), 190 deletions(-) diff --git a/rust/operator-binary/src/controller/build/properties/hbase_site.rs b/rust/operator-binary/src/controller/build/properties/hbase_site.rs index 200de3d4..2d28fad4 100644 --- a/rust/operator-binary/src/controller/build/properties/hbase_site.rs +++ b/rust/operator-binary/src/controller/build/properties/hbase_site.rs @@ -4,12 +4,10 @@ use std::collections::BTreeMap; -use stackable_operator::v2::{ - config_file_writer::to_hadoop_xml, config_overrides::KeyValueConfigOverrides, -}; +use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; use crate::{ - controller::build::properties::resolved_overrides, + controller::build::properties::build_xml_config, crd::{ AnyServiceConfig, HBASE_CLUSTER_DISTRIBUTED, HBASE_MASTER_PORT, HBASE_MASTER_UI_PORT, HBASE_REGIONSERVER_PORT, HBASE_REGIONSERVER_UI_PORT, HBASE_ROOTDIR, HbaseRole, @@ -110,9 +108,7 @@ pub fn build( }; // configOverride come last - config.extend(resolved_overrides(overrides)); - - to_hadoop_xml(config.iter()) + build_xml_config(config, overrides) } #[cfg(test)] diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index c4cf5bb8..770fce49 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -5,7 +5,9 @@ use std::collections::BTreeMap; -use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; +use stackable_operator::v2::{ + config_file_writer::to_hadoop_xml, config_overrides::KeyValueConfigOverrides, +}; pub mod hbase_env; pub mod hbase_site; @@ -31,6 +33,16 @@ fn resolved_overrides( defined_entries(overrides.overrides) } +/// Render an XML config file from base `settings` merged with user `overrides` +/// (overrides applied last, so users win), serialized to the Hadoop-XML on-wire format. +fn build_xml_config( + mut config: BTreeMap, + overrides: KeyValueConfigOverrides, +) -> String { + config.extend(resolved_overrides(overrides)); + to_hadoop_xml(config.iter()) +} + /// The names of the HBase config files assembled into the rolegroup `ConfigMap`. #[derive(Clone, Copy, Debug, strum::Display)] pub enum ConfigFileName { diff --git a/rust/operator-binary/src/controller/build/properties/ssl_client.rs b/rust/operator-binary/src/controller/build/properties/ssl_client.rs index 7cc7afbc..fa3231ac 100644 --- a/rust/operator-binary/src/controller/build/properties/ssl_client.rs +++ b/rust/operator-binary/src/controller/build/properties/ssl_client.rs @@ -1,18 +1,13 @@ //! Builds the `ssl-client.xml` config file: kerberos/TLS client settings + overrides. use std::collections::BTreeMap; -use stackable_operator::v2::{ - config_file_writer::to_hadoop_xml, config_overrides::KeyValueConfigOverrides, -}; +use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; -use crate::controller::build::properties::resolved_overrides; +use crate::controller::build::properties::build_xml_config; /// Renders `ssl-client.xml`. pub fn build(settings: BTreeMap, overrides: KeyValueConfigOverrides) -> String { - let mut config: BTreeMap = BTreeMap::new(); - config.extend(settings); - config.extend(resolved_overrides(overrides)); - to_hadoop_xml(config.iter()) + build_xml_config(settings, overrides) } #[cfg(test)] diff --git a/rust/operator-binary/src/controller/build/properties/ssl_server.rs b/rust/operator-binary/src/controller/build/properties/ssl_server.rs index b14917d3..45812736 100644 --- a/rust/operator-binary/src/controller/build/properties/ssl_server.rs +++ b/rust/operator-binary/src/controller/build/properties/ssl_server.rs @@ -1,18 +1,13 @@ //! Builds the `ssl-server.xml` config file: kerberos/TLS server settings + overrides. use std::collections::BTreeMap; -use stackable_operator::v2::{ - config_file_writer::to_hadoop_xml, config_overrides::KeyValueConfigOverrides, -}; +use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; -use crate::controller::build::properties::resolved_overrides; +use crate::controller::build::properties::build_xml_config; /// Renders `ssl-server.xml`. pub fn build(settings: BTreeMap, overrides: KeyValueConfigOverrides) -> String { - let mut config: BTreeMap = BTreeMap::new(); - config.extend(settings); - config.extend(resolved_overrides(overrides)); - to_hadoop_xml(config.iter()) + build_xml_config(settings, overrides) } #[cfg(test)] diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 792acd00..a10ca0a7 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -72,8 +72,7 @@ pub fn validate_cluster( for hbase_role in HbaseRole::iter() { let role_group_names = role_group_names(hbase, &hbase_role); - // masters and region servers are required (preserves the old build_role_properties check); - // rest servers are optional. + // masters and region servers are required; rest servers are optional. if role_group_names.is_empty() { match hbase_role { HbaseRole::Master | HbaseRole::RegionServer => { diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 1d1a5036..eb68a4d6 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -47,6 +47,7 @@ pub mod security; pub const APP_NAME: &str = "hbase"; pub const FIELD_MANAGER: &str = "hbase-operator"; +pub const OPERATOR_NAME: &str = "hbase.stackable.com"; // This constant is hard coded in hbase-entrypoint.sh // You need to change it there too. @@ -107,12 +108,6 @@ pub type RestServerRoleType = #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("the role [{role}] is invalid and does not exist in HBase"))] - InvalidRole { - source: strum::ParseError, - role: String, - }, - #[snafu(display("the HBase role [{role}] is missing from spec"))] MissingHbaseRole { role: String }, @@ -128,11 +123,6 @@ pub enum Error { #[snafu(display("role-group not found by name"))] RoleGroupNotFound, - #[snafu(display("failed to build listener volume source"))] - ListenerVolumeSource { - source: ListenerOperatorVolumeSourceBuilderError, - }, - #[snafu(display("failed to build listener volume"))] BuildListenerVolume { source: ListenerOperatorVolumeSourceBuilderError, @@ -441,11 +431,6 @@ impl v1alpha1::HbaseCluster { } } - /// The name of the role-level load-balanced Kubernetes `Service` - pub fn server_role_service_name(&self) -> Option { - self.metadata.name.clone() - } - /// Metadata about a server rolegroup pub fn server_rolegroup_ref( &self, @@ -544,70 +529,6 @@ impl HbaseRole { const DEFAULT_REST_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(5); - pub fn default_config( - &self, - cluster_name: &str, - hdfs_discovery_cm_name: &str, - ) -> HbaseConfigFragment { - let resources = match &self { - HbaseRole::Master => ResourcesFragment { - cpu: CpuLimitsFragment { - min: Some(Quantity("250m".to_owned())), - max: Some(Quantity("1".to_owned())), - }, - memory: MemoryLimitsFragment { - limit: Some(Quantity("1Gi".to_owned())), - runtime_limits: NoRuntimeLimitsFragment {}, - }, - storage: HbaseStorageConfigFragment {}, - }, - HbaseRole::RegionServer => ResourcesFragment { - cpu: CpuLimitsFragment { - min: Some(Quantity("250m".to_owned())), - max: Some(Quantity("1".to_owned())), - }, - memory: MemoryLimitsFragment { - limit: Some(Quantity("1Gi".to_owned())), - runtime_limits: NoRuntimeLimitsFragment {}, - }, - storage: HbaseStorageConfigFragment {}, - }, - HbaseRole::RestServer => ResourcesFragment { - cpu: CpuLimitsFragment { - min: Some(Quantity("100m".to_owned())), - max: Some(Quantity("400m".to_owned())), - }, - memory: MemoryLimitsFragment { - limit: Some(Quantity("512Mi".to_owned())), - runtime_limits: NoRuntimeLimitsFragment {}, - }, - storage: HbaseStorageConfigFragment {}, - }, - }; - - let graceful_shutdown_timeout = match &self { - HbaseRole::Master => Self::DEFAULT_MASTER_GRACEFUL_SHUTDOWN_TIMEOUT, - HbaseRole::RegionServer => Self::DEFAULT_REGION_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT, - HbaseRole::RestServer => Self::DEFAULT_REST_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT, - }; - - let requested_secret_lifetime = match &self { - HbaseRole::Master => Self::DEFAULT_MASTER_SECRET_LIFETIME, - HbaseRole::RegionServer => Self::DEFAULT_REGION_SECRET_LIFETIME, - HbaseRole::RestServer => Self::DEFAULT_REST_SECRET_LIFETIME, - }; - - HbaseConfigFragment { - hbase_rootdir: Some(default_hbase_rootdir()), - resources, - logging: product_logging::spec::default_logging(), - affinity: get_affinity(cluster_name, self, hdfs_discovery_cm_name), - graceful_shutdown_timeout: Some(graceful_shutdown_timeout), - requested_secret_lifetime: Some(requested_secret_lifetime), - listener_class: Some(DEFAULT_LISTENER_CLASS.to_string()), - } - } - /// Returns the name of the role as it is needed by the `bin/hbase {cli_role_name} start` command. pub fn cli_role_name(&self) -> String { match self { diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index d76d8768..5efccdf8 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -63,14 +63,13 @@ use stackable_operator::{ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ - OPERATOR_NAME, controller::build::{ discovery::build_discovery_config_map, properties::logging::{MAX_HBASE_LOG_FILES_SIZE, STACKABLE_LOG_DIR}, }, crd::{ APP_NAME, AnyServiceConfig, CONFIG_DIR_NAME, Container, HbaseClusterStatus, HbaseRole, - LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, merged_env, v1alpha1, + LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, OPERATOR_NAME, merged_env, v1alpha1, }, kerberos::{self, add_kerberos_pod_config}, operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, @@ -585,7 +584,7 @@ fn build_rolegroup_service( } /// The rolegroup metrics [`Service`] is a service that exposes metrics and a prometheus scraping label. -pub fn build_rolegroup_metrics_service( +fn build_rolegroup_metrics_service( hbase: &v1alpha1::HbaseCluster, hbase_role: &HbaseRole, rolegroup: &RoleGroupRef, diff --git a/rust/operator-binary/src/kerberos.rs b/rust/operator-binary/src/kerberos.rs index 998d8400..86f99de0 100644 --- a/rust/operator-binary/src/kerberos.rs +++ b/rust/operator-binary/src/kerberos.rs @@ -55,7 +55,7 @@ pub fn kerberos_config_properties( let principal_host_part = principal_host_part(hbase, cluster_info)?; - Ok(BTreeMap::from([ + let mut config = BTreeMap::from([ // Kerberos settings ( "hbase.security.authentication".to_string(), @@ -74,27 +74,6 @@ pub fn kerberos_config_properties( "hbase.rpc.engine".to_string(), "org.apache.hadoop.hbase.ipc.SecureRpcEngine".to_string(), ), - ( - "hbase.master.kerberos.principal".to_string(), - format!( - "{service_name}/{principal_host_part}", - service_name = kerberos_service_name() - ), - ), - ( - "hbase.regionserver.kerberos.principal".to_string(), - format!( - "{service_name}/{principal_host_part}", - service_name = kerberos_service_name() - ), - ), - ( - "hbase.rest.kerberos.principal".to_string(), - format!( - "{service_name}/{principal_host_part}", - service_name = kerberos_service_name() - ), - ), ( "hbase.master.keytab.file".to_string(), "/stackable/kerberos/keytab".to_string(), @@ -128,14 +107,16 @@ pub fn kerberos_config_properties( ("hbase.http.policy".to_string(), "HTTPS_ONLY".to_string()), // Recommended by the docs https://hbase.apache.org/book.html#hbase.ui.cache ("hbase.http.filter.no-store.enable".to_string(), "true".to_string()), - // Ḱey- and truststore come from ssl-server.xml and ssl-client.xml + // Key- and truststore come from ssl-server.xml and ssl-client.xml // Https for rest server ("hbase.rest.ssl.enabled".to_string(), "true".to_string()), ("hbase.rest.ssl.keystore.store".to_string(), format!("{TLS_STORE_DIR}/keystore.p12")), ("hbase.rest.ssl.keystore.password".to_string(), TLS_STORE_PASSWORD.to_string()), ("hbase.rest.ssl.keystore.type".to_string(), "pkcs12".to_string()), - ])) + ]); + config.extend(kerberos_principals(&principal_host_part)); + Ok(config) } pub fn kerberos_discovery_config_properties( @@ -148,35 +129,16 @@ pub fn kerberos_discovery_config_properties( let principal_host_part = principal_host_part(hbase, cluster_info)?; - Ok(BTreeMap::from([ + let mut config = BTreeMap::from([ ( "hbase.security.authentication".to_string(), "kerberos".to_string(), ), ("hbase.rpc.protection".to_string(), "privacy".to_string()), ("hbase.ssl.enabled".to_string(), "true".to_string()), - ( - "hbase.master.kerberos.principal".to_string(), - format!( - "{service_name}/{principal_host_part}", - service_name = kerberos_service_name() - ), - ), - ( - "hbase.regionserver.kerberos.principal".to_string(), - format!( - "{service_name}/{principal_host_part}", - service_name = kerberos_service_name() - ), - ), - ( - "hbase.rest.kerberos.principal".to_string(), - format!( - "{service_name}/{principal_host_part}", - service_name = kerberos_service_name() - ), - ), - ])) + ]); + config.extend(kerberos_principals(&principal_host_part)); + Ok(config) } pub fn kerberos_ssl_server_settings(hbase: &v1alpha1::HbaseCluster) -> BTreeMap { @@ -184,19 +146,8 @@ pub fn kerberos_ssl_server_settings(hbase: &v1alpha1::HbaseCluster) -> BTreeMap< return BTreeMap::new(); } - BTreeMap::from([ - ( - "ssl.server.truststore.location".to_string(), - format!("{TLS_STORE_DIR}/truststore.p12"), - ), - ( - "ssl.server.truststore.type".to_string(), - "pkcs12".to_string(), - ), - ( - "ssl.server.truststore.password".to_string(), - TLS_STORE_PASSWORD.to_string(), - ), + let mut settings = truststore_settings("server"); + settings.extend([ ( "ssl.server.keystore.location".to_string(), format!("{TLS_STORE_DIR}/keystore.p12"), @@ -206,7 +157,8 @@ pub fn kerberos_ssl_server_settings(hbase: &v1alpha1::HbaseCluster) -> BTreeMap< "ssl.server.keystore.password".to_string(), TLS_STORE_PASSWORD.to_string(), ), - ]) + ]); + settings } pub fn kerberos_ssl_client_settings(hbase: &v1alpha1::HbaseCluster) -> BTreeMap { @@ -214,20 +166,7 @@ pub fn kerberos_ssl_client_settings(hbase: &v1alpha1::HbaseCluster) -> BTreeMap< return BTreeMap::new(); } - BTreeMap::from([ - ( - "ssl.client.truststore.location".to_string(), - format!("{TLS_STORE_DIR}/truststore.p12"), - ), - ( - "ssl.client.truststore.type".to_string(), - "pkcs12".to_string(), - ), - ( - "ssl.client.truststore.password".to_string(), - TLS_STORE_PASSWORD.to_string(), - ), - ]) + truststore_settings("client") } pub fn add_kerberos_pod_config( @@ -292,6 +231,43 @@ pub fn add_kerberos_pod_config( Ok(()) } +/// The `hbase.{master,regionserver,rest}.kerberos.principal` entries shared by the main +/// and discovery config. All roles use the same `hbase` service principal (see +/// [`kerberos_service_name`]). +fn kerberos_principals(principal_host_part: &str) -> [(String, String); 3] { + let principal = format!( + "{service_name}/{principal_host_part}", + service_name = kerberos_service_name() + ); + [ + ( + "hbase.master.kerberos.principal".to_string(), + principal.clone(), + ), + ( + "hbase.regionserver.kerberos.principal".to_string(), + principal.clone(), + ), + ("hbase.rest.kerberos.principal".to_string(), principal), + ] +} + +/// The `ssl.{role}.truststore.*` entries (location/type/password) shared by the server and +/// client TLS settings. `role` is either `"server"` or `"client"`. +fn truststore_settings(role: &str) -> BTreeMap { + BTreeMap::from([ + ( + format!("ssl.{role}.truststore.location"), + format!("{TLS_STORE_DIR}/truststore.p12"), + ), + (format!("ssl.{role}.truststore.type"), "pkcs12".to_string()), + ( + format!("ssl.{role}.truststore.password"), + TLS_STORE_PASSWORD.to_string(), + ), + ]) +} + fn principal_host_part( hbase: &v1alpha1::HbaseCluster, cluster_info: &KubernetesClusterInfo, diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 2bd815b8..aa923145 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -33,7 +33,7 @@ use stackable_operator::{ }; use crate::{ - crd::{HbaseCluster, HbaseClusterVersion, v1alpha1}, + crd::{HbaseCluster, HbaseClusterVersion, OPERATOR_NAME, v1alpha1}, webhooks::conversion::create_webhook_server, }; @@ -51,8 +51,6 @@ mod built_info { include!(concat!(env!("OUT_DIR"), "/built.rs")); } -const OPERATOR_NAME: &str = "hbase.stackable.com"; - #[derive(Parser)] #[clap(about, author)] struct Opts { diff --git a/rust/operator-binary/src/operations/pdb.rs b/rust/operator-binary/src/operations/pdb.rs index db00bcbe..76b06b0f 100644 --- a/rust/operator-binary/src/operations/pdb.rs +++ b/rust/operator-binary/src/operations/pdb.rs @@ -5,8 +5,7 @@ use stackable_operator::{ }; use crate::{ - OPERATOR_NAME, - crd::{APP_NAME, HbaseRole, v1alpha1}, + crd::{APP_NAME, HbaseRole, OPERATOR_NAME, v1alpha1}, hbase_controller::HBASE_CONTROLLER_NAME, }; diff --git a/rust/operator-binary/src/zookeeper.rs b/rust/operator-binary/src/zookeeper.rs index ce558354..51aa27b6 100644 --- a/rust/operator-binary/src/zookeeper.rs +++ b/rust/operator-binary/src/zookeeper.rs @@ -107,7 +107,7 @@ impl ZookeeperConnectionInformation { // Check if a user points to a discovery CM of a HBaseCluster rather than a ZNode. if chroot == "/" { warn!( - "It is recommended to let the HBase cluster point to a discovery ConfigMap of a ZNode rater than a ZookeeperCluster. \ + "It is recommended to let the HBase cluster point to a discovery ConfigMap of a ZNode rather than a ZookeeperCluster. \ This prevents accidental reuse of the same Zookeeper path for multiple product instances. \ See https://docs.stackable.tech/home/stable/zookeeper/getting_started/first_steps for details" ); From f60d49e49ccb823ab957b42ebc04d4c68485d317 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 9 Jun 2026 20:09:43 +0200 Subject: [PATCH 23/56] refactor: consolidate ValidatedRoleGroupConfig --- .../src/controller/validate.rs | 37 +++++++++++++++++-- rust/operator-binary/src/crd/mod.rs | 23 ++++++------ rust/operator-binary/src/hbase_controller.rs | 25 +++++++++---- 3 files changed, 63 insertions(+), 22 deletions(-) diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index a10ca0a7..dcdc5dd6 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, str::FromStr}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ @@ -6,7 +6,10 @@ use stackable_operator::{ config::merge::Merge, role_utils::GenericRoleConfig, utils::cluster_info::KubernetesClusterInfo, - v2::controller_utils::{get_cluster_name, get_namespace, get_uid}, + v2::{ + builder::pod::container::{self, EnvVarName, EnvVarSet}, + controller_utils::{get_cluster_name, get_namespace, get_uid}, + }, }; use strum::IntoEnumIterator; @@ -48,6 +51,12 @@ pub enum Error { #[snafu(display("failed to construct role-specific JVM arguments"))] ConstructJvmArgument { source: crate::config::jvm::Error }, + + #[snafu(display("the environment variable override name {name:?} is invalid"))] + InvalidEnvVarName { + source: container::Error, + name: String, + }, } pub fn validate_cluster( @@ -102,12 +111,21 @@ pub fn validate_cluster( ) .context(FailedToResolveConfigSnafu)?; + let rolegroup_ref = + hbase.server_rolegroup_ref(hbase_role.to_string(), rolegroup_name.clone()); + group_configs.insert( rolegroup_name.clone(), ValidatedRoleGroupConfig { + replicas: hbase.replicas(&hbase_role, &rolegroup_ref), merged_config, config_overrides: merged_config_overrides(hbase, &hbase_role, &rolegroup_name), - env_overrides: merged_env_overrides(hbase, &hbase_role, &rolegroup_name), + env_overrides: env_var_set(merged_env_overrides( + hbase, + &hbase_role, + &rolegroup_name, + ))?, + pod_overrides: hbase.merged_pod_overrides(&hbase_role, &rolegroup_ref), non_heap_jvm_args: construct_role_specific_non_heap_jvm_args( hbase, &hbase_role, @@ -285,6 +303,19 @@ fn merged_env_overrides( env_overrides } +/// Converts merged env override pairs into a type-safe [`EnvVarSet`], validating each name so that +/// invalid environment variable names are rejected during validation instead of producing a broken +/// Pod. +fn env_var_set(env_overrides: BTreeMap) -> Result { + let mut set = EnvVarSet::new(); + for (name, value) in env_overrides { + let env_var_name = + EnvVarName::from_str(&name).context(InvalidEnvVarNameSnafu { name: name.clone() })?; + set = set.with_value(&env_var_name, value); + } + Ok(set) +} + #[cfg(test)] mod tests { use indoc::indoc; diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index eb68a4d6..cb80e3fd 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -351,12 +351,12 @@ impl v1alpha1::HbaseCluster { }) } - pub fn merge_pod_overrides( + /// Returns the merged (role <- role group) pod template overrides for the given role group. + pub fn merged_pod_overrides( &self, - pod_template: &mut PodTemplateSpec, role: &HbaseRole, role_group_ref: &RoleGroupRef, - ) { + ) -> PodTemplateSpec { let (role_pod_overrides, role_group_pod_overrides) = match role { HbaseRole::Master => ( self.spec @@ -393,41 +393,40 @@ impl v1alpha1::HbaseCluster { ), }; + let mut merged = PodTemplateSpec::default(); if let Some(rpo) = role_pod_overrides { - pod_template.merge_from(rpo); + merged.merge_from(rpo); } if let Some(rgpo) = role_group_pod_overrides { - pod_template.merge_from(rgpo); + merged.merge_from(rgpo); } + merged } pub fn replicas( &self, hbase_role: &HbaseRole, role_group_ref: &RoleGroupRef, - ) -> Option { + ) -> Option { match hbase_role { HbaseRole::Master => self .spec .masters .as_ref() .and_then(|r| r.role_groups.get(&role_group_ref.role_group)) - .and_then(|rg| rg.replicas) - .map(i32::from), + .and_then(|rg| rg.replicas), HbaseRole::RegionServer => self .spec .region_servers .as_ref() .and_then(|r| r.role_groups.get(&role_group_ref.role_group)) - .and_then(|rg| rg.replicas) - .map(i32::from), + .and_then(|rg| rg.replicas), HbaseRole::RestServer => self .spec .rest_servers .as_ref() .and_then(|r| r.role_groups.get(&role_group_ref.role_group)) - .and_then(|rg| rg.replicas) - .map(i32::from), + .and_then(|rg| rg.replicas), } } diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 5efccdf8..b487ca83 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -19,11 +19,12 @@ use stackable_operator::{ commons::{product_image_selection::ResolvedProductImage, rbac::build_rbac_resources}, constants::RESTART_CONTROLLER_ENABLED_LABEL, k8s_openapi::{ + DeepMerge, api::{ apps::v1::{StatefulSet, StatefulSetSpec}, core::v1::{ - ConfigMapVolumeSource, ContainerPort, EnvVar, Probe, Service, ServiceAccount, - ServicePort, ServiceSpec, TCPSocketAction, Volume, + ConfigMapVolumeSource, ContainerPort, EnvVar, PodTemplateSpec, Probe, Service, + ServiceAccount, ServicePort, ServiceSpec, TCPSocketAction, Volume, }, }, apimachinery::pkg::{ @@ -54,6 +55,7 @@ use stackable_operator::{ }, v2::{ HasName, HasUid, NameIsValidLabelValue, + builder::pod::container::EnvVarSet, types::{ kubernetes::{NamespaceName, Uid}, operator::ClusterName, @@ -218,12 +220,19 @@ pub struct ValidatedRoleConfig { } /// Per-rolegroup configuration: the merged CRD config plus the merged -/// (role <- role group) `configOverrides` and `envOverrides`. +/// (role <- role group) `configOverrides`, `envOverrides` and `podOverrides`. +/// +/// This carries every override channel so that the build step is a pure function of +/// [`ValidatedCluster`] and never has to reach back into the raw `HbaseCluster`. #[derive(Clone, Debug)] pub struct ValidatedRoleGroupConfig { + /// The desired number of replicas (`None` lets Kubernetes default to 1). + pub replicas: Option, pub merged_config: AnyServiceConfig, pub config_overrides: v1alpha1::HbaseConfigOverrides, - pub env_overrides: BTreeMap, + pub env_overrides: EnvVarSet, + /// Merged (role <- role group) pod template overrides. + pub pod_overrides: PodTemplateSpec, /// Pre-resolved role-specific non-heap JVM args (operator-generated + role/role-group overrides). pub non_heap_jvm_args: String, } @@ -713,7 +722,9 @@ fn build_rolegroup_statefulset( ("HBASE_CONF_DIR".to_string(), CONFIG_DIR_NAME.to_string()), ("HADOOP_CONF_DIR".to_string(), CONFIG_DIR_NAME.to_string()), ]); - env_map.extend(validated_rg_config.env_overrides.clone()); + for env_var in validated_rg_config.env_overrides.clone() { + env_map.insert(env_var.name, env_var.value.unwrap_or_default()); + } let mut merged_env = merged_env(&env_map); // This env var is set for all roles to avoid bash's "unbound variable" errors merged_env.extend([ @@ -904,7 +915,7 @@ fn build_rolegroup_statefulset( let mut pod_template = pod_builder.build_template(); - hbase.merge_pod_overrides(&mut pod_template, hbase_role, rolegroup_ref); + pod_template.merge_from(validated_rg_config.pod_overrides.clone()); let metadata = ObjectMetaBuilder::new() .name_and_namespace(hbase) @@ -931,7 +942,7 @@ fn build_rolegroup_statefulset( let statefulset_spec = StatefulSetSpec { pod_management_policy: Some("Parallel".to_string()), - replicas: hbase.replicas(hbase_role, rolegroup_ref), + replicas: validated_rg_config.replicas.map(i32::from), selector: LabelSelector { match_labels: Some(statefulset_match_labels.into()), ..LabelSelector::default() From 0a8047d62b0a20bf3b8334d7f5d83c8dd0ed5448 Mon Sep 17 00:00:00 2001 From: maltesander Date: Wed, 10 Jun 2026 08:26:10 +0200 Subject: [PATCH 24/56] Update rust/operator-binary/src/hbase_controller.rs Co-authored-by: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> --- rust/operator-binary/src/hbase_controller.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index b487ca83..aa56e540 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -720,6 +720,7 @@ fn build_rolegroup_statefulset( let mut env_map: BTreeMap = BTreeMap::from([ ("HBASE_CONF_DIR".to_string(), CONFIG_DIR_NAME.to_string()), + // required by phoenix (for cases where Kerberos is enabled): see https://issues.apache.org/jira/browse/PHOENIX-2369 ("HADOOP_CONF_DIR".to_string(), CONFIG_DIR_NAME.to_string()), ]); for env_var in validated_rg_config.env_overrides.clone() { From 63c17e44e40bc4b6ae05c28d8831df3f653126d2 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 10 Jun 2026 11:26:19 +0200 Subject: [PATCH 25/56] fix: bump stackable-operator for non optional KeyValueConfigOverrides --- Cargo.lock | 34 +-- Cargo.nix | 34 +-- extra/crds.yaml | 210 +++--------------- .../controller/build/properties/hbase_env.rs | 3 +- .../src/controller/build/properties/mod.rs | 21 +- .../build/properties/security_properties.rs | 4 +- 6 files changed, 69 insertions(+), 237 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 15c16088..f71f836e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1514,7 +1514,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "darling", "regex", @@ -2321,9 +2321,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.3" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ "aho-corasick", "memchr", @@ -2344,9 +2344,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "relative-path" @@ -2899,7 +2899,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "const-oid", "ecdsa", @@ -2946,7 +2946,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.111.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "base64", "clap", @@ -2990,7 +2990,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "darling", "proc-macro2", @@ -3001,7 +3001,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "jiff", "k8s-openapi", @@ -3018,7 +3018,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.4" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "axum", "clap", @@ -3042,7 +3042,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "kube", "schemars", @@ -3056,7 +3056,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "convert_case", "convert_case_extras", @@ -3074,7 +3074,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "arc-swap", "async-trait", @@ -4000,18 +4000,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.50" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.50" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.nix b/Cargo.nix index cfa748a7..e44f843f 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4826,7 +4826,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "k8s_version"; @@ -7563,9 +7563,9 @@ rec { }; "regex" = rec { crateName = "regex"; - version = "1.12.3"; + version = "1.12.4"; edition = "2021"; - sha256 = "0xp2q0x7ybmpa5zlgaz00p8zswcirj9h8nry3rxxsdwi9fhm81z1"; + sha256 = "1fm6si2xpmhwqflabdqsakc0qkq718wx2ljl37nbj75fb5vjnagi"; authors = [ "The Rust Project Developers" "Andrew Gallant " @@ -7682,9 +7682,9 @@ rec { }; "regex-syntax" = rec { crateName = "regex-syntax"; - version = "0.8.10"; + version = "0.8.11"; edition = "2021"; - sha256 = "02jx311ka0daxxc7v45ikzhcl3iydjbbb0mdrpc1xgg8v7c7v2fw"; + sha256 = "1m25h5q2wp976fb9gc3dsc9l99svcvd5cri8lncb51c46ydgzxnn"; libName = "regex_syntax"; authors = [ "The Rust Project Developers" @@ -9520,7 +9520,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_certs"; @@ -9720,7 +9720,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_operator"; @@ -9914,7 +9914,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; procMacro = true; @@ -9949,7 +9949,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_shared"; @@ -10030,7 +10030,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_telemetry"; @@ -10140,7 +10140,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_versioned"; @@ -10190,7 +10190,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; procMacro = true; @@ -10258,7 +10258,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; }; libName = "stackable_webhook"; @@ -14098,9 +14098,9 @@ rec { }; "zerocopy" = rec { crateName = "zerocopy"; - version = "0.8.50"; + version = "0.8.52"; edition = "2021"; - sha256 = "1laahnfxs4qyfb1fdf5nbb2qfshi72b1hbi0ffp2zy2m1r7ms1iv"; + sha256 = "0gv563swc1yn3k8w3wjj07a8q293rkx99nfp3a25vzzmbycj446f"; authors = [ "Joshua Liebow-Feeser " "Jack Wrenn " @@ -14134,9 +14134,9 @@ rec { }; "zerocopy-derive" = rec { crateName = "zerocopy-derive"; - version = "0.8.50"; + version = "0.8.52"; edition = "2021"; - sha256 = "0fdnr9qslx1hbn2i9rsvy9s95mychfy2vj90ajsjm2basccinqqb"; + sha256 = "0c3rhsh4sd9kdym4z55zprybjkydy9y2gvw75d72aapcfa5z7rqs"; procMacro = true; libName = "zerocopy_derive"; authors = [ diff --git a/extra/crds.yaml b/extra/crds.yaml index 13eb43b5..d941cff2 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -495,58 +495,33 @@ spec: properties: hbase-env.sh: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object hbase-site.xml: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object security.properties: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object ssl-client.xml: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object ssl-server.xml: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object type: object envOverrides: @@ -941,58 +916,33 @@ spec: properties: hbase-env.sh: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object hbase-site.xml: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object security.properties: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object ssl-client.xml: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object ssl-server.xml: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object type: object envOverrides: @@ -1409,58 +1359,33 @@ spec: properties: hbase-env.sh: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object hbase-site.xml: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object security.properties: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object ssl-client.xml: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object ssl-server.xml: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object type: object envOverrides: @@ -1890,58 +1815,33 @@ spec: properties: hbase-env.sh: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object hbase-site.xml: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object security.properties: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object ssl-client.xml: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object ssl-server.xml: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object type: object envOverrides: @@ -2309,58 +2209,33 @@ spec: properties: hbase-env.sh: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object hbase-site.xml: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object security.properties: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object ssl-client.xml: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object ssl-server.xml: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object type: object envOverrides: @@ -2755,58 +2630,33 @@ spec: properties: hbase-env.sh: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object hbase-site.xml: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object security.properties: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object ssl-client.xml: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object ssl-server.xml: additionalProperties: - nullable: true type: string default: {} - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. + description: Flat key-value overrides for `*.properties`, Hadoop XML, etc. type: object type: object envOverrides: diff --git a/rust/operator-binary/src/controller/build/properties/hbase_env.rs b/rust/operator-binary/src/controller/build/properties/hbase_env.rs index bed4d356..15ec10a5 100644 --- a/rust/operator-binary/src/controller/build/properties/hbase_env.rs +++ b/rust/operator-binary/src/controller/build/properties/hbase_env.rs @@ -7,7 +7,6 @@ use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; use crate::{ config::jvm::{self, construct_global_jvm_args, construct_hbase_heapsize_env}, - controller::build::properties::resolved_overrides, crd::{AnyServiceConfig, HbaseRole}, }; @@ -50,7 +49,7 @@ pub fn build( } // configOverride come last - env.extend(resolved_overrides(overrides)); + env.extend(overrides); Ok(env .iter() diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index 770fce49..fad65a89 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -16,30 +16,13 @@ pub mod security_properties; pub mod ssl_client; pub mod ssl_server; -/// Keep only the set (`Some`) entries of a `key -> optional value` map, as `(key, value)` pairs. -fn defined_entries( - entries: BTreeMap>, -) -> impl Iterator { - entries - .into_iter() - .filter_map(|(key, value)| value.map(|value| (key, value))) -} - -/// Resolve user-provided [`KeyValueConfigOverrides`] into the key/value pairs to merge -/// into a config file, dropping entries whose value is unset (`None`). -fn resolved_overrides( - overrides: KeyValueConfigOverrides, -) -> impl Iterator { - defined_entries(overrides.overrides) -} - /// Render an XML config file from base `settings` merged with user `overrides` /// (overrides applied last, so users win), serialized to the Hadoop-XML on-wire format. fn build_xml_config( mut config: BTreeMap, overrides: KeyValueConfigOverrides, ) -> String { - config.extend(resolved_overrides(overrides)); + config.extend(overrides); to_hadoop_xml(config.iter()) } @@ -86,7 +69,7 @@ pub(crate) mod test_support { KeyValueConfigOverrides { overrides: pairs .iter() - .map(|(k, v)| (k.to_string(), Some(v.to_string()))) + .map(|(k, v)| (k.to_string(), v.to_string())) .collect(), } } diff --git a/rust/operator-binary/src/controller/build/properties/security_properties.rs b/rust/operator-binary/src/controller/build/properties/security_properties.rs index 69246812..82e18868 100644 --- a/rust/operator-binary/src/controller/build/properties/security_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/security_properties.rs @@ -11,7 +11,7 @@ use stackable_operator::v2::{ config_overrides::KeyValueConfigOverrides, }; -use crate::{controller::build::properties::resolved_overrides, crd::HbaseRole}; +use crate::crd::HbaseRole; /// Renders `security.properties`: role-specific DNS cache TTLs plus user overrides. pub fn build( @@ -38,7 +38,7 @@ pub fn build( ), ]); // Overrides applied last so users win. - config.extend(resolved_overrides(overrides)); + config.extend(overrides); to_java_properties_string(config.iter()) } From 2a4064aeeac943057df4803ee2c73b9fb49214b4 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 10 Jun 2026 11:28:34 +0200 Subject: [PATCH 26/56] fix: update hashes --- Cargo.nix | 18 +++++++++--------- crate-hashes.json | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index c418ff62..f222688f 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4827,7 +4827,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "451088f77acee6c3d296754698260256c250ecb2"; - sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "k8s_version"; authors = [ @@ -9521,7 +9521,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "451088f77acee6c3d296754698260256c250ecb2"; - sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_certs"; authors = [ @@ -9721,7 +9721,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "451088f77acee6c3d296754698260256c250ecb2"; - sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_operator"; authors = [ @@ -9915,7 +9915,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "451088f77acee6c3d296754698260256c250ecb2"; - sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -9950,7 +9950,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "451088f77acee6c3d296754698260256c250ecb2"; - sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_shared"; authors = [ @@ -10031,7 +10031,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "451088f77acee6c3d296754698260256c250ecb2"; - sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_telemetry"; authors = [ @@ -10141,7 +10141,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "451088f77acee6c3d296754698260256c250ecb2"; - sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_versioned"; authors = [ @@ -10191,7 +10191,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "451088f77acee6c3d296754698260256c250ecb2"; - sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -10259,7 +10259,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "451088f77acee6c3d296754698260256c250ecb2"; - sha256 = "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_webhook"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index cd03561e..c9a6e6a9 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "0idpq1xdkr94zrd95xsvrwkj3bvzbii9a7qmw23rn5w4yiwgmj96", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file From b32e0977ff0d0b7758b9aa8849cf922e8c21b437 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 10 Jun 2026 20:03:16 +0200 Subject: [PATCH 27/56] refactor: move ValidatedCluster structs to controller/mod.rs --- .../src/controller/build/config_map.rs | 10 +- .../src/controller/build/discovery.rs | 4 +- rust/operator-binary/src/controller/mod.rs | 163 +++++++++++++++++ .../src/controller/validate.rs | 6 +- rust/operator-binary/src/hbase_controller.rs | 173 ++---------------- 5 files changed, 186 insertions(+), 170 deletions(-) diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 199f991f..b011e10c 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -10,11 +10,15 @@ use stackable_operator::{ }; use crate::{ - controller::build::properties::{ - ConfigFileName, hbase_env, hbase_site, logging, security_properties, ssl_client, ssl_server, + controller::{ + ValidatedCluster, + build::properties::{ + ConfigFileName, hbase_env, hbase_site, logging, security_properties, ssl_client, + ssl_server, + }, }, crd::{HbaseRole, v1alpha1}, - hbase_controller::{ValidatedCluster, build_recommended_labels}, + hbase_controller::build_recommended_labels, }; #[derive(Snafu, Debug)] diff --git a/rust/operator-binary/src/controller/build/discovery.rs b/rust/operator-binary/src/controller/build/discovery.rs index f7409471..0e488ca6 100644 --- a/rust/operator-binary/src/controller/build/discovery.rs +++ b/rust/operator-binary/src/controller/build/discovery.rs @@ -8,9 +8,9 @@ use stackable_operator::{ }; use crate::{ - controller::build::properties::ConfigFileName, + controller::{ValidatedCluster, build::properties::ConfigFileName}, crd::HbaseRole, - hbase_controller::{ValidatedCluster, build_recommended_labels}, + hbase_controller::build_recommended_labels, }; type Result = std::result::Result; diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index a1196d5d..b91db5d2 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -1,3 +1,166 @@ pub mod build; pub mod dereference; pub mod validate; + +use std::collections::BTreeMap; + +use stackable_operator::{ + commons::product_image_selection::ResolvedProductImage, + k8s_openapi::{api::core::v1::PodTemplateSpec, apimachinery::pkg::apis::meta::v1::ObjectMeta}, + kube::Resource, + v2::{ + HasName, HasUid, NameIsValidLabelValue, + builder::pod::container::EnvVarSet, + types::{ + kubernetes::{NamespaceName, Uid}, + operator::ClusterName, + }, + }, +}; + +use crate::{ + crd::{AnyServiceConfig, HbaseRole, v1alpha1}, + security::opa::HbaseOpaConfig, + zookeeper::ZookeeperConnectionInformation, +}; + +/// The validated cluster: proves that config merging and validation succeeded for +/// every role and role group before any resources are created. +#[derive(Clone, Debug)] +pub struct ValidatedCluster { + /// Backs the [`Resource`] implementation (provides `meta()`/`name_any()`) so the build + /// functions can derive `ObjectMeta`, owner references and labels without the full + /// `HbaseCluster` object. Holds only name, namespace and uid. + metadata: ObjectMeta, + /// The logical (and Kubernetes object) name of the cluster. + pub name: ClusterName, + /// The namespace the cluster lives in. Part of the cluster identity; currently consumed via + /// the [`Resource`] metadata (`name_and_namespace`) rather than read directly. + #[allow(dead_code)] + pub namespace: NamespaceName, + /// The UID of the `HbaseCluster` object, used to build owner references. + pub uid: Uid, + pub image: ResolvedProductImage, + pub cluster_config: ValidatedClusterConfig, + pub role_group_configs: BTreeMap>, + pub role_configs: BTreeMap, +} + +impl ValidatedCluster { + #[allow(clippy::too_many_arguments)] + pub fn new( + name: ClusterName, + namespace: NamespaceName, + uid: Uid, + image: ResolvedProductImage, + cluster_config: ValidatedClusterConfig, + role_group_configs: BTreeMap>, + role_configs: BTreeMap, + ) -> Self { + Self { + metadata: ObjectMeta { + name: Some(name.to_string()), + namespace: Some(namespace.to_string()), + uid: Some(uid.to_string()), + ..ObjectMeta::default() + }, + name, + namespace, + uid, + image, + cluster_config, + role_group_configs, + role_configs, + } + } +} + +impl Resource for ValidatedCluster { + type DynamicType = ::DynamicType; + type Scope = ::Scope; + + fn group(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha1::HbaseCluster::group(dt) + } + + fn version(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha1::HbaseCluster::version(dt) + } + + fn kind(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha1::HbaseCluster::kind(dt) + } + + fn plural(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { + v1alpha1::HbaseCluster::plural(dt) + } + + fn meta(&self) -> &ObjectMeta { + &self.metadata + } + + fn meta_mut(&mut self) -> &mut ObjectMeta { + &mut self.metadata + } +} + +impl HasName for ValidatedCluster { + fn to_name(&self) -> String { + self.name.to_string() + } +} + +impl HasUid for ValidatedCluster { + fn to_uid(&self) -> Uid { + self.uid.clone() + } +} + +impl NameIsValidLabelValue for ValidatedCluster { + fn to_label_value(&self) -> String { + self.name.to_label_value() + } +} + +/// Cluster-wide settings resolved once during validation. +#[derive(Clone, Debug)] +pub struct ValidatedClusterConfig { + // Pre-resolved OPA connection configuration. + pub hbase_opa_config: Option, + pub kerberos_enabled: bool, + // Pre-resolved kerberos properties for hbase-site.xml (empty when kerberos is disabled). + pub hbase_site_kerberos_config: BTreeMap, + // Pre-resolved kerberos properties for the discovery `hbase-site.xml` exposed to clients + // (empty when kerberos is disabled). + pub discovery_kerberos_config: BTreeMap, + // Pre-resolved ssl-server.xml settings (empty when HTTPS is disabled). + pub ssl_server_settings: BTreeMap, + // Pre-resolved ssl-client.xml settings (empty when HTTPS is disabled). + pub ssl_client_settings: BTreeMap, + // Pre-resolved zookeeper connection settings. + pub zookeeper_connection_information: ZookeeperConnectionInformation, +} + +/// Per-role configuration extracted during validation. +#[derive(Clone, Debug)] +pub struct ValidatedRoleConfig { + pub pdb: stackable_operator::commons::pdb::PdbConfig, +} + +/// Per-rolegroup configuration: the merged CRD config plus the merged +/// (role <- role group) `configOverrides`, `envOverrides` and `podOverrides`. +/// +/// This carries every override channel so that the build step is a pure function of +/// [`ValidatedCluster`] and never has to reach back into the raw `HbaseCluster`. +#[derive(Clone, Debug)] +pub struct ValidatedRoleGroupConfig { + /// The desired number of replicas (`None` lets Kubernetes default to 1). + pub replicas: Option, + pub merged_config: AnyServiceConfig, + pub config_overrides: v1alpha1::HbaseConfigOverrides, + pub env_overrides: EnvVarSet, + /// Merged (role <- role group) pod template overrides. + pub pod_overrides: PodTemplateSpec, + /// Pre-resolved role-specific non-heap JVM args (operator-generated + role/role-group overrides). + pub non_heap_jvm_args: String, +} diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index dcdc5dd6..b7787329 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -15,11 +15,11 @@ use strum::IntoEnumIterator; use crate::{ config::jvm::construct_role_specific_non_heap_jvm_args, - controller::dereference::DereferencedObjects, - crd::{HbaseRole, v1alpha1}, - hbase_controller::{ + controller::{ ValidatedCluster, ValidatedClusterConfig, ValidatedRoleConfig, ValidatedRoleGroupConfig, + dereference::DereferencedObjects, }, + crd::{HbaseRole, v1alpha1}, kerberos::{ self, kerberos_config_properties, kerberos_discovery_config_properties, kerberos_ssl_client_settings, kerberos_ssl_server_settings, diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index aa56e540..9a4dad6c 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -23,14 +23,11 @@ use stackable_operator::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec}, core::v1::{ - ConfigMapVolumeSource, ContainerPort, EnvVar, PodTemplateSpec, Probe, Service, - ServiceAccount, ServicePort, ServiceSpec, TCPSocketAction, Volume, + ConfigMapVolumeSource, ContainerPort, EnvVar, Probe, Service, ServiceAccount, + ServicePort, ServiceSpec, TCPSocketAction, Volume, }, }, - apimachinery::pkg::{ - apis::meta::v1::{LabelSelector, ObjectMeta}, - util::intstr::IntOrString, - }, + apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, }, kube::{ Resource, ResourceExt, @@ -53,30 +50,23 @@ use stackable_operator::{ compute_conditions, operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, }, - v2::{ - HasName, HasUid, NameIsValidLabelValue, - builder::pod::container::EnvVarSet, - types::{ - kubernetes::{NamespaceName, Uid}, - operator::ClusterName, - }, - }, }; use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ - controller::build::{ - discovery::build_discovery_config_map, - properties::logging::{MAX_HBASE_LOG_FILES_SIZE, STACKABLE_LOG_DIR}, + controller::{ + ValidatedRoleGroupConfig, + build::{ + discovery::build_discovery_config_map, + properties::logging::{MAX_HBASE_LOG_FILES_SIZE, STACKABLE_LOG_DIR}, + }, }, crd::{ - APP_NAME, AnyServiceConfig, CONFIG_DIR_NAME, Container, HbaseClusterStatus, HbaseRole, - LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, OPERATOR_NAME, merged_env, v1alpha1, + APP_NAME, CONFIG_DIR_NAME, Container, HbaseClusterStatus, HbaseRole, LISTENER_VOLUME_DIR, + LISTENER_VOLUME_NAME, OPERATOR_NAME, merged_env, v1alpha1, }, kerberos::{self, add_kerberos_pod_config}, operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, - security::opa::HbaseOpaConfig, - zookeeper::ZookeeperConnectionInformation, }; pub const HBASE_CONTROLLER_NAME: &str = "hbasecluster"; @@ -96,147 +86,6 @@ pub struct Ctx { pub operator_environment: OperatorEnvironmentOptions, } -/// The validated cluster: proves that config merging and validation succeeded for -/// every role and role group before any resources are created. -#[derive(Clone, Debug)] -pub struct ValidatedCluster { - /// Backs the [`Resource`] implementation (provides `meta()`/`name_any()`) so the build - /// functions can derive `ObjectMeta`, owner references and labels without the full - /// `HbaseCluster` object. Holds only name, namespace and uid. - metadata: ObjectMeta, - /// The logical (and Kubernetes object) name of the cluster. - pub name: ClusterName, - /// The namespace the cluster lives in. Part of the cluster identity; currently consumed via - /// the [`Resource`] metadata (`name_and_namespace`) rather than read directly. - #[allow(dead_code)] - pub namespace: NamespaceName, - /// The UID of the `HbaseCluster` object, used to build owner references. - pub uid: Uid, - pub image: ResolvedProductImage, - pub cluster_config: ValidatedClusterConfig, - pub role_group_configs: BTreeMap>, - pub role_configs: BTreeMap, -} - -impl ValidatedCluster { - #[allow(clippy::too_many_arguments)] - pub fn new( - name: ClusterName, - namespace: NamespaceName, - uid: Uid, - image: ResolvedProductImage, - cluster_config: ValidatedClusterConfig, - role_group_configs: BTreeMap>, - role_configs: BTreeMap, - ) -> Self { - Self { - metadata: ObjectMeta { - name: Some(name.to_string()), - namespace: Some(namespace.to_string()), - uid: Some(uid.to_string()), - ..ObjectMeta::default() - }, - name, - namespace, - uid, - image, - cluster_config, - role_group_configs, - role_configs, - } - } -} - -impl Resource for ValidatedCluster { - type DynamicType = ::DynamicType; - type Scope = ::Scope; - - fn group(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { - v1alpha1::HbaseCluster::group(dt) - } - - fn version(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { - v1alpha1::HbaseCluster::version(dt) - } - - fn kind(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { - v1alpha1::HbaseCluster::kind(dt) - } - - fn plural(dt: &Self::DynamicType) -> std::borrow::Cow<'_, str> { - v1alpha1::HbaseCluster::plural(dt) - } - - fn meta(&self) -> &ObjectMeta { - &self.metadata - } - - fn meta_mut(&mut self) -> &mut ObjectMeta { - &mut self.metadata - } -} - -impl HasName for ValidatedCluster { - fn to_name(&self) -> String { - self.name.to_string() - } -} - -impl HasUid for ValidatedCluster { - fn to_uid(&self) -> Uid { - self.uid.clone() - } -} - -impl NameIsValidLabelValue for ValidatedCluster { - fn to_label_value(&self) -> String { - self.name.to_label_value() - } -} - -/// Cluster-wide settings resolved once during validation. -#[derive(Clone, Debug)] -pub struct ValidatedClusterConfig { - // Pre-resolved OPA connection configuration. - pub hbase_opa_config: Option, - pub kerberos_enabled: bool, - // Pre-resolved kerberos properties for hbase-site.xml (empty when kerberos is disabled). - pub hbase_site_kerberos_config: BTreeMap, - // Pre-resolved kerberos properties for the discovery `hbase-site.xml` exposed to clients - // (empty when kerberos is disabled). - pub discovery_kerberos_config: BTreeMap, - // Pre-resolved ssl-server.xml settings (empty when HTTPS is disabled). - pub ssl_server_settings: BTreeMap, - // Pre-resolved ssl-client.xml settings (empty when HTTPS is disabled). - pub ssl_client_settings: BTreeMap, - // Pre-resolved zookeeper connection settings. - pub zookeeper_connection_information: ZookeeperConnectionInformation, -} - -/// Per-role configuration extracted during validation. -#[derive(Clone, Debug)] -pub struct ValidatedRoleConfig { - pub pdb: stackable_operator::commons::pdb::PdbConfig, -} - -/// Per-rolegroup configuration: the merged CRD config plus the merged -/// (role <- role group) `configOverrides`, `envOverrides` and `podOverrides`. -/// -/// This carries every override channel so that the build step is a pure function of -/// [`ValidatedCluster`] and never has to reach back into the raw `HbaseCluster`. -#[derive(Clone, Debug)] -pub struct ValidatedRoleGroupConfig { - /// The desired number of replicas (`None` lets Kubernetes default to 1). - pub replicas: Option, - pub merged_config: AnyServiceConfig, - pub config_overrides: v1alpha1::HbaseConfigOverrides, - pub env_overrides: EnvVarSet, - /// Merged (role <- role group) pod template overrides. - pub pod_overrides: PodTemplateSpec, - /// Pre-resolved role-specific non-heap JVM args (operator-generated + role/role-group overrides). - pub non_heap_jvm_args: String, -} - #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] pub enum Error { From 016c2c34377307552d1946863871742981e3a526 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 10 Jun 2026 20:29:45 +0200 Subject: [PATCH 28/56] refactor: add framework module --- .../src/controller/build/config_map.rs | 10 +- rust/operator-binary/src/controller/mod.rs | 14 +- .../src/controller/validate.rs | 339 +++++++---------- rust/operator-binary/src/crd/mod.rs | 352 ++++++------------ rust/operator-binary/src/framework.rs | 12 + .../src/framework/role_utils.rs | 152 ++++++++ rust/operator-binary/src/hbase_controller.rs | 4 +- rust/operator-binary/src/main.rs | 1 + 8 files changed, 435 insertions(+), 449 deletions(-) create mode 100644 rust/operator-binary/src/framework.rs create mode 100644 rust/operator-binary/src/framework/role_utils.rs diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index b011e10c..f30050ec 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -71,7 +71,7 @@ pub fn build_rolegroup_config_map( let hbase_site_xml = hbase_site::build( role, - &rg.merged_config, + &rg.config, cluster_config .zookeeper_connection_information .as_hbase_settings(), @@ -81,7 +81,7 @@ pub fn build_rolegroup_config_map( ); let hbase_env_sh = hbase_env::build( - &rg.merged_config, + &rg.config, role, cluster_config.kerberos_enabled, rg.non_heap_jvm_args.clone(), @@ -134,12 +134,10 @@ pub fn build_rolegroup_config_map( builder.add_data(ConfigFileName::SslClient.to_string(), ssl_client_xml); } - if let Some(log4j2_properties) = logging::build_log4j2(rg.merged_config.logging()) { + if let Some(log4j2_properties) = logging::build_log4j2(rg.config.logging()) { builder.add_data(ConfigFileName::Log4j2.to_string(), log4j2_properties); } - if let Some(vector_config) = - logging::build_vector_config(rolegroup_ref, rg.merged_config.logging()) - { + if let Some(vector_config) = logging::build_vector_config(rolegroup_ref, rg.config.logging()) { builder.add_data(VECTOR_CONFIG_FILE, vector_config); } diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index b91db5d2..4a0eb26e 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -150,13 +150,17 @@ pub struct ValidatedRoleConfig { /// Per-rolegroup configuration: the merged CRD config plus the merged /// (role <- role group) `configOverrides`, `envOverrides` and `podOverrides`. /// -/// This carries every override channel so that the build step is a pure function of -/// [`ValidatedCluster`] and never has to reach back into the raw `HbaseCluster`. +/// The merge and validation is performed by +/// [`with_validated_config`](crate::framework::role_utils::with_validated_config); the +/// result is flattened into this struct and augmented with the pre-resolved +/// `non_heap_jvm_args`. Carrying every override channel (and the JVM args) keeps the +/// build step a pure function of [`ValidatedCluster`] that never has to reach back into +/// the raw `HbaseCluster`. #[derive(Clone, Debug)] pub struct ValidatedRoleGroupConfig { - /// The desired number of replicas (`None` lets Kubernetes default to 1). - pub replicas: Option, - pub merged_config: AnyServiceConfig, + /// The desired number of replicas (defaulted to 1 during validation). + pub replicas: u16, + pub config: AnyServiceConfig, pub config_overrides: v1alpha1::HbaseConfigOverrides, pub env_overrides: EnvVarSet, /// Merged (role <- role group) pod template overrides. diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index b7787329..e9ed4e96 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -1,15 +1,13 @@ -use std::{collections::BTreeMap, str::FromStr}; +use std::collections::BTreeMap; use snafu::{ResultExt, Snafu}; use stackable_operator::{ commons::product_image_selection::{self}, - config::merge::Merge, - role_utils::GenericRoleConfig, + config::{fragment::FromFragment, merge::Merge}, + kube::ResourceExt, + role_utils::{GenericRoleConfig, JavaCommonConfig, Role}, utils::cluster_info::KubernetesClusterInfo, - v2::{ - builder::pod::container::{self, EnvVarName, EnvVarSet}, - controller_utils::{get_cluster_name, get_namespace, get_uid}, - }, + v2::controller_utils::{get_cluster_name, get_namespace, get_uid}, }; use strum::IntoEnumIterator; @@ -19,7 +17,8 @@ use crate::{ ValidatedCluster, ValidatedClusterConfig, ValidatedRoleConfig, ValidatedRoleGroupConfig, dereference::DereferencedObjects, }, - crd::{HbaseRole, v1alpha1}, + crd::{AnyServiceConfig, HbaseConfigFragment, HbaseRole, RegionServerConfigFragment, v1alpha1}, + framework::role_utils::with_validated_config, kerberos::{ self, kerberos_config_properties, kerberos_discovery_config_properties, kerberos_ssl_client_settings, kerberos_ssl_server_settings, @@ -43,20 +42,16 @@ pub enum Error { #[snafu(display("the HbaseCluster has no {role} role defined"))] MissingRequiredRole { role: String }, - #[snafu(display("failed to resolve and merge config for role and role group"))] - FailedToResolveConfig { source: crate::crd::Error }, + #[snafu(display("failed to merge and validate the role group config"))] + ValidateRoleGroupConfig { + source: crate::framework::role_utils::Error, + }, #[snafu(display("failed to resolve kerberos config"))] AddKerberosConfig { source: kerberos::Error }, #[snafu(display("failed to construct role-specific JVM arguments"))] ConstructJvmArgument { source: crate::config::jvm::Error }, - - #[snafu(display("the environment variable override name {name:?} is invalid"))] - InvalidEnvVarName { - source: container::Error, - name: String, - }, } pub fn validate_cluster( @@ -78,11 +73,48 @@ pub fn validate_cluster( let mut role_groups = BTreeMap::new(); let mut role_configs = BTreeMap::new(); + let hdfs_discovery_cm_name = &hbase.spec.cluster_config.hdfs_config_map_name; + let cluster_name = hbase.name_any(); + for hbase_role in HbaseRole::iter() { - let role_group_names = role_group_names(hbase, &hbase_role); + let group_configs = match hbase_role { + HbaseRole::Master => validate_role_group_configs( + hbase, + &hbase_role, + hbase.spec.masters.as_ref(), + HbaseConfigFragment::default_config( + &hbase_role, + &cluster_name, + hdfs_discovery_cm_name, + ), + AnyServiceConfig::Master, + )?, + HbaseRole::RegionServer => validate_role_group_configs( + hbase, + &hbase_role, + hbase.spec.region_servers.as_ref(), + RegionServerConfigFragment::default_config( + &hbase_role, + &cluster_name, + hdfs_discovery_cm_name, + ), + AnyServiceConfig::RegionServer, + )?, + HbaseRole::RestServer => validate_role_group_configs( + hbase, + &hbase_role, + hbase.spec.rest_servers.as_ref(), + HbaseConfigFragment::default_config( + &hbase_role, + &cluster_name, + hdfs_discovery_cm_name, + ), + AnyServiceConfig::RestServer, + )?, + }; // masters and region servers are required; rest servers are optional. - if role_group_names.is_empty() { + if group_configs.is_empty() { match hbase_role { HbaseRole::Master | HbaseRole::RegionServer => { return MissingRequiredRoleSnafu { @@ -101,41 +133,6 @@ pub fn validate_cluster( role_configs.insert(hbase_role.clone(), ValidatedRoleConfig { pdb: pdb.clone() }); } - let mut group_configs = BTreeMap::new(); - for rolegroup_name in role_group_names { - let merged_config = hbase - .merged_config( - &hbase_role, - &rolegroup_name, - &hbase.spec.cluster_config.hdfs_config_map_name, - ) - .context(FailedToResolveConfigSnafu)?; - - let rolegroup_ref = - hbase.server_rolegroup_ref(hbase_role.to_string(), rolegroup_name.clone()); - - group_configs.insert( - rolegroup_name.clone(), - ValidatedRoleGroupConfig { - replicas: hbase.replicas(&hbase_role, &rolegroup_ref), - merged_config, - config_overrides: merged_config_overrides(hbase, &hbase_role, &rolegroup_name), - env_overrides: env_var_set(merged_env_overrides( - hbase, - &hbase_role, - &rolegroup_name, - ))?, - pod_overrides: hbase.merged_pod_overrides(&hbase_role, &rolegroup_ref), - non_heap_jvm_args: construct_role_specific_non_heap_jvm_args( - hbase, - &hbase_role, - &rolegroup_name, - ) - .context(ConstructJvmArgumentSnafu)?, - }, - ); - } - role_groups.insert(hbase_role, group_configs); } @@ -169,151 +166,62 @@ pub fn validate_cluster( )) } -/// The names of the role groups defined for `role` in the spec. -fn role_group_names(hbase: &v1alpha1::HbaseCluster, role: &HbaseRole) -> Vec { - match role { - HbaseRole::Master => hbase - .spec - .masters - .as_ref() - .map(|r| r.role_groups.keys().cloned().collect()), - HbaseRole::RegionServer => hbase - .spec - .region_servers - .as_ref() - .map(|r| r.role_groups.keys().cloned().collect()), - HbaseRole::RestServer => hbase - .spec - .rest_servers - .as_ref() - .map(|r| r.role_groups.keys().cloned().collect()), - } - .unwrap_or_default() -} - -/// Merge role-level then role-group-level `configOverrides` (role group wins). -fn merged_config_overrides( - hbase: &v1alpha1::HbaseCluster, - role: &HbaseRole, - role_group: &str, -) -> v1alpha1::HbaseConfigOverrides { - let (role_overrides, role_group_overrides) = match role { - HbaseRole::Master => ( - hbase - .spec - .masters - .as_ref() - .map(|r| r.config.config_overrides.clone()), - hbase - .spec - .masters - .as_ref() - .and_then(|r| r.role_groups.get(role_group)) - .map(|rg| rg.config.config_overrides.clone()), - ), - HbaseRole::RegionServer => ( - hbase - .spec - .region_servers - .as_ref() - .map(|r| r.config.config_overrides.clone()), - hbase - .spec - .region_servers - .as_ref() - .and_then(|r| r.role_groups.get(role_group)) - .map(|rg| rg.config.config_overrides.clone()), - ), - HbaseRole::RestServer => ( - hbase - .spec - .rest_servers - .as_ref() - .map(|r| r.config.config_overrides.clone()), - hbase - .spec - .rest_servers - .as_ref() - .and_then(|r| r.role_groups.get(role_group)) - .map(|rg| rg.config.config_overrides.clone()), - ), - }; - - let role_overrides = role_overrides.unwrap_or_default(); - let mut merged = role_group_overrides.unwrap_or_default(); - merged.merge(&role_overrides); - merged -} - -/// Merge role-level then role-group-level `envOverrides` (role group wins). -fn merged_env_overrides( +/// Validates every role group of a role into a map keyed by role group name. +/// +/// Each role group is merged and validated via the local-`framework` +/// [`with_validated_config`], which folds the CRD config fragment (default <- role <- +/// role group) plus the `configOverrides`, `envOverrides`, `cliOverrides` and +/// `podOverrides` (role group wins) into a single +/// [`RoleGroupConfig`](crate::framework::role_utils::RoleGroupConfig). The concrete +/// per-role validated config is wrapped into [`AnyServiceConfig`] via `wrap`, and the +/// role-specific non-heap JVM args are pre-resolved so the build step stays a pure +/// function of [`ValidatedCluster`]. +/// +/// Returns an empty map if the role is not configured. +fn validate_role_group_configs( hbase: &v1alpha1::HbaseCluster, - role: &HbaseRole, - role_group: &str, -) -> BTreeMap { - let (role_overrides, role_group_overrides) = match role { - HbaseRole::Master => ( - hbase - .spec - .masters - .as_ref() - .map(|r| r.config.env_overrides.clone()), - hbase - .spec - .masters - .as_ref() - .and_then(|r| r.role_groups.get(role_group)) - .map(|rg| rg.config.env_overrides.clone()), - ), - HbaseRole::RegionServer => ( - hbase - .spec - .region_servers - .as_ref() - .map(|r| r.config.env_overrides.clone()), - hbase - .spec - .region_servers - .as_ref() - .and_then(|r| r.role_groups.get(role_group)) - .map(|rg| rg.config.env_overrides.clone()), - ), - HbaseRole::RestServer => ( - hbase - .spec - .rest_servers - .as_ref() - .map(|r| r.config.env_overrides.clone()), - hbase - .spec - .rest_servers - .as_ref() - .and_then(|r| r.role_groups.get(role_group)) - .map(|rg| rg.config.env_overrides.clone()), - ), + hbase_role: &HbaseRole, + role: Option< + &Role, + >, + default_config: Config, + wrap: fn(ValidatedConfig) -> AnyServiceConfig, +) -> Result, Error> +where + Config: Clone + Merge, + ValidatedConfig: FromFragment, +{ + let Some(role) = role else { + return Ok(BTreeMap::new()); }; - let mut env_overrides = BTreeMap::new(); - if let Some(role_overrides) = role_overrides { - env_overrides.extend(role_overrides); - } - if let Some(role_group_overrides) = role_group_overrides { - env_overrides.extend(role_group_overrides); - } - env_overrides -} - -/// Converts merged env override pairs into a type-safe [`EnvVarSet`], validating each name so that -/// invalid environment variable names are rejected during validation instead of producing a broken -/// Pod. -fn env_var_set(env_overrides: BTreeMap) -> Result { - let mut set = EnvVarSet::new(); - for (name, value) in env_overrides { - let env_var_name = - EnvVarName::from_str(&name).context(InvalidEnvVarNameSnafu { name: name.clone() })?; - set = set.with_value(&env_var_name, value); - } - Ok(set) + role.role_groups + .iter() + .map(|(role_group_name, role_group)| { + let validated = with_validated_config::< + ValidatedConfig, + JavaCommonConfig, + Config, + GenericRoleConfig, + v1alpha1::HbaseConfigOverrides, + >(role_group, role, &default_config) + .context(ValidateRoleGroupConfigSnafu)?; + + let non_heap_jvm_args = + construct_role_specific_non_heap_jvm_args(hbase, hbase_role, role_group_name) + .context(ConstructJvmArgumentSnafu)?; + + let validated = ValidatedRoleGroupConfig { + replicas: validated.replicas, + config: wrap(validated.config), + config_overrides: validated.config_overrides, + env_overrides: validated.env_overrides, + pod_overrides: validated.pod_overrides, + non_heap_jvm_args, + }; + Ok((role_group_name.clone(), validated)) + }) + .collect() } #[cfg(test)] @@ -321,7 +229,10 @@ mod tests { use indoc::indoc; use super::*; + use crate::crd::HbaseConfig; + /// Role-level `envOverrides` are merged with role-group-level ones, with the role + /// group winning on key collisions. #[test] fn test_env_overrides() { let input = indoc! {r#" @@ -371,19 +282,31 @@ spec: let hbase: v1alpha1::HbaseCluster = serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); - let env_overrides = merged_env_overrides(&hbase, &HbaseRole::Master, "default"); - - assert_eq!( - env_overrides.get("TEST_VAR"), - Some(&"MASTER_RG".to_string()) - ); - assert_eq!( - env_overrides.get("TEST_VAR_FROM_MASTER"), - Some(&"MASTER".to_string()) - ); - assert_eq!( - env_overrides.get("TEST_VAR_FROM_MRG"), - Some(&"MASTER".to_string()) + let role = hbase.spec.masters.as_ref().unwrap(); + let role_group = role.role_groups.get("default").unwrap(); + let default_config = HbaseConfigFragment::default_config( + &HbaseRole::Master, + &hbase.name_any(), + &hbase.spec.cluster_config.hdfs_config_map_name, ); + + let validated = with_validated_config::< + HbaseConfig, + JavaCommonConfig, + HbaseConfigFragment, + GenericRoleConfig, + v1alpha1::HbaseConfigOverrides, + >(role_group, role, &default_config) + .unwrap(); + + let env: BTreeMap = validated + .env_overrides + .into_iter() + .map(|env_var| (env_var.name, env_var.value.unwrap_or_default())) + .collect(); + + assert_eq!(env.get("TEST_VAR"), Some(&"MASTER_RG".to_string())); + assert_eq!(env.get("TEST_VAR_FROM_MASTER"), Some(&"MASTER".to_string())); + assert_eq!(env.get("TEST_VAR_FROM_MRG"), Some(&"MASTER".to_string())); } } diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index cb80e3fd..81f65a40 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -24,8 +24,7 @@ use stackable_operator::{ }, deep_merger::ObjectOverrides, k8s_openapi::{ - DeepMerge, - api::core::v1::{EnvVar, PersistentVolumeClaim, PodTemplateSpec, Volume}, + api::core::v1::{EnvVar, PersistentVolumeClaim, Volume}, apimachinery::pkg::api::resource::Quantity, }, kube::{CustomResource, ResourceExt, runtime::reflector::ObjectRef}, @@ -114,9 +113,6 @@ pub enum Error { #[snafu(display("fragment validation failure"))] FragmentValidationFailure { source: ValidationError }, - #[snafu(display("incompatible merge types"))] - IncompatibleMergeTypes, - #[snafu(display("empty values for role-group are not permitted"))] EmptyRoleGroup, @@ -246,187 +242,108 @@ impl HasStatusCondition for v1alpha1::HbaseCluster { impl v1alpha1::HbaseCluster { /// Retrieve and merge resource configs for role and role groups + /// Merges and validates the config for `role`/`role_group`. + /// + /// The role-group config is merged on top of the role config on top of the + /// operator defaults (most specific wins) and the result is validated. pub fn merged_config( &self, role: &HbaseRole, role_group: &str, hdfs_discovery_cm_name: &str, ) -> Result { - // Initialize the result with all default values as baseline - let defaults = - AnyConfigFragment::default_for(role, &self.name_any(), hdfs_discovery_cm_name); - // Trivial values for role-groups are not allowed if role_group.is_empty() { return Err(Error::EmptyRoleGroup); } - let (mut role_config, mut role_group_config) = match role { - HbaseRole::RegionServer => { - let role = self - .spec - .region_servers - .clone() - .context(MissingHbaseRoleSnafu { - role: role.to_string(), - })?; + match role { + HbaseRole::Master => { + let default_config = HbaseConfigFragment::default_config( + role, + &self.name_any(), + hdfs_discovery_cm_name, + ); + let role_obj = self.spec.masters.as_ref().context(MissingHbaseRoleSnafu { + role: role.to_string(), + })?; - let role_config = role.config.config.to_owned(); - let role_group_config = role + let mut role_config = role_obj.config.config.clone(); + let mut role_group_config = role_obj .role_groups .get(role_group) - .map(|rg| rg.config.config.clone()) - .context(RoleGroupNotFoundSnafu)?; - - ( - AnyConfigFragment::RegionServer(role_config), - AnyConfigFragment::RegionServer(role_group_config), - ) + .context(RoleGroupNotFoundSnafu)? + .config + .config + .clone(); + + role_config.merge(&default_config); + role_group_config.merge(&role_config); + Ok(AnyServiceConfig::Master( + fragment::validate(role_group_config) + .context(FragmentValidationFailureSnafu)?, + )) + } + HbaseRole::RegionServer => { + let default_config = RegionServerConfigFragment::default_config( + role, + &self.name_any(), + hdfs_discovery_cm_name, + ); + let role_obj = + self.spec + .region_servers + .as_ref() + .context(MissingHbaseRoleSnafu { + role: role.to_string(), + })?; + + let mut role_config = role_obj.config.config.clone(); + let mut role_group_config = role_obj + .role_groups + .get(role_group) + .context(RoleGroupNotFoundSnafu)? + .config + .config + .clone(); + + role_config.merge(&default_config); + role_group_config.merge(&role_config); + Ok(AnyServiceConfig::RegionServer( + fragment::validate(role_group_config) + .context(FragmentValidationFailureSnafu)?, + )) } HbaseRole::RestServer => { - let role = self + let default_config = HbaseConfigFragment::default_config( + role, + &self.name_any(), + hdfs_discovery_cm_name, + ); + let role_obj = self .spec .rest_servers - .clone() + .as_ref() .context(MissingHbaseRoleSnafu { role: role.to_string(), })?; - let role_config = role.config.config.to_owned(); - - let role_group_config = role + let mut role_config = role_obj.config.config.clone(); + let mut role_group_config = role_obj .role_groups .get(role_group) - .map(|rg| rg.config.config.clone()) - .context(RoleGroupNotFoundSnafu)?; - - // Retrieve role resource config - ( - AnyConfigFragment::RestServer(role_config), - AnyConfigFragment::RestServer(role_group_config), - ) + .context(RoleGroupNotFoundSnafu)? + .config + .config + .clone(); + + role_config.merge(&default_config); + role_group_config.merge(&role_config); + Ok(AnyServiceConfig::RestServer( + fragment::validate(role_group_config) + .context(FragmentValidationFailureSnafu)?, + )) } - HbaseRole::Master => { - let role = self.spec.masters.clone().context(MissingHbaseRoleSnafu { - role: role.to_string(), - })?; - - let role_config = role.config.config.to_owned(); - - // Retrieve rolegroup specific resource config - let role_group_config = role - .role_groups - .get(role_group) - .map(|rg| rg.config.config.clone()) - .context(RoleGroupNotFoundSnafu)?; - - // Retrieve role resource config - ( - AnyConfigFragment::Master(role_config), - AnyConfigFragment::Master(role_group_config), - ) - } - }; - - // Merge more specific configs into default config - // Hierarchy is: - // 1. RoleGroup - // 2. Role - // 3. Default - role_config = role_config.merge(&defaults)?; - role_group_config = role_group_config.merge(&role_config)?; - - tracing::debug!("Merged config: {:?}", role_group_config); - - Ok(match role_group_config { - AnyConfigFragment::RegionServer(conf) => AnyServiceConfig::RegionServer( - fragment::validate(conf).context(FragmentValidationFailureSnafu)?, - ), - AnyConfigFragment::RestServer(conf) => AnyServiceConfig::RestServer( - fragment::validate(conf).context(FragmentValidationFailureSnafu)?, - ), - AnyConfigFragment::Master(conf) => AnyServiceConfig::Master( - fragment::validate(conf).context(FragmentValidationFailureSnafu)?, - ), - }) - } - - /// Returns the merged (role <- role group) pod template overrides for the given role group. - pub fn merged_pod_overrides( - &self, - role: &HbaseRole, - role_group_ref: &RoleGroupRef, - ) -> PodTemplateSpec { - let (role_pod_overrides, role_group_pod_overrides) = match role { - HbaseRole::Master => ( - self.spec - .masters - .as_ref() - .map(|r| r.config.pod_overrides.clone()), - self.spec - .masters - .as_ref() - .and_then(|r| r.role_groups.get(&role_group_ref.role_group)) - .map(|r| r.config.pod_overrides.clone()), - ), - HbaseRole::RegionServer => ( - self.spec - .region_servers - .as_ref() - .map(|r| r.config.pod_overrides.clone()), - self.spec - .region_servers - .as_ref() - .and_then(|r| r.role_groups.get(&role_group_ref.role_group)) - .map(|r| r.config.pod_overrides.clone()), - ), - HbaseRole::RestServer => ( - self.spec - .rest_servers - .as_ref() - .map(|r| r.config.pod_overrides.clone()), - self.spec - .rest_servers - .as_ref() - .and_then(|r| r.role_groups.get(&role_group_ref.role_group)) - .map(|r| r.config.pod_overrides.clone()), - ), - }; - - let mut merged = PodTemplateSpec::default(); - if let Some(rpo) = role_pod_overrides { - merged.merge_from(rpo); - } - if let Some(rgpo) = role_group_pod_overrides { - merged.merge_from(rgpo); - } - merged - } - - pub fn replicas( - &self, - hbase_role: &HbaseRole, - role_group_ref: &RoleGroupRef, - ) -> Option { - match hbase_role { - HbaseRole::Master => self - .spec - .masters - .as_ref() - .and_then(|r| r.role_groups.get(&role_group_ref.role_group)) - .and_then(|rg| rg.replicas), - HbaseRole::RegionServer => self - .spec - .region_servers - .as_ref() - .and_then(|r| r.role_groups.get(&role_group_ref.role_group)) - .and_then(|rg| rg.replicas), - HbaseRole::RestServer => self - .spec - .rest_servers - .as_ref() - .and_then(|r| r.role_groups.get(&role_group_ref.role_group)) - .and_then(|rg| rg.replicas), } } @@ -687,79 +604,58 @@ fn default_resources(role: &HbaseRole) -> ResourcesFragment Result { - match (self, other) { - (AnyConfigFragment::RegionServer(mut me), AnyConfigFragment::RegionServer(you)) => { - me.merge(you); - Ok(AnyConfigFragment::RegionServer(me.clone())) - } - (AnyConfigFragment::RestServer(mut me), AnyConfigFragment::RestServer(you)) => { - me.merge(you); - Ok(AnyConfigFragment::RestServer(me.clone())) - } - (AnyConfigFragment::Master(mut me), AnyConfigFragment::Master(you)) => { - me.merge(you); - Ok(AnyConfigFragment::Master(me.clone())) - } - (_, _) => Err(Error::IncompatibleMergeTypes), +impl HbaseConfigFragment { + /// The operator defaults for a `masters` or `restServers` role group. + pub fn default_config( + role: &HbaseRole, + cluster_name: &str, + hdfs_discovery_cm_name: &str, + ) -> Self { + let graceful_shutdown_timeout = match role { + HbaseRole::Master => HbaseRole::DEFAULT_MASTER_GRACEFUL_SHUTDOWN_TIMEOUT, + HbaseRole::RegionServer => HbaseRole::DEFAULT_REGION_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT, + HbaseRole::RestServer => HbaseRole::DEFAULT_REST_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT, + }; + let requested_secret_lifetime = match role { + HbaseRole::Master => HbaseRole::DEFAULT_MASTER_SECRET_LIFETIME, + HbaseRole::RegionServer => HbaseRole::DEFAULT_REGION_SECRET_LIFETIME, + HbaseRole::RestServer => HbaseRole::DEFAULT_REST_SECRET_LIFETIME, + }; + HbaseConfigFragment { + hbase_rootdir: Some(default_hbase_rootdir()), + resources: default_resources(role), + logging: product_logging::spec::default_logging(), + affinity: get_affinity(cluster_name, role, hdfs_discovery_cm_name), + graceful_shutdown_timeout: Some(graceful_shutdown_timeout), + requested_secret_lifetime: Some(requested_secret_lifetime), + listener_class: Some(DEFAULT_LISTENER_CLASS.to_string()), } } +} - fn default_for( +impl RegionServerConfigFragment { + /// The operator defaults for a `regionServers` role group. + pub fn default_config( role: &HbaseRole, cluster_name: &str, hdfs_discovery_cm_name: &str, - ) -> AnyConfigFragment { - match role { - HbaseRole::RegionServer => { - AnyConfigFragment::RegionServer(RegionServerConfigFragment { - hbase_rootdir: Some(default_hbase_rootdir()), - resources: default_resources(role), - logging: product_logging::spec::default_logging(), - affinity: get_affinity(cluster_name, role, hdfs_discovery_cm_name), - graceful_shutdown_timeout: Some( - HbaseRole::DEFAULT_REGION_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT, - ), - region_mover: RegionMoverFragment { - run_before_shutdown: Some(false), - max_threads: Some(1), - ack: Some(true), - cli_opts: None, - }, - requested_secret_lifetime: Some(HbaseRole::DEFAULT_REGION_SECRET_LIFETIME), - listener_class: Some(DEFAULT_LISTENER_CLASS.to_string()), - }) - } - HbaseRole::RestServer => AnyConfigFragment::RestServer(HbaseConfigFragment { - hbase_rootdir: Some(default_hbase_rootdir()), - resources: default_resources(role), - logging: product_logging::spec::default_logging(), - affinity: get_affinity(cluster_name, role, hdfs_discovery_cm_name), - graceful_shutdown_timeout: Some( - HbaseRole::DEFAULT_REST_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT, - ), - requested_secret_lifetime: Some(HbaseRole::DEFAULT_REST_SECRET_LIFETIME), - listener_class: Some(DEFAULT_LISTENER_CLASS.to_string()), - }), - HbaseRole::Master => AnyConfigFragment::Master(HbaseConfigFragment { - hbase_rootdir: Some(default_hbase_rootdir()), - resources: default_resources(role), - logging: product_logging::spec::default_logging(), - affinity: get_affinity(cluster_name, role, hdfs_discovery_cm_name), - graceful_shutdown_timeout: Some( - HbaseRole::DEFAULT_MASTER_GRACEFUL_SHUTDOWN_TIMEOUT, - ), - requested_secret_lifetime: Some(HbaseRole::DEFAULT_MASTER_SECRET_LIFETIME), - listener_class: Some(DEFAULT_LISTENER_CLASS.to_string()), - }), + ) -> Self { + RegionServerConfigFragment { + hbase_rootdir: Some(default_hbase_rootdir()), + resources: default_resources(role), + logging: product_logging::spec::default_logging(), + affinity: get_affinity(cluster_name, role, hdfs_discovery_cm_name), + graceful_shutdown_timeout: Some( + HbaseRole::DEFAULT_REGION_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT, + ), + region_mover: RegionMoverFragment { + run_before_shutdown: Some(false), + max_threads: Some(1), + ack: Some(true), + cli_opts: None, + }, + requested_secret_lifetime: Some(HbaseRole::DEFAULT_REGION_SECRET_LIFETIME), + listener_class: Some(DEFAULT_LISTENER_CLASS.to_string()), } } } diff --git a/rust/operator-binary/src/framework.rs b/rust/operator-binary/src/framework.rs new file mode 100644 index 00000000..568fc008 --- /dev/null +++ b/rust/operator-binary/src/framework.rs @@ -0,0 +1,12 @@ +//! Local framework helpers that mirror the work-in-progress upstream +//! `stackable_operator::v2::*` modules. +//! +//! We vendor `role_utils` because the upstream `v2::role_utils` requires +//! `CommonConfig: Merge`. HBase (like hdfs, hive and trino) uses +//! `JavaCommonConfig`, whose JVM-argument merge is fallible and so does not +//! implement `Merge`. +//! +//! Follow-up: replace with `stackable_operator::v2::role_utils::*` once upstream +//! relaxes the `Merge` bound. + +pub mod role_utils; diff --git a/rust/operator-binary/src/framework/role_utils.rs b/rust/operator-binary/src/framework/role_utils.rs new file mode 100644 index 00000000..65980875 --- /dev/null +++ b/rust/operator-binary/src/framework/role_utils.rs @@ -0,0 +1,152 @@ +//! Vendored variant of `stackable_operator::v2::role_utils` from the +//! `smooth-operator` branch, with simplifications appropriate for hbase-operator. +//! +//! Differences from upstream: +//! - No `cli_overrides_to_vec` helper, `ResourceNames`, or service-account helpers. +//! - The `CommonConfig` (a.k.a. `product_specific_common_config`) does NOT need to +//! implement `Merge`. HBase uses `JavaCommonConfig`, which intentionally does not +//! implement `Merge` because its inner `JvmArgumentOverrides::try_merge` is +//! fallible (regex validation). The `RoleGroupConfig::product_specific_common_config` +//! field here simply carries the role-group level value through. +//! +//! Replace with `stackable_operator::v2::role_utils::*` once upstream relaxes the +//! `Merge` bound. + +use std::{ + collections::{BTreeMap, HashMap}, + str::FromStr, +}; + +use serde::Serialize; +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + config::{ + fragment::{self, FromFragment}, + merge::{Merge, merge}, + }, + k8s_openapi::{DeepMerge, api::core::v1::PodTemplateSpec}, + role_utils::{Role, RoleGroup}, + schemars::JsonSchema, + v2::builder::pod::container::{self, EnvVarName, EnvVarSet}, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to validate the role group config"))] + ValidateConfig { source: fragment::ValidationError }, + + #[snafu(display("invalid environment variable override name"))] + ParseEnvVarName { source: container::Error }, +} + +/// HBase-friendly view of a validated, merged `RoleGroup`. +#[derive(Clone, Debug, PartialEq)] +pub struct RoleGroupConfig { + pub replicas: u16, + pub config: Config, + pub config_overrides: ConfigOverrides, + pub env_overrides: EnvVarSet, + pub cli_overrides: BTreeMap, + pub pod_overrides: PodTemplateSpec, + pub product_specific_common_config: CommonConfig, +} + +/// Merges and validates the `RoleGroup` with the given `role` and `default_config`. +pub fn with_validated_config( + role_group: &RoleGroup, + role: &Role, + default_config: &Config, +) -> Result, Error> +where + ValidatedConfig: FromFragment, + CommonConfig: Clone + Default + JsonSchema + Serialize, + Config: Clone + Merge, + RoleConfig: Default + JsonSchema + Serialize, + ConfigOverrides: Clone + Default + JsonSchema + Merge + Serialize, +{ + let validated_config = + validate_config(role_group, role, default_config).context(ValidateConfigSnafu)?; + Ok(RoleGroupConfig { + replicas: role_group.replicas.unwrap_or(1), + config: validated_config, + config_overrides: merged_config_overrides( + &role.config.config_overrides, + role_group.config.config_overrides.clone(), + ), + env_overrides: merged_env_overrides( + &role.config.env_overrides, + &role_group.config.env_overrides, + )?, + cli_overrides: merged_cli_overrides( + role.config.cli_overrides.clone(), + role_group.config.cli_overrides.clone(), + ), + pod_overrides: merged_pod_overrides( + role.config.pod_overrides.clone(), + role_group.config.pod_overrides.clone(), + ), + product_specific_common_config: role_group.config.product_specific_common_config.clone(), + }) +} + +fn validate_config( + role_group: &RoleGroup, + role: &Role, + default_config: &Config, +) -> Result +where + ValidatedConfig: FromFragment, + CommonConfig: Default + JsonSchema + Serialize, + Config: Clone + Merge, + RoleConfig: Default + JsonSchema + Serialize, + ConfigOverrides: Default + JsonSchema + Serialize, +{ + role_group.validate_config(role, default_config) +} + +fn merged_config_overrides( + role_config_overrides: &ConfigOverrides, + role_group_config_overrides: ConfigOverrides, +) -> ConfigOverrides +where + ConfigOverrides: Merge, +{ + merge(role_group_config_overrides, role_config_overrides) +} + +fn merged_env_overrides( + role_env_overrides: &HashMap, + role_group_env_overrides: &HashMap, +) -> Result { + // Process the role first, then the role group, so that role-group overrides win on key + // collisions (`EnvVarSet::with_value` overrides earlier entries with the same name). + let mut env_overrides = EnvVarSet::new(); + for (name, value) in role_env_overrides + .iter() + .chain(role_group_env_overrides.iter()) + { + env_overrides = env_overrides.with_value( + &EnvVarName::from_str(name).context(ParseEnvVarNameSnafu)?, + value.clone(), + ); + } + Ok(env_overrides) +} + +fn merged_cli_overrides( + role_cli_overrides: BTreeMap, + role_group_cli_overrides: BTreeMap, +) -> BTreeMap { + let mut merged = role_cli_overrides; + merged.extend(role_group_cli_overrides); + merged +} + +fn merged_pod_overrides( + role_pod_overrides: PodTemplateSpec, + role_group_pod_overrides: PodTemplateSpec, +) -> PodTemplateSpec { + let mut merged = role_pod_overrides; + merged.merge_from(role_group_pod_overrides); + merged +} diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 9a4dad6c..d39a95cb 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -525,7 +525,7 @@ fn build_rolegroup_statefulset( resolved_product_image: &ResolvedProductImage, service_account: &ServiceAccount, ) -> Result { - let merged_config = &validated_rg_config.merged_config; + let merged_config = &validated_rg_config.config; let hbase_version = &resolved_product_image.app_version_label_value; let ports = hbase_role @@ -792,7 +792,7 @@ fn build_rolegroup_statefulset( let statefulset_spec = StatefulSetSpec { pod_management_policy: Some("Parallel".to_string()), - replicas: validated_rg_config.replicas.map(i32::from), + replicas: Some(i32::from(validated_rg_config.replicas)), selector: LabelSelector { match_labels: Some(statefulset_match_labels.into()), ..LabelSelector::default() diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index aa923145..3b0a7858 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -40,6 +40,7 @@ use crate::{ mod config; mod controller; mod crd; +mod framework; mod hbase_controller; mod kerberos; mod operations; From 09377a5483d31b48c1334e9d7ac2dd78646c1cca Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 11 Jun 2026 09:37:17 +0200 Subject: [PATCH 29/56] drop dead code: namespace --- rust/operator-binary/src/controller/mod.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index 4a0eb26e..730fa2ba 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -34,10 +34,6 @@ pub struct ValidatedCluster { metadata: ObjectMeta, /// The logical (and Kubernetes object) name of the cluster. pub name: ClusterName, - /// The namespace the cluster lives in. Part of the cluster identity; currently consumed via - /// the [`Resource`] metadata (`name_and_namespace`) rather than read directly. - #[allow(dead_code)] - pub namespace: NamespaceName, /// The UID of the `HbaseCluster` object, used to build owner references. pub uid: Uid, pub image: ResolvedProductImage, @@ -65,7 +61,6 @@ impl ValidatedCluster { ..ObjectMeta::default() }, name, - namespace, uid, image, cluster_config, From 9e903b1e3571d3e153d8510f92098c23adf86dc5 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 11 Jun 2026 13:02:59 +0200 Subject: [PATCH 30/56] fix: remove product-config references in comments --- .../src/controller/build/properties/hbase_site.rs | 3 +-- .../src/controller/build/properties/security_properties.rs | 7 +++---- rust/operator-binary/src/crd/mod.rs | 3 +-- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/rust/operator-binary/src/controller/build/properties/hbase_site.rs b/rust/operator-binary/src/controller/build/properties/hbase_site.rs index 2d28fad4..8d37330c 100644 --- a/rust/operator-binary/src/controller/build/properties/hbase_site.rs +++ b/rust/operator-binary/src/controller/build/properties/hbase_site.rs @@ -16,7 +16,6 @@ use crate::{ }; /// Renders `hbase-site.xml`. -#[allow(clippy::too_many_arguments)] pub fn build( role: &HbaseRole, merged_config: &AnyServiceConfig, @@ -27,7 +26,7 @@ pub fn build( ) -> String { let mut config: BTreeMap = BTreeMap::new(); - // Defaults previously injected by product-config's `compute_files`. + // Defaults config.insert(HBASE_CLUSTER_DISTRIBUTED.to_string(), "true".to_string()); config.insert(HBASE_ROOTDIR.to_string(), merged_config.hbase_rootdir()); diff --git a/rust/operator-binary/src/controller/build/properties/security_properties.rs b/rust/operator-binary/src/controller/build/properties/security_properties.rs index 82e18868..85421c8f 100644 --- a/rust/operator-binary/src/controller/build/properties/security_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/security_properties.rs @@ -1,8 +1,7 @@ //! Builds the `security.properties` (JVM security) config file. //! -//! The operator injects role-specific JVM DNS cache TTLs (preserving the -//! behaviour previously defined in `properties.yaml`). User `configOverrides` -//! are applied on top. +//! The operator injects role-specific JVM DNS cache TTLs. +//! User `configOverrides` are applied on top. use std::collections::BTreeMap; @@ -37,7 +36,7 @@ pub fn build( "0".to_string(), ), ]); - // Overrides applied last so users win. + // Overrides applied last. config.extend(overrides); to_java_properties_string(config.iter()) } diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 81f65a40..969329e1 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -886,8 +886,7 @@ impl AnyServiceConfig { } } - /// The configured `hbase.rootdir`. Previously injected into - /// `hbase-site.xml` via product-config's `compute_files`. + /// The configured `hbase.rootdir`. pub fn hbase_rootdir(&self) -> String { match self { AnyServiceConfig::Master(config) => config.hbase_rootdir.clone(), From aaa318169a016d2dcda9da8fe0771012bd72e3a4 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 11 Jun 2026 18:43:58 +0200 Subject: [PATCH 31/56] refactor: drop vendored framework, use mergeable v2 JavaCommonConfig --- Cargo.lock | 18 +-- rust/operator-binary/src/config/jvm.rs | 97 +++++------ rust/operator-binary/src/controller/mod.rs | 2 +- .../src/controller/validate.rs | 90 ++++++----- rust/operator-binary/src/crd/mod.rs | 4 +- rust/operator-binary/src/framework.rs | 12 -- .../src/framework/role_utils.rs | 152 ------------------ rust/operator-binary/src/main.rs | 1 - 8 files changed, 105 insertions(+), 271 deletions(-) delete mode 100644 rust/operator-binary/src/framework.rs delete mode 100644 rust/operator-binary/src/framework/role_utils.rs diff --git a/Cargo.lock b/Cargo.lock index f71f836e..569b529f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1514,7 +1514,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "darling", "regex", @@ -2899,7 +2899,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "const-oid", "ecdsa", @@ -2946,7 +2946,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.111.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "base64", "clap", @@ -2990,7 +2990,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "darling", "proc-macro2", @@ -3001,7 +3001,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "jiff", "k8s-openapi", @@ -3018,7 +3018,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.4" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "axum", "clap", @@ -3042,7 +3042,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "kube", "schemars", @@ -3056,7 +3056,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "convert_case", "convert_case_extras", @@ -3074,7 +3074,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "arc-swap", "async-trait", diff --git a/rust/operator-binary/src/config/jvm.rs b/rust/operator-binary/src/config/jvm.rs index b1c4d56a..11c06579 100644 --- a/rust/operator-binary/src/config/jvm.rs +++ b/rust/operator-binary/src/config/jvm.rs @@ -1,12 +1,10 @@ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ memory::{BinaryMultiple, MemoryQuantity}, - role_utils::{self, JvmArgumentOverrides}, + v2::jvm_argument_overrides::JvmArgumentOverrides, }; -use crate::crd::{ - AnyServiceConfig, CONFIG_DIR_NAME, HbaseRole, JVM_SECURITY_PROPERTIES_FILE, v1alpha1, -}; +use crate::crd::{AnyServiceConfig, CONFIG_DIR_NAME, JVM_SECURITY_PROPERTIES_FILE, v1alpha1}; const JAVA_HEAP_FACTOR: f32 = 0.8; @@ -19,12 +17,6 @@ pub enum Error { InvalidMemoryConfig { source: stackable_operator::memory::Error, }, - - #[snafu(display("failed to merge jvm argument overrides"))] - MergeJvmArgumentOverrides { source: role_utils::Error }, - - #[snafu(display("the HBase role [{role}] is missing from spec"))] - MissingHbaseRole { role: String }, } // Applies to both the servers and the CLI @@ -48,58 +40,28 @@ pub fn construct_global_jvm_args(kerberos_enabled: bool) -> String { /// JVM arguments that are specifically for the role (server), so will *not* be used e.g. by CLI tools. /// Heap settings are excluded, as they go into `HBASE_HEAPSIZE`. +/// +/// `merged_jvm_argument_overrides` is the role <- role-group merged [`JvmArgumentOverrides`] +/// produced by +/// [`with_validated_config`](stackable_operator::v2::role_utils::with_validated_config). The +/// operator-generated arguments below form the base that the user overrides are applied on top of. pub fn construct_role_specific_non_heap_jvm_args( hbase: &v1alpha1::HbaseCluster, - hbase_role: &HbaseRole, - role_group: &str, -) -> Result { - let mut jvm_args = vec![format!( + merged_jvm_argument_overrides: &JvmArgumentOverrides, +) -> String { + let mut operator_generated = vec![format!( "-Djava.security.properties={CONFIG_DIR_NAME}/{JVM_SECURITY_PROPERTIES_FILE}" )]; if hbase.has_kerberos_enabled() { - jvm_args.push("-Djava.security.krb5.conf=/stackable/kerberos/krb5.conf".to_owned()); + operator_generated + .push("-Djava.security.krb5.conf=/stackable/kerberos/krb5.conf".to_owned()); } - let operator_generated = JvmArgumentOverrides::new_with_only_additions(jvm_args); - - let merged = match hbase_role { - HbaseRole::Master => hbase - .spec - .masters - .as_ref() - .context(MissingHbaseRoleSnafu { - role: hbase_role.to_string(), - })? - .get_merged_jvm_argument_overrides(role_group, &operator_generated) - .context(MergeJvmArgumentOverridesSnafu)?, - HbaseRole::RegionServer => hbase - .spec - .region_servers - .as_ref() - .context(MissingHbaseRoleSnafu { - role: hbase_role.to_string(), - })? - .get_merged_jvm_argument_overrides(role_group, &operator_generated) - .context(MergeJvmArgumentOverridesSnafu)?, - HbaseRole::RestServer => hbase - .spec - .rest_servers - .as_ref() - .context(MissingHbaseRoleSnafu { - role: hbase_role.to_string(), - })? - .get_merged_jvm_argument_overrides(role_group, &operator_generated) - .context(MergeJvmArgumentOverridesSnafu)?, - }; - jvm_args = merged - .effective_jvm_config_after_merging() - // Sorry for the clone, that's how operator-rs is currently modelled :P - .clone(); - + let mut jvm_args = merged_jvm_argument_overrides.apply_to(operator_generated); jvm_args.retain(|arg| !is_heap_jvm_argument(arg)); - Ok(jvm_args.join(" ")) + jvm_args.join(" ") } /// This will be put into `HBASE_HEAPSIZE`, which is just the heap size in megabytes (with the `m` @@ -135,6 +97,8 @@ fn is_heap_jvm_argument(jvm_argument: &str) -> bool { #[cfg(test)] mod tests { + use stackable_operator::config::merge::Merge; + use super::*; use crate::crd::{HbaseRole, v1alpha1}; @@ -160,11 +124,11 @@ mod tests { default: replicas: 1 "#; - let (hbase, hbase_role, merged_config, role_group) = construct_boilerplate(input); + let (hbase, merged_config, merged_jvm_argument_overrides) = construct_boilerplate(input); let global_jvm_args = construct_global_jvm_args(false); let role_specific_non_heap_jvm_args = - construct_role_specific_non_heap_jvm_args(&hbase, &hbase_role, &role_group).unwrap(); + construct_role_specific_non_heap_jvm_args(&hbase, &merged_jvm_argument_overrides); let hbase_heapsize_env = construct_hbase_heapsize_env(&merged_config).unwrap(); assert_eq!(global_jvm_args, ""); @@ -216,11 +180,11 @@ mod tests { - -Xmx40000m # This has no effect! - -Dhttps.proxyPort=1234 "#; - let (hbase, hbase_role, merged_config, role_group) = construct_boilerplate(input); + let (hbase, merged_config, merged_jvm_argument_overrides) = construct_boilerplate(input); let global_jvm_args = construct_global_jvm_args(hbase.has_kerberos_enabled()); let role_specific_non_heap_jvm_args = - construct_role_specific_non_heap_jvm_args(&hbase, &hbase_role, &role_group).unwrap(); + construct_role_specific_non_heap_jvm_args(&hbase, &merged_jvm_argument_overrides); let hbase_heapsize_env = construct_hbase_heapsize_env(&merged_config).unwrap(); assert_eq!( @@ -240,7 +204,11 @@ mod tests { fn construct_boilerplate( hbase_cluster: &str, - ) -> (v1alpha1::HbaseCluster, HbaseRole, AnyServiceConfig, String) { + ) -> ( + v1alpha1::HbaseCluster, + AnyServiceConfig, + JvmArgumentOverrides, + ) { let hbase: v1alpha1::HbaseCluster = serde_yaml::from_str(hbase_cluster).expect("illegal test input"); @@ -249,6 +217,19 @@ mod tests { .merged_config(&hbase_role, "default", "my-hdfs") .unwrap(); - (hbase, hbase_role, merged_config, "default".to_owned()) + // Merge the role <- role-group JVM argument overrides the same way + // `with_validated_config` does, so the tests exercise the real merge path. + let role = hbase.spec.region_servers.as_ref().unwrap(); + let mut merged_common = role + .role_groups + .get("default") + .unwrap() + .config + .product_specific_common_config + .clone(); + merged_common.merge(&role.config.product_specific_common_config); + let merged_jvm_argument_overrides = merged_common.jvm_argument_overrides; + + (hbase, merged_config, merged_jvm_argument_overrides) } } diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index 730fa2ba..588495a7 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -146,7 +146,7 @@ pub struct ValidatedRoleConfig { /// (role <- role group) `configOverrides`, `envOverrides` and `podOverrides`. /// /// The merge and validation is performed by -/// [`with_validated_config`](crate::framework::role_utils::with_validated_config); the +/// [`with_validated_config`](stackable_operator::v2::role_utils::with_validated_config); the /// result is flattened into this struct and augmented with the pre-resolved /// `non_heap_jvm_args`. Carrying every override channel (and the JVM args) keeps the /// build step a pure function of [`ValidatedCluster`] that never has to reach back into diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index e9ed4e96..7e1d89a3 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -1,13 +1,20 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, str::FromStr}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ commons::product_image_selection::{self}, - config::{fragment::FromFragment, merge::Merge}, + config::{ + fragment::{self, FromFragment}, + merge::Merge, + }, kube::ResourceExt, - role_utils::{GenericRoleConfig, JavaCommonConfig, Role}, + role_utils::{CommonConfiguration, GenericRoleConfig, Role}, utils::cluster_info::KubernetesClusterInfo, - v2::controller_utils::{get_cluster_name, get_namespace, get_uid}, + v2::{ + builder::pod::container::{self, EnvVarName, EnvVarSet}, + controller_utils::{get_cluster_name, get_namespace, get_uid}, + role_utils::{JavaCommonConfig, with_validated_config}, + }, }; use strum::IntoEnumIterator; @@ -18,7 +25,6 @@ use crate::{ dereference::DereferencedObjects, }, crd::{AnyServiceConfig, HbaseConfigFragment, HbaseRole, RegionServerConfigFragment, v1alpha1}, - framework::role_utils::with_validated_config, kerberos::{ self, kerberos_config_properties, kerberos_discovery_config_properties, kerberos_ssl_client_settings, kerberos_ssl_server_settings, @@ -43,15 +49,13 @@ pub enum Error { MissingRequiredRole { role: String }, #[snafu(display("failed to merge and validate the role group config"))] - ValidateRoleGroupConfig { - source: crate::framework::role_utils::Error, - }, + ValidateRoleGroupConfig { source: fragment::ValidationError }, + + #[snafu(display("invalid environment variable override name"))] + ParseEnvVarName { source: container::Error }, #[snafu(display("failed to resolve kerberos config"))] AddKerberosConfig { source: kerberos::Error }, - - #[snafu(display("failed to construct role-specific JVM arguments"))] - ConstructJvmArgument { source: crate::config::jvm::Error }, } pub fn validate_cluster( @@ -80,7 +84,6 @@ pub fn validate_cluster( let group_configs = match hbase_role { HbaseRole::Master => validate_role_group_configs( hbase, - &hbase_role, hbase.spec.masters.as_ref(), HbaseConfigFragment::default_config( &hbase_role, @@ -91,7 +94,6 @@ pub fn validate_cluster( )?, HbaseRole::RegionServer => validate_role_group_configs( hbase, - &hbase_role, hbase.spec.region_servers.as_ref(), RegionServerConfigFragment::default_config( &hbase_role, @@ -102,7 +104,6 @@ pub fn validate_cluster( )?, HbaseRole::RestServer => validate_role_group_configs( hbase, - &hbase_role, hbase.spec.rest_servers.as_ref(), HbaseConfigFragment::default_config( &hbase_role, @@ -168,19 +169,20 @@ pub fn validate_cluster( /// Validates every role group of a role into a map keyed by role group name. /// -/// Each role group is merged and validated via the local-`framework` -/// [`with_validated_config`], which folds the CRD config fragment (default <- role <- -/// role group) plus the `configOverrides`, `envOverrides`, `cliOverrides` and -/// `podOverrides` (role group wins) into a single -/// [`RoleGroupConfig`](crate::framework::role_utils::RoleGroupConfig). The concrete -/// per-role validated config is wrapped into [`AnyServiceConfig`] via `wrap`, and the -/// role-specific non-heap JVM args are pre-resolved so the build step stays a pure -/// function of [`ValidatedCluster`]. +/// Each role group is merged and validated via +/// [`with_validated_config`](stackable_operator::v2::role_utils::with_validated_config), +/// which folds the CRD config fragment (default <- role <- role group) plus the +/// `configOverrides`, `envOverrides`, `cliOverrides`, `podOverrides` and the +/// `jvmArgumentOverrides` (role group wins) into a single merged +/// [`RoleGroup`](stackable_operator::role_utils::RoleGroup). The per-role validated config +/// is wrapped into [`AnyServiceConfig`] via `wrap`; the merged `envOverrides` are converted +/// into an [`EnvVarSet`] (validating each name eagerly), and the role-specific non-heap JVM +/// args are pre-resolved from the merged `jvmArgumentOverrides` so the build step stays a +/// pure function of [`ValidatedCluster`]. /// /// Returns an empty map if the role is not configured. fn validate_role_group_configs( hbase: &v1alpha1::HbaseCluster, - hbase_role: &HbaseRole, role: Option< &Role, >, @@ -207,16 +209,36 @@ where >(role_group, role, &default_config) .context(ValidateRoleGroupConfigSnafu)?; - let non_heap_jvm_args = - construct_role_specific_non_heap_jvm_args(hbase, hbase_role, role_group_name) - .context(ConstructJvmArgumentSnafu)?; + let CommonConfiguration { + config, + config_overrides, + env_overrides, + cli_overrides: _, + pod_overrides, + product_specific_common_config, + } = validated.config; + + let non_heap_jvm_args = construct_role_specific_non_heap_jvm_args( + hbase, + &product_specific_common_config.jvm_argument_overrides, + ); + + // Convert the merged env-override HashMap into an EnvVarSet, validating each name + // eagerly. Keys are unique (HashMap), so insertion order is irrelevant. + let mut env_overrides_set = EnvVarSet::new(); + for (name, value) in env_overrides { + env_overrides_set = env_overrides_set.with_value( + &EnvVarName::from_str(&name).context(ParseEnvVarNameSnafu)?, + value, + ); + } let validated = ValidatedRoleGroupConfig { - replicas: validated.replicas, - config: wrap(validated.config), - config_overrides: validated.config_overrides, - env_overrides: validated.env_overrides, - pod_overrides: validated.pod_overrides, + replicas: validated.replicas.unwrap_or(1), + config: wrap(config), + config_overrides, + env_overrides: env_overrides_set, + pod_overrides, non_heap_jvm_args, }; Ok((role_group_name.clone(), validated)) @@ -299,11 +321,7 @@ spec: >(role_group, role, &default_config) .unwrap(); - let env: BTreeMap = validated - .env_overrides - .into_iter() - .map(|env_var| (env_var.name, env_var.value.unwrap_or_default())) - .collect(); + let env = validated.config.env_overrides; assert_eq!(env.get("TEST_VAR"), Some(&"MASTER_RG".to_string())); assert_eq!(env.get("TEST_VAR_FROM_MASTER"), Some(&"MASTER".to_string())); diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 969329e1..8be2ee1a 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -30,11 +30,11 @@ use stackable_operator::{ kube::{CustomResource, ResourceExt, runtime::reflector::ObjectRef}, kvp::Labels, product_logging::{self, spec::Logging}, - role_utils::{GenericRoleConfig, JavaCommonConfig, Role, RoleGroupRef}, + role_utils::{GenericRoleConfig, Role, RoleGroupRef}, schemars::{self, JsonSchema}, shared::time::Duration, status::condition::{ClusterCondition, HasStatusCondition}, - v2::config_overrides::KeyValueConfigOverrides, + v2::{config_overrides::KeyValueConfigOverrides, role_utils::JavaCommonConfig}, versioned::versioned, }; use strum::{Display, EnumIter, EnumString}; diff --git a/rust/operator-binary/src/framework.rs b/rust/operator-binary/src/framework.rs deleted file mode 100644 index 568fc008..00000000 --- a/rust/operator-binary/src/framework.rs +++ /dev/null @@ -1,12 +0,0 @@ -//! Local framework helpers that mirror the work-in-progress upstream -//! `stackable_operator::v2::*` modules. -//! -//! We vendor `role_utils` because the upstream `v2::role_utils` requires -//! `CommonConfig: Merge`. HBase (like hdfs, hive and trino) uses -//! `JavaCommonConfig`, whose JVM-argument merge is fallible and so does not -//! implement `Merge`. -//! -//! Follow-up: replace with `stackable_operator::v2::role_utils::*` once upstream -//! relaxes the `Merge` bound. - -pub mod role_utils; diff --git a/rust/operator-binary/src/framework/role_utils.rs b/rust/operator-binary/src/framework/role_utils.rs deleted file mode 100644 index 65980875..00000000 --- a/rust/operator-binary/src/framework/role_utils.rs +++ /dev/null @@ -1,152 +0,0 @@ -//! Vendored variant of `stackable_operator::v2::role_utils` from the -//! `smooth-operator` branch, with simplifications appropriate for hbase-operator. -//! -//! Differences from upstream: -//! - No `cli_overrides_to_vec` helper, `ResourceNames`, or service-account helpers. -//! - The `CommonConfig` (a.k.a. `product_specific_common_config`) does NOT need to -//! implement `Merge`. HBase uses `JavaCommonConfig`, which intentionally does not -//! implement `Merge` because its inner `JvmArgumentOverrides::try_merge` is -//! fallible (regex validation). The `RoleGroupConfig::product_specific_common_config` -//! field here simply carries the role-group level value through. -//! -//! Replace with `stackable_operator::v2::role_utils::*` once upstream relaxes the -//! `Merge` bound. - -use std::{ - collections::{BTreeMap, HashMap}, - str::FromStr, -}; - -use serde::Serialize; -use snafu::{ResultExt, Snafu}; -use stackable_operator::{ - config::{ - fragment::{self, FromFragment}, - merge::{Merge, merge}, - }, - k8s_openapi::{DeepMerge, api::core::v1::PodTemplateSpec}, - role_utils::{Role, RoleGroup}, - schemars::JsonSchema, - v2::builder::pod::container::{self, EnvVarName, EnvVarSet}, -}; - -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("failed to validate the role group config"))] - ValidateConfig { source: fragment::ValidationError }, - - #[snafu(display("invalid environment variable override name"))] - ParseEnvVarName { source: container::Error }, -} - -/// HBase-friendly view of a validated, merged `RoleGroup`. -#[derive(Clone, Debug, PartialEq)] -pub struct RoleGroupConfig { - pub replicas: u16, - pub config: Config, - pub config_overrides: ConfigOverrides, - pub env_overrides: EnvVarSet, - pub cli_overrides: BTreeMap, - pub pod_overrides: PodTemplateSpec, - pub product_specific_common_config: CommonConfig, -} - -/// Merges and validates the `RoleGroup` with the given `role` and `default_config`. -pub fn with_validated_config( - role_group: &RoleGroup, - role: &Role, - default_config: &Config, -) -> Result, Error> -where - ValidatedConfig: FromFragment, - CommonConfig: Clone + Default + JsonSchema + Serialize, - Config: Clone + Merge, - RoleConfig: Default + JsonSchema + Serialize, - ConfigOverrides: Clone + Default + JsonSchema + Merge + Serialize, -{ - let validated_config = - validate_config(role_group, role, default_config).context(ValidateConfigSnafu)?; - Ok(RoleGroupConfig { - replicas: role_group.replicas.unwrap_or(1), - config: validated_config, - config_overrides: merged_config_overrides( - &role.config.config_overrides, - role_group.config.config_overrides.clone(), - ), - env_overrides: merged_env_overrides( - &role.config.env_overrides, - &role_group.config.env_overrides, - )?, - cli_overrides: merged_cli_overrides( - role.config.cli_overrides.clone(), - role_group.config.cli_overrides.clone(), - ), - pod_overrides: merged_pod_overrides( - role.config.pod_overrides.clone(), - role_group.config.pod_overrides.clone(), - ), - product_specific_common_config: role_group.config.product_specific_common_config.clone(), - }) -} - -fn validate_config( - role_group: &RoleGroup, - role: &Role, - default_config: &Config, -) -> Result -where - ValidatedConfig: FromFragment, - CommonConfig: Default + JsonSchema + Serialize, - Config: Clone + Merge, - RoleConfig: Default + JsonSchema + Serialize, - ConfigOverrides: Default + JsonSchema + Serialize, -{ - role_group.validate_config(role, default_config) -} - -fn merged_config_overrides( - role_config_overrides: &ConfigOverrides, - role_group_config_overrides: ConfigOverrides, -) -> ConfigOverrides -where - ConfigOverrides: Merge, -{ - merge(role_group_config_overrides, role_config_overrides) -} - -fn merged_env_overrides( - role_env_overrides: &HashMap, - role_group_env_overrides: &HashMap, -) -> Result { - // Process the role first, then the role group, so that role-group overrides win on key - // collisions (`EnvVarSet::with_value` overrides earlier entries with the same name). - let mut env_overrides = EnvVarSet::new(); - for (name, value) in role_env_overrides - .iter() - .chain(role_group_env_overrides.iter()) - { - env_overrides = env_overrides.with_value( - &EnvVarName::from_str(name).context(ParseEnvVarNameSnafu)?, - value.clone(), - ); - } - Ok(env_overrides) -} - -fn merged_cli_overrides( - role_cli_overrides: BTreeMap, - role_group_cli_overrides: BTreeMap, -) -> BTreeMap { - let mut merged = role_cli_overrides; - merged.extend(role_group_cli_overrides); - merged -} - -fn merged_pod_overrides( - role_pod_overrides: PodTemplateSpec, - role_group_pod_overrides: PodTemplateSpec, -) -> PodTemplateSpec { - let mut merged = role_pod_overrides; - merged.merge_from(role_group_pod_overrides); - merged -} diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 3b0a7858..aa923145 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -40,7 +40,6 @@ use crate::{ mod config; mod controller; mod crd; -mod framework; mod hbase_controller; mod kerberos; mod operations; From aba6e440272c2310be9a4f0cb78981f27a7c02cf Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 11 Jun 2026 18:57:18 +0200 Subject: [PATCH 32/56] test: decouple builder tests from crd::merged_config via validated_cluster() --- .../controller/build/properties/hbase_env.rs | 41 +++++--------- .../controller/build/properties/hbase_site.rs | 55 ++++++------------- .../src/controller/build/properties/mod.rs | 54 ++++++++++++++---- .../build/properties/security_properties.rs | 9 ++- .../controller/build/properties/ssl_client.rs | 5 +- .../controller/build/properties/ssl_server.rs | 5 +- rust/operator-binary/src/zookeeper.rs | 14 +++++ 7 files changed, 94 insertions(+), 89 deletions(-) diff --git a/rust/operator-binary/src/controller/build/properties/hbase_env.rs b/rust/operator-binary/src/controller/build/properties/hbase_env.rs index 15ec10a5..6cabcb60 100644 --- a/rust/operator-binary/src/controller/build/properties/hbase_env.rs +++ b/rust/operator-binary/src/controller/build/properties/hbase_env.rs @@ -62,33 +62,18 @@ pub fn build( #[cfg(test)] mod tests { use super::*; - use crate::{ - controller::build::properties::test_support::{config_overrides, minimal_hbase}, - crd::v1alpha1, - }; - - fn master_merged_config(hbase: &v1alpha1::HbaseCluster) -> AnyServiceConfig { - hbase - .merged_config(&HbaseRole::Master, "default", "simple-hdfs") - .expect("merged config for the minimal master group") - } - - fn region_server_merged_config(hbase: &v1alpha1::HbaseCluster) -> AnyServiceConfig { - hbase - .merged_config(&HbaseRole::RegionServer, "default", "simple-hdfs") - .expect("merged config for the minimal region server group") - } + use crate::controller::build::properties::test_support::{merged_config, validated_cluster}; #[test] fn renders_operator_defaults() { - let hbase = minimal_hbase(); - let merged = master_merged_config(&hbase); + let validated_cluster = validated_cluster(); + let merged = merged_config(&validated_cluster, &HbaseRole::Master); let env = build( - &merged, + merged, &HbaseRole::Master, false, "-Xtest".to_string(), - config_overrides(&[]), + KeyValueConfigOverrides::default(), ) .unwrap(); assert!(env.contains("export HBASE_MANAGES_ZK=\"false\""), "{env}"); @@ -97,14 +82,14 @@ mod tests { #[test] fn renders_region_server_opts() { - let hbase = minimal_hbase(); - let merged = region_server_merged_config(&hbase); + let validated_cluster = validated_cluster(); + let merged = merged_config(&validated_cluster, &HbaseRole::RegionServer); let env = build( - &merged, + merged, &HbaseRole::RegionServer, false, "-Xtest".to_string(), - config_overrides(&[]), + KeyValueConfigOverrides::default(), ) .unwrap(); assert!(env.contains("export HBASE_REGIONSERVER_OPTS="), "{env}"); @@ -112,14 +97,14 @@ mod tests { #[test] fn user_override_appears() { - let hbase = minimal_hbase(); - let merged = master_merged_config(&hbase); + let validated_cluster = validated_cluster(); + let merged = merged_config(&validated_cluster, &HbaseRole::Master); let env = build( - &merged, + merged, &HbaseRole::Master, false, "-Xtest".to_string(), - config_overrides(&[("CUSTOM_VAR", "custom_value")]), + [("CUSTOM_VAR", "custom_value")].into(), ) .unwrap(); assert!(env.contains("export CUSTOM_VAR=\"custom_value\""), "{env}"); diff --git a/rust/operator-binary/src/controller/build/properties/hbase_site.rs b/rust/operator-binary/src/controller/build/properties/hbase_site.rs index 8d37330c..2d1b813a 100644 --- a/rust/operator-binary/src/controller/build/properties/hbase_site.rs +++ b/rust/operator-binary/src/controller/build/properties/hbase_site.rs @@ -113,40 +113,19 @@ pub fn build( #[cfg(test)] mod tests { use super::*; - use crate::{ - controller::build::properties::test_support::{config_overrides, minimal_hbase}, - crd::v1alpha1, - }; - - fn master_merged_config(hbase: &v1alpha1::HbaseCluster) -> AnyServiceConfig { - hbase - .merged_config(&HbaseRole::Master, "default", "simple-hdfs") - .expect("merged config for the minimal master group") - } - - fn region_server_merged_config(hbase: &v1alpha1::HbaseCluster) -> AnyServiceConfig { - hbase - .merged_config(&HbaseRole::RegionServer, "default", "simple-hdfs") - .expect("merged config for the minimal region server group") - } - - fn rest_server_merged_config(hbase: &v1alpha1::HbaseCluster) -> AnyServiceConfig { - hbase - .merged_config(&HbaseRole::RestServer, "default", "simple-hdfs") - .expect("merged config for the minimal rest server group") - } + use crate::controller::build::properties::test_support::{merged_config, validated_cluster}; #[test] fn renders_operator_defaults() { - let hbase = minimal_hbase(); - let merged = master_merged_config(&hbase); + let validated_cluster = validated_cluster(); + let merged = merged_config(&validated_cluster, &HbaseRole::Master); let xml = build( &HbaseRole::Master, - &merged, + merged, BTreeMap::new(), BTreeMap::new(), None, - config_overrides(&[]), + KeyValueConfigOverrides::default(), ); assert!( xml.contains("hbase.cluster.distributed\n true"), @@ -160,15 +139,15 @@ mod tests { #[test] fn renders_region_server_bind_settings() { - let hbase = minimal_hbase(); - let merged = region_server_merged_config(&hbase); + let validated_cluster = validated_cluster(); + let merged = merged_config(&validated_cluster, &HbaseRole::RegionServer); let xml = build( &HbaseRole::RegionServer, - &merged, + merged, BTreeMap::new(), BTreeMap::new(), None, - config_overrides(&[]), + KeyValueConfigOverrides::default(), ); assert!( xml.contains("hbase.regionserver.ipc.address\n 0.0.0.0"), @@ -184,15 +163,15 @@ mod tests { #[test] fn renders_rest_server_endpoint() { - let hbase = minimal_hbase(); - let merged = rest_server_merged_config(&hbase); + let validated_cluster = validated_cluster(); + let merged = merged_config(&validated_cluster, &HbaseRole::RestServer); let xml = build( &HbaseRole::RestServer, - &merged, + merged, BTreeMap::new(), BTreeMap::new(), None, - config_overrides(&[]), + KeyValueConfigOverrides::default(), ); assert!( xml.contains( @@ -204,15 +183,15 @@ mod tests { #[test] fn user_override_wins() { - let hbase = minimal_hbase(); - let merged = master_merged_config(&hbase); + let validated_cluster = validated_cluster(); + let merged = merged_config(&validated_cluster, &HbaseRole::Master); let xml = build( &HbaseRole::Master, - &merged, + merged, BTreeMap::new(), BTreeMap::new(), None, - config_overrides(&[("hbase.cluster.distributed", "false")]), + [("hbase.cluster.distributed", "false")].into(), ); assert!( xml.contains("hbase.cluster.distributed\n false"), diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index fad65a89..e7dbd8c2 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -60,19 +60,17 @@ mod tests { #[cfg(test)] pub(crate) mod test_support { - use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; + use stackable_operator::{ + commons::networking::DomainName, utils::cluster_info::KubernetesClusterInfo, + }; - use crate::crd::v1alpha1; - - /// Builds a [`KeyValueConfigOverrides`] from `(key, value)` pairs for tests. - pub fn config_overrides(pairs: &[(&str, &str)]) -> KeyValueConfigOverrides { - KeyValueConfigOverrides { - overrides: pairs - .iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect(), - } - } + use crate::{ + controller::{ + ValidatedCluster, dereference::DereferencedObjects, validate::validate_cluster, + }, + crd::{AnyServiceConfig, HbaseRole, v1alpha1}, + zookeeper::ZookeeperConnectionInformation, + }; /// A minimal three-role HbaseCluster used to drive the per-file builder tests. pub const MINIMAL_HBASE_YAML: &str = r#" @@ -82,6 +80,7 @@ kind: HbaseCluster metadata: name: hbase namespace: default + uid: c2c8c5c0-0b5a-4b1e-9f3e-1a2b3c4d5e6f spec: image: productVersion: 2.6.3 @@ -105,4 +104,35 @@ spec: pub fn minimal_hbase() -> v1alpha1::HbaseCluster { serde_yaml::from_str(MINIMAL_HBASE_YAML).expect("invalid test HbaseCluster YAML") } + + pub fn cluster_info() -> KubernetesClusterInfo { + KubernetesClusterInfo { + cluster_domain: DomainName::try_from("cluster.local").unwrap(), + } + } + + /// Runs the real validation pipeline once over [`minimal_hbase`], with a fixed + /// dereferenced ZooKeeper connection (and no OPA), so the per-file builder tests can + /// pull merged configs straight from the [`ValidatedCluster`] instead of re-merging by + /// hand via `crd::merged_config`. + pub fn validated_cluster() -> ValidatedCluster { + validate_cluster( + &minimal_hbase(), + "oci.example.org", + &cluster_info(), + DereferencedObjects { + zookeeper_connection_information: ZookeeperConnectionInformation::for_tests(), + hbase_opa_config: None, + }, + ) + .expect("validate should succeed for the minimal fixture") + } + + /// The merged [`AnyServiceConfig`] for the `default` role group of `role`. + pub fn merged_config<'a>( + validated_cluster: &'a ValidatedCluster, + role: &HbaseRole, + ) -> &'a AnyServiceConfig { + &validated_cluster.role_group_configs[role]["default"].config + } } diff --git a/rust/operator-binary/src/controller/build/properties/security_properties.rs b/rust/operator-binary/src/controller/build/properties/security_properties.rs index 85421c8f..289e8a4b 100644 --- a/rust/operator-binary/src/controller/build/properties/security_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/security_properties.rs @@ -44,12 +44,11 @@ pub fn build( #[cfg(test)] mod tests { use super::*; - use crate::controller::build::properties::test_support::config_overrides; #[test] fn injects_master_dns_cache_ttl() { assert_eq!( - build(&HbaseRole::Master, config_overrides(&[])).unwrap(), + build(&HbaseRole::Master, KeyValueConfigOverrides::default()).unwrap(), "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=5\n" ); } @@ -57,7 +56,7 @@ mod tests { #[test] fn injects_regionserver_dns_cache_ttl() { assert_eq!( - build(&HbaseRole::RegionServer, config_overrides(&[])).unwrap(), + build(&HbaseRole::RegionServer, KeyValueConfigOverrides::default()).unwrap(), "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=10\n" ); } @@ -65,7 +64,7 @@ mod tests { #[test] fn injects_restserver_dns_cache_ttl() { assert_eq!( - build(&HbaseRole::RestServer, config_overrides(&[])).unwrap(), + build(&HbaseRole::RestServer, KeyValueConfigOverrides::default()).unwrap(), "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=30\n" ); } @@ -75,7 +74,7 @@ mod tests { assert_eq!( build( &HbaseRole::Master, - config_overrides(&[("networkaddress.cache.ttl", "60")]) + [("networkaddress.cache.ttl", "60")].into() ) .unwrap(), "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=60\n" diff --git a/rust/operator-binary/src/controller/build/properties/ssl_client.rs b/rust/operator-binary/src/controller/build/properties/ssl_client.rs index fa3231ac..c9faf3e4 100644 --- a/rust/operator-binary/src/controller/build/properties/ssl_client.rs +++ b/rust/operator-binary/src/controller/build/properties/ssl_client.rs @@ -13,7 +13,6 @@ pub fn build(settings: BTreeMap, overrides: KeyValueConfigOverri #[cfg(test)] mod tests { use super::*; - use crate::controller::build::properties::test_support::config_overrides; #[test] fn settings_appear_in_xml() { @@ -22,7 +21,7 @@ mod tests { "ssl.client.truststore.type".to_string(), "pkcs12".to_string(), )]), - config_overrides(&[]), + KeyValueConfigOverrides::default(), ); assert!( xml.contains("ssl.client.truststore.type\n pkcs12"), @@ -34,7 +33,7 @@ mod tests { fn user_override_appears_in_xml() { let xml = build( BTreeMap::new(), - config_overrides(&[("ssl.client.keystore.type", "jks")]), + [("ssl.client.keystore.type", "jks")].into(), ); assert!( xml.contains("ssl.client.keystore.type\n jks"), diff --git a/rust/operator-binary/src/controller/build/properties/ssl_server.rs b/rust/operator-binary/src/controller/build/properties/ssl_server.rs index 45812736..e6d4b446 100644 --- a/rust/operator-binary/src/controller/build/properties/ssl_server.rs +++ b/rust/operator-binary/src/controller/build/properties/ssl_server.rs @@ -13,13 +13,12 @@ pub fn build(settings: BTreeMap, overrides: KeyValueConfigOverri #[cfg(test)] mod tests { use super::*; - use crate::controller::build::properties::test_support::config_overrides; #[test] fn settings_appear_in_xml() { let xml = build( BTreeMap::from([("ssl.server.keystore.type".to_string(), "pkcs12".to_string())]), - config_overrides(&[]), + KeyValueConfigOverrides::default(), ); assert!( xml.contains("ssl.server.keystore.type\n pkcs12"), @@ -31,7 +30,7 @@ mod tests { fn user_override_appears_in_xml() { let xml = build( BTreeMap::new(), - config_overrides(&[("ssl.server.keystore.type", "jks")]), + [("ssl.server.keystore.type", "jks")].into(), ); assert!( xml.contains("ssl.server.keystore.type\n jks"), diff --git a/rust/operator-binary/src/zookeeper.rs b/rust/operator-binary/src/zookeeper.rs index 51aa27b6..398da1f3 100644 --- a/rust/operator-binary/src/zookeeper.rs +++ b/rust/operator-binary/src/zookeeper.rs @@ -53,6 +53,20 @@ pub struct ZookeeperConnectionInformation { port: u16, } +#[cfg(test)] +impl ZookeeperConnectionInformation { + /// A fixed connection used by the per-file builder tests, so they don't need a live + /// ZooKeeper/ZNode discovery `ConfigMap`. + pub(crate) fn for_tests() -> Self { + Self { + hosts: "simple-zk-server-default-0.simple-zk-server-default.default.svc.cluster.local" + .to_owned(), + chroot: "/znode-test".to_owned(), + port: 2282, + } + } +} + impl ZookeeperConnectionInformation { pub async fn retrieve(hbase: &v1alpha1::HbaseCluster, client: &Client) -> Result { let zk_discovery_cm_name = &hbase.spec.cluster_config.zookeeper_config_map_name; From 96a63295239f77ff2e3ce885bb63afe7afafcc82 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 11 Jun 2026 18:57:30 +0200 Subject: [PATCH 33/56] chore: bump stackable-operator --- Cargo.nix | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index f222688f..7cd80758 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4826,7 +4826,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "k8s_version"; @@ -9520,7 +9520,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_certs"; @@ -9720,7 +9720,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_operator"; @@ -9914,7 +9914,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; procMacro = true; @@ -9949,7 +9949,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_shared"; @@ -10030,7 +10030,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_telemetry"; @@ -10140,7 +10140,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_versioned"; @@ -10190,7 +10190,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; procMacro = true; @@ -10258,7 +10258,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_webhook"; From 1a4e70830ee0c0d97146ea132e490109d7f17c09 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 11 Jun 2026 19:03:29 +0200 Subject: [PATCH 34/56] refactor: move jvm config builder into controller/build --- rust/operator-binary/src/config/mod.rs | 1 - rust/operator-binary/src/{config => controller/build}/jvm.rs | 0 rust/operator-binary/src/controller/build/mod.rs | 1 + .../src/controller/build/properties/hbase_env.rs | 2 +- rust/operator-binary/src/controller/validate.rs | 3 +-- rust/operator-binary/src/main.rs | 1 - 6 files changed, 3 insertions(+), 5 deletions(-) delete mode 100644 rust/operator-binary/src/config/mod.rs rename rust/operator-binary/src/{config => controller/build}/jvm.rs (100%) diff --git a/rust/operator-binary/src/config/mod.rs b/rust/operator-binary/src/config/mod.rs deleted file mode 100644 index 271c6d99..00000000 --- a/rust/operator-binary/src/config/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod jvm; diff --git a/rust/operator-binary/src/config/jvm.rs b/rust/operator-binary/src/controller/build/jvm.rs similarity index 100% rename from rust/operator-binary/src/config/jvm.rs rename to rust/operator-binary/src/controller/build/jvm.rs diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs index 4a7bba72..f3c2259c 100644 --- a/rust/operator-binary/src/controller/build/mod.rs +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -1,3 +1,4 @@ pub mod config_map; pub mod discovery; +pub mod jvm; pub mod properties; diff --git a/rust/operator-binary/src/controller/build/properties/hbase_env.rs b/rust/operator-binary/src/controller/build/properties/hbase_env.rs index 6cabcb60..e19a1807 100644 --- a/rust/operator-binary/src/controller/build/properties/hbase_env.rs +++ b/rust/operator-binary/src/controller/build/properties/hbase_env.rs @@ -6,7 +6,7 @@ use snafu::{ResultExt, Snafu}; use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; use crate::{ - config::jvm::{self, construct_global_jvm_args, construct_hbase_heapsize_env}, + controller::build::jvm::{self, construct_global_jvm_args, construct_hbase_heapsize_env}, crd::{AnyServiceConfig, HbaseRole}, }; diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 7e1d89a3..9001706f 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -19,10 +19,9 @@ use stackable_operator::{ use strum::IntoEnumIterator; use crate::{ - config::jvm::construct_role_specific_non_heap_jvm_args, controller::{ ValidatedCluster, ValidatedClusterConfig, ValidatedRoleConfig, ValidatedRoleGroupConfig, - dereference::DereferencedObjects, + build::jvm::construct_role_specific_non_heap_jvm_args, dereference::DereferencedObjects, }, crd::{AnyServiceConfig, HbaseConfigFragment, HbaseRole, RegionServerConfigFragment, v1alpha1}, kerberos::{ diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index aa923145..7dd470c4 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -37,7 +37,6 @@ use crate::{ webhooks::conversion::create_webhook_server, }; -mod config; mod controller; mod crd; mod hbase_controller; From 6210963b595844235136266eea7c3d0ae19ea85e Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 11 Jun 2026 19:13:31 +0200 Subject: [PATCH 35/56] refactor: remove HbaseCluster::merged_config, dedup merge via with_validated_config --- .../src/controller/build/jvm.rs | 29 +-- rust/operator-binary/src/crd/affinity.rs | 16 +- rust/operator-binary/src/crd/mod.rs | 241 ++++++++---------- 3 files changed, 130 insertions(+), 156 deletions(-) diff --git a/rust/operator-binary/src/controller/build/jvm.rs b/rust/operator-binary/src/controller/build/jvm.rs index 11c06579..205f5af2 100644 --- a/rust/operator-binary/src/controller/build/jvm.rs +++ b/rust/operator-binary/src/controller/build/jvm.rs @@ -97,8 +97,6 @@ fn is_heap_jvm_argument(jvm_argument: &str) -> bool { #[cfg(test)] mod tests { - use stackable_operator::config::merge::Merge; - use super::*; use crate::crd::{HbaseRole, v1alpha1}; @@ -212,23 +210,16 @@ mod tests { let hbase: v1alpha1::HbaseCluster = serde_yaml::from_str(hbase_cluster).expect("illegal test input"); - let hbase_role = HbaseRole::RegionServer; - let merged_config = hbase - .merged_config(&hbase_role, "default", "my-hdfs") - .unwrap(); - - // Merge the role <- role-group JVM argument overrides the same way - // `with_validated_config` does, so the tests exercise the real merge path. - let role = hbase.spec.region_servers.as_ref().unwrap(); - let mut merged_common = role - .role_groups - .get("default") - .unwrap() - .config - .product_specific_common_config - .clone(); - merged_common.merge(&role.config.product_specific_common_config); - let merged_jvm_argument_overrides = merged_common.jvm_argument_overrides; + // Merge + validate the region server `default` role group via the real + // `with_validated_config` path, returning the merged config (for heap sizing) and the + // merged JVM argument overrides. + let (merged_config, merged_jvm_argument_overrides) = + crate::crd::test_helpers::merged_role_group_config( + &hbase, + &HbaseRole::RegionServer, + "default", + "my-hdfs", + ); (hbase, merged_config, merged_jvm_argument_overrides) } diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 4d7e52b3..6302ea95 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -124,15 +124,13 @@ mod tests { "#; let hbase: v1alpha1::HbaseCluster = serde_yaml::from_str(input).expect("illegal test input"); - let affinity = hbase - .merged_config( - &role, - "default", - &hbase.spec.cluster_config.hdfs_config_map_name, - ) - .unwrap() - .affinity() - .clone(); + let (merged_config, _) = crate::crd::test_helpers::merged_role_group_config( + &hbase, + &role, + "default", + &hbase.spec.cluster_config.hdfs_config_map_name, + ); + let affinity = merged_config.affinity().clone(); let mut expected_affinities = vec![WeightedPodAffinityTerm { pod_affinity_term: PodAffinityTerm { diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 8be2ee1a..a278c3eb 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use security::AuthenticationConfig; use serde::{Deserialize, Serialize}; use shell_escape::escape; -use snafu::{OptionExt, ResultExt, Snafu}; +use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::pod::volume::{ ListenerOperatorVolumeSourceBuilder, ListenerOperatorVolumeSourceBuilderError, @@ -19,7 +19,7 @@ use stackable_operator::{ }, }, config::{ - fragment::{self, Fragment, ValidationError}, + fragment::Fragment, merge::{Atomic, Merge}, }, deep_merger::ObjectOverrides, @@ -27,7 +27,7 @@ use stackable_operator::{ api::core::v1::{EnvVar, PersistentVolumeClaim, Volume}, apimachinery::pkg::api::resource::Quantity, }, - kube::{CustomResource, ResourceExt, runtime::reflector::ObjectRef}, + kube::{CustomResource, runtime::reflector::ObjectRef}, kvp::Labels, product_logging::{self, spec::Logging}, role_utils::{GenericRoleConfig, Role, RoleGroupRef}, @@ -107,18 +107,6 @@ pub type RestServerRoleType = #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("the HBase role [{role}] is missing from spec"))] - MissingHbaseRole { role: String }, - - #[snafu(display("fragment validation failure"))] - FragmentValidationFailure { source: ValidationError }, - - #[snafu(display("empty values for role-group are not permitted"))] - EmptyRoleGroup, - - #[snafu(display("role-group not found by name"))] - RoleGroupNotFound, - #[snafu(display("failed to build listener volume"))] BuildListenerVolume { source: ListenerOperatorVolumeSourceBuilderError, @@ -241,112 +229,6 @@ impl HasStatusCondition for v1alpha1::HbaseCluster { } impl v1alpha1::HbaseCluster { - /// Retrieve and merge resource configs for role and role groups - /// Merges and validates the config for `role`/`role_group`. - /// - /// The role-group config is merged on top of the role config on top of the - /// operator defaults (most specific wins) and the result is validated. - pub fn merged_config( - &self, - role: &HbaseRole, - role_group: &str, - hdfs_discovery_cm_name: &str, - ) -> Result { - // Trivial values for role-groups are not allowed - if role_group.is_empty() { - return Err(Error::EmptyRoleGroup); - } - - match role { - HbaseRole::Master => { - let default_config = HbaseConfigFragment::default_config( - role, - &self.name_any(), - hdfs_discovery_cm_name, - ); - let role_obj = self.spec.masters.as_ref().context(MissingHbaseRoleSnafu { - role: role.to_string(), - })?; - - let mut role_config = role_obj.config.config.clone(); - let mut role_group_config = role_obj - .role_groups - .get(role_group) - .context(RoleGroupNotFoundSnafu)? - .config - .config - .clone(); - - role_config.merge(&default_config); - role_group_config.merge(&role_config); - Ok(AnyServiceConfig::Master( - fragment::validate(role_group_config) - .context(FragmentValidationFailureSnafu)?, - )) - } - HbaseRole::RegionServer => { - let default_config = RegionServerConfigFragment::default_config( - role, - &self.name_any(), - hdfs_discovery_cm_name, - ); - let role_obj = - self.spec - .region_servers - .as_ref() - .context(MissingHbaseRoleSnafu { - role: role.to_string(), - })?; - - let mut role_config = role_obj.config.config.clone(); - let mut role_group_config = role_obj - .role_groups - .get(role_group) - .context(RoleGroupNotFoundSnafu)? - .config - .config - .clone(); - - role_config.merge(&default_config); - role_group_config.merge(&role_config); - Ok(AnyServiceConfig::RegionServer( - fragment::validate(role_group_config) - .context(FragmentValidationFailureSnafu)?, - )) - } - HbaseRole::RestServer => { - let default_config = HbaseConfigFragment::default_config( - role, - &self.name_any(), - hdfs_discovery_cm_name, - ); - let role_obj = self - .spec - .rest_servers - .as_ref() - .context(MissingHbaseRoleSnafu { - role: role.to_string(), - })?; - - let mut role_config = role_obj.config.config.clone(); - let mut role_group_config = role_obj - .role_groups - .get(role_group) - .context(RoleGroupNotFoundSnafu)? - .config - .config - .clone(); - - role_config.merge(&default_config); - role_group_config.merge(&role_config); - Ok(AnyServiceConfig::RestServer( - fragment::validate(role_group_config) - .context(FragmentValidationFailureSnafu)?, - )) - } - } - } - /// Metadata about a server rolegroup pub fn server_rolegroup_ref( &self, @@ -951,6 +833,110 @@ impl AnyServiceConfig { } } +#[cfg(test)] +pub(crate) mod test_helpers { + use stackable_operator::{ + config::{fragment::FromFragment, merge::Merge}, + kube::ResourceExt, + role_utils::{GenericRoleConfig, Role}, + v2::{ + jvm_argument_overrides::JvmArgumentOverrides, + role_utils::{JavaCommonConfig, with_validated_config}, + }, + }; + + use super::{ + AnyServiceConfig, HbaseConfig, HbaseConfigFragment, HbaseRole, RegionServerConfig, + RegionServerConfigFragment, v1alpha1, + }; + + /// Test helper: merge + validate a single role group via the production + /// [`with_validated_config`] path (the same merge the controller runs), returning the + /// role-specific [`AnyServiceConfig`] and the merged [`JvmArgumentOverrides`]. + pub(crate) fn merged_role_group_config( + hbase: &v1alpha1::HbaseCluster, + role: &HbaseRole, + role_group: &str, + hdfs_discovery_cm_name: &str, + ) -> (AnyServiceConfig, JvmArgumentOverrides) { + match role { + HbaseRole::Master => merge::( + hbase + .spec + .masters + .as_ref() + .expect("master role must be defined"), + role_group, + HbaseConfigFragment::default_config( + role, + &hbase.name_any(), + hdfs_discovery_cm_name, + ), + AnyServiceConfig::Master, + ), + HbaseRole::RegionServer => merge::( + hbase + .spec + .region_servers + .as_ref() + .expect("region server role must be defined"), + role_group, + RegionServerConfigFragment::default_config( + role, + &hbase.name_any(), + hdfs_discovery_cm_name, + ), + AnyServiceConfig::RegionServer, + ), + HbaseRole::RestServer => merge::( + hbase + .spec + .rest_servers + .as_ref() + .expect("rest server role must be defined"), + role_group, + HbaseConfigFragment::default_config( + role, + &hbase.name_any(), + hdfs_discovery_cm_name, + ), + AnyServiceConfig::RestServer, + ), + } + } + + fn merge( + role: &Role, + role_group: &str, + default_config: Config, + wrap: fn(ValidatedConfig) -> AnyServiceConfig, + ) -> (AnyServiceConfig, JvmArgumentOverrides) + where + Config: Clone + Merge, + ValidatedConfig: FromFragment, + { + let role_group = role + .role_groups + .get(role_group) + .expect("role group must be defined"); + let validated = with_validated_config::< + ValidatedConfig, + JavaCommonConfig, + Config, + GenericRoleConfig, + v1alpha1::HbaseConfigOverrides, + >(role_group, role, &default_config) + .expect("role group config should merge and validate"); + ( + wrap(validated.config.config), + validated + .config + .product_specific_common_config + .jvm_argument_overrides, + ) + } +} + #[cfg(test)] mod tests { use indoc::indoc; @@ -1011,13 +997,12 @@ spec: let hbase_role = HbaseRole::RegionServer; let rolegroup = hbase.server_rolegroup_ref(hbase_role.to_string(), role_group_name); - let merged_config = hbase - .merged_config( - &hbase_role, - &rolegroup.role_group, - &hbase.spec.cluster_config.hdfs_config_map_name, - ) - .unwrap(); + let (merged_config, _) = super::test_helpers::merged_role_group_config( + &hbase, + &hbase_role, + &rolegroup.role_group, + &hbase.spec.cluster_config.hdfs_config_map_name, + ); if let AnyServiceConfig::RegionServer(config) = merged_config { assert_eq!(run_before_shutdown, config.region_mover.run_before_shutdown); assert_eq!(max_threads, config.region_mover.max_threads); From 1acf727fa141fb5c91843f5885cd12f9bd67d08a Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 12:11:29 +0200 Subject: [PATCH 36/56] chore: bump stackable-operator --- Cargo.lock | 87 +++++++++++++++---------------- Cargo.nix | 130 ++++++++++++++++++++++------------------------ crate-hashes.json | 18 +++---- 3 files changed, 112 insertions(+), 123 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 569b529f..a6f79e52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -302,9 +302,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "cc" -version = "1.2.63" +version = "1.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" +checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" dependencies = [ "find-msvc-tools", "jobserver", @@ -574,9 +574,6 @@ name = "deranged" version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" -dependencies = [ - "powerfmt", -] [[package]] name = "derive_more" @@ -1453,9 +1450,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.100" +version = "0.3.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2025f20d7a4fa7785846e7b63d10a76d3f1cee98ee5cb79ea59703f95e42162" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" dependencies = [ "cfg-if", "futures-util", @@ -1514,7 +1511,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "darling", "regex", @@ -1727,9 +1724,9 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "memchr" -version = "2.8.1" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "mime" @@ -2797,9 +2794,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.1" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "snafu" @@ -2899,7 +2896,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "const-oid", "ecdsa", @@ -2946,7 +2943,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.111.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "base64", "clap", @@ -2971,6 +2968,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "sha2", "snafu 0.9.1", "stackable-operator-derive", "stackable-shared", @@ -2990,7 +2988,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "darling", "proc-macro2", @@ -3001,7 +2999,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "jiff", "k8s-openapi", @@ -3018,7 +3016,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.4" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "axum", "clap", @@ -3042,7 +3040,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "kube", "schemars", @@ -3056,7 +3054,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "convert_case", "convert_case_extras", @@ -3074,7 +3072,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "arc-swap", "async-trait", @@ -3235,12 +3233,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.47" +version = "0.3.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" dependencies = [ "deranged", - "itoa", "num-conv", "powerfmt", "serde_core", @@ -3250,15 +3247,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.27" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +checksum = "71c652a3727a9cbb9a02f707f530b618ce00d0ccd762009c8c23bd191df3c17d" dependencies = [ "num-conv", "time-core", @@ -3711,18 +3708,18 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.3+wasi-0.2.9" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ "wit-bindgen", ] [[package]] name = "wasm-bindgen" -version = "0.2.123" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a254a4b10c19a76f09a27640e7ffbf9bc30bf67e16a3bf28aaefa4920fe81563" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" dependencies = [ "cfg-if", "once_cell", @@ -3733,9 +3730,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.73" +version = "0.4.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54568702fabf5d4849ce2b90fadfa64168a097eaf4b351ce9df8b687a0086aaf" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" dependencies = [ "js-sys", "wasm-bindgen", @@ -3743,9 +3740,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.123" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24a40fc75b0ec6f3746ceb10d36f53a93dcd68a93b11b6445983945d79eba0dc" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3753,9 +3750,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.123" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "908f34bd9b9ce3d4caf07b72dfab63d61504d156856c6bd3cd87fa350cf3985b" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" dependencies = [ "bumpalo", "proc-macro2", @@ -3766,18 +3763,18 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.123" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7acbf7616c27b194bbb550bf77ed0c2c3e5b7fd1260a93082b95fb7f47959b92" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" -version = "0.3.100" +version = "0.3.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e0871acf327f283dc6da28a1696cdc64fb355ba9f935d052021fa77f35cce69" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" dependencies = [ "js-sys", "wasm-bindgen", @@ -4041,18 +4038,18 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.8.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" dependencies = [ "zeroize_derive", ] [[package]] name = "zeroize_derive" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.nix b/Cargo.nix index 7cd80758..17caabab 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -962,9 +962,9 @@ rec { }; "cc" = rec { crateName = "cc"; - version = "1.2.63"; + version = "1.2.64"; edition = "2018"; - sha256 = "0zy2bqc4nvj6bv2cipx4h4bn65wf1zqf1fw1hsh64mmvg1hh2vjm"; + sha256 = "07shcd8faxw7csz13m3cg2mj6i8z07pqs960k181pscbjpyqgn6s"; authors = [ "Alex Crichton " ]; @@ -1740,14 +1740,6 @@ rec { authors = [ "Jacob Pratt " ]; - dependencies = [ - { - name = "powerfmt"; - packageId = "powerfmt"; - optional = true; - usesDefaultFeatures = false; - } - ]; features = { "macros" = [ "dep:deranged-macros" ]; "num" = [ "dep:num-traits" ]; @@ -1759,7 +1751,7 @@ rec { "rand09" = [ "dep:rand09" ]; "serde" = [ "dep:serde_core" ]; }; - resolvedDefaultFeatures = [ "default" "powerfmt" ]; + resolvedDefaultFeatures = [ "default" ]; }; "derive_more" = rec { crateName = "derive_more"; @@ -4618,9 +4610,9 @@ rec { }; "js-sys" = rec { crateName = "js-sys"; - version = "0.3.100"; + version = "0.3.102"; edition = "2021"; - sha256 = "0qi1wjakyw2rx9wwprcfx77g3lvn1b8n6yvfhj2pgym4swh5y0pj"; + sha256 = "0cgxklnyrfpzvf32cvdl3x5d070kfsv7ykdxfl3yizwdjqq4rl03"; libName = "js_sys"; authors = [ "The wasm-bindgen Developers" @@ -4826,8 +4818,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; libName = "k8s_version"; authors = [ @@ -5690,9 +5682,9 @@ rec { }; "memchr" = rec { crateName = "memchr"; - version = "2.8.1"; + version = "2.8.2"; edition = "2021"; - sha256 = "1n448jx01h5z2xknj6x2dhxgr8s8fb717cf6vfqj5lmhkpj7m53b"; + sha256 = "1i33wr49pcz2sbd12nds3n9fszay8kq5bk78gwciz462mcs49448"; authors = [ "Andrew Gallant " "bluss" @@ -9211,9 +9203,9 @@ rec { }; "smallvec" = rec { crateName = "smallvec"; - version = "1.15.1"; + version = "1.15.2"; edition = "2018"; - sha256 = "00xxdxxpgyq5vjnpljvkmy99xij5rxgh913ii1v16kzynnivgcb7"; + sha256 = "143wzbqf6vgapdp2z4qpl0yvlqcn17s8cnk8m28rqly808zsdmlf"; authors = [ "The Servo Project Developers" ]; @@ -9520,8 +9512,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; libName = "stackable_certs"; authors = [ @@ -9720,8 +9712,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; libName = "stackable_operator"; authors = [ @@ -9830,6 +9822,11 @@ rec { name = "serde_yaml"; packageId = "serde_yaml"; } + { + name = "sha2"; + packageId = "sha2"; + features = [ "oid" ]; + } { name = "snafu"; packageId = "snafu 0.9.1"; @@ -9914,8 +9911,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -9949,8 +9946,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; libName = "stackable_shared"; authors = [ @@ -10030,8 +10027,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; libName = "stackable_telemetry"; authors = [ @@ -10140,8 +10137,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; libName = "stackable_versioned"; authors = [ @@ -10190,8 +10187,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -10258,8 +10255,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; libName = "stackable_webhook"; authors = [ @@ -10698,9 +10695,9 @@ rec { }; "time" = rec { crateName = "time"; - version = "0.3.47"; + version = "0.3.49"; edition = "2024"; - sha256 = "0b7g9ly2iabrlgizliz6v5x23yq5d6bpp0mqz6407z1s526d8fvl"; + sha256 = "0sc4dgw6g187gvz5qj9iqqk2ashqzvdwi664b2183gbvsk1566ki"; authors = [ "Jacob Pratt " "Time contributors" @@ -10709,12 +10706,6 @@ rec { { name = "deranged"; packageId = "deranged"; - features = [ "powerfmt" ]; - } - { - name = "itoa"; - packageId = "itoa"; - optional = true; } { name = "num-conv"; @@ -10754,13 +10745,14 @@ rec { features = { "alloc" = [ "serde_core?/alloc" ]; "default" = [ "std" ]; - "formatting" = [ "dep:itoa" "std" "time-macros?/formatting" ]; + "formatting" = [ "std" "time-macros?/formatting" ]; "large-dates" = [ "time-core/large-dates" "time-macros?/large-dates" ]; "local-offset" = [ "std" "dep:libc" "dep:num_threads" ]; "macros" = [ "dep:time-macros" ]; "parsing" = [ "time-macros?/parsing" ]; "quickcheck" = [ "dep:quickcheck" "alloc" "deranged/quickcheck" ]; - "rand" = [ "rand08" "rand09" ]; + "rand" = [ "rand08" "rand09" "rand010" ]; + "rand010" = [ "dep:rand010" "deranged/rand010" ]; "rand08" = [ "dep:rand08" "deranged/rand08" ]; "rand09" = [ "dep:rand09" "deranged/rand09" ]; "serde" = [ "dep:serde_core" "time-macros?/serde" "deranged/serde" ]; @@ -10773,9 +10765,9 @@ rec { }; "time-core" = rec { crateName = "time-core"; - version = "0.1.8"; + version = "0.1.9"; edition = "2024"; - sha256 = "1jidl426mw48i7hjj4hs9vxgd9lwqq4vyalm4q8d7y4iwz7y353n"; + sha256 = "028ix0ax7ixp1h1k5zsqwgw85w6y1q32irslma7ci6ddd5kr074y"; libName = "time_core"; authors = [ "Jacob Pratt " @@ -10786,9 +10778,9 @@ rec { }; "time-macros" = rec { crateName = "time-macros"; - version = "0.2.27"; + version = "0.2.29"; edition = "2024"; - sha256 = "058ja265waq275wxvnfwavbz9r1hd4dgwpfn7a1a9a70l32y8w1f"; + sha256 = "0zf1ycfikg93ijf00qnprk801khqnqqga1zp0adbp73sfaim5iki"; procMacro = true; libName = "time_macros"; authors = [ @@ -12430,9 +12422,9 @@ rec { }; "wasip2" = rec { crateName = "wasip2"; - version = "1.0.3+wasi-0.2.9"; + version = "1.0.4+wasi-0.2.12"; edition = "2021"; - sha256 = "1mi3w855dz99xzjqc4aa8c9q5b6z1y5c963pkk4cvmr6vdr4c1i0"; + sha256 = "11wl7lqwq4pbmlmzr6n7bwz0hzy1z6sxc4554bkmrr86w4vznzmn"; dependencies = [ { name = "wit-bindgen"; @@ -12450,9 +12442,9 @@ rec { }; "wasm-bindgen" = rec { crateName = "wasm-bindgen"; - version = "0.2.123"; + version = "0.2.125"; edition = "2021"; - sha256 = "0qqmx07r597gm8lbz8qngvv0phwvpzzyfh3nl84nz9qr1jqs8m52"; + sha256 = "06nakz7nfy0ymyp7a27wfbjwx69659i12117hkgddkiv2iwkznwd"; libName = "wasm_bindgen"; authors = [ "The wasm-bindgen Developers" @@ -12501,9 +12493,9 @@ rec { }; "wasm-bindgen-futures" = rec { crateName = "wasm-bindgen-futures"; - version = "0.4.73"; + version = "0.4.75"; edition = "2021"; - sha256 = "1bva12h8gdpqkp753czlxabs0s21lvgzm41brr4lhpdzz818fmjl"; + sha256 = "104jssshr6cm5hmkn6c66mbkyxgaaphng6c17g0dmj7jhk918fsh"; libName = "wasm_bindgen_futures"; authors = [ "The wasm-bindgen Developers" @@ -12529,9 +12521,9 @@ rec { }; "wasm-bindgen-macro" = rec { crateName = "wasm-bindgen-macro"; - version = "0.2.123"; + version = "0.2.125"; edition = "2021"; - sha256 = "1p50xdwmv543b52bc49vm5lcsgd9adpx647bdisg7ihfbg3hz914"; + sha256 = "0g9w68dwcs4ylm5kxf7schi0kjdfarhc9qlnf8arxc9zn62a28af"; procMacro = true; libName = "wasm_bindgen_macro"; authors = [ @@ -12553,9 +12545,9 @@ rec { }; "wasm-bindgen-macro-support" = rec { crateName = "wasm-bindgen-macro-support"; - version = "0.2.123"; + version = "0.2.125"; edition = "2021"; - sha256 = "0nwqyc63byl7rp9nnv45av8h85fncfmxywkvy35d9qwwkfyk93wh"; + sha256 = "1gayzdx5iwl8gllh7ys79wg9cf4iyasl9hrzzhh5m4xx6nfgvkpy"; libName = "wasm_bindgen_macro_support"; authors = [ "The wasm-bindgen Developers" @@ -12589,10 +12581,10 @@ rec { }; "wasm-bindgen-shared" = rec { crateName = "wasm-bindgen-shared"; - version = "0.2.123"; + version = "0.2.125"; edition = "2021"; links = "wasm_bindgen"; - sha256 = "14lvjm3pzywm5c4962i6s5zmngic1knpggshnnxr9c97dihzgjvs"; + sha256 = "07w7fy5qa14ys3p8v2p84h98yqinw713smibz9v7apcspd29x4r3"; libName = "wasm_bindgen_shared"; authors = [ "The wasm-bindgen Developers" @@ -12607,9 +12599,9 @@ rec { }; "web-sys" = rec { crateName = "web-sys"; - version = "0.3.100"; + version = "0.3.102"; edition = "2021"; - sha256 = "0sffbkrpgyi1402mv4wzp9av6ky6rnb1d2m2dpf87wi7yfn7223f"; + sha256 = "0786aybrnwsgdmcynhc2k5ii291a02rq9zk054j35csyvxr0lhx6"; libName = "web_sys"; authors = [ "The wasm-bindgen Developers" @@ -14222,9 +14214,9 @@ rec { }; "zeroize" = rec { crateName = "zeroize"; - version = "1.8.2"; - edition = "2021"; - sha256 = "1l48zxgcv34d7kjskr610zqsm6j2b4fcr2vfh9jm9j1jgvk58wdr"; + version = "1.9.0"; + edition = "2024"; + sha256 = "0kpnij2v1ig6g2mhc0bnci0lrdfdhiq40afbc0fahajqc9jiag71"; authors = [ "The RustCrypto Project Developers" ]; @@ -14246,9 +14238,9 @@ rec { }; "zeroize_derive" = rec { crateName = "zeroize_derive"; - version = "1.4.3"; - edition = "2021"; - sha256 = "0bl5vd1lz27p4z336nximg5wrlw5j7jc8fxh7iv6r1wrhhav99c5"; + version = "1.5.0"; + edition = "2024"; + sha256 = "0a7kq8srk81pn23xqn7c9jw1jpnfy41ffn802x1zrqqgpdf6al1w"; procMacro = true; authors = [ "The RustCrypto Project Developers" diff --git a/crate-hashes.json b/crate-hashes.json index c9a6e6a9..bb8b51c7 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file From 6c503b9aa8f30d7a087d9434f0de8a351de674e5 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 12:12:51 +0200 Subject: [PATCH 37/56] refactor: Add ValidatedCluster methods and use RoleGroupName. --- .../src/controller/build/config_map.rs | 56 +++-- .../controller/build/properties/logging.rs | 19 +- .../src/controller/build/properties/mod.rs | 7 +- rust/operator-binary/src/controller/mod.rs | 123 ++++++++++- .../src/controller/validate.rs | 14 +- rust/operator-binary/src/crd/mod.rs | 20 +- rust/operator-binary/src/hbase_controller.rs | 192 +++++++----------- rust/operator-binary/src/kerberos.rs | 5 +- 8 files changed, 253 insertions(+), 183 deletions(-) diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index f30050ec..d481c614 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -2,11 +2,10 @@ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ - builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, + builder::configmap::ConfigMapBuilder, k8s_openapi::api::core::v1::ConfigMap, product_logging::framework::VECTOR_CONFIG_FILE, - role_utils::RoleGroupRef, - v2::{builder::meta::ownerreference_from_resource, config_file_writer::PropertiesWriterError}, + v2::{config_file_writer::PropertiesWriterError, types::operator::RoleGroupName}, }; use crate::{ @@ -17,8 +16,7 @@ use crate::{ ssl_server, }, }, - crd::{HbaseRole, v1alpha1}, - hbase_controller::build_recommended_labels, + crd::HbaseRole, }; #[derive(Snafu, Debug)] @@ -35,11 +33,6 @@ pub enum Error { role_group: String, }, - #[snafu(display("failed to build object meta data"))] - ObjectMeta { - source: stackable_operator::builder::meta::Error, - }, - #[snafu(display("cannot build config map for role {role:?} and role group {role_group:?}"))] Assemble { source: stackable_operator::builder::configmap::Error, @@ -53,17 +46,20 @@ type Result = std::result::Result; pub fn build_rolegroup_config_map( cluster: &ValidatedCluster, role: &HbaseRole, - rolegroup_ref: &RoleGroupRef, + role_group_name: &RoleGroupName, ) -> Result { - tracing::info!("Setting up ConfigMap for {:?}", rolegroup_ref); + tracing::info!( + "Setting up ConfigMap for {role}/{role_group_name}", + role = role.to_string() + ); let rg = cluster .role_group_configs .get(role) - .and_then(|groups| groups.get(&rolegroup_ref.role_group)) + .and_then(|groups| groups.get(role_group_name)) .with_context(|| MissingRoleGroupSnafu { - role: rolegroup_ref.role.clone(), - role_group: rolegroup_ref.role_group.clone(), + role: role.to_string(), + role_group: role_group_name.to_string(), })?; let cluster_config = &cluster.cluster_config; @@ -101,21 +97,19 @@ pub fn build_rolegroup_config_map( let security_properties = security_properties::build(role, overrides.security_properties.clone()).with_context( |_| JvmSecurityPropertiesSnafu { - role_group: rolegroup_ref.role_group.clone(), + role_group: role_group_name.to_string(), }, )?; - let cm_metadata = ObjectMetaBuilder::new() - .name_and_namespace(cluster) - .name(rolegroup_ref.object_name()) - .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) - .with_recommended_labels(&build_recommended_labels( - cluster, - &cluster.image.app_version_label_value, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - )) - .context(ObjectMetaSnafu)? + let cm_metadata = cluster + .object_meta( + cluster + .resource_names(role, role_group_name) + .role_group_config_map() + .to_string(), + role, + role_group_name, + ) .build(); let mut builder = ConfigMapBuilder::new(); @@ -137,12 +131,14 @@ pub fn build_rolegroup_config_map( if let Some(log4j2_properties) = logging::build_log4j2(rg.config.logging()) { builder.add_data(ConfigFileName::Log4j2.to_string(), log4j2_properties); } - if let Some(vector_config) = logging::build_vector_config(rolegroup_ref, rg.config.logging()) { + if let Some(vector_config) = + logging::build_vector_config(cluster, role, role_group_name, rg.config.logging()) + { builder.add_data(VECTOR_CONFIG_FILE, vector_config); } builder.build().with_context(|_| AssembleSnafu { - role: rolegroup_ref.role.clone(), - role_group: rolegroup_ref.role_group.clone(), + role: role.to_string(), + role_group: role_group_name.to_string(), }) } diff --git a/rust/operator-binary/src/controller/build/properties/logging.rs b/rust/operator-binary/src/controller/build/properties/logging.rs index 5e105d24..c62bfe0e 100644 --- a/rust/operator-binary/src/controller/build/properties/logging.rs +++ b/rust/operator-binary/src/controller/build/properties/logging.rs @@ -1,4 +1,5 @@ use stackable_operator::{ + kube::runtime::reflector::ObjectRef, memory::{BinaryMultiple, MemoryQuantity}, product_logging::{ self, @@ -7,9 +8,13 @@ use stackable_operator::{ }, }, role_utils::RoleGroupRef, + v2::types::operator::RoleGroupName, }; -use crate::crd::{Container, v1alpha1}; +use crate::{ + controller::ValidatedCluster, + crd::{Container, HbaseRole}, +}; pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; pub const MAX_HBASE_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { @@ -38,7 +43,9 @@ pub fn build_log4j2(logging: &Logging) -> Option { /// /// Returns `None` when the Vector agent is disabled for this role group. pub fn build_vector_config( - rolegroup: &RoleGroupRef, + cluster: &ValidatedCluster, + role: &HbaseRole, + role_group_name: &RoleGroupName, logging: &Logging, ) -> Option { if !logging.enable_vector_agent { @@ -52,8 +59,14 @@ pub fn build_vector_config( _ => None, }; + let rolegroup = RoleGroupRef { + cluster: ObjectRef::from_obj(cluster), + role: role.to_string(), + role_group: role_group_name.to_string(), + }; + Some(product_logging::framework::create_vector_config( - rolegroup, + &rolegroup, vector_log_config, )) } diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index e7dbd8c2..712dcf46 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -60,6 +60,8 @@ mod tests { #[cfg(test)] pub(crate) mod test_support { + use std::str::FromStr; + use stackable_operator::{ commons::networking::DomainName, utils::cluster_info::KubernetesClusterInfo, }; @@ -133,6 +135,9 @@ spec: validated_cluster: &'a ValidatedCluster, role: &HbaseRole, ) -> &'a AnyServiceConfig { - &validated_cluster.role_group_configs[role]["default"].config + let default_role_group = + stackable_operator::v2::types::operator::RoleGroupName::from_str("default") + .expect("'default' is a valid role group name"); + &validated_cluster.role_group_configs[role][&default_role_group].config } } diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index 588495a7..0d620fda 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -2,28 +2,52 @@ pub mod build; pub mod dereference; pub mod validate; -use std::collections::BTreeMap; +use std::{collections::BTreeMap, str::FromStr}; use stackable_operator::{ + builder::meta::ObjectMetaBuilder, commons::product_image_selection::ResolvedProductImage, k8s_openapi::{api::core::v1::PodTemplateSpec, apimachinery::pkg::apis::meta::v1::ObjectMeta}, kube::Resource, + kvp::Labels, v2::{ HasName, HasUid, NameIsValidLabelValue, - builder::pod::container::EnvVarSet, + builder::{meta::ownerreference_from_resource, pod::container::EnvVarSet}, + kvp::label::{recommended_labels, role_group_selector}, + role_group_utils::ResourceNames, types::{ kubernetes::{NamespaceName, Uid}, - operator::ClusterName, + operator::{ + ClusterName, ControllerName, OperatorName, ProductName, ProductVersion, + RoleGroupName, RoleName, + }, }, }, }; use crate::{ - crd::{AnyServiceConfig, HbaseRole, v1alpha1}, + crd::{APP_NAME, AnyServiceConfig, HbaseRole, OPERATOR_NAME, v1alpha1}, + hbase_controller::HBASE_CONTROLLER_NAME, security::opa::HbaseOpaConfig, zookeeper::ZookeeperConnectionInformation, }; +/// The product name (`hbase`) as a type-safe label value. +pub(crate) fn product_name() -> ProductName { + ProductName::from_str(APP_NAME).expect("'hbase' is a valid product name") +} + +/// The operator name as a type-safe label value. +pub(crate) fn operator_name() -> OperatorName { + OperatorName::from_str(OPERATOR_NAME).expect("the operator name is a valid label value") +} + +/// The controller name as a type-safe label value. +pub(crate) fn controller_name() -> ControllerName { + ControllerName::from_str(HBASE_CONTROLLER_NAME) + .expect("the controller name is a valid label value") +} + /// The validated cluster: proves that config merging and validation succeeded for /// every role and role group before any resources are created. #[derive(Clone, Debug)] @@ -37,8 +61,12 @@ pub struct ValidatedCluster { /// The UID of the `HbaseCluster` object, used to build owner references. pub uid: Uid, pub image: ResolvedProductImage, + /// The product version as a valid label value, used for the recommended + /// `app.kubernetes.io/version` label. Derived from the resolved image's app version label + /// value. + pub product_version: ProductVersion, pub cluster_config: ValidatedClusterConfig, - pub role_group_configs: BTreeMap>, + pub role_group_configs: BTreeMap>, pub role_configs: BTreeMap, } @@ -50,9 +78,13 @@ impl ValidatedCluster { uid: Uid, image: ResolvedProductImage, cluster_config: ValidatedClusterConfig, - role_group_configs: BTreeMap>, + role_group_configs: BTreeMap>, role_configs: BTreeMap, ) -> Self { + // `app_version_label_value` is constructed to be a valid label value, so it is also a + // valid `ProductVersion`. + let product_version = ProductVersion::from_str(&image.app_version_label_value) + .expect("the app version label value is a valid product version"); Self { metadata: ObjectMeta { name: Some(name.to_string()), @@ -63,11 +95,90 @@ impl ValidatedCluster { name, uid, image, + product_version, cluster_config, role_group_configs, role_configs, } } + + /// The Kubernetes role name for an [`HbaseRole`] (e.g. `master`, `regionserver`, + /// `restserver`). + pub fn role_name(hbase_role: &HbaseRole) -> RoleName { + RoleName::from_str(&hbase_role.to_string()).expect("an HbaseRole name is a valid role name") + } + + /// Type-safe names for the resources of a given role group. + pub(crate) fn resource_names( + &self, + hbase_role: &HbaseRole, + role_group_name: &RoleGroupName, + ) -> ResourceNames { + ResourceNames { + cluster_name: self.name.clone(), + role_name: Self::role_name(hbase_role), + role_group_name: role_group_name.clone(), + } + } + + /// Recommended labels for a role-group resource. + pub fn recommended_labels( + &self, + hbase_role: &HbaseRole, + role_group_name: &RoleGroupName, + ) -> Labels { + recommended_labels( + self, + &product_name(), + &self.product_version, + &operator_name(), + &controller_name(), + &Self::role_name(hbase_role), + role_group_name, + ) + } + + /// Selector labels matching the pods of a role group. + pub fn role_group_selector( + &self, + hbase_role: &HbaseRole, + role_group_name: &RoleGroupName, + ) -> Labels { + role_group_selector( + self, + &product_name(), + &Self::role_name(hbase_role), + role_group_name, + ) + } + + /// Returns an [`ObjectMetaBuilder`] pre-filled with the namespace, an owner reference back to + /// this cluster, and the recommended labels for a resource named `name` in `role_group_name`. + /// + /// Consolidates the metadata chain repeated by the child-resource builders. Call sites that + /// need extra labels/annotations chain them onto the returned builder. + pub(crate) fn object_meta( + &self, + name: impl Into, + hbase_role: &HbaseRole, + role_group_name: &RoleGroupName, + ) -> ObjectMetaBuilder { + let mut builder = ObjectMetaBuilder::new(); + builder + .name_and_namespace(self) + .name(name) + .ownerreference(ownerreference_from_resource(self, None, Some(true))) + .with_labels(self.recommended_labels(hbase_role, role_group_name)); + builder + } + + /// Whether Kerberos is enabled for this cluster. + /// + /// Mirrors [`v1alpha1::HbaseCluster::has_kerberos_enabled`], derived here from the validated + /// config so build steps don't have to re-read the raw cluster. + pub fn has_kerberos_enabled(&self) -> bool { + self.cluster_config.kerberos_enabled + } } impl Resource for ValidatedCluster { diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 9001706f..26bce2cb 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -14,6 +14,7 @@ use stackable_operator::{ builder::pod::container::{self, EnvVarName, EnvVarSet}, controller_utils::{get_cluster_name, get_namespace, get_uid}, role_utils::{JavaCommonConfig, with_validated_config}, + types::operator::RoleGroupName, }, }; use strum::IntoEnumIterator; @@ -53,6 +54,12 @@ pub enum Error { #[snafu(display("invalid environment variable override name"))] ParseEnvVarName { source: container::Error }, + #[snafu(display("invalid role group name {role_group}"))] + ParseRoleGroupName { + source: stackable_operator::v2::macros::attributed_string_type::Error, + role_group: String, + }, + #[snafu(display("failed to resolve kerberos config"))] AddKerberosConfig { source: kerberos::Error }, } @@ -187,7 +194,7 @@ fn validate_role_group_configs( >, default_config: Config, wrap: fn(ValidatedConfig) -> AnyServiceConfig, -) -> Result, Error> +) -> Result, Error> where Config: Clone + Merge, ValidatedConfig: FromFragment, @@ -199,6 +206,11 @@ where role.role_groups .iter() .map(|(role_group_name, role_group)| { + let role_group_name = RoleGroupName::from_str(role_group_name).with_context(|_| { + ParseRoleGroupNameSnafu { + role_group: role_group_name.clone(), + } + })?; let validated = with_validated_config::< ValidatedConfig, JavaCommonConfig, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index a278c3eb..f9222468 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -27,10 +27,10 @@ use stackable_operator::{ api::core::v1::{EnvVar, PersistentVolumeClaim, Volume}, apimachinery::pkg::api::resource::Quantity, }, - kube::{CustomResource, runtime::reflector::ObjectRef}, + kube::CustomResource, kvp::Labels, product_logging::{self, spec::Logging}, - role_utils::{GenericRoleConfig, Role, RoleGroupRef}, + role_utils::{GenericRoleConfig, Role}, schemars::{self, JsonSchema}, shared::time::Duration, status::condition::{ClusterCondition, HasStatusCondition}, @@ -229,19 +229,6 @@ impl HasStatusCondition for v1alpha1::HbaseCluster { } impl v1alpha1::HbaseCluster { - /// Metadata about a server rolegroup - pub fn server_rolegroup_ref( - &self, - role_name: impl Into, - group_name: impl Into, - ) -> RoleGroupRef { - RoleGroupRef { - cluster: ObjectRef::from_obj(self), - role: role_name.into(), - role_group: group_name.into(), - } - } - pub fn role_config(&self, role: &HbaseRole) -> Option<&GenericRoleConfig> { match role { HbaseRole::Master => self.spec.masters.as_ref().map(|m| &m.role_config), @@ -995,12 +982,11 @@ spec: serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); let hbase_role = HbaseRole::RegionServer; - let rolegroup = hbase.server_rolegroup_ref(hbase_role.to_string(), role_group_name); let (merged_config, _) = super::test_helpers::merged_role_group_config( &hbase, &hbase_role, - &rolegroup.role_group, + role_group_name, &hbase.spec.cluster_config.hdfs_config_map_name, ); if let AnyServiceConfig::RegionServer(config) = merged_config { diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index d39a95cb..92060cfb 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -16,7 +16,7 @@ use stackable_operator::{ }, cli::OperatorEnvironmentOptions, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, - commons::{product_image_selection::ResolvedProductImage, rbac::build_rbac_resources}, + commons::rbac::build_rbac_resources, constants::RESTART_CONTROLLER_ENABLED_LABEL, k8s_openapi::{ DeepMerge, @@ -34,7 +34,7 @@ use stackable_operator::{ core::{DeserializeGuard, error_boundary}, runtime::controller::Action, }, - kvp::{Annotations, Label, LabelError, Labels, ObjectLabels}, + kvp::{Annotations, Label, LabelError, ObjectLabels}, logging::controller::ReconcilerError, product_logging::{ self, @@ -44,18 +44,18 @@ use stackable_operator::{ CustomContainerLogConfig, }, }, - role_utils::RoleGroupRef, shared::time::Duration, status::condition::{ compute_conditions, operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, }, + v2::types::operator::RoleGroupName, }; use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ controller::{ - ValidatedRoleGroupConfig, + ValidatedCluster, ValidatedRoleGroupConfig, build::{ discovery::build_discovery_config_map, properties::logging::{MAX_HBASE_LOG_FILES_SIZE, STACKABLE_LOG_DIR}, @@ -102,10 +102,10 @@ pub enum Error { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("failed to apply Service for {}", rolegroup))] + #[snafu(display("failed to apply Service for role group {role_group}"))] ApplyRoleGroupService { source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, + role_group: RoleGroupName, }, #[snafu(display("failed to apply discovery configmap"))] @@ -123,21 +123,16 @@ pub enum Error { source: crate::controller::build::config_map::Error, }, - #[snafu(display("failed to apply ConfigMap for {}", rolegroup))] + #[snafu(display("failed to apply ConfigMap for role group {role_group}"))] ApplyRoleGroupConfig { source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, + role_group: RoleGroupName, }, - #[snafu(display("failed to apply StatefulSet for {}", rolegroup))] + #[snafu(display("failed to apply StatefulSet for role group {role_group}"))] ApplyRoleGroupStatefulSet { source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, - }, - - #[snafu(display("object is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, + role_group: RoleGroupName, }, #[snafu(display("failed to patch service account"))] @@ -179,11 +174,6 @@ pub enum Error { #[snafu(display("failed to build label"))] BuildLabel { source: LabelError }, - #[snafu(display("failed to build object meta data"))] - ObjectMeta { - source: stackable_operator::builder::meta::Error, - }, - #[snafu(display("failed to configure logging"))] ConfigureLogging { source: LoggingError }, @@ -286,50 +276,48 @@ pub async fn reconcile_hbase( let mut ss_cond_builder = StatefulSetConditionBuilder::default(); for (hbase_role, role_group_configs) in &validated_cluster.role_group_configs { - for (rolegroup_name, validated_rg_config) in role_group_configs { - let rolegroup = hbase.server_rolegroup_ref(hbase_role.to_string(), rolegroup_name); - + for (role_group_name, validated_rg_config) in role_group_configs { let rg_service = - build_rolegroup_service(hbase, hbase_role, &rolegroup, &validated_cluster.image)?; + build_rolegroup_service(hbase, &validated_cluster, hbase_role, role_group_name)?; let rg_metrics_service = build_rolegroup_metrics_service( hbase, + &validated_cluster, hbase_role, - &rolegroup, - &validated_cluster.image, + role_group_name, )?; let rg_configmap = crate::controller::build::config_map::build_rolegroup_config_map( &validated_cluster, hbase_role, - &rolegroup, + role_group_name, ) .context(BuildRolegroupConfigMapSnafu)?; let rg_statefulset = build_rolegroup_statefulset( hbase, + &validated_cluster, hbase_role, - &rolegroup, + role_group_name, validated_rg_config, - &validated_cluster.image, &rbac_sa, )?; cluster_resources .add(client, rg_service) .await .with_context(|_| ApplyRoleGroupServiceSnafu { - rolegroup: rolegroup.clone(), + role_group: role_group_name.clone(), })?; cluster_resources .add(client, rg_metrics_service) .await .with_context(|_| ApplyRoleGroupServiceSnafu { - rolegroup: rolegroup.clone(), + role_group: role_group_name.clone(), })?; cluster_resources .add(client, rg_configmap) .await .with_context(|_| ApplyRoleGroupConfigSnafu { - rolegroup: rolegroup.clone(), + role_group: role_group_name.clone(), })?; // Note: The StatefulSet needs to be applied after all ConfigMaps and Secrets it mounts @@ -340,7 +328,7 @@ pub async fn reconcile_hbase( .add(client, rg_statefulset) .await .with_context(|_| ApplyRoleGroupStatefulSetSnafu { - rolegroup: rolegroup.clone(), + role_group: role_group_name.clone(), })?, ); } @@ -391,9 +379,9 @@ pub async fn reconcile_hbase( /// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. fn build_rolegroup_service( hbase: &v1alpha1::HbaseCluster, + cluster: &ValidatedCluster, hbase_role: &HbaseRole, - rolegroup: &RoleGroupRef, - resolved_product_image: &ResolvedProductImage, + role_group_name: &RoleGroupName, ) -> Result { let ports = hbase_role .ports(hbase) @@ -406,23 +394,18 @@ fn build_rolegroup_service( }) .collect(); - let metadata = ObjectMetaBuilder::new() - .name_and_namespace(hbase) - .name(rolegroup.rolegroup_headless_service_name()) - .ownerreference_from_resource(hbase, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&build_recommended_labels( - hbase, - &resolved_product_image.app_version_label_value, - &rolegroup.role, - &rolegroup.role_group, - )) - .context(ObjectMetaSnafu)? + let metadata = cluster + .object_meta( + cluster + .resource_names(hbase_role, role_group_name) + .headless_service_name() + .to_string(), + hbase_role, + role_group_name, + ) .build(); - let service_selector = - Labels::role_group_selector(hbase, APP_NAME, &rolegroup.role, &rolegroup.role_group) - .context(BuildLabelSnafu)?; + let service_selector = cluster.role_group_selector(hbase_role, role_group_name); let service_spec = ServiceSpec { // Internal communication does not need to be exposed @@ -444,9 +427,9 @@ fn build_rolegroup_service( /// The rolegroup metrics [`Service`] is a service that exposes metrics and a prometheus scraping label. fn build_rolegroup_metrics_service( hbase: &v1alpha1::HbaseCluster, + cluster: &ValidatedCluster, hbase_role: &HbaseRole, - rolegroup: &RoleGroupRef, - resolved_product_image: &ResolvedProductImage, + role_group_name: &RoleGroupName, ) -> Result { let ports = vec![ServicePort { name: Some(HbaseRole::metrics_port_name().to_owned()), @@ -455,23 +438,18 @@ fn build_rolegroup_metrics_service( ..ServicePort::default() }]; - let service_selector = - Labels::role_group_selector(hbase, APP_NAME, &rolegroup.role, &rolegroup.role_group) - .context(BuildLabelSnafu)?; + let service_selector = cluster.role_group_selector(hbase_role, role_group_name); Ok(Service { - metadata: ObjectMetaBuilder::new() - .name_and_namespace(hbase) - .name(rolegroup.rolegroup_metrics_service_name()) - .ownerreference_from_resource(hbase, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&build_recommended_labels( - hbase, - &resolved_product_image.app_version_label_value, - &rolegroup.role, - &rolegroup.role_group, - )) - .context(ObjectMetaSnafu)? + metadata: cluster + .object_meta( + cluster + .resource_names(hbase_role, role_group_name) + .metrics_service_name() + .to_string(), + hbase_role, + role_group_name, + ) .with_label(Label::try_from(("prometheus.io/scrape", "true")).context(LabelBuildSnafu)?) .with_annotations(prometheus_annotations(hbase, hbase_role)) .build(), @@ -516,17 +494,17 @@ fn prometheus_annotations(hbase: &v1alpha1::HbaseCluster, hbase_role: &HbaseRole /// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. /// /// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the corresponding [`Service`] (from [`build_rolegroup_service`]). -#[allow(clippy::too_many_arguments)] fn build_rolegroup_statefulset( hbase: &v1alpha1::HbaseCluster, + cluster: &ValidatedCluster, hbase_role: &HbaseRole, - rolegroup_ref: &RoleGroupRef, + role_group_name: &RoleGroupName, validated_rg_config: &ValidatedRoleGroupConfig, - resolved_product_image: &ResolvedProductImage, service_account: &ServiceAccount, ) -> Result { + let resolved_product_image = &cluster.image; let merged_config = &validated_rg_config.config; - let hbase_version = &resolved_product_image.app_version_label_value; + let resource_names = cluster.resource_names(hbase_role, role_group_name); let ports = hbase_role .ports(hbase) @@ -633,18 +611,10 @@ fn build_rolegroup_statefulset( let mut pod_builder = PodBuilder::new(); - let recommended_object_labels = build_recommended_labels( - hbase, - hbase_version, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - ); - let recommended_labels = - Labels::recommended(&recommended_object_labels).context(LabelBuildSnafu)?; + let recommended_labels = cluster.recommended_labels(hbase_role, role_group_name); let pb_metadata = ObjectMetaBuilder::new() - .with_recommended_labels(&recommended_object_labels) - .context(ObjectMetaSnafu)? + .with_labels(recommended_labels.clone()) .build(); pod_builder @@ -654,7 +624,7 @@ fn build_rolegroup_statefulset( .add_volume(stackable_operator::k8s_openapi::api::core::v1::Volume { name: "hbase-config".to_string(), config_map: Some(ConfigMapVolumeSource { - name: rolegroup_ref.object_name(), + name: resource_names.role_group_config_map().to_string(), ..Default::default() }), ..Default::default() @@ -701,7 +671,7 @@ fn build_rolegroup_statefulset( .add_volume(Volume { name: "log-config".to_string(), config_map: Some(ConfigMapVolumeSource { - name: rolegroup_ref.object_name(), + name: resource_names.role_group_config_map().to_string(), ..ConfigMapVolumeSource::default() }), ..Volume::default() @@ -710,10 +680,10 @@ fn build_rolegroup_statefulset( } add_graceful_shutdown_config(merged_config, &mut pod_builder).context(GracefulShutdownSnafu)?; - if hbase.has_kerberos_enabled() { + if cluster.has_kerberos_enabled() { add_kerberos_pod_config( hbase, - rolegroup_ref, + resource_names.metrics_service_name().as_ref(), &mut hbase_container, &mut pod_builder, merged_config @@ -767,28 +737,16 @@ fn build_rolegroup_statefulset( pod_template.merge_from(validated_rg_config.pod_overrides.clone()); - let metadata = ObjectMetaBuilder::new() - .name_and_namespace(hbase) - .name(rolegroup_ref.object_name()) - .ownerreference_from_resource(hbase, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&build_recommended_labels( - hbase, - hbase_version, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - )) - .context(ObjectMetaSnafu)? + let metadata = cluster + .object_meta( + resource_names.stateful_set_name().to_string(), + hbase_role, + role_group_name, + ) .with_label(RESTART_CONTROLLER_ENABLED_LABEL.to_owned()) .build(); - let statefulset_match_labels = Labels::role_group_selector( - hbase, - APP_NAME, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - ) - .context(BuildLabelSnafu)?; + let statefulset_match_labels = cluster.role_group_selector(hbase_role, role_group_name); let statefulset_spec = StatefulSetSpec { pod_management_policy: Some("Parallel".to_string()), @@ -797,7 +755,7 @@ fn build_rolegroup_statefulset( match_labels: Some(statefulset_match_labels.into()), ..LabelSelector::default() }, - service_name: Some(rolegroup_ref.rolegroup_headless_service_name()), + service_name: Some(resource_names.headless_service_name().to_string()), template: pod_template, volume_claim_templates: listener_pvc, ..StatefulSetSpec::default() @@ -852,10 +810,13 @@ pub fn build_recommended_labels<'a, R>( #[cfg(test)] mod test { + use std::str::FromStr; + use rstest::rstest; - use stackable_operator::kube::runtime::reflector::ObjectRef; + use stackable_operator::v2::types::operator::RoleGroupName; use super::*; + use crate::controller::build::properties::test_support; #[rstest] #[case("2.6.3", HbaseRole::Master, vec!["master", "ui-http"])] @@ -899,22 +860,9 @@ mod test { let hbase: v1alpha1::HbaseCluster = serde_yaml::from_str(&input).expect("illegal test input"); - let resolved_image = ResolvedProductImage { - image: format!("oci.stackable.tech/sdp/hbase:{hbase_version}-stackable0.0.0-dev"), - app_version_label_value: hbase_version - .parse() - .expect("test: hbase version is always valid"), - product_version: hbase_version.to_string(), - image_pull_policy: "Never".to_string(), - pull_secrets: None, - }; - - let role_group_ref = RoleGroupRef { - cluster: ObjectRef::::from_obj(&hbase), - role: role.to_string(), - role_group: "default".to_string(), - }; - let service = build_rolegroup_service(&hbase, &role, &role_group_ref, &resolved_image) + let cluster = test_support::validated_cluster(); + let role_group_name = RoleGroupName::from_str("default").expect("valid role group name"); + let service = build_rolegroup_service(&hbase, &cluster, &role, &role_group_name) .expect("failed to build service"); assert_eq!( diff --git a/rust/operator-binary/src/kerberos.rs b/rust/operator-binary/src/kerberos.rs index 86f99de0..4bc18339 100644 --- a/rust/operator-binary/src/kerberos.rs +++ b/rust/operator-binary/src/kerberos.rs @@ -12,7 +12,6 @@ use stackable_operator::{ }, commons::secret_class::SecretClassVolumeProvisionParts, kube::{ResourceExt, runtime::reflector::ObjectRef}, - role_utils::RoleGroupRef, shared::time::Duration, utils::cluster_info::KubernetesClusterInfo, }; @@ -171,7 +170,7 @@ pub fn kerberos_ssl_client_settings(hbase: &v1alpha1::HbaseCluster) -> BTreeMap< pub fn add_kerberos_pod_config( hbase: &v1alpha1::HbaseCluster, - rolegroup_ref: &RoleGroupRef, + metrics_service_name: &str, cb: &mut ContainerBuilder, pb: &mut PodBuilder, requested_secret_lifetime: Duration, @@ -215,7 +214,7 @@ pub fn add_kerberos_pod_config( .with_pod_scope() .with_node_scope() // We need to add the metrics service for scraping - .with_service_scope(rolegroup_ref.rolegroup_metrics_service_name()) + .with_service_scope(metrics_service_name) .with_format(SecretFormat::TlsPkcs12) .with_tls_pkcs12_password(TLS_STORE_PASSWORD) .with_auto_tls_cert_lifetime(requested_secret_lifetime) From 467c3d5f1d24ccf23daeb52688c55e542533228a Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 12:31:42 +0200 Subject: [PATCH 38/56] refactor: move k8s resources to resource folder, config etc to build --- .../build}/graceful_shutdown.rs | 0 .../src/{ => controller/build}/kerberos.rs | 0 .../src/controller/build/mod.rs | 9 +- .../src/{security => controller/build}/opa.rs | 0 .../controller/build/properties/hbase_site.rs | 3 +- .../src/controller/build/properties/mod.rs | 2 +- .../build/{ => resource}/config_map.rs | 0 .../build/{ => resource}/discovery.rs | 32 +- .../src/controller/build/resource/mod.rs | 8 + .../src/controller/build/resource/pdb.rs | 54 ++ .../src/controller/build/resource/service.rs | 206 ++++++ .../controller/build/resource/statefulset.rs | 385 ++++++++++ .../src/controller/dereference.rs | 11 +- rust/operator-binary/src/controller/mod.rs | 13 +- .../src/controller/validate.rs | 8 +- .../src/{ => controller}/zookeeper.rs | 0 rust/operator-binary/src/hbase_controller.rs | 696 ++---------------- rust/operator-binary/src/main.rs | 6 +- rust/operator-binary/src/operations/mod.rs | 2 - rust/operator-binary/src/operations/pdb.rs | 77 -- rust/operator-binary/src/security/mod.rs | 1 - 21 files changed, 777 insertions(+), 736 deletions(-) rename rust/operator-binary/src/{operations => controller/build}/graceful_shutdown.rs (100%) rename rust/operator-binary/src/{ => controller/build}/kerberos.rs (100%) rename rust/operator-binary/src/{security => controller/build}/opa.rs (100%) rename rust/operator-binary/src/controller/build/{ => resource}/config_map.rs (100%) rename rust/operator-binary/src/controller/build/{ => resource}/discovery.rs (69%) create mode 100644 rust/operator-binary/src/controller/build/resource/mod.rs create mode 100644 rust/operator-binary/src/controller/build/resource/pdb.rs create mode 100644 rust/operator-binary/src/controller/build/resource/service.rs create mode 100644 rust/operator-binary/src/controller/build/resource/statefulset.rs rename rust/operator-binary/src/{ => controller}/zookeeper.rs (100%) delete mode 100644 rust/operator-binary/src/operations/mod.rs delete mode 100644 rust/operator-binary/src/operations/pdb.rs delete mode 100644 rust/operator-binary/src/security/mod.rs diff --git a/rust/operator-binary/src/operations/graceful_shutdown.rs b/rust/operator-binary/src/controller/build/graceful_shutdown.rs similarity index 100% rename from rust/operator-binary/src/operations/graceful_shutdown.rs rename to rust/operator-binary/src/controller/build/graceful_shutdown.rs diff --git a/rust/operator-binary/src/kerberos.rs b/rust/operator-binary/src/controller/build/kerberos.rs similarity index 100% rename from rust/operator-binary/src/kerberos.rs rename to rust/operator-binary/src/controller/build/kerberos.rs diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs index f3c2259c..794be0af 100644 --- a/rust/operator-binary/src/controller/build/mod.rs +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -1,4 +1,9 @@ -pub mod config_map; -pub mod discovery; +//! Builders that turn a [`ValidatedCluster`](crate::controller::ValidatedCluster) into +//! Kubernetes resources. + +pub mod graceful_shutdown; pub mod jvm; +pub mod kerberos; +pub mod opa; pub mod properties; +pub mod resource; diff --git a/rust/operator-binary/src/security/opa.rs b/rust/operator-binary/src/controller/build/opa.rs similarity index 100% rename from rust/operator-binary/src/security/opa.rs rename to rust/operator-binary/src/controller/build/opa.rs diff --git a/rust/operator-binary/src/controller/build/properties/hbase_site.rs b/rust/operator-binary/src/controller/build/properties/hbase_site.rs index 2d1b813a..faabbeeb 100644 --- a/rust/operator-binary/src/controller/build/properties/hbase_site.rs +++ b/rust/operator-binary/src/controller/build/properties/hbase_site.rs @@ -7,12 +7,11 @@ use std::collections::BTreeMap; use stackable_operator::v2::config_overrides::KeyValueConfigOverrides; use crate::{ - controller::build::properties::build_xml_config, + controller::build::{opa::HbaseOpaConfig, properties::build_xml_config}, crd::{ AnyServiceConfig, HBASE_CLUSTER_DISTRIBUTED, HBASE_MASTER_PORT, HBASE_MASTER_UI_PORT, HBASE_REGIONSERVER_PORT, HBASE_REGIONSERVER_UI_PORT, HBASE_ROOTDIR, HbaseRole, }, - security::opa::HbaseOpaConfig, }; /// Renders `hbase-site.xml`. diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index 712dcf46..8cf844d7 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -69,9 +69,9 @@ pub(crate) mod test_support { use crate::{ controller::{ ValidatedCluster, dereference::DereferencedObjects, validate::validate_cluster, + zookeeper::ZookeeperConnectionInformation, }, crd::{AnyServiceConfig, HbaseRole, v1alpha1}, - zookeeper::ZookeeperConnectionInformation, }; /// A minimal three-role HbaseCluster used to drive the per-file builder tests. diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/resource/config_map.rs similarity index 100% rename from rust/operator-binary/src/controller/build/config_map.rs rename to rust/operator-binary/src/controller/build/resource/config_map.rs diff --git a/rust/operator-binary/src/controller/build/discovery.rs b/rust/operator-binary/src/controller/build/resource/discovery.rs similarity index 69% rename from rust/operator-binary/src/controller/build/discovery.rs rename to rust/operator-binary/src/controller/build/resource/discovery.rs index 0e488ca6..ec14446d 100644 --- a/rust/operator-binary/src/controller/build/discovery.rs +++ b/rust/operator-binary/src/controller/build/resource/discovery.rs @@ -4,15 +4,20 @@ use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, k8s_openapi::api::core::v1::ConfigMap, + kube::Resource, + kvp::ObjectLabels, v2::{builder::meta::ownerreference_from_resource, config_file_writer::to_hadoop_xml}, }; use crate::{ controller::{ValidatedCluster, build::properties::ConfigFileName}, - crd::HbaseRole, - hbase_controller::build_recommended_labels, + crd::{APP_NAME, HbaseRole, OPERATOR_NAME}, }; +// The discovery `ConfigMap` is a cluster-wide object (not tied to a single role group), so it is +// labelled with the region-server role and a `discovery` placeholder role-group. +const DISCOVERY_ROLE_GROUP: &str = "discovery"; + type Result = std::result::Result; #[derive(Snafu, Debug)] @@ -46,7 +51,7 @@ pub fn build_discovery_config_map(cluster: &ValidatedCluster) -> Result Result( + owner: &'a R, + app_version: &'a str, + role: &'a str, + role_group: &'a str, +) -> ObjectLabels<'a, R> +where + R: Resource, +{ + ObjectLabels { + owner, + app_name: APP_NAME, + app_version, + operator_name: OPERATOR_NAME, + controller_name: crate::controller::HBASE_CONTROLLER_NAME, + role, + role_group, + } +} diff --git a/rust/operator-binary/src/controller/build/resource/mod.rs b/rust/operator-binary/src/controller/build/resource/mod.rs new file mode 100644 index 00000000..2aa3bec6 --- /dev/null +++ b/rust/operator-binary/src/controller/build/resource/mod.rs @@ -0,0 +1,8 @@ +//! Builders that turn a [`ValidatedCluster`](crate::controller::ValidatedCluster) into +//! Kubernetes resources. + +pub mod config_map; +pub mod discovery; +pub mod pdb; +pub mod service; +pub mod statefulset; diff --git a/rust/operator-binary/src/controller/build/resource/pdb.rs b/rust/operator-binary/src/controller/build/resource/pdb.rs new file mode 100644 index 00000000..e1a57531 --- /dev/null +++ b/rust/operator-binary/src/controller/build/resource/pdb.rs @@ -0,0 +1,54 @@ +//! Build the per-role `PodDisruptionBudget` for the HbaseCluster. + +use stackable_operator::{ + commons::pdb::PdbConfig, k8s_openapi::api::policy::v1::PodDisruptionBudget, + v2::builder::pdb::pod_disruption_budget_builder_with_role, +}; + +use crate::{ + controller::{ValidatedCluster, controller_name, operator_name, product_name}, + crd::HbaseRole, +}; + +/// Builds the [`PodDisruptionBudget`] for the given `role`, or `None` if PDBs are disabled. +pub fn build_pdb( + pdb: &PdbConfig, + cluster: &ValidatedCluster, + role: &HbaseRole, +) -> Option { + if !pdb.enabled { + return None; + } + let max_unavailable = pdb.max_unavailable.unwrap_or(match role { + HbaseRole::Master => max_unavailable_masters(), + HbaseRole::RegionServer => max_unavailable_region_servers(), + HbaseRole::RestServer => max_unavailable_rest_servers(), + }); + let pdb = pod_disruption_budget_builder_with_role( + cluster, + &product_name(), + &ValidatedCluster::role_name(role), + &operator_name(), + &controller_name(), + ) + .with_max_unavailable(max_unavailable) + .build(); + + Some(pdb) +} + +fn max_unavailable_masters() -> u16 { + 1 +} + +fn max_unavailable_region_servers() -> u16 { + 1 +} + +fn max_unavailable_rest_servers() -> u16 { + // RestServers are stateless, we only need to make sure we have two available, so we don't have a single point of failure. + // However, users probably deploy multiple rest servers for both - availability and performance. As there is the use-case + // of having multiple RestServers for availability reasons, we need to be restrictive and stick to our `maxUnavailable: 1` + // for `Multiple replicas to increase availability` rolegroups guideline. + 1 +} diff --git a/rust/operator-binary/src/controller/build/resource/service.rs b/rust/operator-binary/src/controller/build/resource/service.rs new file mode 100644 index 00000000..163ad261 --- /dev/null +++ b/rust/operator-binary/src/controller/build/resource/service.rs @@ -0,0 +1,206 @@ +//! Build the per-rolegroup `Service`s for the HbaseCluster. + +use stackable_operator::{ + k8s_openapi::api::core::v1::{Service, ServicePort, ServiceSpec}, + kvp::{Annotations, Labels}, +}; + +use crate::{ + controller::{RoleGroupName, ValidatedCluster}, + crd::{HbaseRole, v1alpha1}, +}; + +/// The rolegroup [`Service`] is a headless service that allows direct access to the instances of a +/// certain rolegroup. +/// +/// This is mostly useful for internal communication between peers, or for clients that perform +/// client-side load balancing. +pub fn build_rolegroup_service( + hbase: &v1alpha1::HbaseCluster, + cluster: &ValidatedCluster, + hbase_role: &HbaseRole, + role_group_name: &RoleGroupName, +) -> Service { + let ports = hbase_role + .ports(hbase) + .into_iter() + .map(|(name, value)| ServicePort { + name: Some(name), + port: i32::from(value), + protocol: Some("TCP".to_string()), + ..ServicePort::default() + }) + .collect(); + + Service { + metadata: cluster + .object_meta( + cluster + .resource_names(hbase_role, role_group_name) + .headless_service_name() + .to_string(), + hbase_role, + role_group_name, + ) + .build(), + spec: Some(ServiceSpec { + // Internal communication does not need to be exposed + type_: Some("ClusterIP".to_string()), + cluster_ip: Some("None".to_string()), + ports: Some(ports), + selector: Some( + cluster + .role_group_selector(hbase_role, role_group_name) + .into(), + ), + publish_not_ready_addresses: Some(true), + ..ServiceSpec::default() + }), + status: None, + } +} + +/// The rolegroup metrics [`Service`] is a service that exposes metrics and a prometheus scraping +/// label. +pub fn build_rolegroup_metrics_service( + hbase: &v1alpha1::HbaseCluster, + cluster: &ValidatedCluster, + hbase_role: &HbaseRole, + role_group_name: &RoleGroupName, +) -> Service { + let ports = vec![ServicePort { + name: Some(HbaseRole::metrics_port_name().to_owned()), + port: i32::from(hbase_role.metrics_port()), + protocol: Some("TCP".to_owned()), + ..ServicePort::default() + }]; + + Service { + metadata: cluster + .object_meta( + cluster + .resource_names(hbase_role, role_group_name) + .metrics_service_name() + .to_string(), + hbase_role, + role_group_name, + ) + .with_labels(prometheus_labels()) + .with_annotations(prometheus_annotations(hbase, hbase_role)) + .build(), + spec: Some(ServiceSpec { + // Internal communication does not need to be exposed + type_: Some("ClusterIP".to_owned()), + cluster_ip: Some("None".to_owned()), + ports: Some(ports), + selector: Some( + cluster + .role_group_selector(hbase_role, role_group_name) + .into(), + ), + publish_not_ready_addresses: Some(true), + ..ServiceSpec::default() + }), + status: None, + } +} + +/// Common labels for Prometheus. +fn prometheus_labels() -> Labels { + Labels::try_from([("prometheus.io/scrape", "true")]).expect("should be a valid label") +} + +/// Common annotations for Prometheus. +/// +/// These annotations can be used in a ServiceMonitor. +/// +/// see also +fn prometheus_annotations(hbase: &v1alpha1::HbaseCluster, hbase_role: &HbaseRole) -> Annotations { + Annotations::try_from([ + ("prometheus.io/path".to_owned(), "/prometheus".to_owned()), + ( + "prometheus.io/port".to_owned(), + hbase_role.metrics_port().to_string(), + ), + ( + "prometheus.io/scheme".to_owned(), + if hbase.has_https_enabled() { + "https".to_owned() + } else { + "http".to_owned() + }, + ), + ("prometheus.io/scrape".to_owned(), "true".to_owned()), + ]) + .expect("should be valid annotations") +} + +#[cfg(test)] +mod test { + use std::str::FromStr; + + use rstest::rstest; + use stackable_operator::v2::types::operator::RoleGroupName; + + use super::*; + use crate::controller::build::properties::test_support; + + #[rstest] + #[case("2.6.3", HbaseRole::Master, vec!["master", "ui-http"])] + #[case("2.6.3", HbaseRole::RegionServer, vec!["regionserver", "ui-http"])] + #[case("2.6.3", HbaseRole::RestServer, vec!["rest-http", "ui-http"])] + #[case("2.6.4", HbaseRole::Master, vec!["master", "ui-http"])] + #[case("2.6.4", HbaseRole::RegionServer, vec!["regionserver", "ui-http"])] + #[case("2.6.4", HbaseRole::RestServer, vec!["rest-http", "ui-http"])] + fn test_rolegroup_service_ports( + #[case] hbase_version: &str, + #[case] role: HbaseRole, + #[case] expected_ports: Vec<&str>, + ) { + let input = format!( + " + apiVersion: hbase.stackable.tech/v1alpha1 + kind: HbaseCluster + metadata: + name: hbase + uid: c2e98fc1-6b88-4d11-9381-52530e3f431e + spec: + image: + productVersion: {hbase_version} + clusterConfig: + hdfsConfigMapName: simple-hdfs + zookeeperConfigMapName: simple-znode + masters: + roleGroups: + default: + replicas: 1 + regionServers: + roleGroups: + default: + replicas: 1 + restServers: + roleGroups: + default: + replicas: 1 + " + ); + let hbase: v1alpha1::HbaseCluster = + serde_yaml::from_str(&input).expect("illegal test input"); + + let cluster = test_support::validated_cluster(); + let role_group_name = RoleGroupName::from_str("default").expect("valid role group name"); + let service = build_rolegroup_service(&hbase, &cluster, &role, &role_group_name); + + assert_eq!( + expected_ports, + service + .spec + .unwrap() + .ports + .unwrap() + .iter() + .map(|port| { port.clone().name.unwrap() }) + .collect::>() + ); + } +} diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs new file mode 100644 index 00000000..7dd535a0 --- /dev/null +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -0,0 +1,385 @@ +//! Build the per-rolegroup `StatefulSet` for the HbaseCluster. + +use std::collections::BTreeMap; + +use indoc::formatdoc; +use snafu::{OptionExt, ResultExt, Snafu}; +use stackable_operator::{ + builder::{ + self, + meta::ObjectMetaBuilder, + pod::{ + PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, + security::PodSecurityContextBuilder, + }, + }, + constants::RESTART_CONTROLLER_ENABLED_LABEL, + k8s_openapi::{ + DeepMerge, + api::{ + apps::v1::{StatefulSet, StatefulSetSpec}, + core::v1::{ + ConfigMapVolumeSource, ContainerPort, EnvVar, Probe, ServiceAccount, + TCPSocketAction, Volume, + }, + }, + apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, + }, + kube::ResourceExt, + product_logging::{ + self, + framework::LoggingError, + spec::{ + ConfigMapLogConfig, ContainerLogConfig, ContainerLogConfigChoice, + CustomContainerLogConfig, + }, + }, + v2::types::operator::RoleGroupName, +}; + +use crate::{ + controller::{ + ValidatedCluster, ValidatedRoleGroupConfig, + build::{ + graceful_shutdown::{self, add_graceful_shutdown_config}, + kerberos::{self, add_kerberos_pod_config}, + properties::logging::{MAX_HBASE_LOG_FILES_SIZE, STACKABLE_LOG_DIR}, + }, + }, + crd::{ + CONFIG_DIR_NAME, Container, HbaseRole, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, + merged_env, v1alpha1, + }, +}; + +pub static CONTAINERDEBUG_LOG_DIRECTORY: std::sync::LazyLock = + std::sync::LazyLock::new(|| format!("{STACKABLE_LOG_DIR}/containerdebug")); + +// These constants are hard coded in hbase-entrypoint.sh +// You need to change them there too. +const HDFS_DISCOVERY_TMP_DIR: &str = "/stackable/tmp/hdfs"; +const HBASE_CONFIG_TMP_DIR: &str = "/stackable/tmp/hbase"; +const HBASE_LOG_CONFIG_TMP_DIR: &str = "/stackable/tmp/log_config"; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("missing secret lifetime"))] + MissingSecretLifetime, + + #[snafu(display("failed to add kerberos config"))] + AddKerberosConfig { source: kerberos::Error }, + + #[snafu(display("failed to configure graceful shutdown"))] + GracefulShutdown { source: graceful_shutdown::Error }, + + #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] + VectorAggregatorConfigMapMissing, + + #[snafu(display("failed to configure logging"))] + ConfigureLogging { source: LoggingError }, + + #[snafu(display("failed to add needed volume"))] + AddVolume { source: builder::pod::Error }, + + #[snafu(display("failed to add needed volumeMount"))] + AddVolumeMount { + source: builder::pod::container::Error, + }, + + #[snafu(display("failed to build listener volume"))] + ListenerVolume { source: crate::crd::Error }, + + #[snafu(display("failed to build listener persistent volume claim"))] + ListenerPersistentVolumeClaim { source: crate::crd::Error }, +} + +type Result = std::result::Result; + +/// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. +/// +/// The [`Pod`](stackable_operator::k8s_openapi::api::core::v1::Pod)s are accessible through the +/// corresponding headless [`Service`](stackable_operator::k8s_openapi::api::core::v1::Service). +pub fn build_rolegroup_statefulset( + hbase: &v1alpha1::HbaseCluster, + cluster: &ValidatedCluster, + hbase_role: &HbaseRole, + role_group_name: &RoleGroupName, + validated_rg_config: &ValidatedRoleGroupConfig, + service_account: &ServiceAccount, +) -> Result { + let resolved_product_image = &cluster.image; + let merged_config = &validated_rg_config.config; + let resource_names = cluster.resource_names(hbase_role, role_group_name); + + let ports = hbase_role + .ports(hbase) + .into_iter() + .map(|(name, value)| ContainerPort { + name: Some(name), + container_port: i32::from(value), + protocol: Some("TCP".to_string()), + ..ContainerPort::default() + }) + .collect(); + + let probe_template = Probe { + tcp_socket: Some(TCPSocketAction { + port: IntOrString::String(hbase_role.data_port_name(hbase)), + ..TCPSocketAction::default() + }), + ..Probe::default() + }; + + let startup_probe = Probe { + failure_threshold: Some(120), + initial_delay_seconds: Some(4), + period_seconds: Some(5), + timeout_seconds: Some(3), + ..probe_template.clone() + }; + let liveness_probe = Probe { + failure_threshold: Some(3), + period_seconds: Some(10), + timeout_seconds: Some(3), + ..probe_template.clone() + }; + let readiness_probe = Probe { + failure_threshold: Some(1), + period_seconds: Some(10), + timeout_seconds: Some(2), + ..probe_template + }; + + let mut env_map: BTreeMap = BTreeMap::from([ + ("HBASE_CONF_DIR".to_string(), CONFIG_DIR_NAME.to_string()), + // required by phoenix (for cases where Kerberos is enabled): see https://issues.apache.org/jira/browse/PHOENIX-2369 + ("HADOOP_CONF_DIR".to_string(), CONFIG_DIR_NAME.to_string()), + ]); + for env_var in validated_rg_config.env_overrides.clone() { + env_map.insert(env_var.name, env_var.value.unwrap_or_default()); + } + let mut merged_env = merged_env(&env_map); + // This env var is set for all roles to avoid bash's "unbound variable" errors + merged_env.extend([ + EnvVar { + name: "REGION_MOVER_OPTS".to_string(), + value: Some(merged_config.region_mover_args()), + ..EnvVar::default() + }, + EnvVar { + name: "RUN_REGION_MOVER".to_string(), + value: Some(merged_config.run_region_mover().to_string()), + ..EnvVar::default() + }, + EnvVar { + name: "STACKABLE_LOG_DIR".to_string(), + value: Some(STACKABLE_LOG_DIR.to_string()), + ..EnvVar::default() + }, + ]); + + let role_name = hbase_role.cli_role_name(); + let mut hbase_container = ContainerBuilder::new("hbase").expect("ContainerBuilder not created"); + + hbase_container + .image_from_product_image(resolved_product_image) + .command(command()) + .args(vec![formatdoc! {" + {entrypoint} {role} {port} {port_name} {ui_port_name}", + entrypoint = "/stackable/hbase/bin/hbase-entrypoint.sh".to_string(), + role = role_name, + port = hbase_role.data_port(), + port_name = hbase_role.data_port_name(hbase), + ui_port_name = HbaseRole::ui_port_name(hbase.has_https_enabled()), + }]) + .add_env_vars(merged_env) + // Needed for the `containerdebug` process to log it's tracing information to. + .add_env_var( + "CONTAINERDEBUG_LOG_DIRECTORY", + &*CONTAINERDEBUG_LOG_DIRECTORY, + ) + .add_volume_mount("hbase-config", HBASE_CONFIG_TMP_DIR) + .context(AddVolumeMountSnafu)? + .add_volume_mount("hdfs-discovery", HDFS_DISCOVERY_TMP_DIR) + .context(AddVolumeMountSnafu)? + .add_volume_mount("log-config", HBASE_LOG_CONFIG_TMP_DIR) + .context(AddVolumeMountSnafu)? + .add_volume_mount("log", STACKABLE_LOG_DIR) + .context(AddVolumeMountSnafu)? + .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) + .context(AddVolumeMountSnafu)? + .add_container_ports(ports) + .resources(merged_config.resources().clone().into()) + .startup_probe(startup_probe) + .liveness_probe(liveness_probe) + .readiness_probe(readiness_probe); + + let mut pod_builder = PodBuilder::new(); + + let recommended_labels = cluster.recommended_labels(hbase_role, role_group_name); + + let pb_metadata = ObjectMetaBuilder::new() + .with_labels(recommended_labels.clone()) + .build(); + + pod_builder + .metadata(pb_metadata) + .image_pull_secrets_from_product_image(resolved_product_image) + .affinity(merged_config.affinity()) + .add_volume(Volume { + name: "hbase-config".to_string(), + config_map: Some(ConfigMapVolumeSource { + name: resource_names.role_group_config_map().to_string(), + ..Default::default() + }), + ..Default::default() + }) + .context(AddVolumeSnafu)? + .add_volume(Volume { + name: "hdfs-discovery".to_string(), + config_map: Some(ConfigMapVolumeSource { + name: hbase.spec.cluster_config.hdfs_config_map_name.clone(), + ..Default::default() + }), + ..Default::default() + }) + .context(AddVolumeSnafu)? + .add_empty_dir_volume( + "log", + Some(product_logging::framework::calculate_log_volume_size_limit( + &[MAX_HBASE_LOG_FILES_SIZE], + )), + ) + .context(AddVolumeSnafu)? + .service_account_name(service_account.name_any()) + .security_context(PodSecurityContextBuilder::new().fs_group(1000).build()); + + if let Some(ContainerLogConfig { + choice: + Some(ContainerLogConfigChoice::Custom(CustomContainerLogConfig { + custom: ConfigMapLogConfig { config_map }, + })), + }) = merged_config.logging().containers.get(&Container::Hbase) + { + pod_builder + .add_volume(Volume { + name: "log-config".to_string(), + config_map: Some(ConfigMapVolumeSource { + name: config_map.into(), + ..ConfigMapVolumeSource::default() + }), + ..Volume::default() + }) + .context(AddVolumeSnafu)?; + } else { + pod_builder + .add_volume(Volume { + name: "log-config".to_string(), + config_map: Some(ConfigMapVolumeSource { + name: resource_names.role_group_config_map().to_string(), + ..ConfigMapVolumeSource::default() + }), + ..Volume::default() + }) + .context(AddVolumeSnafu)?; + } + + add_graceful_shutdown_config(merged_config, &mut pod_builder).context(GracefulShutdownSnafu)?; + if cluster.has_kerberos_enabled() { + add_kerberos_pod_config( + hbase, + resource_names.metrics_service_name().as_ref(), + &mut hbase_container, + &mut pod_builder, + merged_config + .requested_secret_lifetime() + .context(MissingSecretLifetimeSnafu)?, + ) + .context(AddKerberosConfigSnafu)?; + } + pod_builder.add_container(hbase_container.build()); + + // Vector sidecar shall be the last container in the list + if merged_config.logging().enable_vector_agent { + if let Some(vector_aggregator_config_map_name) = + &hbase.spec.cluster_config.vector_aggregator_config_map_name + { + pod_builder.add_container( + product_logging::framework::vector_container( + resolved_product_image, + "hbase-config", + "log", + merged_config.logging().containers.get(&Container::Vector), + ResourceRequirementsBuilder::new() + .with_cpu_request("250m") + .with_cpu_limit("500m") + .with_memory_request("128Mi") + .with_memory_limit("128Mi") + .build(), + vector_aggregator_config_map_name, + ) + .context(ConfigureLoggingSnafu)?, + ); + } else { + VectorAggregatorConfigMapMissingSnafu.fail()?; + } + } + + let listener_pvc = hbase_role + .listener_pvc(merged_config, &recommended_labels) + .context(ListenerPersistentVolumeClaimSnafu)?; + + if let Some(listener_volume) = hbase_role + .listener_volume(merged_config, &recommended_labels) + .context(ListenerVolumeSnafu)? + { + pod_builder + .add_volume(listener_volume) + .context(AddVolumeSnafu)?; + }; + + let mut pod_template = pod_builder.build_template(); + + pod_template.merge_from(validated_rg_config.pod_overrides.clone()); + + let metadata = cluster + .object_meta( + resource_names.stateful_set_name().to_string(), + hbase_role, + role_group_name, + ) + .with_label(RESTART_CONTROLLER_ENABLED_LABEL.to_owned()) + .build(); + + let statefulset_match_labels = cluster.role_group_selector(hbase_role, role_group_name); + + let statefulset_spec = StatefulSetSpec { + pod_management_policy: Some("Parallel".to_string()), + replicas: Some(i32::from(validated_rg_config.replicas)), + selector: LabelSelector { + match_labels: Some(statefulset_match_labels.into()), + ..LabelSelector::default() + }, + service_name: Some(resource_names.headless_service_name().to_string()), + template: pod_template, + volume_claim_templates: listener_pvc, + ..StatefulSetSpec::default() + }; + + Ok(StatefulSet { + metadata, + spec: Some(statefulset_spec), + status: None, + }) +} + +/// Returns the container command. +fn command() -> Vec { + vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ] +} diff --git a/rust/operator-binary/src/controller/dereference.rs b/rust/operator-binary/src/controller/dereference.rs index 78615bd9..2cbc4886 100644 --- a/rust/operator-binary/src/controller/dereference.rs +++ b/rust/operator-binary/src/controller/dereference.rs @@ -1,16 +1,21 @@ use snafu::{ResultExt, Snafu}; use crate::{ - crd::v1alpha1, security::opa::HbaseOpaConfig, zookeeper::ZookeeperConnectionInformation, + controller::{build::opa::HbaseOpaConfig, zookeeper::ZookeeperConnectionInformation}, + crd::v1alpha1, }; #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("failed to retrieve zookeeper connection information"))] - RetrieveZookeeperConnectionInformation { source: crate::zookeeper::Error }, + RetrieveZookeeperConnectionInformation { + source: crate::controller::zookeeper::Error, + }, #[snafu(display("invalid OPA configuration"))] - InvalidOpaConfig { source: crate::security::opa::Error }, + InvalidOpaConfig { + source: crate::controller::build::opa::Error, + }, } /// External references resolved during the dereference step. diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index 0d620fda..e97c3a68 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -1,9 +1,12 @@ pub mod build; pub mod dereference; pub mod validate; +pub mod zookeeper; use std::{collections::BTreeMap, str::FromStr}; +use const_format::concatcp; +pub use stackable_operator::v2::types::operator::RoleGroupName; use stackable_operator::{ builder::meta::ObjectMetaBuilder, commons::product_image_selection::ResolvedProductImage, @@ -18,20 +21,20 @@ use stackable_operator::{ types::{ kubernetes::{NamespaceName, Uid}, operator::{ - ClusterName, ControllerName, OperatorName, ProductName, ProductVersion, - RoleGroupName, RoleName, + ClusterName, ControllerName, OperatorName, ProductName, ProductVersion, RoleName, }, }, }, }; use crate::{ + controller::{build::opa::HbaseOpaConfig, zookeeper::ZookeeperConnectionInformation}, crd::{APP_NAME, AnyServiceConfig, HbaseRole, OPERATOR_NAME, v1alpha1}, - hbase_controller::HBASE_CONTROLLER_NAME, - security::opa::HbaseOpaConfig, - zookeeper::ZookeeperConnectionInformation, }; +pub const HBASE_CONTROLLER_NAME: &str = "hbasecluster"; +pub const FULL_HBASE_CONTROLLER_NAME: &str = concatcp!(HBASE_CONTROLLER_NAME, '.', OPERATOR_NAME); + /// The product name (`hbase`) as a type-safe label value. pub(crate) fn product_name() -> ProductName { ProductName::from_str(APP_NAME).expect("'hbase' is a valid product name") diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 26bce2cb..b8ae632f 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -20,15 +20,15 @@ use stackable_operator::{ use strum::IntoEnumIterator; use crate::{ + controller::build::kerberos::{ + self, kerberos_config_properties, kerberos_discovery_config_properties, + kerberos_ssl_client_settings, kerberos_ssl_server_settings, + }, controller::{ ValidatedCluster, ValidatedClusterConfig, ValidatedRoleConfig, ValidatedRoleGroupConfig, build::jvm::construct_role_specific_non_heap_jvm_args, dereference::DereferencedObjects, }, crd::{AnyServiceConfig, HbaseConfigFragment, HbaseRole, RegionServerConfigFragment, v1alpha1}, - kerberos::{ - self, kerberos_config_properties, kerberos_discovery_config_properties, - kerberos_ssl_client_settings, kerberos_ssl_server_settings, - }, }; const CONTAINER_IMAGE_BASE_NAME: &str = "hbase"; diff --git a/rust/operator-binary/src/zookeeper.rs b/rust/operator-binary/src/controller/zookeeper.rs similarity index 100% rename from rust/operator-binary/src/zookeeper.rs rename to rust/operator-binary/src/controller/zookeeper.rs diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 92060cfb..c6590d21 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -1,86 +1,46 @@ -//! Ensures that `Pod`s are configured and running for each [`v1alpha1::HbaseCluster`] +//! Ensures that `Pod`s are configured and running for each [`v1alpha1::HbaseCluster`]. +//! +//! This is the controller driver: it runs the `dereference -> validate -> build -> apply` +//! pipeline. The validated cluster type and the resource builders live under the +//! [`crate::controller`] module tree; this file is kept next to `main.rs` for consistency with +//! the other Stackable operators. -use std::{collections::BTreeMap, sync::Arc}; +use std::sync::Arc; -use const_format::concatcp; -use indoc::formatdoc; -use snafu::{OptionExt, ResultExt, Snafu}; +use snafu::{ResultExt, Snafu}; use stackable_operator::{ - builder::{ - self, - meta::ObjectMetaBuilder, - pod::{ - PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, - security::PodSecurityContextBuilder, - }, - }, cli::OperatorEnvironmentOptions, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, commons::rbac::build_rbac_resources, - constants::RESTART_CONTROLLER_ENABLED_LABEL, - k8s_openapi::{ - DeepMerge, - api::{ - apps::v1::{StatefulSet, StatefulSetSpec}, - core::v1::{ - ConfigMapVolumeSource, ContainerPort, EnvVar, Probe, Service, ServiceAccount, - ServicePort, ServiceSpec, TCPSocketAction, Volume, - }, - }, - apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, - }, kube::{ - Resource, ResourceExt, + Resource, core::{DeserializeGuard, error_boundary}, runtime::controller::Action, }, - kvp::{Annotations, Label, LabelError, ObjectLabels}, + kvp::LabelError, logging::controller::ReconcilerError, - product_logging::{ - self, - framework::LoggingError, - spec::{ - ConfigMapLogConfig, ContainerLogConfig, ContainerLogConfigChoice, - CustomContainerLogConfig, - }, - }, shared::time::Duration, status::condition::{ compute_conditions, operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, }, - v2::types::operator::RoleGroupName, }; use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ controller::{ - ValidatedCluster, ValidatedRoleGroupConfig, - build::{ + HBASE_CONTROLLER_NAME, RoleGroupName, + build::resource::{ + config_map::build_rolegroup_config_map, discovery::build_discovery_config_map, - properties::logging::{MAX_HBASE_LOG_FILES_SIZE, STACKABLE_LOG_DIR}, + pdb::build_pdb, + service::{build_rolegroup_metrics_service, build_rolegroup_service}, + statefulset::build_rolegroup_statefulset, }, }, - crd::{ - APP_NAME, CONFIG_DIR_NAME, Container, HbaseClusterStatus, HbaseRole, LISTENER_VOLUME_DIR, - LISTENER_VOLUME_NAME, OPERATOR_NAME, merged_env, v1alpha1, - }, - kerberos::{self, add_kerberos_pod_config}, - operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, + crd::{APP_NAME, HbaseClusterStatus, OPERATOR_NAME, v1alpha1}, }; -pub const HBASE_CONTROLLER_NAME: &str = "hbasecluster"; -pub const FULL_HBASE_CONTROLLER_NAME: &str = concatcp!(HBASE_CONTROLLER_NAME, '.', OPERATOR_NAME); - -pub static CONTAINERDEBUG_LOG_DIRECTORY: std::sync::LazyLock = - std::sync::LazyLock::new(|| format!("{STACKABLE_LOG_DIR}/containerdebug")); - -// These constants are hard coded in hbase-entrypoint.sh -// You need to change them there too. -const HDFS_DISCOVERY_TMP_DIR: &str = "/stackable/tmp/hdfs"; -const HBASE_CONFIG_TMP_DIR: &str = "/stackable/tmp/hbase"; -const HBASE_LOG_CONFIG_TMP_DIR: &str = "/stackable/tmp/log_config"; - pub struct Ctx { pub client: stackable_operator::client::Client, pub operator_environment: OperatorEnvironmentOptions, @@ -89,9 +49,6 @@ pub struct Ctx { #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] pub enum Error { - #[snafu(display("missing secret lifetime"))] - MissingSecretLifetime, - #[snafu(display("failed to create cluster resources"))] CreateClusterResources { source: stackable_operator::cluster_resources::Error, @@ -102,25 +59,33 @@ pub enum Error { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("failed to apply Service for role group {role_group}"))] - ApplyRoleGroupService { + #[snafu(display("failed to build RBAC resources"))] + BuildRbacResources { + source: stackable_operator::commons::rbac::Error, + }, + + #[snafu(display("failed to build label"))] + BuildLabel { source: LabelError }, + + #[snafu(display("failed to patch service account"))] + ApplyServiceAccount { source: stackable_operator::cluster_resources::Error, - role_group: RoleGroupName, }, - #[snafu(display("failed to apply discovery configmap"))] - ApplyDiscoveryConfigMap { + #[snafu(display("failed to patch role binding"))] + ApplyRoleBinding { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("failed to build discovery configmap"))] - BuildDiscoveryConfigMap { - source: crate::controller::build::discovery::Error, + #[snafu(display("failed to apply Service for role group {role_group}"))] + ApplyRoleGroupService { + source: stackable_operator::cluster_resources::Error, + role_group: RoleGroupName, }, #[snafu(display("failed to build rolegroup ConfigMap"))] BuildRolegroupConfigMap { - source: crate::controller::build::config_map::Error, + source: crate::controller::build::resource::config_map::Error, }, #[snafu(display("failed to apply ConfigMap for role group {role_group}"))] @@ -129,78 +94,43 @@ pub enum Error { role_group: RoleGroupName, }, + #[snafu(display("failed to build StatefulSet for role group {role_group}"))] + BuildRoleGroupStatefulSet { + source: crate::controller::build::resource::statefulset::Error, + role_group: RoleGroupName, + }, + #[snafu(display("failed to apply StatefulSet for role group {role_group}"))] ApplyRoleGroupStatefulSet { source: stackable_operator::cluster_resources::Error, role_group: RoleGroupName, }, - #[snafu(display("failed to patch service account"))] - ApplyServiceAccount { + #[snafu(display("failed to apply PodDisruptionBudget"))] + ApplyPdb { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("failed to patch role binding"))] - ApplyRoleBinding { - source: stackable_operator::cluster_resources::Error, + #[snafu(display("failed to build discovery configmap"))] + BuildDiscoveryConfigMap { + source: crate::controller::build::resource::discovery::Error, }, - #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] - VectorAggregatorConfigMapMissing, - - #[snafu(display("failed to add kerberos config"))] - AddKerberosConfig { source: kerberos::Error }, + #[snafu(display("failed to apply discovery configmap"))] + ApplyDiscoveryConfigMap { + source: stackable_operator::cluster_resources::Error, + }, #[snafu(display("failed to update status"))] ApplyStatus { source: stackable_operator::client::Error, }, - #[snafu(display("failed to build RBAC resources"))] - BuildRbacResources { - source: stackable_operator::commons::rbac::Error, - }, - - #[snafu(display("failed to create PodDisruptionBudget"))] - FailedToCreatePdb { - source: crate::operations::pdb::Error, - }, - - #[snafu(display("failed to configure graceful shutdown"))] - GracefulShutdown { - source: crate::operations::graceful_shutdown::Error, - }, - - #[snafu(display("failed to build label"))] - BuildLabel { source: LabelError }, - - #[snafu(display("failed to configure logging"))] - ConfigureLogging { source: LoggingError }, - - #[snafu(display("failed to add needed volume"))] - AddVolume { source: builder::pod::Error }, - - #[snafu(display("failed to add needed volumeMount"))] - AddVolumeMount { - source: builder::pod::container::Error, - }, - #[snafu(display("HBaseCluster object is invalid"))] InvalidHBaseCluster { source: error_boundary::InvalidObject, }, - #[snafu(display("failed to build Labels"))] - LabelBuild { - source: stackable_operator::kvp::LabelError, - }, - - #[snafu(display("failed to build listener volume"))] - ListenerVolume { source: crate::crd::Error }, - - #[snafu(display("failed to build listener persistent volume claim"))] - ListenerPersistentVolumeClaim { source: crate::crd::Error }, - #[snafu(display("failed to dereference cluster resources"))] Dereference { source: crate::controller::dereference::Error, @@ -278,21 +208,18 @@ pub async fn reconcile_hbase( for (hbase_role, role_group_configs) in &validated_cluster.role_group_configs { for (role_group_name, validated_rg_config) in role_group_configs { let rg_service = - build_rolegroup_service(hbase, &validated_cluster, hbase_role, role_group_name)?; + build_rolegroup_service(hbase, &validated_cluster, hbase_role, role_group_name); let rg_metrics_service = build_rolegroup_metrics_service( hbase, &validated_cluster, hbase_role, role_group_name, - )?; + ); - let rg_configmap = crate::controller::build::config_map::build_rolegroup_config_map( - &validated_cluster, - hbase_role, - role_group_name, - ) - .context(BuildRolegroupConfigMapSnafu)?; + let rg_configmap = + build_rolegroup_config_map(&validated_cluster, hbase_role, role_group_name) + .context(BuildRolegroupConfigMapSnafu)?; let rg_statefulset = build_rolegroup_statefulset( hbase, &validated_cluster, @@ -300,7 +227,10 @@ pub async fn reconcile_hbase( role_group_name, validated_rg_config, &rbac_sa, - )?; + ) + .with_context(|_| BuildRoleGroupStatefulSetSnafu { + role_group: role_group_name.clone(), + })?; cluster_resources .add(client, rg_service) .await @@ -333,16 +263,13 @@ pub async fn reconcile_hbase( ); } - if let Some(role_config) = validated_cluster.role_configs.get(hbase_role) { - add_pdbs( - &role_config.pdb, - hbase, - hbase_role, - client, - &mut cluster_resources, - ) - .await - .context(FailedToCreatePdbSnafu)?; + if let Some(role_config) = validated_cluster.role_configs.get(hbase_role) + && let Some(pdb) = build_pdb(&role_config.pdb, &validated_cluster, hbase_role) + { + cluster_resources + .add(client, pdb) + .await + .context(ApplyPdbSnafu)?; } } @@ -374,411 +301,6 @@ pub async fn reconcile_hbase( Ok(Action::await_change()) } -/// The rolegroup [`Service`] is a headless service that allows direct access to the instances of a certain rolegroup -/// -/// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. -fn build_rolegroup_service( - hbase: &v1alpha1::HbaseCluster, - cluster: &ValidatedCluster, - hbase_role: &HbaseRole, - role_group_name: &RoleGroupName, -) -> Result { - let ports = hbase_role - .ports(hbase) - .into_iter() - .map(|(name, value)| ServicePort { - name: Some(name), - port: i32::from(value), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }) - .collect(); - - let metadata = cluster - .object_meta( - cluster - .resource_names(hbase_role, role_group_name) - .headless_service_name() - .to_string(), - hbase_role, - role_group_name, - ) - .build(); - - let service_selector = cluster.role_group_selector(hbase_role, role_group_name); - - let service_spec = ServiceSpec { - // Internal communication does not need to be exposed - type_: Some("ClusterIP".to_string()), - cluster_ip: Some("None".to_string()), - ports: Some(ports), - selector: Some(service_selector.into()), - publish_not_ready_addresses: Some(true), - ..ServiceSpec::default() - }; - - Ok(Service { - metadata, - spec: Some(service_spec), - status: None, - }) -} - -/// The rolegroup metrics [`Service`] is a service that exposes metrics and a prometheus scraping label. -fn build_rolegroup_metrics_service( - hbase: &v1alpha1::HbaseCluster, - cluster: &ValidatedCluster, - hbase_role: &HbaseRole, - role_group_name: &RoleGroupName, -) -> Result { - let ports = vec![ServicePort { - name: Some(HbaseRole::metrics_port_name().to_owned()), - port: i32::from(hbase_role.metrics_port()), - protocol: Some("TCP".to_owned()), - ..ServicePort::default() - }]; - - let service_selector = cluster.role_group_selector(hbase_role, role_group_name); - - Ok(Service { - metadata: cluster - .object_meta( - cluster - .resource_names(hbase_role, role_group_name) - .metrics_service_name() - .to_string(), - hbase_role, - role_group_name, - ) - .with_label(Label::try_from(("prometheus.io/scrape", "true")).context(LabelBuildSnafu)?) - .with_annotations(prometheus_annotations(hbase, hbase_role)) - .build(), - spec: Some(ServiceSpec { - // Internal communication does not need to be exposed - type_: Some("ClusterIP".to_owned()), - cluster_ip: Some("None".to_owned()), - ports: Some(ports), - selector: Some(service_selector.into()), - publish_not_ready_addresses: Some(true), - ..ServiceSpec::default() - }), - status: None, - }) -} - -/// Common annotations for Prometheus -/// -/// These annotations can be used in a ServiceMonitor. -/// -/// see also -fn prometheus_annotations(hbase: &v1alpha1::HbaseCluster, hbase_role: &HbaseRole) -> Annotations { - Annotations::try_from([ - ("prometheus.io/path".to_owned(), "/prometheus".to_owned()), - ( - "prometheus.io/port".to_owned(), - hbase_role.metrics_port().to_string(), - ), - ( - "prometheus.io/scheme".to_owned(), - if hbase.has_https_enabled() { - "https".to_owned() - } else { - "http".to_owned() - }, - ), - ("prometheus.io/scrape".to_owned(), "true".to_owned()), - ]) - .expect("should be valid annotations") -} - -/// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. -/// -/// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the corresponding [`Service`] (from [`build_rolegroup_service`]). -fn build_rolegroup_statefulset( - hbase: &v1alpha1::HbaseCluster, - cluster: &ValidatedCluster, - hbase_role: &HbaseRole, - role_group_name: &RoleGroupName, - validated_rg_config: &ValidatedRoleGroupConfig, - service_account: &ServiceAccount, -) -> Result { - let resolved_product_image = &cluster.image; - let merged_config = &validated_rg_config.config; - let resource_names = cluster.resource_names(hbase_role, role_group_name); - - let ports = hbase_role - .ports(hbase) - .into_iter() - .map(|(name, value)| ContainerPort { - name: Some(name), - container_port: i32::from(value), - protocol: Some("TCP".to_string()), - ..ContainerPort::default() - }) - .collect(); - - let probe_template = Probe { - tcp_socket: Some(TCPSocketAction { - port: IntOrString::String(hbase_role.data_port_name(hbase)), - ..TCPSocketAction::default() - }), - ..Probe::default() - }; - - let startup_probe = Probe { - failure_threshold: Some(120), - initial_delay_seconds: Some(4), - period_seconds: Some(5), - timeout_seconds: Some(3), - ..probe_template.clone() - }; - let liveness_probe = Probe { - failure_threshold: Some(3), - period_seconds: Some(10), - timeout_seconds: Some(3), - ..probe_template.clone() - }; - let readiness_probe = Probe { - failure_threshold: Some(1), - period_seconds: Some(10), - timeout_seconds: Some(2), - ..probe_template - }; - - let mut env_map: BTreeMap = BTreeMap::from([ - ("HBASE_CONF_DIR".to_string(), CONFIG_DIR_NAME.to_string()), - // required by phoenix (for cases where Kerberos is enabled): see https://issues.apache.org/jira/browse/PHOENIX-2369 - ("HADOOP_CONF_DIR".to_string(), CONFIG_DIR_NAME.to_string()), - ]); - for env_var in validated_rg_config.env_overrides.clone() { - env_map.insert(env_var.name, env_var.value.unwrap_or_default()); - } - let mut merged_env = merged_env(&env_map); - // This env var is set for all roles to avoid bash's "unbound variable" errors - merged_env.extend([ - EnvVar { - name: "REGION_MOVER_OPTS".to_string(), - value: Some(merged_config.region_mover_args()), - ..EnvVar::default() - }, - EnvVar { - name: "RUN_REGION_MOVER".to_string(), - value: Some(merged_config.run_region_mover().to_string()), - ..EnvVar::default() - }, - EnvVar { - name: "STACKABLE_LOG_DIR".to_string(), - value: Some(STACKABLE_LOG_DIR.to_string()), - ..EnvVar::default() - }, - ]); - - let role_name = hbase_role.cli_role_name(); - let mut hbase_container = ContainerBuilder::new("hbase").expect("ContainerBuilder not created"); - - hbase_container - .image_from_product_image(resolved_product_image) - .command(command()) - .args(vec![formatdoc! {" - {entrypoint} {role} {port} {port_name} {ui_port_name}", - entrypoint = "/stackable/hbase/bin/hbase-entrypoint.sh".to_string(), - role = role_name, - port = hbase_role.data_port(), - port_name = hbase_role.data_port_name(hbase), - ui_port_name = HbaseRole::ui_port_name(hbase.has_https_enabled()), - }]) - .add_env_vars(merged_env) - // Needed for the `containerdebug` process to log it's tracing information to. - .add_env_var( - "CONTAINERDEBUG_LOG_DIRECTORY", - &*CONTAINERDEBUG_LOG_DIRECTORY, - ) - .add_volume_mount("hbase-config", HBASE_CONFIG_TMP_DIR) - .context(AddVolumeMountSnafu)? - .add_volume_mount("hdfs-discovery", HDFS_DISCOVERY_TMP_DIR) - .context(AddVolumeMountSnafu)? - .add_volume_mount("log-config", HBASE_LOG_CONFIG_TMP_DIR) - .context(AddVolumeMountSnafu)? - .add_volume_mount("log", STACKABLE_LOG_DIR) - .context(AddVolumeMountSnafu)? - .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) - .context(AddVolumeMountSnafu)? - .add_container_ports(ports) - .resources(merged_config.resources().clone().into()) - .startup_probe(startup_probe) - .liveness_probe(liveness_probe) - .readiness_probe(readiness_probe); - - let mut pod_builder = PodBuilder::new(); - - let recommended_labels = cluster.recommended_labels(hbase_role, role_group_name); - - let pb_metadata = ObjectMetaBuilder::new() - .with_labels(recommended_labels.clone()) - .build(); - - pod_builder - .metadata(pb_metadata) - .image_pull_secrets_from_product_image(resolved_product_image) - .affinity(merged_config.affinity()) - .add_volume(stackable_operator::k8s_openapi::api::core::v1::Volume { - name: "hbase-config".to_string(), - config_map: Some(ConfigMapVolumeSource { - name: resource_names.role_group_config_map().to_string(), - ..Default::default() - }), - ..Default::default() - }) - .context(AddVolumeSnafu)? - .add_volume(stackable_operator::k8s_openapi::api::core::v1::Volume { - name: "hdfs-discovery".to_string(), - config_map: Some(ConfigMapVolumeSource { - name: hbase.spec.cluster_config.hdfs_config_map_name.clone(), - ..Default::default() - }), - ..Default::default() - }) - .context(AddVolumeSnafu)? - .add_empty_dir_volume( - "log", - Some(product_logging::framework::calculate_log_volume_size_limit( - &[MAX_HBASE_LOG_FILES_SIZE], - )), - ) - .context(AddVolumeSnafu)? - .service_account_name(service_account.name_any()) - .security_context(PodSecurityContextBuilder::new().fs_group(1000).build()); - - if let Some(ContainerLogConfig { - choice: - Some(ContainerLogConfigChoice::Custom(CustomContainerLogConfig { - custom: ConfigMapLogConfig { config_map }, - })), - }) = merged_config.logging().containers.get(&Container::Hbase) - { - pod_builder - .add_volume(Volume { - name: "log-config".to_string(), - config_map: Some(ConfigMapVolumeSource { - name: config_map.into(), - ..ConfigMapVolumeSource::default() - }), - ..Volume::default() - }) - .context(AddVolumeSnafu)?; - } else { - pod_builder - .add_volume(Volume { - name: "log-config".to_string(), - config_map: Some(ConfigMapVolumeSource { - name: resource_names.role_group_config_map().to_string(), - ..ConfigMapVolumeSource::default() - }), - ..Volume::default() - }) - .context(AddVolumeSnafu)?; - } - - add_graceful_shutdown_config(merged_config, &mut pod_builder).context(GracefulShutdownSnafu)?; - if cluster.has_kerberos_enabled() { - add_kerberos_pod_config( - hbase, - resource_names.metrics_service_name().as_ref(), - &mut hbase_container, - &mut pod_builder, - merged_config - .requested_secret_lifetime() - .context(MissingSecretLifetimeSnafu)?, - ) - .context(AddKerberosConfigSnafu)?; - } - pod_builder.add_container(hbase_container.build()); - - // Vector sidecar shall be the last container in the list - if merged_config.logging().enable_vector_agent { - if let Some(vector_aggregator_config_map_name) = - &hbase.spec.cluster_config.vector_aggregator_config_map_name - { - pod_builder.add_container( - product_logging::framework::vector_container( - resolved_product_image, - "hbase-config", - "log", - merged_config.logging().containers.get(&Container::Vector), - ResourceRequirementsBuilder::new() - .with_cpu_request("250m") - .with_cpu_limit("500m") - .with_memory_request("128Mi") - .with_memory_limit("128Mi") - .build(), - vector_aggregator_config_map_name, - ) - .context(ConfigureLoggingSnafu)?, - ); - } else { - VectorAggregatorConfigMapMissingSnafu.fail()?; - } - } - - let listener_pvc = hbase_role - .listener_pvc(merged_config, &recommended_labels) - .context(ListenerPersistentVolumeClaimSnafu)?; - - if let Some(listener_volume) = hbase_role - .listener_volume(merged_config, &recommended_labels) - .context(ListenerVolumeSnafu)? - { - pod_builder - .add_volume(listener_volume) - .context(AddVolumeSnafu)?; - }; - - let mut pod_template = pod_builder.build_template(); - - pod_template.merge_from(validated_rg_config.pod_overrides.clone()); - - let metadata = cluster - .object_meta( - resource_names.stateful_set_name().to_string(), - hbase_role, - role_group_name, - ) - .with_label(RESTART_CONTROLLER_ENABLED_LABEL.to_owned()) - .build(); - - let statefulset_match_labels = cluster.role_group_selector(hbase_role, role_group_name); - - let statefulset_spec = StatefulSetSpec { - pod_management_policy: Some("Parallel".to_string()), - replicas: Some(i32::from(validated_rg_config.replicas)), - selector: LabelSelector { - match_labels: Some(statefulset_match_labels.into()), - ..LabelSelector::default() - }, - service_name: Some(resource_names.headless_service_name().to_string()), - template: pod_template, - volume_claim_templates: listener_pvc, - ..StatefulSetSpec::default() - }; - - Ok(StatefulSet { - metadata, - spec: Some(statefulset_spec), - status: None, - }) -} - -/// Returns the container command. -fn command() -> Vec { - vec![ - "/bin/bash".to_string(), - "-x".to_string(), - "-euo".to_string(), - "pipefail".to_string(), - "-c".to_string(), - ] -} - pub fn error_policy( _obj: Arc>, error: &Error, @@ -790,91 +312,3 @@ pub fn error_policy( _ => Action::requeue(*Duration::from_secs(5)), } } - -pub fn build_recommended_labels<'a, R>( - owner: &'a R, - app_version: &'a str, - role: &'a str, - role_group: &'a str, -) -> ObjectLabels<'a, R> { - ObjectLabels { - owner, - app_name: APP_NAME, - app_version, - operator_name: OPERATOR_NAME, - controller_name: HBASE_CONTROLLER_NAME, - role, - role_group, - } -} - -#[cfg(test)] -mod test { - use std::str::FromStr; - - use rstest::rstest; - use stackable_operator::v2::types::operator::RoleGroupName; - - use super::*; - use crate::controller::build::properties::test_support; - - #[rstest] - #[case("2.6.3", HbaseRole::Master, vec!["master", "ui-http"])] - #[case("2.6.3", HbaseRole::RegionServer, vec!["regionserver", "ui-http"])] - #[case("2.6.3", HbaseRole::RestServer, vec!["rest-http", "ui-http"])] - #[case("2.6.4", HbaseRole::Master, vec!["master", "ui-http"])] - #[case("2.6.4", HbaseRole::RegionServer, vec!["regionserver", "ui-http"])] - #[case("2.6.4", HbaseRole::RestServer, vec!["rest-http", "ui-http"])] - fn test_rolegroup_service_ports( - #[case] hbase_version: &str, - #[case] role: HbaseRole, - #[case] expected_ports: Vec<&str>, - ) { - let input = format!( - " - apiVersion: hbase.stackable.tech/v1alpha1 - kind: HbaseCluster - metadata: - name: hbase - uid: c2e98fc1-6b88-4d11-9381-52530e3f431e - spec: - image: - productVersion: {hbase_version} - clusterConfig: - hdfsConfigMapName: simple-hdfs - zookeeperConfigMapName: simple-znode - masters: - roleGroups: - default: - replicas: 1 - regionServers: - roleGroups: - default: - replicas: 1 - restServers: - roleGroups: - default: - replicas: 1 - " - ); - let hbase: v1alpha1::HbaseCluster = - serde_yaml::from_str(&input).expect("illegal test input"); - - let cluster = test_support::validated_cluster(); - let role_group_name = RoleGroupName::from_str("default").expect("valid role group name"); - let service = build_rolegroup_service(&hbase, &cluster, &role, &role_group_name) - .expect("failed to build service"); - - assert_eq!( - expected_ports, - service - .spec - .unwrap() - .ports - .unwrap() - .iter() - .map(|port| { port.clone().name.unwrap() }) - .collect::>() - ); - } -} diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 7dd470c4..1784da58 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -6,8 +6,8 @@ use std::sync::Arc; use anyhow::anyhow; use clap::Parser; +use controller::FULL_HBASE_CONTROLLER_NAME; use futures::{FutureExt, StreamExt, TryFutureExt}; -use hbase_controller::FULL_HBASE_CONTROLLER_NAME; use stackable_operator::{ YamlSchema, cli::{Command, RunArguments}, @@ -40,11 +40,7 @@ use crate::{ mod controller; mod crd; mod hbase_controller; -mod kerberos; -mod operations; -mod security; mod webhooks; -mod zookeeper; mod built_info { include!(concat!(env!("OUT_DIR"), "/built.rs")); diff --git a/rust/operator-binary/src/operations/mod.rs b/rust/operator-binary/src/operations/mod.rs deleted file mode 100644 index 92ca2ec7..00000000 --- a/rust/operator-binary/src/operations/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod graceful_shutdown; -pub mod pdb; diff --git a/rust/operator-binary/src/operations/pdb.rs b/rust/operator-binary/src/operations/pdb.rs deleted file mode 100644 index 76b06b0f..00000000 --- a/rust/operator-binary/src/operations/pdb.rs +++ /dev/null @@ -1,77 +0,0 @@ -use snafu::{ResultExt, Snafu}; -use stackable_operator::{ - builder::pdb::PodDisruptionBudgetBuilder, client::Client, cluster_resources::ClusterResources, - commons::pdb::PdbConfig, kube::ResourceExt, -}; - -use crate::{ - crd::{APP_NAME, HbaseRole, OPERATOR_NAME, v1alpha1}, - hbase_controller::HBASE_CONTROLLER_NAME, -}; - -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("Cannot create PodDisruptionBudget for role [{role}]"))] - CreatePdb { - source: stackable_operator::builder::pdb::Error, - role: String, - }, - - #[snafu(display("Cannot apply PodDisruptionBudget [{name}]"))] - ApplyPdb { - source: stackable_operator::cluster_resources::Error, - name: String, - }, -} - -pub async fn add_pdbs( - pdb: &PdbConfig, - hbase: &v1alpha1::HbaseCluster, - role: &HbaseRole, - client: &Client, - cluster_resources: &mut ClusterResources<'_>, -) -> Result<(), Error> { - if !pdb.enabled { - return Ok(()); - } - let max_unavailable = pdb.max_unavailable.unwrap_or(match role { - HbaseRole::Master => max_unavailable_masters(), - HbaseRole::RegionServer => max_unavailable_region_servers(), - HbaseRole::RestServer => max_unavailable_rest_servers(), - }); - let pdb = PodDisruptionBudgetBuilder::new_with_role( - hbase, - APP_NAME, - &role.to_string(), - OPERATOR_NAME, - HBASE_CONTROLLER_NAME, - ) - .with_context(|_| CreatePdbSnafu { - role: role.to_string(), - })? - .with_max_unavailable(max_unavailable) - .build(); - let pdb_name = pdb.name_any(); - cluster_resources - .add(client, pdb) - .await - .with_context(|_| ApplyPdbSnafu { name: pdb_name })?; - - Ok(()) -} - -fn max_unavailable_masters() -> u16 { - 1 -} - -fn max_unavailable_region_servers() -> u16 { - 1 -} - -fn max_unavailable_rest_servers() -> u16 { - // RestServers are stateless, we only need to make sure we have two available, so we don't have a single point of failure. - // However, users probably deploy multiple rest servers for both - availability and performance. As there is the use-case - // of having multiple RestServers for availability reasons, we need to be restrictive and stick to our `maxUnavailable: 1` - // for `Multiple replicas to increase availability` rolegroups guideline. - 1 -} diff --git a/rust/operator-binary/src/security/mod.rs b/rust/operator-binary/src/security/mod.rs deleted file mode 100644 index 932ba472..00000000 --- a/rust/operator-binary/src/security/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod opa; From 7ac68206fa18a62ed0fdfe93b5f358d23a6c9d80 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 13:19:28 +0200 Subject: [PATCH 39/56] refactor: Add ValidatedHbaseConfig, adapt to v2 logging & add vector tests --- .../controller/build/properties/logging.rs | 85 ------- .../build/properties/logging/mod.rs | 68 ++++++ .../build/properties/logging/test-vector.sh | 11 + .../build/properties/logging/vector-test.yaml | 162 ++++++++++++ .../build/properties/logging/vector.yaml | 231 ++++++++++++++++++ .../src/controller/build/properties/mod.rs | 4 +- .../controller/build/resource/config_map.rs | 13 +- .../controller/build/resource/statefulset.rs | 124 ++++------ rust/operator-binary/src/controller/mod.rs | 18 +- .../src/controller/validate.rs | 108 +++++++- 10 files changed, 639 insertions(+), 185 deletions(-) delete mode 100644 rust/operator-binary/src/controller/build/properties/logging.rs create mode 100644 rust/operator-binary/src/controller/build/properties/logging/mod.rs create mode 100755 rust/operator-binary/src/controller/build/properties/logging/test-vector.sh create mode 100644 rust/operator-binary/src/controller/build/properties/logging/vector-test.yaml create mode 100644 rust/operator-binary/src/controller/build/properties/logging/vector.yaml diff --git a/rust/operator-binary/src/controller/build/properties/logging.rs b/rust/operator-binary/src/controller/build/properties/logging.rs deleted file mode 100644 index c62bfe0e..00000000 --- a/rust/operator-binary/src/controller/build/properties/logging.rs +++ /dev/null @@ -1,85 +0,0 @@ -use stackable_operator::{ - kube::runtime::reflector::ObjectRef, - memory::{BinaryMultiple, MemoryQuantity}, - product_logging::{ - self, - spec::{ - AutomaticContainerLogConfig, ContainerLogConfig, ContainerLogConfigChoice, Logging, - }, - }, - role_utils::RoleGroupRef, - v2::types::operator::RoleGroupName, -}; - -use crate::{ - controller::ValidatedCluster, - crd::{Container, HbaseRole}, -}; - -pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; -pub const MAX_HBASE_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { - value: 10.0, - unit: BinaryMultiple::Mebi, -}; - -const CONSOLE_CONVERSION_PATTERN: &str = "%d{ISO8601} %-5p [%t] %c{2}: %.1000m%n"; -const HBASE_LOG4J2_FILE: &str = "hbase.log4j2.xml"; - -/// Renders `log4j2.properties` for the HBase container. -/// -/// Returns `None` when the HBase container does not use the operator's automatic logging -/// configuration (e.g. a custom log ConfigMap is referenced instead), in which case no -/// `log4j2.properties` should be added to the rolegroup `ConfigMap`. -pub fn build_log4j2(logging: &Logging) -> Option { - match logging.containers.get(&Container::Hbase) { - Some(ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic(log_config)), - }) => Some(log4j_config(log_config)), - _ => None, - } -} - -/// Renders the Vector agent config (`vector.yaml`). -/// -/// Returns `None` when the Vector agent is disabled for this role group. -pub fn build_vector_config( - cluster: &ValidatedCluster, - role: &HbaseRole, - role_group_name: &RoleGroupName, - logging: &Logging, -) -> Option { - if !logging.enable_vector_agent { - return None; - } - - let vector_log_config = match logging.containers.get(&Container::Vector) { - Some(ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic(log_config)), - }) => Some(log_config), - _ => None, - }; - - let rolegroup = RoleGroupRef { - cluster: ObjectRef::from_obj(cluster), - role: role.to_string(), - role_group: role_group_name.to_string(), - }; - - Some(product_logging::framework::create_vector_config( - &rolegroup, - vector_log_config, - )) -} - -fn log4j_config(log_config: &AutomaticContainerLogConfig) -> String { - product_logging::framework::create_log4j2_config( - &format!("{STACKABLE_LOG_DIR}/hbase"), - HBASE_LOG4J2_FILE, - MAX_HBASE_LOG_FILES_SIZE - .scale_to(BinaryMultiple::Mebi) - .floor() - .value as u32, - CONSOLE_CONVERSION_PATTERN, - log_config, - ) -} diff --git a/rust/operator-binary/src/controller/build/properties/logging/mod.rs b/rust/operator-binary/src/controller/build/properties/logging/mod.rs new file mode 100644 index 00000000..ed006a02 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/logging/mod.rs @@ -0,0 +1,68 @@ +//! Renders the logging config files (`log4j2.properties` and the Vector agent config) +//! assembled into the rolegroup `ConfigMap`. + +pub use stackable_operator::v2::product_logging::framework::STACKABLE_LOG_DIR; +use stackable_operator::{ + memory::{BinaryMultiple, MemoryQuantity}, + product_logging::{self, spec::AutomaticContainerLogConfig}, + v2::product_logging::framework::ValidatedContainerLogConfigChoice, +}; + +pub const MAX_HBASE_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { + value: 10.0, + unit: BinaryMultiple::Mebi, +}; + +const CONSOLE_CONVERSION_PATTERN: &str = "%d{ISO8601} %-5p [%t] %c{2}: %.1000m%n"; +const HBASE_LOG4J2_FILE: &str = "hbase.log4j2.xml"; + +/// The Vector agent configuration (`vector.yaml`). +/// +/// This is a static, env-var-parameterized file (mirroring the hive- and opensearch-operators). +/// It is validated by the `test-vector.sh` / `vector-test.yaml` harness next to this file. +const VECTOR_CONFIG: &str = include_str!("vector.yaml"); + +/// Returns the Vector agent config (`vector.yaml`) content. +pub fn vector_config_file_content() -> String { + VECTOR_CONFIG.to_owned() +} + +/// Renders `log4j2.properties` for the HBase container. +/// +/// Returns `None` when the HBase container uses a custom log ConfigMap instead of the operator's +/// automatic logging configuration, in which case no `log4j2.properties` should be added to the +/// rolegroup `ConfigMap`. +pub fn build_log4j2(hbase_container: &ValidatedContainerLogConfigChoice) -> Option { + match hbase_container { + ValidatedContainerLogConfigChoice::Automatic(log_config) => Some(log4j_config(log_config)), + ValidatedContainerLogConfigChoice::Custom(_) => None, + } +} + +fn log4j_config(log_config: &AutomaticContainerLogConfig) -> String { + product_logging::framework::create_log4j2_config( + &format!("{STACKABLE_LOG_DIR}/hbase"), + HBASE_LOG4J2_FILE, + MAX_HBASE_LOG_FILES_SIZE + .scale_to(BinaryMultiple::Mebi) + .floor() + .value as u32, + CONSOLE_CONVERSION_PATTERN, + log_config, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_vector_config_file_content() { + let content = vector_config_file_content(); + assert!(!content.is_empty()); + // A kept source must be present ... + assert!(content.contains("files_log4j2")); + // ... while a product-specific source we don't emit must not. + assert!(!content.contains("files_tracing_rs")); + } +} diff --git a/rust/operator-binary/src/controller/build/properties/logging/test-vector.sh b/rust/operator-binary/src/controller/build/properties/logging/test-vector.sh new file mode 100755 index 00000000..c2072269 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/logging/test-vector.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env sh + +DATA_DIR=/stackable/log/_vector-state \ +LOG_DIR=/stackable/log \ +NAMESPACE=default \ +CLUSTER_NAME=hbase \ +ROLE_NAME=regionserver \ +ROLE_GROUP_NAME=default \ +VECTOR_AGGREGATOR_ADDRESS=vector-aggregator \ +VECTOR_FILE_LOG_LEVEL=info \ +vector test vector.yaml vector-test.yaml diff --git a/rust/operator-binary/src/controller/build/properties/logging/vector-test.yaml b/rust/operator-binary/src/controller/build/properties/logging/vector-test.yaml new file mode 100644 index 00000000..8266dca5 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/logging/vector-test.yaml @@ -0,0 +1,162 @@ +# Run tests with `./test-vector.sh` +# +# A downside of these test cases is that they compare the whole event and that the message can +# contain source code positions in vector.yaml, e.g. "function call error for \"parse_xml\" at +# (584:643)". Please adapt the tests if you change VRL code in vector.yaml. +--- +tests: + - name: Test stdout log entry + inputs: + - type: log + insert_at: processed_files_stdout + log_fields: + file: /stackable/log/hbase/hbase.stdout.log + message: Starting HRegionServer + pod: hbase-regionserver-default-0 + source_type: file + timestamp: 2025-10-02T09:27:28.582Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "hbase", + "container": "hbase", + "file": "hbase.stdout.log", + "level": "INFO", + "logger": "ROOT", + "message": "Starting HRegionServer", + "namespace": "default", + "pod": "hbase-regionserver-default-0", + "role": "regionserver", + "roleGroup": "default", + "timestamp": "2025-10-02T09:27:28.582Z" + } + + assert_eq!(expected_log_event, .) + - name: Test stderr log entry + inputs: + - type: log + insert_at: processed_files_stderr + log_fields: + file: /stackable/log/hbase/hbase.stderr.log + message: "Exception in thread \"main\"" + pod: hbase-regionserver-default-0 + source_type: file + timestamp: 2025-10-02T09:27:28.582Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "hbase", + "container": "hbase", + "file": "hbase.stderr.log", + "level": "ERROR", + "logger": "ROOT", + "message": "Exception in thread \"main\"", + "namespace": "default", + "pod": "hbase-regionserver-default-0", + "role": "regionserver", + "roleGroup": "default", + "timestamp": "2025-10-02T09:27:28.582Z" + } + + assert_eq!(expected_log_event, .) + - name: Test log4j2 XML log entry without stacktrace + inputs: + - type: log + insert_at: processed_files_log4j2 + log_fields: + file: /stackable/log/hbase/hbase.log4j2.xml + message: > + Serving as + regionserver... + pod: hbase-regionserver-default-0 + source_type: file + timestamp: 2025-10-02T09:27:29.473487331Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "hbase", + "container": "hbase", + "file": "hbase.log4j2.xml", + "level": "INFO", + "logger": "org.apache.hadoop.hbase.regionserver.HRegionServer", + "message": "Serving as regionserver...", + "namespace": "default", + "pod": "hbase-regionserver-default-0", + "role": "regionserver", + "roleGroup": "default", + "timestamp": t'2025-10-02T09:27:28.582Z' + } + + assert_eq!(expected_log_event, .) + - name: Test Vector internal logs + inputs: + - type: log + insert_at: filtered_logs_vector + log_fields: + arch: x86_64 + message: Vector has started. + metadata: + kind: event + level: INFO + module_path: vector::internal_events::process + target: vector + pid: 14 + pod: hbase-regionserver-default-0 + source_type: internal_logs + timestamp: 2025-10-02T09:46:14.479381097Z + version: 0.49.0 + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "arch": "x86_64", + "cluster": "hbase", + "container": "vector", + "level": "INFO", + "logger": "vector::internal_events::process", + "message": "Vector has started.", + "namespace": "default", + "pod": "hbase-regionserver-default-0", + "role": "regionserver", + "roleGroup": "default", + "timestamp": "2025-10-02T09:46:14.479381097Z", + "version": "0.49.0" + } + + assert_eq!(expected_log_event, .) + - name: Test Vector internal log level filtering - INFO passes + inputs: + - type: log + insert_at: filtered_logs_vector + log_fields: + metadata: + level: INFO + outputs: + - extract_from: filtered_logs_vector + conditions: + - type: vrl + source: | + assert_eq!("INFO", .metadata.level) + - name: Test Vector internal log level filtering - DEBUG dropped + inputs: + - type: log + insert_at: filtered_logs_vector + log_fields: + metadata: + level: DEBUG + no_outputs_from: + - filtered_logs_vector diff --git a/rust/operator-binary/src/controller/build/properties/logging/vector.yaml b/rust/operator-binary/src/controller/build/properties/logging/vector.yaml new file mode 100644 index 00000000..e4d40b6e --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/logging/vector.yaml @@ -0,0 +1,231 @@ +--- +data_dir: ${DATA_DIR} + +log_schema: + host_key: pod + +sources: + # Reads the internal Vector logs + vector: + type: internal_logs + + files_stdout: + type: file + include: + - ${LOG_DIR}/*/*.stdout.log + + files_stderr: + type: file + include: + - ${LOG_DIR}/*/*.stderr.log + + files_log4j2: + type: file + include: + - ${LOG_DIR}/*/*.log4j2.xml + line_delimiter: "\r\n" + +transforms: + processed_files_stdout: + inputs: + - files_stdout + type: remap + source: | + .logger = "ROOT" + .level = "INFO" + + processed_files_stderr: + inputs: + - files_stderr + type: remap + source: | + .logger = "ROOT" + .level = "ERROR" + + processed_files_log4j2: + inputs: + - files_log4j2 + type: remap + source: | + raw_message = string!(.message) + + .timestamp = now() + .logger = "" + .level = "INFO" + .message = "" + .errors = [] + + event = {} + parsed_event, err = parse_xml(raw_message) + if err != null { + error = "XML not parsable: " + err + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + if !is_object(parsed_event.Event) { + error = "Parsed event contains no \"Event\" tag." + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + event = object!(parsed_event.Event) + + tag_instant_valid = false + instant, err = object(event.Instant) + if err == null { + epoch_nanoseconds, err = to_int(instant.@epochSecond) * 1_000_000_000 + to_int(instant.@nanoOfSecond) + if err == null && epoch_nanoseconds != 0 { + converted_timestamp, err = from_unix_timestamp(epoch_nanoseconds, "nanoseconds") + if err == null { + .timestamp = converted_timestamp + tag_instant_valid = true + } else { + .errors = push(.errors, "Instant invalid, trying property timeMillis instead: " + err) + } + } else { + .errors = push(.errors, "Instant invalid, trying property timeMillis instead: " + err) + } + } + if !tag_instant_valid { + epoch_milliseconds, err = to_int(event.@timeMillis) + if err == null && epoch_milliseconds != 0 { + converted_timestamp, err = from_unix_timestamp(epoch_milliseconds, "milliseconds") + if err == null { + .timestamp = converted_timestamp + } else { + .errors = push(.errors, "timeMillis not parsable, using current time instead: " + err) + } + } else { + .errors = push(.errors, "timeMillis not parsable, using current time instead: " + err) + } + } + + .logger, err = string(event.@loggerName) + if err != null || is_empty(.logger) { + .errors = push(.errors, "Logger not found.") + } + + level, err = string(event.@level) + if err != null { + .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") + } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], level) { + .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") + } else { + .level = level + } + + exception = null + thrown = event.Thrown + if is_object(thrown) { + exception = "Exception" + thread, err = string(event.@thread) + if err == null && !is_empty(thread) { + exception = exception + " in thread \"" + thread + "\"" + } + thrown_name, err = string(thrown.@name) + if err == null && !is_empty(exception) { + exception = exception + " " + thrown_name + } + message = string(thrown.@localizedMessage) ?? + string(thrown.@message) ?? + "" + if !is_empty(message) { + exception = exception + ": " + message + } + stacktrace_items = array(thrown.ExtendedStackTrace.ExtendedStackTraceItem) ?? [] + stacktrace = "" + for_each(stacktrace_items) -> |_index, value| { + stacktrace = stacktrace + " " + class = string(value.@class) ?? "" + method = string(value.@method) ?? "" + if !is_empty(class) && !is_empty(method) { + stacktrace = stacktrace + "at " + class + "." + method + } + file = string(value.@file) ?? "" + line = string(value.@line) ?? "" + if !is_empty(file) && !is_empty(line) { + stacktrace = stacktrace + "(" + file + ":" + line + ")" + } + exact = to_bool(value.@exact) ?? false + location = string(value.@location) ?? "" + version = string(value.@version) ?? "" + if !is_empty(location) && !is_empty(version) { + stacktrace = stacktrace + " " + if !exact { + stacktrace = stacktrace + "~" + } + stacktrace = stacktrace + "[" + location + ":" + version + "]" + } + stacktrace = stacktrace + "\n" + } + if stacktrace != "" { + exception = exception + "\n" + stacktrace + } + } + + message, err = string(event.Message) + if err != null || is_empty(message) { + message = null + .errors = push(.errors, "Message not found.") + } + .message = join!(compact([message, exception]), "\n") + } + } + + # Extends the processed files with the fields "container" and "file" + extended_logs_files: + inputs: + - processed_files_* + type: remap + source: | + del(.source_type) + if .errors == [] { + del(.errors) + } + . |= parse_regex!(.file, r'^${LOG_DIR}/(?P.*?)/(?P.*?)$') + + # Filters the logs of the Vector agent according to the defined log level + filtered_logs_vector: + inputs: + - vector + type: filter + condition: > + (.metadata.level == "TRACE" && "${VECTOR_FILE_LOG_LEVEL}" == "trace") || + (.metadata.level == "DEBUG" && includes(["trace", "debug"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "INFO" && includes(["trace", "debug", "info"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "WARN" && includes(["trace", "debug", "info", "warn"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "ERROR" && includes(["trace", "debug", "info", "warn", "error"], "${VECTOR_FILE_LOG_LEVEL}")) + + # Aligns the logs of the Vector agent with the common format + extended_logs_vector: + inputs: + - filtered_logs_vector + type: remap + source: | + .container = "vector" + .level = .metadata.level + .logger = .metadata.module_path + if exists(.file) { .processed_file = del(.file) } + del(.metadata) + del(.pid) + del(.source_type) + + # Add the fields "namespace", "cluster", "role" and "roleGroup" to all logs + extended_logs: + inputs: + - extended_logs_* + type: remap + source: | + .namespace = "${NAMESPACE}" + .cluster = "${CLUSTER_NAME}" + .role = "${ROLE_NAME}" + .roleGroup = "${ROLE_GROUP_NAME}" + +sinks: + # Forward the logs to the Vector aggregator + aggregator: + inputs: + - extended_logs + type: vector + address: ${VECTOR_AGGREGATOR_ADDRESS} diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index 8cf844d7..e28511f8 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -138,6 +138,8 @@ spec: let default_role_group = stackable_operator::v2::types::operator::RoleGroupName::from_str("default") .expect("'default' is a valid role group name"); - &validated_cluster.role_group_configs[role][&default_role_group].config + &validated_cluster.role_group_configs[role][&default_role_group] + .config + .config } } diff --git a/rust/operator-binary/src/controller/build/resource/config_map.rs b/rust/operator-binary/src/controller/build/resource/config_map.rs index d481c614..879530d6 100644 --- a/rust/operator-binary/src/controller/build/resource/config_map.rs +++ b/rust/operator-binary/src/controller/build/resource/config_map.rs @@ -63,11 +63,12 @@ pub fn build_rolegroup_config_map( })?; let cluster_config = &cluster.cluster_config; + let merged_config = &rg.config.config; let overrides = &rg.config_overrides; let hbase_site_xml = hbase_site::build( role, - &rg.config, + merged_config, cluster_config .zookeeper_connection_information .as_hbase_settings(), @@ -77,7 +78,7 @@ pub fn build_rolegroup_config_map( ); let hbase_env_sh = hbase_env::build( - &rg.config, + merged_config, role, cluster_config.kerberos_enabled, rg.non_heap_jvm_args.clone(), @@ -128,13 +129,11 @@ pub fn build_rolegroup_config_map( builder.add_data(ConfigFileName::SslClient.to_string(), ssl_client_xml); } - if let Some(log4j2_properties) = logging::build_log4j2(rg.config.logging()) { + if let Some(log4j2_properties) = logging::build_log4j2(&rg.config.logging.hbase_container) { builder.add_data(ConfigFileName::Log4j2.to_string(), log4j2_properties); } - if let Some(vector_config) = - logging::build_vector_config(cluster, role, role_group_name, rg.config.logging()) - { - builder.add_data(VECTOR_CONFIG_FILE, vector_config); + if rg.config.logging.enable_vector_agent { + builder.add_data(VECTOR_CONFIG_FILE, logging::vector_config_file_content()); } builder.build().with_context(|_| AssembleSnafu { diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 7dd535a0..f8d2a4f7 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -1,6 +1,6 @@ //! Build the per-rolegroup `StatefulSet` for the HbaseCluster. -use std::collections::BTreeMap; +use std::{collections::BTreeMap, str::FromStr}; use indoc::formatdoc; use snafu::{OptionExt, ResultExt, Snafu}; @@ -8,10 +8,7 @@ use stackable_operator::{ builder::{ self, meta::ObjectMetaBuilder, - pod::{ - PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, - security::PodSecurityContextBuilder, - }, + pod::{PodBuilder, container::ContainerBuilder, security::PodSecurityContextBuilder}, }, constants::RESTART_CONTROLLER_ENABLED_LABEL, k8s_openapi::{ @@ -26,15 +23,15 @@ use stackable_operator::{ apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, }, kube::ResourceExt, - product_logging::{ - self, - framework::LoggingError, - spec::{ - ConfigMapLogConfig, ContainerLogConfig, ContainerLogConfigChoice, - CustomContainerLogConfig, + product_logging, + v2::{ + builder::pod::container::EnvVarSet, + product_logging::framework::{ValidatedContainerLogConfigChoice, vector_container}, + types::{ + kubernetes::{ContainerName, VolumeName}, + operator::RoleGroupName, }, }, - v2::types::operator::RoleGroupName, }; use crate::{ @@ -47,11 +44,18 @@ use crate::{ }, }, crd::{ - CONFIG_DIR_NAME, Container, HbaseRole, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, - merged_env, v1alpha1, + CONFIG_DIR_NAME, HbaseRole, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, merged_env, v1alpha1, }, }; +stackable_operator::constant!(VECTOR_CONTAINER_NAME: ContainerName = "vector"); + +// The Vector container reads `vector.yaml` from the rolegroup ConfigMap (mounted as the +// `hbase-config` volume) and writes to the shared `log` volume. These reuse the existing +// volume-name string values so the produced volume mounts match the rest of the Pod. +stackable_operator::constant!(VECTOR_LOG_CONFIG_VOLUME_NAME: VolumeName = "hbase-config"); +stackable_operator::constant!(VECTOR_LOG_VOLUME_NAME: VolumeName = "log"); + pub static CONTAINERDEBUG_LOG_DIRECTORY: std::sync::LazyLock = std::sync::LazyLock::new(|| format!("{STACKABLE_LOG_DIR}/containerdebug")); @@ -72,12 +76,6 @@ pub enum Error { #[snafu(display("failed to configure graceful shutdown"))] GracefulShutdown { source: graceful_shutdown::Error }, - #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] - VectorAggregatorConfigMapMissing, - - #[snafu(display("failed to configure logging"))] - ConfigureLogging { source: LoggingError }, - #[snafu(display("failed to add needed volume"))] AddVolume { source: builder::pod::Error }, @@ -108,7 +106,8 @@ pub fn build_rolegroup_statefulset( service_account: &ServiceAccount, ) -> Result { let resolved_product_image = &cluster.image; - let merged_config = &validated_rg_config.config; + let merged_config = &validated_rg_config.config.config; + let logging = &validated_rg_config.config.logging; let resource_names = cluster.resource_names(hbase_role, role_group_name); let ports = hbase_role @@ -254,35 +253,25 @@ pub fn build_rolegroup_statefulset( .service_account_name(service_account.name_any()) .security_context(PodSecurityContextBuilder::new().fs_group(1000).build()); - if let Some(ContainerLogConfig { - choice: - Some(ContainerLogConfigChoice::Custom(CustomContainerLogConfig { - custom: ConfigMapLogConfig { config_map }, - })), - }) = merged_config.logging().containers.get(&Container::Hbase) - { - pod_builder - .add_volume(Volume { - name: "log-config".to_string(), - config_map: Some(ConfigMapVolumeSource { - name: config_map.into(), - ..ConfigMapVolumeSource::default() - }), - ..Volume::default() - }) - .context(AddVolumeSnafu)?; - } else { - pod_builder - .add_volume(Volume { - name: "log-config".to_string(), - config_map: Some(ConfigMapVolumeSource { - name: resource_names.role_group_config_map().to_string(), - ..ConfigMapVolumeSource::default() - }), - ..Volume::default() - }) - .context(AddVolumeSnafu)?; - } + // The HBase container's log config ConfigMap: either the operator-generated one (the + // rolegroup ConfigMap, which carries the automatic `log4j2.properties`) or a user-provided + // custom ConfigMap. This branches on the *validated* logging choice (see `ValidatedLogging`). + let log_config_config_map = match &logging.hbase_container { + ValidatedContainerLogConfigChoice::Custom(config_map_name) => config_map_name.to_string(), + ValidatedContainerLogConfigChoice::Automatic(_) => { + resource_names.role_group_config_map().to_string() + } + }; + pod_builder + .add_volume(Volume { + name: "log-config".to_string(), + config_map: Some(ConfigMapVolumeSource { + name: log_config_config_map, + ..ConfigMapVolumeSource::default() + }), + ..Volume::default() + }) + .context(AddVolumeSnafu)?; add_graceful_shutdown_config(merged_config, &mut pod_builder).context(GracefulShutdownSnafu)?; if cluster.has_kerberos_enabled() { @@ -299,30 +288,17 @@ pub fn build_rolegroup_statefulset( } pod_builder.add_container(hbase_container.build()); - // Vector sidecar shall be the last container in the list - if merged_config.logging().enable_vector_agent { - if let Some(vector_aggregator_config_map_name) = - &hbase.spec.cluster_config.vector_aggregator_config_map_name - { - pod_builder.add_container( - product_logging::framework::vector_container( - resolved_product_image, - "hbase-config", - "log", - merged_config.logging().containers.get(&Container::Vector), - ResourceRequirementsBuilder::new() - .with_cpu_request("250m") - .with_cpu_limit("500m") - .with_memory_request("128Mi") - .with_memory_limit("128Mi") - .build(), - vector_aggregator_config_map_name, - ) - .context(ConfigureLoggingSnafu)?, - ); - } else { - VectorAggregatorConfigMapMissingSnafu.fail()?; - } + // Vector sidecar shall be the last container in the list. + if let Some(vector_log_config) = &logging.vector_container { + pod_builder.add_container(vector_container( + &VECTOR_CONTAINER_NAME, + resolved_product_image, + vector_log_config, + &resource_names, + &VECTOR_LOG_CONFIG_VOLUME_NAME, + &VECTOR_LOG_VOLUME_NAME, + EnvVarSet::new(), + )); } let listener_pvc = hbase_role diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index e97c3a68..9de726f2 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -258,18 +258,20 @@ pub struct ValidatedRoleConfig { /// Per-rolegroup configuration: the merged CRD config plus the merged /// (role <- role group) `configOverrides`, `envOverrides` and `podOverrides`. -/// -/// The merge and validation is performed by -/// [`with_validated_config`](stackable_operator::v2::role_utils::with_validated_config); the -/// result is flattened into this struct and augmented with the pre-resolved -/// `non_heap_jvm_args`. Carrying every override channel (and the JVM args) keeps the -/// build step a pure function of [`ValidatedCluster`] that never has to reach back into -/// the raw `HbaseCluster`. +#[derive(Clone, Debug)] +pub struct ValidatedHbaseConfig { + /// The merged, role-specific product config. + pub config: AnyServiceConfig, + /// The validated logging configuration (HBase + optional Vector container), resolved up-front + /// during validation. + pub logging: validate::ValidatedLogging, +} + #[derive(Clone, Debug)] pub struct ValidatedRoleGroupConfig { /// The desired number of replicas (defaulted to 1 during validation). pub replicas: u16, - pub config: AnyServiceConfig, + pub config: ValidatedHbaseConfig, pub config_overrides: v1alpha1::HbaseConfigOverrides, pub env_overrides: EnvVarSet, /// Merged (role <- role group) pod template overrides. diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index b8ae632f..913c33a6 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, str::FromStr}; -use snafu::{ResultExt, Snafu}; +use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ commons::product_image_selection::{self}, config::{ @@ -8,27 +8,39 @@ use stackable_operator::{ merge::Merge, }, kube::ResourceExt, + product_logging::spec::Logging, role_utils::{CommonConfiguration, GenericRoleConfig, Role}, utils::cluster_info::KubernetesClusterInfo, v2::{ builder::pod::container::{self, EnvVarName, EnvVarSet}, controller_utils::{get_cluster_name, get_namespace, get_uid}, + product_logging::framework::{ + ValidatedContainerLogConfigChoice, VectorContainerLogConfig, + validate_logging_configuration_for_container, + }, role_utils::{JavaCommonConfig, with_validated_config}, - types::operator::RoleGroupName, + types::{kubernetes::ConfigMapName, operator::RoleGroupName}, }, }; use strum::IntoEnumIterator; use crate::{ - controller::build::kerberos::{ - self, kerberos_config_properties, kerberos_discovery_config_properties, - kerberos_ssl_client_settings, kerberos_ssl_server_settings, - }, controller::{ - ValidatedCluster, ValidatedClusterConfig, ValidatedRoleConfig, ValidatedRoleGroupConfig, - build::jvm::construct_role_specific_non_heap_jvm_args, dereference::DereferencedObjects, + ValidatedCluster, ValidatedClusterConfig, ValidatedHbaseConfig, ValidatedRoleConfig, + ValidatedRoleGroupConfig, + build::{ + jvm::construct_role_specific_non_heap_jvm_args, + kerberos::{ + self, kerberos_config_properties, kerberos_discovery_config_properties, + kerberos_ssl_client_settings, kerberos_ssl_server_settings, + }, + }, + dereference::DereferencedObjects, + }, + crd::{ + AnyServiceConfig, Container, HbaseConfigFragment, HbaseRole, RegionServerConfigFragment, + v1alpha1, }, - crd::{AnyServiceConfig, HbaseConfigFragment, HbaseRole, RegionServerConfigFragment, v1alpha1}, }; const CONTAINER_IMAGE_BASE_NAME: &str = "hbase"; @@ -62,6 +74,60 @@ pub enum Error { #[snafu(display("failed to resolve kerberos config"))] AddKerberosConfig { source: kerberos::Error }, + + #[snafu(display("failed to validate logging configuration"))] + ValidateLoggingConfig { + source: stackable_operator::v2::product_logging::framework::Error, + }, + + #[snafu(display( + "the Vector aggregator discovery ConfigMap name is required when the Vector agent is enabled" + ))] + MissingVectorAggregatorConfigMapName, + + #[snafu(display("invalid Vector aggregator discovery ConfigMap name"))] + ParseVectorAggregatorConfigMapName { + source: stackable_operator::v2::macros::attributed_string_type::Error, + }, +} + +/// Validated logging configuration for the HBase and (optional) Vector container. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ValidatedLogging { + pub hbase_container: ValidatedContainerLogConfigChoice, + pub vector_container: Option, + pub enable_vector_agent: bool, +} + +/// Validates the logging configuration for the HBase (and optional Vector) container. +/// +/// `vector_aggregator_config_map_name` is the discovery ConfigMap name of the Vector aggregator; +/// it is required (and validated) only when the Vector agent is enabled. +fn validate_logging( + logging: &Logging, + vector_aggregator_config_map_name: &Option, +) -> Result { + let hbase_container = validate_logging_configuration_for_container(logging, &Container::Hbase) + .context(ValidateLoggingConfigSnafu)?; + + let vector_container = if logging.enable_vector_agent { + let vector_aggregator_config_map_name = vector_aggregator_config_map_name + .clone() + .context(MissingVectorAggregatorConfigMapNameSnafu)?; + Some(VectorContainerLogConfig { + log_config: validate_logging_configuration_for_container(logging, &Container::Vector) + .context(ValidateLoggingConfigSnafu)?, + vector_aggregator_config_map_name, + }) + } else { + None + }; + + Ok(ValidatedLogging { + hbase_container, + vector_container, + enable_vector_agent: logging.enable_vector_agent, + }) } pub fn validate_cluster( @@ -86,6 +152,17 @@ pub fn validate_cluster( let hdfs_discovery_cm_name = &hbase.spec.cluster_config.hdfs_config_map_name; let cluster_name = hbase.name_any(); + // The Vector aggregator discovery ConfigMap name (validated here so an invalid name fails + // up-front). It is only required when the Vector agent is enabled for a role group. + let vector_aggregator_config_map_name = hbase + .spec + .cluster_config + .vector_aggregator_config_map_name + .as_deref() + .map(ConfigMapName::from_str) + .transpose() + .context(ParseVectorAggregatorConfigMapNameSnafu)?; + for hbase_role in HbaseRole::iter() { let group_configs = match hbase_role { HbaseRole::Master => validate_role_group_configs( @@ -97,6 +174,7 @@ pub fn validate_cluster( hdfs_discovery_cm_name, ), AnyServiceConfig::Master, + &vector_aggregator_config_map_name, )?, HbaseRole::RegionServer => validate_role_group_configs( hbase, @@ -107,6 +185,7 @@ pub fn validate_cluster( hdfs_discovery_cm_name, ), AnyServiceConfig::RegionServer, + &vector_aggregator_config_map_name, )?, HbaseRole::RestServer => validate_role_group_configs( hbase, @@ -117,6 +196,7 @@ pub fn validate_cluster( hdfs_discovery_cm_name, ), AnyServiceConfig::RestServer, + &vector_aggregator_config_map_name, )?, }; @@ -194,6 +274,7 @@ fn validate_role_group_configs( >, default_config: Config, wrap: fn(ValidatedConfig) -> AnyServiceConfig, + vector_aggregator_config_map_name: &Option, ) -> Result, Error> where Config: Clone + Merge, @@ -244,9 +325,16 @@ where ); } + let config = wrap(config); + + // Validate the logging configuration up-front so an invalid custom log ConfigMap name + // or a missing Vector aggregator discovery ConfigMap fails here rather than at build + // time. The build step then consumes the validated logging instead of the raw config. + let logging = validate_logging(config.logging(), vector_aggregator_config_map_name)?; + let validated = ValidatedRoleGroupConfig { replicas: validated.replicas.unwrap_or(1), - config: wrap(config), + config: ValidatedHbaseConfig { config, logging }, config_overrides, env_overrides: env_overrides_set, pod_overrides, From c941688c377aaea74d131d326ed2ad258e6b1a71 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 13:23:40 +0200 Subject: [PATCH 40/56] refactor: switch to v2 Clusterresources --- rust/operator-binary/src/controller/mod.rs | 3 +++ rust/operator-binary/src/hbase_controller.rs | 27 +++++++++----------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index 9de726f2..07c1fb99 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -61,6 +61,8 @@ pub struct ValidatedCluster { metadata: ObjectMeta, /// The logical (and Kubernetes object) name of the cluster. pub name: ClusterName, + /// The namespace the cluster lives in. + pub namespace: NamespaceName, /// The UID of the `HbaseCluster` object, used to build owner references. pub uid: Uid, pub image: ResolvedProductImage, @@ -96,6 +98,7 @@ impl ValidatedCluster { ..ObjectMeta::default() }, name, + namespace, uid, image, product_version, diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index c6590d21..33bc9006 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -10,10 +10,9 @@ use std::sync::Arc; use snafu::{ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, - cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, + cluster_resources::ClusterResourceApplyStrategy, commons::rbac::build_rbac_resources, kube::{ - Resource, core::{DeserializeGuard, error_boundary}, runtime::controller::Action, }, @@ -24,12 +23,13 @@ use stackable_operator::{ compute_conditions, operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, }, + v2::cluster_resources::cluster_resources_new, }; use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ controller::{ - HBASE_CONTROLLER_NAME, RoleGroupName, + RoleGroupName, build::resource::{ config_map::build_rolegroup_config_map, discovery::build_discovery_config_map, @@ -37,6 +37,7 @@ use crate::{ service::{build_rolegroup_metrics_service, build_rolegroup_service}, statefulset::build_rolegroup_statefulset, }, + controller_name, operator_name, product_name, }, crd::{APP_NAME, HbaseClusterStatus, OPERATOR_NAME, v1alpha1}, }; @@ -49,11 +50,6 @@ pub struct Ctx { #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] pub enum Error { - #[snafu(display("failed to create cluster resources"))] - CreateClusterResources { - source: stackable_operator::cluster_resources::Error, - }, - #[snafu(display("failed to delete orphaned resources"))] DeleteOrphanedResources { source: stackable_operator::cluster_resources::Error, @@ -176,15 +172,16 @@ pub async fn reconcile_hbase( ) .context(ValidateSnafu)?; - let mut cluster_resources = ClusterResources::new( - APP_NAME, - OPERATOR_NAME, - HBASE_CONTROLLER_NAME, - &hbase.object_ref(&()), + let mut cluster_resources = cluster_resources_new( + &product_name(), + &operator_name(), + &controller_name(), + &validated_cluster.name, + &validated_cluster.namespace, + &validated_cluster.uid, ClusterResourceApplyStrategy::from(&hbase.spec.cluster_operation), &hbase.spec.object_overrides, - ) - .context(CreateClusterResourcesSnafu)?; + ); let (rbac_sa, rbac_rolebinding) = build_rbac_resources( hbase, From 064e3f4eef3079b34333a15de9a314c028b54307 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 13:36:05 +0200 Subject: [PATCH 41/56] chore: cleanup constants --- .../src/controller/build/jvm.rs | 10 +-- .../src/controller/build/kerberos.rs | 14 ++++- .../controller/build/resource/statefulset.rs | 61 +++++++++---------- rust/operator-binary/src/crd/mod.rs | 15 +---- 4 files changed, 46 insertions(+), 54 deletions(-) diff --git a/rust/operator-binary/src/controller/build/jvm.rs b/rust/operator-binary/src/controller/build/jvm.rs index 205f5af2..cf4c7ce3 100644 --- a/rust/operator-binary/src/controller/build/jvm.rs +++ b/rust/operator-binary/src/controller/build/jvm.rs @@ -4,7 +4,10 @@ use stackable_operator::{ v2::jvm_argument_overrides::JvmArgumentOverrides, }; -use crate::crd::{AnyServiceConfig, CONFIG_DIR_NAME, JVM_SECURITY_PROPERTIES_FILE, v1alpha1}; +use crate::{ + controller::build::kerberos::KRB5_CONFIG_PATH, + crd::{AnyServiceConfig, CONFIG_DIR_NAME, JVM_SECURITY_PROPERTIES_FILE, v1alpha1}, +}; const JAVA_HEAP_FACTOR: f32 = 0.8; @@ -24,7 +27,7 @@ pub fn construct_global_jvm_args(kerberos_enabled: bool) -> String { let mut jvm_args = Vec::new(); if kerberos_enabled { - jvm_args.push("-Djava.security.krb5.conf=/stackable/kerberos/krb5.conf"); + jvm_args.push(format!("-Djava.security.krb5.conf={KRB5_CONFIG_PATH}")); } // We do *not* add user overrides to the global JVM args, but only the role specific JVM arguments. @@ -54,8 +57,7 @@ pub fn construct_role_specific_non_heap_jvm_args( )]; if hbase.has_kerberos_enabled() { - operator_generated - .push("-Djava.security.krb5.conf=/stackable/kerberos/krb5.conf".to_owned()); + operator_generated.push(format!("-Djava.security.krb5.conf={KRB5_CONFIG_PATH}")); } let mut jvm_args = merged_jvm_argument_overrides.apply_to(operator_generated); diff --git a/rust/operator-binary/src/controller/build/kerberos.rs b/rust/operator-binary/src/controller/build/kerberos.rs index 4bc18339..aa9cf603 100644 --- a/rust/operator-binary/src/controller/build/kerberos.rs +++ b/rust/operator-binary/src/controller/build/kerberos.rs @@ -18,6 +18,14 @@ use stackable_operator::{ use crate::crd::{TLS_STORE_DIR, TLS_STORE_PASSWORD, TLS_STORE_VOLUME_NAME, v1alpha1}; +/// Mount path of the Kerberos secret volume (keytab + `krb5.conf`). +pub const STACKABLE_KERBEROS_DIR: &str = "/stackable/kerberos"; +/// Path of the `krb5.conf` rendered into the Kerberos secret volume. Referenced both here (the +/// `KRB5_CONFIG` env var) and by the JVM args builder. +pub const KRB5_CONFIG_PATH: &str = const_format::concatcp!(STACKABLE_KERBEROS_DIR, "/krb5.conf"); +/// Name of the Kerberos secret volume. +const KERBEROS_VOLUME_NAME: &str = "kerberos"; + #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("object {hbase} is missing namespace"))] @@ -188,16 +196,16 @@ pub fn add_kerberos_pod_config( .build() .context(AddKerberosSecretVolumeSnafu)?; pb.add_volume( - VolumeBuilder::new("kerberos") + VolumeBuilder::new(KERBEROS_VOLUME_NAME) .ephemeral(kerberos_secret_operator_volume) .build(), ) .context(AddVolumeSnafu)?; - cb.add_volume_mount("kerberos", "/stackable/kerberos") + cb.add_volume_mount(KERBEROS_VOLUME_NAME, STACKABLE_KERBEROS_DIR) .context(AddVolumeMountSnafu)?; // Needed env vars - cb.add_env_var("KRB5_CONFIG", "/stackable/kerberos/krb5.conf"); + cb.add_env_var("KRB5_CONFIG", KRB5_CONFIG_PATH); } if let Some(https_secret_class) = hbase.https_secret_class() { diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index f8d2a4f7..5d652960 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -1,6 +1,6 @@ //! Build the per-rolegroup `StatefulSet` for the HbaseCluster. -use std::{collections::BTreeMap, str::FromStr}; +use std::str::FromStr; use indoc::formatdoc; use snafu::{OptionExt, ResultExt, Snafu}; @@ -16,8 +16,8 @@ use stackable_operator::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec}, core::v1::{ - ConfigMapVolumeSource, ContainerPort, EnvVar, Probe, ServiceAccount, - TCPSocketAction, Volume, + ConfigMapVolumeSource, ContainerPort, Probe, ServiceAccount, TCPSocketAction, + Volume, }, }, apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, @@ -25,7 +25,7 @@ use stackable_operator::{ kube::ResourceExt, product_logging, v2::{ - builder::pod::container::EnvVarSet, + builder::pod::container::{EnvVarName, EnvVarSet}, product_logging::framework::{ValidatedContainerLogConfigChoice, vector_container}, types::{ kubernetes::{ContainerName, VolumeName}, @@ -43,9 +43,7 @@ use crate::{ properties::logging::{MAX_HBASE_LOG_FILES_SIZE, STACKABLE_LOG_DIR}, }, }, - crd::{ - CONFIG_DIR_NAME, HbaseRole, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, merged_env, v1alpha1, - }, + crd::{CONFIG_DIR_NAME, HbaseRole, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, v1alpha1}, }; stackable_operator::constant!(VECTOR_CONTAINER_NAME: ContainerName = "vector"); @@ -149,33 +147,30 @@ pub fn build_rolegroup_statefulset( ..probe_template }; - let mut env_map: BTreeMap = BTreeMap::from([ - ("HBASE_CONF_DIR".to_string(), CONFIG_DIR_NAME.to_string()), + let merged_env = EnvVarSet::new() + .with_value( + &EnvVarName::from_str_unsafe("HBASE_CONF_DIR"), + CONFIG_DIR_NAME, + ) // required by phoenix (for cases where Kerberos is enabled): see https://issues.apache.org/jira/browse/PHOENIX-2369 - ("HADOOP_CONF_DIR".to_string(), CONFIG_DIR_NAME.to_string()), - ]); - for env_var in validated_rg_config.env_overrides.clone() { - env_map.insert(env_var.name, env_var.value.unwrap_or_default()); - } - let mut merged_env = merged_env(&env_map); - // This env var is set for all roles to avoid bash's "unbound variable" errors - merged_env.extend([ - EnvVar { - name: "REGION_MOVER_OPTS".to_string(), - value: Some(merged_config.region_mover_args()), - ..EnvVar::default() - }, - EnvVar { - name: "RUN_REGION_MOVER".to_string(), - value: Some(merged_config.run_region_mover().to_string()), - ..EnvVar::default() - }, - EnvVar { - name: "STACKABLE_LOG_DIR".to_string(), - value: Some(STACKABLE_LOG_DIR.to_string()), - ..EnvVar::default() - }, - ]); + .with_value( + &EnvVarName::from_str_unsafe("HADOOP_CONF_DIR"), + CONFIG_DIR_NAME, + ) + .merge(validated_rg_config.env_overrides.clone()) + // These env vars are set for all roles to avoid bash's "unbound variable" errors. + .with_value( + &EnvVarName::from_str_unsafe("REGION_MOVER_OPTS"), + merged_config.region_mover_args(), + ) + .with_value( + &EnvVarName::from_str_unsafe("RUN_REGION_MOVER"), + merged_config.run_region_mover().to_string(), + ) + .with_value( + &EnvVarName::from_str_unsafe("STACKABLE_LOG_DIR"), + STACKABLE_LOG_DIR, + ); let role_name = hbase_role.cli_role_name(); let mut hbase_container = ContainerBuilder::new("hbase").expect("ContainerBuilder not created"); diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index f9222468..f66824a0 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,5 +1,3 @@ -use std::collections::BTreeMap; - use security::AuthenticationConfig; use serde::{Deserialize, Serialize}; use shell_escape::escape; @@ -24,7 +22,7 @@ use stackable_operator::{ }, deep_merger::ObjectOverrides, k8s_openapi::{ - api::core::v1::{EnvVar, PersistentVolumeClaim, Volume}, + api::core::v1::{PersistentVolumeClaim, Volume}, apimachinery::pkg::api::resource::Quantity, }, kube::CustomResource, @@ -263,17 +261,6 @@ impl v1alpha1::HbaseCluster { } } -pub fn merged_env(rolegroup_config: &BTreeMap) -> Vec { - rolegroup_config - .iter() - .map(|(env_name, env_value)| EnvVar { - name: env_name.clone(), - value: Some(env_value.to_owned()), - value_from: None, - }) - .collect() -} - #[derive( Clone, Debug, From 40979117d6e9a1ca16227a90576b63c66e08e240 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 13:55:28 +0200 Subject: [PATCH 42/56] refactor: Use typed constants, remove duplicated merge --- .../src/controller/build/jvm.rs | 58 ++++----- .../controller/build/properties/hbase_env.rs | 2 +- .../controller/build/properties/hbase_site.rs | 2 +- .../src/controller/build/properties/mod.rs | 86 ------------- .../src/controller/build/resource/service.rs | 9 +- .../controller/build/resource/statefulset.rs | 32 ++--- .../src/controller/validate.rs | 54 ++++++++ rust/operator-binary/src/crd/affinity.rs | 13 +- rust/operator-binary/src/crd/mod.rs | 120 ++---------------- rust/operator-binary/src/main.rs | 2 + rust/operator-binary/src/test_utils.rs | 110 ++++++++++++++++ 11 files changed, 224 insertions(+), 264 deletions(-) create mode 100644 rust/operator-binary/src/test_utils.rs diff --git a/rust/operator-binary/src/controller/build/jvm.rs b/rust/operator-binary/src/controller/build/jvm.rs index cf4c7ce3..e7d9ba24 100644 --- a/rust/operator-binary/src/controller/build/jvm.rs +++ b/rust/operator-binary/src/controller/build/jvm.rs @@ -100,7 +100,7 @@ fn is_heap_jvm_argument(jvm_argument: &str) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::crd::{HbaseRole, v1alpha1}; + use crate::{crd::HbaseRole, test_utils}; #[test] fn test_construct_jvm_arguments_defaults() { @@ -109,6 +109,8 @@ mod tests { kind: HbaseCluster metadata: name: simple-hbase + namespace: default + uid: 12345678-1234-1234-1234-123456789012 spec: image: productVersion: 2.6.4 @@ -124,16 +126,20 @@ mod tests { default: replicas: 1 "#; - let (hbase, merged_config, merged_jvm_argument_overrides) = construct_boilerplate(input); + let hbase = test_utils::hbase_from_yaml(input); + let validated_cluster = test_utils::validated_cluster_from(&hbase); + let region_server = &validated_cluster.role_group_configs[&HbaseRole::RegionServer] + [&test_utils::role_group_name("default")]; - let global_jvm_args = construct_global_jvm_args(false); - let role_specific_non_heap_jvm_args = - construct_role_specific_non_heap_jvm_args(&hbase, &merged_jvm_argument_overrides); - let hbase_heapsize_env = construct_hbase_heapsize_env(&merged_config).unwrap(); + let global_jvm_args = construct_global_jvm_args(hbase.has_kerberos_enabled()); + let hbase_heapsize_env = + construct_hbase_heapsize_env(®ion_server.config.config).unwrap(); assert_eq!(global_jvm_args, ""); + // `non_heap_jvm_args` is the output of `construct_role_specific_non_heap_jvm_args`, + // pre-resolved during validation. assert_eq!( - role_specific_non_heap_jvm_args, + region_server.non_heap_jvm_args, "-Djava.security.properties=/stackable/conf/security.properties" ); assert_eq!(hbase_heapsize_env, "819m"); @@ -146,6 +152,8 @@ mod tests { kind: HbaseCluster metadata: name: simple-hbase + namespace: default + uid: 12345678-1234-1234-1234-123456789012 spec: image: productVersion: 2.6.4 @@ -180,19 +188,21 @@ mod tests { - -Xmx40000m # This has no effect! - -Dhttps.proxyPort=1234 "#; - let (hbase, merged_config, merged_jvm_argument_overrides) = construct_boilerplate(input); + let hbase = test_utils::hbase_from_yaml(input); + let validated_cluster = test_utils::validated_cluster_from(&hbase); + let region_server = &validated_cluster.role_group_configs[&HbaseRole::RegionServer] + [&test_utils::role_group_name("default")]; let global_jvm_args = construct_global_jvm_args(hbase.has_kerberos_enabled()); - let role_specific_non_heap_jvm_args = - construct_role_specific_non_heap_jvm_args(&hbase, &merged_jvm_argument_overrides); - let hbase_heapsize_env = construct_hbase_heapsize_env(&merged_config).unwrap(); + let hbase_heapsize_env = + construct_hbase_heapsize_env(®ion_server.config.config).unwrap(); assert_eq!( global_jvm_args, "-Djava.security.krb5.conf=/stackable/kerberos/krb5.conf" ); assert_eq!( - role_specific_non_heap_jvm_args, + region_server.non_heap_jvm_args, "-Djava.security.properties=/stackable/conf/security.properties \ -Djava.security.krb5.conf=/stackable/kerberos/krb5.conf \ -Dhttps.proxyHost=proxy.my.corp \ @@ -201,28 +211,4 @@ mod tests { ); assert_eq!(hbase_heapsize_env, "34406m"); } - - fn construct_boilerplate( - hbase_cluster: &str, - ) -> ( - v1alpha1::HbaseCluster, - AnyServiceConfig, - JvmArgumentOverrides, - ) { - let hbase: v1alpha1::HbaseCluster = - serde_yaml::from_str(hbase_cluster).expect("illegal test input"); - - // Merge + validate the region server `default` role group via the real - // `with_validated_config` path, returning the merged config (for heap sizing) and the - // merged JVM argument overrides. - let (merged_config, merged_jvm_argument_overrides) = - crate::crd::test_helpers::merged_role_group_config( - &hbase, - &HbaseRole::RegionServer, - "default", - "my-hdfs", - ); - - (hbase, merged_config, merged_jvm_argument_overrides) - } } diff --git a/rust/operator-binary/src/controller/build/properties/hbase_env.rs b/rust/operator-binary/src/controller/build/properties/hbase_env.rs index e19a1807..8c0af6c5 100644 --- a/rust/operator-binary/src/controller/build/properties/hbase_env.rs +++ b/rust/operator-binary/src/controller/build/properties/hbase_env.rs @@ -62,7 +62,7 @@ pub fn build( #[cfg(test)] mod tests { use super::*; - use crate::controller::build::properties::test_support::{merged_config, validated_cluster}; + use crate::test_utils::{merged_config, validated_cluster}; #[test] fn renders_operator_defaults() { diff --git a/rust/operator-binary/src/controller/build/properties/hbase_site.rs b/rust/operator-binary/src/controller/build/properties/hbase_site.rs index faabbeeb..16aa1f4b 100644 --- a/rust/operator-binary/src/controller/build/properties/hbase_site.rs +++ b/rust/operator-binary/src/controller/build/properties/hbase_site.rs @@ -112,7 +112,7 @@ pub fn build( #[cfg(test)] mod tests { use super::*; - use crate::controller::build::properties::test_support::{merged_config, validated_cluster}; + use crate::test_utils::{merged_config, validated_cluster}; #[test] fn renders_operator_defaults() { diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index e28511f8..aae2b891 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -57,89 +57,3 @@ mod tests { assert_eq!(ConfigFileName::Log4j2.to_string(), "log4j2.properties"); } } - -#[cfg(test)] -pub(crate) mod test_support { - use std::str::FromStr; - - use stackable_operator::{ - commons::networking::DomainName, utils::cluster_info::KubernetesClusterInfo, - }; - - use crate::{ - controller::{ - ValidatedCluster, dereference::DereferencedObjects, validate::validate_cluster, - zookeeper::ZookeeperConnectionInformation, - }, - crd::{AnyServiceConfig, HbaseRole, v1alpha1}, - }; - - /// A minimal three-role HbaseCluster used to drive the per-file builder tests. - pub const MINIMAL_HBASE_YAML: &str = r#" ---- -apiVersion: hbase.stackable.tech/v1alpha1 -kind: HbaseCluster -metadata: - name: hbase - namespace: default - uid: c2c8c5c0-0b5a-4b1e-9f3e-1a2b3c4d5e6f -spec: - image: - productVersion: 2.6.3 - clusterConfig: - hdfsConfigMapName: simple-hdfs - zookeeperConfigMapName: simple-znode - masters: - roleGroups: - default: - replicas: 1 - regionServers: - roleGroups: - default: - replicas: 1 - restServers: - roleGroups: - default: - replicas: 1 -"#; - - pub fn minimal_hbase() -> v1alpha1::HbaseCluster { - serde_yaml::from_str(MINIMAL_HBASE_YAML).expect("invalid test HbaseCluster YAML") - } - - pub fn cluster_info() -> KubernetesClusterInfo { - KubernetesClusterInfo { - cluster_domain: DomainName::try_from("cluster.local").unwrap(), - } - } - - /// Runs the real validation pipeline once over [`minimal_hbase`], with a fixed - /// dereferenced ZooKeeper connection (and no OPA), so the per-file builder tests can - /// pull merged configs straight from the [`ValidatedCluster`] instead of re-merging by - /// hand via `crd::merged_config`. - pub fn validated_cluster() -> ValidatedCluster { - validate_cluster( - &minimal_hbase(), - "oci.example.org", - &cluster_info(), - DereferencedObjects { - zookeeper_connection_information: ZookeeperConnectionInformation::for_tests(), - hbase_opa_config: None, - }, - ) - .expect("validate should succeed for the minimal fixture") - } - - /// The merged [`AnyServiceConfig`] for the `default` role group of `role`. - pub fn merged_config<'a>( - validated_cluster: &'a ValidatedCluster, - role: &HbaseRole, - ) -> &'a AnyServiceConfig { - let default_role_group = - stackable_operator::v2::types::operator::RoleGroupName::from_str("default") - .expect("'default' is a valid role group name"); - &validated_cluster.role_group_configs[role][&default_role_group] - .config - .config - } -} diff --git a/rust/operator-binary/src/controller/build/resource/service.rs b/rust/operator-binary/src/controller/build/resource/service.rs index 163ad261..ec752e6a 100644 --- a/rust/operator-binary/src/controller/build/resource/service.rs +++ b/rust/operator-binary/src/controller/build/resource/service.rs @@ -137,13 +137,10 @@ fn prometheus_annotations(hbase: &v1alpha1::HbaseCluster, hbase_role: &HbaseRole #[cfg(test)] mod test { - use std::str::FromStr; - use rstest::rstest; - use stackable_operator::v2::types::operator::RoleGroupName; use super::*; - use crate::controller::build::properties::test_support; + use crate::test_utils; #[rstest] #[case("2.6.3", HbaseRole::Master, vec!["master", "ui-http"])] @@ -187,8 +184,8 @@ mod test { let hbase: v1alpha1::HbaseCluster = serde_yaml::from_str(&input).expect("illegal test input"); - let cluster = test_support::validated_cluster(); - let role_group_name = RoleGroupName::from_str("default").expect("valid role group name"); + let cluster = test_utils::validated_cluster(); + let role_group_name = test_utils::role_group_name("default"); let service = build_rolegroup_service(&hbase, &cluster, &role, &role_group_name); assert_eq!( diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 5d652960..d99e40ad 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -48,11 +48,13 @@ use crate::{ stackable_operator::constant!(VECTOR_CONTAINER_NAME: ContainerName = "vector"); -// The Vector container reads `vector.yaml` from the rolegroup ConfigMap (mounted as the -// `hbase-config` volume) and writes to the shared `log` volume. These reuse the existing -// volume-name string values so the produced volume mounts match the rest of the Pod. -stackable_operator::constant!(VECTOR_LOG_CONFIG_VOLUME_NAME: VolumeName = "hbase-config"); -stackable_operator::constant!(VECTOR_LOG_VOLUME_NAME: VolumeName = "log"); +// Pod volume names. The Vector container reuses the `hbase-config` (rolegroup ConfigMap, which +// carries `vector.yaml`) and `log` volumes, so the produced volume mounts match the rest of the +// Pod. +stackable_operator::constant!(HBASE_CONFIG_VOLUME_NAME: VolumeName = "hbase-config"); +stackable_operator::constant!(HDFS_DISCOVERY_VOLUME_NAME: VolumeName = "hdfs-discovery"); +stackable_operator::constant!(LOG_CONFIG_VOLUME_NAME: VolumeName = "log-config"); +stackable_operator::constant!(LOG_VOLUME_NAME: VolumeName = "log"); pub static CONTAINERDEBUG_LOG_DIRECTORY: std::sync::LazyLock = std::sync::LazyLock::new(|| format!("{STACKABLE_LOG_DIR}/containerdebug")); @@ -192,13 +194,13 @@ pub fn build_rolegroup_statefulset( "CONTAINERDEBUG_LOG_DIRECTORY", &*CONTAINERDEBUG_LOG_DIRECTORY, ) - .add_volume_mount("hbase-config", HBASE_CONFIG_TMP_DIR) + .add_volume_mount(&*HBASE_CONFIG_VOLUME_NAME, HBASE_CONFIG_TMP_DIR) .context(AddVolumeMountSnafu)? - .add_volume_mount("hdfs-discovery", HDFS_DISCOVERY_TMP_DIR) + .add_volume_mount(&*HDFS_DISCOVERY_VOLUME_NAME, HDFS_DISCOVERY_TMP_DIR) .context(AddVolumeMountSnafu)? - .add_volume_mount("log-config", HBASE_LOG_CONFIG_TMP_DIR) + .add_volume_mount(&*LOG_CONFIG_VOLUME_NAME, HBASE_LOG_CONFIG_TMP_DIR) .context(AddVolumeMountSnafu)? - .add_volume_mount("log", STACKABLE_LOG_DIR) + .add_volume_mount(&*LOG_VOLUME_NAME, STACKABLE_LOG_DIR) .context(AddVolumeMountSnafu)? .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) .context(AddVolumeMountSnafu)? @@ -221,7 +223,7 @@ pub fn build_rolegroup_statefulset( .image_pull_secrets_from_product_image(resolved_product_image) .affinity(merged_config.affinity()) .add_volume(Volume { - name: "hbase-config".to_string(), + name: HBASE_CONFIG_VOLUME_NAME.to_string(), config_map: Some(ConfigMapVolumeSource { name: resource_names.role_group_config_map().to_string(), ..Default::default() @@ -230,7 +232,7 @@ pub fn build_rolegroup_statefulset( }) .context(AddVolumeSnafu)? .add_volume(Volume { - name: "hdfs-discovery".to_string(), + name: HDFS_DISCOVERY_VOLUME_NAME.to_string(), config_map: Some(ConfigMapVolumeSource { name: hbase.spec.cluster_config.hdfs_config_map_name.clone(), ..Default::default() @@ -239,7 +241,7 @@ pub fn build_rolegroup_statefulset( }) .context(AddVolumeSnafu)? .add_empty_dir_volume( - "log", + &*LOG_VOLUME_NAME, Some(product_logging::framework::calculate_log_volume_size_limit( &[MAX_HBASE_LOG_FILES_SIZE], )), @@ -259,7 +261,7 @@ pub fn build_rolegroup_statefulset( }; pod_builder .add_volume(Volume { - name: "log-config".to_string(), + name: LOG_CONFIG_VOLUME_NAME.to_string(), config_map: Some(ConfigMapVolumeSource { name: log_config_config_map, ..ConfigMapVolumeSource::default() @@ -290,8 +292,8 @@ pub fn build_rolegroup_statefulset( resolved_product_image, vector_log_config, &resource_names, - &VECTOR_LOG_CONFIG_VOLUME_NAME, - &VECTOR_LOG_VOLUME_NAME, + &HBASE_CONFIG_VOLUME_NAME, + &LOG_VOLUME_NAME, EnvVarSet::new(), )); } diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 913c33a6..20039be9 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -426,4 +426,58 @@ spec: assert_eq!(env.get("TEST_VAR_FROM_MASTER"), Some(&"MASTER".to_string())); assert_eq!(env.get("TEST_VAR_FROM_MRG"), Some(&"MASTER".to_string())); } + + /// A custom log ConfigMap name that is not a valid Kubernetes name is rejected up-front. + #[test] + fn validate_logging_rejects_invalid_custom_config_map_name() { + use stackable_operator::product_logging::spec::{ + ConfigMapLogConfig, ContainerLogConfig, ContainerLogConfigChoice, + CustomContainerLogConfig, + }; + + let logging = Logging { + enable_vector_agent: false, + containers: [( + Container::Hbase, + ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Custom(CustomContainerLogConfig { + custom: ConfigMapLogConfig { + config_map: "invalid ConfigMap name".to_owned(), + }, + })), + }, + )] + .into(), + }; + + assert!(validate_logging(&logging, &None).is_err()); + } + + /// Enabling the Vector agent without a Vector aggregator discovery ConfigMap name fails, but + /// succeeds once a valid name is provided. + #[test] + fn validate_logging_requires_vector_aggregator_when_enabled() { + use stackable_operator::product_logging::spec::{ + AutomaticContainerLogConfig, ContainerLogConfig, ContainerLogConfigChoice, + }; + + let automatic = || ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Automatic( + AutomaticContainerLogConfig::default(), + )), + }; + let logging = Logging { + enable_vector_agent: true, + containers: [ + (Container::Hbase, automatic()), + (Container::Vector, automatic()), + ] + .into(), + }; + + assert!(validate_logging(&logging, &None).is_err()); + + let aggregator = ConfigMapName::from_str("vector-aggregator").expect("valid name"); + assert!(validate_logging(&logging, &Some(aggregator)).is_ok()); + } } diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 6302ea95..540bee68 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -103,6 +103,8 @@ mod tests { kind: HbaseCluster metadata: name: simple-hbase + namespace: default + uid: 12345678-1234-1234-1234-123456789012 spec: image: productVersion: 2.6.4 @@ -124,13 +126,10 @@ mod tests { "#; let hbase: v1alpha1::HbaseCluster = serde_yaml::from_str(input).expect("illegal test input"); - let (merged_config, _) = crate::crd::test_helpers::merged_role_group_config( - &hbase, - &role, - "default", - &hbase.spec.cluster_config.hdfs_config_map_name, - ); - let affinity = merged_config.affinity().clone(); + let validated_cluster = crate::test_utils::validated_cluster_from(&hbase); + let affinity = crate::test_utils::merged_config(&validated_cluster, &role) + .affinity() + .clone(); let mut expected_affinities = vec![WeightedPodAffinityTerm { pod_affinity_term: PodAffinityTerm { diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index f66824a0..e14d21d3 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -807,110 +807,6 @@ impl AnyServiceConfig { } } -#[cfg(test)] -pub(crate) mod test_helpers { - use stackable_operator::{ - config::{fragment::FromFragment, merge::Merge}, - kube::ResourceExt, - role_utils::{GenericRoleConfig, Role}, - v2::{ - jvm_argument_overrides::JvmArgumentOverrides, - role_utils::{JavaCommonConfig, with_validated_config}, - }, - }; - - use super::{ - AnyServiceConfig, HbaseConfig, HbaseConfigFragment, HbaseRole, RegionServerConfig, - RegionServerConfigFragment, v1alpha1, - }; - - /// Test helper: merge + validate a single role group via the production - /// [`with_validated_config`] path (the same merge the controller runs), returning the - /// role-specific [`AnyServiceConfig`] and the merged [`JvmArgumentOverrides`]. - pub(crate) fn merged_role_group_config( - hbase: &v1alpha1::HbaseCluster, - role: &HbaseRole, - role_group: &str, - hdfs_discovery_cm_name: &str, - ) -> (AnyServiceConfig, JvmArgumentOverrides) { - match role { - HbaseRole::Master => merge::( - hbase - .spec - .masters - .as_ref() - .expect("master role must be defined"), - role_group, - HbaseConfigFragment::default_config( - role, - &hbase.name_any(), - hdfs_discovery_cm_name, - ), - AnyServiceConfig::Master, - ), - HbaseRole::RegionServer => merge::( - hbase - .spec - .region_servers - .as_ref() - .expect("region server role must be defined"), - role_group, - RegionServerConfigFragment::default_config( - role, - &hbase.name_any(), - hdfs_discovery_cm_name, - ), - AnyServiceConfig::RegionServer, - ), - HbaseRole::RestServer => merge::( - hbase - .spec - .rest_servers - .as_ref() - .expect("rest server role must be defined"), - role_group, - HbaseConfigFragment::default_config( - role, - &hbase.name_any(), - hdfs_discovery_cm_name, - ), - AnyServiceConfig::RestServer, - ), - } - } - - fn merge( - role: &Role, - role_group: &str, - default_config: Config, - wrap: fn(ValidatedConfig) -> AnyServiceConfig, - ) -> (AnyServiceConfig, JvmArgumentOverrides) - where - Config: Clone + Merge, - ValidatedConfig: FromFragment, - { - let role_group = role - .role_groups - .get(role_group) - .expect("role group must be defined"); - let validated = with_validated_config::< - ValidatedConfig, - JavaCommonConfig, - Config, - GenericRoleConfig, - v1alpha1::HbaseConfigOverrides, - >(role_group, role, &default_config) - .expect("role group config should merge and validate"); - ( - wrap(validated.config.config), - validated - .config - .product_specific_common_config - .jvm_argument_overrides, - ) - } -} - #[cfg(test)] mod tests { use indoc::indoc; @@ -921,7 +817,7 @@ mod tests { #[rstest] #[case("default", false, 1, vec![])] - #[case("groupRegionMover", true, 5, vec!["--some".to_string(), "extra".to_string()])] + #[case("group-region-mover", true, 5, vec!["--some".to_string(), "extra".to_string()])] pub fn test_region_mover_merge( #[case] role_group_name: &str, #[case] run_before_shutdown: bool, @@ -934,6 +830,8 @@ apiVersion: hbase.stackable.tech/v1alpha1 kind: HbaseCluster metadata: name: test-hbase + namespace: default + uid: 12345678-1234-1234-1234-123456789012 spec: image: productVersion: 2.6.4 @@ -955,7 +853,7 @@ spec: roleGroups: default: replicas: 1 - groupRegionMover: + group-region-mover: replicas: 1 config: regionMover: @@ -968,13 +866,11 @@ spec: let hbase: v1alpha1::HbaseCluster = serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); - let hbase_role = HbaseRole::RegionServer; - - let (merged_config, _) = super::test_helpers::merged_role_group_config( - &hbase, - &hbase_role, + let validated_cluster = crate::test_utils::validated_cluster_from(&hbase); + let merged_config = crate::test_utils::merged_config_for( + &validated_cluster, + &HbaseRole::RegionServer, role_group_name, - &hbase.spec.cluster_config.hdfs_config_map_name, ); if let AnyServiceConfig::RegionServer(config) = merged_config { assert_eq!(run_before_shutdown, config.region_mover.run_before_shutdown); diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 1784da58..0eb35a0d 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -40,6 +40,8 @@ use crate::{ mod controller; mod crd; mod hbase_controller; +#[cfg(test)] +mod test_utils; mod webhooks; mod built_info { diff --git a/rust/operator-binary/src/test_utils.rs b/rust/operator-binary/src/test_utils.rs new file mode 100644 index 00000000..f08653e8 --- /dev/null +++ b/rust/operator-binary/src/test_utils.rs @@ -0,0 +1,110 @@ +//! Shared test fixtures. +//! +//! These run the real validation pipeline (`validate_cluster`) against in-memory `HbaseCluster` +//! YAML with a fixed dereferenced ZooKeeper connection (and no OPA), so unit tests can pull +//! merged, validated configs straight from a [`ValidatedCluster`] instead of re-implementing the +//! merge by hand. + +use std::str::FromStr; + +use stackable_operator::{ + commons::networking::DomainName, utils::cluster_info::KubernetesClusterInfo, + v2::types::operator::RoleGroupName, +}; + +use crate::{ + controller::{ + ValidatedCluster, dereference::DereferencedObjects, validate::validate_cluster, + zookeeper::ZookeeperConnectionInformation, + }, + crd::{AnyServiceConfig, HbaseRole, v1alpha1}, +}; + +/// A minimal three-role `HbaseCluster` used to drive the builder/property tests. +pub const MINIMAL_HBASE_YAML: &str = r#" +--- +apiVersion: hbase.stackable.tech/v1alpha1 +kind: HbaseCluster +metadata: + name: hbase + namespace: default + uid: c2c8c5c0-0b5a-4b1e-9f3e-1a2b3c4d5e6f +spec: + image: + productVersion: 2.6.3 + clusterConfig: + hdfsConfigMapName: simple-hdfs + zookeeperConfigMapName: simple-znode + masters: + roleGroups: + default: + replicas: 1 + regionServers: + roleGroups: + default: + replicas: 1 + restServers: + roleGroups: + default: + replicas: 1 +"#; + +/// Parses an `HbaseCluster` from YAML, panicking on invalid input. +pub fn hbase_from_yaml(yaml: &str) -> v1alpha1::HbaseCluster { + serde_yaml::from_str(yaml).expect("invalid test HbaseCluster YAML") +} + +/// The [`MINIMAL_HBASE_YAML`] cluster. +pub fn minimal_hbase() -> v1alpha1::HbaseCluster { + hbase_from_yaml(MINIMAL_HBASE_YAML) +} + +pub fn cluster_info() -> KubernetesClusterInfo { + KubernetesClusterInfo { + cluster_domain: DomainName::try_from("cluster.local").expect("valid domain"), + } +} + +/// Runs the real validation pipeline over `hbase`, with a fixed dereferenced ZooKeeper connection +/// and no OPA. +pub fn validated_cluster_from(hbase: &v1alpha1::HbaseCluster) -> ValidatedCluster { + validate_cluster( + hbase, + "oci.example.org", + &cluster_info(), + DereferencedObjects { + zookeeper_connection_information: ZookeeperConnectionInformation::for_tests(), + hbase_opa_config: None, + }, + ) + .expect("validate should succeed for the test fixture") +} + +/// Runs the real validation pipeline over [`minimal_hbase`]. +pub fn validated_cluster() -> ValidatedCluster { + validated_cluster_from(&minimal_hbase()) +} + +/// Parses a [`RoleGroupName`], panicking on invalid input. +pub fn role_group_name(name: &str) -> RoleGroupName { + RoleGroupName::from_str(name).expect("valid role group name") +} + +/// The merged [`AnyServiceConfig`] for the given `role` and `role_group`. +pub fn merged_config_for<'a>( + validated_cluster: &'a ValidatedCluster, + role: &HbaseRole, + role_group: &str, +) -> &'a AnyServiceConfig { + &validated_cluster.role_group_configs[role][&role_group_name(role_group)] + .config + .config +} + +/// The merged [`AnyServiceConfig`] for the `default` role group of `role`. +pub fn merged_config<'a>( + validated_cluster: &'a ValidatedCluster, + role: &HbaseRole, +) -> &'a AnyServiceConfig { + merged_config_for(validated_cluster, role, "default") +} From d1d7f0c71a635885b0cd74259037af10016dfa75 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 14:15:21 +0200 Subject: [PATCH 43/56] refactor: remove raw cluster references from k8s resources --- .../src/controller/build/kerberos.rs | 13 ++-- .../src/controller/build/resource/service.rs | 62 ++++--------------- .../controller/build/resource/statefulset.rs | 16 ++--- rust/operator-binary/src/controller/mod.rs | 13 ++++ .../src/controller/validate.rs | 4 ++ rust/operator-binary/src/crd/mod.rs | 10 +-- rust/operator-binary/src/hbase_controller.rs | 11 +--- 7 files changed, 54 insertions(+), 75 deletions(-) diff --git a/rust/operator-binary/src/controller/build/kerberos.rs b/rust/operator-binary/src/controller/build/kerberos.rs index aa9cf603..50507078 100644 --- a/rust/operator-binary/src/controller/build/kerberos.rs +++ b/rust/operator-binary/src/controller/build/kerberos.rs @@ -16,7 +16,10 @@ use stackable_operator::{ utils::cluster_info::KubernetesClusterInfo, }; -use crate::crd::{TLS_STORE_DIR, TLS_STORE_PASSWORD, TLS_STORE_VOLUME_NAME, v1alpha1}; +use crate::{ + controller::ValidatedCluster, + crd::{TLS_STORE_DIR, TLS_STORE_PASSWORD, TLS_STORE_VOLUME_NAME, v1alpha1}, +}; /// Mount path of the Kerberos secret volume (keytab + `krb5.conf`). pub const STACKABLE_KERBEROS_DIR: &str = "/stackable/kerberos"; @@ -177,20 +180,20 @@ pub fn kerberos_ssl_client_settings(hbase: &v1alpha1::HbaseCluster) -> BTreeMap< } pub fn add_kerberos_pod_config( - hbase: &v1alpha1::HbaseCluster, + cluster: &ValidatedCluster, metrics_service_name: &str, cb: &mut ContainerBuilder, pb: &mut PodBuilder, requested_secret_lifetime: Duration, ) -> Result<(), Error> { - if let Some(kerberos_secret_class) = hbase.kerberos_secret_class() { + if let Some(kerberos_secret_class) = cluster.cluster_config.kerberos_secret_class.clone() { // Mount keytab let kerberos_secret_operator_volume = SecretOperatorVolumeSourceBuilder::new( kerberos_secret_class, // We need both public (krb5.conf) and private (keytab) parts. SecretClassVolumeProvisionParts::PublicPrivate, ) - .with_service_scope(hbase.name_any()) + .with_service_scope(cluster.name.to_string()) .with_kerberos_service_name(kerberos_service_name()) .with_kerberos_service_name("HTTP") .build() @@ -208,7 +211,7 @@ pub fn add_kerberos_pod_config( cb.add_env_var("KRB5_CONFIG", KRB5_CONFIG_PATH); } - if let Some(https_secret_class) = hbase.https_secret_class() { + if let Some(https_secret_class) = cluster.cluster_config.https_secret_class.clone() { // Mount TLS keystore pb.add_volume( VolumeBuilder::new(TLS_STORE_VOLUME_NAME) diff --git a/rust/operator-binary/src/controller/build/resource/service.rs b/rust/operator-binary/src/controller/build/resource/service.rs index ec752e6a..569b37fa 100644 --- a/rust/operator-binary/src/controller/build/resource/service.rs +++ b/rust/operator-binary/src/controller/build/resource/service.rs @@ -7,7 +7,7 @@ use stackable_operator::{ use crate::{ controller::{RoleGroupName, ValidatedCluster}, - crd::{HbaseRole, v1alpha1}, + crd::HbaseRole, }; /// The rolegroup [`Service`] is a headless service that allows direct access to the instances of a @@ -16,13 +16,12 @@ use crate::{ /// This is mostly useful for internal communication between peers, or for clients that perform /// client-side load balancing. pub fn build_rolegroup_service( - hbase: &v1alpha1::HbaseCluster, cluster: &ValidatedCluster, hbase_role: &HbaseRole, role_group_name: &RoleGroupName, ) -> Service { let ports = hbase_role - .ports(hbase) + .ports(cluster.has_https_enabled()) .into_iter() .map(|(name, value)| ServicePort { name: Some(name), @@ -63,7 +62,6 @@ pub fn build_rolegroup_service( /// The rolegroup metrics [`Service`] is a service that exposes metrics and a prometheus scraping /// label. pub fn build_rolegroup_metrics_service( - hbase: &v1alpha1::HbaseCluster, cluster: &ValidatedCluster, hbase_role: &HbaseRole, role_group_name: &RoleGroupName, @@ -86,7 +84,10 @@ pub fn build_rolegroup_metrics_service( role_group_name, ) .with_labels(prometheus_labels()) - .with_annotations(prometheus_annotations(hbase, hbase_role)) + .with_annotations(prometheus_annotations( + cluster.has_https_enabled(), + hbase_role, + )) .build(), spec: Some(ServiceSpec { // Internal communication does not need to be exposed @@ -115,7 +116,7 @@ fn prometheus_labels() -> Labels { /// These annotations can be used in a ServiceMonitor. /// /// see also -fn prometheus_annotations(hbase: &v1alpha1::HbaseCluster, hbase_role: &HbaseRole) -> Annotations { +fn prometheus_annotations(https_enabled: bool, hbase_role: &HbaseRole) -> Annotations { Annotations::try_from([ ("prometheus.io/path".to_owned(), "/prometheus".to_owned()), ( @@ -124,7 +125,7 @@ fn prometheus_annotations(hbase: &v1alpha1::HbaseCluster, hbase_role: &HbaseRole ), ( "prometheus.io/scheme".to_owned(), - if hbase.has_https_enabled() { + if https_enabled { "https".to_owned() } else { "http".to_owned() @@ -143,50 +144,13 @@ mod test { use crate::test_utils; #[rstest] - #[case("2.6.3", HbaseRole::Master, vec!["master", "ui-http"])] - #[case("2.6.3", HbaseRole::RegionServer, vec!["regionserver", "ui-http"])] - #[case("2.6.3", HbaseRole::RestServer, vec!["rest-http", "ui-http"])] - #[case("2.6.4", HbaseRole::Master, vec!["master", "ui-http"])] - #[case("2.6.4", HbaseRole::RegionServer, vec!["regionserver", "ui-http"])] - #[case("2.6.4", HbaseRole::RestServer, vec!["rest-http", "ui-http"])] - fn test_rolegroup_service_ports( - #[case] hbase_version: &str, - #[case] role: HbaseRole, - #[case] expected_ports: Vec<&str>, - ) { - let input = format!( - " - apiVersion: hbase.stackable.tech/v1alpha1 - kind: HbaseCluster - metadata: - name: hbase - uid: c2e98fc1-6b88-4d11-9381-52530e3f431e - spec: - image: - productVersion: {hbase_version} - clusterConfig: - hdfsConfigMapName: simple-hdfs - zookeeperConfigMapName: simple-znode - masters: - roleGroups: - default: - replicas: 1 - regionServers: - roleGroups: - default: - replicas: 1 - restServers: - roleGroups: - default: - replicas: 1 - " - ); - let hbase: v1alpha1::HbaseCluster = - serde_yaml::from_str(&input).expect("illegal test input"); - + #[case(HbaseRole::Master, vec!["master", "ui-http"])] + #[case(HbaseRole::RegionServer, vec!["regionserver", "ui-http"])] + #[case(HbaseRole::RestServer, vec!["rest-http", "ui-http"])] + fn test_rolegroup_service_ports(#[case] role: HbaseRole, #[case] expected_ports: Vec<&str>) { let cluster = test_utils::validated_cluster(); let role_group_name = test_utils::role_group_name("default"); - let service = build_rolegroup_service(&hbase, &cluster, &role, &role_group_name); + let service = build_rolegroup_service(&cluster, &role, &role_group_name); assert_eq!( expected_ports, diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index d99e40ad..a5e3197b 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -43,7 +43,7 @@ use crate::{ properties::logging::{MAX_HBASE_LOG_FILES_SIZE, STACKABLE_LOG_DIR}, }, }, - crd::{CONFIG_DIR_NAME, HbaseRole, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, v1alpha1}, + crd::{CONFIG_DIR_NAME, HbaseRole, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME}, }; stackable_operator::constant!(VECTOR_CONTAINER_NAME: ContainerName = "vector"); @@ -98,7 +98,6 @@ type Result = std::result::Result; /// The [`Pod`](stackable_operator::k8s_openapi::api::core::v1::Pod)s are accessible through the /// corresponding headless [`Service`](stackable_operator::k8s_openapi::api::core::v1::Service). pub fn build_rolegroup_statefulset( - hbase: &v1alpha1::HbaseCluster, cluster: &ValidatedCluster, hbase_role: &HbaseRole, role_group_name: &RoleGroupName, @@ -109,9 +108,10 @@ pub fn build_rolegroup_statefulset( let merged_config = &validated_rg_config.config.config; let logging = &validated_rg_config.config.logging; let resource_names = cluster.resource_names(hbase_role, role_group_name); + let https_enabled = cluster.has_https_enabled(); let ports = hbase_role - .ports(hbase) + .ports(https_enabled) .into_iter() .map(|(name, value)| ContainerPort { name: Some(name), @@ -123,7 +123,7 @@ pub fn build_rolegroup_statefulset( let probe_template = Probe { tcp_socket: Some(TCPSocketAction { - port: IntOrString::String(hbase_role.data_port_name(hbase)), + port: IntOrString::String(hbase_role.data_port_name(https_enabled)), ..TCPSocketAction::default() }), ..Probe::default() @@ -185,8 +185,8 @@ pub fn build_rolegroup_statefulset( entrypoint = "/stackable/hbase/bin/hbase-entrypoint.sh".to_string(), role = role_name, port = hbase_role.data_port(), - port_name = hbase_role.data_port_name(hbase), - ui_port_name = HbaseRole::ui_port_name(hbase.has_https_enabled()), + port_name = hbase_role.data_port_name(https_enabled), + ui_port_name = HbaseRole::ui_port_name(https_enabled), }]) .add_env_vars(merged_env) // Needed for the `containerdebug` process to log it's tracing information to. @@ -234,7 +234,7 @@ pub fn build_rolegroup_statefulset( .add_volume(Volume { name: HDFS_DISCOVERY_VOLUME_NAME.to_string(), config_map: Some(ConfigMapVolumeSource { - name: hbase.spec.cluster_config.hdfs_config_map_name.clone(), + name: cluster.cluster_config.hdfs_config_map_name.clone(), ..Default::default() }), ..Default::default() @@ -273,7 +273,7 @@ pub fn build_rolegroup_statefulset( add_graceful_shutdown_config(merged_config, &mut pod_builder).context(GracefulShutdownSnafu)?; if cluster.has_kerberos_enabled() { add_kerberos_pod_config( - hbase, + cluster, resource_names.metrics_service_name().as_ref(), &mut hbase_container, &mut pod_builder, diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index 07c1fb99..2749dcb7 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -185,6 +185,11 @@ impl ValidatedCluster { pub fn has_kerberos_enabled(&self) -> bool { self.cluster_config.kerberos_enabled } + + /// Whether HTTPS is enabled for this cluster. + pub fn has_https_enabled(&self) -> bool { + self.cluster_config.https_enabled + } } impl Resource for ValidatedCluster { @@ -240,6 +245,14 @@ pub struct ValidatedClusterConfig { // Pre-resolved OPA connection configuration. pub hbase_opa_config: Option, pub kerberos_enabled: bool, + /// Whether HTTPS is enabled (a TLS `SecretClass` was configured). + pub https_enabled: bool, + /// The Kerberos `SecretClass` name, if Kerberos is enabled. + pub kerberos_secret_class: Option, + /// The HTTPS/TLS `SecretClass` name, if HTTPS is enabled. + pub https_secret_class: Option, + /// The HDFS discovery ConfigMap name the cluster connects to. + pub hdfs_config_map_name: String, // Pre-resolved kerberos properties for hbase-site.xml (empty when kerberos is disabled). pub hbase_site_kerberos_config: BTreeMap, // Pre-resolved kerberos properties for the discovery `hbase-site.xml` exposed to clients diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 20039be9..ee0fa7ec 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -243,6 +243,10 @@ pub fn validate_cluster( zookeeper_connection_information: dereferenced_objects.zookeeper_connection_information, hbase_opa_config: dereferenced_objects.hbase_opa_config, kerberos_enabled: hbase.has_kerberos_enabled(), + https_enabled: hbase.has_https_enabled(), + kerberos_secret_class: hbase.kerberos_secret_class(), + https_secret_class: hbase.https_secret_class(), + hdfs_config_map_name: hbase.spec.cluster_config.hdfs_config_map_name.clone(), hbase_site_kerberos_config, discovery_kerberos_config, ssl_server_settings, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index e14d21d3..439d41e2 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -361,11 +361,11 @@ impl HbaseRole { /// /// Hbase versions 2.6.* will have two ports for each role. The metrics are available on the /// UI port. - pub fn ports(&self, hbase: &v1alpha1::HbaseCluster) -> Vec<(String, u16)> { + pub fn ports(&self, https_enabled: bool) -> Vec<(String, u16)> { vec![ - (self.data_port_name(hbase), self.data_port()), + (self.data_port_name(https_enabled), self.data_port()), ( - Self::ui_port_name(hbase.has_https_enabled()).to_string(), + Self::ui_port_name(https_enabled).to_string(), self.ui_port(), ), ] @@ -379,11 +379,11 @@ impl HbaseRole { } } - pub fn data_port_name(&self, hbase: &v1alpha1::HbaseCluster) -> String { + pub fn data_port_name(&self, https_enabled: bool) -> String { match self { HbaseRole::Master | HbaseRole::RegionServer => self.to_string(), HbaseRole::RestServer => { - if hbase.has_https_enabled() { + if https_enabled { HBASE_REST_PORT_NAME_HTTPS.to_owned() } else { HBASE_REST_PORT_NAME_HTTP.to_owned() diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 33bc9006..26c358d5 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -205,20 +205,15 @@ pub async fn reconcile_hbase( for (hbase_role, role_group_configs) in &validated_cluster.role_group_configs { for (role_group_name, validated_rg_config) in role_group_configs { let rg_service = - build_rolegroup_service(hbase, &validated_cluster, hbase_role, role_group_name); + build_rolegroup_service(&validated_cluster, hbase_role, role_group_name); - let rg_metrics_service = build_rolegroup_metrics_service( - hbase, - &validated_cluster, - hbase_role, - role_group_name, - ); + let rg_metrics_service = + build_rolegroup_metrics_service(&validated_cluster, hbase_role, role_group_name); let rg_configmap = build_rolegroup_config_map(&validated_cluster, hbase_role, role_group_name) .context(BuildRolegroupConfigMapSnafu)?; let rg_statefulset = build_rolegroup_statefulset( - hbase, &validated_cluster, hbase_role, role_group_name, From 4822af4cbfb9a0382b1f8fffe2dfe009e821f157 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 14:24:54 +0200 Subject: [PATCH 44/56] refactor: move listener build out of crd --- .../src/controller/build/resource/listener.rs | 78 +++++++++++++++++++ .../src/controller/build/resource/mod.rs | 1 + .../controller/build/resource/statefulset.rs | 16 ++-- rust/operator-binary/src/crd/mod.rs | 70 +---------------- 4 files changed, 88 insertions(+), 77 deletions(-) create mode 100644 rust/operator-binary/src/controller/build/resource/listener.rs diff --git a/rust/operator-binary/src/controller/build/resource/listener.rs b/rust/operator-binary/src/controller/build/resource/listener.rs new file mode 100644 index 00000000..de937095 --- /dev/null +++ b/rust/operator-binary/src/controller/build/resource/listener.rs @@ -0,0 +1,78 @@ +//! Build the listener `Volume`/`PersistentVolumeClaim` exposing a rolegroup. + +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::pod::volume::{ + ListenerOperatorVolumeSourceBuilder, ListenerOperatorVolumeSourceBuilderError, + ListenerReference, VolumeBuilder, + }, + k8s_openapi::api::core::v1::{PersistentVolumeClaim, Volume}, + kvp::Labels, +}; + +use crate::crd::{AnyServiceConfig, HbaseRole, LISTENER_VOLUME_NAME}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to build listener volume"))] + BuildListenerVolume { + source: ListenerOperatorVolumeSourceBuilderError, + }, + + #[snafu(display("failed to build listener pvc"))] + BuildListenerPvc { + source: ListenerOperatorVolumeSourceBuilderError, + }, +} + +type Result = std::result::Result; + +/// The ephemeral listener [`Volume`] for the masters and region servers, or `None` for the rest +/// servers (which use a [`PersistentVolumeClaim`] instead, see [`build_listener_pvc`]). +pub fn build_listener_volume( + role: &HbaseRole, + merged_config: &AnyServiceConfig, + recommended_labels: &Labels, +) -> Result> { + let volume = match role { + // Master and regionservers should use ephemeral listener volumes + // since clients pull the latest address from ZooKeeper + HbaseRole::Master | HbaseRole::RegionServer => Some( + VolumeBuilder::new(LISTENER_VOLUME_NAME) + .ephemeral( + ListenerOperatorVolumeSourceBuilder::new( + &ListenerReference::ListenerClass( + merged_config.listener_class().to_string(), + ), + recommended_labels, + ) + .build_ephemeral() + .context(BuildListenerVolumeSnafu)?, + ) + .build(), + ), + HbaseRole::RestServer => None, + }; + Ok(volume) +} + +/// The listener [`PersistentVolumeClaim`] template for the rest servers, or `None` for the masters +/// and region servers (which use an ephemeral [`Volume`] instead, see [`build_listener_volume`]). +pub fn build_listener_pvc( + role: &HbaseRole, + merged_config: &AnyServiceConfig, + recommended_labels: &Labels, +) -> Result>> { + let pvc = match role { + HbaseRole::Master | HbaseRole::RegionServer => None, + HbaseRole::RestServer => Some(vec![ + ListenerOperatorVolumeSourceBuilder::new( + &ListenerReference::ListenerClass(merged_config.listener_class().to_string()), + recommended_labels, + ) + .build_pvc(LISTENER_VOLUME_NAME.to_string()) + .context(BuildListenerPvcSnafu)?, + ]), + }; + Ok(pvc) +} diff --git a/rust/operator-binary/src/controller/build/resource/mod.rs b/rust/operator-binary/src/controller/build/resource/mod.rs index 2aa3bec6..f9b822cd 100644 --- a/rust/operator-binary/src/controller/build/resource/mod.rs +++ b/rust/operator-binary/src/controller/build/resource/mod.rs @@ -3,6 +3,7 @@ pub mod config_map; pub mod discovery; +pub mod listener; pub mod pdb; pub mod service; pub mod statefulset; diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index a5e3197b..b1d74b25 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -85,10 +85,10 @@ pub enum Error { }, #[snafu(display("failed to build listener volume"))] - ListenerVolume { source: crate::crd::Error }, + ListenerVolume { source: super::listener::Error }, #[snafu(display("failed to build listener persistent volume claim"))] - ListenerPersistentVolumeClaim { source: crate::crd::Error }, + ListenerPersistentVolumeClaim { source: super::listener::Error }, } type Result = std::result::Result; @@ -298,13 +298,13 @@ pub fn build_rolegroup_statefulset( )); } - let listener_pvc = hbase_role - .listener_pvc(merged_config, &recommended_labels) - .context(ListenerPersistentVolumeClaimSnafu)?; + let listener_pvc = + super::listener::build_listener_pvc(hbase_role, merged_config, &recommended_labels) + .context(ListenerPersistentVolumeClaimSnafu)?; - if let Some(listener_volume) = hbase_role - .listener_volume(merged_config, &recommended_labels) - .context(ListenerVolumeSnafu)? + if let Some(listener_volume) = + super::listener::build_listener_volume(hbase_role, merged_config, &recommended_labels) + .context(ListenerVolumeSnafu)? { pod_builder .add_volume(listener_volume) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 439d41e2..59aef86d 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,12 +1,7 @@ use security::AuthenticationConfig; use serde::{Deserialize, Serialize}; use shell_escape::escape; -use snafu::{ResultExt, Snafu}; use stackable_operator::{ - builder::pod::volume::{ - ListenerOperatorVolumeSourceBuilder, ListenerOperatorVolumeSourceBuilderError, - ListenerReference, VolumeBuilder, - }, commons::{ affinity::StackableAffinity, cluster_operation::ClusterOperation, @@ -21,12 +16,8 @@ use stackable_operator::{ merge::{Atomic, Merge}, }, deep_merger::ObjectOverrides, - k8s_openapi::{ - api::core::v1::{PersistentVolumeClaim, Volume}, - apimachinery::pkg::api::resource::Quantity, - }, + k8s_openapi::apimachinery::pkg::api::resource::Quantity, kube::CustomResource, - kvp::Labels, product_logging::{self, spec::Logging}, role_utils::{GenericRoleConfig, Role}, schemars::{self, JsonSchema}, @@ -103,19 +94,6 @@ pub type RegionServerRoleType = Role< pub type RestServerRoleType = Role; -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("failed to build listener volume"))] - BuildListenerVolume { - source: ListenerOperatorVolumeSourceBuilderError, - }, - - #[snafu(display("failed to build listener pvc"))] - BuildListenerPvc { - source: ListenerOperatorVolumeSourceBuilderError, - }, -} - #[versioned( version(name = "v1alpha1"), crates( @@ -311,52 +289,6 @@ impl HbaseRole { } } - pub fn listener_volume( - &self, - merged_config: &AnyServiceConfig, - recommended_labels: &Labels, - ) -> Result, Error> { - let volume = match &self { - // Master and regionservers should use ephemeral listener volumes - // since clients pull the latest address from ZooKeeper - HbaseRole::Master | HbaseRole::RegionServer => Some( - VolumeBuilder::new(LISTENER_VOLUME_NAME) - .ephemeral( - ListenerOperatorVolumeSourceBuilder::new( - &ListenerReference::ListenerClass( - merged_config.listener_class().to_string(), - ), - recommended_labels, - ) - .build_ephemeral() - .context(BuildListenerVolumeSnafu)?, - ) - .build(), - ), - HbaseRole::RestServer => None, - }; - Ok(volume) - } - - pub fn listener_pvc( - &self, - merged_config: &AnyServiceConfig, - recommended_labels: &Labels, - ) -> Result>, Error> { - let pvc = match &self { - HbaseRole::Master | HbaseRole::RegionServer => None, - HbaseRole::RestServer => Some(vec![ - ListenerOperatorVolumeSourceBuilder::new( - &ListenerReference::ListenerClass(merged_config.listener_class().to_string()), - recommended_labels, - ) - .build_pvc(LISTENER_VOLUME_NAME.to_string()) - .context(BuildListenerPvcSnafu)?, - ]), - }; - Ok(pvc) - } - /// Returns required port name and port number tuples depending on the role. /// /// Hbase versions 2.6.* will have two ports for each role. The metrics are available on the From 3817a5f0e383c8bfa261e0dae58b54092b159d61 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 14:34:04 +0200 Subject: [PATCH 45/56] refactor: move role and port constants to builder --- .../src/controller/build/mod.rs | 1 + .../src/controller/build/role.rs | 91 +++++++++++++++++++ rust/operator-binary/src/crd/mod.rs | 80 ---------------- 3 files changed, 92 insertions(+), 80 deletions(-) create mode 100644 rust/operator-binary/src/controller/build/role.rs diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs index 794be0af..b1bd5742 100644 --- a/rust/operator-binary/src/controller/build/mod.rs +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -7,3 +7,4 @@ pub mod kerberos; pub mod opa; pub mod properties; pub mod resource; +pub mod role; diff --git a/rust/operator-binary/src/controller/build/role.rs b/rust/operator-binary/src/controller/build/role.rs new file mode 100644 index 00000000..a6de1986 --- /dev/null +++ b/rust/operator-binary/src/controller/build/role.rs @@ -0,0 +1,91 @@ +//! Build-side behaviour of [`HbaseRole`]: the `bin/hbase` CLI subcommand name and the +//! port name/number mapping. These are operator/product knowledge, kept out of the `crd` module +//! so it stays a pure API definition. + +use crate::crd::{ + HBASE_MASTER_METRICS_PORT, HBASE_MASTER_PORT, HBASE_MASTER_UI_PORT, + HBASE_REGIONSERVER_METRICS_PORT, HBASE_REGIONSERVER_PORT, HBASE_REGIONSERVER_UI_PORT, + HBASE_REST_METRICS_PORT, HBASE_REST_PORT, HBASE_REST_UI_PORT, HbaseRole, +}; + +const HBASE_UI_PORT_NAME_HTTP: &str = "ui-http"; +const HBASE_UI_PORT_NAME_HTTPS: &str = "ui-https"; +const HBASE_REST_PORT_NAME_HTTP: &str = "rest-http"; +const HBASE_REST_PORT_NAME_HTTPS: &str = "rest-https"; +const HBASE_METRICS_PORT_NAME: &str = "metrics"; + +impl HbaseRole { + /// Returns the name of the role as it is needed by the `bin/hbase {cli_role_name} start` command. + pub fn cli_role_name(&self) -> String { + match self { + HbaseRole::Master | HbaseRole::RegionServer => self.to_string(), + // Of course it is not called "restserver", so we need to have this match + // instead of just letting the Display impl do it's thing ;P + HbaseRole::RestServer => "rest".to_string(), + } + } + + /// Returns required port name and port number tuples depending on the role. + /// + /// Hbase versions 2.6.* will have two ports for each role. The metrics are available on the + /// UI port. + pub fn ports(&self, https_enabled: bool) -> Vec<(String, u16)> { + vec![ + (self.data_port_name(https_enabled), self.data_port()), + ( + Self::ui_port_name(https_enabled).to_string(), + self.ui_port(), + ), + ] + } + + pub fn data_port(&self) -> u16 { + match self { + HbaseRole::Master => HBASE_MASTER_PORT, + HbaseRole::RegionServer => HBASE_REGIONSERVER_PORT, + HbaseRole::RestServer => HBASE_REST_PORT, + } + } + + pub fn data_port_name(&self, https_enabled: bool) -> String { + match self { + HbaseRole::Master | HbaseRole::RegionServer => self.to_string(), + HbaseRole::RestServer => { + if https_enabled { + HBASE_REST_PORT_NAME_HTTPS.to_owned() + } else { + HBASE_REST_PORT_NAME_HTTP.to_owned() + } + } + } + } + + pub fn ui_port(&self) -> u16 { + match self { + HbaseRole::Master => HBASE_MASTER_UI_PORT, + HbaseRole::RegionServer => HBASE_REGIONSERVER_UI_PORT, + HbaseRole::RestServer => HBASE_REST_UI_PORT, + } + } + + /// Name of the port used by the Web UI, which depends on HTTPS usage + pub fn ui_port_name(has_https_enabled: bool) -> &'static str { + if has_https_enabled { + HBASE_UI_PORT_NAME_HTTPS + } else { + HBASE_UI_PORT_NAME_HTTP + } + } + + pub fn metrics_port(&self) -> u16 { + match self { + HbaseRole::Master => HBASE_MASTER_METRICS_PORT, + HbaseRole::RegionServer => HBASE_REGIONSERVER_METRICS_PORT, + HbaseRole::RestServer => HBASE_REST_METRICS_PORT, + } + } + + pub fn metrics_port_name() -> &'static str { + HBASE_METRICS_PORT_NAME + } +} diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 59aef86d..68f00ab8 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -51,12 +51,6 @@ pub const HBASE_CLUSTER_DISTRIBUTED: &str = "hbase.cluster.distributed"; pub const HBASE_ROOTDIR: &str = "hbase.rootdir"; const DEFAULT_HBASE_ROOTDIR: &str = "/hbase"; -const HBASE_UI_PORT_NAME_HTTP: &str = "ui-http"; -const HBASE_UI_PORT_NAME_HTTPS: &str = "ui-https"; -const HBASE_REST_PORT_NAME_HTTP: &str = "rest-http"; -const HBASE_REST_PORT_NAME_HTTPS: &str = "rest-https"; -const HBASE_METRICS_PORT_NAME: &str = "metrics"; - pub const HBASE_MASTER_PORT: u16 = 16000; // HBase always uses 16010, regardless of http or https. On 2024-01-17 we decided in Arch-meeting that we want to stick // the port numbers to what the product is doing, so we get the least surprise for users - even when this means we have @@ -278,80 +272,6 @@ impl HbaseRole { const DEFAULT_REST_SECRET_LIFETIME: Duration = Duration::from_days_unchecked(1); const DEFAULT_REST_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(5); - - /// Returns the name of the role as it is needed by the `bin/hbase {cli_role_name} start` command. - pub fn cli_role_name(&self) -> String { - match self { - HbaseRole::Master | HbaseRole::RegionServer => self.to_string(), - // Of course it is not called "restserver", so we need to have this match - // instead of just letting the Display impl do it's thing ;P - HbaseRole::RestServer => "rest".to_string(), - } - } - - /// Returns required port name and port number tuples depending on the role. - /// - /// Hbase versions 2.6.* will have two ports for each role. The metrics are available on the - /// UI port. - pub fn ports(&self, https_enabled: bool) -> Vec<(String, u16)> { - vec![ - (self.data_port_name(https_enabled), self.data_port()), - ( - Self::ui_port_name(https_enabled).to_string(), - self.ui_port(), - ), - ] - } - - pub fn data_port(&self) -> u16 { - match self { - HbaseRole::Master => HBASE_MASTER_PORT, - HbaseRole::RegionServer => HBASE_REGIONSERVER_PORT, - HbaseRole::RestServer => HBASE_REST_PORT, - } - } - - pub fn data_port_name(&self, https_enabled: bool) -> String { - match self { - HbaseRole::Master | HbaseRole::RegionServer => self.to_string(), - HbaseRole::RestServer => { - if https_enabled { - HBASE_REST_PORT_NAME_HTTPS.to_owned() - } else { - HBASE_REST_PORT_NAME_HTTP.to_owned() - } - } - } - } - - pub fn ui_port(&self) -> u16 { - match self { - HbaseRole::Master => HBASE_MASTER_UI_PORT, - HbaseRole::RegionServer => HBASE_REGIONSERVER_UI_PORT, - HbaseRole::RestServer => HBASE_REST_UI_PORT, - } - } - - /// Name of the port used by the Web UI, which depends on HTTPS usage - pub fn ui_port_name(has_https_enabled: bool) -> &'static str { - if has_https_enabled { - HBASE_UI_PORT_NAME_HTTPS - } else { - HBASE_UI_PORT_NAME_HTTP - } - } - - pub fn metrics_port(&self) -> u16 { - match self { - HbaseRole::Master => HBASE_MASTER_METRICS_PORT, - HbaseRole::RegionServer => HBASE_REGIONSERVER_METRICS_PORT, - HbaseRole::RestServer => HBASE_REST_METRICS_PORT, - } - } - - pub fn metrics_port_name() -> &'static str { - HBASE_METRICS_PORT_NAME - } } fn default_resources(role: &HbaseRole) -> ResourcesFragment { From fcebf3d90b0d22286c362b9e1c5289e0a7a66734 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 14:44:05 +0200 Subject: [PATCH 46/56] refactor: move region_mover to builder --- .../src/controller/build/mod.rs | 1 + .../src/controller/build/region_mover.rs | 68 +++++++++++++++++++ rust/operator-binary/src/crd/mod.rs | 67 ++---------------- 3 files changed, 73 insertions(+), 63 deletions(-) create mode 100644 rust/operator-binary/src/controller/build/region_mover.rs diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs index b1bd5742..e642a741 100644 --- a/rust/operator-binary/src/controller/build/mod.rs +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -6,5 +6,6 @@ pub mod jvm; pub mod kerberos; pub mod opa; pub mod properties; +pub mod region_mover; pub mod resource; pub mod role; diff --git a/rust/operator-binary/src/controller/build/region_mover.rs b/rust/operator-binary/src/controller/build/region_mover.rs new file mode 100644 index 00000000..9c3ead75 --- /dev/null +++ b/rust/operator-binary/src/controller/build/region_mover.rs @@ -0,0 +1,68 @@ +//! Build-side region-mover logic for [`AnyServiceConfig`]: derives the `bin/hbase` region-mover +//! CLI arguments from the merged config. Kept out of the `crd` module so it stays a pure API +//! definition. + +use shell_escape::escape; +use stackable_operator::shared::time::Duration; + +use crate::crd::AnyServiceConfig; + +const DEFAULT_REGION_MOVER_TIMEOUT: Duration = Duration::from_minutes_unchecked(59); +const DEFAULT_REGION_MOVER_DELTA_TO_SHUTDOWN: Duration = Duration::from_minutes_unchecked(1); + +impl AnyServiceConfig { + /// Returns command line arguments to pass on to the region mover tool. + /// The following arguments are excluded because they are already part of the + /// hbase-entrypoint.sh script. + /// The most important argument, '--regionserverhost' can only be computed on the Pod + /// because it contains the pod's hostname. + /// + /// Returns an empty string if the region mover is disabled or any other role is "self". + pub fn region_mover_args(&self) -> String { + match self { + AnyServiceConfig::RegionServer(config) => { + if config.region_mover.run_before_shutdown { + let timeout = config + .graceful_shutdown_timeout + .map(|d| { + if d.as_secs() <= DEFAULT_REGION_MOVER_DELTA_TO_SHUTDOWN.as_secs() { + d.as_secs() + } else { + d.as_secs() - DEFAULT_REGION_MOVER_DELTA_TO_SHUTDOWN.as_secs() + } + }) + .unwrap_or(DEFAULT_REGION_MOVER_TIMEOUT.as_secs()); + let mut command = vec![ + "--maxthreads".to_string(), + config.region_mover.max_threads.to_string(), + "--timeout".to_string(), + timeout.to_string(), + ]; + if !config.region_mover.ack { + command.push("--noack".to_string()); + } + + command.extend( + config + .region_mover + .cli_opts + .iter() + .flat_map(|o| o.additional_mover_options.clone()) + .map(|s| escape(std::borrow::Cow::Borrowed(&s)).to_string()), + ); + command.join(" ") + } else { + "".to_string() + } + } + _ => "".to_string(), + } + } + + pub fn run_region_mover(&self) -> bool { + match self { + AnyServiceConfig::RegionServer(config) => config.region_mover.run_before_shutdown, + _ => false, + } + } +} diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 68f00ab8..2cac8596 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,6 +1,5 @@ use security::AuthenticationConfig; use serde::{Deserialize, Serialize}; -use shell_escape::escape; use stackable_operator::{ commons::{ affinity::StackableAffinity, @@ -66,9 +65,6 @@ pub const HBASE_REST_METRICS_PORT: u16 = 8085; pub const LISTENER_VOLUME_NAME: &str = "listener"; pub const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; -const DEFAULT_REGION_MOVER_TIMEOUT: Duration = Duration::from_minutes_unchecked(59); -const DEFAULT_REGION_MOVER_DELTA_TO_SHUTDOWN: Duration = Duration::from_minutes_unchecked(1); - const DEFAULT_LISTENER_CLASS: &str = "cluster-internal"; fn default_hbase_rootdir() -> String { @@ -462,17 +458,17 @@ pub struct HbaseConfig { )] pub struct RegionMover { /// Move local regions to other servers before terminating a region server's pod. - run_before_shutdown: bool, + pub run_before_shutdown: bool, /// Maximum number of threads to use for moving regions. - max_threads: u16, + pub max_threads: u16, /// If enabled (default), the region mover will confirm that regions are available on the /// source as well as the target pods before and after the move. - ack: bool, + pub ack: bool, #[fragment_attrs(serde(flatten))] - cli_opts: Option, + pub cli_opts: Option, } #[derive(Clone, Debug, Eq, Deserialize, JsonSchema, PartialEq, Serialize)] @@ -602,61 +598,6 @@ impl AnyServiceConfig { AnyServiceConfig::RestServer(config) => config.hbase_rootdir.clone(), } } - - /// Returns command line arguments to pass on to the region mover tool. - /// The following arguments are excluded because they are already part of the - /// hbase-entrypoint.sh script. - /// The most important argument, '--regionserverhost' can only be computed on the Pod - /// because it contains the pod's hostname. - /// - /// Returns an empty string if the region mover is disabled or any other role is "self". - pub fn region_mover_args(&self) -> String { - match self { - AnyServiceConfig::RegionServer(config) => { - if config.region_mover.run_before_shutdown { - let timeout = config - .graceful_shutdown_timeout - .map(|d| { - if d.as_secs() <= DEFAULT_REGION_MOVER_DELTA_TO_SHUTDOWN.as_secs() { - d.as_secs() - } else { - d.as_secs() - DEFAULT_REGION_MOVER_DELTA_TO_SHUTDOWN.as_secs() - } - }) - .unwrap_or(DEFAULT_REGION_MOVER_TIMEOUT.as_secs()); - let mut command = vec![ - "--maxthreads".to_string(), - config.region_mover.max_threads.to_string(), - "--timeout".to_string(), - timeout.to_string(), - ]; - if !config.region_mover.ack { - command.push("--noack".to_string()); - } - - command.extend( - config - .region_mover - .cli_opts - .iter() - .flat_map(|o| o.additional_mover_options.clone()) - .map(|s| escape(std::borrow::Cow::Borrowed(&s)).to_string()), - ); - command.join(" ") - } else { - "".to_string() - } - } - _ => "".to_string(), - } - } - - pub fn run_region_mover(&self) -> bool { - match self { - AnyServiceConfig::RegionServer(config) => config.region_mover.run_before_shutdown, - _ => false, - } - } } #[cfg(test)] From e40a044bb51e70046aa391876f5c0d368720f7a3 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 16:56:40 +0200 Subject: [PATCH 47/56] fix(tests): add vector templates & fix vars --- tests/templates/kuttl/smoke/30-assert.yaml.j2 | 102 ++++++-- tests/templates/kuttl/smoke/32-assert.yaml | 60 ++++- .../32_cm-test-hbase-master-default.yaml.j2 | 238 +++++++++++++++++- ...cm-test-hbase-regionserver-default.yaml.j2 | 238 +++++++++++++++++- ...2_cm-test-hbase-restserver-default.yaml.j2 | 238 +++++++++++++++++- 5 files changed, 840 insertions(+), 36 deletions(-) diff --git a/tests/templates/kuttl/smoke/30-assert.yaml.j2 b/tests/templates/kuttl/smoke/30-assert.yaml.j2 index dc9d64ea..067fc721 100644 --- a/tests/templates/kuttl/smoke/30-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/30-assert.yaml.j2 @@ -321,12 +321,14 @@ spec: name: listener {% if lookup('env', 'VECTOR_AGGREGATOR') %} - args: - - | + - |- mkdir --parents /stackable/log/_vector-state # Vector will ignore SIGTERM (as PID != 1) and must be shut down by writing a shutdown trigger file - vector --config /stackable/config/vector.yaml & vector_pid=$! + vector & vector_pid=$! if [ ! -f "/stackable/log/_vector/shutdown" ]; then - mkdir -p /stackable/log/_vector && inotifywait -qq --event create /stackable/log/_vector; fi + mkdir -p /stackable/log/_vector + inotifywait -qq --event create /stackable/log/_vector; + fi sleep 1 kill $vector_pid command: @@ -336,13 +338,31 @@ spec: - pipefail - -c env: - - name: VECTOR_LOG - value: info + - name: CLUSTER_NAME + value: test-hbase + - name: DATA_DIR + value: /stackable/log/_vector-state + - name: LOG_DIR + value: /stackable/log + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ROLE_GROUP_NAME + value: default + - name: ROLE_NAME + value: master - name: VECTOR_AGGREGATOR_ADDRESS valueFrom: configMapKeyRef: key: ADDRESS name: vector-aggregator-discovery + - name: VECTOR_CONFIG_YAML + value: /stackable/config/vector.yaml + - name: VECTOR_FILE_LOG_LEVEL + value: info + - name: VECTOR_LOG + value: info imagePullPolicy: IfNotPresent name: vector resources: @@ -353,8 +373,10 @@ spec: cpu: 250m memory: 128Mi volumeMounts: - - mountPath: /stackable/config + - mountPath: /stackable/config/vector.yaml name: hbase-config + readOnly: true + subPath: vector.yaml - mountPath: /stackable/log name: log {% endif %} @@ -534,12 +556,14 @@ spec: name: listener {% if lookup('env', 'VECTOR_AGGREGATOR') %} - args: - - | + - |- mkdir --parents /stackable/log/_vector-state # Vector will ignore SIGTERM (as PID != 1) and must be shut down by writing a shutdown trigger file - vector --config /stackable/config/vector.yaml & vector_pid=$! + vector & vector_pid=$! if [ ! -f "/stackable/log/_vector/shutdown" ]; then - mkdir -p /stackable/log/_vector && inotifywait -qq --event create /stackable/log/_vector; fi + mkdir -p /stackable/log/_vector + inotifywait -qq --event create /stackable/log/_vector; + fi sleep 1 kill $vector_pid command: @@ -549,13 +573,31 @@ spec: - pipefail - -c env: - - name: VECTOR_LOG - value: info + - name: CLUSTER_NAME + value: test-hbase + - name: DATA_DIR + value: /stackable/log/_vector-state + - name: LOG_DIR + value: /stackable/log + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ROLE_GROUP_NAME + value: default + - name: ROLE_NAME + value: regionserver - name: VECTOR_AGGREGATOR_ADDRESS valueFrom: configMapKeyRef: key: ADDRESS name: vector-aggregator-discovery + - name: VECTOR_CONFIG_YAML + value: /stackable/config/vector.yaml + - name: VECTOR_FILE_LOG_LEVEL + value: info + - name: VECTOR_LOG + value: info imagePullPolicy: IfNotPresent name: vector resources: @@ -566,8 +608,10 @@ spec: cpu: 250m memory: 128Mi volumeMounts: - - mountPath: /stackable/config + - mountPath: /stackable/config/vector.yaml name: hbase-config + readOnly: true + subPath: vector.yaml - mountPath: /stackable/log name: log {% endif %} @@ -743,12 +787,14 @@ spec: name: listener {% if lookup('env', 'VECTOR_AGGREGATOR') %} - args: - - | + - |- mkdir --parents /stackable/log/_vector-state # Vector will ignore SIGTERM (as PID != 1) and must be shut down by writing a shutdown trigger file - vector --config /stackable/config/vector.yaml & vector_pid=$! + vector & vector_pid=$! if [ ! -f "/stackable/log/_vector/shutdown" ]; then - mkdir -p /stackable/log/_vector && inotifywait -qq --event create /stackable/log/_vector; fi + mkdir -p /stackable/log/_vector + inotifywait -qq --event create /stackable/log/_vector; + fi sleep 1 kill $vector_pid command: @@ -758,13 +804,31 @@ spec: - pipefail - -c env: - - name: VECTOR_LOG - value: info + - name: CLUSTER_NAME + value: test-hbase + - name: DATA_DIR + value: /stackable/log/_vector-state + - name: LOG_DIR + value: /stackable/log + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ROLE_GROUP_NAME + value: default + - name: ROLE_NAME + value: restserver - name: VECTOR_AGGREGATOR_ADDRESS valueFrom: configMapKeyRef: key: ADDRESS name: vector-aggregator-discovery + - name: VECTOR_CONFIG_YAML + value: /stackable/config/vector.yaml + - name: VECTOR_FILE_LOG_LEVEL + value: info + - name: VECTOR_LOG + value: info imagePullPolicy: IfNotPresent name: vector resources: @@ -775,8 +839,10 @@ spec: cpu: 250m memory: 128Mi volumeMounts: - - mountPath: /stackable/config + - mountPath: /stackable/config/vector.yaml name: hbase-config + readOnly: true + subPath: vector.yaml - mountPath: /stackable/log name: log {% endif %} diff --git a/tests/templates/kuttl/smoke/32-assert.yaml b/tests/templates/kuttl/smoke/32-assert.yaml index 230ffd91..72c53b03 100644 --- a/tests/templates/kuttl/smoke/32-assert.yaml +++ b/tests/templates/kuttl/smoke/32-assert.yaml @@ -1,4 +1,19 @@ --- +# Snapshot the full `.data` of each operator-managed rolegroup ConfigMap. +# Any code change that alters a rendered config value will fail these diffs. +# +# How the comparison stays robust: +# * The snapshot files (32_cm-*.yaml) are read straight from disk, so no shell +# `$`-expansion happens: the literal ${NAMESPACE}/${DATA_DIR}/... that the +# operator writes into vector.yaml (Vector resolves them at runtime) survive +# verbatim. Only the two genuinely dynamic values use sentinels: +# - __NAMESPACE__ -> substituted with the random kuttl namespace via sed. +# - __ZNODE__ -> the zk operator mints a fresh znode UUID per install, +# so we normalize the *actual* side instead of guessing +# it (see the sed on the `actual=` pipeline). +# * Both sides are canonicalized to JSON with `yq -o=json`, so block-scalar +# style and trailing-newline differences never cause false positives. +# * On drift we print a `diff -u` so the failing keys are obvious in the logs. apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 60 @@ -7,34 +22,55 @@ commands: # ConfigMap data snapshot: test-hbase-master-default # - script: | - export ZNODE_PATH=$(kubectl -n $NAMESPACE get zookeeperznode test-znode -o jsonpath='{.status.znodePath}') - envsubst '$NAMESPACE $ZNODE_PATH' < 32_cm-test-hbase-master-default.yaml > /tmp/$NAMESPACE-expected.yaml - kubectl -n $NAMESPACE get cm test-hbase-master-default -o yaml | yq '.data' > /tmp/$NAMESPACE-actual.yaml - if ! diff /tmp/$NAMESPACE-expected.yaml /tmp/$NAMESPACE-actual.yaml; then + expected=$(sed "s|__NAMESPACE__|$NAMESPACE|g" 32_cm-test-hbase-master-default.yaml | yq -o=json) + actual=$(kubectl -n $NAMESPACE get cm test-hbase-master-default -o yaml \ + | yq -o=json '.data' \ + | sed -E 's|/znode-[a-f0-9-]+|/znode-__ZNODE__|') + expected_file=$(mktemp) && actual_file=$(mktemp) + printf '%s\n' "$expected" > "$expected_file" + printf '%s\n' "$actual" > "$actual_file" + if ! diff_out=$(diff -u "$expected_file" "$actual_file"); then echo "ERROR: ConfigMap test-hbase-master-default data drifted from snapshot." + printf '%s\n' "$diff_out" + rm -f "$expected_file" "$actual_file" exit 1 fi + rm -f "$expected_file" "$actual_file" # # ConfigMap data snapshot: test-hbase-regionserver-default # - script: | - export ZNODE_PATH=$(kubectl -n $NAMESPACE get zookeeperznode test-znode -o jsonpath='{.status.znodePath}') - envsubst '$NAMESPACE $ZNODE_PATH' < 32_cm-test-hbase-regionserver-default.yaml > /tmp/$NAMESPACE-expected.yaml - kubectl -n $NAMESPACE get cm test-hbase-regionserver-default -o yaml | yq '.data' > /tmp/$NAMESPACE-actual.yaml - if ! diff /tmp/$NAMESPACE-expected.yaml /tmp/$NAMESPACE-actual.yaml; then + expected=$(sed "s|__NAMESPACE__|$NAMESPACE|g" 32_cm-test-hbase-regionserver-default.yaml | yq -o=json) + actual=$(kubectl -n $NAMESPACE get cm test-hbase-regionserver-default -o yaml \ + | yq -o=json '.data' \ + | sed -E 's|/znode-[a-f0-9-]+|/znode-__ZNODE__|') + expected_file=$(mktemp) && actual_file=$(mktemp) + printf '%s\n' "$expected" > "$expected_file" + printf '%s\n' "$actual" > "$actual_file" + if ! diff_out=$(diff -u "$expected_file" "$actual_file"); then echo "ERROR: ConfigMap test-hbase-regionserver-default data drifted from snapshot." + printf '%s\n' "$diff_out" + rm -f "$expected_file" "$actual_file" exit 1 fi + rm -f "$expected_file" "$actual_file" # # ConfigMap data snapshot: test-hbase-restserver-default # - script: | - export ZNODE_PATH=$(kubectl -n $NAMESPACE get zookeeperznode test-znode -o jsonpath='{.status.znodePath}') - envsubst '$NAMESPACE $ZNODE_PATH' < 32_cm-test-hbase-restserver-default.yaml > /tmp/$NAMESPACE-expected.yaml - kubectl -n $NAMESPACE get cm test-hbase-restserver-default -o yaml | yq '.data' > /tmp/$NAMESPACE-actual.yaml - if ! diff /tmp/$NAMESPACE-expected.yaml /tmp/$NAMESPACE-actual.yaml; then + expected=$(sed "s|__NAMESPACE__|$NAMESPACE|g" 32_cm-test-hbase-restserver-default.yaml | yq -o=json) + actual=$(kubectl -n $NAMESPACE get cm test-hbase-restserver-default -o yaml \ + | yq -o=json '.data' \ + | sed -E 's|/znode-[a-f0-9-]+|/znode-__ZNODE__|') + expected_file=$(mktemp) && actual_file=$(mktemp) + printf '%s\n' "$expected" > "$expected_file" + printf '%s\n' "$actual" > "$actual_file" + if ! diff_out=$(diff -u "$expected_file" "$actual_file"); then echo "ERROR: ConfigMap test-hbase-restserver-default data drifted from snapshot." + printf '%s\n' "$diff_out" + rm -f "$expected_file" "$actual_file" exit 1 fi + rm -f "$expected_file" "$actual_file" diff --git a/tests/templates/kuttl/smoke/32_cm-test-hbase-master-default.yaml.j2 b/tests/templates/kuttl/smoke/32_cm-test-hbase-master-default.yaml.j2 index adadeee8..fb9b7c08 100644 --- a/tests/templates/kuttl/smoke/32_cm-test-hbase-master-default.yaml.j2 +++ b/tests/templates/kuttl/smoke/32_cm-test-hbase-master-default.yaml.j2 @@ -52,7 +52,7 @@ hbase-site.xml: |- hbase.zookeeper.quorum - test-zk-server.$NAMESPACE.svc.cluster.local:2282 + test-zk-server.__NAMESPACE__.svc.cluster.local:2282 phoenix.log.saltBuckets @@ -60,7 +60,7 @@ hbase-site.xml: |- zookeeper.znode.parent - $ZNODE_PATH/hbase + /znode-__ZNODE__/hbase log4j2.properties: |- @@ -103,3 +103,237 @@ ssl-server.xml: |- +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +vector.yaml: | + --- + data_dir: ${DATA_DIR} + + log_schema: + host_key: pod + + sources: + # Reads the internal Vector logs + vector: + type: internal_logs + + files_stdout: + type: file + include: + - ${LOG_DIR}/*/*.stdout.log + + files_stderr: + type: file + include: + - ${LOG_DIR}/*/*.stderr.log + + files_log4j2: + type: file + include: + - ${LOG_DIR}/*/*.log4j2.xml + line_delimiter: "\r\n" + + transforms: + processed_files_stdout: + inputs: + - files_stdout + type: remap + source: | + .logger = "ROOT" + .level = "INFO" + + processed_files_stderr: + inputs: + - files_stderr + type: remap + source: | + .logger = "ROOT" + .level = "ERROR" + + processed_files_log4j2: + inputs: + - files_log4j2 + type: remap + source: | + raw_message = string!(.message) + + .timestamp = now() + .logger = "" + .level = "INFO" + .message = "" + .errors = [] + + event = {} + parsed_event, err = parse_xml(raw_message) + if err != null { + error = "XML not parsable: " + err + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + if !is_object(parsed_event.Event) { + error = "Parsed event contains no \"Event\" tag." + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + event = object!(parsed_event.Event) + + tag_instant_valid = false + instant, err = object(event.Instant) + if err == null { + epoch_nanoseconds, err = to_int(instant.@epochSecond) * 1_000_000_000 + to_int(instant.@nanoOfSecond) + if err == null && epoch_nanoseconds != 0 { + converted_timestamp, err = from_unix_timestamp(epoch_nanoseconds, "nanoseconds") + if err == null { + .timestamp = converted_timestamp + tag_instant_valid = true + } else { + .errors = push(.errors, "Instant invalid, trying property timeMillis instead: " + err) + } + } else { + .errors = push(.errors, "Instant invalid, trying property timeMillis instead: " + err) + } + } + if !tag_instant_valid { + epoch_milliseconds, err = to_int(event.@timeMillis) + if err == null && epoch_milliseconds != 0 { + converted_timestamp, err = from_unix_timestamp(epoch_milliseconds, "milliseconds") + if err == null { + .timestamp = converted_timestamp + } else { + .errors = push(.errors, "timeMillis not parsable, using current time instead: " + err) + } + } else { + .errors = push(.errors, "timeMillis not parsable, using current time instead: " + err) + } + } + + .logger, err = string(event.@loggerName) + if err != null || is_empty(.logger) { + .errors = push(.errors, "Logger not found.") + } + + level, err = string(event.@level) + if err != null { + .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") + } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], level) { + .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") + } else { + .level = level + } + + exception = null + thrown = event.Thrown + if is_object(thrown) { + exception = "Exception" + thread, err = string(event.@thread) + if err == null && !is_empty(thread) { + exception = exception + " in thread \"" + thread + "\"" + } + thrown_name, err = string(thrown.@name) + if err == null && !is_empty(exception) { + exception = exception + " " + thrown_name + } + message = string(thrown.@localizedMessage) ?? + string(thrown.@message) ?? + "" + if !is_empty(message) { + exception = exception + ": " + message + } + stacktrace_items = array(thrown.ExtendedStackTrace.ExtendedStackTraceItem) ?? [] + stacktrace = "" + for_each(stacktrace_items) -> |_index, value| { + stacktrace = stacktrace + " " + class = string(value.@class) ?? "" + method = string(value.@method) ?? "" + if !is_empty(class) && !is_empty(method) { + stacktrace = stacktrace + "at " + class + "." + method + } + file = string(value.@file) ?? "" + line = string(value.@line) ?? "" + if !is_empty(file) && !is_empty(line) { + stacktrace = stacktrace + "(" + file + ":" + line + ")" + } + exact = to_bool(value.@exact) ?? false + location = string(value.@location) ?? "" + version = string(value.@version) ?? "" + if !is_empty(location) && !is_empty(version) { + stacktrace = stacktrace + " " + if !exact { + stacktrace = stacktrace + "~" + } + stacktrace = stacktrace + "[" + location + ":" + version + "]" + } + stacktrace = stacktrace + "\n" + } + if stacktrace != "" { + exception = exception + "\n" + stacktrace + } + } + + message, err = string(event.Message) + if err != null || is_empty(message) { + message = null + .errors = push(.errors, "Message not found.") + } + .message = join!(compact([message, exception]), "\n") + } + } + + # Extends the processed files with the fields "container" and "file" + extended_logs_files: + inputs: + - processed_files_* + type: remap + source: | + del(.source_type) + if .errors == [] { + del(.errors) + } + . |= parse_regex!(.file, r'^${LOG_DIR}/(?P.*?)/(?P.*?)$') + + # Filters the logs of the Vector agent according to the defined log level + filtered_logs_vector: + inputs: + - vector + type: filter + condition: > + (.metadata.level == "TRACE" && "${VECTOR_FILE_LOG_LEVEL}" == "trace") || + (.metadata.level == "DEBUG" && includes(["trace", "debug"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "INFO" && includes(["trace", "debug", "info"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "WARN" && includes(["trace", "debug", "info", "warn"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "ERROR" && includes(["trace", "debug", "info", "warn", "error"], "${VECTOR_FILE_LOG_LEVEL}")) + + # Aligns the logs of the Vector agent with the common format + extended_logs_vector: + inputs: + - filtered_logs_vector + type: remap + source: | + .container = "vector" + .level = .metadata.level + .logger = .metadata.module_path + if exists(.file) { .processed_file = del(.file) } + del(.metadata) + del(.pid) + del(.source_type) + + # Add the fields "namespace", "cluster", "role" and "roleGroup" to all logs + extended_logs: + inputs: + - extended_logs_* + type: remap + source: | + .namespace = "${NAMESPACE}" + .cluster = "${CLUSTER_NAME}" + .role = "${ROLE_NAME}" + .roleGroup = "${ROLE_GROUP_NAME}" + + sinks: + # Forward the logs to the Vector aggregator + aggregator: + inputs: + - extended_logs + type: vector + address: ${VECTOR_AGGREGATOR_ADDRESS} +{%- endif %} diff --git a/tests/templates/kuttl/smoke/32_cm-test-hbase-regionserver-default.yaml.j2 b/tests/templates/kuttl/smoke/32_cm-test-hbase-regionserver-default.yaml.j2 index 5b44d0ca..a708f757 100644 --- a/tests/templates/kuttl/smoke/32_cm-test-hbase-regionserver-default.yaml.j2 +++ b/tests/templates/kuttl/smoke/32_cm-test-hbase-regionserver-default.yaml.j2 @@ -52,7 +52,7 @@ hbase-site.xml: |- hbase.zookeeper.quorum - test-zk-server.$NAMESPACE.svc.cluster.local:2282 + test-zk-server.__NAMESPACE__.svc.cluster.local:2282 phoenix.log.saltBuckets @@ -60,7 +60,7 @@ hbase-site.xml: |- zookeeper.znode.parent - $ZNODE_PATH/hbase + /znode-__ZNODE__/hbase log4j2.properties: |- @@ -103,3 +103,237 @@ ssl-server.xml: |- +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +vector.yaml: | + --- + data_dir: ${DATA_DIR} + + log_schema: + host_key: pod + + sources: + # Reads the internal Vector logs + vector: + type: internal_logs + + files_stdout: + type: file + include: + - ${LOG_DIR}/*/*.stdout.log + + files_stderr: + type: file + include: + - ${LOG_DIR}/*/*.stderr.log + + files_log4j2: + type: file + include: + - ${LOG_DIR}/*/*.log4j2.xml + line_delimiter: "\r\n" + + transforms: + processed_files_stdout: + inputs: + - files_stdout + type: remap + source: | + .logger = "ROOT" + .level = "INFO" + + processed_files_stderr: + inputs: + - files_stderr + type: remap + source: | + .logger = "ROOT" + .level = "ERROR" + + processed_files_log4j2: + inputs: + - files_log4j2 + type: remap + source: | + raw_message = string!(.message) + + .timestamp = now() + .logger = "" + .level = "INFO" + .message = "" + .errors = [] + + event = {} + parsed_event, err = parse_xml(raw_message) + if err != null { + error = "XML not parsable: " + err + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + if !is_object(parsed_event.Event) { + error = "Parsed event contains no \"Event\" tag." + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + event = object!(parsed_event.Event) + + tag_instant_valid = false + instant, err = object(event.Instant) + if err == null { + epoch_nanoseconds, err = to_int(instant.@epochSecond) * 1_000_000_000 + to_int(instant.@nanoOfSecond) + if err == null && epoch_nanoseconds != 0 { + converted_timestamp, err = from_unix_timestamp(epoch_nanoseconds, "nanoseconds") + if err == null { + .timestamp = converted_timestamp + tag_instant_valid = true + } else { + .errors = push(.errors, "Instant invalid, trying property timeMillis instead: " + err) + } + } else { + .errors = push(.errors, "Instant invalid, trying property timeMillis instead: " + err) + } + } + if !tag_instant_valid { + epoch_milliseconds, err = to_int(event.@timeMillis) + if err == null && epoch_milliseconds != 0 { + converted_timestamp, err = from_unix_timestamp(epoch_milliseconds, "milliseconds") + if err == null { + .timestamp = converted_timestamp + } else { + .errors = push(.errors, "timeMillis not parsable, using current time instead: " + err) + } + } else { + .errors = push(.errors, "timeMillis not parsable, using current time instead: " + err) + } + } + + .logger, err = string(event.@loggerName) + if err != null || is_empty(.logger) { + .errors = push(.errors, "Logger not found.") + } + + level, err = string(event.@level) + if err != null { + .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") + } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], level) { + .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") + } else { + .level = level + } + + exception = null + thrown = event.Thrown + if is_object(thrown) { + exception = "Exception" + thread, err = string(event.@thread) + if err == null && !is_empty(thread) { + exception = exception + " in thread \"" + thread + "\"" + } + thrown_name, err = string(thrown.@name) + if err == null && !is_empty(exception) { + exception = exception + " " + thrown_name + } + message = string(thrown.@localizedMessage) ?? + string(thrown.@message) ?? + "" + if !is_empty(message) { + exception = exception + ": " + message + } + stacktrace_items = array(thrown.ExtendedStackTrace.ExtendedStackTraceItem) ?? [] + stacktrace = "" + for_each(stacktrace_items) -> |_index, value| { + stacktrace = stacktrace + " " + class = string(value.@class) ?? "" + method = string(value.@method) ?? "" + if !is_empty(class) && !is_empty(method) { + stacktrace = stacktrace + "at " + class + "." + method + } + file = string(value.@file) ?? "" + line = string(value.@line) ?? "" + if !is_empty(file) && !is_empty(line) { + stacktrace = stacktrace + "(" + file + ":" + line + ")" + } + exact = to_bool(value.@exact) ?? false + location = string(value.@location) ?? "" + version = string(value.@version) ?? "" + if !is_empty(location) && !is_empty(version) { + stacktrace = stacktrace + " " + if !exact { + stacktrace = stacktrace + "~" + } + stacktrace = stacktrace + "[" + location + ":" + version + "]" + } + stacktrace = stacktrace + "\n" + } + if stacktrace != "" { + exception = exception + "\n" + stacktrace + } + } + + message, err = string(event.Message) + if err != null || is_empty(message) { + message = null + .errors = push(.errors, "Message not found.") + } + .message = join!(compact([message, exception]), "\n") + } + } + + # Extends the processed files with the fields "container" and "file" + extended_logs_files: + inputs: + - processed_files_* + type: remap + source: | + del(.source_type) + if .errors == [] { + del(.errors) + } + . |= parse_regex!(.file, r'^${LOG_DIR}/(?P.*?)/(?P.*?)$') + + # Filters the logs of the Vector agent according to the defined log level + filtered_logs_vector: + inputs: + - vector + type: filter + condition: > + (.metadata.level == "TRACE" && "${VECTOR_FILE_LOG_LEVEL}" == "trace") || + (.metadata.level == "DEBUG" && includes(["trace", "debug"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "INFO" && includes(["trace", "debug", "info"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "WARN" && includes(["trace", "debug", "info", "warn"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "ERROR" && includes(["trace", "debug", "info", "warn", "error"], "${VECTOR_FILE_LOG_LEVEL}")) + + # Aligns the logs of the Vector agent with the common format + extended_logs_vector: + inputs: + - filtered_logs_vector + type: remap + source: | + .container = "vector" + .level = .metadata.level + .logger = .metadata.module_path + if exists(.file) { .processed_file = del(.file) } + del(.metadata) + del(.pid) + del(.source_type) + + # Add the fields "namespace", "cluster", "role" and "roleGroup" to all logs + extended_logs: + inputs: + - extended_logs_* + type: remap + source: | + .namespace = "${NAMESPACE}" + .cluster = "${CLUSTER_NAME}" + .role = "${ROLE_NAME}" + .roleGroup = "${ROLE_GROUP_NAME}" + + sinks: + # Forward the logs to the Vector aggregator + aggregator: + inputs: + - extended_logs + type: vector + address: ${VECTOR_AGGREGATOR_ADDRESS} +{%- endif %} diff --git a/tests/templates/kuttl/smoke/32_cm-test-hbase-restserver-default.yaml.j2 b/tests/templates/kuttl/smoke/32_cm-test-hbase-restserver-default.yaml.j2 index c38c12b1..8a97fa8e 100644 --- a/tests/templates/kuttl/smoke/32_cm-test-hbase-restserver-default.yaml.j2 +++ b/tests/templates/kuttl/smoke/32_cm-test-hbase-restserver-default.yaml.j2 @@ -32,7 +32,7 @@ hbase-site.xml: |- hbase.zookeeper.quorum - test-zk-server.$NAMESPACE.svc.cluster.local:2282 + test-zk-server.__NAMESPACE__.svc.cluster.local:2282 phoenix.log.saltBuckets @@ -40,7 +40,7 @@ hbase-site.xml: |- zookeeper.znode.parent - $ZNODE_PATH/hbase + /znode-__ZNODE__/hbase log4j2.properties: |- @@ -83,3 +83,237 @@ ssl-server.xml: |- +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +vector.yaml: | + --- + data_dir: ${DATA_DIR} + + log_schema: + host_key: pod + + sources: + # Reads the internal Vector logs + vector: + type: internal_logs + + files_stdout: + type: file + include: + - ${LOG_DIR}/*/*.stdout.log + + files_stderr: + type: file + include: + - ${LOG_DIR}/*/*.stderr.log + + files_log4j2: + type: file + include: + - ${LOG_DIR}/*/*.log4j2.xml + line_delimiter: "\r\n" + + transforms: + processed_files_stdout: + inputs: + - files_stdout + type: remap + source: | + .logger = "ROOT" + .level = "INFO" + + processed_files_stderr: + inputs: + - files_stderr + type: remap + source: | + .logger = "ROOT" + .level = "ERROR" + + processed_files_log4j2: + inputs: + - files_log4j2 + type: remap + source: | + raw_message = string!(.message) + + .timestamp = now() + .logger = "" + .level = "INFO" + .message = "" + .errors = [] + + event = {} + parsed_event, err = parse_xml(raw_message) + if err != null { + error = "XML not parsable: " + err + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + if !is_object(parsed_event.Event) { + error = "Parsed event contains no \"Event\" tag." + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + event = object!(parsed_event.Event) + + tag_instant_valid = false + instant, err = object(event.Instant) + if err == null { + epoch_nanoseconds, err = to_int(instant.@epochSecond) * 1_000_000_000 + to_int(instant.@nanoOfSecond) + if err == null && epoch_nanoseconds != 0 { + converted_timestamp, err = from_unix_timestamp(epoch_nanoseconds, "nanoseconds") + if err == null { + .timestamp = converted_timestamp + tag_instant_valid = true + } else { + .errors = push(.errors, "Instant invalid, trying property timeMillis instead: " + err) + } + } else { + .errors = push(.errors, "Instant invalid, trying property timeMillis instead: " + err) + } + } + if !tag_instant_valid { + epoch_milliseconds, err = to_int(event.@timeMillis) + if err == null && epoch_milliseconds != 0 { + converted_timestamp, err = from_unix_timestamp(epoch_milliseconds, "milliseconds") + if err == null { + .timestamp = converted_timestamp + } else { + .errors = push(.errors, "timeMillis not parsable, using current time instead: " + err) + } + } else { + .errors = push(.errors, "timeMillis not parsable, using current time instead: " + err) + } + } + + .logger, err = string(event.@loggerName) + if err != null || is_empty(.logger) { + .errors = push(.errors, "Logger not found.") + } + + level, err = string(event.@level) + if err != null { + .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") + } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], level) { + .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") + } else { + .level = level + } + + exception = null + thrown = event.Thrown + if is_object(thrown) { + exception = "Exception" + thread, err = string(event.@thread) + if err == null && !is_empty(thread) { + exception = exception + " in thread \"" + thread + "\"" + } + thrown_name, err = string(thrown.@name) + if err == null && !is_empty(exception) { + exception = exception + " " + thrown_name + } + message = string(thrown.@localizedMessage) ?? + string(thrown.@message) ?? + "" + if !is_empty(message) { + exception = exception + ": " + message + } + stacktrace_items = array(thrown.ExtendedStackTrace.ExtendedStackTraceItem) ?? [] + stacktrace = "" + for_each(stacktrace_items) -> |_index, value| { + stacktrace = stacktrace + " " + class = string(value.@class) ?? "" + method = string(value.@method) ?? "" + if !is_empty(class) && !is_empty(method) { + stacktrace = stacktrace + "at " + class + "." + method + } + file = string(value.@file) ?? "" + line = string(value.@line) ?? "" + if !is_empty(file) && !is_empty(line) { + stacktrace = stacktrace + "(" + file + ":" + line + ")" + } + exact = to_bool(value.@exact) ?? false + location = string(value.@location) ?? "" + version = string(value.@version) ?? "" + if !is_empty(location) && !is_empty(version) { + stacktrace = stacktrace + " " + if !exact { + stacktrace = stacktrace + "~" + } + stacktrace = stacktrace + "[" + location + ":" + version + "]" + } + stacktrace = stacktrace + "\n" + } + if stacktrace != "" { + exception = exception + "\n" + stacktrace + } + } + + message, err = string(event.Message) + if err != null || is_empty(message) { + message = null + .errors = push(.errors, "Message not found.") + } + .message = join!(compact([message, exception]), "\n") + } + } + + # Extends the processed files with the fields "container" and "file" + extended_logs_files: + inputs: + - processed_files_* + type: remap + source: | + del(.source_type) + if .errors == [] { + del(.errors) + } + . |= parse_regex!(.file, r'^${LOG_DIR}/(?P.*?)/(?P.*?)$') + + # Filters the logs of the Vector agent according to the defined log level + filtered_logs_vector: + inputs: + - vector + type: filter + condition: > + (.metadata.level == "TRACE" && "${VECTOR_FILE_LOG_LEVEL}" == "trace") || + (.metadata.level == "DEBUG" && includes(["trace", "debug"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "INFO" && includes(["trace", "debug", "info"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "WARN" && includes(["trace", "debug", "info", "warn"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "ERROR" && includes(["trace", "debug", "info", "warn", "error"], "${VECTOR_FILE_LOG_LEVEL}")) + + # Aligns the logs of the Vector agent with the common format + extended_logs_vector: + inputs: + - filtered_logs_vector + type: remap + source: | + .container = "vector" + .level = .metadata.level + .logger = .metadata.module_path + if exists(.file) { .processed_file = del(.file) } + del(.metadata) + del(.pid) + del(.source_type) + + # Add the fields "namespace", "cluster", "role" and "roleGroup" to all logs + extended_logs: + inputs: + - extended_logs_* + type: remap + source: | + .namespace = "${NAMESPACE}" + .cluster = "${CLUSTER_NAME}" + .role = "${ROLE_NAME}" + .roleGroup = "${ROLE_GROUP_NAME}" + + sinks: + # Forward the logs to the Vector aggregator + aggregator: + inputs: + - extended_logs + type: vector + address: ${VECTOR_AGGREGATOR_ADDRESS} +{%- endif %} From 89dafb2d29e0cc9b4bcd2b828738f782793b2a11 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 21:06:31 +0200 Subject: [PATCH 48/56] refactor: use v2 ConfigMapName, SecretName, ListenerName in CRD --- extra/crds.yaml | 33 ++++++++++++++++ .../controller/build/resource/statefulset.rs | 2 +- rust/operator-binary/src/controller/mod.rs | 8 ++-- .../src/controller/validate.rs | 19 +++------- .../src/controller/zookeeper.rs | 10 ++--- rust/operator-binary/src/crd/mod.rs | 38 ++++++++++++------- rust/operator-binary/src/crd/security.rs | 15 +++++--- rust/operator-binary/src/main.rs | 9 ++++- 8 files changed, 90 insertions(+), 44 deletions(-) diff --git a/extra/crds.yaml b/extra/crds.yaml index d941cff2..b13b66a1 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -42,6 +42,9 @@ spec: properties: secretClass: description: Name of the SecretClass providing the keytab for the HBase services. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string required: - secretClass @@ -49,6 +52,9 @@ spec: tlsSecretClass: default: tls description: Name of the SecretClass providing the tls certificates for the WebUIs. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string required: - kerberos @@ -82,6 +88,9 @@ spec: description: |- Name of the [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) for an HDFS cluster. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string vectorAggregatorConfigMapName: description: |- @@ -89,12 +98,18 @@ spec: It must contain the key `ADDRESS` with the address of the Vector aggregator. Follow the [logging tutorial](https://docs.stackable.tech/home/nightly/tutorials/logging-vector-aggregator) to learn how to configure log aggregation with Vector. + maxLength: 253 + minLength: 1 nullable: true + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string zookeeperConfigMapName: description: |- Name of the [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) for a ZooKeeper cluster. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string required: - hdfsConfigMapName @@ -247,7 +262,10 @@ spec: type: string listenerClass: description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. + maxLength: 253 + minLength: 1 nullable: true + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string logging: default: @@ -668,7 +686,10 @@ spec: type: string listenerClass: description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. + maxLength: 253 + minLength: 1 nullable: true + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string logging: default: @@ -1075,7 +1096,10 @@ spec: type: string listenerClass: description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. + maxLength: 253 + minLength: 1 nullable: true + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string logging: default: @@ -1531,7 +1555,10 @@ spec: type: string listenerClass: description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. + maxLength: 253 + minLength: 1 nullable: true + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string logging: default: @@ -1961,7 +1988,10 @@ spec: type: string listenerClass: description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. + maxLength: 253 + minLength: 1 nullable: true + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string logging: default: @@ -2382,7 +2412,10 @@ spec: type: string listenerClass: description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. + maxLength: 253 + minLength: 1 nullable: true + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string logging: default: diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index b1d74b25..21e3d11d 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -234,7 +234,7 @@ pub fn build_rolegroup_statefulset( .add_volume(Volume { name: HDFS_DISCOVERY_VOLUME_NAME.to_string(), config_map: Some(ConfigMapVolumeSource { - name: cluster.cluster_config.hdfs_config_map_name.clone(), + name: cluster.cluster_config.hdfs_config_map_name.to_string(), ..Default::default() }), ..Default::default() diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index 2749dcb7..3c4b5f0c 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -19,7 +19,7 @@ use stackable_operator::{ kvp::label::{recommended_labels, role_group_selector}, role_group_utils::ResourceNames, types::{ - kubernetes::{NamespaceName, Uid}, + kubernetes::{ConfigMapName, NamespaceName, SecretClassName, Uid}, operator::{ ClusterName, ControllerName, OperatorName, ProductName, ProductVersion, RoleName, }, @@ -248,11 +248,11 @@ pub struct ValidatedClusterConfig { /// Whether HTTPS is enabled (a TLS `SecretClass` was configured). pub https_enabled: bool, /// The Kerberos `SecretClass` name, if Kerberos is enabled. - pub kerberos_secret_class: Option, + pub kerberos_secret_class: Option, /// The HTTPS/TLS `SecretClass` name, if HTTPS is enabled. - pub https_secret_class: Option, + pub https_secret_class: Option, /// The HDFS discovery ConfigMap name the cluster connects to. - pub hdfs_config_map_name: String, + pub hdfs_config_map_name: ConfigMapName, // Pre-resolved kerberos properties for hbase-site.xml (empty when kerberos is disabled). pub hbase_site_kerberos_config: BTreeMap, // Pre-resolved kerberos properties for the discovery `hbase-site.xml` exposed to clients diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index ee0fa7ec..cd810bad 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -84,11 +84,6 @@ pub enum Error { "the Vector aggregator discovery ConfigMap name is required when the Vector agent is enabled" ))] MissingVectorAggregatorConfigMapName, - - #[snafu(display("invalid Vector aggregator discovery ConfigMap name"))] - ParseVectorAggregatorConfigMapName { - source: stackable_operator::v2::macros::attributed_string_type::Error, - }, } /// Validated logging configuration for the HBase and (optional) Vector container. @@ -149,19 +144,17 @@ pub fn validate_cluster( let mut role_groups = BTreeMap::new(); let mut role_configs = BTreeMap::new(); - let hdfs_discovery_cm_name = &hbase.spec.cluster_config.hdfs_config_map_name; + let hdfs_discovery_cm_name = hbase.spec.cluster_config.hdfs_config_map_name.as_ref(); let cluster_name = hbase.name_any(); - // The Vector aggregator discovery ConfigMap name (validated here so an invalid name fails - // up-front). It is only required when the Vector agent is enabled for a role group. + // The Vector aggregator discovery ConfigMap name. It is validated already at deserialize time + // (it is a `ConfigMapName`), and only required when the Vector agent is enabled for a role + // group. let vector_aggregator_config_map_name = hbase .spec .cluster_config .vector_aggregator_config_map_name - .as_deref() - .map(ConfigMapName::from_str) - .transpose() - .context(ParseVectorAggregatorConfigMapNameSnafu)?; + .clone(); for hbase_role in HbaseRole::iter() { let group_configs = match hbase_role { @@ -412,7 +405,7 @@ spec: let default_config = HbaseConfigFragment::default_config( &HbaseRole::Master, &hbase.name_any(), - &hbase.spec.cluster_config.hdfs_config_map_name, + hbase.spec.cluster_config.hdfs_config_map_name.as_ref(), ); let validated = with_validated_config::< diff --git a/rust/operator-binary/src/controller/zookeeper.rs b/rust/operator-binary/src/controller/zookeeper.rs index 398da1f3..a824fa12 100644 --- a/rust/operator-binary/src/controller/zookeeper.rs +++ b/rust/operator-binary/src/controller/zookeeper.rs @@ -72,7 +72,7 @@ impl ZookeeperConnectionInformation { let zk_discovery_cm_name = &hbase.spec.cluster_config.zookeeper_config_map_name; let mut zk_discovery_cm = client .get::( - zk_discovery_cm_name, + zk_discovery_cm_name.as_ref(), hbase .namespace() .as_deref() @@ -88,7 +88,7 @@ impl ZookeeperConnectionInformation { .as_mut() .and_then(|data| data.remove(ZOOKEEPER_DISCOVERY_CM_HOSTS_ENTRY)) .context(MissingConfigMapEntrySnafu { - cm_name: zk_discovery_cm_name.as_str(), + cm_name: zk_discovery_cm_name.to_string(), entry: ZOOKEEPER_DISCOVERY_CM_HOSTS_ENTRY, })?; let mut chroot = zk_discovery_cm @@ -96,7 +96,7 @@ impl ZookeeperConnectionInformation { .as_mut() .and_then(|data| data.remove(ZOOKEEPER_DISCOVERY_CM_CHROOT_ENTRY)) .context(MissingConfigMapEntrySnafu { - cm_name: zk_discovery_cm_name.as_str(), + cm_name: zk_discovery_cm_name.to_string(), entry: ZOOKEEPER_DISCOVERY_CM_CHROOT_ENTRY, })?; let port = zk_discovery_cm @@ -104,12 +104,12 @@ impl ZookeeperConnectionInformation { .as_mut() .and_then(|data| data.remove(ZOOKEEPER_DISCOVERY_CM_CLIENT_PORT_ENTRY)) .context(MissingConfigMapEntrySnafu { - cm_name: zk_discovery_cm_name.as_str(), + cm_name: zk_discovery_cm_name.to_string(), entry: ZOOKEEPER_DISCOVERY_CM_CLIENT_PORT_ENTRY, })? .parse() .context(ParseZookeeperPortSnafu { - cm_name: zk_discovery_cm_name.as_str(), + cm_name: zk_discovery_cm_name.to_string(), entry: ZOOKEEPER_DISCOVERY_CM_CLIENT_PORT_ENTRY, })?; diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 2cac8596..dc67d512 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,3 +1,5 @@ +use std::str::FromStr; + use security::AuthenticationConfig; use serde::{Deserialize, Serialize}; use stackable_operator::{ @@ -22,7 +24,11 @@ use stackable_operator::{ schemars::{self, JsonSchema}, shared::time::Duration, status::condition::{ClusterCondition, HasStatusCondition}, - v2::{config_overrides::KeyValueConfigOverrides, role_utils::JavaCommonConfig}, + v2::{ + config_overrides::KeyValueConfigOverrides, + role_utils::JavaCommonConfig, + types::kubernetes::{ConfigMapName, ListenerClassName, SecretClassName}, + }, versioned::versioned, }; use strum::{Display, EnumIter, EnumString}; @@ -140,23 +146,23 @@ pub mod versioned { pub rest_servers: Option, } - #[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] + #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct HbaseClusterConfig { /// Name of the [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery) /// for an HDFS cluster. - pub hdfs_config_map_name: String, + pub hdfs_config_map_name: ConfigMapName, /// Name of the Vector aggregator [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery). /// It must contain the key `ADDRESS` with the address of the Vector aggregator. /// Follow the [logging tutorial](DOCS_BASE_URL_PLACEHOLDER/tutorials/logging-vector-aggregator) /// to learn how to configure log aggregation with Vector. #[serde(skip_serializing_if = "Option::is_none")] - pub vector_aggregator_config_map_name: Option, + pub vector_aggregator_config_map_name: Option, /// Name of the [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery) /// for a ZooKeeper cluster. - pub zookeeper_config_map_name: String, + pub zookeeper_config_map_name: ConfigMapName, /// Settings related to user [authentication](DOCS_BASE_URL_PLACEHOLDER/usage-guide/security). pub authentication: Option, @@ -207,7 +213,7 @@ impl v1alpha1::HbaseCluster { self.kerberos_secret_class().is_some() } - pub fn kerberos_secret_class(&self) -> Option { + pub fn kerberos_secret_class(&self) -> Option { self.spec .cluster_config .authentication @@ -220,7 +226,7 @@ impl v1alpha1::HbaseCluster { self.https_secret_class().is_some() } - pub fn https_secret_class(&self) -> Option { + pub fn https_secret_class(&self) -> Option { self.spec .cluster_config .authentication @@ -332,7 +338,10 @@ impl HbaseConfigFragment { affinity: get_affinity(cluster_name, role, hdfs_discovery_cm_name), graceful_shutdown_timeout: Some(graceful_shutdown_timeout), requested_secret_lifetime: Some(requested_secret_lifetime), - listener_class: Some(DEFAULT_LISTENER_CLASS.to_string()), + listener_class: Some( + ListenerClassName::from_str(DEFAULT_LISTENER_CLASS) + .expect("DEFAULT_LISTENER_CLASS is a valid listener class name"), + ), } } } @@ -359,7 +368,10 @@ impl RegionServerConfigFragment { cli_opts: None, }, requested_secret_lifetime: Some(HbaseRole::DEFAULT_REGION_SECRET_LIFETIME), - listener_class: Some(DEFAULT_LISTENER_CLASS.to_string()), + listener_class: Some( + ListenerClassName::from_str(DEFAULT_LISTENER_CLASS) + .expect("DEFAULT_LISTENER_CLASS is a valid listener class name"), + ), } } } @@ -401,7 +413,7 @@ pub enum Container { Vector, } -#[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] +#[derive(Clone, Debug, Fragment, JsonSchema, PartialEq)] #[fragment_attrs( derive( Clone, @@ -439,7 +451,7 @@ pub struct HbaseConfig { pub requested_secret_lifetime: Option, /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose this rolegroup. - pub listener_class: String, + pub listener_class: ListenerClassName, } #[derive(Fragment, Clone, Debug, JsonSchema, PartialEq, Serialize, Deserialize)] @@ -524,7 +536,7 @@ pub struct RegionServerConfig { pub region_mover: RegionMover, /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose this rolegroup. - pub listener_class: String, + pub listener_class: ListenerClassName, } #[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] @@ -582,7 +594,7 @@ impl AnyServiceConfig { } } - pub fn listener_class(&self) -> String { + pub fn listener_class(&self) -> ListenerClassName { match self { AnyServiceConfig::Master(config) => config.listener_class.clone(), AnyServiceConfig::RegionServer(config) => config.listener_class.clone(), diff --git a/rust/operator-binary/src/crd/security.rs b/rust/operator-binary/src/crd/security.rs index 85919efe..839ceb9f 100644 --- a/rust/operator-binary/src/crd/security.rs +++ b/rust/operator-binary/src/crd/security.rs @@ -1,29 +1,32 @@ +use std::str::FromStr; + use serde::{Deserialize, Serialize}; use stackable_operator::{ commons::opa::OpaConfig, schemars::{self, JsonSchema}, + v2::types::kubernetes::SecretClassName, }; -#[derive(Clone, Debug, Deserialize, Eq, Hash, JsonSchema, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct AuthenticationConfig { /// Name of the SecretClass providing the tls certificates for the WebUIs. #[serde(default = "default_tls_secret_class")] - pub tls_secret_class: String, + pub tls_secret_class: SecretClassName, /// Kerberos configuration. pub kerberos: KerberosConfig, } -fn default_tls_secret_class() -> String { - "tls".to_string() +fn default_tls_secret_class() -> SecretClassName { + SecretClassName::from_str("tls").expect("\"tls\" is a valid secret class name") } -#[derive(Clone, Debug, Deserialize, Eq, Hash, JsonSchema, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct KerberosConfig { /// Name of the SecretClass providing the keytab for the HBase services. - pub secret_class: String, + pub secret_class: SecretClassName, } #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 0eb35a0d..d0df5dd7 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -193,8 +193,13 @@ fn references_config_map( return false; }; - hbase.spec.cluster_config.zookeeper_config_map_name == config_map.name_any() - || hbase.spec.cluster_config.hdfs_config_map_name == config_map.name_any() + hbase + .spec + .cluster_config + .zookeeper_config_map_name + .to_string() + == config_map.name_any() + || hbase.spec.cluster_config.hdfs_config_map_name.to_string() == config_map.name_any() || match &hbase.spec.cluster_config.authorization { Some(hbase_authorization) => { hbase_authorization.opa.config_map_name == config_map.name_any() From 5901236c78dad6b007c4053d566c3b3c8d431f8d Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 21:10:39 +0200 Subject: [PATCH 49/56] refactor: switch to v2 listener builder --- .../src/controller/build/resource/listener.rs | 35 ++++++++++--------- .../controller/build/resource/statefulset.rs | 6 +--- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/rust/operator-binary/src/controller/build/resource/listener.rs b/rust/operator-binary/src/controller/build/resource/listener.rs index de937095..8cd99290 100644 --- a/rust/operator-binary/src/controller/build/resource/listener.rs +++ b/rust/operator-binary/src/controller/build/resource/listener.rs @@ -1,5 +1,7 @@ //! Build the listener `Volume`/`PersistentVolumeClaim` exposing a rolegroup. +use std::str::FromStr; + use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::pod::volume::{ @@ -8,6 +10,13 @@ use stackable_operator::{ }, k8s_openapi::api::core::v1::{PersistentVolumeClaim, Volume}, kvp::Labels, + v2::{ + builder::pod::volume::{ + ListenerReference as TypedListenerReference, + listener_operator_volume_source_builder_build_pvc, + }, + types::kubernetes::PersistentVolumeClaimName, + }, }; use crate::crd::{AnyServiceConfig, HbaseRole, LISTENER_VOLUME_NAME}; @@ -18,11 +27,6 @@ pub enum Error { BuildListenerVolume { source: ListenerOperatorVolumeSourceBuilderError, }, - - #[snafu(display("failed to build listener pvc"))] - BuildListenerPvc { - source: ListenerOperatorVolumeSourceBuilderError, - }, } type Result = std::result::Result; @@ -62,17 +66,14 @@ pub fn build_listener_pvc( role: &HbaseRole, merged_config: &AnyServiceConfig, recommended_labels: &Labels, -) -> Result>> { - let pvc = match role { +) -> Option> { + match role { HbaseRole::Master | HbaseRole::RegionServer => None, - HbaseRole::RestServer => Some(vec![ - ListenerOperatorVolumeSourceBuilder::new( - &ListenerReference::ListenerClass(merged_config.listener_class().to_string()), - recommended_labels, - ) - .build_pvc(LISTENER_VOLUME_NAME.to_string()) - .context(BuildListenerPvcSnafu)?, - ]), - }; - Ok(pvc) + HbaseRole::RestServer => Some(vec![listener_operator_volume_source_builder_build_pvc( + &TypedListenerReference::ListenerClass(merged_config.listener_class()), + recommended_labels, + &PersistentVolumeClaimName::from_str(LISTENER_VOLUME_NAME) + .expect("LISTENER_VOLUME_NAME is a valid PersistentVolumeClaim name"), + )]), + } } diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 21e3d11d..e47e7ed8 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -86,9 +86,6 @@ pub enum Error { #[snafu(display("failed to build listener volume"))] ListenerVolume { source: super::listener::Error }, - - #[snafu(display("failed to build listener persistent volume claim"))] - ListenerPersistentVolumeClaim { source: super::listener::Error }, } type Result = std::result::Result; @@ -299,8 +296,7 @@ pub fn build_rolegroup_statefulset( } let listener_pvc = - super::listener::build_listener_pvc(hbase_role, merged_config, &recommended_labels) - .context(ListenerPersistentVolumeClaimSnafu)?; + super::listener::build_listener_pvc(hbase_role, merged_config, &recommended_labels); if let Some(listener_volume) = super::listener::build_listener_volume(hbase_role, merged_config, &recommended_labels) From d60da5900aff7af1d728a582d8fe35fed0519335 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 21:14:41 +0200 Subject: [PATCH 50/56] refactor: use v2 container builder --- .../src/controller/build/resource/statefulset.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index e47e7ed8..1fcf7dc6 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -8,7 +8,7 @@ use stackable_operator::{ builder::{ self, meta::ObjectMetaBuilder, - pod::{PodBuilder, container::ContainerBuilder, security::PodSecurityContextBuilder}, + pod::{PodBuilder, security::PodSecurityContextBuilder}, }, constants::RESTART_CONTROLLER_ENABLED_LABEL, k8s_openapi::{ @@ -25,7 +25,7 @@ use stackable_operator::{ kube::ResourceExt, product_logging, v2::{ - builder::pod::container::{EnvVarName, EnvVarSet}, + builder::pod::container::{EnvVarName, EnvVarSet, new_container_builder}, product_logging::framework::{ValidatedContainerLogConfigChoice, vector_container}, types::{ kubernetes::{ContainerName, VolumeName}, @@ -46,6 +46,7 @@ use crate::{ crd::{CONFIG_DIR_NAME, HbaseRole, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME}, }; +stackable_operator::constant!(HBASE_CONTAINER_NAME: ContainerName = "hbase"); stackable_operator::constant!(VECTOR_CONTAINER_NAME: ContainerName = "vector"); // Pod volume names. The Vector container reuses the `hbase-config` (rolegroup ConfigMap, which @@ -172,7 +173,7 @@ pub fn build_rolegroup_statefulset( ); let role_name = hbase_role.cli_role_name(); - let mut hbase_container = ContainerBuilder::new("hbase").expect("ContainerBuilder not created"); + let mut hbase_container = new_container_builder(&HBASE_CONTAINER_NAME); hbase_container .image_from_product_image(resolved_product_image) From b90fe2cd929a8f4094ce94f7be97becf57b6c264 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 21:22:27 +0200 Subject: [PATCH 51/56] refactor: use v2 Port type --- .../src/controller/build/role.rs | 10 ++++---- rust/operator-binary/src/crd/mod.rs | 23 +++++++++++-------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/rust/operator-binary/src/controller/build/role.rs b/rust/operator-binary/src/controller/build/role.rs index a6de1986..b478200d 100644 --- a/rust/operator-binary/src/controller/build/role.rs +++ b/rust/operator-binary/src/controller/build/role.rs @@ -2,6 +2,8 @@ //! port name/number mapping. These are operator/product knowledge, kept out of the `crd` module //! so it stays a pure API definition. +use stackable_operator::v2::types::common::Port; + use crate::crd::{ HBASE_MASTER_METRICS_PORT, HBASE_MASTER_PORT, HBASE_MASTER_UI_PORT, HBASE_REGIONSERVER_METRICS_PORT, HBASE_REGIONSERVER_PORT, HBASE_REGIONSERVER_UI_PORT, @@ -29,7 +31,7 @@ impl HbaseRole { /// /// Hbase versions 2.6.* will have two ports for each role. The metrics are available on the /// UI port. - pub fn ports(&self, https_enabled: bool) -> Vec<(String, u16)> { + pub fn ports(&self, https_enabled: bool) -> Vec<(String, Port)> { vec![ (self.data_port_name(https_enabled), self.data_port()), ( @@ -39,7 +41,7 @@ impl HbaseRole { ] } - pub fn data_port(&self) -> u16 { + pub fn data_port(&self) -> Port { match self { HbaseRole::Master => HBASE_MASTER_PORT, HbaseRole::RegionServer => HBASE_REGIONSERVER_PORT, @@ -60,7 +62,7 @@ impl HbaseRole { } } - pub fn ui_port(&self) -> u16 { + pub fn ui_port(&self) -> Port { match self { HbaseRole::Master => HBASE_MASTER_UI_PORT, HbaseRole::RegionServer => HBASE_REGIONSERVER_UI_PORT, @@ -77,7 +79,7 @@ impl HbaseRole { } } - pub fn metrics_port(&self) -> u16 { + pub fn metrics_port(&self) -> Port { match self { HbaseRole::Master => HBASE_MASTER_METRICS_PORT, HbaseRole::RegionServer => HBASE_REGIONSERVER_METRICS_PORT, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index dc67d512..d660788e 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -27,7 +27,10 @@ use stackable_operator::{ v2::{ config_overrides::KeyValueConfigOverrides, role_utils::JavaCommonConfig, - types::kubernetes::{ConfigMapName, ListenerClassName, SecretClassName}, + types::{ + common::Port, + kubernetes::{ConfigMapName, ListenerClassName, SecretClassName}, + }, }, versioned::versioned, }; @@ -56,18 +59,18 @@ pub const HBASE_CLUSTER_DISTRIBUTED: &str = "hbase.cluster.distributed"; pub const HBASE_ROOTDIR: &str = "hbase.rootdir"; const DEFAULT_HBASE_ROOTDIR: &str = "/hbase"; -pub const HBASE_MASTER_PORT: u16 = 16000; +pub const HBASE_MASTER_PORT: Port = Port(16000); // HBase always uses 16010, regardless of http or https. On 2024-01-17 we decided in Arch-meeting that we want to stick // the port numbers to what the product is doing, so we get the least surprise for users - even when this means we have // inconsistency between Stackable products. -pub const HBASE_MASTER_UI_PORT: u16 = 16010; -pub const HBASE_MASTER_METRICS_PORT: u16 = 16010; -pub const HBASE_REGIONSERVER_PORT: u16 = 16020; -pub const HBASE_REGIONSERVER_UI_PORT: u16 = 16030; -pub const HBASE_REGIONSERVER_METRICS_PORT: u16 = 16030; -pub const HBASE_REST_PORT: u16 = 8080; -pub const HBASE_REST_UI_PORT: u16 = 8085; -pub const HBASE_REST_METRICS_PORT: u16 = 8085; +pub const HBASE_MASTER_UI_PORT: Port = Port(16010); +pub const HBASE_MASTER_METRICS_PORT: Port = Port(16010); +pub const HBASE_REGIONSERVER_PORT: Port = Port(16020); +pub const HBASE_REGIONSERVER_UI_PORT: Port = Port(16030); +pub const HBASE_REGIONSERVER_METRICS_PORT: Port = Port(16030); +pub const HBASE_REST_PORT: Port = Port(8080); +pub const HBASE_REST_UI_PORT: Port = Port(8085); +pub const HBASE_REST_METRICS_PORT: Port = Port(8085); pub const LISTENER_VOLUME_NAME: &str = "listener"; pub const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; From f9056351990f0c60d0b6df74045755539735075c Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 16 Jun 2026 18:53:59 +0200 Subject: [PATCH 52/56] refactor: remove raw cluster ref from kerberos and jvm --- .../src/controller/build/jvm.rs | 6 +- .../src/controller/build/kerberos.rs | 73 ++++++------------- .../src/controller/validate.rs | 44 +++++++---- 3 files changed, 57 insertions(+), 66 deletions(-) diff --git a/rust/operator-binary/src/controller/build/jvm.rs b/rust/operator-binary/src/controller/build/jvm.rs index e7d9ba24..a51d1eab 100644 --- a/rust/operator-binary/src/controller/build/jvm.rs +++ b/rust/operator-binary/src/controller/build/jvm.rs @@ -6,7 +6,7 @@ use stackable_operator::{ use crate::{ controller::build::kerberos::KRB5_CONFIG_PATH, - crd::{AnyServiceConfig, CONFIG_DIR_NAME, JVM_SECURITY_PROPERTIES_FILE, v1alpha1}, + crd::{AnyServiceConfig, CONFIG_DIR_NAME, JVM_SECURITY_PROPERTIES_FILE}, }; const JAVA_HEAP_FACTOR: f32 = 0.8; @@ -49,14 +49,14 @@ pub fn construct_global_jvm_args(kerberos_enabled: bool) -> String { /// [`with_validated_config`](stackable_operator::v2::role_utils::with_validated_config). The /// operator-generated arguments below form the base that the user overrides are applied on top of. pub fn construct_role_specific_non_heap_jvm_args( - hbase: &v1alpha1::HbaseCluster, + kerberos_enabled: bool, merged_jvm_argument_overrides: &JvmArgumentOverrides, ) -> String { let mut operator_generated = vec![format!( "-Djava.security.properties={CONFIG_DIR_NAME}/{JVM_SECURITY_PROPERTIES_FILE}" )]; - if hbase.has_kerberos_enabled() { + if kerberos_enabled { operator_generated.push(format!("-Djava.security.krb5.conf={KRB5_CONFIG_PATH}")); } diff --git a/rust/operator-binary/src/controller/build/kerberos.rs b/rust/operator-binary/src/controller/build/kerberos.rs index 50507078..1a8fa0b3 100644 --- a/rust/operator-binary/src/controller/build/kerberos.rs +++ b/rust/operator-binary/src/controller/build/kerberos.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use snafu::{OptionExt, ResultExt, Snafu}; +use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::{ self, @@ -11,14 +11,13 @@ use stackable_operator::{ }, }, commons::secret_class::SecretClassVolumeProvisionParts, - kube::{ResourceExt, runtime::reflector::ObjectRef}, shared::time::Duration, utils::cluster_info::KubernetesClusterInfo, }; use crate::{ controller::ValidatedCluster, - crd::{TLS_STORE_DIR, TLS_STORE_PASSWORD, TLS_STORE_VOLUME_NAME, v1alpha1}, + crd::{TLS_STORE_DIR, TLS_STORE_PASSWORD, TLS_STORE_VOLUME_NAME}, }; /// Mount path of the Kerberos secret volume (keytab + `krb5.conf`). @@ -31,18 +30,13 @@ const KERBEROS_VOLUME_NAME: &str = "kerberos"; #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("object {hbase} is missing namespace"))] - ObjectMissingNamespace { - hbase: ObjectRef, - }, - - #[snafu(display("failed to add Kerberos secret volume"))] - AddKerberosSecretVolume { + #[snafu(display("failed to build Kerberos secret volume"))] + BuildKerberosSecretVolume { source: stackable_operator::builder::pod::volume::SecretOperatorVolumeSourceBuilderError, }, - #[snafu(display("failed to add TLS secret volume"))] - AddTlsSecretVolume { + #[snafu(display("failed to build TLS secret volume"))] + BuildTlsSecretVolume { source: stackable_operator::builder::pod::volume::SecretOperatorVolumeSourceBuilderError, }, @@ -56,14 +50,11 @@ pub enum Error { } pub fn kerberos_config_properties( - hbase: &v1alpha1::HbaseCluster, + hbase_name: &str, + hbase_namespace: &str, cluster_info: &KubernetesClusterInfo, -) -> Result, Error> { - if !hbase.has_kerberos_enabled() { - return Ok(BTreeMap::new()); - } - - let principal_host_part = principal_host_part(hbase, cluster_info)?; +) -> BTreeMap { + let principal_host_part = principal_host_part(hbase_name, hbase_namespace, cluster_info); let mut config = BTreeMap::from([ // Kerberos settings @@ -126,18 +117,15 @@ pub fn kerberos_config_properties( ("hbase.rest.ssl.keystore.type".to_string(), "pkcs12".to_string()), ]); config.extend(kerberos_principals(&principal_host_part)); - Ok(config) + config } pub fn kerberos_discovery_config_properties( - hbase: &v1alpha1::HbaseCluster, + hbase_name: &str, + hbase_namespace: &str, cluster_info: &KubernetesClusterInfo, -) -> Result, Error> { - if !hbase.has_kerberos_enabled() { - return Ok(BTreeMap::new()); - } - - let principal_host_part = principal_host_part(hbase, cluster_info)?; +) -> BTreeMap { + let principal_host_part = principal_host_part(hbase_name, hbase_namespace, cluster_info); let mut config = BTreeMap::from([ ( @@ -148,14 +136,10 @@ pub fn kerberos_discovery_config_properties( ("hbase.ssl.enabled".to_string(), "true".to_string()), ]); config.extend(kerberos_principals(&principal_host_part)); - Ok(config) + config } -pub fn kerberos_ssl_server_settings(hbase: &v1alpha1::HbaseCluster) -> BTreeMap { - if !hbase.has_https_enabled() { - return BTreeMap::new(); - } - +pub fn kerberos_ssl_server_settings() -> BTreeMap { let mut settings = truststore_settings("server"); settings.extend([ ( @@ -171,11 +155,7 @@ pub fn kerberos_ssl_server_settings(hbase: &v1alpha1::HbaseCluster) -> BTreeMap< settings } -pub fn kerberos_ssl_client_settings(hbase: &v1alpha1::HbaseCluster) -> BTreeMap { - if !hbase.has_https_enabled() { - return BTreeMap::new(); - } - +pub fn kerberos_ssl_client_settings() -> BTreeMap { truststore_settings("client") } @@ -197,7 +177,7 @@ pub fn add_kerberos_pod_config( .with_kerberos_service_name(kerberos_service_name()) .with_kerberos_service_name("HTTP") .build() - .context(AddKerberosSecretVolumeSnafu)?; + .context(BuildKerberosSecretVolumeSnafu)?; pb.add_volume( VolumeBuilder::new(KERBEROS_VOLUME_NAME) .ephemeral(kerberos_secret_operator_volume) @@ -230,7 +210,7 @@ pub fn add_kerberos_pod_config( .with_tls_pkcs12_password(TLS_STORE_PASSWORD) .with_auto_tls_cert_lifetime(requested_secret_lifetime) .build() - .context(AddTlsSecretVolumeSnafu)?, + .context(BuildTlsSecretVolumeSnafu)?, ) .build(), ) @@ -279,17 +259,12 @@ fn truststore_settings(role: &str) -> BTreeMap { } fn principal_host_part( - hbase: &v1alpha1::HbaseCluster, + hbase_name: &str, + hbase_namespace: &str, cluster_info: &KubernetesClusterInfo, -) -> Result { - let hbase_name = hbase.name_any(); - let hbase_namespace = hbase.namespace().context(ObjectMissingNamespaceSnafu { - hbase: ObjectRef::from_obj(hbase), - })?; +) -> String { let cluster_domain = &cluster_info.cluster_domain; - Ok(format!( - "{hbase_name}.{hbase_namespace}.svc.{cluster_domain}@${{env:KERBEROS_REALM}}" - )) + format!("{hbase_name}.{hbase_namespace}.svc.{cluster_domain}@${{env:KERBEROS_REALM}}") } /// We could have different service names depended on the role (e.g. "hbase-master", "hbase-regionserver" and diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index cd810bad..45ea006e 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -31,7 +31,7 @@ use crate::{ build::{ jvm::construct_role_specific_non_heap_jvm_args, kerberos::{ - self, kerberos_config_properties, kerberos_discovery_config_properties, + kerberos_config_properties, kerberos_discovery_config_properties, kerberos_ssl_client_settings, kerberos_ssl_server_settings, }, }, @@ -72,9 +72,6 @@ pub enum Error { role_group: String, }, - #[snafu(display("failed to resolve kerberos config"))] - AddKerberosConfig { source: kerberos::Error }, - #[snafu(display("failed to validate logging configuration"))] ValidateLoggingConfig { source: stackable_operator::v2::product_logging::framework::Error, @@ -216,17 +213,36 @@ pub fn validate_cluster( role_groups.insert(hbase_role, group_configs); } - let hbase_site_kerberos_config = - kerberos_config_properties(hbase, cluster_info).context(AddKerberosConfigSnafu)?; - let discovery_kerberos_config = kerberos_discovery_config_properties(hbase, cluster_info) - .context(AddKerberosConfigSnafu)?; - let ssl_server_settings = kerberos_ssl_server_settings(hbase); - let ssl_client_settings = kerberos_ssl_client_settings(hbase); - let name = get_cluster_name(hbase).context(GetClusterIdentitySnafu)?; let namespace = get_namespace(hbase).context(GetClusterIdentitySnafu)?; let uid = get_uid(hbase).context(GetClusterIdentitySnafu)?; + let kerberos_enabled = hbase.has_kerberos_enabled(); + let https_enabled = hbase.has_https_enabled(); + + // Kerberos- and TLS-related properties, pre-resolved here so the build step stays a pure + // function of `ValidatedCluster` (empty when the respective feature is disabled). + let hbase_site_kerberos_config = if kerberos_enabled { + kerberos_config_properties(name.as_ref(), namespace.as_ref(), cluster_info) + } else { + BTreeMap::new() + }; + let discovery_kerberos_config = if kerberos_enabled { + kerberos_discovery_config_properties(name.as_ref(), namespace.as_ref(), cluster_info) + } else { + BTreeMap::new() + }; + let ssl_server_settings = if https_enabled { + kerberos_ssl_server_settings() + } else { + BTreeMap::new() + }; + let ssl_client_settings = if https_enabled { + kerberos_ssl_client_settings() + } else { + BTreeMap::new() + }; + Ok(ValidatedCluster::new( name, namespace, @@ -235,8 +251,8 @@ pub fn validate_cluster( ValidatedClusterConfig { zookeeper_connection_information: dereferenced_objects.zookeeper_connection_information, hbase_opa_config: dereferenced_objects.hbase_opa_config, - kerberos_enabled: hbase.has_kerberos_enabled(), - https_enabled: hbase.has_https_enabled(), + kerberos_enabled, + https_enabled, kerberos_secret_class: hbase.kerberos_secret_class(), https_secret_class: hbase.https_secret_class(), hdfs_config_map_name: hbase.spec.cluster_config.hdfs_config_map_name.clone(), @@ -308,7 +324,7 @@ where } = validated.config; let non_heap_jvm_args = construct_role_specific_non_heap_jvm_args( - hbase, + hbase.has_kerberos_enabled(), &product_specific_common_config.jvm_argument_overrides, ); From db3a753a95a89d05251eeea8f32ad7a934523cf2 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 16 Jun 2026 19:06:34 +0200 Subject: [PATCH 53/56] fix: move hardcoded reused values to constants --- .../src/controller/build/kerberos.rs | 36 +++++++++++++------ rust/operator-binary/src/crd/mod.rs | 2 ++ 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/rust/operator-binary/src/controller/build/kerberos.rs b/rust/operator-binary/src/controller/build/kerberos.rs index 1a8fa0b3..cb69108a 100644 --- a/rust/operator-binary/src/controller/build/kerberos.rs +++ b/rust/operator-binary/src/controller/build/kerberos.rs @@ -17,7 +17,7 @@ use stackable_operator::{ use crate::{ controller::ValidatedCluster, - crd::{TLS_STORE_DIR, TLS_STORE_PASSWORD, TLS_STORE_VOLUME_NAME}, + crd::{TLS_STORE_DIR, TLS_STORE_PASSWORD, TLS_STORE_TYPE, TLS_STORE_VOLUME_NAME}, }; /// Mount path of the Kerberos secret volume (keytab + `krb5.conf`). @@ -27,6 +27,8 @@ pub const STACKABLE_KERBEROS_DIR: &str = "/stackable/kerberos"; pub const KRB5_CONFIG_PATH: &str = const_format::concatcp!(STACKABLE_KERBEROS_DIR, "/krb5.conf"); /// Name of the Kerberos secret volume. const KERBEROS_VOLUME_NAME: &str = "kerberos"; +/// The RPC/data-transfer quality-of-protection level used when Kerberos is enabled. +const PROTECTION_PRIVACY: &str = "privacy"; #[derive(Snafu, Debug)] pub enum Error { @@ -66,10 +68,13 @@ pub fn kerberos_config_properties( "hbase.security.authorization".to_string(), "true".to_string(), ), - ("hbase.rpc.protection".to_string(), "privacy".to_string()), + ( + "hbase.rpc.protection".to_string(), + PROTECTION_PRIVACY.to_string(), + ), ( "dfs.data.transfer.protection".to_string(), - "privacy".to_string(), + PROTECTION_PRIVACY.to_string(), ), ( "hbase.rpc.engine".to_string(), @@ -77,15 +82,15 @@ pub fn kerberos_config_properties( ), ( "hbase.master.keytab.file".to_string(), - "/stackable/kerberos/keytab".to_string(), + format!("{STACKABLE_KERBEROS_DIR}/keytab"), ), ( "hbase.regionserver.keytab.file".to_string(), - "/stackable/kerberos/keytab".to_string(), + format!("{STACKABLE_KERBEROS_DIR}/keytab"), ), ( "hbase.rest.keytab.file".to_string(), - "/stackable/kerberos/keytab".to_string(), + format!("{STACKABLE_KERBEROS_DIR}/keytab"), ), ( "hbase.coprocessor.master.classes".to_string(), @@ -101,7 +106,7 @@ pub fn kerberos_config_properties( ("hbase.rest.authentication.kerberos.principal".to_string(), format!( "HTTP/{principal_host_part}" )), - ("hbase.rest.authentication.kerberos.keytab".to_string(), "/stackable/kerberos/keytab".to_string()), + ("hbase.rest.authentication.kerberos.keytab".to_string(), format!("{STACKABLE_KERBEROS_DIR}/keytab")), // Enabled https as well ("hbase.ssl.enabled".to_string(), "true".to_string()), @@ -114,7 +119,7 @@ pub fn kerberos_config_properties( ("hbase.rest.ssl.enabled".to_string(), "true".to_string()), ("hbase.rest.ssl.keystore.store".to_string(), format!("{TLS_STORE_DIR}/keystore.p12")), ("hbase.rest.ssl.keystore.password".to_string(), TLS_STORE_PASSWORD.to_string()), - ("hbase.rest.ssl.keystore.type".to_string(), "pkcs12".to_string()), + ("hbase.rest.ssl.keystore.type".to_string(), TLS_STORE_TYPE.to_string()), ]); config.extend(kerberos_principals(&principal_host_part)); config @@ -132,7 +137,10 @@ pub fn kerberos_discovery_config_properties( "hbase.security.authentication".to_string(), "kerberos".to_string(), ), - ("hbase.rpc.protection".to_string(), "privacy".to_string()), + ( + "hbase.rpc.protection".to_string(), + PROTECTION_PRIVACY.to_string(), + ), ("hbase.ssl.enabled".to_string(), "true".to_string()), ]); config.extend(kerberos_principals(&principal_host_part)); @@ -146,7 +154,10 @@ pub fn kerberos_ssl_server_settings() -> BTreeMap { "ssl.server.keystore.location".to_string(), format!("{TLS_STORE_DIR}/keystore.p12"), ), - ("ssl.server.keystore.type".to_string(), "pkcs12".to_string()), + ( + "ssl.server.keystore.type".to_string(), + TLS_STORE_TYPE.to_string(), + ), ( "ssl.server.keystore.password".to_string(), TLS_STORE_PASSWORD.to_string(), @@ -250,7 +261,10 @@ fn truststore_settings(role: &str) -> BTreeMap { format!("ssl.{role}.truststore.location"), format!("{TLS_STORE_DIR}/truststore.p12"), ), - (format!("ssl.{role}.truststore.type"), "pkcs12".to_string()), + ( + format!("ssl.{role}.truststore.type"), + TLS_STORE_TYPE.to_string(), + ), ( format!("ssl.{role}.truststore.password"), TLS_STORE_PASSWORD.to_string(), diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index d660788e..04bf4c96 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -52,6 +52,8 @@ pub const CONFIG_DIR_NAME: &str = "/stackable/conf"; pub const TLS_STORE_DIR: &str = "/stackable/tls"; pub const TLS_STORE_VOLUME_NAME: &str = "tls"; pub const TLS_STORE_PASSWORD: &str = "changeit"; +/// The key- and truststore type used for all HBase TLS stores. +pub const TLS_STORE_TYPE: &str = "pkcs12"; pub const JVM_SECURITY_PROPERTIES_FILE: &str = "security.properties"; From 56ba62a3c7e7b04fd3b3a675ab0b6823813dbc79 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 16 Jun 2026 19:33:13 +0200 Subject: [PATCH 54/56] refactor: switch to generic rolegroupconfig --- .../src/controller/build/jvm.rs | 30 +++++++--------- .../controller/build/resource/config_map.rs | 11 +++--- .../controller/build/resource/statefulset.rs | 4 +-- rust/operator-binary/src/controller/mod.rs | 30 ++++++++-------- .../src/controller/validate.rs | 36 +++++++------------ 5 files changed, 48 insertions(+), 63 deletions(-) diff --git a/rust/operator-binary/src/controller/build/jvm.rs b/rust/operator-binary/src/controller/build/jvm.rs index a51d1eab..b69d2d69 100644 --- a/rust/operator-binary/src/controller/build/jvm.rs +++ b/rust/operator-binary/src/controller/build/jvm.rs @@ -1,11 +1,8 @@ use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_operator::{ - memory::{BinaryMultiple, MemoryQuantity}, - v2::jvm_argument_overrides::JvmArgumentOverrides, -}; +use stackable_operator::memory::{BinaryMultiple, MemoryQuantity}; use crate::{ - controller::build::kerberos::KRB5_CONFIG_PATH, + controller::{HbaseRoleGroupConfig, ValidatedCluster, build::kerberos::KRB5_CONFIG_PATH}, crd::{AnyServiceConfig, CONFIG_DIR_NAME, JVM_SECURITY_PROPERTIES_FILE}, }; @@ -44,23 +41,24 @@ pub fn construct_global_jvm_args(kerberos_enabled: bool) -> String { /// JVM arguments that are specifically for the role (server), so will *not* be used e.g. by CLI tools. /// Heap settings are excluded, as they go into `HBASE_HEAPSIZE`. /// -/// `merged_jvm_argument_overrides` is the role <- role-group merged [`JvmArgumentOverrides`] -/// produced by -/// [`with_validated_config`](stackable_operator::v2::role_utils::with_validated_config). The -/// operator-generated arguments below form the base that the user overrides are applied on top of. +/// The role <- role-group merged `jvmArgumentOverrides` (from `rg.product_specific_common_config`) +/// are applied on top of the operator-generated base arguments below. pub fn construct_role_specific_non_heap_jvm_args( - kerberos_enabled: bool, - merged_jvm_argument_overrides: &JvmArgumentOverrides, + cluster: &ValidatedCluster, + rg: &HbaseRoleGroupConfig, ) -> String { let mut operator_generated = vec![format!( "-Djava.security.properties={CONFIG_DIR_NAME}/{JVM_SECURITY_PROPERTIES_FILE}" )]; - if kerberos_enabled { + if cluster.has_kerberos_enabled() { operator_generated.push(format!("-Djava.security.krb5.conf={KRB5_CONFIG_PATH}")); } - let mut jvm_args = merged_jvm_argument_overrides.apply_to(operator_generated); + let mut jvm_args = rg + .product_specific_common_config + .jvm_argument_overrides + .apply_to(operator_generated); jvm_args.retain(|arg| !is_heap_jvm_argument(arg)); jvm_args.join(" ") @@ -136,10 +134,8 @@ mod tests { construct_hbase_heapsize_env(®ion_server.config.config).unwrap(); assert_eq!(global_jvm_args, ""); - // `non_heap_jvm_args` is the output of `construct_role_specific_non_heap_jvm_args`, - // pre-resolved during validation. assert_eq!( - region_server.non_heap_jvm_args, + construct_role_specific_non_heap_jvm_args(&validated_cluster, region_server), "-Djava.security.properties=/stackable/conf/security.properties" ); assert_eq!(hbase_heapsize_env, "819m"); @@ -202,7 +198,7 @@ mod tests { "-Djava.security.krb5.conf=/stackable/kerberos/krb5.conf" ); assert_eq!( - region_server.non_heap_jvm_args, + construct_role_specific_non_heap_jvm_args(&validated_cluster, region_server), "-Djava.security.properties=/stackable/conf/security.properties \ -Djava.security.krb5.conf=/stackable/kerberos/krb5.conf \ -Dhttps.proxyHost=proxy.my.corp \ diff --git a/rust/operator-binary/src/controller/build/resource/config_map.rs b/rust/operator-binary/src/controller/build/resource/config_map.rs index 879530d6..01ccddd4 100644 --- a/rust/operator-binary/src/controller/build/resource/config_map.rs +++ b/rust/operator-binary/src/controller/build/resource/config_map.rs @@ -11,9 +11,12 @@ use stackable_operator::{ use crate::{ controller::{ ValidatedCluster, - build::properties::{ - ConfigFileName, hbase_env, hbase_site, logging, security_properties, ssl_client, - ssl_server, + build::{ + jvm::construct_role_specific_non_heap_jvm_args, + properties::{ + ConfigFileName, hbase_env, hbase_site, logging, security_properties, ssl_client, + ssl_server, + }, }, }, crd::HbaseRole, @@ -81,7 +84,7 @@ pub fn build_rolegroup_config_map( merged_config, role, cluster_config.kerberos_enabled, - rg.non_heap_jvm_args.clone(), + construct_role_specific_non_heap_jvm_args(cluster, rg), overrides.hbase_env_sh.clone(), ) .context(BuildHbaseEnvSnafu)?; diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 1fcf7dc6..2c04d3f3 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -36,7 +36,7 @@ use stackable_operator::{ use crate::{ controller::{ - ValidatedCluster, ValidatedRoleGroupConfig, + HbaseRoleGroupConfig, ValidatedCluster, build::{ graceful_shutdown::{self, add_graceful_shutdown_config}, kerberos::{self, add_kerberos_pod_config}, @@ -99,7 +99,7 @@ pub fn build_rolegroup_statefulset( cluster: &ValidatedCluster, hbase_role: &HbaseRole, role_group_name: &RoleGroupName, - validated_rg_config: &ValidatedRoleGroupConfig, + validated_rg_config: &HbaseRoleGroupConfig, service_account: &ServiceAccount, ) -> Result { let resolved_product_image = &cluster.image; diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index 3c4b5f0c..ae01b603 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -10,12 +10,12 @@ pub use stackable_operator::v2::types::operator::RoleGroupName; use stackable_operator::{ builder::meta::ObjectMetaBuilder, commons::product_image_selection::ResolvedProductImage, - k8s_openapi::{api::core::v1::PodTemplateSpec, apimachinery::pkg::apis::meta::v1::ObjectMeta}, + k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta, kube::Resource, kvp::Labels, v2::{ HasName, HasUid, NameIsValidLabelValue, - builder::{meta::ownerreference_from_resource, pod::container::EnvVarSet}, + builder::meta::ownerreference_from_resource, kvp::label::{recommended_labels, role_group_selector}, role_group_utils::ResourceNames, types::{ @@ -71,7 +71,7 @@ pub struct ValidatedCluster { /// value. pub product_version: ProductVersion, pub cluster_config: ValidatedClusterConfig, - pub role_group_configs: BTreeMap>, + pub role_group_configs: BTreeMap>, pub role_configs: BTreeMap, } @@ -83,7 +83,7 @@ impl ValidatedCluster { uid: Uid, image: ResolvedProductImage, cluster_config: ValidatedClusterConfig, - role_group_configs: BTreeMap>, + role_group_configs: BTreeMap>, role_configs: BTreeMap, ) -> Self { // `app_version_label_value` is constructed to be a valid label value, so it is also a @@ -283,15 +283,13 @@ pub struct ValidatedHbaseConfig { pub logging: validate::ValidatedLogging, } -#[derive(Clone, Debug)] -pub struct ValidatedRoleGroupConfig { - /// The desired number of replicas (defaulted to 1 during validation). - pub replicas: u16, - pub config: ValidatedHbaseConfig, - pub config_overrides: v1alpha1::HbaseConfigOverrides, - pub env_overrides: EnvVarSet, - /// Merged (role <- role group) pod template overrides. - pub pod_overrides: PodTemplateSpec, - /// Pre-resolved role-specific non-heap JVM args (operator-generated + role/role-group overrides). - pub non_heap_jvm_args: String, -} +/// Per-rolegroup configuration: a v2 +/// [`RoleGroupConfig`](stackable_operator::v2::role_utils::RoleGroupConfig) over the validated +/// HBase config. The merged (role <- role group) `jvmArgumentOverrides` are available via +/// `product_specific_common_config` and applied at build time by +/// [`construct_role_specific_non_heap_jvm_args`](build::jvm::construct_role_specific_non_heap_jvm_args). +pub type HbaseRoleGroupConfig = stackable_operator::v2::role_utils::RoleGroupConfig< + ValidatedHbaseConfig, + stackable_operator::v2::role_utils::JavaCommonConfig, + v1alpha1::HbaseConfigOverrides, +>; diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 45ea006e..5573ca36 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -26,14 +26,11 @@ use strum::IntoEnumIterator; use crate::{ controller::{ - ValidatedCluster, ValidatedClusterConfig, ValidatedHbaseConfig, ValidatedRoleConfig, - ValidatedRoleGroupConfig, - build::{ - jvm::construct_role_specific_non_heap_jvm_args, - kerberos::{ - kerberos_config_properties, kerberos_discovery_config_properties, - kerberos_ssl_client_settings, kerberos_ssl_server_settings, - }, + HbaseRoleGroupConfig, ValidatedCluster, ValidatedClusterConfig, ValidatedHbaseConfig, + ValidatedRoleConfig, + build::kerberos::{ + kerberos_config_properties, kerberos_discovery_config_properties, + kerberos_ssl_client_settings, kerberos_ssl_server_settings, }, dereference::DereferencedObjects, }, @@ -156,7 +153,6 @@ pub fn validate_cluster( for hbase_role in HbaseRole::iter() { let group_configs = match hbase_role { HbaseRole::Master => validate_role_group_configs( - hbase, hbase.spec.masters.as_ref(), HbaseConfigFragment::default_config( &hbase_role, @@ -167,7 +163,6 @@ pub fn validate_cluster( &vector_aggregator_config_map_name, )?, HbaseRole::RegionServer => validate_role_group_configs( - hbase, hbase.spec.region_servers.as_ref(), RegionServerConfigFragment::default_config( &hbase_role, @@ -178,7 +173,6 @@ pub fn validate_cluster( &vector_aggregator_config_map_name, )?, HbaseRole::RestServer => validate_role_group_configs( - hbase, hbase.spec.rest_servers.as_ref(), HbaseConfigFragment::default_config( &hbase_role, @@ -275,20 +269,18 @@ pub fn validate_cluster( /// `jvmArgumentOverrides` (role group wins) into a single merged /// [`RoleGroup`](stackable_operator::role_utils::RoleGroup). The per-role validated config /// is wrapped into [`AnyServiceConfig`] via `wrap`; the merged `envOverrides` are converted -/// into an [`EnvVarSet`] (validating each name eagerly), and the role-specific non-heap JVM -/// args are pre-resolved from the merged `jvmArgumentOverrides` so the build step stays a -/// pure function of [`ValidatedCluster`]. +/// into an [`EnvVarSet`] (validating each name eagerly). The merged `jvmArgumentOverrides` are +/// kept in `product_specific_common_config` and applied at build time. /// /// Returns an empty map if the role is not configured. fn validate_role_group_configs( - hbase: &v1alpha1::HbaseCluster, role: Option< &Role, >, default_config: Config, wrap: fn(ValidatedConfig) -> AnyServiceConfig, vector_aggregator_config_map_name: &Option, -) -> Result, Error> +) -> Result, Error> where Config: Clone + Merge, ValidatedConfig: FromFragment, @@ -318,16 +310,11 @@ where config, config_overrides, env_overrides, - cli_overrides: _, + cli_overrides, pod_overrides, product_specific_common_config, } = validated.config; - let non_heap_jvm_args = construct_role_specific_non_heap_jvm_args( - hbase.has_kerberos_enabled(), - &product_specific_common_config.jvm_argument_overrides, - ); - // Convert the merged env-override HashMap into an EnvVarSet, validating each name // eagerly. Keys are unique (HashMap), so insertion order is irrelevant. let mut env_overrides_set = EnvVarSet::new(); @@ -345,13 +332,14 @@ where // time. The build step then consumes the validated logging instead of the raw config. let logging = validate_logging(config.logging(), vector_aggregator_config_map_name)?; - let validated = ValidatedRoleGroupConfig { + let validated = HbaseRoleGroupConfig { replicas: validated.replicas.unwrap_or(1), config: ValidatedHbaseConfig { config, logging }, config_overrides, env_overrides: env_overrides_set, + cli_overrides, pod_overrides, - non_heap_jvm_args, + product_specific_common_config, }; Ok((role_group_name.clone(), validated)) }) From f5e26764c9d5baa2cf882d46a52cfbfd56862f46 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 17 Jun 2026 12:18:51 +0200 Subject: [PATCH 55/56] refactor: remove has_kerberos_enabled and has_https_enabled from ValidatedClusterConfig --- .../src/controller/build/resource/config_map.rs | 2 +- rust/operator-binary/src/controller/mod.rs | 9 ++++----- rust/operator-binary/src/controller/validate.rs | 2 -- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/rust/operator-binary/src/controller/build/resource/config_map.rs b/rust/operator-binary/src/controller/build/resource/config_map.rs index 01ccddd4..a1048cfb 100644 --- a/rust/operator-binary/src/controller/build/resource/config_map.rs +++ b/rust/operator-binary/src/controller/build/resource/config_map.rs @@ -83,7 +83,7 @@ pub fn build_rolegroup_config_map( let hbase_env_sh = hbase_env::build( merged_config, role, - cluster_config.kerberos_enabled, + cluster.has_kerberos_enabled(), construct_role_specific_non_heap_jvm_args(cluster, rg), overrides.hbase_env_sh.clone(), ) diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index ae01b603..d40f0c24 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -183,12 +183,14 @@ impl ValidatedCluster { /// Mirrors [`v1alpha1::HbaseCluster::has_kerberos_enabled`], derived here from the validated /// config so build steps don't have to re-read the raw cluster. pub fn has_kerberos_enabled(&self) -> bool { - self.cluster_config.kerberos_enabled + self.cluster_config.kerberos_secret_class.is_some() } /// Whether HTTPS is enabled for this cluster. + /// + /// Derived from the validated config (a TLS `SecretClass` was configured). pub fn has_https_enabled(&self) -> bool { - self.cluster_config.https_enabled + self.cluster_config.https_secret_class.is_some() } } @@ -244,9 +246,6 @@ impl NameIsValidLabelValue for ValidatedCluster { pub struct ValidatedClusterConfig { // Pre-resolved OPA connection configuration. pub hbase_opa_config: Option, - pub kerberos_enabled: bool, - /// Whether HTTPS is enabled (a TLS `SecretClass` was configured). - pub https_enabled: bool, /// The Kerberos `SecretClass` name, if Kerberos is enabled. pub kerberos_secret_class: Option, /// The HTTPS/TLS `SecretClass` name, if HTTPS is enabled. diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 5573ca36..5c39e3b3 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -245,8 +245,6 @@ pub fn validate_cluster( ValidatedClusterConfig { zookeeper_connection_information: dereferenced_objects.zookeeper_connection_information, hbase_opa_config: dereferenced_objects.hbase_opa_config, - kerberos_enabled, - https_enabled, kerberos_secret_class: hbase.kerberos_secret_class(), https_secret_class: hbase.https_secret_class(), hdfs_config_map_name: hbase.spec.cluster_config.hdfs_config_map_name.clone(), From 889daf32a555ce8d59c08319a59147ad603f396e Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 17 Jun 2026 12:30:32 +0200 Subject: [PATCH 56/56] refactor: remove calulcated properties from Validated* structs --- .../src/controller/build/jvm.rs | 4 +- .../src/controller/build/kerberos.rs | 52 +++++++++++++++++++ .../controller/build/resource/config_map.rs | 9 ++-- .../controller/build/resource/discovery.rs | 13 +++-- rust/operator-binary/src/controller/mod.rs | 12 ----- .../src/controller/validate.rs | 39 +------------- rust/operator-binary/src/crd/mod.rs | 8 --- rust/operator-binary/src/hbase_controller.rs | 14 +++-- rust/operator-binary/src/test_utils.rs | 12 +---- 9 files changed, 81 insertions(+), 82 deletions(-) diff --git a/rust/operator-binary/src/controller/build/jvm.rs b/rust/operator-binary/src/controller/build/jvm.rs index b69d2d69..a0ca96be 100644 --- a/rust/operator-binary/src/controller/build/jvm.rs +++ b/rust/operator-binary/src/controller/build/jvm.rs @@ -129,7 +129,7 @@ mod tests { let region_server = &validated_cluster.role_group_configs[&HbaseRole::RegionServer] [&test_utils::role_group_name("default")]; - let global_jvm_args = construct_global_jvm_args(hbase.has_kerberos_enabled()); + let global_jvm_args = construct_global_jvm_args(validated_cluster.has_kerberos_enabled()); let hbase_heapsize_env = construct_hbase_heapsize_env(®ion_server.config.config).unwrap(); @@ -189,7 +189,7 @@ mod tests { let region_server = &validated_cluster.role_group_configs[&HbaseRole::RegionServer] [&test_utils::role_group_name("default")]; - let global_jvm_args = construct_global_jvm_args(hbase.has_kerberos_enabled()); + let global_jvm_args = construct_global_jvm_args(validated_cluster.has_kerberos_enabled()); let hbase_heapsize_env = construct_hbase_heapsize_env(®ion_server.config.config).unwrap(); diff --git a/rust/operator-binary/src/controller/build/kerberos.rs b/rust/operator-binary/src/controller/build/kerberos.rs index cb69108a..fd87dc19 100644 --- a/rust/operator-binary/src/controller/build/kerberos.rs +++ b/rust/operator-binary/src/controller/build/kerberos.rs @@ -51,6 +51,58 @@ pub enum Error { }, } +/// The `hbase-site.xml` Kerberos properties for `cluster`, gated on Kerberos being enabled +/// (empty when disabled). Derived in the build step from the validated cluster. +pub fn hbase_site_kerberos_config( + cluster: &ValidatedCluster, + cluster_info: &KubernetesClusterInfo, +) -> BTreeMap { + if cluster.has_kerberos_enabled() { + kerberos_config_properties( + cluster.name.as_ref(), + cluster.namespace.as_ref(), + cluster_info, + ) + } else { + BTreeMap::new() + } +} + +/// The Kerberos properties for the discovery `hbase-site.xml` exposed to clients, gated on +/// Kerberos being enabled (empty when disabled). +pub fn discovery_kerberos_config( + cluster: &ValidatedCluster, + cluster_info: &KubernetesClusterInfo, +) -> BTreeMap { + if cluster.has_kerberos_enabled() { + kerberos_discovery_config_properties( + cluster.name.as_ref(), + cluster.namespace.as_ref(), + cluster_info, + ) + } else { + BTreeMap::new() + } +} + +/// The `ssl-server.xml` settings for `cluster`, gated on HTTPS being enabled (empty when disabled). +pub fn ssl_server_settings(cluster: &ValidatedCluster) -> BTreeMap { + if cluster.has_https_enabled() { + kerberos_ssl_server_settings() + } else { + BTreeMap::new() + } +} + +/// The `ssl-client.xml` settings for `cluster`, gated on HTTPS being enabled (empty when disabled). +pub fn ssl_client_settings(cluster: &ValidatedCluster) -> BTreeMap { + if cluster.has_https_enabled() { + kerberos_ssl_client_settings() + } else { + BTreeMap::new() + } +} + pub fn kerberos_config_properties( hbase_name: &str, hbase_namespace: &str, diff --git a/rust/operator-binary/src/controller/build/resource/config_map.rs b/rust/operator-binary/src/controller/build/resource/config_map.rs index a1048cfb..508b9745 100644 --- a/rust/operator-binary/src/controller/build/resource/config_map.rs +++ b/rust/operator-binary/src/controller/build/resource/config_map.rs @@ -5,6 +5,7 @@ use stackable_operator::{ builder::configmap::ConfigMapBuilder, k8s_openapi::api::core::v1::ConfigMap, product_logging::framework::VECTOR_CONFIG_FILE, + utils::cluster_info::KubernetesClusterInfo, v2::{config_file_writer::PropertiesWriterError, types::operator::RoleGroupName}, }; @@ -13,6 +14,7 @@ use crate::{ ValidatedCluster, build::{ jvm::construct_role_specific_non_heap_jvm_args, + kerberos, properties::{ ConfigFileName, hbase_env, hbase_site, logging, security_properties, ssl_client, ssl_server, @@ -48,6 +50,7 @@ type Result = std::result::Result; pub fn build_rolegroup_config_map( cluster: &ValidatedCluster, + cluster_info: &KubernetesClusterInfo, role: &HbaseRole, role_group_name: &RoleGroupName, ) -> Result { @@ -75,7 +78,7 @@ pub fn build_rolegroup_config_map( cluster_config .zookeeper_connection_information .as_hbase_settings(), - cluster_config.hbase_site_kerberos_config.clone(), + kerberos::hbase_site_kerberos_config(cluster, cluster_info), cluster_config.hbase_opa_config.as_ref(), overrides.hbase_site_xml.clone(), ); @@ -90,11 +93,11 @@ pub fn build_rolegroup_config_map( .context(BuildHbaseEnvSnafu)?; let ssl_server_xml = ssl_server::build( - cluster_config.ssl_server_settings.clone(), + kerberos::ssl_server_settings(cluster), overrides.ssl_server_xml.clone(), ); let ssl_client_xml = ssl_client::build( - cluster_config.ssl_client_settings.clone(), + kerberos::ssl_client_settings(cluster), overrides.ssl_client_xml.clone(), ); diff --git a/rust/operator-binary/src/controller/build/resource/discovery.rs b/rust/operator-binary/src/controller/build/resource/discovery.rs index ec14446d..7113645e 100644 --- a/rust/operator-binary/src/controller/build/resource/discovery.rs +++ b/rust/operator-binary/src/controller/build/resource/discovery.rs @@ -6,11 +6,15 @@ use stackable_operator::{ k8s_openapi::api::core::v1::ConfigMap, kube::Resource, kvp::ObjectLabels, + utils::cluster_info::KubernetesClusterInfo, v2::{builder::meta::ownerreference_from_resource, config_file_writer::to_hadoop_xml}, }; use crate::{ - controller::{ValidatedCluster, build::properties::ConfigFileName}, + controller::{ + ValidatedCluster, + build::{kerberos, properties::ConfigFileName}, + }, crd::{APP_NAME, HbaseRole, OPERATOR_NAME}, }; @@ -34,13 +38,16 @@ pub enum Error { } /// Creates a discovery config map containing the `hbase-site.xml` for clients. -pub fn build_discovery_config_map(cluster: &ValidatedCluster) -> Result { +pub fn build_discovery_config_map( + cluster: &ValidatedCluster, + cluster_info: &KubernetesClusterInfo, +) -> Result { let cluster_config = &cluster.cluster_config; let mut hbase_site = cluster_config .zookeeper_connection_information .as_hbase_settings(); - hbase_site.extend(cluster_config.discovery_kerberos_config.clone()); + hbase_site.extend(kerberos::discovery_kerberos_config(cluster, cluster_info)); ConfigMapBuilder::new() .metadata( diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index d40f0c24..558d8d2a 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -179,9 +179,6 @@ impl ValidatedCluster { } /// Whether Kerberos is enabled for this cluster. - /// - /// Mirrors [`v1alpha1::HbaseCluster::has_kerberos_enabled`], derived here from the validated - /// config so build steps don't have to re-read the raw cluster. pub fn has_kerberos_enabled(&self) -> bool { self.cluster_config.kerberos_secret_class.is_some() } @@ -252,15 +249,6 @@ pub struct ValidatedClusterConfig { pub https_secret_class: Option, /// The HDFS discovery ConfigMap name the cluster connects to. pub hdfs_config_map_name: ConfigMapName, - // Pre-resolved kerberos properties for hbase-site.xml (empty when kerberos is disabled). - pub hbase_site_kerberos_config: BTreeMap, - // Pre-resolved kerberos properties for the discovery `hbase-site.xml` exposed to clients - // (empty when kerberos is disabled). - pub discovery_kerberos_config: BTreeMap, - // Pre-resolved ssl-server.xml settings (empty when HTTPS is disabled). - pub ssl_server_settings: BTreeMap, - // Pre-resolved ssl-client.xml settings (empty when HTTPS is disabled). - pub ssl_client_settings: BTreeMap, // Pre-resolved zookeeper connection settings. pub zookeeper_connection_information: ZookeeperConnectionInformation, } diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 5c39e3b3..7309a685 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -10,7 +10,6 @@ use stackable_operator::{ kube::ResourceExt, product_logging::spec::Logging, role_utils::{CommonConfiguration, GenericRoleConfig, Role}, - utils::cluster_info::KubernetesClusterInfo, v2::{ builder::pod::container::{self, EnvVarName, EnvVarSet}, controller_utils::{get_cluster_name, get_namespace, get_uid}, @@ -27,12 +26,7 @@ use strum::IntoEnumIterator; use crate::{ controller::{ HbaseRoleGroupConfig, ValidatedCluster, ValidatedClusterConfig, ValidatedHbaseConfig, - ValidatedRoleConfig, - build::kerberos::{ - kerberos_config_properties, kerberos_discovery_config_properties, - kerberos_ssl_client_settings, kerberos_ssl_server_settings, - }, - dereference::DereferencedObjects, + ValidatedRoleConfig, dereference::DereferencedObjects, }, crd::{ AnyServiceConfig, Container, HbaseConfigFragment, HbaseRole, RegionServerConfigFragment, @@ -122,7 +116,6 @@ fn validate_logging( pub fn validate_cluster( hbase: &v1alpha1::HbaseCluster, image_repository: &str, - cluster_info: &KubernetesClusterInfo, dereferenced_objects: DereferencedObjects, ) -> Result { let resolved_product_image = hbase @@ -211,32 +204,6 @@ pub fn validate_cluster( let namespace = get_namespace(hbase).context(GetClusterIdentitySnafu)?; let uid = get_uid(hbase).context(GetClusterIdentitySnafu)?; - let kerberos_enabled = hbase.has_kerberos_enabled(); - let https_enabled = hbase.has_https_enabled(); - - // Kerberos- and TLS-related properties, pre-resolved here so the build step stays a pure - // function of `ValidatedCluster` (empty when the respective feature is disabled). - let hbase_site_kerberos_config = if kerberos_enabled { - kerberos_config_properties(name.as_ref(), namespace.as_ref(), cluster_info) - } else { - BTreeMap::new() - }; - let discovery_kerberos_config = if kerberos_enabled { - kerberos_discovery_config_properties(name.as_ref(), namespace.as_ref(), cluster_info) - } else { - BTreeMap::new() - }; - let ssl_server_settings = if https_enabled { - kerberos_ssl_server_settings() - } else { - BTreeMap::new() - }; - let ssl_client_settings = if https_enabled { - kerberos_ssl_client_settings() - } else { - BTreeMap::new() - }; - Ok(ValidatedCluster::new( name, namespace, @@ -248,10 +215,6 @@ pub fn validate_cluster( kerberos_secret_class: hbase.kerberos_secret_class(), https_secret_class: hbase.https_secret_class(), hdfs_config_map_name: hbase.spec.cluster_config.hdfs_config_map_name.clone(), - hbase_site_kerberos_config, - discovery_kerberos_config, - ssl_server_settings, - ssl_client_settings, }, role_groups, role_configs, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 04bf4c96..4cc6a7b5 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -214,10 +214,6 @@ impl v1alpha1::HbaseCluster { } } - pub fn has_kerberos_enabled(&self) -> bool { - self.kerberos_secret_class().is_some() - } - pub fn kerberos_secret_class(&self) -> Option { self.spec .cluster_config @@ -227,10 +223,6 @@ impl v1alpha1::HbaseCluster { .map(|k| k.secret_class.clone()) } - pub fn has_https_enabled(&self) -> bool { - self.https_secret_class().is_some() - } - pub fn https_secret_class(&self) -> Option { self.spec .cluster_config diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 26c358d5..666d58d7 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -167,7 +167,6 @@ pub async fn reconcile_hbase( let validated_cluster = crate::controller::validate::validate_cluster( hbase, &ctx.operator_environment.image_repository, - &client.kubernetes_cluster_info, dereferenced_objects, ) .context(ValidateSnafu)?; @@ -210,9 +209,13 @@ pub async fn reconcile_hbase( let rg_metrics_service = build_rolegroup_metrics_service(&validated_cluster, hbase_role, role_group_name); - let rg_configmap = - build_rolegroup_config_map(&validated_cluster, hbase_role, role_group_name) - .context(BuildRolegroupConfigMapSnafu)?; + let rg_configmap = build_rolegroup_config_map( + &validated_cluster, + &client.kubernetes_cluster_info, + hbase_role, + role_group_name, + ) + .context(BuildRolegroupConfigMapSnafu)?; let rg_statefulset = build_rolegroup_statefulset( &validated_cluster, hbase_role, @@ -268,7 +271,8 @@ pub async fn reconcile_hbase( // Discovery CM will fail to build until the rest of the cluster has been deployed, so do it last // so that failure won't inhibit the rest of the cluster from booting up. let discovery_cm = - build_discovery_config_map(&validated_cluster).context(BuildDiscoveryConfigMapSnafu)?; + build_discovery_config_map(&validated_cluster, &client.kubernetes_cluster_info) + .context(BuildDiscoveryConfigMapSnafu)?; cluster_resources .add(client, discovery_cm) .await diff --git a/rust/operator-binary/src/test_utils.rs b/rust/operator-binary/src/test_utils.rs index f08653e8..ad888f71 100644 --- a/rust/operator-binary/src/test_utils.rs +++ b/rust/operator-binary/src/test_utils.rs @@ -7,10 +7,7 @@ use std::str::FromStr; -use stackable_operator::{ - commons::networking::DomainName, utils::cluster_info::KubernetesClusterInfo, - v2::types::operator::RoleGroupName, -}; +use stackable_operator::v2::types::operator::RoleGroupName; use crate::{ controller::{ @@ -59,19 +56,12 @@ pub fn minimal_hbase() -> v1alpha1::HbaseCluster { hbase_from_yaml(MINIMAL_HBASE_YAML) } -pub fn cluster_info() -> KubernetesClusterInfo { - KubernetesClusterInfo { - cluster_domain: DomainName::try_from("cluster.local").expect("valid domain"), - } -} - /// Runs the real validation pipeline over `hbase`, with a fixed dereferenced ZooKeeper connection /// and no OPA. pub fn validated_cluster_from(hbase: &v1alpha1::HbaseCluster) -> ValidatedCluster { validate_cluster( hbase, "oci.example.org", - &cluster_info(), DereferencedObjects { zookeeper_connection_information: ZookeeperConnectionInformation::for_tests(), hbase_opa_config: None,