From 3ed0b2227a10553baa765dca70f88745321d5dde Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Jun 2026 21:10:10 +0200 Subject: [PATCH 01/48] chore: switch stackable-operator to smooth-operator branch Add a [patch] override redirecting stackable-operator to the smooth-operator branch of operator-rs (mirroring trino/hdfs operators), in preparation for removing product-config. No source changes were required; the framework API surface used by druid-operator is unchanged between the 0.111.0 tag and the smooth-operator branch. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 352 ++++++++++++++++++++++++++++------------------------- Cargo.toml | 1 + 2 files changed, 186 insertions(+), 167 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 39a18400..9af8ae8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -163,9 +163,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "axum" @@ -265,9 +265,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "2.11.1" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "84d7ced0ae9557296835c32bf1b1e02b44c746701f898460fb000d7eaa84f00a" [[package]] name = "block-buffer" @@ -280,9 +280,9 @@ dependencies = [ [[package]] name = "built" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4ad8f11f288f48ca24471bbd51ac257aaeaaa07adae295591266b792902ae64" +checksum = "5c0e531d93d39c34eef561e929e8a7f86d77a5af08aac4f6d6e39976c51858e9" dependencies = [ "chrono", "git2", @@ -290,9 +290,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.20.2" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytes" @@ -302,9 +302,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "cc" -version = "1.2.61" +version = "1.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" dependencies = [ "find-msvc-tools", "jobserver", @@ -320,9 +320,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "num-traits", @@ -613,9 +613,9 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", @@ -676,9 +676,9 @@ dependencies = [ [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "elliptic-curve" @@ -939,9 +939,9 @@ checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-timer" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" [[package]] name = "futures-util" @@ -998,15 +998,14 @@ dependencies = [ [[package]] name = "git2" -version = "0.20.4" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" +checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" dependencies = [ "bitflags", "libc", "libgit2-sys", "log", - "url", ] [[package]] @@ -1040,9 +1039,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" dependencies = [ "atomic-waker", "bytes", @@ -1070,9 +1069,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.17.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "heck" @@ -1102,9 +1101,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" dependencies = [ "bytes", "itoa", @@ -1153,9 +1152,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hyper" -version = "1.9.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" dependencies = [ "atomic-waker", "bytes", @@ -1366,7 +1365,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.17.0", + "hashbrown 0.17.1", ] [[package]] @@ -1384,16 +1383,6 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" -[[package]] -name = "iri-string" -version = "0.7.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -1428,9 +1417,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.24" +version = "0.2.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" +checksum = "4603d3033e49e2b0e31229fcab20a5d40089c607d975cd9c80551dc69eed9102" dependencies = [ "jiff-static", "jiff-tzdb-platform", @@ -1438,14 +1427,14 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.61.2", + "windows-link", ] [[package]] name = "jiff-static" -version = "0.2.24" +version = "0.2.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" +checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47" dependencies = [ "proc-macro2", "quote", @@ -1479,9 +1468,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.97" +version = "0.3.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1840c94c045fbcf8ba2812c95db44499f7c64910a912551aaaa541decebcacf" +checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" dependencies = [ "cfg-if", "futures-util", @@ -1491,14 +1480,15 @@ dependencies = [ [[package]] name = "json-patch" -version = "4.1.0" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f300e415e2134745ef75f04562dd0145405c2f7fd92065db029ac4b16b57fe90" +checksum = "7421438de105a0827e44fadd05377727847d717c80ce29a229f85fd04c427b72" dependencies = [ "jsonptr", + "schemars", "serde", "serde_json", - "thiserror 1.0.69", + "thiserror 2.0.18", ] [[package]] @@ -1540,11 +1530,11 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#128e1afca7761d07058624091217b6d695fa790c" dependencies = [ "darling", "regex", - "snafu 0.9.0", + "snafu 0.9.1", ] [[package]] @@ -1687,9 +1677,9 @@ checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libgit2-sys" -version = "0.18.3+1.9.2" +version = "0.18.5+1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487" +checksum = "005d6ae6eac1912906073e069f7db60b1fa98e052a68227824afe3e3a1c59ca2" dependencies = [ "cc", "libc", @@ -1705,9 +1695,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libz-sys" -version = "1.1.28" +version = "1.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc3a226e576f50782b3305c5ccf458698f92798987f551c6a02efe8276721e22" +checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" dependencies = [ "cc", "libc", @@ -1732,9 +1722,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" [[package]] name = "matchers" @@ -1753,9 +1743,9 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" [[package]] name = "mime" @@ -1775,9 +1765,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi", @@ -1811,9 +1801,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" @@ -1859,15 +1849,14 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "openssl" -version = "0.10.78" +version = "0.10.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38c4372413cdaaf3cc79dd92d29d7d9f5ab09b51b10dded508fb90bb70b9222" +checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" dependencies = [ "bitflags", "cfg-if", "foreign-types", "libc", - "once_cell", "openssl-macros", "openssl-sys", ] @@ -1891,9 +1880,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.114" +version = "0.9.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13ce1245cd07fcc4cfdb438f7507b0c7e4f3849a69fd84d52374c66d83741bb6" +checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4" dependencies = [ "cc", "libc", @@ -1903,9 +1892,9 @@ dependencies = [ [[package]] name = "opentelemetry" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" dependencies = [ "futures-core", "futures-sink", @@ -1917,9 +1906,9 @@ dependencies = [ [[package]] name = "opentelemetry-appender-tracing" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef6a1ac5ca3accf562b8c306fa8483c85f4390f768185ab775f242f7fe8fdcc2" +checksum = "2c0080f0dc1d7c786f467cd85a4e395fcab11ee852004f39a29a18ab7c25d837" dependencies = [ "opentelemetry", "tracing", @@ -1929,9 +1918,9 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" +checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331" dependencies = [ "async-trait", "bytes", @@ -1942,9 +1931,9 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" +checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" dependencies = [ "http", "opentelemetry", @@ -1956,14 +1945,14 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tonic", - "tracing", + "tonic-types", ] [[package]] name = "opentelemetry-proto" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" +checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -1974,21 +1963,22 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e62e29dfe041afb8ed2a6c9737ab57db4907285d999ef8ad3a59092a36bdc846" +checksum = "6ca2f98a0437b427b4b08f19f1caa3c44db885a202bc12cfea13d6c702243d68" [[package]] name = "opentelemetry_sdk" -version = "0.31.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" +checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" dependencies = [ "futures-channel", "futures-executor", "futures-util", "opentelemetry", "percent-encoding", + "portable-atomic", "rand 0.9.4", "thiserror 2.0.18", "tokio", @@ -2115,18 +2105,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.11" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.11" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", @@ -2271,6 +2261,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.45" @@ -2410,9 +2409,9 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" -version = "0.12.28" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64", "bytes", @@ -2428,9 +2427,6 @@ dependencies = [ "log", "percent-encoding", "pin-project-lite", - "serde", - "serde_json", - "serde_urlencoded", "sync_wrapper", "tokio", "tower", @@ -2527,9 +2523,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.39" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c2c118cb077cca2822033836dfb1b975355dfb784b5e8da48f7b6c5db74e60e" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "log", "once_cell", @@ -2542,9 +2538,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -2730,9 +2726,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -2810,9 +2806,9 @@ dependencies = [ [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "signal-hook-registry" @@ -2873,11 +2869,11 @@ dependencies = [ [[package]] name = "snafu" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1d4bced6a69f90b2056c03dcff2c4737f98d6fb9e0853493996e1d253ca29c6" +checksum = "d1a012328be2e3f5d5f6f3218147ca02588cea4cb865e876849ab6debcf36522" dependencies = [ - "snafu-derive 0.9.0", + "snafu-derive 0.9.1", ] [[package]] @@ -2905,9 +2901,9 @@ dependencies = [ [[package]] name = "snafu-derive" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" +checksum = "5f103c50866b8743da9429b8a581d81a27c2d3a9c4ac7df8f8571c1dd7896eda" dependencies = [ "heck", "proc-macro2", @@ -2917,9 +2913,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", "windows-sys 0.61.2", @@ -2950,7 +2946,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#128e1afca7761d07058624091217b6d695fa790c" dependencies = [ "const-oid", "ecdsa", @@ -2962,7 +2958,7 @@ dependencies = [ "rsa", "sha2", "signature", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-shared", "tokio", "tokio-rustls", @@ -2990,7 +2986,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-operator", "strum", "tokio", @@ -2999,8 +2995,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.111.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +version = "0.111.1" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#128e1afca7761d07058624091217b6d695fa790c" dependencies = [ "base64", "clap", @@ -3024,7 +3020,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-operator-derive", "stackable-shared", "stackable-telemetry", @@ -3036,12 +3032,13 @@ dependencies = [ "tracing-appender", "tracing-subscriber", "url", + "uuid", ] [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#128e1afca7761d07058624091217b6d695fa790c" dependencies = [ "darling", "proc-macro2", @@ -3051,8 +3048,8 @@ dependencies = [ [[package]] name = "stackable-shared" -version = "0.1.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +version = "0.1.1" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#128e1afca7761d07058624091217b6d695fa790c" dependencies = [ "jiff", "k8s-openapi", @@ -3061,15 +3058,15 @@ dependencies = [ "semver", "serde", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "strum", "time", ] [[package]] name = "stackable-telemetry" -version = "0.6.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +version = "0.6.4" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#128e1afca7761d07058624091217b6d695fa790c" dependencies = [ "axum", "clap", @@ -3080,7 +3077,7 @@ dependencies = [ "opentelemetry-semantic-conventions", "opentelemetry_sdk", "pin-project", - "snafu 0.9.0", + "snafu 0.9.1", "strum", "tokio", "tower", @@ -3093,21 +3090,21 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#128e1afca7761d07058624091217b6d695fa790c" dependencies = [ "kube", "schemars", "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-versioned-macros", ] [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#128e1afca7761d07058624091217b6d695fa790c" dependencies = [ "convert_case", "convert_case_extras", @@ -3125,7 +3122,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#128e1afca7761d07058624091217b6d695fa790c" dependencies = [ "arc-swap", "async-trait", @@ -3141,7 +3138,7 @@ dependencies = [ "rand 0.9.4", "serde", "serde_json", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-certs", "stackable-shared", "stackable-telemetry", @@ -3348,9 +3345,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.1" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -3420,9 +3417,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.11+spec-1.1.0" +version = "0.25.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" +checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" dependencies = [ "indexmap", "toml_datetime", @@ -3441,9 +3438,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "base64", @@ -3468,15 +3465,26 @@ dependencies = [ [[package]] name = "tonic-prost" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" dependencies = [ "bytes", "prost", "tonic", ] +[[package]] +name = "tonic-types" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab1b02061f83d519bba3caa167f88f261ef05720ab8ebc954ade70de3348e8" +dependencies = [ + "prost", + "prost-types", + "tonic", +] + [[package]] name = "tower" version = "0.5.3" @@ -3498,9 +3506,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "base64", "bitflags", @@ -3508,13 +3516,13 @@ dependencies = [ "futures-util", "http", "http-body", - "iri-string", "mime", "pin-project-lite", "tower", "tower-layer", "tower-service", "tracing", + "url", ] [[package]] @@ -3588,9 +3596,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.32.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac28f2d093c6c477eaa76b23525478f38de514fa9aeb1285738d4b97a9552fc" +checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" dependencies = [ "js-sys", "opentelemetry", @@ -3641,9 +3649,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" -version = "1.20.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "ucd-trie" @@ -3659,9 +3667,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-segmentation" -version = "1.13.2" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] name = "unicode-xid" @@ -3706,6 +3714,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -3750,9 +3768,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.120" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df52b6d9b87e0c74c9edfa1eb2d9bf85e5d63515474513aa50fa181b3c4f5db1" +checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" dependencies = [ "cfg-if", "once_cell", @@ -3763,9 +3781,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.70" +version = "0.4.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af934872acec734c2d80e6617bbb5ff4f12b052dd8e6332b0817bce889516084" +checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" dependencies = [ "js-sys", "wasm-bindgen", @@ -3773,9 +3791,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.120" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b1041f495fb322e64aca85f5756b2172e35cd459376e67f2a6c9dffcedb103" +checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3783,9 +3801,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.120" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dcd0ff20416988a18ac686d4d4d0f6aae9ebf08a389ff5d29012b05af2a1b41" +checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" dependencies = [ "bumpalo", "proc-macro2", @@ -3796,18 +3814,18 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.120" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49757b3c82ebf16c57d69365a142940b384176c24df52a087fb748e2085359ea" +checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" -version = "0.3.97" +version = "0.3.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eadbac71025cd7b0834f20d1fe8472e8495821b4e9801eb0a60bd1f19827602" +checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" dependencies = [ "js-sys", "wasm-bindgen", @@ -3966,9 +3984,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" dependencies = [ "memchr", ] @@ -4001,15 +4019,15 @@ dependencies = [ [[package]] name = "xml" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8aa498d22c9bbaf482329839bc5620c46be275a19a812e9a22a2b07529a642a" +checksum = "636f85e5ca6488e96401b61eb7de54f4e44755c988af0f52cf90230c312a1a89" [[package]] name = "yoke" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -4030,18 +4048,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.48" +version = "0.8.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.48" +version = "0.8.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" dependencies = [ "proc-macro2", "quote", @@ -4050,9 +4068,9 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ "zerofrom-derive", ] diff --git a/Cargo.toml b/Cargo.toml index 3e7fa478..a6955133 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,5 +33,6 @@ tokio = { version = "1.40", features = ["full"] } tracing = "0.1" [patch."https://github.com/stackabletech/operator-rs.git"] +stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "smooth-operator" } # stackable-operator = { path = "../operator-rs/crates/stackable-operator" } # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } From 683e21bf30328d8153ea8ab168e07a672710e33c Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Jun 2026 21:14:20 +0200 Subject: [PATCH 02/48] feat: vendor java-properties writer (no product-config) Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 1 + Cargo.toml | 1 + rust/operator-binary/Cargo.toml | 1 + rust/operator-binary/src/controller.rs | 1 + rust/operator-binary/src/controller/build.rs | 3 + .../src/controller/build/properties/mod.rs | 5 ++ .../src/controller/build/properties/writer.rs | 78 +++++++++++++++++++ 7 files changed, 90 insertions(+) create mode 100644 rust/operator-binary/src/controller/build.rs create mode 100644 rust/operator-binary/src/controller/build/properties/mod.rs create mode 100644 rust/operator-binary/src/controller/build/properties/writer.rs diff --git a/Cargo.lock b/Cargo.lock index 9af8ae8a..e4f32e1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2978,6 +2978,7 @@ dependencies = [ "fnv", "futures 0.3.32", "indoc", + "java-properties", "openssl", "pin-project", "product-config", diff --git a/Cargo.toml b/Cargo.toml index a6955133..f0702dca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ const_format = "0.2" fnv = "1.0" futures = { version = "0.3", features = ["compat"] } indoc = "2.0" +java-properties = "2.0" openssl = "0.10" pin-project = "1.1" rstest = "0.26" diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 6bce1b1f..e66d5be8 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -18,6 +18,7 @@ const_format.workspace = true fnv.workspace = true futures.workspace = true indoc.workspace = true +java-properties.workspace = true openssl.workspace = true pin-project.workspace = true semver.workspace = true diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index e6885eab..af5448fa 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -86,6 +86,7 @@ use crate::{ service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, }; +mod build; mod dereference; mod validate; diff --git a/rust/operator-binary/src/controller/build.rs b/rust/operator-binary/src/controller/build.rs new file mode 100644 index 00000000..9703c659 --- /dev/null +++ b/rust/operator-binary/src/controller/build.rs @@ -0,0 +1,3 @@ +//! Build steps that turn a `ValidatedCluster` into Kubernetes resources. + +pub mod properties; diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs new file mode 100644 index 00000000..d35f0071 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -0,0 +1,5 @@ +//! Per-file builders for Druid `.properties` files. + +// The writer is vendored but not yet consumed; later refactor tasks wire it in. +#[allow(dead_code)] +pub mod writer; diff --git a/rust/operator-binary/src/controller/build/properties/writer.rs b/rust/operator-binary/src/controller/build/properties/writer.rs new file mode 100644 index 00000000..a74babf0 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/writer.rs @@ -0,0 +1,78 @@ +//! Writer for Java `.properties` files. +//! +//! Vendored from the `product-config` crate's `writer` module so the operator no +//! longer depends on `product-config` for rendering. + +use std::io::Write; + +use java_properties::{PropertiesError, PropertiesWriter}; +use snafu::{ResultExt, Snafu}; + +#[derive(Debug, Snafu)] +pub enum PropertiesWriterError { + #[snafu(display("failed to create properties file"))] + Properties { source: PropertiesError }, + + #[snafu(display("failed to convert properties file byte array to UTF-8"))] + FromUtf8 { source: std::string::FromUtf8Error }, +} + +/// Creates a common Java properties file string in the format: +/// `property_1=value_1\nproperty_2=value_2\n`. +pub fn to_java_properties_string<'a, T>(properties: T) -> Result +where + T: Iterator)>, +{ + let mut output = Vec::new(); + write_java_properties(&mut output, properties)?; + String::from_utf8(output).context(FromUtf8Snafu) +} + +/// Writes Java properties to the given writer. A `None` value is written as an +/// empty value (`key=`). +fn write_java_properties<'a, W, T>(writer: W, properties: T) -> Result<(), PropertiesWriterError> +where + W: Write, + T: Iterator)>, +{ + let mut writer = PropertiesWriter::new(writer); + for (k, v) in properties { + let property_value = v.as_deref().unwrap_or_default(); + writer.write(k, property_value).context(PropertiesSnafu)?; + } + writer.flush().context(PropertiesSnafu)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use super::*; + + fn props(pairs: &[(&str, Option<&str>)]) -> String { + let map: BTreeMap> = pairs + .iter() + .map(|(k, v)| (k.to_string(), v.map(str::to_string))) + .collect(); + to_java_properties_string(map.iter()).unwrap() + } + + #[test] + fn java_properties_renders_key_value() { + assert_eq!(props(&[("a", Some("1")), ("b", Some("2"))]), "a=1\nb=2\n"); + } + + #[test] + fn java_properties_renders_none_as_empty() { + assert_eq!(props(&[("none", None)]), "none=\n"); + } + + #[test] + fn java_properties_escapes_colon_in_value() { + assert_eq!( + props(&[("url", Some("file://this/location/file.abc"))]), + "url=file\\://this/location/file.abc\n" + ); + } +} From df1221135f692f97e1f4e1f643842ea4c4f15d13 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Jun 2026 21:27:55 +0200 Subject: [PATCH 03/48] feat: typed runtime/security properties builders with re-encoded defaults Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/controller/build/properties/mod.rs | 7 + .../build/properties/runtime_properties.rs | 206 ++++++++++++++++++ .../build/properties/security_properties.rs | 58 +++++ 3 files changed, 271 insertions(+) create mode 100644 rust/operator-binary/src/controller/build/properties/runtime_properties.rs create mode 100644 rust/operator-binary/src/controller/build/properties/security_properties.rs diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index d35f0071..b189cda3 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -3,3 +3,10 @@ // The writer is vendored but not yet consumed; later refactor tasks wire it in. #[allow(dead_code)] pub mod writer; + +// consumed by config_map builder in a later task +#[allow(dead_code)] +pub mod runtime_properties; +// consumed by config_map builder in a later task +#[allow(dead_code)] +pub mod security_properties; diff --git a/rust/operator-binary/src/controller/build/properties/runtime_properties.rs b/rust/operator-binary/src/controller/build/properties/runtime_properties.rs new file mode 100644 index 00000000..7adb1381 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/runtime_properties.rs @@ -0,0 +1,206 @@ +//! Builder for the static, role-specific `runtime.properties` defaults that were +//! previously injected by product-config's `recommendedValues`. +//! +//! Only includes a recommended value for a role where the property was +//! `required: true` for that role (verified against +//! `tests/templates/kuttl/smoke/53-assert.yaml.j2`). Dynamic, cluster-derived +//! values (ZooKeeper, extensions, metadata DB, deep storage, TLS, auth, ports, +//! resource-derived sizes) are added by the controller, not here. + +use std::collections::BTreeMap; + +use crate::crd::DruidRole; + +/// Defaults rendered for every role. +const ALL_ROLES: &[(&str, &str)] = &[ + ("druid.startup.logging.logProperties", "true"), + ( + "druid.monitoring.monitors", + "[\"org.apache.druid.java.util.metrics.JvmMonitor\"]", + ), + ("druid.emitter", "prometheus"), + ("druid.emitter.prometheus.strategy", "exporter"), + ("druid.emitter.prometheus.namespace", "druid"), + ( + "druid.indexer.logs.directory", + "/stackable/var/druid/indexing-logs", + ), +]; + +const BROKER: &[(&str, &str)] = &[("druid.processing.tmpDir", "/stackable/var/druid/processing")]; + +const COORDINATOR: &[(&str, &str)] = &[ + ("druid.coordinator.startDelay", "PT20S"), + ("druid.coordinator.period", "PT20S"), + ("druid.coordinator.asOverlord.enabled", "true"), + ( + "druid.coordinator.asOverlord.overlordService", + "druid/overlord", + ), + ("druid.indexer.queue.startDelay", "PT20S"), + ("druid.indexer.runner.type", "remote"), + ("druid.indexer.storage.type", "metadata"), +]; + +const HISTORICAL: &[(&str, &str)] = &[ + ("druid.historical.cache.useCache", "true"), + ("druid.historical.cache.populateCache", "true"), + ("druid.processing.tmpDir", "/stackable/var/druid/processing"), +]; + +const MIDDLEMANAGER: &[(&str, &str)] = &[ + ( + "druid.indexer.task.hadoopWorkingPath", + "/stackable/var/druid/hadoop-tmp", + ), + ( + "druid.indexer.task.baseTaskDir", + "/stackable/var/druid/task", + ), + ( + "druid.indexer.runner.javaOpts", + "-server -Xms256m -Xmx256m -XX:MaxDirectMemorySize=300m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager", + ), +]; + +const ROUTER: &[(&str, &str)] = &[ + ("druid.router.managementProxy.enabled", "true"), + ("druid.router.http.numConnections", "25"), +]; + +/// Static `recommendedValues` defaults for a role, as `key -> Some(value)`. +pub fn defaults(role: &DruidRole) -> BTreeMap> { + let role_specific: &[(&str, &str)] = match role { + DruidRole::Broker => BROKER, + DruidRole::Coordinator => COORDINATOR, + DruidRole::Historical => HISTORICAL, + DruidRole::MiddleManager => MIDDLEMANAGER, + DruidRole::Router => ROUTER, + }; + ALL_ROLES + .iter() + .chain(role_specific.iter()) + .map(|(k, v)| (k.to_string(), Some(v.to_string()))) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + // All expected values below are copied verbatim from the runtime.properties + // blocks of tests/templates/kuttl/smoke/53-assert.yaml.j2. + + #[test] + fn common_defaults_present_for_every_role() { + for role in [ + DruidRole::Broker, + DruidRole::Coordinator, + DruidRole::Historical, + DruidRole::MiddleManager, + DruidRole::Router, + ] { + let p = defaults(&role); + assert_eq!(p["druid.emitter"], Some("prometheus".to_string())); + assert_eq!( + p["druid.emitter.prometheus.namespace"], + Some("druid".to_string()) + ); + assert_eq!( + p["druid.emitter.prometheus.strategy"], + Some("exporter".to_string()) + ); + assert_eq!( + p["druid.monitoring.monitors"], + Some("[\"org.apache.druid.java.util.metrics.JvmMonitor\"]".to_string()) + ); + assert_eq!( + p["druid.indexer.logs.directory"], + Some("/stackable/var/druid/indexing-logs".to_string()) + ); + assert_eq!( + p["druid.startup.logging.logProperties"], + Some("true".to_string()) + ); + } + } + + #[test] + fn processing_tmpdir_only_for_broker_and_historical() { + // 53-assert: present for broker + historical, ABSENT for the other three. + assert!(defaults(&DruidRole::Broker).contains_key("druid.processing.tmpDir")); + assert!(defaults(&DruidRole::Historical).contains_key("druid.processing.tmpDir")); + assert!(!defaults(&DruidRole::Coordinator).contains_key("druid.processing.tmpDir")); + assert!(!defaults(&DruidRole::MiddleManager).contains_key("druid.processing.tmpDir")); + assert!(!defaults(&DruidRole::Router).contains_key("druid.processing.tmpDir")); + } + + #[test] + fn coordinator_defaults_match_snapshot() { + let p = defaults(&DruidRole::Coordinator); + assert_eq!( + p["druid.coordinator.asOverlord.enabled"], + Some("true".to_string()) + ); + assert_eq!( + p["druid.coordinator.asOverlord.overlordService"], + Some("druid/overlord".to_string()) + ); + assert_eq!(p["druid.coordinator.period"], Some("PT20S".to_string())); + assert_eq!(p["druid.coordinator.startDelay"], Some("PT20S".to_string())); + assert_eq!( + p["druid.indexer.queue.startDelay"], + Some("PT20S".to_string()) + ); + assert_eq!(p["druid.indexer.runner.type"], Some("remote".to_string())); + assert_eq!( + p["druid.indexer.storage.type"], + Some("metadata".to_string()) + ); + } + + #[test] + fn historical_defaults_match_snapshot() { + let p = defaults(&DruidRole::Historical); + assert_eq!( + p["druid.historical.cache.useCache"], + Some("true".to_string()) + ); + assert_eq!( + p["druid.historical.cache.populateCache"], + Some("true".to_string()) + ); + } + + #[test] + fn middlemanager_javaopts_exact() { + let p = defaults(&DruidRole::MiddleManager); + assert_eq!( + p["druid.indexer.runner.javaOpts"].as_deref(), + Some( + "-server -Xms256m -Xmx256m -XX:MaxDirectMemorySize=300m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager" + ) + ); + assert_eq!( + p["druid.indexer.task.baseTaskDir"], + Some("/stackable/var/druid/task".to_string()) + ); + assert_eq!( + p["druid.indexer.task.hadoopWorkingPath"], + Some("/stackable/var/druid/hadoop-tmp".to_string()) + ); + } + + #[test] + fn router_defaults_match_snapshot() { + let p = defaults(&DruidRole::Router); + assert_eq!( + p["druid.router.managementProxy.enabled"], + Some("true".to_string()) + ); + assert_eq!( + p["druid.router.http.numConnections"], + Some("25".to_string()) + ); + } +} diff --git a/rust/operator-binary/src/controller/build/properties/security_properties.rs b/rust/operator-binary/src/controller/build/properties/security_properties.rs new file mode 100644 index 00000000..eb73e681 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/security_properties.rs @@ -0,0 +1,58 @@ +//! Builder for `security.properties` (Druid's JVM security properties file). + +use std::collections::BTreeMap; + +const NETWORKADDRESS_CACHE_TTL: &str = "networkaddress.cache.ttl"; +const NETWORKADDRESS_CACHE_NEGATIVE_TTL: &str = "networkaddress.cache.negative.ttl"; + +const DEFAULT_NETWORKADDRESS_CACHE_TTL: &str = "30"; +const DEFAULT_NETWORKADDRESS_CACHE_NEGATIVE_TTL: &str = "0"; + +/// Build the `security.properties` key/value pairs (same defaults for all roles). +/// +/// `overrides` are the user's merged (role <- rolegroup) `security.properties` +/// config overrides, as returned by `DruidConfigOverrides::get_key_value_overrides`. +/// Override values win. +pub fn build(overrides: &BTreeMap>) -> BTreeMap> { + let mut props: BTreeMap> = BTreeMap::new(); + props.insert( + NETWORKADDRESS_CACHE_TTL.to_string(), + Some(DEFAULT_NETWORKADDRESS_CACHE_TTL.to_string()), + ); + props.insert( + NETWORKADDRESS_CACHE_NEGATIVE_TTL.to_string(), + Some(DEFAULT_NETWORKADDRESS_CACHE_NEGATIVE_TTL.to_string()), + ); + props.extend(overrides.iter().map(|(k, v)| (k.clone(), v.clone()))); + props +} + +#[cfg(test)] +mod tests { + use super::*; + + // Expected values copied verbatim from tests/templates/kuttl/smoke/53-assert.yaml.j2 + // (security.properties block, identical for every role): + // networkaddress.cache.negative.ttl=0 + // networkaddress.cache.ttl=30 + #[test] + fn defaults_match_snapshot() { + let props = build(&BTreeMap::new()); + assert_eq!(props["networkaddress.cache.ttl"], Some("30".to_string())); + assert_eq!( + props["networkaddress.cache.negative.ttl"], + Some("0".to_string()) + ); + assert_eq!(props.len(), 2); + } + + #[test] + fn override_wins() { + let ov = BTreeMap::from([( + "networkaddress.cache.ttl".to_string(), + Some("60".to_string()), + )]); + let props = build(&ov); + assert_eq!(props["networkaddress.cache.ttl"], Some("60".to_string())); + } +} From 626ba3f048cd96890a4ff4d1516b16c755f97d19 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Jun 2026 21:50:02 +0200 Subject: [PATCH 04/48] refactor: build config from ValidatedCluster instead of product-config Co-Authored-By: Claude Opus 4.8 (1M context) --- rust/operator-binary/src/controller.rs | 532 ++++-------------- rust/operator-binary/src/controller/build.rs | 1 + .../src/controller/build/config_map.rs | 327 +++++++++++ .../src/controller/build/properties/mod.rs | 6 - .../src/controller/validate.rs | 196 +++++-- rust/operator-binary/src/crd/mod.rs | 38 +- 6 files changed, 635 insertions(+), 465 deletions(-) create mode 100644 rust/operator-binary/src/controller/build/config_map.rs diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index af5448fa..5d0d48c7 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -1,23 +1,14 @@ //! Ensures that `Pod`s are configured and running for each [`DruidCluster`][v1alpha1] //! //! [v1alpha1]: v1alpha1::DruidCluster -use std::{ - collections::{BTreeMap, HashMap}, - str::FromStr, - sync::Arc, -}; +use std::sync::Arc; use const_format::concatcp; -use product_config::{ - ProductConfigManager, - types::PropertyNameKind, - writer::{PropertiesWriterError, to_java_properties_string}, -}; +use product_config::{ProductConfigManager, writer::PropertiesWriterError}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::{ self, - configmap::ConfigMapBuilder, meta::ObjectMetaBuilder, pod::{ PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, @@ -34,7 +25,7 @@ use stackable_operator::{ DeepMerge, api::{ apps::v1::{StatefulSet, StatefulSetSpec}, - core::v1::{ConfigMap, EnvVar, PersistentVolumeClaim, ServiceAccount}, + core::v1::{EnvVar, PersistentVolumeClaim, ServiceAccount}, }, apimachinery::pkg::apis::meta::v1::LabelSelector, }, @@ -64,25 +55,20 @@ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ authentication::DruidAuthenticationConfig, - config::jvm::construct_jvm_args, crd::{ - APP_NAME, AUTH_AUTHORIZER_OPA_URI, CommonRoleGroupConfig, Container, - DRUID_CONFIG_DIRECTORY, DS_BUCKET, DeepStorageSpec, DruidClusterStatus, DruidRole, - EXTENSIONS_LOADLIST, HDFS_CONFIG_DIRECTORY, JVM_CONFIG, JVM_SECURITY_PROPERTIES_FILE, + APP_NAME, CommonRoleGroupConfig, Container, DRUID_CONFIG_DIRECTORY, DeepStorageSpec, + DruidClusterStatus, DruidRole, HDFS_CONFIG_DIRECTORY, JVM_SECURITY_PROPERTIES_FILE, LOG_CONFIG_DIRECTORY, MAX_DRUID_LOG_FILES_SIZE, METRICS_PORT, METRICS_PORT_NAME, - OPERATOR_NAME, RUNTIME_PROPS, RW_CONFIG_DIRECTORY, S3_ACCESS_KEY, S3_ENDPOINT_URL, - S3_PATH_STYLE_ACCESS, S3_SECRET_KEY, STACKABLE_LOG_DIR, ZOOKEEPER_CONNECTION_STRING, - build_recommended_labels, build_string_list, security::DruidTlsSecurity, v1alpha1, + OPERATOR_NAME, RW_CONFIG_DIRECTORY, STACKABLE_LOG_DIR, build_recommended_labels, + security::DruidTlsSecurity, v1alpha1, }, discovery::{self, build_discovery_configmaps}, - extensions::get_extension_list, internal_secret::create_shared_internal_secret, listener::{ LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_group_listener, build_group_listener_pvc, group_listener_name, secret_volume_listener_scope, }, operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, - product_logging::extend_role_group_config_map, service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, }; @@ -90,6 +76,8 @@ mod build; mod dereference; mod validate; +use validate::DruidRoleGroupConfig; + pub const DRUID_CONTROLLER_NAME: &str = "druidcluster"; pub const FULL_CONTROLLER_NAME: &str = concatcp!(DRUID_CONTROLLER_NAME, '.', OPERATOR_NAME); @@ -105,6 +93,9 @@ const USERDATA_MOUNTPOINT: &str = "/stackable/userdata"; pub struct Ctx { pub client: stackable_operator::client::Client, + // Still constructed in `main.rs` but no longer consumed after the product-config removal in + // the validate/build steps. Removing it entirely is a follow-up task. + #[allow(dead_code)] pub product_config: ProductConfigManager, pub operator_environment: OperatorEnvironmentOptions, } @@ -312,6 +303,9 @@ pub enum Error { #[snafu(display("failed to validate cluster"))] ValidateCluster { source: validate::Error }, + #[snafu(display("failed to build rolegroup ConfigMap"))] + BuildConfigMap { source: build::config_map::Error }, + #[snafu(display("invalid metadata database connection"))] InvalidMetadataDatabaseConnection { source: stackable_operator::database_connections::Error, @@ -343,13 +337,8 @@ pub async fn reconcile_druid( .await .context(DereferenceSnafu)?; - let validated = validate::validate( - druid, - &dereferenced_objects, - &ctx.operator_environment, - &ctx.product_config, - ) - .context(ValidateClusterSnafu)?; + let validated = validate::validate(druid, &dereferenced_objects, &ctx.operator_environment) + .context(ValidateClusterSnafu)?; let mut cluster_resources = ClusterResources::new( APP_NAME, @@ -361,8 +350,6 @@ pub async fn reconcile_druid( ) .context(CreateClusterResourcesSnafu)?; - let merged_config = druid.merged_config().context(FailedToResolveConfigSnafu)?; - let (rbac_sa, rbac_rolebinding) = build_rbac_resources( druid, APP_NAME, @@ -383,30 +370,24 @@ pub async fn reconcile_druid( let mut ss_cond_builder = StatefulSetConditionBuilder::default(); - for (role_name, role_config) in validated.validated_role_config.iter() { - let druid_role = DruidRole::from_str(role_name).context(UnidentifiedDruidRoleSnafu { - role: role_name.to_string(), - })?; + for (druid_role, groups) in validated.role_group_configs.iter() { + let role_name = druid_role.to_string(); create_shared_internal_secret(druid, client, DRUID_CONTROLLER_NAME) .await .context(FailedInternalSecretCreationSnafu)?; - for (rolegroup_name, rolegroup_config) in role_config.iter() { + for (rolegroup_name, rg) in groups.iter() { let rolegroup = RoleGroupRef { cluster: ObjectRef::from_obj(druid), - role: role_name.into(), + role: role_name.clone(), role_group: rolegroup_name.into(), }; - let merged_rolegroup_config = merged_config - .common_config(&druid_role, rolegroup_name) - .context(FailedToResolveConfigSnafu)?; - let role_group_service_recommended_labels = build_recommended_labels( druid, DRUID_CONTROLLER_NAME, - &validated.resolved_product_image.app_version_label_value, + &validated.image.app_version_label_value, &rolegroup.role, &rolegroup.role_group, ); @@ -421,8 +402,8 @@ pub async fn reconcile_druid( let rg_headless_service = build_rolegroup_headless_service( druid, - &validated.druid_tls_security, - &druid_role, + &validated.cluster_config.druid_tls_security, + druid_role, &rolegroup, role_group_service_recommended_labels.clone(), role_group_service_selector.clone().into(), @@ -436,29 +417,24 @@ pub async fn reconcile_druid( ) .context(ServiceConfigurationSnafu)?; - let rg_configmap = build_rolegroup_config_map( - druid, - &validated.resolved_product_image, + let rg_configmap = build::config_map::build_rolegroup_config_map( + &validated, + druid_role, &rolegroup, - rolegroup_config, - &merged_rolegroup_config, - &validated.zookeeper_connection_string, - validated.opa_connection_string.as_deref(), - validated.s3_connection.as_ref(), - validated.deep_storage_bucket_name.as_deref(), - &validated.druid_tls_security, - &validated.druid_auth_config, - )?; + rg, + &validated.image, + druid, + ) + .context(BuildConfigMapSnafu)?; let rg_statefulset = build_rolegroup_statefulset( druid, - &validated.resolved_product_image, - &druid_role, + &validated.image, + druid_role, &rolegroup, - rolegroup_config, - &merged_rolegroup_config, - validated.s3_connection.as_ref(), - &validated.druid_tls_security, - &validated.druid_auth_config, + rg, + validated.cluster_config.s3_connection.as_ref(), + &validated.cluster_config.druid_tls_security, + &validated.cluster_config.druid_auth_config, &rbac_sa, )?; @@ -495,20 +471,20 @@ pub async fn reconcile_druid( } if let Some(listener_class) = druid_role.listener_class_name(druid) { - if let Some(listener_group_name) = group_listener_name(druid, &druid_role) { + if let Some(listener_group_name) = group_listener_name(druid, druid_role) { let role_group_listener = build_group_listener( druid, build_recommended_labels( druid, DRUID_CONTROLLER_NAME, - &validated.resolved_product_image.app_version_label_value, - role_name, + &validated.image.app_version_label_value, + &role_name, "none", ), listener_class.to_string(), listener_group_name, - &druid_role, - &validated.druid_tls_security, + druid_role, + &validated.cluster_config.druid_tls_security, ) .context(ListenerConfigurationSnafu)?; @@ -517,13 +493,13 @@ pub async fn reconcile_druid( .await .context(ApplyGroupListenerSnafu)?; - if druid_role == DruidRole::Router { + if *druid_role == DruidRole::Router { // discovery for discovery_cm in build_discovery_configmaps( druid, druid, - &validated.resolved_product_image, - &validated.druid_tls_security, + &validated.image, + &validated.cluster_config.druid_tls_security, listener, ) .await @@ -538,12 +514,12 @@ pub async fn reconcile_druid( } } - let role_config = druid.generic_role_config(&druid_role); + let role_config = druid.generic_role_config(druid_role); add_pdbs( &role_config.pod_disruption_budget, druid, - &druid_role, + druid_role, client, &mut cluster_resources, ) @@ -570,247 +546,6 @@ pub async fn reconcile_druid( Ok(Action::await_change()) } -#[allow(clippy::too_many_arguments)] -/// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator -fn build_rolegroup_config_map( - druid: &v1alpha1::DruidCluster, - resolved_product_image: &ResolvedProductImage, - rolegroup: &RoleGroupRef, - rolegroup_config: &HashMap>, - merged_rolegroup_config: &CommonRoleGroupConfig, - zk_connstr: &str, - opa_connstr: Option<&str>, - s3_conn: Option<&s3::v1alpha1::ConnectionSpec>, - deep_storage_bucket_name: Option<&str>, - druid_tls_security: &DruidTlsSecurity, - druid_auth_config: &Option, -) -> Result { - let druid_role = - DruidRole::from_str(&rolegroup.role).with_context(|_| UnidentifiedDruidRoleSnafu { - role: &rolegroup.role, - })?; - let role = druid.get_role(&druid_role); - let mut cm_conf_data = BTreeMap::new(); // filename -> filecontent - let metadata_database_connection_details = druid - .spec - .cluster_config - .metadata_database - .jdbc_connection_details("metadata") - .context(InvalidMetadataDatabaseConnectionSnafu)?; - - for (property_name_kind, config) in rolegroup_config { - let mut conf: BTreeMap> = Default::default(); - - match property_name_kind { - PropertyNameKind::File(file_name) if file_name == RUNTIME_PROPS => { - // Add any properties derived from storage manifests, such as segment cache locations. - // This has to be done here since there is no other suitable place for it. - // Previously such properties were added in the compute_files() function, - // but that code path is now incompatible with the design of fragment merging. - merged_rolegroup_config - .resources - .update_druid_config_file(&mut conf) - .context(UpdateDruidConfigFromResourcesSnafu)?; - // NOTE: druid.host can be set manually - if it isn't, the canonical host name of - // the local host is used. This should work with the agent and k8s host networking - // but might need to be revisited in the future - conf.insert( - ZOOKEEPER_CONNECTION_STRING.to_string(), - Some(zk_connstr.to_string()), - ); - - conf.insert( - EXTENSIONS_LOADLIST.to_string(), - Some(build_string_list(&get_extension_list( - druid, - druid_tls_security, - druid_auth_config, - ))), - ); - - if let Some(opa_str) = opa_connstr { - conf.insert( - AUTH_AUTHORIZER_OPA_URI.to_string(), - Some(opa_str.to_string()), - ); - }; - - conf.insert( - crate::crd::database::METADATA_STORAGE_TYPE.to_string(), - Some( - druid - .spec - .cluster_config - .metadata_database - .as_metadata_storage_type() - .to_string(), - ), - ); - - conf.insert( - crate::crd::database::METADATA_STORAGE_CONNECTOR_CONNECT_URI.to_string(), - Some( - metadata_database_connection_details - .connection_url - .to_string(), - ), - ); - - if let Some(EnvVar { - name: username_env_name, - .. - }) = &metadata_database_connection_details.username_env - { - conf.insert( - crate::crd::database::METADATA_STORAGE_USER.to_string(), - Some(format!("${{env:{username_env_name}}}",)), - ); - } - - if let Some(EnvVar { - name: password_env_name, - .. - }) = &metadata_database_connection_details.password_env - { - conf.insert( - crate::crd::database::METADATA_STORAGE_PASSWORD.to_string(), - Some(format!("${{env:{password_env_name}}}",)), - ); - } - - if let Some(s3) = s3_conn { - if !s3.region.is_default_config() { - // Raising this as warning instead of returning an error, better safe than sorry. - // It might still work out for the user. - tracing::warn!( - region = ?s3.region, - "You configured a non-default region on the S3Connection. - The S3Connection region field is ignored because Druid uses the AWS SDK v1, which ignores the region if the endpoint is set. \ - The host is a required field, therefore the endpoint will always be set." - ) - } - - conf.insert( - S3_ENDPOINT_URL.to_string(), - Some(s3.endpoint().context(ConfigureS3Snafu)?.to_string()), - ); - - if let Some((access_key_file, secret_key_file)) = s3.credentials_mount_paths() { - conf.insert( - S3_ACCESS_KEY.to_string(), - Some(format!("${{file:UTF-8:{access_key_file}}}")), - ); - conf.insert( - S3_SECRET_KEY.to_string(), - Some(format!("${{file:UTF-8:{secret_key_file}}}")), - ); - } - - conf.insert( - S3_PATH_STYLE_ACCESS.to_string(), - Some((s3.access_style == s3::v1alpha1::S3AccessStyle::Path).to_string()), - ); - } - conf.insert( - DS_BUCKET.to_string(), - deep_storage_bucket_name.map(str::to_string), - ); - - // add tls encryption / auth properties - druid_tls_security.add_tls_config_properties(&mut conf, &druid_role); - - if let Some(auth_config) = druid_auth_config { - conf.extend( - auth_config - .generate_runtime_properties_config(&druid_role) - .context(GenerateAuthenticationRuntimeSettingsSnafu)?, - ); - }; - - let transformed_config: BTreeMap> = config - .iter() - .map(|(k, v)| (k.clone(), Some(v.clone()))) - .collect(); - // extend the config to respect overrides - conf.extend(transformed_config); - - let runtime_properties = - to_java_properties_string(conf.iter()).context(PropertiesWriteSnafu)?; - cm_conf_data.insert(RUNTIME_PROPS.to_string(), runtime_properties); - } - - PropertyNameKind::File(file_name) if file_name == JVM_CONFIG => { - let (heap, direct) = merged_rolegroup_config - .resources - .get_memory_sizes(&druid_role) - .context(DeriveMemorySettingsSnafu)?; - let jvm_config = - construct_jvm_args(&druid_role, &role, &rolegroup.role_group, heap, direct) - .context(GetJvmConfigSnafu)?; - cm_conf_data.insert(JVM_CONFIG.to_string(), jvm_config); - } - - PropertyNameKind::File(file_name) if file_name == JVM_SECURITY_PROPERTIES_FILE => { - let jvm_sec_props: BTreeMap> = rolegroup_config - .get(&PropertyNameKind::File( - JVM_SECURITY_PROPERTIES_FILE.to_string(), - )) - .cloned() - .unwrap_or_default() - .into_iter() - .map(|(k, v)| (k, Some(v))) - .collect(); - cm_conf_data.insert( - JVM_SECURITY_PROPERTIES_FILE.to_string(), - to_java_properties_string(jvm_sec_props.iter()).with_context(|_| { - JvmSecurityPropertiesSnafu { - rolegroup: rolegroup.role_group.clone(), - } - })?, - ); - } - _ => {} - } - } - - let mut config_map_builder = ConfigMapBuilder::new(); - config_map_builder.metadata( - ObjectMetaBuilder::new() - .name_and_namespace(druid) - .name(rolegroup.object_name()) - .ownerreference_from_resource(druid, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&build_recommended_labels( - druid, - DRUID_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - &rolegroup.role, - &rolegroup.role_group, - )) - .context(MetadataBuildSnafu)? - .build(), - ); - - for (filename, file_content) in cm_conf_data.iter() { - config_map_builder.add_data(filename, file_content); - } - - extend_role_group_config_map( - rolegroup, - &merged_rolegroup_config.logging, - &mut config_map_builder, - ) - .context(InvalidLoggingConfigSnafu { - cm_name: rolegroup.object_name(), - })?; - - config_map_builder - .build() - .with_context(|_| BuildRoleGroupConfigSnafu { - rolegroup: rolegroup.clone(), - }) -} - #[allow(clippy::too_many_arguments)] /// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. /// @@ -821,13 +556,13 @@ fn build_rolegroup_statefulset( resolved_product_image: &ResolvedProductImage, role: &DruidRole, rolegroup_ref: &RoleGroupRef, - rolegroup_config: &HashMap>, - merged_rolegroup_config: &CommonRoleGroupConfig, + rg: &DruidRoleGroupConfig, s3_conn: Option<&s3::v1alpha1::ConnectionSpec>, druid_tls_security: &DruidTlsSecurity, druid_auth_config: &Option, service_account: &ServiceAccount, ) -> Result { + let merged_rolegroup_config = &rg.merged_config; // prepare container builder let prepare_container_name = Container::Prepare.to_string(); let mut cb_prepare = ContainerBuilder::new(&prepare_container_name).context( @@ -948,10 +683,9 @@ fn build_rolegroup_statefulset( metadata_database_connection_details.add_to_container(&mut cb_druid); // rest of env - let mut rest_env = rolegroup_config - .get(&PropertyNameKind::Env) + let mut rest_env = rg + .env .iter() - .flat_map(|env_vars| env_vars.iter()) .map(|(k, v)| EnvVar { name: k.clone(), value: Some(v.clone()), @@ -1258,46 +992,22 @@ pub fn error_policy( #[cfg(test)] mod test { - use product_config::{ProductConfigManager, writer}; + use std::collections::BTreeMap; + use rstest::*; - use stackable_operator::product_config_utils::{ - transform_all_roles_to_config, validate_all_roles_and_groups_config, - }; use super::*; - use crate::crd::{PROP_SEGMENT_CACHE_LOCATIONS, authentication::AuthenticationClassesResolved}; - - #[derive(Snafu, Debug, EnumDiscriminants)] - #[strum_discriminants(derive(IntoStaticStr))] - #[allow(clippy::enum_variant_names)] - pub enum Error { - #[snafu(display("controller error"))] - Controller { - #[snafu(source(from(super::Error, Box::new)))] - source: Box, - }, - - #[snafu(display("product config error"))] - ProductConfig { - source: product_config::error::Error, + use crate::controller::build::properties::writer; + use crate::{ + controller::{ + build::{config_map::build_rolegroup_config_map, properties::runtime_properties}, + validate::{DruidRoleGroupConfig, ValidatedCluster, ValidatedClusterConfig}, }, - - #[snafu(display("product config utils error"))] - ProductConfigUtils { - source: stackable_operator::product_config_utils::Error, - }, - - #[snafu(display("operator framework error"))] - OperatorFramework { - source: stackable_operator::product_config_utils::Error, + crd::{ + PROP_SEGMENT_CACHE_LOCATIONS, RUNTIME_PROPS, + authentication::AuthenticationClassesResolved, }, - - #[snafu(display("failed to resolve and merge config for role and role group"))] - FailedToResolveConfig { source: crate::crd::Error }, - - #[snafu(display("invalid configuration"))] - InvalidConfiguration { source: crate::crd::Error }, - } + }; #[rstest] #[case( @@ -1314,7 +1024,7 @@ mod test { #[case] druid_manifest: &str, #[case] tested_rolegroup_name: &str, #[case] expected_druid_segment_cache_property: &str, - ) -> Result<(), Box> { + ) { let cluster_cr = std::fs::File::open(format!("test/resources/druid_controller/{druid_manifest}")) .unwrap(); @@ -1331,20 +1041,6 @@ mod test { crate::built_info::PKG_VERSION, ) .expect("test: resolved product image is always valid"); - let role_config = transform_all_roles_to_config(&druid, &druid.build_role_properties()); - - let product_config_manager = - ProductConfigManager::from_yaml_file("test/resources/druid_controller/properties.yaml") - .context(ProductConfigSnafu)?; - - let validated_role_config = validate_all_roles_and_groups_config( - &resolved_product_image.product_version, - &role_config.context(ProductConfigUtilsSnafu)?, - &product_config_manager, - false, - false, - ) - .context(OperatorFrameworkSnafu)?; let druid_tls_security = DruidTlsSecurity::new( &AuthenticationClassesResolved { @@ -1353,53 +1049,58 @@ mod test { Some("tls".to_string()), ); - let mut druid_segment_cache_property = "invalid".to_string(); - - let config = druid.merged_config().context(FailedToResolveConfigSnafu)?; - - for (role_name, role_config) in validated_role_config.iter() { - for (rolegroup_name, rolegroup_config) in role_config.iter() { - if rolegroup_name == tested_rolegroup_name - && role_name == &DruidRole::Historical.to_string() - { - let rolegroup_ref = RoleGroupRef { - cluster: ObjectRef::from_obj(&druid), - role: role_name.into(), - role_group: rolegroup_name.clone(), - }; - - let merged_rolegroup_config = config - .common_config(&DruidRole::Historical, rolegroup_name) - .context(InvalidConfigurationSnafu)?; - - let auth_settings: Option = None; - - let rg_configmap = build_rolegroup_config_map( - &druid, - &resolved_product_image, - &rolegroup_ref, - rolegroup_config, - &merged_rolegroup_config, - "zookeeper-connection-string", - None, - None, - None, - &druid_tls_security, - &auth_settings, - ) - .context(ControllerSnafu)?; + let merged = druid.merged_config().expect("merged config"); + let merged_config = merged + .common_config(&DruidRole::Historical, tested_rolegroup_name) + .expect("common config for tested rolegroup"); + + // The segment cache property is injected dynamically by the config_map builder from the + // merged resources, independent of the precomputed runtime_config. We still populate the + // runtime_config with the static role defaults to mirror the production path. + let rg = DruidRoleGroupConfig { + merged_config, + runtime_config: runtime_properties::defaults(&DruidRole::Historical), + security_config: BTreeMap::new(), + env: BTreeMap::new(), + }; + + let cluster = ValidatedCluster { + name: druid.name_any(), + image: resolved_product_image.clone(), + cluster_config: ValidatedClusterConfig { + zookeeper_connection_string: "zookeeper-connection-string".to_string(), + opa_connection_string: None, + s3_connection: None, + deep_storage_bucket_name: None, + druid_tls_security, + druid_auth_config: None, + }, + role_group_configs: BTreeMap::new(), + }; + + let rolegroup_ref = RoleGroupRef { + cluster: ObjectRef::from_obj(&druid), + role: DruidRole::Historical.to_string(), + role_group: tested_rolegroup_name.to_string(), + }; + + let rg_configmap = build_rolegroup_config_map( + &cluster, + &DruidRole::Historical, + &rolegroup_ref, + &rg, + &resolved_product_image, + &druid, + ) + .expect("build rolegroup config map"); - druid_segment_cache_property = rg_configmap - .data - .unwrap() - .get(RUNTIME_PROPS) - .unwrap() - .to_string(); + let druid_segment_cache_property = rg_configmap + .data + .unwrap() + .get(RUNTIME_PROPS) + .unwrap() + .to_string(); - break; - } - } - } let escaped_segment_cache_property = writer::to_java_properties_string( vec![( &PROP_SEGMENT_CACHE_LOCATIONS.to_string(), @@ -1411,10 +1112,7 @@ mod test { assert!( druid_segment_cache_property.contains(&escaped_segment_cache_property), - "role group {}", - tested_rolegroup_name + "role group {tested_rolegroup_name}" ); - - Ok(()) } } diff --git a/rust/operator-binary/src/controller/build.rs b/rust/operator-binary/src/controller/build.rs index 9703c659..5a152c2a 100644 --- a/rust/operator-binary/src/controller/build.rs +++ b/rust/operator-binary/src/controller/build.rs @@ -1,3 +1,4 @@ //! Build steps that turn a `ValidatedCluster` into Kubernetes resources. +pub mod config_map; pub mod properties; diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs new file mode 100644 index 00000000..10172107 --- /dev/null +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -0,0 +1,327 @@ +//! Builds the rolegroup [`ConfigMap`] from a [`ValidatedCluster`]. +//! +//! The per-file "validated config" maps (runtime.properties / security.properties) are taken +//! from the [`DruidRoleGroupConfig`] precomputed in the validate step; product-config is no +//! longer involved. +//! +//! Residual reads from the owning [`v1alpha1::DruidCluster`] remain for things that are not yet +//! modelled on `ValidatedCluster`: the owner reference + recommended labels, the extensions load +//! list (`get_extension_list`), the metadata-database connection +//! (`spec.cluster_config.metadata_database` / `as_metadata_storage_type`), and `get_role` for the +//! jvm.config. Fully removing these is a follow-up. + +use std::collections::BTreeMap; + +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, + commons::product_image_selection::ResolvedProductImage, + crd::s3, + database_connections::drivers::jdbc::JdbcDatabaseConnection as _, + k8s_openapi::api::core::v1::{ConfigMap, EnvVar}, + role_utils::RoleGroupRef, +}; + +use crate::{ + config::jvm::construct_jvm_args, + controller::build::properties::writer::to_java_properties_string, + controller::{ + DRUID_CONTROLLER_NAME, + validate::{DruidRoleGroupConfig, ValidatedCluster}, + }, + crd::{ + AUTH_AUTHORIZER_OPA_URI, DS_BUCKET, DruidRole, EXTENSIONS_LOADLIST, JVM_CONFIG, + JVM_SECURITY_PROPERTIES_FILE, RUNTIME_PROPS, S3_ACCESS_KEY, S3_ENDPOINT_URL, + S3_PATH_STYLE_ACCESS, S3_SECRET_KEY, ZOOKEEPER_CONNECTION_STRING, build_recommended_labels, + build_string_list, v1alpha1, + }, + extensions::get_extension_list, + product_logging::extend_role_group_config_map, +}; + +#[derive(Snafu, Debug)] +#[allow(clippy::enum_variant_names)] +pub enum Error { + #[snafu(display("object is missing metadata to build owner reference"))] + ObjectMissingMetadataForOwnerRef { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to build ConfigMap for {}", rolegroup))] + BuildRoleGroupConfig { + source: stackable_operator::builder::configmap::Error, + rolegroup: RoleGroupRef, + }, + + #[snafu(display("failed to configure S3 connection"))] + ConfigureS3 { + source: stackable_operator::crd::s3::v1alpha1::ConnectionError, + }, + + #[snafu(display("failed to format runtime properties"))] + PropertiesWriteError { + source: crate::controller::build::properties::writer::PropertiesWriterError, + }, + + #[snafu(display("failed to serialize [{JVM_SECURITY_PROPERTIES_FILE}] for {rolegroup}"))] + JvmSecurityProperties { + source: crate::controller::build::properties::writer::PropertiesWriterError, + rolegroup: String, + }, + + #[snafu(display("failed to get JVM config"))] + GetJvmConfig { source: crate::config::jvm::Error }, + + #[snafu(display("failed to derive Druid memory settings from resources"))] + DeriveMemorySettings { source: crate::crd::resource::Error }, + + #[snafu(display("failed to update Druid config from resources"))] + UpdateDruidConfigFromResources { source: crate::crd::resource::Error }, + + #[snafu(display("failed to build metadata"))] + MetadataBuild { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("there was an error generating the authentication runtime settings"))] + GenerateAuthenticationRuntimeSettings { + source: crate::authentication::Error, + }, + + #[snafu(display("failed to add the logging configuration to the ConfigMap [{cm_name}]"))] + InvalidLoggingConfig { + source: crate::product_logging::Error, + cm_name: String, + }, + + #[snafu(display("invalid metadata database connection"))] + InvalidMetadataDatabaseConnection { + source: stackable_operator::database_connections::Error, + }, +} + +type Result = std::result::Result; + +/// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator +pub fn build_rolegroup_config_map( + cluster: &ValidatedCluster, + role: &DruidRole, + rolegroup: &RoleGroupRef, + rg: &DruidRoleGroupConfig, + resolved_product_image: &ResolvedProductImage, + owner: &v1alpha1::DruidCluster, +) -> Result { + let cluster_config = &cluster.cluster_config; + let druid_tls_security = &cluster_config.druid_tls_security; + let druid_auth_config = &cluster_config.druid_auth_config; + let zk_connstr = cluster_config.zookeeper_connection_string.as_str(); + let opa_connstr = cluster_config.opa_connection_string.as_deref(); + let s3_conn = cluster_config.s3_connection.as_ref(); + let deep_storage_bucket_name = cluster_config.deep_storage_bucket_name.as_deref(); + + let role_obj = owner.get_role(role); + let mut cm_conf_data = BTreeMap::new(); // filename -> filecontent + let metadata_database_connection_details = owner + .spec + .cluster_config + .metadata_database + .jdbc_connection_details("metadata") + .context(InvalidMetadataDatabaseConnectionSnafu)?; + + // ----- runtime.properties ----- + { + let mut conf: BTreeMap> = Default::default(); + + // Add any properties derived from storage manifests, such as segment cache locations. + // This has to be done here since there is no other suitable place for it. + // Previously such properties were added in the compute_files() function, + // but that code path is now incompatible with the design of fragment merging. + rg.merged_config + .resources + .update_druid_config_file(&mut conf) + .context(UpdateDruidConfigFromResourcesSnafu)?; + // NOTE: druid.host can be set manually - if it isn't, the canonical host name of + // the local host is used. This should work with the agent and k8s host networking + // but might need to be revisited in the future + conf.insert( + ZOOKEEPER_CONNECTION_STRING.to_string(), + Some(zk_connstr.to_string()), + ); + + conf.insert( + EXTENSIONS_LOADLIST.to_string(), + Some(build_string_list(&get_extension_list( + owner, + druid_tls_security, + druid_auth_config, + ))), + ); + + if let Some(opa_str) = opa_connstr { + conf.insert( + AUTH_AUTHORIZER_OPA_URI.to_string(), + Some(opa_str.to_string()), + ); + }; + + conf.insert( + crate::crd::database::METADATA_STORAGE_TYPE.to_string(), + Some( + owner + .spec + .cluster_config + .metadata_database + .as_metadata_storage_type() + .to_string(), + ), + ); + + conf.insert( + crate::crd::database::METADATA_STORAGE_CONNECTOR_CONNECT_URI.to_string(), + Some( + metadata_database_connection_details + .connection_url + .to_string(), + ), + ); + + if let Some(EnvVar { + name: username_env_name, + .. + }) = &metadata_database_connection_details.username_env + { + conf.insert( + crate::crd::database::METADATA_STORAGE_USER.to_string(), + Some(format!("${{env:{username_env_name}}}",)), + ); + } + + if let Some(EnvVar { + name: password_env_name, + .. + }) = &metadata_database_connection_details.password_env + { + conf.insert( + crate::crd::database::METADATA_STORAGE_PASSWORD.to_string(), + Some(format!("${{env:{password_env_name}}}",)), + ); + } + + if let Some(s3) = s3_conn { + if !s3.region.is_default_config() { + // Raising this as warning instead of returning an error, better safe than sorry. + // It might still work out for the user. + tracing::warn!( + region = ?s3.region, + "You configured a non-default region on the S3Connection. + The S3Connection region field is ignored because Druid uses the AWS SDK v1, which ignores the region if the endpoint is set. \ + The host is a required field, therefore the endpoint will always be set." + ) + } + + conf.insert( + S3_ENDPOINT_URL.to_string(), + Some(s3.endpoint().context(ConfigureS3Snafu)?.to_string()), + ); + + if let Some((access_key_file, secret_key_file)) = s3.credentials_mount_paths() { + conf.insert( + S3_ACCESS_KEY.to_string(), + Some(format!("${{file:UTF-8:{access_key_file}}}")), + ); + conf.insert( + S3_SECRET_KEY.to_string(), + Some(format!("${{file:UTF-8:{secret_key_file}}}")), + ); + } + + conf.insert( + S3_PATH_STYLE_ACCESS.to_string(), + Some((s3.access_style == s3::v1alpha1::S3AccessStyle::Path).to_string()), + ); + } + conf.insert( + DS_BUCKET.to_string(), + deep_storage_bucket_name.map(str::to_string), + ); + + // add tls encryption / auth properties + druid_tls_security.add_tls_config_properties(&mut conf, role); + + if let Some(auth_config) = druid_auth_config { + conf.extend( + auth_config + .generate_runtime_properties_config(role) + .context(GenerateAuthenticationRuntimeSettingsSnafu)?, + ); + }; + + // extend the config to respect the precomputed defaults and overrides + conf.extend(rg.runtime_config.clone()); + + let runtime_properties = + to_java_properties_string(conf.iter()).context(PropertiesWriteSnafu)?; + cm_conf_data.insert(RUNTIME_PROPS.to_string(), runtime_properties); + } + + // ----- jvm.config ----- + { + let (heap, direct) = rg + .merged_config + .resources + .get_memory_sizes(role) + .context(DeriveMemorySettingsSnafu)?; + let jvm_config = construct_jvm_args(role, &role_obj, &rolegroup.role_group, heap, direct) + .context(GetJvmConfigSnafu)?; + cm_conf_data.insert(JVM_CONFIG.to_string(), jvm_config); + } + + // ----- security.properties ----- + { + cm_conf_data.insert( + JVM_SECURITY_PROPERTIES_FILE.to_string(), + to_java_properties_string(rg.security_config.iter()).with_context(|_| { + JvmSecurityPropertiesSnafu { + rolegroup: rolegroup.role_group.clone(), + } + })?, + ); + } + + let mut config_map_builder = ConfigMapBuilder::new(); + config_map_builder.metadata( + ObjectMetaBuilder::new() + .name_and_namespace(owner) + .name(rolegroup.object_name()) + .ownerreference_from_resource(owner, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(&build_recommended_labels( + owner, + DRUID_CONTROLLER_NAME, + &resolved_product_image.app_version_label_value, + &rolegroup.role, + &rolegroup.role_group, + )) + .context(MetadataBuildSnafu)? + .build(), + ); + + for (filename, file_content) in cm_conf_data.iter() { + config_map_builder.add_data(filename, file_content); + } + + extend_role_group_config_map( + rolegroup, + &rg.merged_config.logging, + &mut config_map_builder, + ) + .context(InvalidLoggingConfigSnafu { + cm_name: rolegroup.object_name(), + })?; + + config_map_builder + .build() + .with_context(|_| BuildRoleGroupConfigSnafu { + rolegroup: rolegroup.clone(), + }) +} diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index b189cda3..9a1f5510 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -1,12 +1,6 @@ //! Per-file builders for Druid `.properties` files. -// The writer is vendored but not yet consumed; later refactor tasks wire it in. -#[allow(dead_code)] pub mod writer; -// consumed by config_map builder in a later task -#[allow(dead_code)] pub mod runtime_properties; -// consumed by config_map builder in a later task -#[allow(dead_code)] pub mod security_properties; diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 2d55d0e6..37b4ba12 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -1,24 +1,31 @@ //! The validate step in the DruidCluster controller //! //! Synchronously validates inputs that don't require a Kubernetes client. Produces -//! [`ValidatedInputs`], consumed by the rest of `reconcile_druid`. +//! [`ValidatedCluster`], consumed by the rest of `reconcile_druid`. + +use std::collections::BTreeMap; -use product_config::ProductConfigManager; use snafu::{ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, commons::product_image_selection::{self, ResolvedProductImage}, + config_overrides::KeyValueOverridesProvider, crd::s3, - product_config_utils::{ - ValidatedRoleConfigByPropertyKind, transform_all_roles_to_config, - validate_all_roles_and_groups_config, - }, + kube::ResourceExt, }; +use strum::IntoEnumIterator; use crate::{ authentication::DruidAuthenticationConfig, - controller::dereference::DereferencedObjects, - crd::{security::DruidTlsSecurity, v1alpha1}, + controller::{ + build::properties::{runtime_properties, security_properties}, + dereference::DereferencedObjects, + }, + crd::{ + CommonRoleGroupConfig, DruidRole, INDEXER_JAVA_OPTS, JVM_SECURITY_PROPERTIES_FILE, + RUNTIME_PROPS, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, build_string_list, + security::DruidTlsSecurity, v1alpha1, + }, }; #[derive(Snafu, Debug)] @@ -34,29 +41,66 @@ pub enum Error { source: crate::authentication::Error, }, - #[snafu(display("failed to transform configs"))] - ProductConfigTransform { - source: stackable_operator::product_config_utils::Error, - }, + #[snafu(display("failed to resolve and merge config for role and role group"))] + FailedToResolveConfig { source: crate::crd::Error }, - #[snafu(display("invalid product configuration"))] - InvalidProductConfig { + #[snafu(display("failed to compute the runtime.properties compute_files for role [{role}]"))] + ComputeRuntimeProperties { source: stackable_operator::product_config_utils::Error, + role: String, }, } type Result = std::result::Result; -/// Synchronous inputs the rest of `reconcile_druid` needs after dereferencing. -pub struct ValidatedInputs { +pub type RoleGroupName = String; + +/// The merged config plus the per-file "validated config" maps that used to be produced by +/// product-config. These are computed from first principles so that rendered config stays +/// byte-identical. +#[derive(Clone)] +pub struct DruidRoleGroupConfig { + pub merged_config: CommonRoleGroupConfig, + /// The runtime.properties "validated config" (compute_files + recommended defaults + merged overrides). + pub runtime_config: BTreeMap>, + /// The security.properties "validated config". + pub security_config: BTreeMap>, + /// Merged env overrides (role <- rolegroup). compute_env is empty for druid. + pub env: BTreeMap, +} + +/// Cluster-wide resolved fields that are not role/rolegroup specific. +pub struct ValidatedClusterConfig { pub zookeeper_connection_string: String, pub opa_connection_string: Option, pub s3_connection: Option, pub deep_storage_bucket_name: Option, - pub resolved_product_image: ResolvedProductImage, pub druid_tls_security: DruidTlsSecurity, pub druid_auth_config: Option, - pub validated_role_config: ValidatedRoleConfigByPropertyKind, +} + +/// Synchronous inputs the rest of `reconcile_druid` needs after dereferencing. +pub struct ValidatedCluster { + // Currently unused by the build steps, but part of the documented `ValidatedCluster` shape; + // consumed by later tasks. + #[allow(dead_code)] + pub name: String, + pub image: ResolvedProductImage, + pub cluster_config: ValidatedClusterConfig, + pub role_group_configs: BTreeMap>, +} + +/// The `druid.indexer.runner.javaOptsArray` entry that `MiddleManagerConfigFragment::compute_files` +/// adds for *every* file (runtime.properties and security.properties). +fn middlemanager_indexer_java_opts() -> (String, Option) { + ( + INDEXER_JAVA_OPTS.to_string(), + Some(build_string_list(&[ + format!("-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}"), + format!("-Djavax.net.ssl.trustStorePassword={STACKABLE_TRUST_STORE_PASSWORD}"), + "-Djavax.net.ssl.trustStoreType=pkcs12".to_owned(), + ])), + ) } /// Validates the cluster spec and the dereferenced inputs. @@ -64,9 +108,8 @@ pub fn validate( druid: &v1alpha1::DruidCluster, dereferenced_objects: &DereferencedObjects, operator_environment: &OperatorEnvironmentOptions, - product_config: &ProductConfigManager, -) -> Result { - let resolved_product_image = druid +) -> Result { + let image = druid .spec .image .resolve( @@ -86,25 +129,96 @@ pub fn validate( ) .context(InvalidDruidAuthenticationConfigSnafu)?; - let role_config = transform_all_roles_to_config(druid, &druid.build_role_properties()) - .context(ProductConfigTransformSnafu)?; - let validated_role_config = validate_all_roles_and_groups_config( - &resolved_product_image.product_version, - &role_config, - product_config, - false, - false, - ) - .context(InvalidProductConfigSnafu)?; - - Ok(ValidatedInputs { - zookeeper_connection_string: dereferenced_objects.zookeeper_connection_string.clone(), - opa_connection_string: dereferenced_objects.opa_connection_string.clone(), - s3_connection: dereferenced_objects.s3_connection.clone(), - deep_storage_bucket_name: dereferenced_objects.deep_storage_bucket_name.clone(), - resolved_product_image, - druid_tls_security, - druid_auth_config, - validated_role_config, + let merged = druid.merged_config().context(FailedToResolveConfigSnafu)?; + + let mut role_group_configs: BTreeMap> = + BTreeMap::new(); + + for druid_role in DruidRole::iter() { + // The role-level overrides (role <- rolegroup precedence starts here). + let role = druid.get_role(&druid_role); + let role_runtime_overrides = role + .config + .config_overrides + .get_key_value_overrides(RUNTIME_PROPS); + let role_security_overrides = role + .config + .config_overrides + .get_key_value_overrides(JVM_SECURITY_PROPERTIES_FILE); + let role_env_overrides = role.config.env_overrides.clone(); + + let rolegroups = merged.role_group_names(&druid_role); + + let mut group_map: BTreeMap = BTreeMap::new(); + for rg_name in rolegroups { + let merged_config = merged + .common_config(&druid_role, &rg_name) + .context(FailedToResolveConfigSnafu)?; + // The rolegroup-level config/env overrides (rolegroup wins over role). + // The rolegroup is guaranteed to exist because `rg_name` comes from + // `role_group_names` and `common_config` above already resolved it. + let (rg_config_overrides, rg_env_overrides) = merged + .role_group_overrides(&druid_role, &rg_name) + .expect("role group resolved by common_config must exist"); + + // ----- runtime.properties ----- + let mut runtime_config = druid.common_compute_files(RUNTIME_PROPS).context( + ComputeRuntimePropertiesSnafu { + role: druid_role.to_string(), + }, + )?; + if druid_role == DruidRole::MiddleManager { + let (k, v) = middlemanager_indexer_java_opts(); + runtime_config.insert(k, v); + } + runtime_config.extend(runtime_properties::defaults(&druid_role)); + // merged user overrides (role <- rolegroup; rolegroup wins) + let mut runtime_overrides = role_runtime_overrides.clone(); + runtime_overrides.extend(rg_config_overrides.get_key_value_overrides(RUNTIME_PROPS)); + runtime_config.extend(runtime_overrides); + + // ----- security.properties ----- + let mut security_config: BTreeMap> = BTreeMap::new(); + if druid_role == DruidRole::MiddleManager { + let (k, v) = middlemanager_indexer_java_opts(); + security_config.insert(k, v); + } + let mut security_overrides = role_security_overrides.clone(); + security_overrides + .extend(rg_config_overrides.get_key_value_overrides(JVM_SECURITY_PROPERTIES_FILE)); + security_config.extend(security_properties::build(&security_overrides)); + + // ----- env ----- + let mut env: BTreeMap = role_env_overrides + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + env.extend(rg_env_overrides.iter().map(|(k, v)| (k.clone(), v.clone()))); + + group_map.insert( + rg_name, + DruidRoleGroupConfig { + merged_config, + runtime_config, + security_config, + env, + }, + ); + } + role_group_configs.insert(druid_role, group_map); + } + + Ok(ValidatedCluster { + name: druid.name_any(), + image, + cluster_config: ValidatedClusterConfig { + zookeeper_connection_string: dereferenced_objects.zookeeper_connection_string.clone(), + opa_connection_string: dereferenced_objects.opa_connection_string.clone(), + s3_connection: dereferenced_objects.s3_connection.clone(), + deep_storage_bucket_name: dereferenced_objects.deep_storage_bucket_name.clone(), + druid_tls_security, + druid_auth_config, + }, + role_group_configs, }) } diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 11901b6e..9882bee8 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use indoc::formatdoc; use product_config::types::PropertyNameKind; @@ -818,6 +818,7 @@ pub enum Container { } /// Common configuration for all role groups +#[derive(Clone)] pub struct CommonRoleGroupConfig { pub resources: RoleResource, pub logging: Logging, @@ -950,6 +951,39 @@ impl MergedConfig { } } } + + /// Returns the (sorted) role group names defined for the given role. + pub fn role_group_names(&self, role: &DruidRole) -> BTreeSet { + match role { + DruidRole::Broker => self.brokers.keys().cloned().collect(), + DruidRole::Coordinator => self.coordinators.keys().cloned().collect(), + DruidRole::Historical => self.historicals.keys().cloned().collect(), + DruidRole::MiddleManager => self.middle_managers.keys().cloned().collect(), + DruidRole::Router => self.routers.keys().cloned().collect(), + } + } + + /// Returns the rolegroup-level config and env overrides for the given role and rolegroup. + pub fn role_group_overrides( + &self, + role: &DruidRole, + rolegroup_name: &str, + ) -> Option<(&DruidConfigOverrides, &HashMap)> { + macro_rules! get { + ($field:expr) => { + $field + .get(rolegroup_name) + .map(|rg| (&rg.config.config_overrides, &rg.config.env_overrides)) + }; + } + match role { + DruidRole::Broker => get!(self.brokers), + DruidRole::Coordinator => get!(self.coordinators), + DruidRole::Historical => get!(self.historicals), + DruidRole::MiddleManager => get!(self.middle_managers), + DruidRole::Router => get!(self.routers), + } + } } impl Default for v1alpha1::DruidRoleConfig { @@ -974,7 +1008,9 @@ fn druid_default_listener_class() -> String { Eq, Hash, JsonSchema, + Ord, PartialEq, + PartialOrd, Serialize, EnumString, )] From 88311b6573e8c7d39b2f3c5a68237b7802850825 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Jun 2026 22:01:05 +0200 Subject: [PATCH 05/48] chore: prune dead error variants after config_map extraction Co-Authored-By: Claude Opus 4.8 (1M context) --- rust/operator-binary/src/controller.rs | 55 ++------------------------ 1 file changed, 4 insertions(+), 51 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 5d0d48c7..7884bd94 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use const_format::concatcp; -use product_config::{ProductConfigManager, writer::PropertiesWriterError}; +use product_config::ProductConfigManager; use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::{ @@ -57,9 +57,9 @@ use crate::{ authentication::DruidAuthenticationConfig, crd::{ APP_NAME, CommonRoleGroupConfig, Container, DRUID_CONFIG_DIRECTORY, DeepStorageSpec, - DruidClusterStatus, DruidRole, HDFS_CONFIG_DIRECTORY, JVM_SECURITY_PROPERTIES_FILE, - LOG_CONFIG_DIRECTORY, MAX_DRUID_LOG_FILES_SIZE, METRICS_PORT, METRICS_PORT_NAME, - OPERATOR_NAME, RW_CONFIG_DIRECTORY, STACKABLE_LOG_DIR, build_recommended_labels, + DruidClusterStatus, DruidRole, HDFS_CONFIG_DIRECTORY, LOG_CONFIG_DIRECTORY, + MAX_DRUID_LOG_FILES_SIZE, METRICS_PORT, METRICS_PORT_NAME, OPERATOR_NAME, + RW_CONFIG_DIRECTORY, STACKABLE_LOG_DIR, build_recommended_labels, security::DruidTlsSecurity, v1alpha1, }, discovery::{self, build_discovery_configmaps}, @@ -110,12 +110,6 @@ pub enum Error { rolegroup: RoleGroupRef, }, - #[snafu(display("failed to build ConfigMap for {}", rolegroup))] - BuildRoleGroupConfig { - source: stackable_operator::builder::configmap::Error, - rolegroup: RoleGroupRef, - }, - #[snafu(display("failed to apply ConfigMap for {}", rolegroup))] ApplyRoleGroupConfig { source: stackable_operator::cluster_resources::Error, @@ -141,9 +135,6 @@ pub enum Error { source: stackable_operator::crd::s3::v1alpha1::ConnectionError, }, - #[snafu(display("failed to format runtime properties"))] - PropertiesWriteError { source: PropertiesWriterError }, - #[snafu(display("failed to build discovery ConfigMap"))] BuildDiscoveryConfig { source: discovery::Error }, @@ -162,18 +153,6 @@ pub enum Error { ))] S3TlsNoVerificationNotSupported, - #[snafu(display("could not parse Druid role [{role}]"))] - UnidentifiedDruidRole { - source: strum::ParseError, - role: String, - }, - - #[snafu(display("failed to resolve and merge config for role and role group"))] - FailedToResolveConfig { source: crate::crd::Error }, - - #[snafu(display("invalid configuration"))] - InvalidConfiguration { source: crate::crd::Error }, - #[snafu(display("failed to create cluster resources"))] CreateClusterResources { source: stackable_operator::cluster_resources::Error, @@ -193,12 +172,6 @@ pub enum Error { #[snafu(display("failed to initialize security context"))] FailedToInitializeSecurityContext { source: crate::crd::security::Error }, - #[snafu(display("failed to get JVM config"))] - GetJvmConfig { source: crate::config::jvm::Error }, - - #[snafu(display("failed to derive Druid memory settings from resources"))] - DeriveMemorySettings { source: crate::crd::resource::Error }, - #[snafu(display("failed to update Druid config from resources"))] UpdateDruidConfigFromResources { source: crate::crd::resource::Error }, @@ -210,12 +183,6 @@ pub enum Error { #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] VectorAggregatorConfigMapMissing, - #[snafu(display("failed to add the logging configuration to the ConfigMap [{cm_name}]"))] - InvalidLoggingConfig { - source: crate::product_logging::Error, - cm_name: String, - }, - #[snafu(display("failed to create RBAC service account"))] ApplyServiceAccount { source: stackable_operator::cluster_resources::Error, @@ -231,15 +198,6 @@ pub enum Error { source: stackable_operator::commons::rbac::Error, }, - #[snafu(display( - "failed to serialize [{JVM_SECURITY_PROPERTIES_FILE}] for {}", - rolegroup - ))] - JvmSecurityProperties { - source: PropertiesWriterError, - rolegroup: String, - }, - #[snafu(display("failed to create PodDisruptionBudget"))] FailedToCreatePdb { source: crate::operations::pdb::Error, @@ -268,11 +226,6 @@ pub enum Error { source: KeyValuePairError, }, - #[snafu(display("there was an error generating the authentication runtime settings"))] - GenerateAuthenticationRuntimeSettings { - source: crate::authentication::Error, - }, - #[snafu(display("failed to build vector container"))] BuildVectorContainer { source: LoggingError }, From c852a02c8d96c4095e4556445a91041fc8074f40 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Jun 2026 22:12:00 +0200 Subject: [PATCH 06/48] chore: remove product-config dependency and config-spec Removes the last direct uses of the product-config crate now that validation and config-map rendering compute everything from first principles. Deletes the dead `build_role_properties` method and the five `impl Configuration` blocks, makes `common_compute_files` infallible, and replaces the `Configuration`-based `.erase()` in `get_role` with a local `erase_config` helper that erases the typed config to `()`. Also drops the `ProductConfigManager` from the controller `Ctx`, ignores the shared `--product-config` CLI arg, removes the product-config config-spec files plus their Helm/Dockerfile wiring, and removes the crate from Cargo.toml/Cargo.lock (it now only remains transitively via stackable-operator). Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 1 - Cargo.toml | 1 - deploy/config-spec/properties.yaml | 827 ------------------ .../druid-operator/configs/properties.yaml | 827 ------------------ .../druid-operator/templates/configmap.yaml | 9 - .../druid-operator/templates/deployment.yaml | 8 - docker/Dockerfile | 2 - rust/operator-binary/Cargo.toml | 1 - rust/operator-binary/src/controller.rs | 5 - .../src/controller/validate.rs | 12 +- rust/operator-binary/src/crd/mod.rs | 305 ++----- rust/operator-binary/src/main.rs | 8 +- .../druid_controller/properties.yaml | 638 -------------- 13 files changed, 58 insertions(+), 2586 deletions(-) delete mode 100644 deploy/config-spec/properties.yaml delete mode 100644 deploy/helm/druid-operator/configs/properties.yaml delete mode 100644 deploy/helm/druid-operator/templates/configmap.yaml delete mode 100644 rust/operator-binary/test/resources/druid_controller/properties.yaml diff --git a/Cargo.lock b/Cargo.lock index e4f32e1d..302dc4a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2981,7 +2981,6 @@ dependencies = [ "java-properties", "openssl", "pin-project", - "product-config", "rstest", "semver", "serde", diff --git a/Cargo.toml b/Cargo.toml index f0702dca..3165d088 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,6 @@ edition = "2021" repository = "https://github.com/stackabletech/druid-operator" [workspace.dependencies] -product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.8.0" } stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.111.0", features = ["crds", "webhook"] } anyhow = "1.0" diff --git a/deploy/config-spec/properties.yaml b/deploy/config-spec/properties.yaml deleted file mode 100644 index e4ede43b..00000000 --- a/deploy/config-spec/properties.yaml +++ /dev/null @@ -1,827 +0,0 @@ ---- -version: 0.1.0 -spec: - units: - - unit: &unitDirectory - name: "directory" - regex: "^/|(/[\\w-]+)+$" - examples: - - "/tmp/xyz" - - unit: &unitPort - name: "port" - regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]\ - |6553[0-5])$" - - unit: &unitPrometheusNamespace - name: "prometheusNamespace" - regex: "^[a-zA-Z_:][a-zA-Z0-9_:]*$" - - unit: &unitDuration - name: "duration" - regex: "^P(?!$)(\\d+Y)?(\\d+M)?(\\d+W)?(\\d+D)?(T(?=\\d)(\\d+H)?(\\d+M)?(\\d+S)\ - ?)?$" - examples: - - "PT300S" - -################################################################################################### -# runtime.properties -# For information on the properties, see: https://druid.apache.org/docs/latest/configuration/index.html -################################################################################################### - -properties: - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "broker" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "coordinator" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "historical" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "middlemanager" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "router" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: true - asOfVersion: "0.0.0" - comment: "TTL for host names that cannot be resolved." - description: "TTL for host names that cannot be resolved." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "coordinator" - required: true - asOfVersion: "0.0.0" - comment: "TTL for host names that cannot be resolved." - description: "TTL for host names that cannot be resolved." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "historical" - required: true - asOfVersion: "0.0.0" - comment: "TTL for host names that cannot be resolved." - description: "TTL for host names that cannot be resolved." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "middlemanager" - required: true - asOfVersion: "0.0.0" - comment: "TTL for host names that cannot be resolved." - description: "TTL for host names that cannot be resolved." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "router" - required: true - asOfVersion: "0.0.0" - comment: "TTL for host names that cannot be resolved." - description: "TTL for host names that cannot be resolved." - - - property: &plaintextPort - propertyNames: - - name: "druid.plaintextPort" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - min: "1" - max: "65535" - unit: *unitPort - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &tlsPort - propertyNames: - - name: "druid.tlsPort" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - min: "1" - max: "65535" - unit: *unitPort - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &startupLoggingLogProperties - propertyNames: - - name: "druid.startup.logging.logProperties" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &monitoringMonitors - propertyNames: - - name: "druid.monitoring.monitors" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "[\"org.apache.druid.java.util.metrics.JvmMonitor\"]" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitter - propertyNames: - - name: "druid.emitter" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "prometheus" - allowedValues: - - "noop" - - "logging" - - "http" - - "parametrized" - - "composing" - - "prometheus" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitterPrometheusStrategy - propertyNames: - - name: "druid.emitter.prometheus.strategy" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "exporter" - allowedValues: - - "exporter" - - "pushgateway" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitterPrometheusNamespace - propertyNames: - - name: "druid.emitter.prometheus.namespace" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitPrometheusNamespace - defaultValues: - - fromVersion: "0.0.0" - value: "druid" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitterPrometheusPort - propertyNames: - - name: "druid.emitter.prometheus.port" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - min: "1" - max: "65535" - unit: *unitPort - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &indexerLogsDirectory - propertyNames: - - name: "druid.indexer.logs.directory" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/indexing-logs" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &processingTmpDir - propertyNames: - - name: "druid.processing.tmpDir" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/processing" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: false - - name: "historical" - required: true - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerTaskHadoopWorkingPath - propertyNames: - - name: "druid.indexer.task.hadoopWorkingPath" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/hadoop-tmp" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: true - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerTaskBaseTaskDir - propertyNames: - - name: "druid.indexer.task.baseTaskDir" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/task" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: true - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerRunnerJavaOpts - propertyNames: - - name: "druid.indexer.runner.javaOpts" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "-server -Xms256m -Xmx256m -XX:MaxDirectMemorySize=300m - -Duser.timezone=UTC -Dfile.encoding=UTF-8 - -XX:+ExitOnOutOfMemoryError - -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: true - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &routerManagementProxyEnabled - propertyNames: - # Management proxy to coordinator / overlord: required for unified web console. - - name: "druid.router.managementProxy.enabled" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &routerHttpNumConnections - propertyNames: - - name: "druid.router.http.numConnections" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - defaultValues: - - fromVersion: "0.0.0" - value: "25" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &historicalCacheUseCache - propertyNames: - - name: "druid.historical.cache.useCache" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: true - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &historicalCachePopulateCache - propertyNames: - - name: "druid.historical.cache.populateCache" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: true - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorStartDelay - propertyNames: - - name: "druid.coordinator.startDelay" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDuration - defaultValues: - - fromVersion: "0.0.0" - value: "PT20S" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerQueueStartDelay - propertyNames: - - name: "druid.indexer.queue.startDelay" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDuration - defaultValues: - - fromVersion: "0.0.0" - value: "PT20S" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerRunnerType - propertyNames: - - name: "druid.indexer.runner.type" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - allowedValues: - - "local" - - "remote" - - "httpRemote" - defaultValues: - - fromVersion: "0.0.0" - value: "remote" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerStorageType - propertyNames: - - name: "druid.indexer.storage.type" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - allowedValues: - - "local" - - "metadata" - defaultValues: - - fromVersion: "0.0.0" - value: "metadata" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorPeriod - propertyNames: - - name: "druid.coordinator.period" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDuration - defaultValues: - - fromVersion: "0.0.0" - value: "PT20S" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorAsOverlordEnabled - propertyNames: - - name: "druid.coordinator.asOverlord.enabled" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorAsOverlordOverlordService - propertyNames: - - name: "druid.coordinator.asOverlord.overlordService" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "druid/overlord" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - -################################################################################################### -# jvm.config -################################################################################################### diff --git a/deploy/helm/druid-operator/configs/properties.yaml b/deploy/helm/druid-operator/configs/properties.yaml deleted file mode 100644 index e4ede43b..00000000 --- a/deploy/helm/druid-operator/configs/properties.yaml +++ /dev/null @@ -1,827 +0,0 @@ ---- -version: 0.1.0 -spec: - units: - - unit: &unitDirectory - name: "directory" - regex: "^/|(/[\\w-]+)+$" - examples: - - "/tmp/xyz" - - unit: &unitPort - name: "port" - regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]\ - |6553[0-5])$" - - unit: &unitPrometheusNamespace - name: "prometheusNamespace" - regex: "^[a-zA-Z_:][a-zA-Z0-9_:]*$" - - unit: &unitDuration - name: "duration" - regex: "^P(?!$)(\\d+Y)?(\\d+M)?(\\d+W)?(\\d+D)?(T(?=\\d)(\\d+H)?(\\d+M)?(\\d+S)\ - ?)?$" - examples: - - "PT300S" - -################################################################################################### -# runtime.properties -# For information on the properties, see: https://druid.apache.org/docs/latest/configuration/index.html -################################################################################################### - -properties: - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "broker" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "coordinator" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "historical" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "middlemanager" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "router" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: true - asOfVersion: "0.0.0" - comment: "TTL for host names that cannot be resolved." - description: "TTL for host names that cannot be resolved." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "coordinator" - required: true - asOfVersion: "0.0.0" - comment: "TTL for host names that cannot be resolved." - description: "TTL for host names that cannot be resolved." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "historical" - required: true - asOfVersion: "0.0.0" - comment: "TTL for host names that cannot be resolved." - description: "TTL for host names that cannot be resolved." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "middlemanager" - required: true - asOfVersion: "0.0.0" - comment: "TTL for host names that cannot be resolved." - description: "TTL for host names that cannot be resolved." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "router" - required: true - asOfVersion: "0.0.0" - comment: "TTL for host names that cannot be resolved." - description: "TTL for host names that cannot be resolved." - - - property: &plaintextPort - propertyNames: - - name: "druid.plaintextPort" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - min: "1" - max: "65535" - unit: *unitPort - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &tlsPort - propertyNames: - - name: "druid.tlsPort" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - min: "1" - max: "65535" - unit: *unitPort - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &startupLoggingLogProperties - propertyNames: - - name: "druid.startup.logging.logProperties" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &monitoringMonitors - propertyNames: - - name: "druid.monitoring.monitors" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "[\"org.apache.druid.java.util.metrics.JvmMonitor\"]" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitter - propertyNames: - - name: "druid.emitter" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "prometheus" - allowedValues: - - "noop" - - "logging" - - "http" - - "parametrized" - - "composing" - - "prometheus" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitterPrometheusStrategy - propertyNames: - - name: "druid.emitter.prometheus.strategy" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "exporter" - allowedValues: - - "exporter" - - "pushgateway" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitterPrometheusNamespace - propertyNames: - - name: "druid.emitter.prometheus.namespace" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitPrometheusNamespace - defaultValues: - - fromVersion: "0.0.0" - value: "druid" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitterPrometheusPort - propertyNames: - - name: "druid.emitter.prometheus.port" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - min: "1" - max: "65535" - unit: *unitPort - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &indexerLogsDirectory - propertyNames: - - name: "druid.indexer.logs.directory" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/indexing-logs" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &processingTmpDir - propertyNames: - - name: "druid.processing.tmpDir" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/processing" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: false - - name: "historical" - required: true - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerTaskHadoopWorkingPath - propertyNames: - - name: "druid.indexer.task.hadoopWorkingPath" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/hadoop-tmp" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: true - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerTaskBaseTaskDir - propertyNames: - - name: "druid.indexer.task.baseTaskDir" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/task" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: true - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerRunnerJavaOpts - propertyNames: - - name: "druid.indexer.runner.javaOpts" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "-server -Xms256m -Xmx256m -XX:MaxDirectMemorySize=300m - -Duser.timezone=UTC -Dfile.encoding=UTF-8 - -XX:+ExitOnOutOfMemoryError - -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: true - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &routerManagementProxyEnabled - propertyNames: - # Management proxy to coordinator / overlord: required for unified web console. - - name: "druid.router.managementProxy.enabled" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &routerHttpNumConnections - propertyNames: - - name: "druid.router.http.numConnections" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - defaultValues: - - fromVersion: "0.0.0" - value: "25" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &historicalCacheUseCache - propertyNames: - - name: "druid.historical.cache.useCache" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: true - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &historicalCachePopulateCache - propertyNames: - - name: "druid.historical.cache.populateCache" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: true - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorStartDelay - propertyNames: - - name: "druid.coordinator.startDelay" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDuration - defaultValues: - - fromVersion: "0.0.0" - value: "PT20S" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerQueueStartDelay - propertyNames: - - name: "druid.indexer.queue.startDelay" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDuration - defaultValues: - - fromVersion: "0.0.0" - value: "PT20S" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerRunnerType - propertyNames: - - name: "druid.indexer.runner.type" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - allowedValues: - - "local" - - "remote" - - "httpRemote" - defaultValues: - - fromVersion: "0.0.0" - value: "remote" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerStorageType - propertyNames: - - name: "druid.indexer.storage.type" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - allowedValues: - - "local" - - "metadata" - defaultValues: - - fromVersion: "0.0.0" - value: "metadata" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorPeriod - propertyNames: - - name: "druid.coordinator.period" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDuration - defaultValues: - - fromVersion: "0.0.0" - value: "PT20S" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorAsOverlordEnabled - propertyNames: - - name: "druid.coordinator.asOverlord.enabled" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorAsOverlordOverlordService - propertyNames: - - name: "druid.coordinator.asOverlord.overlordService" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "druid/overlord" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - -################################################################################################### -# jvm.config -################################################################################################### diff --git a/deploy/helm/druid-operator/templates/configmap.yaml b/deploy/helm/druid-operator/templates/configmap.yaml deleted file mode 100644 index e75acc25..00000000 --- a/deploy/helm/druid-operator/templates/configmap.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -apiVersion: v1 -data: -{{ (.Files.Glob "configs/*").AsConfig | indent 2 }} -kind: ConfigMap -metadata: - name: {{ include "operator.fullname" . }}-configmap - labels: - {{- include "operator.labels" . | nindent 4 }} diff --git a/deploy/helm/druid-operator/templates/deployment.yaml b/deploy/helm/druid-operator/templates/deployment.yaml index d0cecba4..06dde967 100644 --- a/deploy/helm/druid-operator/templates/deployment.yaml +++ b/deploy/helm/druid-operator/templates/deployment.yaml @@ -16,7 +16,6 @@ spec: metadata: annotations: internal.stackable.tech/image: {{ include "operator.image" . }} - checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} @@ -41,9 +40,6 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} resources: {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - mountPath: /etc/stackable/{{ include "operator.appname" . }}/config-spec - name: config-spec env: # The following env vars are passed as clap (think CLI) arguments to the operator. # They are picked up by clap using the structs defied in the operator. @@ -88,10 +84,6 @@ spec: {{- include "telemetry.envVars" . | nindent 12 }} {{- include "maintenance.envVars" . | nindent 12 }} - volumes: - - name: config-spec - configMap: - name: {{ include "operator.fullname" . }}-configmap {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/docker/Dockerfile b/docker/Dockerfile index 84817207..226bd4db 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -191,8 +191,6 @@ COPY LICENSE /licenses/LICENSE COPY --from=builder --chown=${STACKABLE_USER_UID}:0 /app/* /usr/local/bin/ -COPY deploy/config-spec/properties.yaml /etc/stackable/druid-operator/config-spec/properties.yaml - USER ${STACKABLE_USER_UID} ENTRYPOINT ["stackable-druid-operator"] diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index e66d5be8..ea94f2f3 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -9,7 +9,6 @@ repository.workspace = true publish = false [dependencies] -product-config.workspace = true stackable-operator.workspace = true anyhow.workspace = true diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 7884bd94..808b9f1e 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -4,7 +4,6 @@ use std::sync::Arc; use const_format::concatcp; -use product_config::ProductConfigManager; use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::{ @@ -93,10 +92,6 @@ const USERDATA_MOUNTPOINT: &str = "/stackable/userdata"; pub struct Ctx { pub client: stackable_operator::client::Client, - // Still constructed in `main.rs` but no longer consumed after the product-config removal in - // the validate/build steps. Removing it entirely is a follow-up task. - #[allow(dead_code)] - pub product_config: ProductConfigManager, pub operator_environment: OperatorEnvironmentOptions, } diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 37b4ba12..272283a3 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -43,12 +43,6 @@ pub enum Error { #[snafu(display("failed to resolve and merge config for role and role group"))] FailedToResolveConfig { source: crate::crd::Error }, - - #[snafu(display("failed to compute the runtime.properties compute_files for role [{role}]"))] - ComputeRuntimeProperties { - source: stackable_operator::product_config_utils::Error, - role: String, - }, } type Result = std::result::Result; @@ -162,11 +156,7 @@ pub fn validate( .expect("role group resolved by common_config must exist"); // ----- runtime.properties ----- - let mut runtime_config = druid.common_compute_files(RUNTIME_PROPS).context( - ComputeRuntimePropertiesSnafu { - role: druid_role.to_string(), - }, - )?; + let mut runtime_config = druid.common_compute_files(RUNTIME_PROPS); if druid_role == DruidRole::MiddleManager { let (k, v) = middlemanager_indexer_java_opts(); runtime_config.insert(k, v); diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 9882bee8..5d928d69 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,7 +1,6 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use indoc::formatdoc; -use product_config::types::PropertyNameKind; use security::add_cert_to_jvm_trust_store_cmd; use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt, Snafu}; @@ -28,7 +27,6 @@ use stackable_operator::{ kube::{CustomResource, ResourceExt}, kvp::ObjectLabels, memory::{BinaryMultiple, MemoryQuantity}, - product_config_utils::{Configuration, Error as ConfigError}, product_logging::{ self, framework::{create_vector_shutdown_file_command, remove_vector_shutdown_file_command}, @@ -401,10 +399,7 @@ impl HasStatusCondition for v1alpha1::DruidCluster { } impl v1alpha1::DruidCluster { - pub fn common_compute_files( - &self, - file: &str, - ) -> Result>, ConfigError> { + pub fn common_compute_files(&self, file: &str) -> BTreeMap> { let mut result = BTreeMap::new(); match file { JVM_CONFIG => {} @@ -446,72 +441,7 @@ impl v1alpha1::DruidCluster { _ => {} } - Ok(result) - } - - #[allow(clippy::type_complexity)] - pub fn build_role_properties( - &self, - ) -> HashMap< - String, - ( - Vec, - Role< - impl Configuration, - DruidConfigOverrides, - GenericRoleConfig, - JavaCommonConfig, - >, - ), - > { - let config_files = vec![ - PropertyNameKind::Env, - PropertyNameKind::File(JVM_CONFIG.to_string()), - PropertyNameKind::File(RUNTIME_PROPS.to_string()), - PropertyNameKind::File(JVM_SECURITY_PROPERTIES_FILE.to_string()), - ]; - - vec![ - ( - DruidRole::Broker.to_string(), - ( - config_files.clone(), - extract_role_from_role_config::(self.spec.brokers.clone()) - .erase(), - ), - ), - ( - DruidRole::Coordinator.to_string(), - ( - config_files.clone(), - extract_role_from_role_config::( - self.spec.coordinators.clone(), - ) - .erase(), - ), - ), - ( - DruidRole::Historical.to_string(), - (config_files.clone(), self.spec.historicals.clone().erase()), - ), - ( - DruidRole::MiddleManager.to_string(), - ( - config_files.clone(), - self.spec.middle_managers.clone().erase(), - ), - ), - ( - DruidRole::Router.to_string(), - ( - config_files, - extract_role_from_role_config::(self.spec.routers.clone()) - .erase(), - ), - ), - ] - .into_iter() - .collect() + result } /// If an s3 connection for ingestion is given, as well as an s3 connection for deep storage, they need to be the same. @@ -724,25 +654,19 @@ impl v1alpha1::DruidCluster { pub fn get_role( &self, druid_role: &DruidRole, - ) -> Role< - impl Configuration, - DruidConfigOverrides, - GenericRoleConfig, - JavaCommonConfig, - > { + ) -> Role<(), DruidConfigOverrides, GenericRoleConfig, JavaCommonConfig> { match druid_role { - DruidRole::Broker => { - extract_role_from_role_config::(self.spec.brokers.clone()).erase() - } - DruidRole::Coordinator => { - extract_role_from_role_config::(self.spec.coordinators.clone()) - .erase() - } - DruidRole::Historical => self.spec.historicals.clone().erase(), - DruidRole::MiddleManager => self.spec.middle_managers.clone().erase(), - DruidRole::Router => { - extract_role_from_role_config::(self.spec.routers.clone()).erase() - } + DruidRole::Broker => erase_config(extract_role_from_role_config::( + self.spec.brokers.clone(), + )), + DruidRole::Coordinator => erase_config(extract_role_from_role_config::< + CoordinatorConfig, + >(self.spec.coordinators.clone())), + DruidRole::Historical => erase_config(self.spec.historicals.clone()), + DruidRole::MiddleManager => erase_config(self.spec.middle_managers.clone()), + DruidRole::Router => erase_config(extract_role_from_role_config::( + self.spec.routers.clone(), + )), } } @@ -1454,165 +1378,6 @@ impl HistoricalConfig { } } -impl Configuration for BrokerConfigFragment { - type Configurable = v1alpha1::DruidCluster; - - fn compute_env( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, ConfigError> { - let mut _result = BTreeMap::new(); - Ok(_result) - } - - fn compute_cli( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, ConfigError> { - Ok(BTreeMap::new()) - } - - fn compute_files( - &self, - resource: &Self::Configurable, - _role_name: &str, - file: &str, - ) -> Result>, ConfigError> { - resource.common_compute_files(file) - } -} - -impl Configuration for HistoricalConfigFragment { - type Configurable = v1alpha1::DruidCluster; - - fn compute_env( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, ConfigError> { - let mut _result = BTreeMap::new(); - Ok(_result) - } - - fn compute_cli( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, ConfigError> { - Ok(BTreeMap::new()) - } - - fn compute_files( - &self, - resource: &Self::Configurable, - _role_name: &str, - file: &str, - ) -> Result>, ConfigError> { - resource.common_compute_files(file) - } -} - -impl Configuration for RouterConfigFragment { - type Configurable = v1alpha1::DruidCluster; - - fn compute_env( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, ConfigError> { - let mut _result = BTreeMap::new(); - Ok(_result) - } - - fn compute_cli( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, ConfigError> { - Ok(BTreeMap::new()) - } - - fn compute_files( - &self, - resource: &Self::Configurable, - _role_name: &str, - file: &str, - ) -> Result>, ConfigError> { - resource.common_compute_files(file) - } -} - -impl Configuration for MiddleManagerConfigFragment { - type Configurable = v1alpha1::DruidCluster; - - fn compute_env( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, ConfigError> { - let mut _result = BTreeMap::new(); - Ok(_result) - } - - fn compute_cli( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, ConfigError> { - Ok(BTreeMap::new()) - } - - fn compute_files( - &self, - resource: &Self::Configurable, - _role_name: &str, - file: &str, - ) -> Result>, ConfigError> { - let mut result = resource.common_compute_files(file)?; - result.insert( - INDEXER_JAVA_OPTS.to_string(), - Some(build_string_list(&[ - format!("-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}"), - format!("-Djavax.net.ssl.trustStorePassword={STACKABLE_TRUST_STORE_PASSWORD}"), - "-Djavax.net.ssl.trustStoreType=pkcs12".to_owned(), - ])), - ); - Ok(result) - } -} - -impl Configuration for CoordinatorConfigFragment { - type Configurable = v1alpha1::DruidCluster; - - fn compute_env( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, ConfigError> { - let mut _result = BTreeMap::new(); - Ok(_result) - } - - fn compute_cli( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, ConfigError> { - Ok(BTreeMap::new()) - } - - fn compute_files( - &self, - resource: &Self::Configurable, - _role_name: &str, - file: &str, - ) -> Result>, ConfigError> { - resource.common_compute_files(file) - } -} - #[derive(Clone, Debug, Default, Deserialize, JsonSchema, Serialize)] #[serde(rename_all = "camelCase")] pub struct DruidClusterStatus { @@ -1687,6 +1452,48 @@ where } } +/// Discards the typed `config` of a [`Role`], replacing it with `()`. +/// +/// `get_role` needs to return a single concrete type across all roles, but each role has a +/// different config fragment. Callers only read the role/role-group level overrides +/// (`config_overrides`, `env_overrides`, `product_specific_common_config`), never the typed config +/// itself, so erasing it to `()` yields a uniform return type without needing a trait object. +fn erase_config( + role: Role, +) -> Role<(), DruidConfigOverrides, GenericRoleConfig, JavaCommonConfig> { + Role { + config: CommonConfiguration { + config: (), + config_overrides: role.config.config_overrides, + env_overrides: role.config.env_overrides, + cli_overrides: role.config.cli_overrides, + pod_overrides: role.config.pod_overrides, + product_specific_common_config: role.config.product_specific_common_config, + }, + role_config: role.role_config, + role_groups: role + .role_groups + .into_iter() + .map(|(k, v)| { + ( + k, + RoleGroup { + config: CommonConfiguration { + config: (), + config_overrides: v.config.config_overrides, + env_overrides: v.config.env_overrides, + cli_overrides: v.config.cli_overrides, + pod_overrides: v.config.pod_overrides, + product_specific_common_config: v.config.product_specific_common_config, + }, + replicas: v.replicas, + }, + ) + }) + .collect(), + } +} + #[cfg(test)] mod tests { use stackable_operator::versioned::test_utils::RoundtripTestData; diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 1176b910..a157763b 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -70,7 +70,7 @@ async fn main() -> anyhow::Result<()> { Command::Run(RunArguments { operator_environment, watch_namespace, - product_config, + product_config: _, maintenance, common, }) => { @@ -117,11 +117,6 @@ async fn main() -> anyhow::Result<()> { .run(sigterm_watcher.handle()) .map_err(|err| anyhow!(err).context("failed to run webhook server")); - let product_config = product_config.load(&[ - "deploy/config-spec/properties.yaml", - "/etc/stackable/druid-operator/config-spec/properties.yaml", - ])?; - let event_recorder = Arc::new(Recorder::new( client.as_kube_client(), Reporter { @@ -166,7 +161,6 @@ async fn main() -> anyhow::Result<()> { Arc::new(controller::Ctx { client: client.clone(), operator_environment, - product_config, }), ) // We can let the reporting happen in the background diff --git a/rust/operator-binary/test/resources/druid_controller/properties.yaml b/rust/operator-binary/test/resources/druid_controller/properties.yaml deleted file mode 100644 index e1e167d9..00000000 --- a/rust/operator-binary/test/resources/druid_controller/properties.yaml +++ /dev/null @@ -1,638 +0,0 @@ ---- -version: 0.1.0 -spec: - units: - - unit: &unitDirectory - name: "directory" - regex: "^/|(/[\\w-]+)+$" - examples: - - "/tmp/xyz" - - unit: &unitPort - name: "port" - regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]\ - |6553[0-5])$" - - unit: &unitPrometheusNamespace - name: "prometheusNamespace" - regex: "^[a-zA-Z_:][a-zA-Z0-9_:]*$" - - unit: &unitDuration - name: "duration" - regex: "^P(?!$)(\\d+Y)?(\\d+M)?(\\d+W)?(\\d+D)?(T(?=\\d)(\\d+H)?(\\d+M)?(\\d+S)\ - ?)?$" - examples: - - "PT300S" - -################################################################################################### -# runtime.properties -# For information on the properties, see: https://druid.apache.org/docs/latest/configuration/index.html -################################################################################################### - -properties: - - - property: &plaintextPort - propertyNames: - - name: "druid.plaintextPort" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - min: "1" - max: "65535" - unit: *unitPort - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &tlsPort - propertyNames: - - name: "druid.tlsPort" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - min: "1" - max: "65535" - unit: *unitPort - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &startupLoggingLogProperties - propertyNames: - - name: "druid.startup.logging.logProperties" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &monitoringMonitors - propertyNames: - - name: "druid.monitoring.monitors" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "[\"org.apache.druid.java.util.metrics.JvmMonitor\"]" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitter - propertyNames: - - name: "druid.emitter" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "prometheus" - allowedValues: - - "noop" - - "logging" - - "http" - - "parametrized" - - "composing" - - "prometheus" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitterPrometheusStrategy - propertyNames: - - name: "druid.emitter.prometheus.strategy" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "exporter" - allowedValues: - - "exporter" - - "pushgateway" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitterPrometheusNamespace - propertyNames: - - name: "druid.emitter.prometheus.namespace" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitPrometheusNamespace - defaultValues: - - fromVersion: "0.0.0" - value: "druid" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitterPrometheusPort - propertyNames: - - name: "druid.emitter.prometheus.port" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - min: "1" - max: "65535" - unit: *unitPort - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &indexerLogsDirectory - propertyNames: - - name: "druid.indexer.logs.directory" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/indexing-logs" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &processingTmpDir - propertyNames: - - name: "druid.processing.tmpDir" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/processing" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: false - - name: "historical" - required: true - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerTaskHadoopWorkingPath - propertyNames: - - name: "druid.indexer.task.hadoopWorkingPath" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/hadoop-tmp" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: true - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerTaskBaseTaskDir - propertyNames: - - name: "druid.indexer.task.baseTaskDir" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/task" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: true - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerRunnerJavaOpts - propertyNames: - - name: "druid.indexer.runner.javaOpts" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "-server -Xms256m -Xmx256m -XX:MaxDirectMemorySize=300m - -Duser.timezone=UTC -Dfile.encoding=UTF-8 - -XX:+ExitOnOutOfMemoryError - -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: true - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &routerManagementProxyEnabled - propertyNames: - # Management proxy to coordinator / overlord: required for unified web console. - - name: "druid.router.managementProxy.enabled" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &routerHttpNumConnections - propertyNames: - - name: "druid.router.http.numConnections" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - defaultValues: - - fromVersion: "0.0.0" - value: "25" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &historicalCacheUseCache - propertyNames: - - name: "druid.historical.cache.useCache" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: true - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &historicalCachePopulateCache - propertyNames: - - name: "druid.historical.cache.populateCache" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: true - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorStartDelay - propertyNames: - - name: "druid.coordinator.startDelay" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDuration - defaultValues: - - fromVersion: "0.0.0" - value: "PT20S" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerQueueStartDelay - propertyNames: - - name: "druid.indexer.queue.startDelay" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDuration - defaultValues: - - fromVersion: "0.0.0" - value: "PT20S" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerRunnerType - propertyNames: - - name: "druid.indexer.runner.type" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - allowedValues: - - "local" - - "remote" - - "httpRemote" - defaultValues: - - fromVersion: "0.0.0" - value: "remote" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerStorageType - propertyNames: - - name: "druid.indexer.storage.type" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - allowedValues: - - "local" - - "metadata" - defaultValues: - - fromVersion: "0.0.0" - value: "metadata" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorPeriod - propertyNames: - - name: "druid.coordinator.period" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDuration - defaultValues: - - fromVersion: "0.0.0" - value: "PT20S" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorAsOverlordEnabled - propertyNames: - - name: "druid.coordinator.asOverlord.enabled" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorAsOverlordOverlordService - propertyNames: - - name: "druid.coordinator.asOverlord.overlordService" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "druid/overlord" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - -################################################################################################### -# jvm.config -################################################################################################### From 1045037fbbe965dce439cca4dbf1447b0a240751 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Jun 2026 22:17:10 +0200 Subject: [PATCH 07/48] docs: remove product-config CLI arg and PRODUCT_CONFIG env var references Co-Authored-By: Claude Opus 4.8 (1M context) --- .../reference/commandline-parameters.adoc | 13 ---------- .../reference/environment-variables.adoc | 26 ------------------- 2 files changed, 39 deletions(-) diff --git a/docs/modules/druid/pages/reference/commandline-parameters.adoc b/docs/modules/druid/pages/reference/commandline-parameters.adoc index 2aeb7591..3278c897 100644 --- a/docs/modules/druid/pages/reference/commandline-parameters.adoc +++ b/docs/modules/druid/pages/reference/commandline-parameters.adoc @@ -2,19 +2,6 @@ This operator accepts the following command line parameters: -== product-config - -*Default value*: `/etc/stackable/druid-operator/config-spec/properties.yaml` - -*Required*: false - -*Multiple values:* false - -[source] ----- -stackable-druid-operator run --product-config /foo/bar/properties.yaml ----- - == watch-namespace *Default value*: All namespaces diff --git a/docs/modules/druid/pages/reference/environment-variables.adoc b/docs/modules/druid/pages/reference/environment-variables.adoc index 2083b351..1f603488 100644 --- a/docs/modules/druid/pages/reference/environment-variables.adoc +++ b/docs/modules/druid/pages/reference/environment-variables.adoc @@ -33,32 +33,6 @@ docker run \ oci.stackable.tech/sdp/druid-operator:0.0.0-dev ---- -== PRODUCT_CONFIG - -*Default value*: `/etc/stackable/druid-operator/config-spec/properties.yaml` - -*Required*: false - -*Multiple values*: false - -[source] ----- -export PRODUCT_CONFIG=/foo/bar/properties.yaml -stackable-druid-operator run ----- - -or via docker: - ----- -docker run \ - --name druid-operator \ - --network host \ - --env KUBECONFIG=/home/stackable/.kube/config \ - --env PRODUCT_CONFIG=/my/product/config.yaml \ - --mount type=bind,source="$HOME/.kube/config",target="/home/stackable/.kube/config" \ - oci.stackable.tech/sdp/druid-operator:0.0.0-dev ----- - == WATCH_NAMESPACE *Default value*: All namespaces From 6c0b527a57f01c2eb6c00e03487d2563f6c229a2 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Jun 2026 22:24:02 +0200 Subject: [PATCH 08/48] chore: keep product-config config-spec as empty stubs The operator no longer reads a product-config spec, but the config-spec properties.yaml files must remain (empty) for chart/release tooling, matching trino-operator and hdfs-operator. Revert the Helm/Dockerfile changes so the config-spec wiring is unchanged from main; only the spec contents are emptied. Co-Authored-By: Claude Opus 4.8 (1M context) --- deploy/config-spec/properties.yaml | 5 +++++ deploy/helm/druid-operator/configs/properties.yaml | 5 +++++ deploy/helm/druid-operator/templates/configmap.yaml | 9 +++++++++ deploy/helm/druid-operator/templates/deployment.yaml | 8 ++++++++ docker/Dockerfile | 2 ++ 5 files changed, 29 insertions(+) create mode 100644 deploy/config-spec/properties.yaml create mode 100644 deploy/helm/druid-operator/configs/properties.yaml create mode 100644 deploy/helm/druid-operator/templates/configmap.yaml diff --git a/deploy/config-spec/properties.yaml b/deploy/config-spec/properties.yaml new file mode 100644 index 00000000..9bd8c3b2 --- /dev/null +++ b/deploy/config-spec/properties.yaml @@ -0,0 +1,5 @@ +--- +version: 0.1.0 +spec: + units: [] +properties: [] diff --git a/deploy/helm/druid-operator/configs/properties.yaml b/deploy/helm/druid-operator/configs/properties.yaml new file mode 100644 index 00000000..9bd8c3b2 --- /dev/null +++ b/deploy/helm/druid-operator/configs/properties.yaml @@ -0,0 +1,5 @@ +--- +version: 0.1.0 +spec: + units: [] +properties: [] diff --git a/deploy/helm/druid-operator/templates/configmap.yaml b/deploy/helm/druid-operator/templates/configmap.yaml new file mode 100644 index 00000000..e75acc25 --- /dev/null +++ b/deploy/helm/druid-operator/templates/configmap.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +data: +{{ (.Files.Glob "configs/*").AsConfig | indent 2 }} +kind: ConfigMap +metadata: + name: {{ include "operator.fullname" . }}-configmap + labels: + {{- include "operator.labels" . | nindent 4 }} diff --git a/deploy/helm/druid-operator/templates/deployment.yaml b/deploy/helm/druid-operator/templates/deployment.yaml index 06dde967..d0cecba4 100644 --- a/deploy/helm/druid-operator/templates/deployment.yaml +++ b/deploy/helm/druid-operator/templates/deployment.yaml @@ -16,6 +16,7 @@ spec: metadata: annotations: internal.stackable.tech/image: {{ include "operator.image" . }} + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} @@ -40,6 +41,9 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} resources: {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - mountPath: /etc/stackable/{{ include "operator.appname" . }}/config-spec + name: config-spec env: # The following env vars are passed as clap (think CLI) arguments to the operator. # They are picked up by clap using the structs defied in the operator. @@ -84,6 +88,10 @@ spec: {{- include "telemetry.envVars" . | nindent 12 }} {{- include "maintenance.envVars" . | nindent 12 }} + volumes: + - name: config-spec + configMap: + name: {{ include "operator.fullname" . }}-configmap {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/docker/Dockerfile b/docker/Dockerfile index 226bd4db..84817207 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -191,6 +191,8 @@ COPY LICENSE /licenses/LICENSE COPY --from=builder --chown=${STACKABLE_USER_UID}:0 /app/* /usr/local/bin/ +COPY deploy/config-spec/properties.yaml /etc/stackable/druid-operator/config-spec/properties.yaml + USER ${STACKABLE_USER_UID} ENTRYPOINT ["stackable-druid-operator"] From 3fd7d2b82fee06b2bf030c21e548f9c1e7c56d50 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 4 Jun 2026 22:40:08 +0200 Subject: [PATCH 09/48] refactor: introduce ConfigFileName enum and drop hardcoded file-name constants Co-Authored-By: Claude Opus 4.8 (1M context) --- rust/operator-binary/src/config/jvm.rs | 12 +- rust/operator-binary/src/controller.rs | 12 +- .../src/controller/build/config_map.rs | 28 ++-- .../src/controller/build/properties/mod.rs | 9 ++ .../src/controller/validate.rs | 142 ++++++++++++------ rust/operator-binary/src/crd/mod.rs | 101 +++++-------- rust/operator-binary/src/product_logging.rs | 6 +- 7 files changed, 173 insertions(+), 137 deletions(-) diff --git a/rust/operator-binary/src/config/jvm.rs b/rust/operator-binary/src/config/jvm.rs index 90f8ffa0..e9bcd8a3 100644 --- a/rust/operator-binary/src/config/jvm.rs +++ b/rust/operator-binary/src/config/jvm.rs @@ -6,10 +6,14 @@ use stackable_operator::{ }; use crate::crd::{ - DruidConfigOverrides, DruidRole, JVM_SECURITY_PROPERTIES_FILE, LOG4J2_CONFIG, - RW_CONFIG_DIRECTORY, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, + DruidConfigOverrides, DruidRole, RW_CONFIG_DIRECTORY, STACKABLE_TRUST_STORE, + STACKABLE_TRUST_STORE_PASSWORD, }; +// File names not exported from crd/mod.rs (see ConfigFileName for the properties-builder files). +const SECURITY_PROPERTIES_FILE: &str = "security.properties"; +const LOG4J2_CONFIG_FILE: &str = "log4j2.properties"; + #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("failed to format memory quantity {value:?} for Java"))] @@ -54,12 +58,12 @@ pub fn construct_jvm_args( jvm_args.extend([ "-XX:+ExitOnOutOfMemoryError".to_owned(), "-XX:+UseG1GC".to_owned(), - format!("-Djava.security.properties={RW_CONFIG_DIRECTORY}/{JVM_SECURITY_PROPERTIES_FILE}"), + format!("-Djava.security.properties={RW_CONFIG_DIRECTORY}/{SECURITY_PROPERTIES_FILE}"), "-Duser.timezone=UTC".to_owned(), "-Dfile.encoding=UTF-8".to_owned(), "-Djava.io.tmpdir=/tmp".to_owned(), "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager".to_owned(), - format!("-Dlog4j.configurationFile={RW_CONFIG_DIRECTORY}/{LOG4J2_CONFIG}"), + format!("-Dlog4j.configurationFile={RW_CONFIG_DIRECTORY}/{LOG4J2_CONFIG_FILE}"), format!("-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}"), format!("-Djavax.net.ssl.trustStorePassword={STACKABLE_TRUST_STORE_PASSWORD}"), "-Djavax.net.ssl.trustStoreType=pkcs12".to_owned(), diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 808b9f1e..83d85105 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -948,13 +948,13 @@ mod test { use crate::controller::build::properties::writer; use crate::{ controller::{ - build::{config_map::build_rolegroup_config_map, properties::runtime_properties}, + build::{ + config_map::build_rolegroup_config_map, + properties::{ConfigFileName, runtime_properties}, + }, validate::{DruidRoleGroupConfig, ValidatedCluster, ValidatedClusterConfig}, }, - crd::{ - PROP_SEGMENT_CACHE_LOCATIONS, RUNTIME_PROPS, - authentication::AuthenticationClassesResolved, - }, + crd::{PROP_SEGMENT_CACHE_LOCATIONS, authentication::AuthenticationClassesResolved}, }; #[rstest] @@ -1045,7 +1045,7 @@ mod test { let druid_segment_cache_property = rg_configmap .data .unwrap() - .get(RUNTIME_PROPS) + .get(&ConfigFileName::RuntimeProperties.to_string()) .unwrap() .to_string(); diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 10172107..b54f944f 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -24,21 +24,24 @@ use stackable_operator::{ use crate::{ config::jvm::construct_jvm_args, - controller::build::properties::writer::to_java_properties_string, + controller::build::properties::{ConfigFileName, writer::to_java_properties_string}, controller::{ DRUID_CONTROLLER_NAME, validate::{DruidRoleGroupConfig, ValidatedCluster}, }, crd::{ - AUTH_AUTHORIZER_OPA_URI, DS_BUCKET, DruidRole, EXTENSIONS_LOADLIST, JVM_CONFIG, - JVM_SECURITY_PROPERTIES_FILE, RUNTIME_PROPS, S3_ACCESS_KEY, S3_ENDPOINT_URL, - S3_PATH_STYLE_ACCESS, S3_SECRET_KEY, ZOOKEEPER_CONNECTION_STRING, build_recommended_labels, - build_string_list, v1alpha1, + AUTH_AUTHORIZER_OPA_URI, DS_BUCKET, DruidRole, EXTENSIONS_LOADLIST, S3_ACCESS_KEY, + S3_ENDPOINT_URL, S3_PATH_STYLE_ACCESS, S3_SECRET_KEY, ZOOKEEPER_CONNECTION_STRING, + build_recommended_labels, build_string_list, v1alpha1, }, extensions::get_extension_list, product_logging::extend_role_group_config_map, }; +// jvm.config is built by `config::jvm`, not a properties builder, so it is not part +// of `ConfigFileName`. +const JVM_CONFIG: &str = "jvm.config"; + #[derive(Snafu, Debug)] #[allow(clippy::enum_variant_names)] pub enum Error { @@ -58,12 +61,12 @@ pub enum Error { source: stackable_operator::crd::s3::v1alpha1::ConnectionError, }, - #[snafu(display("failed to format runtime properties"))] - PropertiesWriteError { + #[snafu(display("failed to serialize [runtime.properties]"))] + SerializeRuntimeProperties { source: crate::controller::build::properties::writer::PropertiesWriterError, }, - #[snafu(display("failed to serialize [{JVM_SECURITY_PROPERTIES_FILE}] for {rolegroup}"))] + #[snafu(display("failed to serialize [security.properties] for {rolegroup}"))] JvmSecurityProperties { source: crate::controller::build::properties::writer::PropertiesWriterError, rolegroup: String, @@ -260,8 +263,11 @@ pub fn build_rolegroup_config_map( conf.extend(rg.runtime_config.clone()); let runtime_properties = - to_java_properties_string(conf.iter()).context(PropertiesWriteSnafu)?; - cm_conf_data.insert(RUNTIME_PROPS.to_string(), runtime_properties); + to_java_properties_string(conf.iter()).context(SerializeRuntimePropertiesSnafu)?; + cm_conf_data.insert( + ConfigFileName::RuntimeProperties.to_string(), + runtime_properties, + ); } // ----- jvm.config ----- @@ -279,7 +285,7 @@ pub fn build_rolegroup_config_map( // ----- security.properties ----- { cm_conf_data.insert( - JVM_SECURITY_PROPERTIES_FILE.to_string(), + ConfigFileName::SecurityProperties.to_string(), to_java_properties_string(rg.security_config.iter()).with_context(|_| { JvmSecurityPropertiesSnafu { rolegroup: rolegroup.role_group.clone(), diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index 9a1f5510..426fb4bb 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -4,3 +4,12 @@ pub mod writer; pub mod runtime_properties; pub mod security_properties; + +/// The names of the operator-written Druid config files assembled into the rolegroup ConfigMap. +#[derive(Clone, Copy, Debug, strum::Display)] +pub enum ConfigFileName { + #[strum(serialize = "runtime.properties")] + RuntimeProperties, + #[strum(serialize = "security.properties")] + SecurityProperties, +} diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 272283a3..444db959 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -3,13 +3,12 @@ //! Synchronously validates inputs that don't require a Kubernetes client. Produces //! [`ValidatedCluster`], consumed by the rest of `reconcile_druid`. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, commons::product_image_selection::{self, ResolvedProductImage}, - config_overrides::KeyValueOverridesProvider, crd::s3, kube::ResourceExt, }; @@ -18,12 +17,12 @@ use strum::IntoEnumIterator; use crate::{ authentication::DruidAuthenticationConfig, controller::{ - build::properties::{runtime_properties, security_properties}, + build::properties::{ConfigFileName, runtime_properties, security_properties}, dereference::DereferencedObjects, }, crd::{ - CommonRoleGroupConfig, DruidRole, INDEXER_JAVA_OPTS, JVM_SECURITY_PROPERTIES_FILE, - RUNTIME_PROPS, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, build_string_list, + CommonRoleGroupConfig, DruidConfigOverrides, DruidRole, INDEXER_JAVA_OPTS, + STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, build_string_list, security::DruidTlsSecurity, v1alpha1, }, }; @@ -84,6 +83,79 @@ pub struct ValidatedCluster { pub role_group_configs: BTreeMap>, } +/// Returns the user-supplied key/value overrides for the given config file from a +/// [`DruidConfigOverrides`], as a product-config style map. +fn key_value_overrides( + overrides: &DruidConfigOverrides, + file: ConfigFileName, +) -> BTreeMap> { + let kv = match file { + ConfigFileName::RuntimeProperties => overrides.runtime_properties.as_ref(), + ConfigFileName::SecurityProperties => overrides.security_properties.as_ref(), + }; + kv.map( + stackable_operator::config_overrides::KeyValueConfigOverrides::as_product_config_overrides, + ) + .unwrap_or_default() +} + +/// Builds the precomputed per-file config for a single rolegroup. Pure assembly: combines the +/// role-level overrides with the rolegroup-level overrides (rolegroup wins) on top of the +/// computed defaults. No behavior change vs. the inline loop body it was extracted from. +#[allow(clippy::too_many_arguments)] +fn build_role_group_config( + druid: &v1alpha1::DruidCluster, + druid_role: &DruidRole, + merged_config: CommonRoleGroupConfig, + role_runtime_overrides: &BTreeMap>, + role_security_overrides: &BTreeMap>, + role_env_overrides: &HashMap, + rg_config_overrides: &DruidConfigOverrides, + rg_env_overrides: &HashMap, +) -> DruidRoleGroupConfig { + // ----- runtime.properties ----- + let mut runtime_config = druid.compute_runtime_properties(); + if *druid_role == DruidRole::MiddleManager { + let (k, v) = middlemanager_indexer_java_opts(); + runtime_config.insert(k, v); + } + runtime_config.extend(runtime_properties::defaults(druid_role)); + // merged user overrides (role <- rolegroup; rolegroup wins) + let mut runtime_overrides = role_runtime_overrides.clone(); + runtime_overrides.extend(key_value_overrides( + rg_config_overrides, + ConfigFileName::RuntimeProperties, + )); + runtime_config.extend(runtime_overrides); + + // ----- security.properties ----- + let mut security_config: BTreeMap> = BTreeMap::new(); + if *druid_role == DruidRole::MiddleManager { + let (k, v) = middlemanager_indexer_java_opts(); + security_config.insert(k, v); + } + let mut security_overrides = role_security_overrides.clone(); + security_overrides.extend(key_value_overrides( + rg_config_overrides, + ConfigFileName::SecurityProperties, + )); + security_config.extend(security_properties::build(&security_overrides)); + + // ----- env ----- + let mut env: BTreeMap = role_env_overrides + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + env.extend(rg_env_overrides.iter().map(|(k, v)| (k.clone(), v.clone()))); + + DruidRoleGroupConfig { + merged_config, + runtime_config, + security_config, + env, + } +} + /// The `druid.indexer.runner.javaOptsArray` entry that `MiddleManagerConfigFragment::compute_files` /// adds for *every* file (runtime.properties and security.properties). fn middlemanager_indexer_java_opts() -> (String, Option) { @@ -131,14 +203,14 @@ pub fn validate( for druid_role in DruidRole::iter() { // The role-level overrides (role <- rolegroup precedence starts here). let role = druid.get_role(&druid_role); - let role_runtime_overrides = role - .config - .config_overrides - .get_key_value_overrides(RUNTIME_PROPS); - let role_security_overrides = role - .config - .config_overrides - .get_key_value_overrides(JVM_SECURITY_PROPERTIES_FILE); + let role_runtime_overrides = key_value_overrides( + &role.config.config_overrides, + ConfigFileName::RuntimeProperties, + ); + let role_security_overrides = key_value_overrides( + &role.config.config_overrides, + ConfigFileName::SecurityProperties, + ); let role_env_overrides = role.config.env_overrides.clone(); let rolegroups = merged.role_group_names(&druid_role); @@ -155,44 +227,18 @@ pub fn validate( .role_group_overrides(&druid_role, &rg_name) .expect("role group resolved by common_config must exist"); - // ----- runtime.properties ----- - let mut runtime_config = druid.common_compute_files(RUNTIME_PROPS); - if druid_role == DruidRole::MiddleManager { - let (k, v) = middlemanager_indexer_java_opts(); - runtime_config.insert(k, v); - } - runtime_config.extend(runtime_properties::defaults(&druid_role)); - // merged user overrides (role <- rolegroup; rolegroup wins) - let mut runtime_overrides = role_runtime_overrides.clone(); - runtime_overrides.extend(rg_config_overrides.get_key_value_overrides(RUNTIME_PROPS)); - runtime_config.extend(runtime_overrides); - - // ----- security.properties ----- - let mut security_config: BTreeMap> = BTreeMap::new(); - if druid_role == DruidRole::MiddleManager { - let (k, v) = middlemanager_indexer_java_opts(); - security_config.insert(k, v); - } - let mut security_overrides = role_security_overrides.clone(); - security_overrides - .extend(rg_config_overrides.get_key_value_overrides(JVM_SECURITY_PROPERTIES_FILE)); - security_config.extend(security_properties::build(&security_overrides)); - - // ----- env ----- - let mut env: BTreeMap = role_env_overrides - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - env.extend(rg_env_overrides.iter().map(|(k, v)| (k.clone(), v.clone()))); - group_map.insert( rg_name, - DruidRoleGroupConfig { + build_role_group_config( + druid, + &druid_role, merged_config, - runtime_config, - security_config, - env, - }, + &role_runtime_overrides, + &role_security_overrides, + &role_env_overrides, + rg_config_overrides, + rg_env_overrides, + ), ); } role_group_configs.insert(druid_role, group_map); diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 5d928d69..88650abf 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -17,7 +17,7 @@ use stackable_operator::{ fragment::{self, Fragment, FromFragment, ValidationError}, merge::Merge, }, - config_overrides::{KeyValueConfigOverrides, KeyValueOverridesProvider}, + config_overrides::KeyValueConfigOverrides, crd::{ authentication::{core, oidc}, s3, @@ -69,12 +69,6 @@ pub const HDFS_CONFIG_DIRECTORY: &str = "/stackable/hdfs"; pub const LOG_CONFIG_DIRECTORY: &str = "/stackable/log_config"; pub const RW_CONFIG_DIRECTORY: &str = "/stackable/rwconfig"; -// config file names -pub const JVM_CONFIG: &str = "jvm.config"; -pub const RUNTIME_PROPS: &str = "runtime.properties"; -pub const LOG4J2_CONFIG: &str = "log4j2.properties"; -pub const JVM_SECURITY_PROPERTIES_FILE: &str = "security.properties"; - // store directories pub const STACKABLE_TRUST_STORE: &str = "/stackable/truststore.p12"; pub const STACKABLE_TRUST_STORE_PASSWORD: &str = "changeit"; @@ -147,6 +141,7 @@ const DEFAULT_HISTORICAL_SECRET_LIFETIME: Duration = Duration::from_days_uncheck #[serde(rename_all = "camelCase")] pub struct DruidConfigOverrides { /// Overrides for the `runtime.properties` file. + // File name defined in [`crate::controller::build::properties::ConfigFileName`] #[serde( default, rename = "runtime.properties", @@ -155,6 +150,7 @@ pub struct DruidConfigOverrides { pub runtime_properties: Option, /// Overrides for the `jvm.config` file. + // File name defined in [`crate::controller::build::properties::ConfigFileName`] #[serde( default, rename = "jvm.config", @@ -163,6 +159,7 @@ pub struct DruidConfigOverrides { pub jvm_config: Option, /// Overrides for the `security.properties` file. + // File name defined in [`crate::controller::build::properties::ConfigFileName`] #[serde( default, rename = "security.properties", @@ -171,29 +168,6 @@ pub struct DruidConfigOverrides { pub security_properties: Option, } -impl KeyValueOverridesProvider for DruidConfigOverrides { - fn get_key_value_overrides(&self, file: &str) -> BTreeMap> { - match file { - RUNTIME_PROPS => self - .runtime_properties - .as_ref() - .map(KeyValueConfigOverrides::as_product_config_overrides) - .unwrap_or_default(), - JVM_CONFIG => self - .jvm_config - .as_ref() - .map(KeyValueConfigOverrides::as_product_config_overrides) - .unwrap_or_default(), - JVM_SECURITY_PROPERTIES_FILE => self - .security_properties - .as_ref() - .map(KeyValueConfigOverrides::as_product_config_overrides) - .unwrap_or_default(), - _ => BTreeMap::new(), - } - } -} - #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] #[allow(clippy::enum_variant_names)] @@ -399,48 +373,41 @@ impl HasStatusCondition for v1alpha1::DruidCluster { } impl v1alpha1::DruidCluster { - pub fn common_compute_files(&self, file: &str) -> BTreeMap> { + pub fn compute_runtime_properties(&self) -> BTreeMap> { let mut result = BTreeMap::new(); - match file { - JVM_CONFIG => {} - RUNTIME_PROPS => { - // OPA - if let Some(DruidAuthorization { opa: _ }) = &self.spec.cluster_config.authorization - { - result.insert( - AUTH_AUTHORIZERS.to_string(), - Some(AUTH_AUTHORIZERS_VALUE.to_string()), - ); - result.insert( - AUTH_AUTHORIZER_OPA_TYPE.to_string(), - Some(AUTH_AUTHORIZER_OPA_TYPE_VALUE.to_string()), - ); - // The opaUri still needs to be set, but that requires a discovery config map and is handled in the controller.rs - } - // deep storage - result.insert( - DS_TYPE.to_string(), - Some(self.spec.cluster_config.deep_storage.to_string()), - ); - match self.spec.cluster_config.deep_storage.clone() { - DeepStorageSpec::Hdfs(hdfs) => { - result.insert(DS_DIRECTORY.to_string(), Some(hdfs.directory)); - } - DeepStorageSpec::S3(s3_spec) => { - if let Some(key) = &s3_spec.base_key { - result.insert(DS_BASE_KEY.to_string(), Some(key.to_string())); - } - // bucket information (name, connection) needs to be resolved first, - // that is done directly in the controller - } + // OPA + if let Some(DruidAuthorization { opa: _ }) = &self.spec.cluster_config.authorization { + result.insert( + AUTH_AUTHORIZERS.to_string(), + Some(AUTH_AUTHORIZERS_VALUE.to_string()), + ); + result.insert( + AUTH_AUTHORIZER_OPA_TYPE.to_string(), + Some(AUTH_AUTHORIZER_OPA_TYPE_VALUE.to_string()), + ); + // The opaUri still needs to be set, but that requires a discovery config map and is handled in the controller.rs + } + // deep storage + result.insert( + DS_TYPE.to_string(), + Some(self.spec.cluster_config.deep_storage.to_string()), + ); + match self.spec.cluster_config.deep_storage.clone() { + DeepStorageSpec::Hdfs(hdfs) => { + result.insert(DS_DIRECTORY.to_string(), Some(hdfs.directory)); + } + DeepStorageSpec::S3(s3_spec) => { + if let Some(key) = &s3_spec.base_key { + result.insert(DS_BASE_KEY.to_string(), Some(key.to_string())); } - - // metrics - result.insert(PROMETHEUS_PORT.to_string(), Some(METRICS_PORT.to_string())); + // bucket information (name, connection) needs to be resolved first, + // that is done directly in the controller } - _ => {} } + // metrics + result.insert(PROMETHEUS_PORT.to_string(), Some(METRICS_PORT.to_string())); + result } diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs index b2923da1..a1b0809f 100644 --- a/rust/operator-binary/src/product_logging.rs +++ b/rust/operator-binary/src/product_logging.rs @@ -10,9 +10,13 @@ use stackable_operator::{ }; use crate::crd::{ - Container, DRUID_LOG_FILE, LOG4J2_CONFIG, MAX_DRUID_LOG_FILES_SIZE, STACKABLE_LOG_DIR, v1alpha1, + Container, DRUID_LOG_FILE, MAX_DRUID_LOG_FILES_SIZE, STACKABLE_LOG_DIR, v1alpha1, }; +// log4j2.properties is written by the logging framework, not a properties builder, so it is +// not part of ConfigFileName. File name not exported from crd/mod.rs. +const LOG4J2_CONFIG: &str = "log4j2.properties"; + #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("object has no namespace"))] From cecfef69bb2070ae9414fe43eb92e57b436ffd8e Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 5 Jun 2026 13:14:50 +0200 Subject: [PATCH 10/48] fix(clippy): remove large_varian_names --- rust/operator-binary/src/controller.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 83d85105..1a37e84d 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -97,7 +97,6 @@ pub struct Ctx { #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] -#[allow(clippy::enum_variant_names)] pub enum Error { #[snafu(display("failed to apply Service for {}", rolegroup))] ApplyRoleGroupService { @@ -945,12 +944,11 @@ mod test { use rstest::*; use super::*; - use crate::controller::build::properties::writer; use crate::{ controller::{ build::{ config_map::build_rolegroup_config_map, - properties::{ConfigFileName, runtime_properties}, + properties::{ConfigFileName, runtime_properties, writer}, }, validate::{DruidRoleGroupConfig, ValidatedCluster, ValidatedClusterConfig}, }, From 40dd338163282d711d0be9ef327dce67a2dc9b50 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 5 Jun 2026 13:16:42 +0200 Subject: [PATCH 11/48] fix: rename validated -> validated_cluster --- rust/operator-binary/src/controller.rs | 31 +++++++++++++------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 1a37e84d..b9b47238 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -284,8 +284,9 @@ pub async fn reconcile_druid( .await .context(DereferenceSnafu)?; - let validated = validate::validate(druid, &dereferenced_objects, &ctx.operator_environment) - .context(ValidateClusterSnafu)?; + let validated_cluster = + validate::validate(druid, &dereferenced_objects, &ctx.operator_environment) + .context(ValidateClusterSnafu)?; let mut cluster_resources = ClusterResources::new( APP_NAME, @@ -317,7 +318,7 @@ pub async fn reconcile_druid( let mut ss_cond_builder = StatefulSetConditionBuilder::default(); - for (druid_role, groups) in validated.role_group_configs.iter() { + for (druid_role, groups) in validated_cluster.role_group_configs.iter() { let role_name = druid_role.to_string(); create_shared_internal_secret(druid, client, DRUID_CONTROLLER_NAME) @@ -334,7 +335,7 @@ pub async fn reconcile_druid( let role_group_service_recommended_labels = build_recommended_labels( druid, DRUID_CONTROLLER_NAME, - &validated.image.app_version_label_value, + &validated_cluster.image.app_version_label_value, &rolegroup.role, &rolegroup.role_group, ); @@ -349,7 +350,7 @@ pub async fn reconcile_druid( let rg_headless_service = build_rolegroup_headless_service( druid, - &validated.cluster_config.druid_tls_security, + &validated_cluster.cluster_config.druid_tls_security, druid_role, &rolegroup, role_group_service_recommended_labels.clone(), @@ -365,23 +366,23 @@ pub async fn reconcile_druid( .context(ServiceConfigurationSnafu)?; let rg_configmap = build::config_map::build_rolegroup_config_map( - &validated, + &validated_cluster, druid_role, &rolegroup, rg, - &validated.image, + &validated_cluster.image, druid, ) .context(BuildConfigMapSnafu)?; let rg_statefulset = build_rolegroup_statefulset( druid, - &validated.image, + &validated_cluster.image, druid_role, &rolegroup, rg, - validated.cluster_config.s3_connection.as_ref(), - &validated.cluster_config.druid_tls_security, - &validated.cluster_config.druid_auth_config, + validated_cluster.cluster_config.s3_connection.as_ref(), + &validated_cluster.cluster_config.druid_tls_security, + &validated_cluster.cluster_config.druid_auth_config, &rbac_sa, )?; @@ -424,14 +425,14 @@ pub async fn reconcile_druid( build_recommended_labels( druid, DRUID_CONTROLLER_NAME, - &validated.image.app_version_label_value, + &validated_cluster.image.app_version_label_value, &role_name, "none", ), listener_class.to_string(), listener_group_name, druid_role, - &validated.cluster_config.druid_tls_security, + &validated_cluster.cluster_config.druid_tls_security, ) .context(ListenerConfigurationSnafu)?; @@ -445,8 +446,8 @@ pub async fn reconcile_druid( for discovery_cm in build_discovery_configmaps( druid, druid, - &validated.image, - &validated.cluster_config.druid_tls_security, + &validated_cluster.image, + &validated_cluster.cluster_config.druid_tls_security, listener, ) .await From 3d028db6f0ebfd8a2ac96ae02b2bf02d540a429a Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 5 Jun 2026 13:19:07 +0200 Subject: [PATCH 12/48] fix: remove obsolete image parameter --- rust/operator-binary/src/controller.rs | 1 - rust/operator-binary/src/controller/build/config_map.rs | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index b9b47238..1a72a51f 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -370,7 +370,6 @@ pub async fn reconcile_druid( druid_role, &rolegroup, rg, - &validated_cluster.image, druid, ) .context(BuildConfigMapSnafu)?; diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index b54f944f..d3725028 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -15,7 +15,6 @@ use std::collections::BTreeMap; use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, - commons::product_image_selection::ResolvedProductImage, crd::s3, database_connections::drivers::jdbc::JdbcDatabaseConnection as _, k8s_openapi::api::core::v1::{ConfigMap, EnvVar}, @@ -24,9 +23,9 @@ use stackable_operator::{ use crate::{ config::jvm::construct_jvm_args, - controller::build::properties::{ConfigFileName, writer::to_java_properties_string}, controller::{ DRUID_CONTROLLER_NAME, + build::properties::{ConfigFileName, writer::to_java_properties_string}, validate::{DruidRoleGroupConfig, ValidatedCluster}, }, crd::{ @@ -111,7 +110,6 @@ pub fn build_rolegroup_config_map( role: &DruidRole, rolegroup: &RoleGroupRef, rg: &DruidRoleGroupConfig, - resolved_product_image: &ResolvedProductImage, owner: &v1alpha1::DruidCluster, ) -> Result { let cluster_config = &cluster.cluster_config; @@ -304,7 +302,7 @@ pub fn build_rolegroup_config_map( .with_recommended_labels(&build_recommended_labels( owner, DRUID_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, + &cluster.image.app_version_label_value, &rolegroup.role, &rolegroup.role_group, )) From 09d457ead721ff24c20835bc8cc71884f4f274aa Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 5 Jun 2026 13:20:41 +0200 Subject: [PATCH 13/48] fix: regenerate & fix test --- Cargo.nix | 767 +++++++++++++------------ crate-hashes.json | 18 +- rust/operator-binary/src/controller.rs | 1 - 3 files changed, 422 insertions(+), 364 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index c41d291e..20c06abd 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -489,9 +489,9 @@ rec { }; "autocfg" = rec { crateName = "autocfg"; - version = "1.5.0"; + version = "1.5.1"; edition = "2015"; - sha256 = "1s77f98id9l4af4alklmzq46f21c980v13z2r1pcxx6bqgw0d1n0"; + sha256 = "0lqasy5i30flcgih1b50kvsk6z32g09r1q4ql7q81pj6228jy0zj"; authors = [ "Josh Stone " ]; @@ -866,9 +866,9 @@ rec { }; "bitflags" = rec { crateName = "bitflags"; - version = "2.11.1"; + version = "2.12.1"; edition = "2021"; - sha256 = "1cvqijg3rvwgis20a66vfdxannjsxfy5fgjqkaq3l13gyfcj4lf4"; + sha256 = "02phhjm7w380zdh8928zf13cfi1bw2qz2ay36ml2jmwmmv8cxmw4"; authors = [ "The Rust Project Developers" ]; @@ -898,9 +898,9 @@ rec { }; "built" = rec { crateName = "built"; - version = "0.8.0"; - edition = "2021"; - sha256 = "0r5f08lpjsr6j5ajkbmd0ymfmajpq8ddbfvi8ji8rx48y88qzbgl"; + version = "0.8.1"; + edition = "2024"; + sha256 = "1saq332pd6g3svvc9ah8myjpfvgqlzl2ksb1ypp3976kjcfm63jw"; authors = [ "Lukas Lueg " ]; @@ -924,15 +924,16 @@ rec { "chrono" = [ "dep:chrono" ]; "dependency-tree" = [ "cargo-lock/dependency-tree" ]; "git2" = [ "dep:git2" ]; + "gix" = [ "dep:gix" ]; "semver" = [ "dep:semver" ]; }; resolvedDefaultFeatures = [ "chrono" "git2" ]; }; "bumpalo" = rec { crateName = "bumpalo"; - version = "3.20.2"; + version = "3.20.3"; edition = "2021"; - sha256 = "1jrgxlff76k9glam0akhwpil2fr1w32gbjdf5hpipc7ld2c7h82x"; + sha256 = "0jc6va3nwcqikm7chnpdv1s87my3gs2j7g1sc7g3k91brg3arxbj"; authors = [ "Nick Fitzgerald " ]; @@ -961,9 +962,9 @@ rec { }; "cc" = rec { crateName = "cc"; - version = "1.2.61"; + version = "1.2.63"; edition = "2018"; - sha256 = "0vawvnrrsmi8dygavq3wx085cmlp10sp3fhld5842rlqkqsr0vfi"; + sha256 = "0zy2bqc4nvj6bv2cipx4h4bn65wf1zqf1fw1hsh64mmvg1hh2vjm"; authors = [ "Alex Crichton " ]; @@ -1011,9 +1012,9 @@ rec { }; "chrono" = rec { crateName = "chrono"; - version = "0.4.44"; + version = "0.4.45"; edition = "2021"; - sha256 = "1c64mk9a235271j5g3v4zrzqqmd43vp9vki7vqfllpqf5rd0fwy6"; + sha256 = "09rkcgk6is2sdhqs9142zv8xqnj8ryx8m9hknllqwyv9wxi9x9qs"; dependencies = [ { name = "iana-time-zone"; @@ -1906,9 +1907,9 @@ rec { }; "displaydoc" = rec { crateName = "displaydoc"; - version = "0.2.5"; + version = "0.2.6"; edition = "2021"; - sha256 = "1q0alair462j21iiqwrr21iabkfnb13d6x5w95lkdg21q2xrqdlp"; + sha256 = "0kyxwfbdmagd8afzb2pzja7wj8dhah7smxdsgw00iq8pa2jhmiqs"; procMacro = true; authors = [ "Jane Lusby " @@ -2108,12 +2109,9 @@ rec { }; "either" = rec { crateName = "either"; - version = "1.15.0"; + version = "1.16.0"; edition = "2021"; - sha256 = "069p1fknsmzn9llaizh77kip0pqmcwpdsykv2x30xpjyija5gis8"; - authors = [ - "bluss" - ]; + sha256 = "17k7jfbdz7k440h6lws9baz8p9zlxgb41sig3w81h80nwzsjyqli"; features = { "default" = [ "std" ]; "serde" = [ "dep:serde" ]; @@ -2858,9 +2856,9 @@ rec { }; "futures-timer" = rec { crateName = "futures-timer"; - version = "3.0.3"; + version = "3.0.4"; edition = "2018"; - sha256 = "094vw8k37djpbwv74bwf2qb7n6v6ghif4myss6smd6hgyajb127j"; + sha256 = "0s39in8ivw7g4d37pf31q02y44zd1hpfkd1pgra2slcqibdzlhxg"; libName = "futures_timer"; authors = [ "Alex Crichton " @@ -3118,9 +3116,9 @@ rec { }; "git2" = rec { crateName = "git2"; - version = "0.20.4"; - edition = "2018"; - sha256 = "0azykjpk3j6s354z23jkyq3r3pbmlw9ha1zsxkw5cnnpi1h2b23v"; + version = "0.21.0"; + edition = "2021"; + sha256 = "0bmqga9vlyx5sdlr0i28z0362s89xv9i4qcv20vvx9j54y9vzpfx"; authors = [ "Josh Triplett " "Alex Crichton " @@ -3142,17 +3140,14 @@ rec { name = "log"; packageId = "log"; } - { - name = "url"; - packageId = "url"; - } ]; features = { - "default" = [ "ssh" "https" ]; - "https" = [ "libgit2-sys/https" "openssl-sys" "openssl-probe" ]; + "cred" = [ "dep:url" ]; + "https" = [ "libgit2-sys/https" "openssl-sys" "openssl-probe" "cred" ]; "openssl-probe" = [ "dep:openssl-probe" ]; "openssl-sys" = [ "dep:openssl-sys" ]; - "ssh" = [ "libgit2-sys/ssh" ]; + "ssh" = [ "libgit2-sys/ssh" "cred" ]; + "unstable-sha256" = [ "libgit2-sys/unstable-sha256" ]; "vendored-libgit2" = [ "libgit2-sys/vendored" ]; "vendored-openssl" = [ "openssl-sys/vendored" "libgit2-sys/vendored-openssl" ]; "zlib-ng-compat" = [ "libgit2-sys/zlib-ng-compat" ]; @@ -3242,9 +3237,9 @@ rec { }; "h2" = rec { crateName = "h2"; - version = "0.4.13"; + version = "0.4.14"; edition = "2021"; - sha256 = "0m6w5gg0n0m1m5915bxrv8n4rlazhx5icknkslz719jhh4xdli1g"; + sha256 = "0cw7jk7kn2vn6f8w8ssh6gis1mljnfjxd606gvi4sjpyjayfy7qp"; authors = [ "Carl Lerche " "Sean McArthur " @@ -3355,14 +3350,11 @@ rec { }; resolvedDefaultFeatures = [ "allocator-api2" "default" "default-hasher" "equivalent" "inline-more" "raw-entry" ]; }; - "hashbrown 0.17.0" = rec { + "hashbrown 0.17.1" = rec { crateName = "hashbrown"; - version = "0.17.0"; + version = "0.17.1"; edition = "2024"; - sha256 = "0l8gvcz80lvinb7x22h53cqbi2y1fm603y2jhhh9qwygvkb7sijg"; - authors = [ - "Amanieu d'Antras " - ]; + sha256 = "0jmqz7i4yl6cm7rbn0i2ffkfrmwi6xkmzkaldr2v8bcsx2v0jngd"; features = { "alloc" = [ "dep:alloc" ]; "allocator-api2" = [ "dep:allocator-api2" ]; @@ -3437,9 +3429,9 @@ rec { }; "http" = rec { crateName = "http"; - version = "1.4.0"; + version = "1.4.1"; edition = "2021"; - sha256 = "06iind4cwsj1d6q8c2xgq8i2wka4ps74kmws24gsi1bzdlw2mfp3"; + sha256 = "1l7k2ia57z3q7q3ka497krzps795kd3fymm2k12lr623y4nldrwb"; authors = [ "Alex Crichton " "Carl Lerche " @@ -3556,9 +3548,9 @@ rec { }; "hyper" = rec { crateName = "hyper"; - version = "1.9.0"; + version = "1.10.1"; edition = "2021"; - sha256 = "1jmwbwqcaficskg76kq402gbymbnh2z4v99xwq3l5aa6n8bg16b2"; + sha256 = "1624nwrh1ci34psqcl3q8q266kha8kd6fmqjj14qck49l59iqa2m"; authors = [ "Sean McArthur " ]; @@ -4365,7 +4357,7 @@ rec { } { name = "hashbrown"; - packageId = "hashbrown 0.17.0"; + packageId = "hashbrown 0.17.1"; usesDefaultFeatures = false; } ]; @@ -4423,39 +4415,6 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; - "iri-string" = rec { - crateName = "iri-string"; - version = "0.7.12"; - edition = "2021"; - sha256 = "082fpx6c5ghvmqpwxaf2b268m47z2ic3prajqbmi1s1qpfj5kri5"; - libName = "iri_string"; - authors = [ - "YOSHIOKA Takuma " - ]; - dependencies = [ - { - name = "memchr"; - packageId = "memchr"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "serde"; - packageId = "serde"; - optional = true; - usesDefaultFeatures = false; - features = [ "derive" ]; - } - ]; - features = { - "alloc" = [ "serde?/alloc" ]; - "default" = [ "std" ]; - "memchr" = [ "dep:memchr" ]; - "serde" = [ "dep:serde" ]; - "std" = [ "alloc" "memchr?/std" "serde?/std" ]; - }; - resolvedDefaultFeatures = [ "alloc" "default" "std" ]; - }; "is_terminal_polyfill" = rec { crateName = "is_terminal_polyfill"; version = "1.70.2"; @@ -4525,9 +4484,9 @@ rec { }; "jiff" = rec { crateName = "jiff"; - version = "0.2.24"; + version = "0.2.28"; edition = "2021"; - sha256 = "0g87al8yqp05m63dhqzi359xgsslc0grqz00nvfdyq8dcayms2zh"; + sha256 = "00lixngcc7amh2fcsxfr0z38j06lllhapz192biv1qj97q1x60s6"; authors = [ "Andrew Gallant " ]; @@ -4573,12 +4532,10 @@ rec { usesDefaultFeatures = false; } { - name = "windows-sys"; - packageId = "windows-sys 0.61.2"; + name = "windows-link"; + packageId = "windows-link"; optional = true; - usesDefaultFeatures = false; target = { target, features }: (target."windows" or false); - features = [ "Win32_Foundation" "Win32_System_Time" ]; } ]; devDependencies = [ @@ -4597,7 +4554,7 @@ rec { "static-tz" = [ "dep:jiff-static" ]; "std" = [ "alloc" "log?/std" "serde_core?/std" ]; "tz-fat" = [ "jiff-static?/tz-fat" ]; - "tz-system" = [ "std" "dep:windows-sys" ]; + "tz-system" = [ "std" "dep:windows-link" ]; "tzdb-bundle-always" = [ "dep:jiff-tzdb" "alloc" ]; "tzdb-bundle-platform" = [ "dep:jiff-tzdb-platform" "alloc" ]; "tzdb-concatenated" = [ "std" ]; @@ -4607,9 +4564,9 @@ rec { }; "jiff-static" = rec { crateName = "jiff-static"; - version = "0.2.24"; + version = "0.2.28"; edition = "2021"; - sha256 = "1mz6v0d1hd8wjgfzccgda5g9z01s1yxnyiizvahjw0pq1w1xw070"; + sha256 = "0irbhfh2f4i9w5l53jcmh6ssnhdd92wfy76978chgwnxilvk4bbq"; procMacro = true; libName = "jiff_static"; authors = [ @@ -4689,9 +4646,9 @@ rec { }; "js-sys" = rec { crateName = "js-sys"; - version = "0.3.97"; + version = "0.3.99"; edition = "2021"; - sha256 = "1kyaxgn1sm5am98jb48aj5j7r7s98kdrab41la5wzys5q2a0r151"; + sha256 = "04azrzsz91gr5s3z0ij36lz0kj9ry4lw3jz0mmbiwb251rsc8aql"; libName = "js_sys"; authors = [ "The wasm-bindgen Developers" @@ -4728,9 +4685,9 @@ rec { }; "json-patch" = rec { crateName = "json-patch"; - version = "4.1.0"; + version = "4.2.0"; edition = "2021"; - sha256 = "147yaxmv3i4s0bdna86rgwpmqh2507fn4ighfpplaiqkw8ay807k"; + sha256 = "0wkv896d0pzq56i2kkl0giqpv117fwvhbpgs8iz85805w66l68bl"; libName = "json_patch"; authors = [ "Ivan Dubrov " @@ -4740,6 +4697,11 @@ rec { name = "jsonptr"; packageId = "jsonptr"; } + { + name = "schemars"; + packageId = "schemars"; + optional = true; + } { name = "serde"; packageId = "serde"; @@ -4751,10 +4713,14 @@ rec { } { name = "thiserror"; - packageId = "thiserror 1.0.69"; + packageId = "thiserror 2.0.18"; } ]; devDependencies = [ + { + name = "schemars"; + packageId = "schemars"; + } { name = "serde_json"; packageId = "serde_json"; @@ -4766,7 +4732,7 @@ rec { "schemars" = [ "dep:schemars" ]; "utoipa" = [ "dep:utoipa" ]; }; - resolvedDefaultFeatures = [ "default" "diff" ]; + resolvedDefaultFeatures = [ "default" "diff" "schemars" ]; }; "jsonpath-rust" = rec { crateName = "jsonpath-rust"; @@ -4892,8 +4858,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "128e1afca7761d07058624091217b6d695fa790c"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "k8s_version"; @@ -4912,7 +4878,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } ]; features = { @@ -5560,10 +5526,10 @@ rec { }; "libgit2-sys" = rec { crateName = "libgit2-sys"; - version = "0.18.3+1.9.2"; + version = "0.18.5+1.9.4"; edition = "2021"; links = "git2"; - sha256 = "11rlbyihj3k35mnkxxz4yvsnlx33a4r9srl66c5vp08pp72arcy9"; + sha256 = "18lwqnhy7qxg4iw24s1a0n7aj7qbnryry1iy0w32k4f1xbk6lp80"; libName = "libgit2_sys"; libPath = "lib.rs"; authors = [ @@ -5621,10 +5587,10 @@ rec { }; "libz-sys" = rec { crateName = "libz-sys"; - version = "1.1.28"; + version = "1.1.29"; edition = "2018"; links = "z"; - sha256 = "08hyf9v85zifl3353xc7i5wr53v9b3scri856cmphl3gaxp24fpw"; + sha256 = "1n98kqya7a7a0cxf5n5z3g13rj7a1vqxynk2xc7bja1qfxbrdg45"; libName = "libz_sys"; authors = [ "Alex Crichton " @@ -5701,9 +5667,9 @@ rec { }; "log" = rec { crateName = "log"; - version = "0.4.29"; + version = "0.4.32"; edition = "2021"; - sha256 = "15q8j9c8g5zpkcw0hnd6cf2z7fxqnvsjh3rw5mv5q10r83i34l2y"; + sha256 = "0fmdg0cxig7i4fwf1sw7fmg4d1gdbfzniawcfpwydy1q7320fgwm"; authors = [ "The Rust Project Developers" ]; @@ -5757,9 +5723,9 @@ rec { }; "memchr" = rec { crateName = "memchr"; - version = "2.8.0"; + version = "2.8.1"; edition = "2021"; - sha256 = "0y9zzxcqxvdqg6wyag7vc3h0blhdn7hkq164bxyx2vph8zs5ijpq"; + sha256 = "1n448jx01h5z2xknj6x2dhxgr8s8fb717cf6vfqj5lmhkpj7m53b"; authors = [ "Andrew Gallant " "bluss" @@ -5820,9 +5786,9 @@ rec { }; "mio" = rec { crateName = "mio"; - version = "1.2.0"; + version = "1.2.1"; edition = "2021"; - sha256 = "1hanrh4fwsfkdqdaqfidz48zz1wdix23zwn3r2x78am0garfbdsh"; + sha256 = "1nkggmrlnjs93w8rja4lvjj4aml1xqahgimv1h0p7d373kvhmg82"; authors = [ "Carl Lerche " "Thomas de Zeeuw " @@ -5958,9 +5924,9 @@ rec { }; "num-conv" = rec { crateName = "num-conv"; - version = "0.2.1"; + version = "0.2.2"; edition = "2021"; - sha256 = "0rqrr29brafaa2za352pbmhkk556n7f8z9rrkgmjp1idvdl3fry6"; + sha256 = "0hg4f9bwmy7cwpxdkm165dmkfc8jhkkayci234jsmi5ssb33j5sj"; libName = "num_conv"; authors = [ "Jacob Pratt " @@ -6082,9 +6048,9 @@ rec { }; "openssl" = rec { crateName = "openssl"; - version = "0.10.78"; + version = "0.10.80"; edition = "2021"; - sha256 = "08lj1fvhpfcga3nxs40vnl4spxfrswljvncxqwyazniw85r4737k"; + sha256 = "0ryrcbdd7hq0ydvassn4cr02agii1l54yd6sali7chkci2ma4px4"; authors = [ "Steven Fackler " ]; @@ -6105,10 +6071,6 @@ rec { name = "libc"; packageId = "libc"; } - { - name = "once_cell"; - packageId = "once_cell"; - } { name = "openssl-macros"; packageId = "openssl-macros"; @@ -6165,10 +6127,10 @@ rec { }; "openssl-sys" = rec { crateName = "openssl-sys"; - version = "0.9.114"; + version = "0.9.116"; edition = "2021"; links = "openssl"; - sha256 = "1dhvfj1nvikl4gaq9zb9ka2g7r67n03pb3s3vg7w9z07rm2i5khk"; + sha256 = "1i0qcgsimh8qkfgrglmzz2kq3jk2d5575rz5jvqabka0f7f252pj"; build = "build/main.rs"; libName = "openssl_sys"; authors = [ @@ -6207,9 +6169,9 @@ rec { }; "opentelemetry" = rec { crateName = "opentelemetry"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "18629xsj4rsyiby9aj511q6wcw6s9m09gx3ymw1yjcvix1mcsjxq"; + sha256 = "10ln14d1jgc8rvw97mblc9blzcgpg1bimim4d170b7ia4mijq55h"; dependencies = [ { name = "futures-core"; @@ -6246,24 +6208,24 @@ rec { ]; features = { "default" = [ "trace" "metrics" "logs" "internal-logs" "futures" ]; + "experimental_metrics_bound_instruments" = [ "metrics" ]; "futures" = [ "futures-core" "futures-sink" "pin-project-lite" ]; "futures-core" = [ "dep:futures-core" ]; "futures-sink" = [ "dep:futures-sink" ]; "internal-logs" = [ "tracing" ]; "pin-project-lite" = [ "dep:pin-project-lite" ]; - "spec_unstable_logs_enabled" = [ "logs" ]; "testing" = [ "trace" ]; "thiserror" = [ "dep:thiserror" ]; "trace" = [ "futures" "thiserror" ]; "tracing" = [ "dep:tracing" ]; }; - resolvedDefaultFeatures = [ "default" "futures" "futures-core" "futures-sink" "internal-logs" "logs" "metrics" "pin-project-lite" "spec_unstable_logs_enabled" "thiserror" "trace" "tracing" ]; + resolvedDefaultFeatures = [ "default" "futures" "futures-core" "futures-sink" "internal-logs" "logs" "metrics" "pin-project-lite" "thiserror" "trace" "tracing" ]; }; "opentelemetry-appender-tracing" = rec { crateName = "opentelemetry-appender-tracing"; - version = "0.31.1"; + version = "0.32.0"; edition = "2021"; - sha256 = "1hnwizzgfhpjfnvml638yy846py8hf2gl1n3p1igbk1srb2ilspg"; + sha256 = "0dyq4myan64sl8wly02jx0gb3jjz7575mn3w8rpphz0xvkq8001c"; libName = "opentelemetry_appender_tracing"; dependencies = [ { @@ -6306,18 +6268,15 @@ rec { ]; features = { "experimental_metadata_attributes" = [ "dep:tracing-log" ]; - "experimental_use_tracing_span_context" = [ "tracing-opentelemetry" ]; "log" = [ "dep:log" ]; - "spec_unstable_logs_enabled" = [ "opentelemetry/spec_unstable_logs_enabled" ]; - "tracing-opentelemetry" = [ "dep:tracing-opentelemetry" ]; }; resolvedDefaultFeatures = [ "default" ]; }; "opentelemetry-http" = rec { crateName = "opentelemetry-http"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "0pc5nw1ds8v8w0nvyall39m92v8m1xl1p3vwvxk6nkhrffdd19np"; + sha256 = "0ca3drvm4fx5nskl7yn42dimy3bg35ppzc85y1p27pz215fh30sn"; libName = "opentelemetry_http"; dependencies = [ { @@ -6353,16 +6312,16 @@ rec { "internal-logs" = [ "opentelemetry/internal-logs" ]; "reqwest" = [ "dep:reqwest" ]; "reqwest-blocking" = [ "dep:reqwest" "reqwest/blocking" ]; - "reqwest-rustls" = [ "dep:reqwest" "reqwest/rustls-tls-native-roots" ]; - "reqwest-rustls-webpki-roots" = [ "dep:reqwest" "reqwest/rustls-tls-webpki-roots" ]; + "reqwest-rustls" = [ "dep:reqwest" "reqwest/default-tls" ]; + "reqwest-rustls-webpki-roots" = [ "dep:reqwest" "reqwest/default-tls" "reqwest/webpki-roots" ]; }; - resolvedDefaultFeatures = [ "internal-logs" "reqwest" "reqwest-blocking" ]; + resolvedDefaultFeatures = [ "reqwest" "reqwest-blocking" ]; }; "opentelemetry-otlp" = rec { crateName = "opentelemetry-otlp"; - version = "0.31.1"; + version = "0.32.0"; edition = "2021"; - sha256 = "07zp0b62b9dajnvvcd6j2ppw5zg7wp4ixka9z6fr3bxrrdmcss8z"; + sha256 = "0d9cys2flpidfxbr6h1103hjc633cax47ihnqgbj0xnicscr4rlr"; libName = "opentelemetry_otlp"; dependencies = [ { @@ -6423,10 +6382,9 @@ rec { usesDefaultFeatures = false; } { - name = "tracing"; - packageId = "tracing"; + name = "tonic-types"; + packageId = "tonic-types"; optional = true; - usesDefaultFeatures = false; } ]; devDependencies = [ @@ -6451,16 +6409,19 @@ rec { ]; features = { "default" = [ "http-proto" "reqwest-blocking-client" "trace" "metrics" "logs" "internal-logs" ]; + "experimental-grpc-retry" = [ "grpc-tonic" "opentelemetry_sdk/experimental_async_runtime" "opentelemetry_sdk/rt-tokio" ]; + "experimental-http-retry" = [ "opentelemetry_sdk/experimental_async_runtime" "opentelemetry_sdk/rt-tokio" "tokio" "httpdate" ]; "flate2" = [ "dep:flate2" ]; - "grpc-tonic" = [ "tonic" "prost" "http" "tokio" "opentelemetry-proto/gen-tonic" ]; + "grpc-tonic" = [ "tonic" "tonic-types" "prost" "http" "tokio" "opentelemetry-proto/gen-tonic" ]; "gzip-http" = [ "flate2" ]; "gzip-tonic" = [ "tonic/gzip" ]; "http" = [ "dep:http" ]; "http-json" = [ "serde_json" "prost" "opentelemetry-http" "opentelemetry-proto/gen-tonic-messages" "opentelemetry-proto/with-serde" "http" "trace" "metrics" ]; "http-proto" = [ "prost" "opentelemetry-http" "opentelemetry-proto/gen-tonic-messages" "http" "trace" "metrics" ]; + "httpdate" = [ "dep:httpdate" ]; "hyper-client" = [ "opentelemetry-http/hyper" ]; "integration-testing" = [ "tonic" "prost" "tokio/full" "trace" "logs" ]; - "internal-logs" = [ "tracing" "opentelemetry_sdk/internal-logs" "opentelemetry-http/internal-logs" ]; + "internal-logs" = [ "opentelemetry_sdk/internal-logs" "opentelemetry/internal-logs" ]; "logs" = [ "opentelemetry/logs" "opentelemetry_sdk/logs" "opentelemetry-proto/logs" ]; "metrics" = [ "opentelemetry/metrics" "opentelemetry_sdk/metrics" "opentelemetry-proto/metrics" ]; "opentelemetry-http" = [ "dep:opentelemetry-http" ]; @@ -6473,27 +6434,27 @@ rec { "serde" = [ "dep:serde" ]; "serde_json" = [ "dep:serde_json" ]; "serialize" = [ "serde" "serde_json" ]; - "tls" = [ "tonic/tls-ring" ]; + "tls" = [ "tls-ring" ]; "tls-aws-lc" = [ "tonic/tls-aws-lc" ]; "tls-provider-agnostic" = [ "tonic/_tls-any" ]; "tls-ring" = [ "tonic/tls-ring" ]; - "tls-roots" = [ "tls" "tonic/tls-native-roots" ]; - "tls-webpki-roots" = [ "tls" "tonic/tls-webpki-roots" ]; + "tls-roots" = [ "tonic/tls-native-roots" ]; + "tls-webpki-roots" = [ "tonic/tls-webpki-roots" ]; "tokio" = [ "dep:tokio" ]; "tonic" = [ "dep:tonic" ]; + "tonic-types" = [ "dep:tonic-types" ]; "trace" = [ "opentelemetry/trace" "opentelemetry_sdk/trace" "opentelemetry-proto/trace" ]; - "tracing" = [ "dep:tracing" ]; "zstd" = [ "dep:zstd" ]; "zstd-http" = [ "zstd" ]; "zstd-tonic" = [ "tonic/zstd" ]; }; - resolvedDefaultFeatures = [ "default" "grpc-tonic" "gzip-tonic" "http" "http-proto" "internal-logs" "logs" "metrics" "opentelemetry-http" "prost" "reqwest" "reqwest-blocking-client" "tokio" "tonic" "trace" "tracing" ]; + resolvedDefaultFeatures = [ "default" "grpc-tonic" "gzip-tonic" "http" "http-proto" "internal-logs" "logs" "metrics" "opentelemetry-http" "prost" "reqwest" "reqwest-blocking-client" "tokio" "tonic" "tonic-types" "trace" ]; }; "opentelemetry-proto" = rec { crateName = "opentelemetry-proto"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "03xkjsjrsm7zkkx5gascqd9bg2z20wymm06l16cyxsp5dpq5s5x7"; + sha256 = "0f5ny4rpnpq6q5q34b8k2q548rf31rpbxkwjqjwzfqxg3yx5imjn"; libName = "opentelemetry_proto"; dependencies = [ { @@ -6537,30 +6498,29 @@ rec { "const-hex" = [ "dep:const-hex" ]; "default" = [ "full" ]; "full" = [ "gen-tonic" "trace" "logs" "metrics" "zpages" "with-serde" "internal-logs" ]; - "gen-tonic" = [ "gen-tonic-messages" "tonic/channel" ]; - "gen-tonic-messages" = [ "tonic" "tonic-prost" "prost" ]; + "gen-tonic" = [ "gen-tonic-messages" "tonic" "tonic-prost" "tonic/channel" ]; + "gen-tonic-messages" = [ "prost" ]; "internal-logs" = [ "opentelemetry/internal-logs" ]; "logs" = [ "opentelemetry/logs" "opentelemetry_sdk/logs" ]; "metrics" = [ "opentelemetry/metrics" "opentelemetry_sdk/metrics" ]; "prost" = [ "dep:prost" ]; "schemars" = [ "dep:schemars" ]; "serde" = [ "dep:serde" ]; - "serde_json" = [ "dep:serde_json" ]; "testing" = [ "opentelemetry/testing" ]; "tonic" = [ "dep:tonic" ]; "tonic-prost" = [ "dep:tonic-prost" ]; "trace" = [ "opentelemetry/trace" "opentelemetry_sdk/trace" ]; "with-schemars" = [ "schemars" ]; - "with-serde" = [ "serde" "const-hex" "base64" "serde_json" ]; + "with-serde" = [ "serde" "const-hex" "base64" ]; "zpages" = [ "trace" ]; }; resolvedDefaultFeatures = [ "gen-tonic" "gen-tonic-messages" "logs" "metrics" "prost" "tonic" "tonic-prost" "trace" ]; }; "opentelemetry-semantic-conventions" = rec { crateName = "opentelemetry-semantic-conventions"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "0in8plv2l2ar7anzi7lrbll0fjfvaymkg5vc5bnvibs1w3gjjbp6"; + sha256 = "0s1x4h1cgmhkxb7i5g02la2vhkf4lg5g26cgn2s2gd1p0j5gk8kc"; libName = "opentelemetry_semantic_conventions"; features = { }; @@ -6568,9 +6528,9 @@ rec { }; "opentelemetry_sdk" = rec { crateName = "opentelemetry_sdk"; - version = "0.31.0"; + version = "0.32.1"; edition = "2021"; - sha256 = "1gbjsggdxfpjbanjvaxa3nq32vfa37i3v13dvx4gsxhrk7sy8jp1"; + sha256 = "1ycl11syranrinhgn4c2hlzhyzyvpa06ryxq5mxgzmf4387ghncv"; dependencies = [ { name = "futures-channel"; @@ -6596,6 +6556,13 @@ rec { packageId = "percent-encoding"; optional = true; } + { + name = "portable-atomic"; + packageId = "portable-atomic"; + usesDefaultFeatures = false; + target = { target, features }: (!("64" == target."has_atomic" or null)); + features = [ "fallback" ]; + } { name = "rand"; packageId = "rand 0.9.4"; @@ -6620,10 +6587,18 @@ rec { optional = true; } ]; + devDependencies = [ + { + name = "tokio"; + packageId = "tokio"; + usesDefaultFeatures = false; + features = [ "macros" "rt-multi-thread" ]; + } + ]; features = { "default" = [ "trace" "metrics" "logs" "internal-logs" ]; "experimental_logs_batch_log_processor_with_async_runtime" = [ "logs" "experimental_async_runtime" ]; - "experimental_logs_concurrent_log_processor" = [ "logs" ]; + "experimental_metrics_bound_instruments" = [ "metrics" "opentelemetry/experimental_metrics_bound_instruments" ]; "experimental_metrics_custom_reader" = [ "metrics" ]; "experimental_metrics_disable_name_validation" = [ "metrics" ]; "experimental_metrics_periodicreader_with_async_runtime" = [ "metrics" "experimental_async_runtime" ]; @@ -6640,15 +6615,14 @@ rec { "rt-tokio-current-thread" = [ "tokio/rt" "tokio/time" "tokio-stream" "experimental_async_runtime" ]; "serde" = [ "dep:serde" ]; "serde_json" = [ "dep:serde_json" ]; - "spec_unstable_logs_enabled" = [ "logs" "opentelemetry/spec_unstable_logs_enabled" ]; "spec_unstable_metrics_views" = [ "metrics" ]; - "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "rt-tokio" "rt-tokio-current-thread" "tokio/macros" "tokio/rt-multi-thread" ]; + "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "tokio/sync" ]; "tokio" = [ "dep:tokio" ]; "tokio-stream" = [ "dep:tokio-stream" ]; "trace" = [ "opentelemetry/trace" "rand" "percent-encoding" ]; "url" = [ "dep:url" ]; }; - resolvedDefaultFeatures = [ "default" "experimental_async_runtime" "internal-logs" "logs" "metrics" "percent-encoding" "rand" "rt-tokio" "spec_unstable_logs_enabled" "tokio" "tokio-stream" "trace" ]; + resolvedDefaultFeatures = [ "default" "experimental_async_runtime" "internal-logs" "logs" "metrics" "percent-encoding" "rand" "rt-tokio" "tokio" "tokio-stream" "trace" ]; }; "ordered-float" = rec { crateName = "ordered-float"; @@ -7022,9 +6996,9 @@ rec { }; "pin-project" = rec { crateName = "pin-project"; - version = "1.1.11"; + version = "1.1.13"; edition = "2021"; - sha256 = "05zm3y3bl83ypsr6favxvny2kys4i19jiz1y18ylrbxwsiz9qx7i"; + sha256 = "09091qp946lpmjz4yp0xil1r5v4hgc91fi19dg5csayhdqrv4ri4"; libName = "pin_project"; dependencies = [ { @@ -7036,9 +7010,9 @@ rec { }; "pin-project-internal" = rec { crateName = "pin-project-internal"; - version = "1.1.11"; + version = "1.1.13"; edition = "2021"; - sha256 = "1ik4mpb92da75inmjvxf2qm61vrnwml3x24wddvrjlqh1z9hxcnr"; + sha256 = "12rzlh07i1sdgrvzj6wgkka5bjqyvbfsl8knq6qi7g16m7q9aqy9"; procMacro = true; libName = "pin_project_internal"; dependencies = [ @@ -7157,7 +7131,7 @@ rec { "default" = [ "fallback" ]; "serde" = [ "dep:serde" ]; }; - resolvedDefaultFeatures = [ "require-cas" ]; + resolvedDefaultFeatures = [ "fallback" "require-cas" ]; }; "portable-atomic-util" = rec { crateName = "portable-atomic-util"; @@ -7425,6 +7399,34 @@ rec { ]; }; + "prost-types" = rec { + crateName = "prost-types"; + version = "0.14.3"; + edition = "2021"; + sha256 = "1mrxrciryfgi6a0vmrgyj3g27r9hdhlgwkq71cgv3icbvg5w94c9"; + libName = "prost_types"; + authors = [ + "Dan Burkert " + "Lucio Franco " + "Casper Meijn " + "Tokio Contributors " + ]; + dependencies = [ + { + name = "prost"; + packageId = "prost"; + usesDefaultFeatures = false; + features = [ "derive" ]; + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + "chrono" = [ "dep:chrono" ]; + "default" = [ "std" ]; + "std" = [ "prost/std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "quote" = rec { crateName = "quote"; version = "1.0.45"; @@ -7854,9 +7856,9 @@ rec { }; "reqwest" = rec { crateName = "reqwest"; - version = "0.12.28"; + version = "0.13.4"; edition = "2021"; - sha256 = "0iqidijghgqbzl3bjg5hb4zmigwa4r612bgi0yiq0c90b6jkrpgd"; + sha256 = "1hy1plns9krbh3h1dy2sdjygsfkdcnxm6pbxdi0ya9b5vq8mi711"; authors = [ "Sean McArthur " ]; @@ -7873,7 +7875,7 @@ rec { name = "futures-channel"; packageId = "futures-channel"; optional = true; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "futures-core"; @@ -7893,62 +7895,44 @@ rec { { name = "http-body"; packageId = "http-body"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "http-body-util"; packageId = "http-body-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "hyper"; packageId = "hyper"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "client" ]; } { name = "hyper-util"; packageId = "hyper-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "client" "client-legacy" "client-proxy" "tokio" ]; } { name = "js-sys"; packageId = "js-sys"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "log"; packageId = "log"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "percent-encoding"; packageId = "percent-encoding"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "pin-project-lite"; packageId = "pin-project-lite"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - } - { - name = "serde"; - packageId = "serde"; - } - { - name = "serde_json"; - packageId = "serde_json"; - optional = true; - } - { - name = "serde_json"; - packageId = "serde_json"; - target = { target, features }: ("wasm32" == target."arch" or null); - } - { - name = "serde_urlencoded"; - packageId = "serde_urlencoded"; + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "sync_wrapper"; @@ -7959,27 +7943,27 @@ rec { name = "tokio"; packageId = "tokio"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "net" "time" ]; } { name = "tower"; packageId = "tower"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "retry" "timeout" "util" ]; } { name = "tower-http"; packageId = "tower-http"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "follow-redirect" ]; } { name = "tower-service"; packageId = "tower-service"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "url"; @@ -7988,17 +7972,17 @@ rec { { name = "wasm-bindgen"; packageId = "wasm-bindgen"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "wasm-bindgen-futures"; packageId = "wasm-bindgen-futures"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "web-sys"; packageId = "web-sys"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); features = [ "AbortController" "AbortSignal" "Headers" "Request" "RequestInit" "RequestMode" "Response" "Window" "FormData" "Blob" "BlobPropertyBag" "ServiceWorkerGlobalScope" "RequestCredentials" "File" "ReadableStream" "RequestCache" ]; } ]; @@ -8007,33 +7991,27 @@ rec { name = "futures-util"; packageId = "futures-util"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "std" "alloc" ]; } { name = "hyper"; packageId = "hyper"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "http2" "client" "server" ]; } { name = "hyper-util"; packageId = "hyper-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "http2" "client" "client-legacy" "server-auto" "server-graceful" "tokio" ]; } - { - name = "serde"; - packageId = "serde"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - features = [ "derive" ]; - } { name = "tokio"; packageId = "tokio"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "macros" "rt-multi-thread" ]; } { @@ -8045,40 +8023,37 @@ rec { { name = "wasm-bindgen"; packageId = "wasm-bindgen"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); features = [ "serde-serialize" ]; } ]; features = { + "__native-tls" = [ "dep:hyper-tls" "dep:native-tls-crate" "__tls" "dep:tokio-native-tls" ]; + "__native-tls-alpn" = [ "native-tls-crate?/alpn" "hyper-tls?/alpn" ]; "__rustls" = [ "dep:hyper-rustls" "dep:tokio-rustls" "dep:rustls" "__tls" ]; - "__rustls-ring" = [ "hyper-rustls?/ring" "tokio-rustls?/ring" "rustls?/ring" "quinn?/ring" ]; + "__rustls-aws-lc-rs" = [ "hyper-rustls?/aws-lc-rs" "tokio-rustls?/aws-lc-rs" "rustls?/aws-lc-rs" "quinn?/rustls-aws-lc-rs" ]; "__tls" = [ "dep:rustls-pki-types" "tokio/io-util" ]; "blocking" = [ "dep:futures-channel" "futures-channel?/sink" "dep:futures-util" "futures-util?/io" "futures-util?/sink" "tokio/sync" ]; "brotli" = [ "tower-http/decompression-br" ]; "charset" = [ "dep:encoding_rs" "dep:mime" ]; "cookies" = [ "dep:cookie_crate" "dep:cookie_store" ]; "default" = [ "default-tls" "charset" "http2" "system-proxy" ]; - "default-tls" = [ "dep:hyper-tls" "dep:native-tls-crate" "__tls" "dep:tokio-native-tls" ]; + "default-tls" = [ "rustls" ]; "deflate" = [ "tower-http/decompression-deflate" ]; + "form" = [ "dep:serde" "dep:serde_urlencoded" ]; "gzip" = [ "tower-http/decompression-gzip" ]; - "h2" = [ "dep:h2" ]; "hickory-dns" = [ "dep:hickory-resolver" "dep:once_cell" ]; - "http2" = [ "h2" "hyper/http2" "hyper-util/http2" "hyper-rustls?/http2" ]; - "http3" = [ "rustls-tls-manual-roots" "dep:h3" "dep:h3-quinn" "dep:quinn" "tokio/macros" ]; - "json" = [ "dep:serde_json" ]; - "macos-system-configuration" = [ "system-proxy" ]; + "http2" = [ "dep:h2" "hyper/http2" "hyper-util/http2" "hyper-rustls?/http2" ]; + "http3" = [ "rustls" "dep:h3" "dep:h3-quinn" "dep:quinn" "tokio/macros" ]; + "json" = [ "dep:serde" "dep:serde_json" ]; "multipart" = [ "dep:mime_guess" "dep:futures-util" ]; - "native-tls" = [ "default-tls" ]; - "native-tls-alpn" = [ "native-tls" "native-tls-crate?/alpn" "hyper-tls?/alpn" ]; - "native-tls-vendored" = [ "native-tls" "native-tls-crate?/vendored" ]; - "rustls-tls" = [ "rustls-tls-webpki-roots" ]; - "rustls-tls-manual-roots" = [ "rustls-tls-manual-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-manual-roots-no-provider" = [ "__rustls" ]; - "rustls-tls-native-roots" = [ "rustls-tls-native-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-native-roots-no-provider" = [ "dep:rustls-native-certs" "hyper-rustls?/native-tokio" "__rustls" ]; - "rustls-tls-no-provider" = [ "rustls-tls-manual-roots-no-provider" ]; - "rustls-tls-webpki-roots" = [ "rustls-tls-webpki-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-webpki-roots-no-provider" = [ "dep:webpki-roots" "hyper-rustls?/webpki-tokio" "__rustls" ]; + "native-tls" = [ "__native-tls" "__native-tls-alpn" ]; + "native-tls-no-alpn" = [ "__native-tls" ]; + "native-tls-vendored" = [ "__native-tls" "native-tls-crate?/vendored" "__native-tls-alpn" ]; + "native-tls-vendored-no-alpn" = [ "__native-tls" "native-tls-crate?/vendored" ]; + "query" = [ "dep:serde" "dep:serde_urlencoded" ]; + "rustls" = [ "__rustls-aws-lc-rs" "dep:rustls-platform-verifier" "__rustls" ]; + "rustls-no-provider" = [ "dep:rustls-platform-verifier" "__rustls" ]; "stream" = [ "tokio/fs" "dep:futures-util" "dep:tokio-util" "dep:wasm-streams" ]; "system-proxy" = [ "hyper-util/client-proxy-system" ]; "zstd" = [ "tower-http/decompression-zstd" ]; @@ -8390,9 +8365,9 @@ rec { }; "rustls" = rec { crateName = "rustls"; - version = "0.23.39"; + version = "0.23.40"; edition = "2021"; - sha256 = "03p6fkdwbdpp93dfidc4nzgmalwp3gxnv0rk421a5k3pn2612b3w"; + sha256 = "12qnv3ag4wrw7aj8jng74kgrilpjm2b1rfcjaac8h691frccv1pg"; dependencies = [ { name = "log"; @@ -8459,9 +8434,9 @@ rec { }; "rustls-native-certs" = rec { crateName = "rustls-native-certs"; - version = "0.8.3"; + version = "0.8.4"; edition = "2021"; - sha256 = "0qrajg2n90bcr3bcq6j95gjm7a9lirfkkdmjj32419dyyzan0931"; + sha256 = "0kgazl8zc1sv63qg179bz96ilzh56lzfa5k92ji7d265f4kibdfs"; libName = "rustls_native_certs"; dependencies = [ { @@ -9028,9 +9003,9 @@ rec { }; "serde_json" = rec { crateName = "serde_json"; - version = "1.0.149"; + version = "1.0.150"; edition = "2021"; - sha256 = "11jdx4vilzrjjd1dpgy67x5lgzr0laplz30dhv75lnf5ffa07z43"; + sha256 = "1ffgfhy9kndjnrz8lmy95pr758p2zk8dxv6yi99x0vkkni24w0g8"; authors = [ "Erick Tryzelaar " "David Tolnay " @@ -9271,9 +9246,9 @@ rec { }; "shlex" = rec { crateName = "shlex"; - version = "1.3.0"; - edition = "2015"; - sha256 = "0r1y6bv26c1scpxvhg2cabimrmwgbp4p3wy6syj9n0c4s3q2znhg"; + version = "2.0.1"; + edition = "2018"; + sha256 = "1fjsll1cd7d2bcpdij9kd6w62rpbc7qqzvydvs021vsmr1cxvypq"; authors = [ "comex " "Fenhl " @@ -9452,29 +9427,25 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "rust_1_61" "rust_1_65" "std" ]; }; - "snafu 0.9.0" = rec { + "snafu 0.9.1" = rec { crateName = "snafu"; - version = "0.9.0"; + version = "0.9.1"; edition = "2018"; - sha256 = "1ii9r99x5qcn754m624yzgb9hzvkqkrcygf0aqh0pyb9dbnvrm6i"; + sha256 = "08k5yfydxdlshivfhrdq9km8qn02r93q28gkyvazbqz2icr1586i"; authors = [ "Jake Goulding " ]; dependencies = [ { name = "snafu-derive"; - packageId = "snafu-derive 0.9.0"; + packageId = "snafu-derive 0.9.1"; } ]; features = { - "backtrace" = [ "dep:backtrace" ]; - "backtraces-impl-backtrace-crate" = [ "backtrace" ]; + "backtraces-impl-backtrace-crate" = [ "dep:backtrace" ]; "default" = [ "std" "rust_1_81" ]; - "futures" = [ "futures-core-crate" "pin-project" ]; - "futures-core-crate" = [ "dep:futures-core-crate" ]; - "futures-crate" = [ "dep:futures-crate" ]; - "internal-dev-dependencies" = [ "futures-crate" ]; - "pin-project" = [ "dep:pin-project" ]; + "futures" = [ "dep:futures-core" "dep:pin-project" ]; + "internal-dev-dependencies" = [ "dep:futures" ]; "std" = [ "alloc" ]; "unstable-provider-api" = [ "snafu-derive/unstable-provider-api" ]; }; @@ -9542,11 +9513,11 @@ rec { }; resolvedDefaultFeatures = [ "rust_1_61" ]; }; - "snafu-derive 0.9.0" = rec { + "snafu-derive 0.9.1" = rec { crateName = "snafu-derive"; - version = "0.9.0"; + version = "0.9.1"; edition = "2018"; - sha256 = "0h0x61kyj4fvilcr2nj02l85shw1ika64vq9brf2gyna662ln9al"; + sha256 = "1nkfi7bis72pz3w7vb64m79w49qsv20sbf19jkd471vbhr83q42z"; procMacro = true; libName = "snafu_derive"; authors = [ @@ -9572,7 +9543,7 @@ rec { name = "syn"; packageId = "syn 2.0.117"; usesDefaultFeatures = false; - features = [ "clone-impls" "derive" "full" "parsing" "printing" "proc-macro" ]; + features = [ "clone-impls" "derive" "full" "parsing" "printing" "proc-macro" "visit-mut" ]; } ]; features = { @@ -9580,9 +9551,9 @@ rec { }; "socket2" = rec { crateName = "socket2"; - version = "0.6.3"; + version = "0.6.4"; edition = "2021"; - sha256 = "0gkjjcyn69hqhhlh5kl8byk5m0d7hyrp2aqwzbs3d33q208nwxis"; + sha256 = "0ldyp5rhba15spwxj1n94xh7sjks1398c3vwpwkxkd1087nwzlaj"; authors = [ "Alex Crichton " "Thomas de Zeeuw " @@ -9680,8 +9651,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "128e1afca7761d07058624091217b6d695fa790c"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_certs"; @@ -9740,7 +9711,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-shared"; @@ -9818,6 +9789,10 @@ rec { name = "indoc"; packageId = "indoc"; } + { + name = "java-properties"; + packageId = "java-properties"; + } { name = "openssl"; packageId = "openssl"; @@ -9826,10 +9801,6 @@ rec { name = "pin-project"; packageId = "pin-project"; } - { - name = "product-config"; - packageId = "product-config"; - } { name = "semver"; packageId = "semver"; @@ -9845,7 +9816,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-operator"; @@ -9888,12 +9859,12 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.111.0"; + version = "0.111.1"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "128e1afca7761d07058624091217b6d695fa790c"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_operator"; @@ -9951,6 +9922,7 @@ rec { { name = "json-patch"; packageId = "json-patch"; + features = [ "schemars" ]; } { name = "k8s-openapi"; @@ -10000,7 +9972,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-operator-derive"; @@ -10054,12 +10026,17 @@ rec { packageId = "url"; features = [ "serde" ]; } + { + name = "uuid"; + packageId = "uuid"; + } ]; features = { "certs" = [ "dep:stackable-certs" ]; + "client-feature-gates" = [ "dep:winnow" ]; "crds" = [ "dep:stackable-versioned" ]; "default" = [ "crds" ]; - "full" = [ "crds" "certs" "time" "webhook" "kube-ws" ]; + "full" = [ "client-feature-gates" "crds" "certs" "time" "webhook" "kube-ws" ]; "kube-ws" = [ "kube/ws" ]; "time" = [ "stackable-shared/time" ]; "webhook" = [ "dep:stackable-webhook" ]; @@ -10072,8 +10049,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "128e1afca7761d07058624091217b6d695fa790c"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; procMacro = true; @@ -10103,12 +10080,12 @@ rec { }; "stackable-shared" = rec { crateName = "stackable-shared"; - version = "0.1.0"; + version = "0.1.1"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "128e1afca7761d07058624091217b6d695fa790c"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_shared"; @@ -10153,7 +10130,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "strum"; @@ -10184,12 +10161,12 @@ rec { }; "stackable-telemetry" = rec { crateName = "stackable-telemetry"; - version = "0.6.3"; + version = "0.6.4"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "128e1afca7761d07058624091217b6d695fa790c"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_telemetry"; @@ -10233,7 +10210,7 @@ rec { { name = "opentelemetry_sdk"; packageId = "opentelemetry_sdk"; - features = [ "rt-tokio" "logs" "rt-tokio" "spec_unstable_logs_enabled" ]; + features = [ "rt-tokio" "logs" "rt-tokio" ]; } { name = "pin-project"; @@ -10241,7 +10218,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "strum"; @@ -10298,8 +10275,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "128e1afca7761d07058624091217b6d695fa790c"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_versioned"; @@ -10333,7 +10310,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-versioned-macros"; @@ -10348,8 +10325,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "128e1afca7761d07058624091217b6d695fa790c"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; procMacro = true; @@ -10416,8 +10393,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "128e1afca7761d07058624091217b6d695fa790c"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_webhook"; @@ -10490,7 +10467,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-certs"; @@ -11062,9 +11039,9 @@ rec { }; "tokio" = rec { crateName = "tokio"; - version = "1.52.1"; + version = "1.52.3"; edition = "2021"; - sha256 = "1imw1dkkv38p66i33m5hsyk3d6prsbyrayjvqhndjvz89ybywzdn"; + sha256 = "1zpzazypkg61sw91na1m85x5s4rsjym335fwwhwm1hcs70dz1iwg"; authors = [ "Tokio Contributors " ]; @@ -11374,9 +11351,9 @@ rec { }; "toml_edit" = rec { crateName = "toml_edit"; - version = "0.25.11+spec-1.1.0"; + version = "0.25.12+spec-1.1.0"; edition = "2024"; - sha256 = "0awzffbkx33v9x4h19b5mfrwp3sn4ifr16y58sbk6j6l5v9c8n8b"; + sha256 = "1mx5paq837rjw7w51zprrjynk1vaig9yzxfqz9ac79jmd7f3w5fj"; dependencies = [ { name = "indexmap"; @@ -11429,9 +11406,9 @@ rec { }; "tonic" = rec { crateName = "tonic"; - version = "0.14.5"; - edition = "2021"; - sha256 = "1v4k7aa28m7722gz9qak2jiy7lis1ycm4fdmq63iip4m0qdcdizy"; + version = "0.14.6"; + edition = "2024"; + sha256 = "1vs5ci6z6b9xhfsnx4s8qx6bqi1zzcrxncjp71147a0gqwc5aamc"; authors = [ "Lucio Franco " ]; @@ -11558,9 +11535,9 @@ rec { }; "tonic-prost" = rec { crateName = "tonic-prost"; - version = "0.14.5"; - edition = "2021"; - sha256 = "02fkg2bv87q0yds2wz3w0s7i1x6qcgbrl00dy6ipajdapfh7clx5"; + version = "0.14.6"; + edition = "2024"; + sha256 = "184y40nf0iyzc5rg32ivgd88snv68sqy1kchynn55r1vhml9z12h"; libName = "tonic_prost"; authors = [ "Lucio Franco " @@ -11581,6 +11558,33 @@ rec { } ]; + }; + "tonic-types" = rec { + crateName = "tonic-types"; + version = "0.14.6"; + edition = "2024"; + sha256 = "1s286gg71pjajny8xar0azq1w9lgz1ks3jm3pccxb0qz0q11pavk"; + libName = "tonic_types"; + authors = [ + "Lucio Franco " + "Rafael Lemos " + ]; + dependencies = [ + { + name = "prost"; + packageId = "prost"; + } + { + name = "prost-types"; + packageId = "prost-types"; + } + { + name = "tonic"; + packageId = "tonic"; + usesDefaultFeatures = false; + } + ]; + }; "tower" = rec { crateName = "tower"; @@ -11702,9 +11706,9 @@ rec { }; "tower-http" = rec { crateName = "tower-http"; - version = "0.6.8"; + version = "0.6.11"; edition = "2018"; - sha256 = "1y514jwzbyrmrkbaajpwmss4rg0mak82k16d6588w9ncaffmbrnl"; + sha256 = "0h08wjgs3hwnq11iwwzlmnabn1h4cl0fzd48svaccvqffkiggz2c"; libName = "tower_http"; authors = [ "Tower Maintainers " @@ -11738,11 +11742,6 @@ rec { packageId = "http-body"; optional = true; } - { - name = "iri-string"; - packageId = "iri-string"; - optional = true; - } { name = "mime"; packageId = "mime"; @@ -11772,6 +11771,11 @@ rec { optional = true; usesDefaultFeatures = false; } + { + name = "url"; + packageId = "url"; + optional = true; + } ]; devDependencies = [ { @@ -11793,35 +11797,33 @@ rec { } ]; features = { - "async-compression" = [ "dep:async-compression" ]; "auth" = [ "base64" "validate-request" ]; "base64" = [ "dep:base64" ]; "catch-panic" = [ "tracing" "futures-util/std" "dep:http-body" "dep:http-body-util" ]; - "compression-br" = [ "async-compression/brotli" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; - "compression-deflate" = [ "async-compression/zlib" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; + "compression-br" = [ "dep:async-compression" "async-compression?/brotli" "futures-core" "dep:http-body" "tokio-util" "dep:tokio" ]; + "compression-deflate" = [ "dep:async-compression" "async-compression?/zlib" "futures-core" "dep:http-body" "tokio-util" "dep:tokio" ]; "compression-full" = [ "compression-br" "compression-deflate" "compression-gzip" "compression-zstd" ]; - "compression-gzip" = [ "async-compression/gzip" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; - "compression-zstd" = [ "async-compression/zstd" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; - "decompression-br" = [ "async-compression/brotli" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; - "decompression-deflate" = [ "async-compression/zlib" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; + "compression-gzip" = [ "dep:async-compression" "async-compression?/gzip" "futures-core" "dep:http-body" "tokio-util" "dep:tokio" ]; + "compression-zstd" = [ "dep:async-compression" "async-compression?/zstd" "futures-core" "dep:http-body" "tokio-util" "dep:tokio" ]; + "decompression-br" = [ "dep:async-compression" "async-compression?/brotli" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "dep:tokio" ]; + "decompression-deflate" = [ "dep:async-compression" "async-compression?/zlib" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "dep:tokio" ]; "decompression-full" = [ "decompression-br" "decompression-deflate" "decompression-gzip" "decompression-zstd" ]; - "decompression-gzip" = [ "async-compression/gzip" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; - "decompression-zstd" = [ "async-compression/zstd" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; - "follow-redirect" = [ "futures-util" "dep:http-body" "iri-string" "tower/util" ]; - "fs" = [ "futures-core" "futures-util" "dep:http-body" "dep:http-body-util" "tokio/fs" "tokio-util/io" "tokio/io-util" "dep:http-range-header" "mime_guess" "mime" "percent-encoding" "httpdate" "set-status" "futures-util/alloc" "tracing" ]; - "full" = [ "add-extension" "auth" "catch-panic" "compression-full" "cors" "decompression-full" "follow-redirect" "fs" "limit" "map-request-body" "map-response-body" "metrics" "normalize-path" "propagate-header" "redirect" "request-id" "sensitive-headers" "set-header" "set-status" "timeout" "trace" "util" "validate-request" ]; + "decompression-gzip" = [ "dep:async-compression" "async-compression?/gzip" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "dep:tokio" ]; + "decompression-zstd" = [ "dep:async-compression" "async-compression?/zstd" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "dep:tokio" ]; + "follow-redirect" = [ "futures-util" "dep:http-body" "dep:url" "tower/util" ]; + "fs" = [ "dep:tokio" "tokio?/fs" "tokio?/io-util" "futures-core" "futures-util" "dep:http-body" "dep:http-body-util" "tokio-util/io" "dep:http-range-header" "mime_guess" "mime" "percent-encoding" "httpdate" "set-status" "futures-util/alloc" ]; + "full" = [ "add-extension" "auth" "catch-panic" "compression-full" "cors" "decompression-full" "follow-redirect" "fs" "limit" "map-request-body" "map-response-body" "metrics" "normalize-path" "on-early-drop" "propagate-header" "redirect" "request-id" "sensitive-headers" "set-header" "set-status" "timeout" "trace" "util" "validate-request" ]; "futures-core" = [ "dep:futures-core" ]; "futures-util" = [ "dep:futures-util" ]; "httpdate" = [ "dep:httpdate" ]; - "iri-string" = [ "dep:iri-string" ]; "limit" = [ "dep:http-body" "dep:http-body-util" ]; - "metrics" = [ "dep:http-body" "tokio/time" ]; + "metrics" = [ "dep:http-body" "dep:tokio" "tokio?/time" ]; "mime" = [ "dep:mime" ]; "mime_guess" = [ "dep:mime_guess" ]; + "on-early-drop" = [ "dep:http-body" ]; "percent-encoding" = [ "dep:percent-encoding" ]; "request-id" = [ "uuid" ]; - "timeout" = [ "dep:http-body" "tokio/time" ]; - "tokio" = [ "dep:tokio" ]; + "timeout" = [ "dep:http-body" "dep:tokio" "tokio?/time" ]; "tokio-util" = [ "dep:tokio-util" ]; "tower" = [ "dep:tower" ]; "trace" = [ "dep:http-body" "tracing" ]; @@ -11830,7 +11832,7 @@ rec { "uuid" = [ "dep:uuid" ]; "validate-request" = [ "mime" ]; }; - resolvedDefaultFeatures = [ "auth" "base64" "default" "follow-redirect" "futures-util" "iri-string" "map-response-body" "mime" "tower" "trace" "tracing" "util" "validate-request" ]; + resolvedDefaultFeatures = [ "auth" "base64" "default" "follow-redirect" "futures-util" "map-response-body" "mime" "tower" "trace" "tracing" "util" "validate-request" ]; }; "tower-layer" = rec { crateName = "tower-layer"; @@ -12037,9 +12039,9 @@ rec { }; "tracing-opentelemetry" = rec { crateName = "tracing-opentelemetry"; - version = "0.32.1"; + version = "0.33.0"; edition = "2021"; - sha256 = "1z2jjmxbkm1qawlb3bm99x8xwf4g8wjkbcknm9z4fv1w14nqzhhs"; + sha256 = "09nvxy5m7nxmifz4b6szdcyczapp2jcgxcac0jw4ax8klz5n9g5d"; libName = "tracing_opentelemetry"; dependencies = [ { @@ -12274,13 +12276,9 @@ rec { }; "typenum" = rec { crateName = "typenum"; - version = "1.20.0"; + version = "1.20.1"; edition = "2018"; - sha256 = "1pj35y6q11d3y55gdl6g1h2dfhmybjming0jdi9bh0bpnqm11kj0"; - authors = [ - "Paho Lurie-Gregg " - "Andre Bogus " - ]; + sha256 = "086s9ly0906kw5yw41249fba97w5zfxf03pyfwdkffvcprqfixdn"; features = { "scale-info" = [ "dep:scale-info" ]; "scale_info" = [ "scale-info/derive" ]; @@ -12313,9 +12311,9 @@ rec { }; "unicode-segmentation" = rec { crateName = "unicode-segmentation"; - version = "1.13.2"; + version = "1.13.3"; edition = "2018"; - sha256 = "135a26m4a0wj319gcw28j6a5aqvz00jmgwgmcs6szgxjf942facn"; + sha256 = "1a47zaq83p386r3baq4m018xd5q4q0grdg56i1x042dzn71x7xf6"; libName = "unicode_segmentation"; authors = [ "kwantam " @@ -12441,6 +12439,66 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; + "uuid" = rec { + crateName = "uuid"; + version = "1.23.2"; + edition = "2021"; + sha256 = "1xy942s4z0bi8p3441wvd4ry3hx6ry1c7s6fgrr38462xqybhn6j"; + authors = [ + "Ashley Mannix" + "Dylan DPC" + "Hunar Roop Kahlon" + ]; + dependencies = [ + { + name = "js-sys"; + packageId = "js-sys"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)) && (builtins.elem "atomics" targetFeatures)); + } + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); + } + ]; + devDependencies = [ + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + "atomic" = [ "dep:atomic" ]; + "borsh" = [ "dep:borsh" "dep:borsh-derive" ]; + "bytemuck" = [ "dep:bytemuck" ]; + "default" = [ "std" ]; + "fast-rng" = [ "rng" "dep:rand" ]; + "js" = [ "dep:wasm-bindgen" "dep:js-sys" ]; + "md5" = [ "dep:md-5" ]; + "rng" = [ "dep:getrandom" ]; + "rng-getrandom" = [ "rng" "dep:getrandom" "uuid-rng-internal-lib" "uuid-rng-internal-lib/getrandom" ]; + "rng-rand" = [ "rng" "dep:rand" "uuid-rng-internal-lib" "uuid-rng-internal-lib/rand" ]; + "serde" = [ "dep:serde_core" ]; + "sha1" = [ "dep:sha1_smol" ]; + "slog" = [ "dep:slog" ]; + "std" = [ "wasm-bindgen?/std" "js-sys?/std" ]; + "uuid-rng-internal-lib" = [ "dep:uuid-rng-internal-lib" ]; + "v1" = [ "atomic" ]; + "v3" = [ "md5" ]; + "v4" = [ "rng" ]; + "v5" = [ "sha1" ]; + "v6" = [ "atomic" ]; + "v7" = [ "rng" ]; + "zerocopy" = [ "dep:zerocopy" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "valuable" = rec { crateName = "valuable"; version = "0.1.1"; @@ -12528,9 +12586,9 @@ rec { }; "wasm-bindgen" = rec { crateName = "wasm-bindgen"; - version = "0.2.120"; + version = "0.2.122"; edition = "2021"; - sha256 = "1cax9wy1n67sa2m16ia72lsxdrc5pzcv47psxp4p833yp3cvclnz"; + sha256 = "02flix96brsb2r1i3grnikii302iqpdm337kl3xv5lklz5v4bl1y"; libName = "wasm_bindgen"; authors = [ "The wasm-bindgen Developers" @@ -12579,9 +12637,9 @@ rec { }; "wasm-bindgen-futures" = rec { crateName = "wasm-bindgen-futures"; - version = "0.4.70"; + version = "0.4.72"; edition = "2021"; - sha256 = "1130a64yig0p10mk7rnq5l2jpwglbyxpnqg6h0nlqwzcmir4i4xg"; + sha256 = "03qb24gfr072rk8hb69glfdc8yhqqqq2rhy3j5i0ps8sk79dnwwl"; libName = "wasm_bindgen_futures"; authors = [ "The wasm-bindgen Developers" @@ -12607,9 +12665,9 @@ rec { }; "wasm-bindgen-macro" = rec { crateName = "wasm-bindgen-macro"; - version = "0.2.120"; + version = "0.2.122"; edition = "2021"; - sha256 = "00xixpydzjd6y9knwdsrsiff6wi1ddszb1fa9bk25csz94gh9cbq"; + sha256 = "1inyl55bvdifx7l60q9wl0ivmw7236jg7jqmcqpxhsx3knq52qci"; procMacro = true; libName = "wasm_bindgen_macro"; authors = [ @@ -12631,9 +12689,9 @@ rec { }; "wasm-bindgen-macro-support" = rec { crateName = "wasm-bindgen-macro-support"; - version = "0.2.120"; + version = "0.2.122"; edition = "2021"; - sha256 = "0h8v5aphaaq155fzz2d312zrxbka1x6lsvb8mhc8m60n0kr0zkcx"; + sha256 = "0pjw5kc2mbfz59agk5l21kh4hxzp94rygdvsnr4f3z6b5hv4g419"; libName = "wasm_bindgen_macro_support"; authors = [ "The wasm-bindgen Developers" @@ -12667,10 +12725,10 @@ rec { }; "wasm-bindgen-shared" = rec { crateName = "wasm-bindgen-shared"; - version = "0.2.120"; + version = "0.2.122"; edition = "2021"; links = "wasm_bindgen"; - sha256 = "1sjrac4f4j5pgw42mxadq9v42f0bji1a2rcksrbnrwgbh8y7nxa9"; + sha256 = "0ds4mmfqvxwc5fp33hn0jblf0f6b4lghrd9mpkls66zic4n9p4ls"; libName = "wasm_bindgen_shared"; authors = [ "The wasm-bindgen Developers" @@ -12685,9 +12743,9 @@ rec { }; "web-sys" = rec { crateName = "web-sys"; - version = "0.3.97"; + version = "0.3.99"; edition = "2021"; - sha256 = "00knh8cizgb01bmh362f3f19b11f8zl1y3gj6h47pk95233vmb9f"; + sha256 = "0dilfvl9jnyhi4skl6cry9wc300r693j0w82jjbq8yy3rx0i8qkd"; libName = "web_sys"; authors = [ "The wasm-bindgen Developers" @@ -12771,6 +12829,7 @@ rec { "CssStyleSheet" = [ "StyleSheet" ]; "CssSupportsRule" = [ "CssConditionRule" "CssGroupingRule" "CssRule" ]; "CssTransition" = [ "Animation" "EventTarget" ]; + "CssViewTransitionRule" = [ "CssRule" ]; "CustomEvent" = [ "Event" ]; "DedicatedWorkerGlobalScope" = [ "EventTarget" "WorkerGlobalScope" ]; "DelayNode" = [ "AudioNode" "EventTarget" ]; @@ -13847,7 +13906,7 @@ rec { "Win32_Web" = [ "Win32" ]; "Win32_Web_InternetExplorer" = [ "Win32_Web" ]; }; - resolvedDefaultFeatures = [ "Wdk" "Wdk_Foundation" "Wdk_Storage" "Wdk_Storage_FileSystem" "Wdk_System" "Wdk_System_IO" "Win32" "Win32_Foundation" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_IO" "Win32_System_LibraryLoader" "Win32_System_Memory" "Win32_System_Pipes" "Win32_System_SystemInformation" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_Time" "Win32_System_WindowsProgramming" "default" ]; + resolvedDefaultFeatures = [ "Wdk" "Wdk_Foundation" "Wdk_Storage" "Wdk_Storage_FileSystem" "Wdk_System" "Wdk_System_IO" "Win32" "Win32_Foundation" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_IO" "Win32_System_LibraryLoader" "Win32_System_Memory" "Win32_System_Pipes" "Win32_System_SystemInformation" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_WindowsProgramming" "default" ]; }; "windows-targets" = rec { crateName = "windows-targets"; @@ -13984,9 +14043,9 @@ rec { }; "winnow" = rec { crateName = "winnow"; - version = "1.0.2"; + version = "1.0.3"; edition = "2021"; - sha256 = "1l7xnfvlgy4da6gq5ip2bgcm8i9d0rwzaxg1p88nlw8lxy5p1q9f"; + sha256 = "1wajycd3krn6h699vydjv7hm0ll5l31p899qzpk59y2is74y34h5"; dependencies = [ { name = "memchr"; @@ -14098,9 +14157,9 @@ rec { }; "xml" = rec { crateName = "xml"; - version = "1.2.1"; + version = "1.3.0"; edition = "2021"; - sha256 = "0ak4k990faralbli5a0rb8kvwihccb2rp0r94d4azfy94a6lkamq"; + sha256 = "128s58qhq8whrx90zbw8r5algr7lakgbf7mn05jfk234rbjqavv3"; authors = [ "Vladimir Matveev " "Kornel (https://github.com/kornelski)" @@ -14109,9 +14168,9 @@ rec { }; "yoke" = rec { crateName = "yoke"; - version = "0.8.2"; + version = "0.8.3"; edition = "2021"; - sha256 = "1jprcs7a98a5whvfs6r3jvfh1nnfp6zyijl7y4ywmn88lzywbs5b"; + sha256 = "1xgyj6c2lxj2bp891ynmhws87c6z7yyv2li1v0ss9di40hxf57vh"; authors = [ "Manish Goregaokar " ]; @@ -14175,9 +14234,9 @@ rec { }; "zerocopy" = rec { crateName = "zerocopy"; - version = "0.8.48"; + version = "0.8.50"; edition = "2021"; - sha256 = "1sb8plax8jbrsng1jdval7bdhk7hhrx40dz3hwh074k6knzkgm7f"; + sha256 = "1laahnfxs4qyfb1fdf5nbb2qfshi72b1hbi0ffp2zy2m1r7ms1iv"; authors = [ "Joshua Liebow-Feeser " "Jack Wrenn " @@ -14211,9 +14270,9 @@ rec { }; "zerocopy-derive" = rec { crateName = "zerocopy-derive"; - version = "0.8.48"; + version = "0.8.50"; edition = "2021"; - sha256 = "1m5s0g92cxggqc74j83k1priz24k3z93sj5gadppd20p9c4cvqvh"; + sha256 = "0fdnr9qslx1hbn2i9rsvy9s95mychfy2vj90ajsjm2basccinqqb"; procMacro = true; libName = "zerocopy_derive"; authors = [ @@ -14246,11 +14305,11 @@ rec { }; "zerofrom" = rec { crateName = "zerofrom"; - version = "0.1.7"; + version = "0.1.8"; edition = "2021"; - sha256 = "1py40in4rirc9q8w36q67pld0zk8ssg024xhh0cncxgal7ra3yk9"; + sha256 = "0wjjdj7gdmd0iq91gzkxl7dlv0nhkk80l4bmdpzh3a1yh48mmh0f"; authors = [ - "Manish Goregaokar " + "The ICU4X Project Developers" ]; dependencies = [ { diff --git a/crate-hashes.json b/crate-hashes.json index 71fbc1c3..014d9478 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#k8s-version@0.1.3": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-certs@0.4.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-operator-derive@0.3.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-operator@0.111.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-shared@0.1.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-telemetry@0.6.3": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-versioned-macros@0.10.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-versioned@0.10.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-webhook@0.9.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 1a72a51f..2911021b 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -1035,7 +1035,6 @@ mod test { &DruidRole::Historical, &rolegroup_ref, &rg, - &resolved_product_image, &druid, ) .expect("build rolegroup config map"); From 383b63ccb20906aed830a6b23522a0e51d9a439b Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 5 Jun 2026 13:55:02 +0200 Subject: [PATCH 14/48] refactor: move logging & discovery --- rust/operator-binary/src/controller.rs | 24 +++--- rust/operator-binary/src/controller/build.rs | 1 + .../src/controller/build/config_map.rs | 45 +++++----- .../src/{ => controller/build}/discovery.rs | 40 +++------ .../controller/build/properties/logging.rs | 77 +++++++++++++++++ .../src/controller/build/properties/mod.rs | 7 ++ .../src/controller/validate.rs | 9 +- rust/operator-binary/src/crd/memory.rs | 10 ++- rust/operator-binary/src/crd/mod.rs | 26 ------ rust/operator-binary/src/main.rs | 2 - rust/operator-binary/src/product_logging.rs | 85 ------------------- 11 files changed, 145 insertions(+), 181 deletions(-) rename rust/operator-binary/src/{ => controller/build}/discovery.rs (69%) create mode 100644 rust/operator-binary/src/controller/build/properties/logging.rs delete mode 100644 rust/operator-binary/src/product_logging.rs diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 2911021b..4c7c4bff 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -56,12 +56,10 @@ use crate::{ authentication::DruidAuthenticationConfig, crd::{ APP_NAME, CommonRoleGroupConfig, Container, DRUID_CONFIG_DIRECTORY, DeepStorageSpec, - DruidClusterStatus, DruidRole, HDFS_CONFIG_DIRECTORY, LOG_CONFIG_DIRECTORY, - MAX_DRUID_LOG_FILES_SIZE, METRICS_PORT, METRICS_PORT_NAME, OPERATOR_NAME, - RW_CONFIG_DIRECTORY, STACKABLE_LOG_DIR, build_recommended_labels, - security::DruidTlsSecurity, v1alpha1, + DruidClusterStatus, DruidRole, HDFS_CONFIG_DIRECTORY, LOG_CONFIG_DIRECTORY, METRICS_PORT, + METRICS_PORT_NAME, OPERATOR_NAME, RW_CONFIG_DIRECTORY, STACKABLE_LOG_DIR, + build_recommended_labels, security::DruidTlsSecurity, v1alpha1, }, - discovery::{self, build_discovery_configmaps}, internal_secret::create_shared_internal_secret, listener::{ LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_group_listener, build_group_listener_pvc, @@ -75,6 +73,9 @@ mod build; mod dereference; mod validate; +use build::discovery::{self, build_discovery_configmaps}; +use build::properties::logging::MAX_DRUID_LOG_FILES_SIZE; + use validate::DruidRoleGroupConfig; pub const DRUID_CONTROLLER_NAME: &str = "druidcluster"; @@ -442,15 +443,10 @@ pub async fn reconcile_druid( if *druid_role == DruidRole::Router { // discovery - for discovery_cm in build_discovery_configmaps( - druid, - druid, - &validated_cluster.image, - &validated_cluster.cluster_config.druid_tls_security, - listener, - ) - .await - .context(BuildDiscoveryConfigSnafu)? + for discovery_cm in + build_discovery_configmaps(&validated_cluster, druid, listener) + .await + .context(BuildDiscoveryConfigSnafu)? { cluster_resources .add(client, discovery_cm) diff --git a/rust/operator-binary/src/controller/build.rs b/rust/operator-binary/src/controller/build.rs index 5a152c2a..182778ae 100644 --- a/rust/operator-binary/src/controller/build.rs +++ b/rust/operator-binary/src/controller/build.rs @@ -1,4 +1,5 @@ //! Build steps that turn a `ValidatedCluster` into Kubernetes resources. pub mod config_map; +pub mod discovery; pub mod properties; diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index d3725028..017012a3 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -18,6 +18,7 @@ use stackable_operator::{ crd::s3, database_connections::drivers::jdbc::JdbcDatabaseConnection as _, k8s_openapi::api::core::v1::{ConfigMap, EnvVar}, + product_logging::framework::VECTOR_CONFIG_FILE, role_utils::RoleGroupRef, }; @@ -25,22 +26,31 @@ use crate::{ config::jvm::construct_jvm_args, controller::{ DRUID_CONTROLLER_NAME, - build::properties::{ConfigFileName, writer::to_java_properties_string}, + build::properties::{ + ConfigFileName, + logging::{build_log4j2_config, build_vector_config}, + writer::to_java_properties_string, + }, validate::{DruidRoleGroupConfig, ValidatedCluster}, }, - crd::{ - AUTH_AUTHORIZER_OPA_URI, DS_BUCKET, DruidRole, EXTENSIONS_LOADLIST, S3_ACCESS_KEY, - S3_ENDPOINT_URL, S3_PATH_STYLE_ACCESS, S3_SECRET_KEY, ZOOKEEPER_CONNECTION_STRING, - build_recommended_labels, build_string_list, v1alpha1, - }, + crd::{DruidRole, build_recommended_labels, build_string_list, v1alpha1}, extensions::get_extension_list, - product_logging::extend_role_group_config_map, }; // jvm.config is built by `config::jvm`, not a properties builder, so it is not part // of `ConfigFileName`. const JVM_CONFIG: &str = "jvm.config"; +// Druid `runtime.properties` config-property keys assembled into the rolegroup ConfigMap here. +const EXTENSIONS_LOADLIST: &str = "druid.extensions.loadList"; +const ZOOKEEPER_CONNECTION_STRING: &str = "druid.zk.service.host"; +const DS_BUCKET: &str = "druid.storage.bucket"; +const S3_ENDPOINT_URL: &str = "druid.s3.endpoint.url"; +const S3_ACCESS_KEY: &str = "druid.s3.accessKey"; +const S3_SECRET_KEY: &str = "druid.s3.secretKey"; +const S3_PATH_STYLE_ACCESS: &str = "druid.s3.enablePathStyleAccess"; +const AUTH_AUTHORIZER_OPA_URI: &str = "druid.auth.authorizer.OpaAuthorizer.opaUri"; + #[derive(Snafu, Debug)] #[allow(clippy::enum_variant_names)] pub enum Error { @@ -90,12 +100,6 @@ pub enum Error { source: crate::authentication::Error, }, - #[snafu(display("failed to add the logging configuration to the ConfigMap [{cm_name}]"))] - InvalidLoggingConfig { - source: crate::product_logging::Error, - cm_name: String, - }, - #[snafu(display("invalid metadata database connection"))] InvalidMetadataDatabaseConnection { source: stackable_operator::database_connections::Error, @@ -314,14 +318,13 @@ pub fn build_rolegroup_config_map( config_map_builder.add_data(filename, file_content); } - extend_role_group_config_map( - rolegroup, - &rg.merged_config.logging, - &mut config_map_builder, - ) - .context(InvalidLoggingConfigSnafu { - cm_name: rolegroup.object_name(), - })?; + if let Some(log4j2_config) = build_log4j2_config(&rg.merged_config.logging) { + config_map_builder.add_data(ConfigFileName::Log4j2Properties.to_string(), log4j2_config); + } + + if let Some(vector_config) = build_vector_config(rolegroup, &rg.merged_config.logging) { + config_map_builder.add_data(VECTOR_CONFIG_FILE, vector_config); + } config_map_builder .build() diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/controller/build/discovery.rs similarity index 69% rename from rust/operator-binary/src/discovery.rs rename to rust/operator-binary/src/controller/build/discovery.rs index 95220b22..0e371444 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/controller/build/discovery.rs @@ -4,24 +4,23 @@ use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, - commons::product_image_selection::ResolvedProductImage, crd::listener::v1alpha1::Listener, k8s_openapi::api::core::v1::ConfigMap, - kube::{Resource, ResourceExt, runtime::reflector::ObjectRef}, + kube::{Resource, ResourceExt}, }; use crate::{ DRUID_CONTROLLER_NAME, - crd::{DruidRole, build_recommended_labels, security::DruidTlsSecurity, v1alpha1}, + controller::validate::ValidatedCluster, + crd::{DruidRole, build_recommended_labels}, listener::build_listener_connection_string, }; #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("object {} is missing metadata to build owner reference", druid))] + #[snafu(display("object is missing metadata to build owner reference"))] ObjectMissingMetadataForOwnerRef { source: stackable_operator::builder::meta::Error, - druid: ObjectRef, }, #[snafu(display("failed to get service FQDN"))] @@ -41,37 +40,28 @@ pub enum Error { ListenerConfiguration { source: crate::listener::Error }, } -/// Builds discovery [`ConfigMap`]s for connecting to a [`v1alpha1::DruidCluster`]. +/// Builds discovery [`ConfigMap`]s for connecting to a Druid cluster. pub async fn build_discovery_configmaps( - druid: &v1alpha1::DruidCluster, + cluster: &ValidatedCluster, owner: &impl Resource, - resolved_product_image: &ResolvedProductImage, - druid_tls_security: &DruidTlsSecurity, listener: Listener, ) -> Result, Error> { let name = owner.name_unchecked(); Ok(vec![build_discovery_configmap( - druid, - owner, - resolved_product_image, - druid_tls_security, - &name, - listener, + cluster, owner, &name, listener, )?]) } -/// Build a discovery [`ConfigMap`] containing information about how to connect to a certain [`v1alpha1::DruidCluster`]. +/// Build a discovery [`ConfigMap`] containing information about how to connect to a certain Druid cluster. fn build_discovery_configmap( - druid: &v1alpha1::DruidCluster, + cluster: &ValidatedCluster, owner: &impl Resource, - resolved_product_image: &ResolvedProductImage, - druid_tls_security: &DruidTlsSecurity, name: &str, listener: Listener, ) -> Result { let router_host = build_listener_connection_string( listener, - druid_tls_security, + &cluster.cluster_config.druid_tls_security, &DruidRole::Router.to_string(), ) .context(ListenerConfigurationSnafu)?; @@ -84,16 +74,14 @@ fn build_discovery_configmap( ConfigMapBuilder::new() .metadata( ObjectMetaBuilder::new() - .name_and_namespace(druid) + .name_and_namespace(owner) .name(name) .ownerreference_from_resource(owner, None, Some(true)) - .with_context(|_| ObjectMissingMetadataForOwnerRefSnafu { - druid: ObjectRef::from_obj(druid), - })? + .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(&build_recommended_labels( - druid, + owner, DRUID_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, + &cluster.image.app_version_label_value, &DruidRole::Router.to_string(), "discovery", )) diff --git a/rust/operator-binary/src/controller/build/properties/logging.rs b/rust/operator-binary/src/controller/build/properties/logging.rs new file mode 100644 index 00000000..adbaac20 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/logging.rs @@ -0,0 +1,77 @@ +//! Builds the log4j2 and Vector logging configuration for the rolegroup `ConfigMap`. + +use stackable_operator::{ + memory::{BinaryMultiple, MemoryQuantity}, + product_logging::{ + self, + spec::{ContainerLogConfig, ContainerLogConfigChoice, Logging}, + }, + role_utils::RoleGroupRef, +}; + +use crate::crd::{Container, STACKABLE_LOG_DIR, v1alpha1}; + +const CONSOLE_CONVERSION_PATTERN: &str = "%d{ISO8601} %p [%t] %c - %m%n"; + +/// File that the Druid log4j2 config writes its rolling log output to. +const DRUID_LOG_FILE: &str = "druid.log4j2.xml"; + +/// Maximum size of all Druid log files combined, used both for the log4j2 rollover configuration +/// and to size the log volume in the controller. +pub const MAX_DRUID_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { + value: 10.0, + unit: BinaryMultiple::Mebi, +}; + +/// Renders the `log4j2.properties` content for the Druid container. +/// +/// Returns `None` when the container uses a custom log ConfigMap rather than the operator's +/// automatic logging configuration. +pub fn build_log4j2_config(logging: &Logging) -> Option { + if let Some(ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Automatic(log_config)), + }) = logging.containers.get(&Container::Druid) + { + Some(product_logging::framework::create_log4j2_config( + &format!( + "{STACKABLE_LOG_DIR}/{container}", + container = Container::Druid + ), + DRUID_LOG_FILE, + MAX_DRUID_LOG_FILES_SIZE + .scale_to(BinaryMultiple::Mebi) + .floor() + .value as u32, + CONSOLE_CONVERSION_PATTERN, + log_config, + )) + } else { + None + } +} + +/// Renders the Vector agent config (`vector.yaml`). +/// +/// Returns `None` when the Vector agent is disabled for this role group. +pub fn build_vector_config( + rolegroup: &RoleGroupRef, + logging: &Logging, +) -> Option { + if !logging.enable_vector_agent { + return None; + } + + let vector_log_config = if let Some(ContainerLogConfig { + choice: Some(ContainerLogConfigChoice::Automatic(log_config)), + }) = logging.containers.get(&Container::Vector) + { + Some(log_config) + } else { + None + }; + + Some(product_logging::framework::create_vector_config( + rolegroup, + vector_log_config, + )) +} diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index 426fb4bb..cb7eb945 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -2,14 +2,21 @@ pub mod writer; +pub mod logging; pub mod runtime_properties; pub mod security_properties; /// The names of the operator-written Druid config files assembled into the rolegroup ConfigMap. #[derive(Clone, Copy, Debug, strum::Display)] +// The shared `Properties` suffix mirrors the actual on-disk file names; it is not redundant naming. +#[allow(clippy::enum_variant_names)] pub enum ConfigFileName { #[strum(serialize = "runtime.properties")] RuntimeProperties, #[strum(serialize = "security.properties")] SecurityProperties, + /// `log4j2.properties` is rendered by the logging framework rather than a properties builder, + /// but it is still an operator-written file assembled into the rolegroup ConfigMap. + #[strum(serialize = "log4j2.properties")] + Log4j2Properties, } diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 444db959..d08f9778 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -21,9 +21,8 @@ use crate::{ dereference::DereferencedObjects, }, crd::{ - CommonRoleGroupConfig, DruidConfigOverrides, DruidRole, INDEXER_JAVA_OPTS, - STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, build_string_list, - security::DruidTlsSecurity, v1alpha1, + CommonRoleGroupConfig, DruidConfigOverrides, DruidRole, STACKABLE_TRUST_STORE, + STACKABLE_TRUST_STORE_PASSWORD, build_string_list, security::DruidTlsSecurity, v1alpha1, }, }; @@ -92,6 +91,8 @@ fn key_value_overrides( let kv = match file { ConfigFileName::RuntimeProperties => overrides.runtime_properties.as_ref(), ConfigFileName::SecurityProperties => overrides.security_properties.as_ref(), + // log4j2.properties is rendered by the logging framework and accepts no key/value overrides. + ConfigFileName::Log4j2Properties => None, }; kv.map( stackable_operator::config_overrides::KeyValueConfigOverrides::as_product_config_overrides, @@ -156,6 +157,8 @@ fn build_role_group_config( } } +const INDEXER_JAVA_OPTS: &str = "druid.indexer.runner.javaOptsArray"; + /// The `druid.indexer.runner.javaOptsArray` entry that `MiddleManagerConfigFragment::compute_files` /// adds for *every* file (runtime.properties and security.properties). fn middlemanager_indexer_java_opts() -> (String, Option) { diff --git a/rust/operator-binary/src/crd/memory.rs b/rust/operator-binary/src/crd/memory.rs index 218c4261..376b6b57 100644 --- a/rust/operator-binary/src/crd/memory.rs +++ b/rust/operator-binary/src/crd/memory.rs @@ -7,10 +7,12 @@ use stackable_operator::{ memory::{BinaryMultiple, MemoryQuantity}, }; -use crate::crd::{ - PROCESSING_BUFFER_SIZE_BYTES, PROCESSING_NUM_MERGE_BUFFERS, PROCESSING_NUM_THREADS, - storage::HistoricalStorage, -}; +use crate::crd::storage::HistoricalStorage; + +// Druid historical processing config-property keys, only used for the memory calculations here. +const PROCESSING_BUFFER_SIZE_BYTES: &str = "druid.processing.buffer.sizeBytes"; +const PROCESSING_NUM_MERGE_BUFFERS: &str = "druid.processing.numMergeBuffers"; +const PROCESSING_NUM_THREADS: &str = "druid.processing.numThreads"; static MIN_HEAP_RATIO: f32 = 0.75; diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 88650abf..1cd3689a 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -26,7 +26,6 @@ use stackable_operator::{ k8s_openapi::api::core::v1::{PodTemplateSpec, Volume}, kube::{CustomResource, ResourceExt}, kvp::ObjectLabels, - memory::{BinaryMultiple, MemoryQuantity}, product_logging::{ self, framework::{create_vector_shutdown_file_command, remove_vector_shutdown_file_command}, @@ -74,46 +73,21 @@ pub const STACKABLE_TRUST_STORE: &str = "/stackable/truststore.p12"; pub const STACKABLE_TRUST_STORE_PASSWORD: &str = "changeit"; pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; -// store file names -pub const DRUID_LOG_FILE: &str = "druid.log4j2.xml"; - pub const PROP_SEGMENT_CACHE_LOCATIONS: &str = "druid.segmentCache.locations"; pub const PATH_SEGMENT_CACHE: &str = "/stackable/var/druid/segment-cache"; ///////////////////////////// // CONFIG PROPERTIES // ///////////////////////////// -// extensions -pub const EXTENSIONS_LOADLIST: &str = "druid.extensions.loadList"; -// zookeeper -pub const ZOOKEEPER_CONNECTION_STRING: &str = "druid.zk.service.host"; // deep storage pub const DS_TYPE: &str = "druid.storage.type"; pub const DS_DIRECTORY: &str = "druid.storage.storageDirectory"; -// S3 -pub const DS_BUCKET: &str = "druid.storage.bucket"; pub const DS_BASE_KEY: &str = "druid.storage.baseKey"; -pub const S3_ENDPOINT_URL: &str = "druid.s3.endpoint.url"; -pub const S3_ACCESS_KEY: &str = "druid.s3.accessKey"; -pub const S3_SECRET_KEY: &str = "druid.s3.secretKey"; -pub const S3_PATH_STYLE_ACCESS: &str = "druid.s3.enablePathStyleAccess"; // OPA pub const AUTH_AUTHORIZERS: &str = "druid.auth.authorizers"; pub const AUTH_AUTHORIZERS_VALUE: &str = "[\"OpaAuthorizer\"]"; pub const AUTH_AUTHORIZER_OPA_TYPE: &str = "druid.auth.authorizer.OpaAuthorizer.type"; pub const AUTH_AUTHORIZER_OPA_TYPE_VALUE: &str = "opa"; -pub const AUTH_AUTHORIZER_OPA_URI: &str = "druid.auth.authorizer.OpaAuthorizer.opaUri"; -// indexer properties -pub const INDEXER_JAVA_OPTS: &str = "druid.indexer.runner.javaOptsArray"; -// historical settings -pub const PROCESSING_BUFFER_SIZE_BYTES: &str = "druid.processing.buffer.sizeBytes"; -pub const PROCESSING_NUM_MERGE_BUFFERS: &str = "druid.processing.numMergeBuffers"; -pub const PROCESSING_NUM_THREADS: &str = "druid.processing.numThreads"; -// logs -pub const MAX_DRUID_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { - value: 10.0, - unit: BinaryMultiple::Mebi, -}; // metrics pub const PROMETHEUS_PORT: &str = "druid.emitter.prometheus.port"; pub const METRICS_PORT_NAME: &str = "metrics"; diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index a157763b..aa99e518 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -41,12 +41,10 @@ mod authentication; mod config; mod controller; mod crd; -mod discovery; mod extensions; mod internal_secret; mod listener; mod operations; -mod product_logging; mod service; mod webhooks; diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs deleted file mode 100644 index a1b0809f..00000000 --- a/rust/operator-binary/src/product_logging.rs +++ /dev/null @@ -1,85 +0,0 @@ -use snafu::Snafu; -use stackable_operator::{ - builder::configmap::ConfigMapBuilder, - memory::BinaryMultiple, - product_logging::{ - self, - spec::{ContainerLogConfig, ContainerLogConfigChoice, Logging}, - }, - role_utils::RoleGroupRef, -}; - -use crate::crd::{ - Container, DRUID_LOG_FILE, MAX_DRUID_LOG_FILES_SIZE, STACKABLE_LOG_DIR, v1alpha1, -}; - -// log4j2.properties is written by the logging framework, not a properties builder, so it is -// not part of ConfigFileName. File name not exported from crd/mod.rs. -const LOG4J2_CONFIG: &str = "log4j2.properties"; - -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("object has no namespace"))] - ObjectHasNoNamespace, - #[snafu(display("failed to retrieve the ConfigMap {cm_name}"))] - ConfigMapNotFound { - source: stackable_operator::client::Error, - cm_name: String, - }, - #[snafu(display("failed to retrieve the entry {entry} for ConfigMap {cm_name}"))] - MissingConfigMapEntry { - entry: &'static str, - cm_name: String, - }, -} - -type Result = std::result::Result; - -const CONSOLE_CONVERSION_PATTERN: &str = "%d{ISO8601} %p [%t] %c - %m%n"; - -/// Extend the role group ConfigMap with logging and Vector configurations -pub fn extend_role_group_config_map( - rolegroup: &RoleGroupRef, - logging: &Logging, - cm_builder: &mut ConfigMapBuilder, -) -> Result<()> { - if let Some(ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic(log_config)), - }) = logging.containers.get(&Container::Druid) - { - cm_builder.add_data( - LOG4J2_CONFIG, - product_logging::framework::create_log4j2_config( - &format!( - "{STACKABLE_LOG_DIR}/{container}", - container = Container::Druid - ), - DRUID_LOG_FILE, - MAX_DRUID_LOG_FILES_SIZE - .scale_to(BinaryMultiple::Mebi) - .floor() - .value as u32, - CONSOLE_CONVERSION_PATTERN, - log_config, - ), - ); - } - - let vector_log_config = if let Some(ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic(log_config)), - }) = logging.containers.get(&Container::Vector) - { - Some(log_config) - } else { - None - }; - - if logging.enable_vector_agent { - cm_builder.add_data( - product_logging::framework::VECTOR_CONFIG_FILE, - product_logging::framework::create_vector_config(rolegroup, vector_log_config), - ); - } - - Ok(()) -} From b963c5e168c03988d87c97bb27b189fb7cdd9fd7 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 5 Jun 2026 13:55:13 +0200 Subject: [PATCH 15/48] fix: regenerate hashes --- Cargo.nix | 18 +++++++++--------- crate-hashes.json | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index 20c06abd..c20efc31 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4860,7 +4860,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "128e1afca7761d07058624091217b6d695fa790c"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; }; libName = "k8s_version"; authors = [ @@ -9653,7 +9653,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "128e1afca7761d07058624091217b6d695fa790c"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; }; libName = "stackable_certs"; authors = [ @@ -9865,7 +9865,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "128e1afca7761d07058624091217b6d695fa790c"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; }; libName = "stackable_operator"; authors = [ @@ -10051,7 +10051,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "128e1afca7761d07058624091217b6d695fa790c"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -10086,7 +10086,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "128e1afca7761d07058624091217b6d695fa790c"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; }; libName = "stackable_shared"; authors = [ @@ -10167,7 +10167,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "128e1afca7761d07058624091217b6d695fa790c"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; }; libName = "stackable_telemetry"; authors = [ @@ -10277,7 +10277,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "128e1afca7761d07058624091217b6d695fa790c"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; }; libName = "stackable_versioned"; authors = [ @@ -10327,7 +10327,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "128e1afca7761d07058624091217b6d695fa790c"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -10395,7 +10395,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "128e1afca7761d07058624091217b6d695fa790c"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; }; libName = "stackable_webhook"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index 014d9478..b944c535 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file From 7032fbce5ad40605bcf840efb059e6d3a9d6b778 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 5 Jun 2026 15:41:06 +0200 Subject: [PATCH 16/48] refactor: remove writer.rs & bump dependencies --- Cargo.lock | 21 ++--- Cargo.nix | 30 +++---- Cargo.toml | 5 +- rust/operator-binary/Cargo.toml | 1 - rust/operator-binary/src/controller.rs | 26 ++++--- .../src/controller/build/config_map.rs | 6 +- .../src/controller/build/properties/mod.rs | 2 - .../src/controller/build/properties/writer.rs | 78 ------------------- 8 files changed, 47 insertions(+), 122 deletions(-) delete mode 100644 rust/operator-binary/src/controller/build/properties/writer.rs diff --git a/Cargo.lock b/Cargo.lock index 302dc4a7..f3b0bf82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1530,7 +1530,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#128e1afca7761d07058624091217b6d695fa790c" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "darling", "regex", @@ -2946,7 +2946,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#128e1afca7761d07058624091217b6d695fa790c" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "const-oid", "ecdsa", @@ -2978,7 +2978,6 @@ dependencies = [ "fnv", "futures 0.3.32", "indoc", - "java-properties", "openssl", "pin-project", "rstest", @@ -2996,7 +2995,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.111.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#128e1afca7761d07058624091217b6d695fa790c" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "base64", "clap", @@ -3008,6 +3007,7 @@ dependencies = [ "futures 0.3.32", "http", "indexmap", + "java-properties", "jiff", "json-patch", "k8s-openapi", @@ -3033,12 +3033,13 @@ dependencies = [ "tracing-subscriber", "url", "uuid", + "xml", ] [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#128e1afca7761d07058624091217b6d695fa790c" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "darling", "proc-macro2", @@ -3049,7 +3050,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#128e1afca7761d07058624091217b6d695fa790c" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "jiff", "k8s-openapi", @@ -3066,7 +3067,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.4" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#128e1afca7761d07058624091217b6d695fa790c" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "axum", "clap", @@ -3090,7 +3091,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#128e1afca7761d07058624091217b6d695fa790c" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "kube", "schemars", @@ -3104,7 +3105,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#128e1afca7761d07058624091217b6d695fa790c" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "convert_case", "convert_case_extras", @@ -3122,7 +3123,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#128e1afca7761d07058624091217b6d695fa790c" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.nix b/Cargo.nix index c20efc31..89c87564 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4859,7 +4859,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "128e1afca7761d07058624091217b6d695fa790c"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; }; libName = "k8s_version"; @@ -9652,7 +9652,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "128e1afca7761d07058624091217b6d695fa790c"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; }; libName = "stackable_certs"; @@ -9789,10 +9789,6 @@ rec { name = "indoc"; packageId = "indoc"; } - { - name = "java-properties"; - packageId = "java-properties"; - } { name = "openssl"; packageId = "openssl"; @@ -9864,7 +9860,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "128e1afca7761d07058624091217b6d695fa790c"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; }; libName = "stackable_operator"; @@ -9915,6 +9911,10 @@ rec { name = "indexmap"; packageId = "indexmap"; } + { + name = "java-properties"; + packageId = "java-properties"; + } { name = "jiff"; packageId = "jiff"; @@ -10030,6 +10030,10 @@ rec { name = "uuid"; packageId = "uuid"; } + { + name = "xml"; + packageId = "xml"; + } ]; features = { "certs" = [ "dep:stackable-certs" ]; @@ -10050,7 +10054,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "128e1afca7761d07058624091217b6d695fa790c"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; }; procMacro = true; @@ -10085,7 +10089,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "128e1afca7761d07058624091217b6d695fa790c"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; }; libName = "stackable_shared"; @@ -10166,7 +10170,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "128e1afca7761d07058624091217b6d695fa790c"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; }; libName = "stackable_telemetry"; @@ -10276,7 +10280,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "128e1afca7761d07058624091217b6d695fa790c"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; }; libName = "stackable_versioned"; @@ -10326,7 +10330,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "128e1afca7761d07058624091217b6d695fa790c"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; }; procMacro = true; @@ -10394,7 +10398,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "128e1afca7761d07058624091217b6d695fa790c"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; }; libName = "stackable_webhook"; diff --git a/Cargo.toml b/Cargo.toml index 3165d088..7c5949f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,12 +14,11 @@ stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", anyhow = "1.0" built = { version = "0.8", features = ["chrono", "git2"] } -clap = "4.5" +clap = "4.6" const_format = "0.2" fnv = "1.0" futures = { version = "0.3", features = ["compat"] } indoc = "2.0" -java-properties = "2.0" openssl = "0.10" pin-project = "1.1" rstest = "0.26" @@ -29,7 +28,7 @@ serde_json = "1.0" serde_yaml = "0.9" snafu = "0.9" strum = { version = "0.28", features = ["derive"] } -tokio = { version = "1.40", features = ["full"] } +tokio = { version = "1.52", features = ["full"] } tracing = "0.1" [patch."https://github.com/stackabletech/operator-rs.git"] diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index ea94f2f3..84f0146b 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -17,7 +17,6 @@ const_format.workspace = true fnv.workspace = true futures.workspace = true indoc.workspace = true -java-properties.workspace = true openssl.workspace = true pin-project.workspace = true semver.workspace = true diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 4c7c4bff..014adf9e 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -73,9 +73,10 @@ mod build; mod dereference; mod validate; -use build::discovery::{self, build_discovery_configmaps}; -use build::properties::logging::MAX_DRUID_LOG_FILES_SIZE; - +use build::{ + discovery::{self, build_discovery_configmaps}, + properties::logging::MAX_DRUID_LOG_FILES_SIZE, +}; use validate::DruidRoleGroupConfig; pub const DRUID_CONTROLLER_NAME: &str = "druidcluster"; @@ -944,7 +945,7 @@ mod test { controller::{ build::{ config_map::build_rolegroup_config_map, - properties::{ConfigFileName, runtime_properties, writer}, + properties::{ConfigFileName, runtime_properties}, }, validate::{DruidRoleGroupConfig, ValidatedCluster, ValidatedClusterConfig}, }, @@ -1042,14 +1043,15 @@ mod test { .unwrap() .to_string(); - let escaped_segment_cache_property = writer::to_java_properties_string( - vec![( - &PROP_SEGMENT_CACHE_LOCATIONS.to_string(), - &Some(expected_druid_segment_cache_property.to_string()), - )] - .into_iter(), - ) - .unwrap(); + let escaped_segment_cache_property = + stackable_operator::v2::config_file_writer::to_java_properties_string( + vec![( + &PROP_SEGMENT_CACHE_LOCATIONS.to_string(), + &Some(expected_druid_segment_cache_property.to_string()), + )] + .into_iter(), + ) + .unwrap(); assert!( druid_segment_cache_property.contains(&escaped_segment_cache_property), diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 017012a3..874f0dc1 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -20,6 +20,7 @@ use stackable_operator::{ k8s_openapi::api::core::v1::{ConfigMap, EnvVar}, product_logging::framework::VECTOR_CONFIG_FILE, role_utils::RoleGroupRef, + v2::config_file_writer::to_java_properties_string, }; use crate::{ @@ -29,7 +30,6 @@ use crate::{ build::properties::{ ConfigFileName, logging::{build_log4j2_config, build_vector_config}, - writer::to_java_properties_string, }, validate::{DruidRoleGroupConfig, ValidatedCluster}, }, @@ -72,12 +72,12 @@ pub enum Error { #[snafu(display("failed to serialize [runtime.properties]"))] SerializeRuntimeProperties { - source: crate::controller::build::properties::writer::PropertiesWriterError, + source: stackable_operator::v2::config_file_writer::PropertiesWriterError, }, #[snafu(display("failed to serialize [security.properties] for {rolegroup}"))] JvmSecurityProperties { - source: crate::controller::build::properties::writer::PropertiesWriterError, + source: stackable_operator::v2::config_file_writer::PropertiesWriterError, rolegroup: String, }, diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index cb7eb945..ee12da28 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -1,7 +1,5 @@ //! Per-file builders for Druid `.properties` files. -pub mod writer; - pub mod logging; pub mod runtime_properties; pub mod security_properties; diff --git a/rust/operator-binary/src/controller/build/properties/writer.rs b/rust/operator-binary/src/controller/build/properties/writer.rs deleted file mode 100644 index a74babf0..00000000 --- a/rust/operator-binary/src/controller/build/properties/writer.rs +++ /dev/null @@ -1,78 +0,0 @@ -//! Writer for Java `.properties` files. -//! -//! Vendored from the `product-config` crate's `writer` module so the operator no -//! longer depends on `product-config` for rendering. - -use std::io::Write; - -use java_properties::{PropertiesError, PropertiesWriter}; -use snafu::{ResultExt, Snafu}; - -#[derive(Debug, Snafu)] -pub enum PropertiesWriterError { - #[snafu(display("failed to create properties file"))] - Properties { source: PropertiesError }, - - #[snafu(display("failed to convert properties file byte array to UTF-8"))] - FromUtf8 { source: std::string::FromUtf8Error }, -} - -/// Creates a common Java properties file string in the format: -/// `property_1=value_1\nproperty_2=value_2\n`. -pub fn to_java_properties_string<'a, T>(properties: T) -> Result -where - T: Iterator)>, -{ - let mut output = Vec::new(); - write_java_properties(&mut output, properties)?; - String::from_utf8(output).context(FromUtf8Snafu) -} - -/// Writes Java properties to the given writer. A `None` value is written as an -/// empty value (`key=`). -fn write_java_properties<'a, W, T>(writer: W, properties: T) -> Result<(), PropertiesWriterError> -where - W: Write, - T: Iterator)>, -{ - let mut writer = PropertiesWriter::new(writer); - for (k, v) in properties { - let property_value = v.as_deref().unwrap_or_default(); - writer.write(k, property_value).context(PropertiesSnafu)?; - } - writer.flush().context(PropertiesSnafu)?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use std::collections::BTreeMap; - - use super::*; - - fn props(pairs: &[(&str, Option<&str>)]) -> String { - let map: BTreeMap> = pairs - .iter() - .map(|(k, v)| (k.to_string(), v.map(str::to_string))) - .collect(); - to_java_properties_string(map.iter()).unwrap() - } - - #[test] - fn java_properties_renders_key_value() { - assert_eq!(props(&[("a", Some("1")), ("b", Some("2"))]), "a=1\nb=2\n"); - } - - #[test] - fn java_properties_renders_none_as_empty() { - assert_eq!(props(&[("none", None)]), "none=\n"); - } - - #[test] - fn java_properties_escapes_colon_in_value() { - assert_eq!( - props(&[("url", Some("file://this/location/file.abc"))]), - "url=file\\://this/location/file.abc\n" - ); - } -} From b3a563047f8d3a2332bbbe9d31444118c8016d28 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 5 Jun 2026 15:42:50 +0200 Subject: [PATCH 17/48] chore: regenerate hashes --- Cargo.nix | 18 +++++++++--------- crate-hashes.json | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index 89c87564..45874b46 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4860,7 +4860,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; + sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "k8s_version"; authors = [ @@ -9653,7 +9653,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; + sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_certs"; authors = [ @@ -9861,7 +9861,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; + sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_operator"; authors = [ @@ -10055,7 +10055,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; + sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -10090,7 +10090,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; + sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_shared"; authors = [ @@ -10171,7 +10171,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; + sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_telemetry"; authors = [ @@ -10281,7 +10281,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; + sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_versioned"; authors = [ @@ -10331,7 +10331,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; + sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -10399,7 +10399,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; - sha256 = "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs"; + sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_webhook"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index b944c535..deac3bf4 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "1a7g0rvvinwkm2wl5sxp129dc9agiilbgzfi0pvdp5a81xv01nxs", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file From 6b4506973446b6c4ebd90abb3e4fd3f4a720b270 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 5 Jun 2026 16:59:09 +0200 Subject: [PATCH 18/48] chore: adapt changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a66e0195..736a7351 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ All notable changes to this project will be documented in this file. - Document Helm deployed RBAC permissions and remove unnecessary permissions ([#810]). - Internal operator refactoring: introduce dereference() and validate() steps in the reconciler ([#824]). - test: Bump vector-aggregator to 0.55.0, replace /graphql call with gRPC calls ([#826]). +- BREAKING: Removed product-config machinery. This is a breaking change in terms of configuration. + Users relying on the product-config `properties.yaml` file have to set these properties via the CRD ([#830]). ### Deleted @@ -29,6 +31,7 @@ All notable changes to this project will be documented in this file. [#818]: https://github.com/stackabletech/druid-operator/pull/818 [#824]: https://github.com/stackabletech/druid-operator/pull/824 [#826]: https://github.com/stackabletech/druid-operator/pull/826 +[#830]: https://github.com/stackabletech/druid-operator/pull/830 ## [26.3.0] - 2026-03-16 From eb48ad8e29c14540ac1c8dfad133366c93dd722a Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 5 Jun 2026 17:34:32 +0200 Subject: [PATCH 19/48] fix: make config overrides non optional --- extra/crds.yaml | 90 ++++++++++++------- .../src/controller/validate.rs | 14 ++- rust/operator-binary/src/crd/mod.rs | 26 ++---- 3 files changed, 72 insertions(+), 58 deletions(-) diff --git a/extra/crds.yaml b/extra/crds.yaml index 73ad0b48..063e1a98 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -412,21 +412,24 @@ spec: properties: jvm.config: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `jvm.config` file. - nullable: true type: object runtime.properties: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `runtime.properties` file. - nullable: true type: object security.properties: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `security.properties` file. - nullable: true type: object type: object envOverrides: @@ -903,21 +906,24 @@ spec: properties: jvm.config: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `jvm.config` file. - nullable: true type: object runtime.properties: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `runtime.properties` file. - nullable: true type: object security.properties: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `security.properties` file. - nullable: true type: object type: object envOverrides: @@ -2011,21 +2017,24 @@ spec: properties: jvm.config: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `jvm.config` file. - nullable: true type: object runtime.properties: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `runtime.properties` file. - nullable: true type: object security.properties: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `security.properties` file. - nullable: true type: object type: object envOverrides: @@ -2502,21 +2511,24 @@ spec: properties: jvm.config: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `jvm.config` file. - nullable: true type: object runtime.properties: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `runtime.properties` file. - nullable: true type: object security.properties: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `security.properties` file. - nullable: true type: object type: object envOverrides: @@ -3012,21 +3024,24 @@ spec: properties: jvm.config: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `jvm.config` file. - nullable: true type: object runtime.properties: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `runtime.properties` file. - nullable: true type: object security.properties: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `security.properties` file. - nullable: true type: object type: object envOverrides: @@ -3543,21 +3558,24 @@ spec: properties: jvm.config: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `jvm.config` file. - nullable: true type: object runtime.properties: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `runtime.properties` file. - nullable: true type: object security.properties: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `security.properties` file. - nullable: true type: object type: object envOverrides: @@ -4072,21 +4090,24 @@ spec: properties: jvm.config: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `jvm.config` file. - nullable: true type: object runtime.properties: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `runtime.properties` file. - nullable: true type: object security.properties: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `security.properties` file. - nullable: true type: object type: object envOverrides: @@ -4559,21 +4580,24 @@ spec: properties: jvm.config: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `jvm.config` file. - nullable: true type: object runtime.properties: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `runtime.properties` file. - nullable: true type: object security.properties: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `security.properties` file. - nullable: true type: object type: object envOverrides: @@ -5039,21 +5063,24 @@ spec: properties: jvm.config: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `jvm.config` file. - nullable: true type: object runtime.properties: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `runtime.properties` file. - nullable: true type: object security.properties: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `security.properties` file. - nullable: true type: object type: object envOverrides: @@ -5530,21 +5557,24 @@ spec: properties: jvm.config: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `jvm.config` file. - nullable: true type: object runtime.properties: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `runtime.properties` file. - nullable: true type: object security.properties: additionalProperties: + nullable: true type: string + default: {} description: Overrides for the `security.properties` file. - nullable: true type: object type: object envOverrides: diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index d08f9778..1ea9d68c 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -88,16 +88,12 @@ fn key_value_overrides( overrides: &DruidConfigOverrides, file: ConfigFileName, ) -> BTreeMap> { - let kv = match file { - ConfigFileName::RuntimeProperties => overrides.runtime_properties.as_ref(), - ConfigFileName::SecurityProperties => overrides.security_properties.as_ref(), + match file { + ConfigFileName::RuntimeProperties => overrides.runtime_properties.overrides.clone(), + ConfigFileName::SecurityProperties => overrides.security_properties.overrides.clone(), // log4j2.properties is rendered by the logging framework and accepts no key/value overrides. - ConfigFileName::Log4j2Properties => None, - }; - kv.map( - stackable_operator::config_overrides::KeyValueConfigOverrides::as_product_config_overrides, - ) - .unwrap_or_default() + ConfigFileName::Log4j2Properties => BTreeMap::new(), + } } /// Builds the precomputed per-file config for a single rolegroup. Pure assembly: combines the diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 1cd3689a..39ce9628 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -17,7 +17,6 @@ use stackable_operator::{ fragment::{self, Fragment, FromFragment, ValidationError}, merge::Merge, }, - config_overrides::KeyValueConfigOverrides, crd::{ authentication::{core, oidc}, s3, @@ -36,6 +35,7 @@ use stackable_operator::{ shared::time::Duration, status::condition::{ClusterCondition, HasStatusCondition}, utils::{COMMON_BASH_TRAP_FUNCTIONS, crds::raw_object_list_schema}, + v2::config_overrides::KeyValueConfigOverrides, versioned::versioned, }; use strum::{Display, EnumDiscriminants, EnumIter, EnumString, IntoStaticStr}; @@ -116,30 +116,18 @@ const DEFAULT_HISTORICAL_SECRET_LIFETIME: Duration = Duration::from_days_uncheck pub struct DruidConfigOverrides { /// Overrides for the `runtime.properties` file. // File name defined in [`crate::controller::build::properties::ConfigFileName`] - #[serde( - default, - rename = "runtime.properties", - skip_serializing_if = "Option::is_none" - )] - pub runtime_properties: Option, + #[serde(default, rename = "runtime.properties")] + pub runtime_properties: KeyValueConfigOverrides, /// Overrides for the `jvm.config` file. // File name defined in [`crate::controller::build::properties::ConfigFileName`] - #[serde( - default, - rename = "jvm.config", - skip_serializing_if = "Option::is_none" - )] - pub jvm_config: Option, + #[serde(default, rename = "jvm.config")] + pub jvm_config: KeyValueConfigOverrides, /// Overrides for the `security.properties` file. // File name defined in [`crate::controller::build::properties::ConfigFileName`] - #[serde( - default, - rename = "security.properties", - skip_serializing_if = "Option::is_none" - )] - pub security_properties: Option, + #[serde(default, rename = "security.properties")] + pub security_properties: KeyValueConfigOverrides, } #[derive(Snafu, Debug, EnumDiscriminants)] From fbb29efd09b2f728b9d55e0c4235598b6011a697 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 5 Jun 2026 17:40:23 +0200 Subject: [PATCH 20/48] refactor: use v2 keyconfig overrides with merge --- .../src/controller/validate.rs | 35 +++++++------------ rust/operator-binary/src/crd/mod.rs | 18 +--------- 2 files changed, 13 insertions(+), 40 deletions(-) diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 1ea9d68c..84caa5e3 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -9,6 +9,7 @@ use snafu::{ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, commons::product_image_selection::{self, ResolvedProductImage}, + config::merge::Merge, crd::s3, kube::ResourceExt, }; @@ -99,17 +100,19 @@ fn key_value_overrides( /// Builds the precomputed per-file config for a single rolegroup. Pure assembly: combines the /// role-level overrides with the rolegroup-level overrides (rolegroup wins) on top of the /// computed defaults. No behavior change vs. the inline loop body it was extracted from. -#[allow(clippy::too_many_arguments)] fn build_role_group_config( druid: &v1alpha1::DruidCluster, druid_role: &DruidRole, merged_config: CommonRoleGroupConfig, - role_runtime_overrides: &BTreeMap>, - role_security_overrides: &BTreeMap>, + role_config_overrides: &DruidConfigOverrides, role_env_overrides: &HashMap, rg_config_overrides: &DruidConfigOverrides, rg_env_overrides: &HashMap, ) -> DruidRoleGroupConfig { + // Merge the role-level and rolegroup-level config overrides (rolegroup wins over role). + let mut config_overrides = rg_config_overrides.clone(); + config_overrides.merge(role_config_overrides); + // ----- runtime.properties ----- let mut runtime_config = druid.compute_runtime_properties(); if *druid_role == DruidRole::MiddleManager { @@ -117,13 +120,10 @@ fn build_role_group_config( runtime_config.insert(k, v); } runtime_config.extend(runtime_properties::defaults(druid_role)); - // merged user overrides (role <- rolegroup; rolegroup wins) - let mut runtime_overrides = role_runtime_overrides.clone(); - runtime_overrides.extend(key_value_overrides( - rg_config_overrides, + runtime_config.extend(key_value_overrides( + &config_overrides, ConfigFileName::RuntimeProperties, )); - runtime_config.extend(runtime_overrides); // ----- security.properties ----- let mut security_config: BTreeMap> = BTreeMap::new(); @@ -131,11 +131,8 @@ fn build_role_group_config( let (k, v) = middlemanager_indexer_java_opts(); security_config.insert(k, v); } - let mut security_overrides = role_security_overrides.clone(); - security_overrides.extend(key_value_overrides( - rg_config_overrides, - ConfigFileName::SecurityProperties, - )); + let security_overrides = + key_value_overrides(&config_overrides, ConfigFileName::SecurityProperties); security_config.extend(security_properties::build(&security_overrides)); // ----- env ----- @@ -202,14 +199,7 @@ pub fn validate( for druid_role in DruidRole::iter() { // The role-level overrides (role <- rolegroup precedence starts here). let role = druid.get_role(&druid_role); - let role_runtime_overrides = key_value_overrides( - &role.config.config_overrides, - ConfigFileName::RuntimeProperties, - ); - let role_security_overrides = key_value_overrides( - &role.config.config_overrides, - ConfigFileName::SecurityProperties, - ); + let role_config_overrides = &role.config.config_overrides; let role_env_overrides = role.config.env_overrides.clone(); let rolegroups = merged.role_group_names(&druid_role); @@ -232,8 +222,7 @@ pub fn validate( druid, &druid_role, merged_config, - &role_runtime_overrides, - &role_security_overrides, + role_config_overrides, &role_env_overrides, rg_config_overrides, rg_env_overrides, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 39ce9628..b117f70e 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -5,7 +5,6 @@ use security::add_cert_to_jvm_trust_store_cmd; use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ - builder::pod::volume::ListenerOperatorVolumeSourceBuilderError, client::Client, commons::{ affinity::StackableAffinity, @@ -111,7 +110,7 @@ const DEFAULT_ROUTER_SECRET_LIFETIME: Duration = Duration::from_days_unchecked(1 const DEFAULT_HISTORICAL_SECRET_LIFETIME: Duration = Duration::from_days_unchecked(1); /// Typed config override strategies for Druid config files. -#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, Merge, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct DruidConfigOverrides { /// Overrides for the `runtime.properties` file. @@ -158,21 +157,6 @@ pub enum Error { #[snafu(display("fragment validation failure"))] FragmentValidationFailure { source: ValidationError }, - - #[snafu(display("failed to build Labels"))] - LabelBuild { - source: stackable_operator::kvp::LabelError, - }, - - #[snafu(display("failed to build listener volume"))] - BuildListenerVolume { - source: ListenerOperatorVolumeSourceBuilderError, - }, - - #[snafu(display("failed to apply group listener"))] - ApplyGroupListener { - source: stackable_operator::cluster_resources::Error, - }, } #[versioned( From d01042bb7a85e3fb750973cedfc6dbbad74f984b Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 8 Jun 2026 21:16:48 +0200 Subject: [PATCH 21/48] refactor: add namespace and ui to ValidatedCluster --- Cargo.lock | 21 ++-- Cargo.nix | 24 ++-- Cargo.toml | 1 + rust/operator-binary/Cargo.toml | 1 + rust/operator-binary/src/controller.rs | 27 ++-- .../src/controller/build/config_map.rs | 27 ++-- .../src/controller/build/discovery.rs | 23 +--- .../src/controller/validate.rs | 117 ++++++++++++++++-- 8 files changed, 169 insertions(+), 72 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f3b0bf82..6208aca5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -265,9 +265,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "2.12.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84d7ced0ae9557296835c32bf1b1e02b44c746701f898460fb000d7eaa84f00a" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" [[package]] name = "block-buffer" @@ -1101,9 +1101,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes", "itoa", @@ -2240,9 +2240,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ "bytes", "prost-derive", @@ -2250,9 +2250,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", "itertools", @@ -2263,9 +2263,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" dependencies = [ "prost", ] @@ -2990,6 +2990,7 @@ dependencies = [ "strum", "tokio", "tracing", + "uuid", ] [[package]] diff --git a/Cargo.nix b/Cargo.nix index 45874b46..602bc445 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -866,9 +866,9 @@ rec { }; "bitflags" = rec { crateName = "bitflags"; - version = "2.12.1"; + version = "2.13.0"; edition = "2021"; - sha256 = "02phhjm7w380zdh8928zf13cfi1bw2qz2ay36ml2jmwmmv8cxmw4"; + sha256 = "1y239gpvl061rfvav7jds8mjs42kmwi39is7yx5d1qw3hvp8nf5l"; authors = [ "The Rust Project Developers" ]; @@ -3429,9 +3429,9 @@ rec { }; "http" = rec { crateName = "http"; - version = "1.4.1"; + version = "1.4.2"; edition = "2021"; - sha256 = "1l7k2ia57z3q7q3ka497krzps795kd3fymm2k12lr623y4nldrwb"; + sha256 = "09b4p8fiivkg7wm0b59fyrn1jkm7px298ci7zb9igz6n647gaw39"; authors = [ "Alex Crichton " "Carl Lerche " @@ -7334,9 +7334,9 @@ rec { }; "prost" = rec { crateName = "prost"; - version = "0.14.3"; + version = "0.14.4"; edition = "2021"; - sha256 = "0s057z9nzggzy7x4bbsiar852hg7zb81f4z4phcdb0ig99971snj"; + sha256 = "1qas5v5rap45f43v3ja0jngxrrafrkcwl0iw5a3ld1pz2rscd2jj"; authors = [ "Dan Burkert " "Lucio Franco " @@ -7363,9 +7363,9 @@ rec { }; "prost-derive" = rec { crateName = "prost-derive"; - version = "0.14.3"; + version = "0.14.4"; edition = "2021"; - sha256 = "02zvva6kb0pfvlyc4nac6gd37ncjrs8jq5scxcq4nbqkc8wh5ii7"; + sha256 = "1pqa77d7da5pf6ba3kjj7510m5cynz6902ax01ckvr0pfrgv4w5m"; procMacro = true; libName = "prost_derive"; authors = [ @@ -7401,9 +7401,9 @@ rec { }; "prost-types" = rec { crateName = "prost-types"; - version = "0.14.3"; + version = "0.14.4"; edition = "2021"; - sha256 = "1mrxrciryfgi6a0vmrgyj3g27r9hdhlgwkq71cgv3icbvg5w94c9"; + sha256 = "02ivjvc4cwl5bfgjs3l00hwlrk74z8zlg1xcgx60bww8fvf6fjgr"; libName = "prost_types"; authors = [ "Dan Burkert " @@ -9833,6 +9833,10 @@ rec { name = "tracing"; packageId = "tracing"; } + { + name = "uuid"; + packageId = "uuid"; + } ]; buildDependencies = [ { diff --git a/Cargo.toml b/Cargo.toml index 7c5949f8..e68b525c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ snafu = "0.9" strum = { version = "0.28", features = ["derive"] } tokio = { version = "1.52", features = ["full"] } tracing = "0.1" +uuid = "1.23" [patch."https://github.com/stackabletech/operator-rs.git"] stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "smooth-operator" } diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 84f0146b..225c5d44 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -26,6 +26,7 @@ snafu.workspace = true strum.workspace = true tokio.workspace = true tracing.workspace = true +uuid.workspace = true [build-dependencies] built.workspace = true diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 014adf9e..9587f843 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -444,10 +444,9 @@ pub async fn reconcile_druid( if *druid_role == DruidRole::Router { // discovery - for discovery_cm in - build_discovery_configmaps(&validated_cluster, druid, listener) - .await - .context(BuildDiscoveryConfigSnafu)? + for discovery_cm in build_discovery_configmaps(&validated_cluster, listener) + .await + .context(BuildDiscoveryConfigSnafu)? { cluster_resources .add(client, discovery_cm) @@ -936,9 +935,13 @@ pub fn error_policy( #[cfg(test)] mod test { - use std::collections::BTreeMap; + use std::{collections::BTreeMap, str::FromStr}; use rstest::*; + use stackable_operator::v2::types::{ + kubernetes::{NamespaceName, Uid}, + operator::ClusterName, + }; use super::*; use crate::{ @@ -1007,10 +1010,12 @@ mod test { env: BTreeMap::new(), }; - let cluster = ValidatedCluster { - name: druid.name_any(), - image: resolved_product_image.clone(), - cluster_config: ValidatedClusterConfig { + let cluster = ValidatedCluster::new( + ClusterName::from_str(&druid.name_any()).expect("test: valid cluster name"), + NamespaceName::from_str("default").expect("test: valid namespace"), + Uid::from_str("c27b3971-ca72-42c1-80a4-abdfc1db0ddd").expect("test: valid uid"), + resolved_product_image.clone(), + ValidatedClusterConfig { zookeeper_connection_string: "zookeeper-connection-string".to_string(), opa_connection_string: None, s3_connection: None, @@ -1018,8 +1023,8 @@ mod test { druid_tls_security, druid_auth_config: None, }, - role_group_configs: BTreeMap::new(), - }; + BTreeMap::new(), + ); let rolegroup_ref = RoleGroupRef { cluster: ObjectRef::from_obj(&druid), diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 874f0dc1..65c999fa 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -4,11 +4,14 @@ //! from the [`DruidRoleGroupConfig`] precomputed in the validate step; product-config is no //! longer involved. //! +//! Metadata, owner reference and recommended labels are derived entirely from `ValidatedCluster` +//! (which carries the validated name/namespace/uid and implements `Resource`). +//! //! Residual reads from the owning [`v1alpha1::DruidCluster`] remain for things that are not yet -//! modelled on `ValidatedCluster`: the owner reference + recommended labels, the extensions load -//! list (`get_extension_list`), the metadata-database connection -//! (`spec.cluster_config.metadata_database` / `as_metadata_storage_type`), and `get_role` for the -//! jvm.config. Fully removing these is a follow-up. +//! modelled on `ValidatedCluster`: the extensions load list (`get_extension_list`), the +//! metadata-database connection (`spec.cluster_config.metadata_database` / +//! `as_metadata_storage_type`), and `get_role` for the jvm.config. Fully removing these is a +//! follow-up. use std::collections::BTreeMap; @@ -20,7 +23,9 @@ use stackable_operator::{ k8s_openapi::api::core::v1::{ConfigMap, EnvVar}, product_logging::framework::VECTOR_CONFIG_FILE, role_utils::RoleGroupRef, - v2::config_file_writer::to_java_properties_string, + v2::{ + builder::meta::ownerreference_from_resource, config_file_writer::to_java_properties_string, + }, }; use crate::{ @@ -54,11 +59,6 @@ const AUTH_AUTHORIZER_OPA_URI: &str = "druid.auth.authorizer.OpaAuthorizer.opaUr #[derive(Snafu, Debug)] #[allow(clippy::enum_variant_names)] pub enum Error { - #[snafu(display("object is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, - }, - #[snafu(display("failed to build ConfigMap for {}", rolegroup))] BuildRoleGroupConfig { source: stackable_operator::builder::configmap::Error, @@ -299,12 +299,11 @@ pub fn build_rolegroup_config_map( let mut config_map_builder = ConfigMapBuilder::new(); config_map_builder.metadata( ObjectMetaBuilder::new() - .name_and_namespace(owner) + .name_and_namespace(cluster) .name(rolegroup.object_name()) - .ownerreference_from_resource(owner, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) .with_recommended_labels(&build_recommended_labels( - owner, + cluster, DRUID_CONTROLLER_NAME, &cluster.image.app_version_label_value, &rolegroup.role, diff --git a/rust/operator-binary/src/controller/build/discovery.rs b/rust/operator-binary/src/controller/build/discovery.rs index 0e371444..2be36624 100644 --- a/rust/operator-binary/src/controller/build/discovery.rs +++ b/rust/operator-binary/src/controller/build/discovery.rs @@ -6,7 +6,7 @@ use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, crd::listener::v1alpha1::Listener, k8s_openapi::api::core::v1::ConfigMap, - kube::{Resource, ResourceExt}, + v2::builder::meta::ownerreference_from_resource, }; use crate::{ @@ -18,11 +18,6 @@ use crate::{ #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("object is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, - }, - #[snafu(display("failed to get service FQDN"))] NoServiceFqdn, @@ -43,20 +38,14 @@ pub enum Error { /// Builds discovery [`ConfigMap`]s for connecting to a Druid cluster. pub async fn build_discovery_configmaps( cluster: &ValidatedCluster, - owner: &impl Resource, listener: Listener, ) -> Result, Error> { - let name = owner.name_unchecked(); - Ok(vec![build_discovery_configmap( - cluster, owner, &name, listener, - )?]) + Ok(vec![build_discovery_configmap(cluster, listener)?]) } /// Build a discovery [`ConfigMap`] containing information about how to connect to a certain Druid cluster. fn build_discovery_configmap( cluster: &ValidatedCluster, - owner: &impl Resource, - name: &str, listener: Listener, ) -> Result { let router_host = build_listener_connection_string( @@ -74,12 +63,10 @@ fn build_discovery_configmap( ConfigMapBuilder::new() .metadata( ObjectMetaBuilder::new() - .name_and_namespace(owner) - .name(name) - .ownerreference_from_resource(owner, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? + .name_and_namespace(cluster) + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) .with_recommended_labels(&build_recommended_labels( - owner, + cluster, DRUID_CONTROLLER_NAME, &cluster.image.app_version_label_value, &DruidRole::Router.to_string(), diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 84caa5e3..8d9f2c0b 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -3,7 +3,10 @@ //! Synchronously validates inputs that don't require a Kubernetes client. Produces //! [`ValidatedCluster`], consumed by the rest of `reconcile_druid`. -use std::collections::{BTreeMap, HashMap}; +use std::{ + borrow::Cow, + collections::{BTreeMap, HashMap}, +}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ @@ -11,7 +14,15 @@ use stackable_operator::{ commons::product_image_selection::{self, ResolvedProductImage}, config::merge::Merge, crd::s3, - kube::ResourceExt, + kube::{Resource, api::ObjectMeta}, + v2::{ + HasName, HasUid, + controller_utils::{get_cluster_name, get_namespace, get_uid}, + types::{ + kubernetes::{NamespaceName, Uid}, + operator::ClusterName, + }, + }, }; use strum::IntoEnumIterator; @@ -42,6 +53,11 @@ pub enum Error { #[snafu(display("failed to resolve and merge config for role and role group"))] FailedToResolveConfig { source: crate::crd::Error }, + + #[snafu(display("failed to determine the cluster's name, namespace, or uid"))] + ClusterIdentity { + source: stackable_operator::v2::controller_utils::Error, + }, } type Result = std::result::Result; @@ -74,15 +90,92 @@ pub struct ValidatedClusterConfig { /// Synchronous inputs the rest of `reconcile_druid` needs after dereferencing. pub struct ValidatedCluster { - // Currently unused by the build steps, but part of the documented `ValidatedCluster` shape; - // consumed by later tasks. + /// Mirrors `name`/`namespace`/`uid` below so that `ValidatedCluster` can implement + /// [`Resource`] and be passed directly to the metadata/owner-reference builders. + metadata: ObjectMeta, + pub name: ClusterName, + // Read from the mirrored `metadata` in the configmap path (via `name_and_namespace`); the typed + // field is consumed directly when the service/statefulset builders move onto `ValidatedCluster`. #[allow(dead_code)] - pub name: String, + pub namespace: NamespaceName, + pub uid: Uid, pub image: ResolvedProductImage, pub cluster_config: ValidatedClusterConfig, pub role_group_configs: BTreeMap>, } +impl ValidatedCluster { + pub(crate) fn new( + name: ClusterName, + namespace: NamespaceName, + uid: Uid, + image: ResolvedProductImage, + cluster_config: ValidatedClusterConfig, + role_group_configs: BTreeMap>, + ) -> Self { + let metadata = ObjectMeta { + name: Some(name.to_string()), + namespace: Some(namespace.to_string()), + uid: Some(uid.to_string()), + ..ObjectMeta::default() + }; + Self { + metadata, + name, + namespace, + uid, + image, + cluster_config, + role_group_configs, + } + } +} + +// Implementing `Resource` (plus `HasName`/`HasUid`) lets `ValidatedCluster` stand in for the raw +// `DruidCluster` when building child-object metadata and owner references. The identity-bearing +// methods are backed by the `metadata` built in `new`, while kind/group/version/plural delegate to +// the CRD so produced owner references are byte-identical to the previous raw-cluster ones. +impl Resource for ValidatedCluster { + type DynamicType = ::DynamicType; + type Scope = ::Scope; + + fn kind(dt: &Self::DynamicType) -> Cow<'_, str> { + v1alpha1::DruidCluster::kind(dt) + } + + fn group(dt: &Self::DynamicType) -> Cow<'_, str> { + v1alpha1::DruidCluster::group(dt) + } + + fn version(dt: &Self::DynamicType) -> Cow<'_, str> { + v1alpha1::DruidCluster::version(dt) + } + + fn plural(dt: &Self::DynamicType) -> Cow<'_, str> { + v1alpha1::DruidCluster::plural(dt) + } + + fn meta(&self) -> &ObjectMeta { + &self.metadata + } + + fn meta_mut(&mut self) -> &mut ObjectMeta { + &mut self.metadata + } +} + +impl HasName for ValidatedCluster { + fn to_name(&self) -> String { + self.name.to_string() + } +} + +impl HasUid for ValidatedCluster { + fn to_uid(&self) -> Uid { + self.uid.clone() + } +} + /// Returns the user-supplied key/value overrides for the given config file from a /// [`DruidConfigOverrides`], as a product-config style map. fn key_value_overrides( @@ -232,10 +325,16 @@ pub fn validate( role_group_configs.insert(druid_role, group_map); } - Ok(ValidatedCluster { - name: druid.name_any(), + let name = get_cluster_name(druid).context(ClusterIdentitySnafu)?; + let namespace = get_namespace(druid).context(ClusterIdentitySnafu)?; + let uid = get_uid(druid).context(ClusterIdentitySnafu)?; + + Ok(ValidatedCluster::new( + name, + namespace, + uid, image, - cluster_config: ValidatedClusterConfig { + ValidatedClusterConfig { zookeeper_connection_string: dereferenced_objects.zookeeper_connection_string.clone(), opa_connection_string: dereferenced_objects.opa_connection_string.clone(), s3_connection: dereferenced_objects.s3_connection.clone(), @@ -244,5 +343,5 @@ pub fn validate( druid_auth_config, }, role_group_configs, - }) + )) } From 9e63cff0bedd6095e94ec0e523eb169f60418cf5 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 9 Jun 2026 12:07:55 +0200 Subject: [PATCH 22/48] refactor: use non optional properties writer --- Cargo.lock | 51 ++++++------- Cargo.nix | 55 ++++++-------- .../src/authentication/ldap.rs | 40 +++++----- .../operator-binary/src/authentication/mod.rs | 36 ++++----- .../src/authentication/oidc.rs | 53 ++++++------- rust/operator-binary/src/controller.rs | 2 +- .../src/controller/build/config_map.rs | 51 ++++++------- .../build/properties/runtime_properties.rs | 68 ++++++----------- .../build/properties/security_properties.rs | 22 ++---- .../src/controller/validate.rs | 28 ++++--- rust/operator-binary/src/crd/memory.rs | 8 +- rust/operator-binary/src/crd/mod.rs | 14 ++-- rust/operator-binary/src/crd/resource.rs | 10 +-- rust/operator-binary/src/crd/security.rs | 76 ++++++++----------- 14 files changed, 231 insertions(+), 283 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6208aca5..fa91be38 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1468,13 +1468,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.99" +version = "0.3.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" +checksum = "f2025f20d7a4fa7785846e7b63d10a76d3f1cee98ee5cb79ea59703f95e42162" dependencies = [ "cfg-if", "futures-util", - "once_cell", "wasm-bindgen", ] @@ -1530,7 +1529,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "darling", "regex", @@ -2946,7 +2945,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "const-oid", "ecdsa", @@ -2996,7 +2995,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.111.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "base64", "clap", @@ -3040,7 +3039,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "darling", "proc-macro2", @@ -3051,7 +3050,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "jiff", "k8s-openapi", @@ -3068,7 +3067,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.4" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "axum", "clap", @@ -3092,7 +3091,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "kube", "schemars", @@ -3106,7 +3105,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "convert_case", "convert_case_extras", @@ -3124,7 +3123,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" dependencies = [ "arc-swap", "async-trait", @@ -3718,9 +3717,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.2" +version = "1.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" dependencies = [ "js-sys", "wasm-bindgen", @@ -3770,9 +3769,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.122" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" +checksum = "a254a4b10c19a76f09a27640e7ffbf9bc30bf67e16a3bf28aaefa4920fe81563" dependencies = [ "cfg-if", "once_cell", @@ -3783,9 +3782,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.72" +version = "0.4.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" +checksum = "54568702fabf5d4849ce2b90fadfa64168a097eaf4b351ce9df8b687a0086aaf" dependencies = [ "js-sys", "wasm-bindgen", @@ -3793,9 +3792,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.122" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" +checksum = "24a40fc75b0ec6f3746ceb10d36f53a93dcd68a93b11b6445983945d79eba0dc" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3803,9 +3802,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.122" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" +checksum = "908f34bd9b9ce3d4caf07b72dfab63d61504d156856c6bd3cd87fa350cf3985b" dependencies = [ "bumpalo", "proc-macro2", @@ -3816,18 +3815,18 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.122" +version = "0.2.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" +checksum = "7acbf7616c27b194bbb550bf77ed0c2c3e5b7fd1260a93082b95fb7f47959b92" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" -version = "0.3.99" +version = "0.3.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" +checksum = "6e0871acf327f283dc6da28a1696cdc64fb355ba9f935d052021fa77f35cce69" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.nix b/Cargo.nix index 602bc445..174ff591 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4646,9 +4646,9 @@ rec { }; "js-sys" = rec { crateName = "js-sys"; - version = "0.3.99"; + version = "0.3.100"; edition = "2021"; - sha256 = "04azrzsz91gr5s3z0ij36lz0kj9ry4lw3jz0mmbiwb251rsc8aql"; + sha256 = "0qi1wjakyw2rx9wwprcfx77g3lvn1b8n6yvfhj2pgym4swh5y0pj"; libName = "js_sys"; authors = [ "The wasm-bindgen Developers" @@ -4665,11 +4665,6 @@ rec { usesDefaultFeatures = false; features = [ "std" ]; } - { - name = "once_cell"; - packageId = "once_cell"; - usesDefaultFeatures = false; - } { name = "wasm-bindgen"; packageId = "wasm-bindgen"; @@ -4859,7 +4854,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "k8s_version"; @@ -9652,7 +9647,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_certs"; @@ -9864,7 +9859,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_operator"; @@ -10058,7 +10053,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; procMacro = true; @@ -10093,7 +10088,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_shared"; @@ -10174,7 +10169,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_telemetry"; @@ -10284,7 +10279,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_versioned"; @@ -10334,7 +10329,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; procMacro = true; @@ -10402,7 +10397,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; + rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_webhook"; @@ -12449,9 +12444,9 @@ rec { }; "uuid" = rec { crateName = "uuid"; - version = "1.23.2"; + version = "1.23.3"; edition = "2021"; - sha256 = "1xy942s4z0bi8p3441wvd4ry3hx6ry1c7s6fgrr38462xqybhn6j"; + sha256 = "1drddl03gi12vl1s3l2h371dw39plhn9wappp00v707g7h96nk8l"; authors = [ "Ashley Mannix" "Dylan DPC" @@ -12594,9 +12589,9 @@ rec { }; "wasm-bindgen" = rec { crateName = "wasm-bindgen"; - version = "0.2.122"; + version = "0.2.123"; edition = "2021"; - sha256 = "02flix96brsb2r1i3grnikii302iqpdm337kl3xv5lklz5v4bl1y"; + sha256 = "0qqmx07r597gm8lbz8qngvv0phwvpzzyfh3nl84nz9qr1jqs8m52"; libName = "wasm_bindgen"; authors = [ "The wasm-bindgen Developers" @@ -12645,9 +12640,9 @@ rec { }; "wasm-bindgen-futures" = rec { crateName = "wasm-bindgen-futures"; - version = "0.4.72"; + version = "0.4.73"; edition = "2021"; - sha256 = "03qb24gfr072rk8hb69glfdc8yhqqqq2rhy3j5i0ps8sk79dnwwl"; + sha256 = "1bva12h8gdpqkp753czlxabs0s21lvgzm41brr4lhpdzz818fmjl"; libName = "wasm_bindgen_futures"; authors = [ "The wasm-bindgen Developers" @@ -12673,9 +12668,9 @@ rec { }; "wasm-bindgen-macro" = rec { crateName = "wasm-bindgen-macro"; - version = "0.2.122"; + version = "0.2.123"; edition = "2021"; - sha256 = "1inyl55bvdifx7l60q9wl0ivmw7236jg7jqmcqpxhsx3knq52qci"; + sha256 = "1p50xdwmv543b52bc49vm5lcsgd9adpx647bdisg7ihfbg3hz914"; procMacro = true; libName = "wasm_bindgen_macro"; authors = [ @@ -12697,9 +12692,9 @@ rec { }; "wasm-bindgen-macro-support" = rec { crateName = "wasm-bindgen-macro-support"; - version = "0.2.122"; + version = "0.2.123"; edition = "2021"; - sha256 = "0pjw5kc2mbfz59agk5l21kh4hxzp94rygdvsnr4f3z6b5hv4g419"; + sha256 = "0nwqyc63byl7rp9nnv45av8h85fncfmxywkvy35d9qwwkfyk93wh"; libName = "wasm_bindgen_macro_support"; authors = [ "The wasm-bindgen Developers" @@ -12733,10 +12728,10 @@ rec { }; "wasm-bindgen-shared" = rec { crateName = "wasm-bindgen-shared"; - version = "0.2.122"; + version = "0.2.123"; edition = "2021"; links = "wasm_bindgen"; - sha256 = "0ds4mmfqvxwc5fp33hn0jblf0f6b4lghrd9mpkls66zic4n9p4ls"; + sha256 = "14lvjm3pzywm5c4962i6s5zmngic1knpggshnnxr9c97dihzgjvs"; libName = "wasm_bindgen_shared"; authors = [ "The wasm-bindgen Developers" @@ -12751,9 +12746,9 @@ rec { }; "web-sys" = rec { crateName = "web-sys"; - version = "0.3.99"; + version = "0.3.100"; edition = "2021"; - sha256 = "0dilfvl9jnyhi4skl6cry9wc300r693j0w82jjbq8yy3rx0i8qkd"; + sha256 = "0sffbkrpgyi1402mv4wzp9av6ky6rnb1d2m2dpf87wi7yfn7223f"; libName = "web_sys"; authors = [ "The wasm-bindgen Developers" diff --git a/rust/operator-binary/src/authentication/ldap.rs b/rust/operator-binary/src/authentication/ldap.rs index efc99471..c61cc5f8 100644 --- a/rust/operator-binary/src/authentication/ldap.rs +++ b/rust/operator-binary/src/authentication/ldap.rs @@ -13,28 +13,26 @@ use crate::crd::security::{STACKABLE_TLS_DIR, TLS_STORE_PASSWORD, add_cert_to_tr fn add_authenticator_config( provider: &ldap::v1alpha1::AuthenticationProvider, - config: &mut BTreeMap>, + config: &mut BTreeMap, ) -> Result<(), Error> { config.insert( "druid.auth.authenticator.Ldap.type".to_string(), - Some("basic".to_string()), + "basic".to_string(), ); config.insert( "druid.auth.authenticator.Ldap.enableCacheNotifications".to_string(), - Some("true".to_string()), + "true".to_string(), ); config.insert( "druid.auth.authenticator.Ldap.credentialsValidator.type".to_string(), - Some("ldap".to_string()), + "ldap".to_string(), ); config.insert( "druid.auth.authenticator.Ldap.credentialsValidator.url".to_string(), - Some( - provider - .endpoint_url() - .context(ConstructLdapEndpointUrlSnafu)? - .into(), - ), + provider + .endpoint_url() + .context(ConstructLdapEndpointUrlSnafu)? + .into(), ); if let Some((ldap_bind_user_path, ldap_bind_password_path)) = @@ -42,52 +40,52 @@ fn add_authenticator_config( { config.insert( "druid.auth.authenticator.Ldap.credentialsValidator.bindUser".to_string(), - Some(format!("${{file:UTF-8:{ldap_bind_user_path}}}").to_string()), + format!("${{file:UTF-8:{ldap_bind_user_path}}}").to_string(), ); config.insert( "druid.auth.authenticator.Ldap.credentialsValidator.bindPassword".to_string(), - Some(format!("${{file:UTF-8:{ldap_bind_password_path}}}").to_string()), + format!("${{file:UTF-8:{ldap_bind_password_path}}}").to_string(), ); } config.insert( "druid.auth.authenticator.Ldap.credentialsValidator.baseDn".to_string(), - Some(provider.search_base.to_string()), + provider.search_base.to_string(), ); config.insert( "druid.auth.authenticator.Ldap.credentialsValidator.userAttribute".to_string(), - Some(provider.ldap_field_names.uid.to_string()), + provider.ldap_field_names.uid.to_string(), ); config.insert( "druid.auth.authenticator.Ldap.credentialsValidator.userSearch".to_string(), - Some(provider.search_filter.to_string()), + provider.search_filter.to_string(), ); config.insert( "druid.auth.authenticator.Ldap.authorizerName".to_string(), - Some("LdapAuthorizer".to_string()), + "LdapAuthorizer".to_string(), ); config.insert( "druid.auth.authenticatorChain".to_string(), - Some(r#"["DruidSystemAuthenticator", "Ldap"]"#.to_string()), + r#"["DruidSystemAuthenticator", "Ldap"]"#.to_string(), ); Ok(()) } -fn add_authorizer_config(config: &mut BTreeMap>) { +fn add_authorizer_config(config: &mut BTreeMap) { config.insert( "druid.auth.authorizers".to_string(), - Some(r#"["LdapAuthorizer", "DruidSystemAuthorizer"]"#.to_string()), + r#"["LdapAuthorizer", "DruidSystemAuthorizer"]"#.to_string(), ); config.insert( "druid.auth.authorizer.LdapAuthorizer.type".to_string(), - Some(r#"allowAll"#.to_string()), + r#"allowAll"#.to_string(), ); } pub fn generate_runtime_properties_config( provider: &ldap::v1alpha1::AuthenticationProvider, - config: &mut BTreeMap>, + config: &mut BTreeMap, ) -> Result<(), Error> { add_authenticator_config(provider, config)?; add_authorizer_config(config); diff --git a/rust/operator-binary/src/authentication/mod.rs b/rust/operator-binary/src/authentication/mod.rs index 548c9213..dd933528 100644 --- a/rust/operator-binary/src/authentication/mod.rs +++ b/rust/operator-binary/src/authentication/mod.rs @@ -89,8 +89,8 @@ impl DruidAuthenticationConfig { pub fn generate_runtime_properties_config( &self, role: &DruidRole, - ) -> Result>, Error> { - let mut config: BTreeMap> = BTreeMap::new(); + ) -> Result, Error> { + let mut config: BTreeMap = BTreeMap::new(); match self { DruidAuthenticationConfig::Ldap { provider, .. } => { @@ -107,16 +107,13 @@ impl DruidAuthenticationConfig { } /// Creates authentication config that is required by LDAP and OIDC and doesn't depend on user input. - fn generate_common_runtime_properties_config( - &self, - config: &mut BTreeMap>, - ) { + fn generate_common_runtime_properties_config(&self, config: &mut BTreeMap) { self.add_druid_system_authenticator_config(config); self.add_escalator_config(config); config.insert( "druid.auth.authorizer.DruidSystemAuthorizer.type".to_string(), - Some(r#"allowAll"#.to_string()), + r#"allowAll"#.to_string(), ); } @@ -176,50 +173,47 @@ impl DruidAuthenticationConfig { /// When using LDAP or OIDC the DruidSystemAuthenticator is always tried first and skipped if no basic auth credentials were supplied. /// We don't want to create an admin user for the internal authentication, so this line is left out of the config: /// # druid.auth.authenticator.DruidSystemAuthenticator.initialAdminPassword: XXX - fn add_druid_system_authenticator_config(&self, config: &mut BTreeMap>) { + fn add_druid_system_authenticator_config(&self, config: &mut BTreeMap) { config.insert( "druid.auth.authenticator.DruidSystemAuthenticator.type".to_string(), - Some("basic".to_string()), + "basic".to_string(), ); config.insert( "druid.auth.authenticator.DruidSystemAuthenticator.credentialsValidator.type" .to_string(), - Some("metadata".to_string()), + "metadata".to_string(), ); config.insert( "druid.auth.authenticator.DruidSystemAuthenticator.initialInternalClientPassword" .to_string(), - Some(format!("${{env:{INTERNAL_INITIAL_CLIENT_PASSWORD_ENV}}}").to_string()), + format!("${{env:{INTERNAL_INITIAL_CLIENT_PASSWORD_ENV}}}").to_string(), ); config.insert( "druid.auth.authenticator.DruidSystemAuthenticator.authorizerName".to_string(), - Some("DruidSystemAuthorizer".to_string()), + "DruidSystemAuthorizer".to_string(), ); config.insert( "druid.auth.authenticator.DruidSystemAuthenticator.skipOnFailure".to_string(), - Some("true".to_string()), + "true".to_string(), ); } /// Creates the escalator config: . /// This configures Druid processes to use the basic auth authentication added in `add_druid_system_authenticator_config` for internal communication. - fn add_escalator_config(&self, config: &mut BTreeMap>) { - config.insert( - "druid.escalator.type".to_string(), - Some("basic".to_string()), - ); + fn add_escalator_config(&self, config: &mut BTreeMap) { + config.insert("druid.escalator.type".to_string(), "basic".to_string()); config.insert( "druid.escalator.internalClientUsername".to_string(), - Some("druid_system".to_string()), + "druid_system".to_string(), ); config.insert( "druid.escalator.internalClientPassword".to_string(), - Some(format!("${{env:{ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV}}}").to_string()), + format!("${{env:{ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV}}}").to_string(), ); config.insert( "druid.escalator.authorizerName".to_string(), - Some("DruidSystemAuthorizer".to_string()), + "DruidSystemAuthorizer".to_string(), ); } } diff --git a/rust/operator-binary/src/authentication/oidc.rs b/rust/operator-binary/src/authentication/oidc.rs index e37e45a8..3aeca1be 100644 --- a/rust/operator-binary/src/authentication/oidc.rs +++ b/rust/operator-binary/src/authentication/oidc.rs @@ -22,7 +22,7 @@ pub type DruidClientAuthenticationOptions = fn add_authenticator_config( provider: &oidc::v1alpha1::AuthenticationProvider, oidc: &DruidClientAuthenticationOptions, - config: &mut BTreeMap>, + config: &mut BTreeMap, ) -> Result<(), Error> { let well_known_url = &provider .well_known_config_url() @@ -38,36 +38,33 @@ fn add_authenticator_config( config.insert( "druid.auth.authenticator.Oidc.type".to_string(), - Some(r#"pac4j"#.to_string()), + r#"pac4j"#.to_string(), ); config.insert( "druid.auth.authenticator.Oidc.authorizerName".to_string(), - Some(r#"OidcAuthorizer"#.to_string()), + r#"OidcAuthorizer"#.to_string(), ); config.insert( "druid.auth.pac4j.cookiePassphrase".to_string(), - Some(format!("${{env:{COOKIE_PASSPHRASE_ENV}}}").to_string()), + format!("${{env:{COOKIE_PASSPHRASE_ENV}}}").to_string(), ); config.insert( "druid.auth.pac4j.oidc.clientID".to_string(), - Some(format!("${{env:{oidc_client_id_env}}}").to_string()), + format!("${{env:{oidc_client_id_env}}}").to_string(), ); config.insert( "druid.auth.pac4j.oidc.clientSecret".to_string(), - Some(format!("${{env:{oidc_client_secret_env}}}").to_string()), + format!("${{env:{oidc_client_secret_env}}}").to_string(), ); config.insert( "druid.auth.pac4j.oidc.discoveryURI".to_string(), - Some(well_known_url.to_string()), + well_known_url.to_string(), ); config.insert( "druid.auth.pac4j.oidc.oidcClaim".to_string(), - Some(provider.principal_claim.to_string()), - ); - config.insert( - "druid.auth.pac4j.oidc.scope".to_string(), - Some(scopes.join(" ")), + provider.principal_claim.to_string(), ); + config.insert("druid.auth.pac4j.oidc.scope".to_string(), scopes.join(" ")); let method_string = oidc .product_specific_fields @@ -75,25 +72,25 @@ fn add_authenticator_config( .as_ref(); config.insert( "druid.auth.pac4j.oidc.clientAuthenticationMethod".to_string(), - Some(method_string.to_string()), + method_string.to_string(), ); config.insert( "druid.auth.authenticatorChain".to_string(), - Some(r#"["DruidSystemAuthenticator", "Oidc"]"#.to_string()), + r#"["DruidSystemAuthenticator", "Oidc"]"#.to_string(), ); Ok(()) } -fn add_authorizer_config(config: &mut BTreeMap>) { +fn add_authorizer_config(config: &mut BTreeMap) { config.insert( "druid.auth.authorizers".to_string(), - Some(r#"["OidcAuthorizer", "DruidSystemAuthorizer"]"#.to_string()), + r#"["OidcAuthorizer", "DruidSystemAuthorizer"]"#.to_string(), ); config.insert( "druid.auth.authorizer.OidcAuthorizer.type".to_string(), - Some(r#"allowAll"#.to_string()), + r#"allowAll"#.to_string(), ); } @@ -104,13 +101,13 @@ pub fn generate_runtime_properties_config( provider: &oidc::v1alpha1::AuthenticationProvider, oidc: &DruidClientAuthenticationOptions, role: &DruidRole, - config: &mut BTreeMap>, + config: &mut BTreeMap, ) -> Result<(), Error> { match role { DruidRole::MiddleManager => { config.insert( "druid.auth.authenticatorChain".to_string(), - Some(r#"["DruidSystemAuthenticator"]"#.to_string()), + r#"["DruidSystemAuthenticator"]"#.to_string(), ); } _ => { @@ -215,35 +212,35 @@ mod tests { assert_eq!( properties.get("druid.auth.authenticator.Oidc.type"), - Some(&Some("pac4j".to_owned())) + Some(&"pac4j".to_owned()) ); assert_eq!( properties.get("druid.auth.pac4j.oidc.oidcClaim"), - Some(&Some("preferred_username".to_owned())) + Some(&"preferred_username".to_owned()) ); assert_eq!( properties.get("druid.auth.pac4j.oidc.scope"), - Some(&Some("openid".to_owned())) + Some(&"openid".to_owned()) ); assert_eq!( properties.get("druid.auth.authenticator.Oidc.authorizerName"), - Some(&Some("OidcAuthorizer".to_owned())) + Some(&"OidcAuthorizer".to_owned()) ); assert_eq!( properties.get("druid.auth.authenticatorChain"), - Some(&Some("[\"DruidSystemAuthenticator\", \"Oidc\"]".to_owned())) + Some(&"[\"DruidSystemAuthenticator\", \"Oidc\"]".to_owned()) ); assert_eq!( properties.get("druid.auth.pac4j.oidc.discoveryURI"), - Some(&Some( - "https://keycloak.mycorp.org/realms/sdp/.well-known/openid-configuration" + Some( + &"https://keycloak.mycorp.org/realms/sdp/.well-known/openid-configuration" .to_owned() - )) + ) ); assert_eq!( properties.get("druid.auth.pac4j.oidc.clientAuthenticationMethod"), - Some(&Some("client_secret_post".to_owned())) + Some(&"client_secret_post".to_owned()) ); assert!(properties.contains_key("druid.auth.pac4j.oidc.clientID")); assert!(properties.contains_key("druid.auth.pac4j.oidc.clientSecret")); diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 9587f843..8667d743 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -1052,7 +1052,7 @@ mod test { stackable_operator::v2::config_file_writer::to_java_properties_string( vec![( &PROP_SEGMENT_CACHE_LOCATIONS.to_string(), - &Some(expected_druid_segment_cache_property.to_string()), + &expected_druid_segment_cache_property.to_string(), )] .into_iter(), ) diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 65c999fa..1492dd9d 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -135,7 +135,7 @@ pub fn build_rolegroup_config_map( // ----- runtime.properties ----- { - let mut conf: BTreeMap> = Default::default(); + let mut conf: BTreeMap = Default::default(); // Add any properties derived from storage manifests, such as segment cache locations. // This has to be done here since there is no other suitable place for it. @@ -150,44 +150,37 @@ pub fn build_rolegroup_config_map( // but might need to be revisited in the future conf.insert( ZOOKEEPER_CONNECTION_STRING.to_string(), - Some(zk_connstr.to_string()), + zk_connstr.to_string(), ); conf.insert( EXTENSIONS_LOADLIST.to_string(), - Some(build_string_list(&get_extension_list( + build_string_list(&get_extension_list( owner, druid_tls_security, druid_auth_config, - ))), + )), ); if let Some(opa_str) = opa_connstr { - conf.insert( - AUTH_AUTHORIZER_OPA_URI.to_string(), - Some(opa_str.to_string()), - ); + conf.insert(AUTH_AUTHORIZER_OPA_URI.to_string(), opa_str.to_string()); }; conf.insert( crate::crd::database::METADATA_STORAGE_TYPE.to_string(), - Some( - owner - .spec - .cluster_config - .metadata_database - .as_metadata_storage_type() - .to_string(), - ), + owner + .spec + .cluster_config + .metadata_database + .as_metadata_storage_type() + .to_string(), ); conf.insert( crate::crd::database::METADATA_STORAGE_CONNECTOR_CONNECT_URI.to_string(), - Some( - metadata_database_connection_details - .connection_url - .to_string(), - ), + metadata_database_connection_details + .connection_url + .to_string(), ); if let Some(EnvVar { @@ -197,7 +190,7 @@ pub fn build_rolegroup_config_map( { conf.insert( crate::crd::database::METADATA_STORAGE_USER.to_string(), - Some(format!("${{env:{username_env_name}}}",)), + format!("${{env:{username_env_name}}}",), ); } @@ -208,7 +201,7 @@ pub fn build_rolegroup_config_map( { conf.insert( crate::crd::database::METADATA_STORAGE_PASSWORD.to_string(), - Some(format!("${{env:{password_env_name}}}",)), + format!("${{env:{password_env_name}}}",), ); } @@ -226,28 +219,30 @@ pub fn build_rolegroup_config_map( conf.insert( S3_ENDPOINT_URL.to_string(), - Some(s3.endpoint().context(ConfigureS3Snafu)?.to_string()), + s3.endpoint().context(ConfigureS3Snafu)?.to_string(), ); if let Some((access_key_file, secret_key_file)) = s3.credentials_mount_paths() { conf.insert( S3_ACCESS_KEY.to_string(), - Some(format!("${{file:UTF-8:{access_key_file}}}")), + format!("${{file:UTF-8:{access_key_file}}}"), ); conf.insert( S3_SECRET_KEY.to_string(), - Some(format!("${{file:UTF-8:{secret_key_file}}}")), + format!("${{file:UTF-8:{secret_key_file}}}"), ); } conf.insert( S3_PATH_STYLE_ACCESS.to_string(), - Some((s3.access_style == s3::v1alpha1::S3AccessStyle::Path).to_string()), + (s3.access_style == s3::v1alpha1::S3AccessStyle::Path).to_string(), ); } + // When no deep-storage bucket is set (e.g. HDFS deep storage) this renders an empty + // `druid.storage.bucket=`, matching the previous `None` -> empty-value behavior. conf.insert( DS_BUCKET.to_string(), - deep_storage_bucket_name.map(str::to_string), + deep_storage_bucket_name.unwrap_or_default().to_string(), ); // add tls encryption / auth properties diff --git a/rust/operator-binary/src/controller/build/properties/runtime_properties.rs b/rust/operator-binary/src/controller/build/properties/runtime_properties.rs index 7adb1381..f2f91968 100644 --- a/rust/operator-binary/src/controller/build/properties/runtime_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/runtime_properties.rs @@ -68,8 +68,8 @@ const ROUTER: &[(&str, &str)] = &[ ("druid.router.http.numConnections", "25"), ]; -/// Static `recommendedValues` defaults for a role, as `key -> Some(value)`. -pub fn defaults(role: &DruidRole) -> BTreeMap> { +/// Static `recommendedValues` defaults for a role, as `key -> value`. +pub fn defaults(role: &DruidRole) -> BTreeMap { let role_specific: &[(&str, &str)] = match role { DruidRole::Broker => BROKER, DruidRole::Coordinator => COORDINATOR, @@ -80,7 +80,7 @@ pub fn defaults(role: &DruidRole) -> BTreeMap> { ALL_ROLES .iter() .chain(role_specific.iter()) - .map(|(k, v)| (k.to_string(), Some(v.to_string()))) + .map(|(k, v)| (k.to_string(), v.to_string())) .collect() } @@ -101,27 +101,21 @@ mod tests { DruidRole::Router, ] { let p = defaults(&role); - assert_eq!(p["druid.emitter"], Some("prometheus".to_string())); - assert_eq!( - p["druid.emitter.prometheus.namespace"], - Some("druid".to_string()) - ); + assert_eq!(p["druid.emitter"], "prometheus".to_string()); + assert_eq!(p["druid.emitter.prometheus.namespace"], "druid".to_string()); assert_eq!( p["druid.emitter.prometheus.strategy"], - Some("exporter".to_string()) + "exporter".to_string() ); assert_eq!( p["druid.monitoring.monitors"], - Some("[\"org.apache.druid.java.util.metrics.JvmMonitor\"]".to_string()) + "[\"org.apache.druid.java.util.metrics.JvmMonitor\"]".to_string() ); assert_eq!( p["druid.indexer.logs.directory"], - Some("/stackable/var/druid/indexing-logs".to_string()) - ); - assert_eq!( - p["druid.startup.logging.logProperties"], - Some("true".to_string()) + "/stackable/var/druid/indexing-logs".to_string() ); + assert_eq!(p["druid.startup.logging.logProperties"], "true".to_string()); } } @@ -140,35 +134,26 @@ mod tests { let p = defaults(&DruidRole::Coordinator); assert_eq!( p["druid.coordinator.asOverlord.enabled"], - Some("true".to_string()) + "true".to_string() ); assert_eq!( p["druid.coordinator.asOverlord.overlordService"], - Some("druid/overlord".to_string()) - ); - assert_eq!(p["druid.coordinator.period"], Some("PT20S".to_string())); - assert_eq!(p["druid.coordinator.startDelay"], Some("PT20S".to_string())); - assert_eq!( - p["druid.indexer.queue.startDelay"], - Some("PT20S".to_string()) - ); - assert_eq!(p["druid.indexer.runner.type"], Some("remote".to_string())); - assert_eq!( - p["druid.indexer.storage.type"], - Some("metadata".to_string()) + "druid/overlord".to_string() ); + assert_eq!(p["druid.coordinator.period"], "PT20S".to_string()); + assert_eq!(p["druid.coordinator.startDelay"], "PT20S".to_string()); + assert_eq!(p["druid.indexer.queue.startDelay"], "PT20S".to_string()); + assert_eq!(p["druid.indexer.runner.type"], "remote".to_string()); + assert_eq!(p["druid.indexer.storage.type"], "metadata".to_string()); } #[test] fn historical_defaults_match_snapshot() { let p = defaults(&DruidRole::Historical); - assert_eq!( - p["druid.historical.cache.useCache"], - Some("true".to_string()) - ); + assert_eq!(p["druid.historical.cache.useCache"], "true".to_string()); assert_eq!( p["druid.historical.cache.populateCache"], - Some("true".to_string()) + "true".to_string() ); } @@ -176,18 +161,16 @@ mod tests { fn middlemanager_javaopts_exact() { let p = defaults(&DruidRole::MiddleManager); assert_eq!( - p["druid.indexer.runner.javaOpts"].as_deref(), - Some( - "-server -Xms256m -Xmx256m -XX:MaxDirectMemorySize=300m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager" - ) + p["druid.indexer.runner.javaOpts"], + "-server -Xms256m -Xmx256m -XX:MaxDirectMemorySize=300m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager" ); assert_eq!( p["druid.indexer.task.baseTaskDir"], - Some("/stackable/var/druid/task".to_string()) + "/stackable/var/druid/task".to_string() ); assert_eq!( p["druid.indexer.task.hadoopWorkingPath"], - Some("/stackable/var/druid/hadoop-tmp".to_string()) + "/stackable/var/druid/hadoop-tmp".to_string() ); } @@ -196,11 +179,8 @@ mod tests { let p = defaults(&DruidRole::Router); assert_eq!( p["druid.router.managementProxy.enabled"], - Some("true".to_string()) - ); - assert_eq!( - p["druid.router.http.numConnections"], - Some("25".to_string()) + "true".to_string() ); + assert_eq!(p["druid.router.http.numConnections"], "25".to_string()); } } diff --git a/rust/operator-binary/src/controller/build/properties/security_properties.rs b/rust/operator-binary/src/controller/build/properties/security_properties.rs index eb73e681..1bed42a2 100644 --- a/rust/operator-binary/src/controller/build/properties/security_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/security_properties.rs @@ -13,15 +13,15 @@ const DEFAULT_NETWORKADDRESS_CACHE_NEGATIVE_TTL: &str = "0"; /// `overrides` are the user's merged (role <- rolegroup) `security.properties` /// config overrides, as returned by `DruidConfigOverrides::get_key_value_overrides`. /// Override values win. -pub fn build(overrides: &BTreeMap>) -> BTreeMap> { - let mut props: BTreeMap> = BTreeMap::new(); +pub fn build(overrides: &BTreeMap) -> BTreeMap { + let mut props: BTreeMap = BTreeMap::new(); props.insert( NETWORKADDRESS_CACHE_TTL.to_string(), - Some(DEFAULT_NETWORKADDRESS_CACHE_TTL.to_string()), + DEFAULT_NETWORKADDRESS_CACHE_TTL.to_string(), ); props.insert( NETWORKADDRESS_CACHE_NEGATIVE_TTL.to_string(), - Some(DEFAULT_NETWORKADDRESS_CACHE_NEGATIVE_TTL.to_string()), + DEFAULT_NETWORKADDRESS_CACHE_NEGATIVE_TTL.to_string(), ); props.extend(overrides.iter().map(|(k, v)| (k.clone(), v.clone()))); props @@ -38,21 +38,15 @@ mod tests { #[test] fn defaults_match_snapshot() { let props = build(&BTreeMap::new()); - assert_eq!(props["networkaddress.cache.ttl"], Some("30".to_string())); - assert_eq!( - props["networkaddress.cache.negative.ttl"], - Some("0".to_string()) - ); + assert_eq!(props["networkaddress.cache.ttl"], "30".to_string()); + assert_eq!(props["networkaddress.cache.negative.ttl"], "0".to_string()); assert_eq!(props.len(), 2); } #[test] fn override_wins() { - let ov = BTreeMap::from([( - "networkaddress.cache.ttl".to_string(), - Some("60".to_string()), - )]); + let ov = BTreeMap::from([("networkaddress.cache.ttl".to_string(), "60".to_string())]); let props = build(&ov); - assert_eq!(props["networkaddress.cache.ttl"], Some("60".to_string())); + assert_eq!(props["networkaddress.cache.ttl"], "60".to_string()); } } diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 8d9f2c0b..b2225132 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -71,9 +71,9 @@ pub type RoleGroupName = String; pub struct DruidRoleGroupConfig { pub merged_config: CommonRoleGroupConfig, /// The runtime.properties "validated config" (compute_files + recommended defaults + merged overrides). - pub runtime_config: BTreeMap>, + pub runtime_config: BTreeMap, /// The security.properties "validated config". - pub security_config: BTreeMap>, + pub security_config: BTreeMap, /// Merged env overrides (role <- rolegroup). compute_env is empty for druid. pub env: BTreeMap, } @@ -177,17 +177,25 @@ impl HasUid for ValidatedCluster { } /// Returns the user-supplied key/value overrides for the given config file from a -/// [`DruidConfigOverrides`], as a product-config style map. +/// [`DruidConfigOverrides`]. +/// +/// The CRD override map allows a value-less key (`someKey:` / null in YAML), modelled as +/// `Option`. We flatten `None` to an empty string here, matching how the Java +/// properties writer rendered a missing value (`key=`), so the rest of the pipeline can work +/// with plain `String` values. fn key_value_overrides( overrides: &DruidConfigOverrides, file: ConfigFileName, -) -> BTreeMap> { - match file { +) -> BTreeMap { + let raw = match file { ConfigFileName::RuntimeProperties => overrides.runtime_properties.overrides.clone(), ConfigFileName::SecurityProperties => overrides.security_properties.overrides.clone(), // log4j2.properties is rendered by the logging framework and accepts no key/value overrides. ConfigFileName::Log4j2Properties => BTreeMap::new(), - } + }; + raw.into_iter() + .map(|(k, v)| (k, v.unwrap_or_default())) + .collect() } /// Builds the precomputed per-file config for a single rolegroup. Pure assembly: combines the @@ -219,7 +227,7 @@ fn build_role_group_config( )); // ----- security.properties ----- - let mut security_config: BTreeMap> = BTreeMap::new(); + let mut security_config: BTreeMap = BTreeMap::new(); if *druid_role == DruidRole::MiddleManager { let (k, v) = middlemanager_indexer_java_opts(); security_config.insert(k, v); @@ -247,14 +255,14 @@ const INDEXER_JAVA_OPTS: &str = "druid.indexer.runner.javaOptsArray"; /// The `druid.indexer.runner.javaOptsArray` entry that `MiddleManagerConfigFragment::compute_files` /// adds for *every* file (runtime.properties and security.properties). -fn middlemanager_indexer_java_opts() -> (String, Option) { +fn middlemanager_indexer_java_opts() -> (String, String) { ( INDEXER_JAVA_OPTS.to_string(), - Some(build_string_list(&[ + build_string_list(&[ format!("-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}"), format!("-Djavax.net.ssl.trustStorePassword={STACKABLE_TRUST_STORE_PASSWORD}"), "-Djavax.net.ssl.trustStoreType=pkcs12".to_owned(), - ])), + ]), ) } diff --git a/rust/operator-binary/src/crd/memory.rs b/rust/operator-binary/src/crd/memory.rs index 376b6b57..892bcdc8 100644 --- a/rust/operator-binary/src/crd/memory.rs +++ b/rust/operator-binary/src/crd/memory.rs @@ -122,18 +122,18 @@ impl HistoricalDerivedSettings { } /// Adds derived runtime settings to the given config - pub fn add_settings(&self, config: &mut BTreeMap>) { + pub fn add_settings(&self, config: &mut BTreeMap) { config.insert( PROCESSING_NUM_THREADS.to_owned(), - Some(self.num_threads().to_string()), + self.num_threads().to_string(), ); config.insert( PROCESSING_NUM_MERGE_BUFFERS.to_owned(), - Some(self.num_merge_buffers().to_string()), + self.num_merge_buffers().to_string(), ); config.insert( PROCESSING_BUFFER_SIZE_BYTES.to_owned(), - Some(format_for_druid(&self.buffer_size())), + format_for_druid(&self.buffer_size()), ); } } diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index b117f70e..cd0182f2 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -319,32 +319,32 @@ impl HasStatusCondition for v1alpha1::DruidCluster { } impl v1alpha1::DruidCluster { - pub fn compute_runtime_properties(&self) -> BTreeMap> { + pub fn compute_runtime_properties(&self) -> BTreeMap { let mut result = BTreeMap::new(); // OPA if let Some(DruidAuthorization { opa: _ }) = &self.spec.cluster_config.authorization { result.insert( AUTH_AUTHORIZERS.to_string(), - Some(AUTH_AUTHORIZERS_VALUE.to_string()), + AUTH_AUTHORIZERS_VALUE.to_string(), ); result.insert( AUTH_AUTHORIZER_OPA_TYPE.to_string(), - Some(AUTH_AUTHORIZER_OPA_TYPE_VALUE.to_string()), + AUTH_AUTHORIZER_OPA_TYPE_VALUE.to_string(), ); // The opaUri still needs to be set, but that requires a discovery config map and is handled in the controller.rs } // deep storage result.insert( DS_TYPE.to_string(), - Some(self.spec.cluster_config.deep_storage.to_string()), + self.spec.cluster_config.deep_storage.to_string(), ); match self.spec.cluster_config.deep_storage.clone() { DeepStorageSpec::Hdfs(hdfs) => { - result.insert(DS_DIRECTORY.to_string(), Some(hdfs.directory)); + result.insert(DS_DIRECTORY.to_string(), hdfs.directory); } DeepStorageSpec::S3(s3_spec) => { if let Some(key) = &s3_spec.base_key { - result.insert(DS_BASE_KEY.to_string(), Some(key.to_string())); + result.insert(DS_BASE_KEY.to_string(), key.to_string()); } // bucket information (name, connection) needs to be resolved first, // that is done directly in the controller @@ -352,7 +352,7 @@ impl v1alpha1::DruidCluster { } // metrics - result.insert(PROMETHEUS_PORT.to_string(), Some(METRICS_PORT.to_string())); + result.insert(PROMETHEUS_PORT.to_string(), METRICS_PORT.to_string()); result } diff --git a/rust/operator-binary/src/crd/resource.rs b/rust/operator-binary/src/crd/resource.rs index e2d6bfb2..148eb38a 100644 --- a/rust/operator-binary/src/crd/resource.rs +++ b/rust/operator-binary/src/crd/resource.rs @@ -78,7 +78,7 @@ impl RoleResource { /// Currently it only adds historical-specific configs for direct memory buffers, thread counts and segment cache. pub fn update_druid_config_file( &self, - config: &mut BTreeMap>, + config: &mut BTreeMap, ) -> Result<(), Error> { match self { Self::Historical(r) => { @@ -87,10 +87,10 @@ impl RoleResource { config .entry(PROP_SEGMENT_CACHE_LOCATIONS.to_string()) .or_insert_with(|| { - Some(format!( + format!( r#"[{{"path":"{}","maxSize":"{}","freeSpacePercent":"{}"}}]"#, PATH_SEGMENT_CACHE, capacity.0, free_percentage - )) + ) }); let settings = @@ -500,7 +500,7 @@ mod test { assert!(got.contains_key(PROP_SEGMENT_CACHE_LOCATIONS)); let value = got.get(PROP_SEGMENT_CACHE_LOCATIONS).unwrap(); - let expected = Some(r#"[{"path":"/stackable/var/druid/segment-cache","maxSize":"5g","freeSpacePercent":"3"}]"#.to_string()); + let expected = r#"[{"path":"/stackable/var/druid/segment-cache","maxSize":"5g","freeSpacePercent":"3"}]"#.to_string(); assert_eq!(value, &expected, "primary"); // ---------- secondary role group @@ -514,7 +514,7 @@ mod test { assert!(got.contains_key(PROP_SEGMENT_CACHE_LOCATIONS)); let value = got.get(PROP_SEGMENT_CACHE_LOCATIONS).unwrap(); - let expected = Some(r#"[{"path":"/stackable/var/druid/segment-cache","maxSize":"2g","freeSpacePercent":"7"}]"#.to_string()); + let expected = r#"[{"path":"/stackable/var/druid/segment-cache","maxSize":"2g","freeSpacePercent":"7"}]"#.to_string(); assert_eq!(value, &expected, "secondary"); Ok(()) diff --git a/rust/operator-binary/src/crd/security.rs b/rust/operator-binary/src/crd/security.rs index bb610b30..7f9e18ad 100644 --- a/rust/operator-binary/src/crd/security.rs +++ b/rust/operator-binary/src/crd/security.rs @@ -265,33 +265,27 @@ impl DruidTlsSecurity { fn add_tls_port_config_properties( &self, - config: &mut BTreeMap>, + config: &mut BTreeMap, role: &DruidRole, ) { // no secure communication if !self.tls_enabled() { - config.insert(ENABLE_PLAINTEXT_PORT.to_string(), Some("true".to_string())); - config.insert(ENABLE_TLS_PORT.to_string(), Some("false".to_string())); - config.insert( - PLAINTEXT_PORT.to_string(), - Some(role.get_http_port().to_string()), - ); + config.insert(ENABLE_PLAINTEXT_PORT.to_string(), "true".to_string()); + config.insert(ENABLE_TLS_PORT.to_string(), "false".to_string()); + config.insert(PLAINTEXT_PORT.to_string(), role.get_http_port().to_string()); } // only allow secure communication else { - config.insert(ENABLE_PLAINTEXT_PORT.to_string(), Some("false".to_string())); - config.insert(ENABLE_TLS_PORT.to_string(), Some("true".to_string())); - config.insert( - TLS_PORT.to_string(), - Some(role.get_https_port().to_string()), - ); + config.insert(ENABLE_PLAINTEXT_PORT.to_string(), "false".to_string()); + config.insert(ENABLE_TLS_PORT.to_string(), "true".to_string()); + config.insert(TLS_PORT.to_string(), role.get_https_port().to_string()); } } /// Add required TLS ports, trust/key store properties pub fn add_tls_config_properties( &self, - config: &mut BTreeMap>, + config: &mut BTreeMap, role: &DruidRole, ) { self.add_tls_port_config_properties(config, role); @@ -306,7 +300,7 @@ impl DruidTlsSecurity { } fn add_tls_encryption_config_properties( - config: &mut BTreeMap>, + config: &mut BTreeMap, store_directory: &str, store_alias: &str, ) { @@ -314,66 +308,63 @@ impl DruidTlsSecurity { // can only be enabled/disabled together config.insert( CLIENT_HTTPS_TRUST_STORE_PATH.to_string(), - Some(format!("{}/truststore.p12", store_directory)), + format!("{}/truststore.p12", store_directory), ); config.insert( CLIENT_HTTPS_TRUST_STORE_TYPE.to_string(), - Some(TLS_STORE_TYPE.to_string()), + TLS_STORE_TYPE.to_string(), ); config.insert( CLIENT_HTTPS_TRUST_STORE_PASSWORD.to_string(), - Some(TLS_STORE_PASSWORD.to_string()), + TLS_STORE_PASSWORD.to_string(), ); config.insert( SERVER_HTTPS_KEY_STORE_PATH.to_string(), - Some(format!("{}/keystore.p12", store_directory)), + format!("{}/keystore.p12", store_directory), ); config.insert( SERVER_HTTPS_KEY_STORE_TYPE.to_string(), - Some(TLS_STORE_TYPE.to_string()), + TLS_STORE_TYPE.to_string(), ); config.insert( SERVER_HTTPS_KEY_STORE_PASSWORD.to_string(), - Some(TLS_STORE_PASSWORD.to_string()), - ); - config.insert( - SERVER_HTTPS_CERT_ALIAS.to_string(), - Some(store_alias.to_string()), + TLS_STORE_PASSWORD.to_string(), ); + config.insert(SERVER_HTTPS_CERT_ALIAS.to_string(), store_alias.to_string()); // We also need to configure the truststore for authentication related stuff, // such as verifying the LDAP server config.insert( AUTH_TRUST_STORE_PATH.to_string(), - Some(format!("{}/truststore.p12", store_directory)), + format!("{}/truststore.p12", store_directory), ); config.insert( AUTH_TRUST_STORE_TYPE.to_string(), - Some(TLS_STORE_TYPE.to_string()), + TLS_STORE_TYPE.to_string(), ); config.insert( AUTH_TRUST_STORE_PASSWORD.to_string(), - Some(TLS_STORE_PASSWORD.to_string()), + TLS_STORE_PASSWORD.to_string(), ); } fn add_tls_auth_config_properties( - config: &mut BTreeMap>, + config: &mut BTreeMap, store_directory: &str, store_alias: &str, ) { config.insert( CLIENT_HTTPS_KEY_STORE_PATH.to_string(), - Some(format!("{store_directory}/keystore.p12")), + format!("{store_directory}/keystore.p12"), ); config.insert( CLIENT_HTTPS_KEY_STORE_TYPE.to_string(), - Some(TLS_STORE_TYPE.to_string()), + TLS_STORE_TYPE.to_string(), ); config.insert( CLIENT_HTTPS_KEY_STORE_PASSWORD.to_string(), - Some(TLS_STORE_PASSWORD.to_string()), + TLS_STORE_PASSWORD.to_string(), ); // This is required because PKCS12 does not use any key passwords but it will // be checked and would lead to an exception: @@ -382,36 +373,33 @@ impl DruidTlsSecurity { // javax.crypto.BadPaddingException: Given final block not properly padded. Such issues can arise if a bad key is used during decryption. config.insert( CLIENT_HTTPS_KEY_MANAGER_PASSWORD.to_string(), - Some(TLS_STORE_PASSWORD.to_string()), - ); - config.insert( - CLIENT_HTTPS_CERT_ALIAS.to_string(), - Some(store_alias.to_string()), + TLS_STORE_PASSWORD.to_string(), ); + config.insert(CLIENT_HTTPS_CERT_ALIAS.to_string(), store_alias.to_string()); // FIXME: https://github.com/stackabletech/druid-operator/issues/372 // This is required because the server will send its pod ip which is not in the SANs of the certificates config.insert( CLIENT_HTTPS_VALIDATE_HOST_NAMES.to_string(), - Some("false".to_string()), + "false".to_string(), ); // This will enforce the client to authenticate itself config.insert( SERVER_HTTPS_REQUIRE_CLIENT_CERTIFICATE.to_string(), - Some("true".to_string()), + "true".to_string(), ); config.insert( SERVER_HTTPS_TRUST_STORE_PATH.to_string(), - Some(format!("{store_directory}/truststore.p12")), + format!("{store_directory}/truststore.p12"), ); config.insert( SERVER_HTTPS_TRUST_STORE_TYPE.to_string(), - Some(TLS_STORE_TYPE.to_string()), + TLS_STORE_TYPE.to_string(), ); config.insert( SERVER_HTTPS_TRUST_STORE_PASSWORD.to_string(), - Some(TLS_STORE_PASSWORD.to_string()), + TLS_STORE_PASSWORD.to_string(), ); // This is required because PKCS12 does not use any key passwords but it will // be checked and would lead to an exception: @@ -420,13 +408,13 @@ impl DruidTlsSecurity { // javax.crypto.BadPaddingException: Given final block not properly padded. Such issues can arise if a bad key is used during decryption. config.insert( SERVER_HTTPS_KEY_MANAGER_PASSWORD.to_string(), - Some(TLS_STORE_PASSWORD.to_string()), + TLS_STORE_PASSWORD.to_string(), ); // FIXME: https://github.com/stackabletech/druid-operator/issues/372 // This is required because the client will send its pod ip which is not in the SANs of the certificates config.insert( SERVER_HTTPS_VALIDATE_HOST_NAMES.to_string(), - Some("false".to_string()), + "false".to_string(), ); } From c0bd30117dcc2b32cc424651eac133c8a2ff81d8 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 9 Jun 2026 19:34:35 +0200 Subject: [PATCH 23/48] refactor: remove ownerref from build configmap --- rust/operator-binary/src/controller.rs | 31 +++++-- .../src/controller/build/config_map.rs | 56 ++---------- .../src/controller/validate.rs | 88 +++++++++++++++---- 3 files changed, 107 insertions(+), 68 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 8667d743..058a92e3 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -372,7 +372,6 @@ pub async fn reconcile_druid( druid_role, &rolegroup, rg, - druid, ) .context(BuildConfigMapSnafu)?; let rg_statefulset = build_rolegroup_statefulset( @@ -938,9 +937,12 @@ mod test { use std::{collections::BTreeMap, str::FromStr}; use rstest::*; - use stackable_operator::v2::types::{ - kubernetes::{NamespaceName, Uid}, - operator::ClusterName, + use stackable_operator::{ + database_connections::drivers::jdbc::JdbcDatabaseConnection, + v2::types::{ + kubernetes::{NamespaceName, Uid}, + operator::ClusterName, + }, }; use super::*; @@ -953,6 +955,7 @@ mod test { validate::{DruidRoleGroupConfig, ValidatedCluster, ValidatedClusterConfig}, }, crd::{PROP_SEGMENT_CACHE_LOCATIONS, authentication::AuthenticationClassesResolved}, + extensions::get_extension_list, }; #[rstest] @@ -1008,8 +1011,24 @@ mod test { runtime_config: runtime_properties::defaults(&DruidRole::Historical), security_config: BTreeMap::new(), env: BTreeMap::new(), + // The test only asserts on runtime.properties, so the rendered jvm.config is irrelevant. + jvm_config: String::new(), }; + let extensions = get_extension_list(&druid, &druid_tls_security, &None); + let metadata_storage_type = druid + .spec + .cluster_config + .metadata_database + .as_metadata_storage_type() + .to_string(); + let metadata_db_connection = druid + .spec + .cluster_config + .metadata_database + .jdbc_connection_details("metadata") + .expect("test: valid metadata db connection"); + let cluster = ValidatedCluster::new( ClusterName::from_str(&druid.name_any()).expect("test: valid cluster name"), NamespaceName::from_str("default").expect("test: valid namespace"), @@ -1022,6 +1041,9 @@ mod test { deep_storage_bucket_name: None, druid_tls_security, druid_auth_config: None, + extensions, + metadata_storage_type, + metadata_db_connection, }, BTreeMap::new(), ); @@ -1037,7 +1059,6 @@ mod test { &DruidRole::Historical, &rolegroup_ref, &rg, - &druid, ) .expect("build rolegroup config map"); diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 1492dd9d..6ef8cad5 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -7,11 +7,9 @@ //! Metadata, owner reference and recommended labels are derived entirely from `ValidatedCluster` //! (which carries the validated name/namespace/uid and implements `Resource`). //! -//! Residual reads from the owning [`v1alpha1::DruidCluster`] remain for things that are not yet -//! modelled on `ValidatedCluster`: the extensions load list (`get_extension_list`), the -//! metadata-database connection (`spec.cluster_config.metadata_database` / -//! `as_metadata_storage_type`), and `get_role` for the jvm.config. Fully removing these is a -//! follow-up. +//! The builder no longer reads the raw [`v1alpha1::DruidCluster`] at all: the extensions load +//! list, the metadata-database connection / storage type and the rendered `jvm.config` are all +//! precomputed on `ValidatedCluster` during the validate step. use std::collections::BTreeMap; @@ -19,7 +17,6 @@ use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, crd::s3, - database_connections::drivers::jdbc::JdbcDatabaseConnection as _, k8s_openapi::api::core::v1::{ConfigMap, EnvVar}, product_logging::framework::VECTOR_CONFIG_FILE, role_utils::RoleGroupRef, @@ -29,7 +26,6 @@ use stackable_operator::{ }; use crate::{ - config::jvm::construct_jvm_args, controller::{ DRUID_CONTROLLER_NAME, build::properties::{ @@ -39,7 +35,6 @@ use crate::{ validate::{DruidRoleGroupConfig, ValidatedCluster}, }, crd::{DruidRole, build_recommended_labels, build_string_list, v1alpha1}, - extensions::get_extension_list, }; // jvm.config is built by `config::jvm`, not a properties builder, so it is not part @@ -81,12 +76,6 @@ pub enum Error { rolegroup: String, }, - #[snafu(display("failed to get JVM config"))] - GetJvmConfig { source: crate::config::jvm::Error }, - - #[snafu(display("failed to derive Druid memory settings from resources"))] - DeriveMemorySettings { source: crate::crd::resource::Error }, - #[snafu(display("failed to update Druid config from resources"))] UpdateDruidConfigFromResources { source: crate::crd::resource::Error }, @@ -99,11 +88,6 @@ pub enum Error { GenerateAuthenticationRuntimeSettings { source: crate::authentication::Error, }, - - #[snafu(display("invalid metadata database connection"))] - InvalidMetadataDatabaseConnection { - source: stackable_operator::database_connections::Error, - }, } type Result = std::result::Result; @@ -114,7 +98,6 @@ pub fn build_rolegroup_config_map( role: &DruidRole, rolegroup: &RoleGroupRef, rg: &DruidRoleGroupConfig, - owner: &v1alpha1::DruidCluster, ) -> Result { let cluster_config = &cluster.cluster_config; let druid_tls_security = &cluster_config.druid_tls_security; @@ -124,14 +107,8 @@ pub fn build_rolegroup_config_map( let s3_conn = cluster_config.s3_connection.as_ref(); let deep_storage_bucket_name = cluster_config.deep_storage_bucket_name.as_deref(); - let role_obj = owner.get_role(role); let mut cm_conf_data = BTreeMap::new(); // filename -> filecontent - let metadata_database_connection_details = owner - .spec - .cluster_config - .metadata_database - .jdbc_connection_details("metadata") - .context(InvalidMetadataDatabaseConnectionSnafu)?; + let metadata_database_connection_details = &cluster_config.metadata_db_connection; // ----- runtime.properties ----- { @@ -155,11 +132,7 @@ pub fn build_rolegroup_config_map( conf.insert( EXTENSIONS_LOADLIST.to_string(), - build_string_list(&get_extension_list( - owner, - druid_tls_security, - druid_auth_config, - )), + build_string_list(&cluster_config.extensions), ); if let Some(opa_str) = opa_connstr { @@ -168,12 +141,7 @@ pub fn build_rolegroup_config_map( conf.insert( crate::crd::database::METADATA_STORAGE_TYPE.to_string(), - owner - .spec - .cluster_config - .metadata_database - .as_metadata_storage_type() - .to_string(), + cluster_config.metadata_storage_type.clone(), ); conf.insert( @@ -268,16 +236,8 @@ pub fn build_rolegroup_config_map( } // ----- jvm.config ----- - { - let (heap, direct) = rg - .merged_config - .resources - .get_memory_sizes(role) - .context(DeriveMemorySettingsSnafu)?; - let jvm_config = construct_jvm_args(role, &role_obj, &rolegroup.role_group, heap, direct) - .context(GetJvmConfigSnafu)?; - cm_conf_data.insert(JVM_CONFIG.to_string(), jvm_config); - } + // Precomputed during validation; see `DruidRoleGroupConfig::jvm_config`. + cm_conf_data.insert(JVM_CONFIG.to_string(), rg.jvm_config.clone()); // ----- security.properties ----- { diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index b2225132..54be5626 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -14,6 +14,7 @@ use stackable_operator::{ commons::product_image_selection::{self, ResolvedProductImage}, config::merge::Merge, crd::s3, + database_connections::drivers::jdbc::{JdbcDatabaseConnection, JdbcDatabaseConnectionDetails}, kube::{Resource, api::ObjectMeta}, v2::{ HasName, HasUid, @@ -28,6 +29,7 @@ use strum::IntoEnumIterator; use crate::{ authentication::DruidAuthenticationConfig, + config::jvm::construct_jvm_args, controller::{ build::properties::{ConfigFileName, runtime_properties, security_properties}, dereference::DereferencedObjects, @@ -36,6 +38,7 @@ use crate::{ CommonRoleGroupConfig, DruidConfigOverrides, DruidRole, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, build_string_list, security::DruidTlsSecurity, v1alpha1, }, + extensions::get_extension_list, }; #[derive(Snafu, Debug)] @@ -58,6 +61,17 @@ pub enum Error { ClusterIdentity { source: stackable_operator::v2::controller_utils::Error, }, + + #[snafu(display("failed to get JVM config"))] + GetJvmConfig { source: crate::config::jvm::Error }, + + #[snafu(display("failed to derive Druid memory settings from resources"))] + DeriveMemorySettings { source: crate::crd::resource::Error }, + + #[snafu(display("invalid metadata database connection"))] + InvalidMetadataDatabaseConnection { + source: stackable_operator::database_connections::Error, + }, } type Result = std::result::Result; @@ -76,6 +90,10 @@ pub struct DruidRoleGroupConfig { pub security_config: BTreeMap, /// Merged env overrides (role <- rolegroup). compute_env is empty for druid. pub env: BTreeMap, + /// The fully rendered `jvm.config` (operator defaults merged with the role/rolegroup JVM + /// argument overrides). Precomputed here so the config-map builder no longer needs the raw + /// cluster's `get_role`. + pub jvm_config: String, } /// Cluster-wide resolved fields that are not role/rolegroup specific. @@ -86,6 +104,13 @@ pub struct ValidatedClusterConfig { pub deep_storage_bucket_name: Option, pub druid_tls_security: DruidTlsSecurity, pub druid_auth_config: Option, + /// The `druid.extensions.loadList` entries, resolved from the metadata database, TLS, S3 and + /// authentication settings during validation. + pub extensions: Vec, + /// The `druid.metadata.storage.type` value derived from the configured metadata database. + pub metadata_storage_type: String, + /// The JDBC connection details (URL plus credential env vars) for the metadata database. + pub metadata_db_connection: JdbcDatabaseConnectionDetails, } /// Synchronous inputs the rest of `reconcile_druid` needs after dereferencing. @@ -201,15 +226,17 @@ fn key_value_overrides( /// Builds the precomputed per-file config for a single rolegroup. Pure assembly: combines the /// role-level overrides with the rolegroup-level overrides (rolegroup wins) on top of the /// computed defaults. No behavior change vs. the inline loop body it was extracted from. +#[allow(clippy::too_many_arguments)] fn build_role_group_config( druid: &v1alpha1::DruidCluster, druid_role: &DruidRole, + rg_name: &str, merged_config: CommonRoleGroupConfig, role_config_overrides: &DruidConfigOverrides, role_env_overrides: &HashMap, rg_config_overrides: &DruidConfigOverrides, rg_env_overrides: &HashMap, -) -> DruidRoleGroupConfig { +) -> Result { // Merge the role-level and rolegroup-level config overrides (rolegroup wins over role). let mut config_overrides = rg_config_overrides.clone(); config_overrides.merge(role_config_overrides); @@ -243,12 +270,27 @@ fn build_role_group_config( .collect(); env.extend(rg_env_overrides.iter().map(|(k, v)| (k.clone(), v.clone()))); - DruidRoleGroupConfig { + // ----- jvm.config ----- + let (heap, direct) = merged_config + .resources + .get_memory_sizes(druid_role) + .context(DeriveMemorySettingsSnafu)?; + let jvm_config = construct_jvm_args( + druid_role, + &druid.get_role(druid_role), + rg_name, + heap, + direct, + ) + .context(GetJvmConfigSnafu)?; + + Ok(DruidRoleGroupConfig { merged_config, runtime_config, security_config, env, - } + jvm_config, + }) } const INDEXER_JAVA_OPTS: &str = "druid.indexer.runner.javaOptsArray"; @@ -317,18 +359,17 @@ pub fn validate( .role_group_overrides(&druid_role, &rg_name) .expect("role group resolved by common_config must exist"); - group_map.insert( - rg_name, - build_role_group_config( - druid, - &druid_role, - merged_config, - role_config_overrides, - &role_env_overrides, - rg_config_overrides, - rg_env_overrides, - ), - ); + let rg_config = build_role_group_config( + druid, + &druid_role, + &rg_name, + merged_config, + role_config_overrides, + &role_env_overrides, + rg_config_overrides, + rg_env_overrides, + )?; + group_map.insert(rg_name, rg_config); } role_group_configs.insert(druid_role, group_map); } @@ -337,6 +378,20 @@ pub fn validate( let namespace = get_namespace(druid).context(ClusterIdentitySnafu)?; let uid = get_uid(druid).context(ClusterIdentitySnafu)?; + let extensions = get_extension_list(druid, &druid_tls_security, &druid_auth_config); + let metadata_storage_type = druid + .spec + .cluster_config + .metadata_database + .as_metadata_storage_type() + .to_string(); + let metadata_db_connection = druid + .spec + .cluster_config + .metadata_database + .jdbc_connection_details("metadata") + .context(InvalidMetadataDatabaseConnectionSnafu)?; + Ok(ValidatedCluster::new( name, namespace, @@ -349,6 +404,9 @@ pub fn validate( deep_storage_bucket_name: dereferenced_objects.deep_storage_bucket_name.clone(), druid_tls_security, druid_auth_config, + extensions, + metadata_storage_type, + metadata_db_connection, }, role_group_configs, )) From 83bfe01f8f5d22ee6d558d406f0ef43126a65320 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 9 Jun 2026 19:47:47 +0200 Subject: [PATCH 24/48] refactor: move jvm from config to build --- rust/operator-binary/src/config/mod.rs | 1 - rust/operator-binary/src/controller.rs | 10 +++------- rust/operator-binary/src/controller/build.rs | 1 + .../src/controller/build/config_map.rs | 6 +----- .../src/{config => controller/build}/jvm.rs | 11 +++++------ .../src/controller/build/properties/mod.rs | 7 +++++-- rust/operator-binary/src/controller/validate.rs | 15 ++++++++++----- rust/operator-binary/src/main.rs | 1 - 8 files changed, 25 insertions(+), 27 deletions(-) delete mode 100644 rust/operator-binary/src/config/mod.rs rename rust/operator-binary/src/{config => controller/build}/jvm.rs (97%) diff --git a/rust/operator-binary/src/config/mod.rs b/rust/operator-binary/src/config/mod.rs deleted file mode 100644 index 271c6d99..00000000 --- a/rust/operator-binary/src/config/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod jvm; diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 058a92e3..84bf2a8f 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -1054,13 +1054,9 @@ mod test { role_group: tested_rolegroup_name.to_string(), }; - let rg_configmap = build_rolegroup_config_map( - &cluster, - &DruidRole::Historical, - &rolegroup_ref, - &rg, - ) - .expect("build rolegroup config map"); + let rg_configmap = + build_rolegroup_config_map(&cluster, &DruidRole::Historical, &rolegroup_ref, &rg) + .expect("build rolegroup config map"); let druid_segment_cache_property = rg_configmap .data diff --git a/rust/operator-binary/src/controller/build.rs b/rust/operator-binary/src/controller/build.rs index 182778ae..c090434d 100644 --- a/rust/operator-binary/src/controller/build.rs +++ b/rust/operator-binary/src/controller/build.rs @@ -2,4 +2,5 @@ pub mod config_map; pub mod discovery; +pub mod jvm; pub mod properties; diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 6ef8cad5..ad971122 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -37,10 +37,6 @@ use crate::{ crd::{DruidRole, build_recommended_labels, build_string_list, v1alpha1}, }; -// jvm.config is built by `config::jvm`, not a properties builder, so it is not part -// of `ConfigFileName`. -const JVM_CONFIG: &str = "jvm.config"; - // Druid `runtime.properties` config-property keys assembled into the rolegroup ConfigMap here. const EXTENSIONS_LOADLIST: &str = "druid.extensions.loadList"; const ZOOKEEPER_CONNECTION_STRING: &str = "druid.zk.service.host"; @@ -237,7 +233,7 @@ pub fn build_rolegroup_config_map( // ----- jvm.config ----- // Precomputed during validation; see `DruidRoleGroupConfig::jvm_config`. - cm_conf_data.insert(JVM_CONFIG.to_string(), rg.jvm_config.clone()); + cm_conf_data.insert(ConfigFileName::JvmConfig.to_string(), rg.jvm_config.clone()); // ----- security.properties ----- { diff --git a/rust/operator-binary/src/config/jvm.rs b/rust/operator-binary/src/controller/build/jvm.rs similarity index 97% rename from rust/operator-binary/src/config/jvm.rs rename to rust/operator-binary/src/controller/build/jvm.rs index e9bcd8a3..4026db9f 100644 --- a/rust/operator-binary/src/config/jvm.rs +++ b/rust/operator-binary/src/controller/build/jvm.rs @@ -5,15 +5,12 @@ use stackable_operator::{ role_utils::{GenericRoleConfig, JavaCommonConfig, JvmArgumentOverrides, Role}, }; +use super::properties::ConfigFileName; use crate::crd::{ DruidConfigOverrides, DruidRole, RW_CONFIG_DIRECTORY, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, }; -// File names not exported from crd/mod.rs (see ConfigFileName for the properties-builder files). -const SECURITY_PROPERTIES_FILE: &str = "security.properties"; -const LOG4J2_CONFIG_FILE: &str = "log4j2.properties"; - #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("failed to format memory quantity {value:?} for Java"))] @@ -55,15 +52,17 @@ pub fn construct_jvm_args( if let Some(direct_memory) = direct_memory_str { jvm_args.push(format!("-XX:MaxDirectMemorySize={direct_memory}")); } + let security_properties_file = ConfigFileName::SecurityProperties; + let log4j2_config_file = ConfigFileName::Log4j2Properties; jvm_args.extend([ "-XX:+ExitOnOutOfMemoryError".to_owned(), "-XX:+UseG1GC".to_owned(), - format!("-Djava.security.properties={RW_CONFIG_DIRECTORY}/{SECURITY_PROPERTIES_FILE}"), + format!("-Djava.security.properties={RW_CONFIG_DIRECTORY}/{security_properties_file}"), "-Duser.timezone=UTC".to_owned(), "-Dfile.encoding=UTF-8".to_owned(), "-Djava.io.tmpdir=/tmp".to_owned(), "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager".to_owned(), - format!("-Dlog4j.configurationFile={RW_CONFIG_DIRECTORY}/{LOG4J2_CONFIG_FILE}"), + format!("-Dlog4j.configurationFile={RW_CONFIG_DIRECTORY}/{log4j2_config_file}"), format!("-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}"), format!("-Djavax.net.ssl.trustStorePassword={STACKABLE_TRUST_STORE_PASSWORD}"), "-Djavax.net.ssl.trustStoreType=pkcs12".to_owned(), diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index ee12da28..1f3cf715 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -6,8 +6,6 @@ pub mod security_properties; /// The names of the operator-written Druid config files assembled into the rolegroup ConfigMap. #[derive(Clone, Copy, Debug, strum::Display)] -// The shared `Properties` suffix mirrors the actual on-disk file names; it is not redundant naming. -#[allow(clippy::enum_variant_names)] pub enum ConfigFileName { #[strum(serialize = "runtime.properties")] RuntimeProperties, @@ -17,4 +15,9 @@ pub enum ConfigFileName { /// but it is still an operator-written file assembled into the rolegroup ConfigMap. #[strum(serialize = "log4j2.properties")] Log4j2Properties, + /// `jvm.config` is rendered from JVM argument overrides by [`super::jvm`] rather than a + /// properties builder, but it is still an operator-written file assembled into the rolegroup + /// ConfigMap. + #[strum(serialize = "jvm.config")] + JvmConfig, } diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 54be5626..082dd145 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -29,9 +29,11 @@ use strum::IntoEnumIterator; use crate::{ authentication::DruidAuthenticationConfig, - config::jvm::construct_jvm_args, controller::{ - build::properties::{ConfigFileName, runtime_properties, security_properties}, + build::{ + jvm::construct_jvm_args, + properties::{ConfigFileName, runtime_properties, security_properties}, + }, dereference::DereferencedObjects, }, crd::{ @@ -63,7 +65,9 @@ pub enum Error { }, #[snafu(display("failed to get JVM config"))] - GetJvmConfig { source: crate::config::jvm::Error }, + GetJvmConfig { + source: crate::controller::build::jvm::Error, + }, #[snafu(display("failed to derive Druid memory settings from resources"))] DeriveMemorySettings { source: crate::crd::resource::Error }, @@ -215,8 +219,9 @@ fn key_value_overrides( let raw = match file { ConfigFileName::RuntimeProperties => overrides.runtime_properties.overrides.clone(), ConfigFileName::SecurityProperties => overrides.security_properties.overrides.clone(), - // log4j2.properties is rendered by the logging framework and accepts no key/value overrides. - ConfigFileName::Log4j2Properties => BTreeMap::new(), + // log4j2.properties is rendered by the logging framework, and jvm.config is rendered from + // JVM argument overrides; neither is assembled from key/value overrides here. + ConfigFileName::Log4j2Properties | ConfigFileName::JvmConfig => BTreeMap::new(), }; raw.into_iter() .map(|(k, v)| (k, v.unwrap_or_default())) diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index aa99e518..32d08118 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -38,7 +38,6 @@ use crate::{ }; mod authentication; -mod config; mod controller; mod crd; mod extensions; From 6e203e9d3872eb4a81e53056a495e24de4dcc112 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 9 Jun 2026 20:24:54 +0200 Subject: [PATCH 25/48] fix: cleanup, visibility, constants --- .../src/authentication/ldap.rs | 6 +- .../operator-binary/src/authentication/mod.rs | 5 +- .../src/authentication/oidc.rs | 8 +-- .../src/controller/build/discovery.rs | 3 - .../src/controller/validate.rs | 8 +-- .../operator-binary/src/crd/authentication.rs | 4 +- rust/operator-binary/src/crd/database.rs | 8 +-- rust/operator-binary/src/crd/memory.rs | 4 +- rust/operator-binary/src/crd/mod.rs | 28 ++++----- rust/operator-binary/src/crd/resource.rs | 60 ++++++++++--------- rust/operator-binary/src/crd/security.rs | 32 ++++------ rust/operator-binary/src/internal_secret.rs | 4 +- rust/operator-binary/src/listener.rs | 2 +- .../src/webhooks/conversion.rs | 4 +- 14 files changed, 82 insertions(+), 94 deletions(-) diff --git a/rust/operator-binary/src/authentication/ldap.rs b/rust/operator-binary/src/authentication/ldap.rs index c61cc5f8..7411fdea 100644 --- a/rust/operator-binary/src/authentication/ldap.rs +++ b/rust/operator-binary/src/authentication/ldap.rs @@ -83,7 +83,7 @@ fn add_authorizer_config(config: &mut BTreeMap) { ); } -pub fn generate_runtime_properties_config( +pub(super) fn generate_runtime_properties_config( provider: &ldap::v1alpha1::AuthenticationProvider, config: &mut BTreeMap, ) -> Result<(), Error> { @@ -93,7 +93,7 @@ pub fn generate_runtime_properties_config( Ok(()) } -pub fn prepare_container_commands( +pub(super) fn prepare_container_commands( provider: &ldap::v1alpha1::AuthenticationProvider, command: &mut Vec, ) { @@ -106,7 +106,7 @@ pub fn prepare_container_commands( } } -pub fn add_volumes_and_mounts( +pub(super) fn add_volumes_and_mounts( provider: &ldap::v1alpha1::AuthenticationProvider, pb: &mut PodBuilder, cb_druid: &mut ContainerBuilder, diff --git a/rust/operator-binary/src/authentication/mod.rs b/rust/operator-binary/src/authentication/mod.rs index dd933528..0342fcc1 100644 --- a/rust/operator-binary/src/authentication/mod.rs +++ b/rust/operator-binary/src/authentication/mod.rs @@ -11,7 +11,7 @@ use crate::{ crd::{ DruidRole, authentication::{AuthenticationClassResolved, AuthenticationClassesResolved}, - security::{ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV, INTERNAL_INITIAL_CLIENT_PASSWORD_ENV}, + security::INTERNAL_INITIAL_CLIENT_PASSWORD_ENV, v1alpha1, }, internal_secret::{build_shared_internal_secret_name, env_var_from_secret}, @@ -20,6 +20,9 @@ use crate::{ pub mod ldap; pub mod oidc; +// It seems this needs to be the same password for Druid to work, so we re-use the existing env variable. +const ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV: &str = INTERNAL_INITIAL_CLIENT_PASSWORD_ENV; + type Result = std::result::Result; #[derive(Snafu, Debug)] diff --git a/rust/operator-binary/src/authentication/oidc.rs b/rust/operator-binary/src/authentication/oidc.rs index 3aeca1be..cb018426 100644 --- a/rust/operator-binary/src/authentication/oidc.rs +++ b/rust/operator-binary/src/authentication/oidc.rs @@ -97,7 +97,7 @@ fn add_authorizer_config(config: &mut BTreeMap) { /// Creates the OIDC parts of the runtime.properties config file. /// OIDC authentication is not configured on middlemanagers, because end users don't interact with them directly using the web console and /// turning on OIDC will lead to problems with the communication with coordinators during data ingest. -pub fn generate_runtime_properties_config( +pub(super) fn generate_runtime_properties_config( provider: &oidc::v1alpha1::AuthenticationProvider, oidc: &DruidClientAuthenticationOptions, role: &DruidRole, @@ -118,7 +118,7 @@ pub fn generate_runtime_properties_config( Ok(()) } -pub fn main_container_commands( +pub(super) fn main_container_commands( provider: &oidc::v1alpha1::AuthenticationProvider, command: &mut Vec, ) { @@ -129,7 +129,7 @@ pub fn main_container_commands( /// Mounts the OIDC credentials secret and the auto-generated internal secret containing the cookie passphrase. /// Not necessary on middlemanagers, because OIDC is not configured on them. -pub fn get_env_var_mounts( +pub(super) fn get_env_var_mounts( role: &DruidRole, oidc: &DruidClientAuthenticationOptions, internal_secret_name: &str, @@ -153,7 +153,7 @@ pub fn get_env_var_mounts( envs } -pub fn add_volumes_and_mounts( +pub(super) fn add_volumes_and_mounts( provider: &oidc::v1alpha1::AuthenticationProvider, pb: &mut PodBuilder, cb_druid: &mut ContainerBuilder, diff --git a/rust/operator-binary/src/controller/build/discovery.rs b/rust/operator-binary/src/controller/build/discovery.rs index 2be36624..388684b8 100644 --- a/rust/operator-binary/src/controller/build/discovery.rs +++ b/rust/operator-binary/src/controller/build/discovery.rs @@ -18,9 +18,6 @@ use crate::{ #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("failed to get service FQDN"))] - NoServiceFqdn, - #[snafu(display("failed to build ConfigMap"))] BuildConfigMap { source: stackable_operator::builder::configmap::Error, diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 082dd145..0438652d 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -88,11 +88,11 @@ pub type RoleGroupName = String; #[derive(Clone)] pub struct DruidRoleGroupConfig { pub merged_config: CommonRoleGroupConfig, - /// The runtime.properties "validated config" (compute_files + recommended defaults + merged overrides). + /// The runtime.properties config: recommended defaults plus merged overrides. pub runtime_config: BTreeMap, /// The security.properties "validated config". pub security_config: BTreeMap, - /// Merged env overrides (role <- rolegroup). compute_env is empty for druid. + /// Merged env overrides (role <- rolegroup). Druid has no computed env vars. pub env: BTreeMap, /// The fully rendered `jvm.config` (operator defaults merged with the role/rolegroup JVM /// argument overrides). Precomputed here so the config-map builder no longer needs the raw @@ -300,8 +300,8 @@ fn build_role_group_config( const INDEXER_JAVA_OPTS: &str = "druid.indexer.runner.javaOptsArray"; -/// The `druid.indexer.runner.javaOptsArray` entry that `MiddleManagerConfigFragment::compute_files` -/// adds for *every* file (runtime.properties and security.properties). +/// The `druid.indexer.runner.javaOptsArray` entry that must be present in *every* rendered file +/// (runtime.properties and security.properties) for MiddleManagers. fn middlemanager_indexer_java_opts() -> (String, String) { ( INDEXER_JAVA_OPTS.to_string(), diff --git a/rust/operator-binary/src/crd/authentication.rs b/rust/operator-binary/src/crd/authentication.rs index 034723e5..3f77d205 100644 --- a/rust/operator-binary/src/crd/authentication.rs +++ b/rust/operator-binary/src/crd/authentication.rs @@ -13,7 +13,7 @@ use crate::crd::v1alpha1::DruidClusterConfig; type Result = std::result::Result; // The assumed OIDC provider if no hint is given in the AuthClass -pub const DEFAULT_OIDC_PROVIDER: oidc::v1alpha1::IdentityProviderHint = +const DEFAULT_OIDC_PROVIDER: oidc::v1alpha1::IdentityProviderHint = oidc::v1alpha1::IdentityProviderHint::Keycloak; const SUPPORTED_OIDC_PROVIDERS: &[oidc::v1alpha1::IdentityProviderHint] = @@ -101,7 +101,7 @@ impl AuthenticationClassesResolved { } /// Retrieves all provided `AuthenticationClass` references and checks if the configuration (TLS settings, secret class, OIDC config, etc.) is valid. - pub async fn resolve( + async fn resolve( cluster_config: &DruidClusterConfig, resolve_auth_class: impl Fn( core::v1alpha1::ClientAuthenticationDetails< diff --git a/rust/operator-binary/src/crd/database.rs b/rust/operator-binary/src/crd/database.rs index 91116627..f24fc92c 100644 --- a/rust/operator-binary/src/crd/database.rs +++ b/rust/operator-binary/src/crd/database.rs @@ -11,11 +11,11 @@ use stackable_operator::{ }; // metadata storage config properties -pub const METADATA_STORAGE_TYPE: &str = "druid.metadata.storage.type"; -pub const METADATA_STORAGE_CONNECTOR_CONNECT_URI: &str = +pub(crate) const METADATA_STORAGE_TYPE: &str = "druid.metadata.storage.type"; +pub(crate) const METADATA_STORAGE_CONNECTOR_CONNECT_URI: &str = "druid.metadata.storage.connector.connectURI"; -pub const METADATA_STORAGE_USER: &str = "druid.metadata.storage.connector.user"; -pub const METADATA_STORAGE_PASSWORD: &str = "druid.metadata.storage.connector.password"; +pub(crate) const METADATA_STORAGE_USER: &str = "druid.metadata.storage.connector.user"; +pub(crate) const METADATA_STORAGE_PASSWORD: &str = "druid.metadata.storage.connector.password"; #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] diff --git a/rust/operator-binary/src/crd/memory.rs b/rust/operator-binary/src/crd/memory.rs index 892bcdc8..88c21b5c 100644 --- a/rust/operator-binary/src/crd/memory.rs +++ b/rust/operator-binary/src/crd/memory.rs @@ -21,7 +21,7 @@ pub static RESERVED_OS_MEMORY: LazyLock = /// Max size for direct access buffers. This is defined in Druid to be 2GB: /// -pub static MAX_DIRECT_BUFFER_SIZE: LazyLock = +static MAX_DIRECT_BUFFER_SIZE: LazyLock = LazyLock::new(|| MemoryQuantity::from_gibi(2.)); #[derive(Snafu, Debug)] @@ -55,7 +55,7 @@ pub struct HistoricalDerivedSettings { } impl HistoricalDerivedSettings { - pub fn new(total_memory: MemoryQuantity, cpu_millis: CpuQuantity) -> Self { + fn new(total_memory: MemoryQuantity, cpu_millis: CpuQuantity) -> Self { Self { total_memory, cpu_millis, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index cd0182f2..abb01520 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -59,7 +59,6 @@ pub mod tls; pub const APP_NAME: &str = "druid"; pub const OPERATOR_NAME: &str = "druid.stackable.tech"; -pub const FIELD_MANAGER: &str = "druid-operator"; // config directories pub const DRUID_CONFIG_DIRECTORY: &str = "/stackable/config"; @@ -73,22 +72,21 @@ pub const STACKABLE_TRUST_STORE_PASSWORD: &str = "changeit"; pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; pub const PROP_SEGMENT_CACHE_LOCATIONS: &str = "druid.segmentCache.locations"; -pub const PATH_SEGMENT_CACHE: &str = "/stackable/var/druid/segment-cache"; ///////////////////////////// // CONFIG PROPERTIES // ///////////////////////////// // deep storage -pub const DS_TYPE: &str = "druid.storage.type"; -pub const DS_DIRECTORY: &str = "druid.storage.storageDirectory"; -pub const DS_BASE_KEY: &str = "druid.storage.baseKey"; +const DS_TYPE: &str = "druid.storage.type"; +const DS_DIRECTORY: &str = "druid.storage.storageDirectory"; +const DS_BASE_KEY: &str = "druid.storage.baseKey"; // OPA -pub const AUTH_AUTHORIZERS: &str = "druid.auth.authorizers"; -pub const AUTH_AUTHORIZERS_VALUE: &str = "[\"OpaAuthorizer\"]"; -pub const AUTH_AUTHORIZER_OPA_TYPE: &str = "druid.auth.authorizer.OpaAuthorizer.type"; -pub const AUTH_AUTHORIZER_OPA_TYPE_VALUE: &str = "opa"; +const AUTH_AUTHORIZERS: &str = "druid.auth.authorizers"; +const AUTH_AUTHORIZERS_VALUE: &str = "[\"OpaAuthorizer\"]"; +const AUTH_AUTHORIZER_OPA_TYPE: &str = "druid.auth.authorizer.OpaAuthorizer.type"; +const AUTH_AUTHORIZER_OPA_TYPE_VALUE: &str = "opa"; // metrics -pub const PROMETHEUS_PORT: &str = "druid.emitter.prometheus.port"; +const PROMETHEUS_PORT: &str = "druid.emitter.prometheus.port"; pub const METRICS_PORT_NAME: &str = "metrics"; pub const METRICS_PORT: u16 = 9090; @@ -546,7 +544,7 @@ impl v1alpha1::DruidCluster { } /// Merges and validates the given role group, role, and default configurations - pub fn merged_rolegroup_config( + pub(crate) fn merged_rolegroup_config( rolegroup_config: &T::Fragment, role_config: &T::Fragment, default_config: &T::Fragment, @@ -900,7 +898,7 @@ impl DruidRole { } /// Return the default graceful shutdown timeout - pub fn default_graceful_shutdown_timeout(&self) -> Duration { + fn default_graceful_shutdown_timeout(&self) -> Duration { match &self { DruidRole::Coordinator => DEFAULT_COORDINATOR_GRACEFUL_SHUTDOWN_TIMEOUT, DruidRole::Broker => DEFAULT_BROKER_GRACEFUL_SHUTDOWN_TIMEOUT, @@ -1003,11 +1001,7 @@ pub enum DeepStorageSpec { } impl DeepStorageSpec { - pub fn is_hdfs(&self) -> bool { - matches!(self, DeepStorageSpec::Hdfs(_)) - } - - pub fn is_s3(&self) -> bool { + fn is_s3(&self) -> bool { matches!(self, DeepStorageSpec::S3(_)) } } diff --git a/rust/operator-binary/src/crd/resource.rs b/rust/operator-binary/src/crd/resource.rs index 148eb38a..71b5a357 100644 --- a/rust/operator-binary/src/crd/resource.rs +++ b/rust/operator-binary/src/crd/resource.rs @@ -17,11 +17,13 @@ use stackable_operator::{ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::crd::{ - DruidRole, PATH_SEGMENT_CACHE, PROP_SEGMENT_CACHE_LOCATIONS, + DruidRole, PROP_SEGMENT_CACHE_LOCATIONS, memory::{HistoricalDerivedSettings, RESERVED_OS_MEMORY}, storage::{self, default_free_percentage_empty_dir_fragment}, }; +const PATH_SEGMENT_CACHE: &str = "/stackable/var/druid/segment-cache"; + // volume names const SEGMENT_CACHE_VOLUME_NAME: &str = "segment-cache"; @@ -172,7 +174,7 @@ impl RoleResource { } } -pub static MIDDLE_MANAGER_RESOURCES: LazyLock< +pub(crate) static MIDDLE_MANAGER_RESOURCES: LazyLock< ResourcesFragment, > = LazyLock::new(|| ResourcesFragment { cpu: CpuLimitsFragment { @@ -186,20 +188,21 @@ pub static MIDDLE_MANAGER_RESOURCES: LazyLock< storage: storage::DruidStorageFragment {}, }); -pub static BROKER_RESOURCES: LazyLock> = - LazyLock::new(|| ResourcesFragment { - cpu: CpuLimitsFragment { - min: Some(Quantity("300m".to_owned())), - max: Some(Quantity("1200m".to_owned())), - }, - memory: MemoryLimitsFragment { - limit: Some(Quantity("1500Mi".to_owned())), - runtime_limits: NoRuntimeLimitsFragment {}, - }, - storage: storage::DruidStorageFragment {}, - }); +pub(crate) static BROKER_RESOURCES: LazyLock< + ResourcesFragment, +> = LazyLock::new(|| ResourcesFragment { + cpu: CpuLimitsFragment { + min: Some(Quantity("300m".to_owned())), + max: Some(Quantity("1200m".to_owned())), + }, + memory: MemoryLimitsFragment { + limit: Some(Quantity("1500Mi".to_owned())), + runtime_limits: NoRuntimeLimitsFragment {}, + }, + storage: storage::DruidStorageFragment {}, +}); -pub static HISTORICAL_RESOURCES: LazyLock< +pub(crate) static HISTORICAL_RESOURCES: LazyLock< ResourcesFragment, > = LazyLock::new(|| ResourcesFragment { cpu: CpuLimitsFragment { @@ -215,7 +218,7 @@ pub static HISTORICAL_RESOURCES: LazyLock< }, }); -pub static COORDINATOR_RESOURCES: LazyLock< +pub(crate) static COORDINATOR_RESOURCES: LazyLock< ResourcesFragment, > = LazyLock::new(|| ResourcesFragment { cpu: CpuLimitsFragment { @@ -229,18 +232,19 @@ pub static COORDINATOR_RESOURCES: LazyLock< storage: storage::DruidStorageFragment {}, }); -pub static ROUTER_RESOURCES: LazyLock> = - LazyLock::new(|| ResourcesFragment { - cpu: CpuLimitsFragment { - min: Some(Quantity("300m".to_owned())), - max: Some(Quantity("1200m".to_owned())), - }, - memory: MemoryLimitsFragment { - limit: Some(Quantity("512Mi".to_owned())), - runtime_limits: NoRuntimeLimitsFragment {}, - }, - storage: storage::DruidStorageFragment {}, - }); +pub(crate) static ROUTER_RESOURCES: LazyLock< + ResourcesFragment, +> = LazyLock::new(|| ResourcesFragment { + cpu: CpuLimitsFragment { + min: Some(Quantity("300m".to_owned())), + max: Some(Quantity("1200m".to_owned())), + }, + memory: MemoryLimitsFragment { + limit: Some(Quantity("512Mi".to_owned())), + runtime_limits: NoRuntimeLimitsFragment {}, + }, + storage: storage::DruidStorageFragment {}, +}); #[cfg(test)] mod test { diff --git a/rust/operator-binary/src/crd/security.rs b/rust/operator-binary/src/crd/security.rs index 7f9e18ad..2756d72b 100644 --- a/rust/operator-binary/src/crd/security.rs +++ b/rust/operator-binary/src/crd/security.rs @@ -23,16 +23,12 @@ use stackable_operator::{ }; use crate::crd::{ - DruidRole, STACKABLE_TRUST_STORE_PASSWORD, - authentication::{self, AuthenticationClassesResolved}, + DruidRole, STACKABLE_TRUST_STORE_PASSWORD, authentication::AuthenticationClassesResolved, v1alpha1, }; #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("failed to process authentication class"))] - InvalidAuthenticationClassConfiguration { source: authentication::Error }, - #[snafu(display("failed to build the Secret operator Volume"))] SecretVolumeBuild { source: SecretOperatorVolumeSourceBuilderError, @@ -86,12 +82,12 @@ const SERVER_HTTPS_REQUIRE_CLIENT_CERTIFICATE: &str = "druid.server.https.requir /// All secret-op generated keystores have one entry with the alias "1". /// (side node: I think technically they don't have an alias and the JVm counts them, but not sure) const TLS_ALIAS_NAME: &str = "1"; -pub const AUTH_TRUST_STORE_PATH: &str = "druid.auth.basic.ssl.trustStorePath"; -pub const AUTH_TRUST_STORE_TYPE: &str = "druid.auth.basic.ssl.trustStoreType"; -pub const AUTH_TRUST_STORE_PASSWORD: &str = "druid.auth.basic.ssl.trustStorePassword"; +const AUTH_TRUST_STORE_PATH: &str = "druid.auth.basic.ssl.trustStorePath"; +const AUTH_TRUST_STORE_TYPE: &str = "druid.auth.basic.ssl.trustStoreType"; +const AUTH_TRUST_STORE_PASSWORD: &str = "druid.auth.basic.ssl.trustStorePassword"; // Misc TLS pub const TLS_STORE_PASSWORD: &str = "changeit"; -pub const TLS_STORE_TYPE: &str = "pkcs12"; +const TLS_STORE_TYPE: &str = "pkcs12"; // directories const STACKABLE_MOUNT_TLS_DIR: &str = "/stackable/mount_tls"; @@ -102,12 +98,10 @@ const TLS_VOLUME_NAME: &str = "tls"; const TLS_MOUNT_VOLUME_NAME: &str = "tls-mount"; pub const INTERNAL_INITIAL_CLIENT_PASSWORD_ENV: &str = "INTERNAL_INITIAL_CLIENT_PASSWORD"; -// It seems this needs to be the same password for Druid to work, so we re-use the existing env variable from above. -pub const ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV: &str = INTERNAL_INITIAL_CLIENT_PASSWORD_ENV; impl DruidTlsSecurity { #[cfg(test)] - pub fn new( + pub(crate) fn new( auth_classes: &AuthenticationClassesResolved, server_and_internal_secret_class: Option, ) -> Self { @@ -148,7 +142,7 @@ impl DruidTlsSecurity { } /// Retrieve an optional TLS secret class for external client -> server and server <-> server communications. - pub fn tls_server_and_internal_secret_class(&self) -> Option<&str> { + fn tls_server_and_internal_secret_class(&self) -> Option<&str> { self.server_and_internal_secret_class.as_deref() } @@ -176,21 +170,15 @@ impl DruidTlsSecurity { .collect() } - pub fn listener_ports( - &self, - role: &DruidRole, - ) -> Option> { - let listener_ports = self - .exposed_ports(role) + pub fn listener_ports(&self, role: &DruidRole) -> Vec { + self.exposed_ports(role) .into_iter() .map(|(name, val)| listener::v1alpha1::ListenerPort { name, port: val.into(), protocol: Some("TCP".to_string()), }) - .collect(); - - Some(listener_ports) + .collect() } fn exposed_ports(&self, role: &DruidRole) -> Vec<(String, u16)> { diff --git a/rust/operator-binary/src/internal_secret.rs b/rust/operator-binary/src/internal_secret.rs index 0f398312..629c3ae6 100644 --- a/rust/operator-binary/src/internal_secret.rs +++ b/rust/operator-binary/src/internal_secret.rs @@ -144,7 +144,7 @@ pub async fn create_shared_internal_secret( Ok(()) } -pub fn build_shared_internal_secret(druid: &v1alpha1::DruidCluster) -> Result { +fn build_shared_internal_secret(druid: &v1alpha1::DruidCluster) -> Result { let mut internal_secret = BTreeMap::new(); internal_secret.insert( INTERNAL_INITIAL_CLIENT_PASSWORD_ENV.to_string(), @@ -164,7 +164,7 @@ pub fn build_shared_internal_secret(druid: &v1alpha1::DruidCluster) -> Result String { +fn build_immutable_shared_internal_secret_name(druid: &v1alpha1::DruidCluster) -> String { format!("{}-internal-secret", druid.name_any()) } diff --git a/rust/operator-binary/src/listener.rs b/rust/operator-binary/src/listener.rs index 2283cd76..6bc9247a 100644 --- a/rust/operator-binary/src/listener.rs +++ b/rust/operator-binary/src/listener.rs @@ -65,7 +65,7 @@ pub fn build_group_listener( .build(), spec: listener::v1alpha1::ListenerSpec { class_name: Some(listener_class), - ports: druid_tls_security.listener_ports(druid_role), + ports: Some(druid_tls_security.listener_ports(druid_role)), ..listener::v1alpha1::ListenerSpec::default() }, status: None, diff --git a/rust/operator-binary/src/webhooks/conversion.rs b/rust/operator-binary/src/webhooks/conversion.rs index 474a243f..67952135 100644 --- a/rust/operator-binary/src/webhooks/conversion.rs +++ b/rust/operator-binary/src/webhooks/conversion.rs @@ -8,7 +8,9 @@ use stackable_operator::{ }, }; -use crate::crd::{DruidCluster, DruidClusterVersion, FIELD_MANAGER}; +use crate::crd::{DruidCluster, DruidClusterVersion}; + +const FIELD_MANAGER: &str = "druid-operator"; /// Contains errors which can be encountered when creating the conversion webhook server and the /// CRD maintainer. From f753aeb821cd2b427ed0f30fe32f09772df2f65a Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 9 Jun 2026 20:44:33 +0200 Subject: [PATCH 26/48] refactor: cleanup duplication, use helpers --- .../src/authentication/ldap.rs | 27 +- .../operator-binary/src/authentication/mod.rs | 34 +- .../src/authentication/oidc.rs | 34 +- .../src/controller/build/config_map.rs | 19 +- rust/operator-binary/src/crd/mod.rs | 479 +++++------------- rust/operator-binary/src/crd/security.rs | 110 ++-- 6 files changed, 245 insertions(+), 458 deletions(-) diff --git a/rust/operator-binary/src/authentication/ldap.rs b/rust/operator-binary/src/authentication/ldap.rs index 7411fdea..730de69b 100644 --- a/rust/operator-binary/src/authentication/ldap.rs +++ b/rust/operator-binary/src/authentication/ldap.rs @@ -9,7 +9,10 @@ use stackable_operator::{ use super::{ AddLdapVolumesSnafu, ConstructLdapEndpointUrlSnafu, Error, MissingLdapBindCredentialsSnafu, }; -use crate::crd::security::{STACKABLE_TLS_DIR, TLS_STORE_PASSWORD, add_cert_to_trust_store_cmd}; +use crate::crd::{ + file_reference, + security::{STACKABLE_TLS_DIR, TLS_STORE_PASSWORD, add_cert_to_trust_store_cmd}, +}; fn add_authenticator_config( provider: &ldap::v1alpha1::AuthenticationProvider, @@ -40,11 +43,11 @@ fn add_authenticator_config( { config.insert( "druid.auth.authenticator.Ldap.credentialsValidator.bindUser".to_string(), - format!("${{file:UTF-8:{ldap_bind_user_path}}}").to_string(), + file_reference(ldap_bind_user_path), ); config.insert( "druid.auth.authenticator.Ldap.credentialsValidator.bindPassword".to_string(), - format!("${{file:UTF-8:{ldap_bind_password_path}}}").to_string(), + file_reference(ldap_bind_password_path), ); } @@ -64,31 +67,17 @@ fn add_authenticator_config( "druid.auth.authenticator.Ldap.authorizerName".to_string(), "LdapAuthorizer".to_string(), ); - config.insert( - "druid.auth.authenticatorChain".to_string(), - r#"["DruidSystemAuthenticator", "Ldap"]"#.to_string(), - ); + super::set_authenticator_chain(config, &["Ldap"]); Ok(()) } -fn add_authorizer_config(config: &mut BTreeMap) { - config.insert( - "druid.auth.authorizers".to_string(), - r#"["LdapAuthorizer", "DruidSystemAuthorizer"]"#.to_string(), - ); - config.insert( - "druid.auth.authorizer.LdapAuthorizer.type".to_string(), - r#"allowAll"#.to_string(), - ); -} - pub(super) fn generate_runtime_properties_config( provider: &ldap::v1alpha1::AuthenticationProvider, config: &mut BTreeMap, ) -> Result<(), Error> { add_authenticator_config(provider, config)?; - add_authorizer_config(config); + super::add_authorizer_config(config, "LdapAuthorizer"); Ok(()) } diff --git a/rust/operator-binary/src/authentication/mod.rs b/rust/operator-binary/src/authentication/mod.rs index 0342fcc1..73dc67ce 100644 --- a/rust/operator-binary/src/authentication/mod.rs +++ b/rust/operator-binary/src/authentication/mod.rs @@ -11,6 +11,7 @@ use crate::{ crd::{ DruidRole, authentication::{AuthenticationClassResolved, AuthenticationClassesResolved}, + env_var_reference, security::INTERNAL_INITIAL_CLIENT_PASSWORD_ENV, v1alpha1, }, @@ -23,6 +24,35 @@ pub mod oidc; // It seems this needs to be the same password for Druid to work, so we re-use the existing env variable. const ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV: &str = INTERNAL_INITIAL_CLIENT_PASSWORD_ENV; +/// Configures the given provider authorizer (e.g. `"LdapAuthorizer"`) alongside the Druid system +/// authorizer, both with the `allowAll` authorizer type. Shared by the LDAP and OIDC providers. +fn add_authorizer_config(config: &mut BTreeMap, authorizer_name: &str) { + config.insert( + "druid.auth.authorizers".to_string(), + format!(r#"["{authorizer_name}", "DruidSystemAuthorizer"]"#), + ); + config.insert( + format!("druid.auth.authorizer.{authorizer_name}.type"), + "allowAll".to_string(), + ); +} + +/// Sets `druid.auth.authenticatorChain` to the Druid system authenticator followed by the given +/// provider authenticators (e.g. `["Ldap"]`). Shared by the LDAP and OIDC providers. +fn set_authenticator_chain( + config: &mut BTreeMap, + provider_authenticators: &[&str], +) { + let authenticators: Vec = std::iter::once("DruidSystemAuthenticator") + .chain(provider_authenticators.iter().copied()) + .map(|name| format!("\"{name}\"")) + .collect(); + config.insert( + "druid.auth.authenticatorChain".to_string(), + format!("[{}]", authenticators.join(", ")), + ); +} + type Result = std::result::Result; #[derive(Snafu, Debug)] @@ -190,7 +220,7 @@ impl DruidAuthenticationConfig { config.insert( "druid.auth.authenticator.DruidSystemAuthenticator.initialInternalClientPassword" .to_string(), - format!("${{env:{INTERNAL_INITIAL_CLIENT_PASSWORD_ENV}}}").to_string(), + env_var_reference(INTERNAL_INITIAL_CLIENT_PASSWORD_ENV), ); config.insert( "druid.auth.authenticator.DruidSystemAuthenticator.authorizerName".to_string(), @@ -212,7 +242,7 @@ impl DruidAuthenticationConfig { ); config.insert( "druid.escalator.internalClientPassword".to_string(), - format!("${{env:{ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV}}}").to_string(), + env_var_reference(ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV), ); config.insert( "druid.escalator.authorizerName".to_string(), diff --git a/rust/operator-binary/src/authentication/oidc.rs b/rust/operator-binary/src/authentication/oidc.rs index cb018426..7540531f 100644 --- a/rust/operator-binary/src/authentication/oidc.rs +++ b/rust/operator-binary/src/authentication/oidc.rs @@ -9,7 +9,10 @@ use stackable_operator::{ use super::{AddOidcVolumesSnafu, ConstructOidcWellKnownUrlSnafu, Error}; use crate::{ - crd::{COOKIE_PASSPHRASE_ENV, DruidRole, security::add_cert_to_jvm_trust_store_cmd}, + crd::{ + COOKIE_PASSPHRASE_ENV, DruidRole, env_var_reference, + security::add_cert_to_jvm_trust_store_cmd, + }, internal_secret::env_var_from_secret, }; @@ -46,15 +49,15 @@ fn add_authenticator_config( ); config.insert( "druid.auth.pac4j.cookiePassphrase".to_string(), - format!("${{env:{COOKIE_PASSPHRASE_ENV}}}").to_string(), + env_var_reference(COOKIE_PASSPHRASE_ENV), ); config.insert( "druid.auth.pac4j.oidc.clientID".to_string(), - format!("${{env:{oidc_client_id_env}}}").to_string(), + env_var_reference(oidc_client_id_env), ); config.insert( "druid.auth.pac4j.oidc.clientSecret".to_string(), - format!("${{env:{oidc_client_secret_env}}}").to_string(), + env_var_reference(oidc_client_secret_env), ); config.insert( "druid.auth.pac4j.oidc.discoveryURI".to_string(), @@ -75,25 +78,11 @@ fn add_authenticator_config( method_string.to_string(), ); - config.insert( - "druid.auth.authenticatorChain".to_string(), - r#"["DruidSystemAuthenticator", "Oidc"]"#.to_string(), - ); + super::set_authenticator_chain(config, &["Oidc"]); Ok(()) } -fn add_authorizer_config(config: &mut BTreeMap) { - config.insert( - "druid.auth.authorizers".to_string(), - r#"["OidcAuthorizer", "DruidSystemAuthorizer"]"#.to_string(), - ); - config.insert( - "druid.auth.authorizer.OidcAuthorizer.type".to_string(), - r#"allowAll"#.to_string(), - ); -} - /// Creates the OIDC parts of the runtime.properties config file. /// OIDC authentication is not configured on middlemanagers, because end users don't interact with them directly using the web console and /// turning on OIDC will lead to problems with the communication with coordinators during data ingest. @@ -105,14 +94,11 @@ pub(super) fn generate_runtime_properties_config( ) -> Result<(), Error> { match role { DruidRole::MiddleManager => { - config.insert( - "druid.auth.authenticatorChain".to_string(), - r#"["DruidSystemAuthenticator"]"#.to_string(), - ); + super::set_authenticator_chain(config, &[]); } _ => { add_authenticator_config(provider, oidc, config)?; - add_authorizer_config(config) + super::add_authorizer_config(config, "OidcAuthorizer"); } } Ok(()) diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index ad971122..a86423f1 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -34,7 +34,10 @@ use crate::{ }, validate::{DruidRoleGroupConfig, ValidatedCluster}, }, - crd::{DruidRole, build_recommended_labels, build_string_list, v1alpha1}, + crd::{ + DruidRole, build_recommended_labels, build_string_list, env_var_reference, file_reference, + v1alpha1, + }, }; // Druid `runtime.properties` config-property keys assembled into the rolegroup ConfigMap here. @@ -154,7 +157,7 @@ pub fn build_rolegroup_config_map( { conf.insert( crate::crd::database::METADATA_STORAGE_USER.to_string(), - format!("${{env:{username_env_name}}}",), + env_var_reference(username_env_name), ); } @@ -165,7 +168,7 @@ pub fn build_rolegroup_config_map( { conf.insert( crate::crd::database::METADATA_STORAGE_PASSWORD.to_string(), - format!("${{env:{password_env_name}}}",), + env_var_reference(password_env_name), ); } @@ -187,14 +190,8 @@ pub fn build_rolegroup_config_map( ); if let Some((access_key_file, secret_key_file)) = s3.credentials_mount_paths() { - conf.insert( - S3_ACCESS_KEY.to_string(), - format!("${{file:UTF-8:{access_key_file}}}"), - ); - conf.insert( - S3_SECRET_KEY.to_string(), - format!("${{file:UTF-8:{secret_key_file}}}"), - ); + conf.insert(S3_ACCESS_KEY.to_string(), file_reference(access_key_file)); + conf.insert(S3_SECRET_KEY.to_string(), file_reference(secret_key_file)); } conf.insert( diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index abb01520..794eba3c 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -92,6 +92,18 @@ pub const METRICS_PORT: u16 = 9090; pub const COOKIE_PASSPHRASE_ENV: &str = "OIDC_COOKIE_PASSPHRASE"; +/// Formats a Druid [dynamic config](https://druid.apache.org/docs/latest/operations/dynamic-config-provider) +/// reference to an environment variable, i.e. `${env:NAME}`. +pub(crate) fn env_var_reference(name: impl std::fmt::Display) -> String { + format!("${{env:{name}}}") +} + +/// Formats a Druid [dynamic config](https://druid.apache.org/docs/latest/operations/dynamic-config-provider) +/// reference to the UTF-8 contents of a file, i.e. `${file:UTF-8:PATH}`. +pub(crate) fn file_reference(path: impl std::fmt::Display) -> String { + format!("${{file:UTF-8:{path}}}") +} + // Graceful shutdown timeouts const DEFAULT_BROKER_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(5); const DEFAULT_COORDINATOR_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(5); @@ -596,37 +608,21 @@ impl v1alpha1::DruidCluster { role: &DruidRole, role_group: &str, ) -> Option<&PodTemplateSpec> { + macro_rules! pod_overrides { + ($field:ident) => { + self.spec + .$field + .role_groups + .get(role_group) + .map(|rg| &rg.config.pod_overrides) + }; + } match role { - DruidRole::Broker => self - .spec - .brokers - .role_groups - .get(role_group) - .map(|rg| &rg.config.pod_overrides), - DruidRole::Coordinator => self - .spec - .coordinators - .role_groups - .get(role_group) - .map(|rg| &rg.config.pod_overrides), - DruidRole::Historical => self - .spec - .historicals - .role_groups - .get(role_group) - .map(|rg| &rg.config.pod_overrides), - DruidRole::MiddleManager => self - .spec - .middle_managers - .role_groups - .get(role_group) - .map(|rg| &rg.config.pod_overrides), - DruidRole::Router => self - .spec - .routers - .role_groups - .get(role_group) - .map(|rg| &rg.config.pod_overrides), + DruidRole::Broker => pod_overrides!(brokers), + DruidRole::Coordinator => pod_overrides!(coordinators), + DruidRole::Historical => pod_overrides!(historicals), + DruidRole::MiddleManager => pod_overrides!(middle_managers), + DruidRole::Router => pod_overrides!(routers), } } } @@ -691,88 +687,16 @@ impl MergedConfig { role: &DruidRole, rolegroup_name: &str, ) -> Result { - match role { - DruidRole::Broker => { - let rolegroup = self - .brokers - .get(rolegroup_name) - .context(CannotRetrieveRoleGroupSnafu { rolegroup_name })?; - Ok(CommonRoleGroupConfig { - resources: RoleResource::Druid(rolegroup.config.config.resources.to_owned()), - logging: rolegroup.config.config.logging.to_owned(), - replicas: rolegroup.replicas, - affinity: rolegroup.config.config.affinity.clone(), - graceful_shutdown_timeout: rolegroup.config.config.graceful_shutdown_timeout, - requested_secret_lifetime: rolegroup - .config - .config - .requested_secret_lifetime - .context(MissingSecretLifetimeSnafu)?, - }) - } - DruidRole::Coordinator => { - let rolegroup = self - .coordinators - .get(rolegroup_name) - .context(CannotRetrieveRoleGroupSnafu { rolegroup_name })?; - Ok(CommonRoleGroupConfig { - resources: RoleResource::Druid(rolegroup.config.config.resources.to_owned()), - logging: rolegroup.config.config.logging.to_owned(), - replicas: rolegroup.replicas, - affinity: rolegroup.config.config.affinity.clone(), - graceful_shutdown_timeout: rolegroup.config.config.graceful_shutdown_timeout, - requested_secret_lifetime: rolegroup - .config - .config - .requested_secret_lifetime - .context(MissingSecretLifetimeSnafu)?, - }) - } - DruidRole::Historical => { - let rolegroup = self - .historicals - .get(rolegroup_name) - .context(CannotRetrieveRoleGroupSnafu { rolegroup_name })?; - Ok(CommonRoleGroupConfig { - resources: RoleResource::Historical( - rolegroup.config.config.resources.to_owned(), - ), - logging: rolegroup.config.config.logging.to_owned(), - replicas: rolegroup.replicas, - affinity: rolegroup.config.config.affinity.clone(), - graceful_shutdown_timeout: rolegroup.config.config.graceful_shutdown_timeout, - requested_secret_lifetime: rolegroup - .config - .config - .requested_secret_lifetime - .context(MissingSecretLifetimeSnafu)?, - }) - } - DruidRole::MiddleManager => { - let rolegroup = self - .middle_managers - .get(rolegroup_name) - .context(CannotRetrieveRoleGroupSnafu { rolegroup_name })?; - Ok(CommonRoleGroupConfig { - resources: RoleResource::Druid(rolegroup.config.config.resources.to_owned()), - logging: rolegroup.config.config.logging.to_owned(), - replicas: rolegroup.replicas, - affinity: rolegroup.config.config.affinity.clone(), - graceful_shutdown_timeout: rolegroup.config.config.graceful_shutdown_timeout, - requested_secret_lifetime: rolegroup - .config - .config - .requested_secret_lifetime - .context(MissingSecretLifetimeSnafu)?, - }) - } - DruidRole::Router => { + // All roles build an identical `CommonRoleGroupConfig`; only the role-group map and the + // `RoleResource` variant (historicals carry typed storage) differ. + macro_rules! common_config { + ($field:ident, $resource:path) => {{ let rolegroup = self - .routers + .$field .get(rolegroup_name) .context(CannotRetrieveRoleGroupSnafu { rolegroup_name })?; Ok(CommonRoleGroupConfig { - resources: RoleResource::Druid(rolegroup.config.config.resources.to_owned()), + resources: $resource(rolegroup.config.config.resources.to_owned()), logging: rolegroup.config.config.logging.to_owned(), replicas: rolegroup.replicas, affinity: rolegroup.config.config.affinity.clone(), @@ -783,7 +707,14 @@ impl MergedConfig { .requested_secret_lifetime .context(MissingSecretLifetimeSnafu)?, }) - } + }}; + } + match role { + DruidRole::Broker => common_config!(brokers, RoleResource::Druid), + DruidRole::Coordinator => common_config!(coordinators, RoleResource::Druid), + DruidRole::Historical => common_config!(historicals, RoleResource::Historical), + DruidRole::MiddleManager => common_config!(middle_managers, RoleResource::Druid), + DruidRole::Router => common_config!(routers, RoleResource::Druid), } } @@ -1040,250 +971,102 @@ pub struct IngestionSpec { pub s3connection: Option, } -#[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] -#[fragment_attrs( - derive( - Clone, - Debug, - Default, - Deserialize, - Merge, - JsonSchema, - PartialEq, - Serialize - ), - serde(rename_all = "camelCase") -)] -pub struct BrokerConfig { - #[fragment_attrs(serde(default))] - resources: Resources, - #[fragment_attrs(serde(default))] - pub logging: Logging, - #[fragment_attrs(serde(default))] - pub affinity: StackableAffinity, - /// The time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. - /// Read more about graceful shutdown in the - /// [graceful shutdown documentation](DOCS_BASE_URL_PLACEHOLDER/druid/usage-guide/operations/graceful-shutdown). - #[fragment_attrs(serde(default))] - pub graceful_shutdown_timeout: Option, - - /// Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. - /// This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. - #[fragment_attrs(serde(default))] - pub requested_secret_lifetime: Option, -} - -impl BrokerConfig { - fn default_config( - cluster_name: &str, - role: &DruidRole, - deep_storage: &DeepStorageSpec, - ) -> BrokerConfigFragment { - BrokerConfigFragment { - resources: resource::BROKER_RESOURCES.to_owned(), - logging: product_logging::spec::default_logging(), - affinity: get_affinity(cluster_name, role, deep_storage), - graceful_shutdown_timeout: Some(role.default_graceful_shutdown_timeout()), - requested_secret_lifetime: Some(DEFAULT_BROKER_SECRET_LIFETIME), - } - } -} - -#[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] -#[fragment_attrs( - derive( - Clone, - Debug, - Default, - Deserialize, - Merge, - JsonSchema, - PartialEq, - Serialize - ), - serde(rename_all = "camelCase") -)] -pub struct CoordinatorConfig { - #[fragment_attrs(serde(default))] - resources: Resources, - #[fragment_attrs(serde(default))] - pub logging: Logging, - #[fragment_attrs(serde(default))] - pub affinity: StackableAffinity, - /// The time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. - /// Read more about graceful shutdown in the - /// [graceful shutdown documentation](DOCS_BASE_URL_PLACEHOLDER/druid/usage-guide/operations/graceful-shutdown). - #[fragment_attrs(serde(default))] - pub graceful_shutdown_timeout: Option, - - /// Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. - /// This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. - #[fragment_attrs(serde(default))] - pub requested_secret_lifetime: Option, -} - -impl CoordinatorConfig { - fn default_config( - cluster_name: &str, - role: &DruidRole, - deep_storage: &DeepStorageSpec, - ) -> CoordinatorConfigFragment { - CoordinatorConfigFragment { - resources: resource::COORDINATOR_RESOURCES.to_owned(), - logging: product_logging::spec::default_logging(), - affinity: get_affinity(cluster_name, role, deep_storage), - graceful_shutdown_timeout: Some(role.default_graceful_shutdown_timeout()), - requested_secret_lifetime: Some(DEFAULT_COORDINATOR_SECRET_LIFETIME), - } - } -} - -#[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] -#[fragment_attrs( - derive( - Clone, - Debug, - Default, - Deserialize, - Merge, - JsonSchema, - PartialEq, - Serialize - ), - serde(rename_all = "camelCase") -)] -pub struct MiddleManagerConfig { - #[fragment_attrs(serde(default))] - resources: Resources, - #[fragment_attrs(serde(default))] - pub logging: Logging, - #[fragment_attrs(serde(default))] - pub affinity: StackableAffinity, - /// The time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. - /// Read more about graceful shutdown in the - /// [graceful shutdown documentation](DOCS_BASE_URL_PLACEHOLDER/druid/usage-guide/operations/graceful-shutdown). - #[fragment_attrs(serde(default))] - pub graceful_shutdown_timeout: Option, - - /// Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. - /// This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. - #[fragment_attrs(serde(default))] - pub requested_secret_lifetime: Option, -} - -impl MiddleManagerConfig { - fn default_config( - cluster_name: &str, - role: &DruidRole, - deep_storage: &DeepStorageSpec, - ) -> MiddleManagerConfigFragment { - MiddleManagerConfigFragment { - resources: resource::MIDDLE_MANAGER_RESOURCES.to_owned(), - logging: product_logging::spec::default_logging(), - affinity: get_affinity(cluster_name, role, deep_storage), - graceful_shutdown_timeout: Some(role.default_graceful_shutdown_timeout()), - requested_secret_lifetime: Some(DEFAULT_MIDDLE_SECRET_LIFETIME), +/// Generates a per-role config struct and its `default_config`. The four non-historical roles share +/// an identical shape; only the storage type, default [`resource`] profile and default secret +/// lifetime differ. The `Fragment` type (named `Fragment` by the [`Fragment`] derive) is +/// passed explicitly because `macro_rules!` cannot synthesize identifiers. +macro_rules! role_group_config { + ($name:ident, $fragment:ident, $storage:ty, $resources:expr, $secret_lifetime:expr $(,)?) => { + #[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] + #[fragment_attrs( + derive( + Clone, + Debug, + Default, + Deserialize, + Merge, + JsonSchema, + PartialEq, + Serialize + ), + serde(rename_all = "camelCase") + )] + pub struct $name { + #[fragment_attrs(serde(default))] + resources: Resources<$storage, NoRuntimeLimits>, + #[fragment_attrs(serde(default))] + pub logging: Logging, + #[fragment_attrs(serde(default))] + pub affinity: StackableAffinity, + /// The time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. + /// Read more about graceful shutdown in the + /// [graceful shutdown documentation](DOCS_BASE_URL_PLACEHOLDER/druid/usage-guide/operations/graceful-shutdown). + #[fragment_attrs(serde(default))] + pub graceful_shutdown_timeout: Option, + + /// Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. + /// This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. + #[fragment_attrs(serde(default))] + pub requested_secret_lifetime: Option, } - } -} - -#[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] -#[fragment_attrs( - derive( - Clone, - Debug, - Default, - Deserialize, - Merge, - JsonSchema, - PartialEq, - Serialize - ), - serde(rename_all = "camelCase") -)] -pub struct RouterConfig { - #[fragment_attrs(serde(default))] - resources: Resources, - #[fragment_attrs(serde(default))] - pub logging: Logging, - #[fragment_attrs(serde(default))] - pub affinity: StackableAffinity, - /// The time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. - /// Read more about graceful shutdown in the - /// [graceful shutdown documentation](DOCS_BASE_URL_PLACEHOLDER/druid/usage-guide/operations/graceful-shutdown). - #[fragment_attrs(serde(default))] - pub graceful_shutdown_timeout: Option, - - /// Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. - /// This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. - #[fragment_attrs(serde(default))] - pub requested_secret_lifetime: Option, -} -impl RouterConfig { - fn default_config( - cluster_name: &str, - role: &DruidRole, - deep_storage: &DeepStorageSpec, - ) -> RouterConfigFragment { - RouterConfigFragment { - resources: resource::ROUTER_RESOURCES.to_owned(), - logging: product_logging::spec::default_logging(), - affinity: get_affinity(cluster_name, role, deep_storage), - graceful_shutdown_timeout: Some(role.default_graceful_shutdown_timeout()), - requested_secret_lifetime: Some(DEFAULT_ROUTER_SECRET_LIFETIME), + impl $name { + fn default_config( + cluster_name: &str, + role: &DruidRole, + deep_storage: &DeepStorageSpec, + ) -> $fragment { + $fragment { + resources: $resources.to_owned(), + logging: product_logging::spec::default_logging(), + affinity: get_affinity(cluster_name, role, deep_storage), + graceful_shutdown_timeout: Some(role.default_graceful_shutdown_timeout()), + requested_secret_lifetime: Some($secret_lifetime), + } + } } - } -} - -#[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] -#[fragment_attrs( - derive( - Clone, - Debug, - Default, - Deserialize, - Merge, - JsonSchema, - PartialEq, - Serialize - ), - serde(rename_all = "camelCase") -)] -pub struct HistoricalConfig { - #[fragment_attrs(serde(default))] - resources: Resources, - #[fragment_attrs(serde(default))] - pub logging: Logging, - #[fragment_attrs(serde(default))] - pub affinity: StackableAffinity, - /// The time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. - /// Read more about graceful shutdown in the - /// [graceful shutdown documentation](DOCS_BASE_URL_PLACEHOLDER/druid/usage-guide/operations/graceful-shutdown). - #[fragment_attrs(serde(default))] - pub graceful_shutdown_timeout: Option, - - /// Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. - /// This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. - #[fragment_attrs(serde(default))] - pub requested_secret_lifetime: Option, + }; } -impl HistoricalConfig { - fn default_config( - cluster_name: &str, - role: &DruidRole, - deep_storage: &DeepStorageSpec, - ) -> HistoricalConfigFragment { - HistoricalConfigFragment { - resources: resource::HISTORICAL_RESOURCES.to_owned(), - logging: product_logging::spec::default_logging(), - affinity: get_affinity(cluster_name, role, deep_storage), - graceful_shutdown_timeout: Some(role.default_graceful_shutdown_timeout()), - requested_secret_lifetime: Some(DEFAULT_HISTORICAL_SECRET_LIFETIME), - } - } -} +role_group_config!( + BrokerConfig, + BrokerConfigFragment, + storage::DruidStorage, + resource::BROKER_RESOURCES, + DEFAULT_BROKER_SECRET_LIFETIME, +); + +role_group_config!( + CoordinatorConfig, + CoordinatorConfigFragment, + storage::DruidStorage, + resource::COORDINATOR_RESOURCES, + DEFAULT_COORDINATOR_SECRET_LIFETIME, +); + +role_group_config!( + MiddleManagerConfig, + MiddleManagerConfigFragment, + storage::DruidStorage, + resource::MIDDLE_MANAGER_RESOURCES, + DEFAULT_MIDDLE_SECRET_LIFETIME, +); + +role_group_config!( + RouterConfig, + RouterConfigFragment, + storage::DruidStorage, + resource::ROUTER_RESOURCES, + DEFAULT_ROUTER_SECRET_LIFETIME, +); + +role_group_config!( + HistoricalConfig, + HistoricalConfigFragment, + storage::HistoricalStorage, + resource::HISTORICAL_RESOURCES, + DEFAULT_HISTORICAL_SECRET_LIFETIME, +); #[derive(Clone, Debug, Default, Deserialize, JsonSchema, Serialize)] #[serde(rename_all = "camelCase")] diff --git a/rust/operator-binary/src/crd/security.rs b/rust/operator-binary/src/crd/security.rs index 2756d72b..9e67b8dd 100644 --- a/rust/operator-binary/src/crd/security.rs +++ b/rust/operator-binary/src/crd/security.rs @@ -287,53 +287,63 @@ impl DruidTlsSecurity { } } - fn add_tls_encryption_config_properties( + /// Inserts the path/type/password triple describing a single PKCS12 keystore or truststore + /// (`/`), keeping the store type and password consistent across + /// all of Druid's HTTPS store settings. + fn add_pkcs12_store_properties( config: &mut BTreeMap, store_directory: &str, - store_alias: &str, + store_file: &str, + path_property: &str, + type_property: &str, + password_property: &str, ) { - // We need a truststore in addition to a keystore here, because server and internal tls - // can only be enabled/disabled together - config.insert( - CLIENT_HTTPS_TRUST_STORE_PATH.to_string(), - format!("{}/truststore.p12", store_directory), - ); config.insert( - CLIENT_HTTPS_TRUST_STORE_TYPE.to_string(), - TLS_STORE_TYPE.to_string(), + path_property.to_string(), + format!("{store_directory}/{store_file}"), ); + config.insert(type_property.to_string(), TLS_STORE_TYPE.to_string()); config.insert( - CLIENT_HTTPS_TRUST_STORE_PASSWORD.to_string(), + password_property.to_string(), TLS_STORE_PASSWORD.to_string(), ); + } - config.insert( - SERVER_HTTPS_KEY_STORE_PATH.to_string(), - format!("{}/keystore.p12", store_directory), - ); - config.insert( - SERVER_HTTPS_KEY_STORE_TYPE.to_string(), - TLS_STORE_TYPE.to_string(), + fn add_tls_encryption_config_properties( + config: &mut BTreeMap, + store_directory: &str, + store_alias: &str, + ) { + // We need a truststore in addition to a keystore here, because server and internal tls + // can only be enabled/disabled together + Self::add_pkcs12_store_properties( + config, + store_directory, + "truststore.p12", + CLIENT_HTTPS_TRUST_STORE_PATH, + CLIENT_HTTPS_TRUST_STORE_TYPE, + CLIENT_HTTPS_TRUST_STORE_PASSWORD, ); - config.insert( - SERVER_HTTPS_KEY_STORE_PASSWORD.to_string(), - TLS_STORE_PASSWORD.to_string(), + + Self::add_pkcs12_store_properties( + config, + store_directory, + "keystore.p12", + SERVER_HTTPS_KEY_STORE_PATH, + SERVER_HTTPS_KEY_STORE_TYPE, + SERVER_HTTPS_KEY_STORE_PASSWORD, ); config.insert(SERVER_HTTPS_CERT_ALIAS.to_string(), store_alias.to_string()); // We also need to configure the truststore for authentication related stuff, // such as verifying the LDAP server - config.insert( - AUTH_TRUST_STORE_PATH.to_string(), - format!("{}/truststore.p12", store_directory), - ); - config.insert( - AUTH_TRUST_STORE_TYPE.to_string(), - TLS_STORE_TYPE.to_string(), - ); - config.insert( - AUTH_TRUST_STORE_PASSWORD.to_string(), - TLS_STORE_PASSWORD.to_string(), + Self::add_pkcs12_store_properties( + config, + store_directory, + "truststore.p12", + AUTH_TRUST_STORE_PATH, + AUTH_TRUST_STORE_TYPE, + AUTH_TRUST_STORE_PASSWORD, ); } @@ -342,17 +352,13 @@ impl DruidTlsSecurity { store_directory: &str, store_alias: &str, ) { - config.insert( - CLIENT_HTTPS_KEY_STORE_PATH.to_string(), - format!("{store_directory}/keystore.p12"), - ); - config.insert( - CLIENT_HTTPS_KEY_STORE_TYPE.to_string(), - TLS_STORE_TYPE.to_string(), - ); - config.insert( - CLIENT_HTTPS_KEY_STORE_PASSWORD.to_string(), - TLS_STORE_PASSWORD.to_string(), + Self::add_pkcs12_store_properties( + config, + store_directory, + "keystore.p12", + CLIENT_HTTPS_KEY_STORE_PATH, + CLIENT_HTTPS_KEY_STORE_TYPE, + CLIENT_HTTPS_KEY_STORE_PASSWORD, ); // This is required because PKCS12 does not use any key passwords but it will // be checked and would lead to an exception: @@ -377,17 +383,13 @@ impl DruidTlsSecurity { "true".to_string(), ); - config.insert( - SERVER_HTTPS_TRUST_STORE_PATH.to_string(), - format!("{store_directory}/truststore.p12"), - ); - config.insert( - SERVER_HTTPS_TRUST_STORE_TYPE.to_string(), - TLS_STORE_TYPE.to_string(), - ); - config.insert( - SERVER_HTTPS_TRUST_STORE_PASSWORD.to_string(), - TLS_STORE_PASSWORD.to_string(), + Self::add_pkcs12_store_properties( + config, + store_directory, + "truststore.p12", + SERVER_HTTPS_TRUST_STORE_PATH, + SERVER_HTTPS_TRUST_STORE_TYPE, + SERVER_HTTPS_TRUST_STORE_PASSWORD, ); // This is required because PKCS12 does not use any key passwords but it will // be checked and would lead to an exception: From 8f92ec9045856c919a9f31ee89d88636247bf9a9 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 9 Jun 2026 21:06:33 +0200 Subject: [PATCH 27/48] refactor: switch to EnvVarSet --- rust/operator-binary/src/controller.rs | 23 +++++------- .../src/controller/validate.rs | 35 +++++++++++++++---- 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 84bf2a8f..c0dc29af 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -624,16 +624,8 @@ fn build_rolegroup_statefulset( metadata_database_connection_details.add_to_container(&mut cb_druid); - // rest of env - let mut rest_env = rg - .env - .iter() - .map(|(k, v)| EnvVar { - name: k.clone(), - value: Some(v.clone()), - ..EnvVar::default() - }) - .collect::>(); + // rest of env: the validated env overrides, rendered in sorted-by-name order. + let mut rest_env: Vec = rg.env.clone().into(); if let Some(auth_config) = druid_auth_config { rest_env.extend(auth_config.get_env_var_mounts(druid, role)) @@ -939,9 +931,12 @@ mod test { use rstest::*; use stackable_operator::{ database_connections::drivers::jdbc::JdbcDatabaseConnection, - v2::types::{ - kubernetes::{NamespaceName, Uid}, - operator::ClusterName, + v2::{ + builder::pod::container::EnvVarSet, + types::{ + kubernetes::{NamespaceName, Uid}, + operator::ClusterName, + }, }, }; @@ -1010,7 +1005,7 @@ mod test { merged_config, runtime_config: runtime_properties::defaults(&DruidRole::Historical), security_config: BTreeMap::new(), - env: BTreeMap::new(), + env: EnvVarSet::new(), // The test only asserts on runtime.properties, so the rendered jvm.config is irrelevant. jvm_config: String::new(), }; diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 0438652d..85e71e71 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -6,6 +6,7 @@ use std::{ borrow::Cow, collections::{BTreeMap, HashMap}, + str::FromStr, }; use snafu::{ResultExt, Snafu}; @@ -18,6 +19,7 @@ use stackable_operator::{ kube::{Resource, api::ObjectMeta}, v2::{ HasName, HasUid, + builder::pod::container::{self, EnvVarName, EnvVarSet}, controller_utils::{get_cluster_name, get_namespace, get_uid}, types::{ kubernetes::{NamespaceName, Uid}, @@ -76,6 +78,9 @@ pub enum Error { InvalidMetadataDatabaseConnection { source: stackable_operator::database_connections::Error, }, + + #[snafu(display("invalid environment variable override name"))] + ParseEnvVarName { source: container::Error }, } type Result = std::result::Result; @@ -92,8 +97,9 @@ pub struct DruidRoleGroupConfig { pub runtime_config: BTreeMap, /// The security.properties "validated config". pub security_config: BTreeMap, - /// Merged env overrides (role <- rolegroup). Druid has no computed env vars. - pub env: BTreeMap, + /// Merged env overrides (role <- rolegroup). Druid has no computed env vars. Names are + /// validated here so the build step can render them directly into the container. + pub env: EnvVarSet, /// The fully rendered `jvm.config` (operator defaults merged with the role/rolegroup JVM /// argument overrides). Precomputed here so the config-map builder no longer needs the raw /// cluster's `get_role`. @@ -228,6 +234,25 @@ fn key_value_overrides( .collect() } +/// Merges the role-level and rolegroup-level env overrides into a validated [`EnvVarSet`]. +/// +/// The role is processed first, then the rolegroup, so that rolegroup overrides win on key +/// collisions ([`EnvVarSet::with_value`] overrides earlier entries with the same name). The +/// override names are validated here so the build step can render them directly. +fn merged_env_overrides( + role_env_overrides: &HashMap, + rg_env_overrides: &HashMap, +) -> Result { + let mut env = EnvVarSet::new(); + for (name, value) in role_env_overrides.iter().chain(rg_env_overrides.iter()) { + env = env.with_value( + &EnvVarName::from_str(name).context(ParseEnvVarNameSnafu)?, + value.clone(), + ); + } + Ok(env) +} + /// Builds the precomputed per-file config for a single rolegroup. Pure assembly: combines the /// role-level overrides with the rolegroup-level overrides (rolegroup wins) on top of the /// computed defaults. No behavior change vs. the inline loop body it was extracted from. @@ -269,11 +294,7 @@ fn build_role_group_config( security_config.extend(security_properties::build(&security_overrides)); // ----- env ----- - let mut env: BTreeMap = role_env_overrides - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - env.extend(rg_env_overrides.iter().map(|(k, v)| (k.clone(), v.clone()))); + let env = merged_env_overrides(role_env_overrides, rg_env_overrides)?; // ----- jvm.config ----- let (heap, direct) = merged_config From e1ebe064e6931f37f4d09ed0c9c25295cab60e8b Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 9 Jun 2026 21:59:26 +0200 Subject: [PATCH 28/48] refactor: switch to EnvVarSet, introduce role_utils --- rust/operator-binary/src/controller.rs | 59 ++- .../src/controller/build/config_map.rs | 125 +++++- .../src/controller/build/jvm.rs | 7 +- .../build/properties/runtime_properties.rs | 58 ++- .../src/controller/validate.rs | 244 ++-------- rust/operator-binary/src/crd/affinity.rs | 9 +- rust/operator-binary/src/crd/mod.rs | 422 ++++++------------ rust/operator-binary/src/crd/resource.rs | 62 +-- rust/operator-binary/src/framework.rs | 11 + .../src/framework/role_utils.rs | 152 +++++++ rust/operator-binary/src/main.rs | 1 + 11 files changed, 549 insertions(+), 601 deletions(-) create mode 100644 rust/operator-binary/src/framework.rs create mode 100644 rust/operator-binary/src/framework/role_utils.rs diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index c0dc29af..a569b7ee 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -504,7 +504,7 @@ fn build_rolegroup_statefulset( druid_auth_config: &Option, service_account: &ServiceAccount, ) -> Result { - let merged_rolegroup_config = &rg.merged_config; + let merged_rolegroup_config = &rg.config; // prepare container builder let prepare_container_name = Container::Prepare.to_string(); let mut cb_prepare = ContainerBuilder::new(&prepare_container_name).context( @@ -625,7 +625,7 @@ fn build_rolegroup_statefulset( metadata_database_connection_details.add_to_container(&mut cb_druid); // rest of env: the validated env overrides, rendered in sorted-by-name order. - let mut rest_env: Vec = rg.env.clone().into(); + let mut rest_env: Vec = rg.env_overrides.clone().into(); if let Some(auth_config) = druid_auth_config { rest_env.extend(auth_config.get_env_var_mounts(druid, role)) @@ -757,11 +757,8 @@ fn build_rolegroup_statefulset( } let mut pod_template = pb.build_template(); - pod_template.merge_from(druid.pod_overrides_for_role(role).clone()); - if let Some(pod_overrides) = druid.pod_overrides_for_role_group(role, &rolegroup_ref.role_group) - { - pod_template.merge_from(pod_overrides.clone()); - } + // The role and rolegroup pod overrides were already merged (rolegroup wins) during validation. + pod_template.merge_from(rg.pod_overrides.clone()); Ok(StatefulSet { metadata: ObjectMetaBuilder::new() @@ -931,23 +928,17 @@ mod test { use rstest::*; use stackable_operator::{ database_connections::drivers::jdbc::JdbcDatabaseConnection, - v2::{ - builder::pod::container::EnvVarSet, - types::{ - kubernetes::{NamespaceName, Uid}, - operator::ClusterName, - }, + v2::types::{ + kubernetes::{NamespaceName, Uid}, + operator::ClusterName, }, }; use super::*; use crate::{ controller::{ - build::{ - config_map::build_rolegroup_config_map, - properties::{ConfigFileName, runtime_properties}, - }, - validate::{DruidRoleGroupConfig, ValidatedCluster, ValidatedClusterConfig}, + build::{config_map::build_rolegroup_config_map, properties::ConfigFileName}, + validate::{ValidatedCluster, ValidatedClusterConfig}, }, crd::{PROP_SEGMENT_CACHE_LOCATIONS, authentication::AuthenticationClassesResolved}, extensions::get_extension_list, @@ -993,22 +984,22 @@ mod test { Some("tls".to_string()), ); - let merged = druid.merged_config().expect("merged config"); - let merged_config = merged - .common_config(&DruidRole::Historical, tested_rolegroup_name) - .expect("common config for tested rolegroup"); - // The segment cache property is injected dynamically by the config_map builder from the - // merged resources, independent of the precomputed runtime_config. We still populate the - // runtime_config with the static role defaults to mirror the production path. - let rg = DruidRoleGroupConfig { - merged_config, - runtime_config: runtime_properties::defaults(&DruidRole::Historical), - security_config: BTreeMap::new(), - env: EnvVarSet::new(), - // The test only asserts on runtime.properties, so the rendered jvm.config is irrelevant. - jvm_config: String::new(), - }; + // merged resources of the validated role group config. + let rg = druid + .merged_role(&DruidRole::Historical) + .expect("merged historical role") + .get(tested_rolegroup_name) + .expect("tested rolegroup") + .clone(); + + // The build step renders jvm.config from the erased role; populate the one role the test + // exercises. + let mut roles = BTreeMap::new(); + roles.insert( + DruidRole::Historical, + druid.get_role(&DruidRole::Historical), + ); let extensions = get_extension_list(&druid, &druid_tls_security, &None); let metadata_storage_type = druid @@ -1039,8 +1030,10 @@ mod test { extensions, metadata_storage_type, metadata_db_connection, + deep_storage: druid.spec.cluster_config.deep_storage.clone(), }, BTreeMap::new(), + roles, ); let rolegroup_ref = RoleGroupRef { diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index a86423f1..b17dc841 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -1,15 +1,15 @@ //! Builds the rolegroup [`ConfigMap`] from a [`ValidatedCluster`]. //! -//! The per-file "validated config" maps (runtime.properties / security.properties) are taken -//! from the [`DruidRoleGroupConfig`] precomputed in the validate step; product-config is no -//! longer involved. +//! The per-file configs (runtime.properties / security.properties / jvm.config) are rendered here +//! from the merged [`DruidRoleGroupConfig`] (config plus the merged config overrides); the +//! recommended cluster-level runtime properties and the erased roles needed for `jvm.config` are +//! carried on `ValidatedCluster`. Product-config is no longer involved. //! //! Metadata, owner reference and recommended labels are derived entirely from `ValidatedCluster` //! (which carries the validated name/namespace/uid and implements `Resource`). //! -//! The builder no longer reads the raw [`v1alpha1::DruidCluster`] at all: the extensions load -//! list, the metadata-database connection / storage type and the rendered `jvm.config` are all -//! precomputed on `ValidatedCluster` during the validate step. +//! The builder does not read the raw [`v1alpha1::DruidCluster`] at all: everything it needs is +//! carried on `ValidatedCluster` (resolved during the validate step). use std::collections::BTreeMap; @@ -28,15 +28,19 @@ use stackable_operator::{ use crate::{ controller::{ DRUID_CONTROLLER_NAME, - build::properties::{ - ConfigFileName, - logging::{build_log4j2_config, build_vector_config}, + build::{ + jvm::construct_jvm_args, + properties::{ + ConfigFileName, + logging::{build_log4j2_config, build_vector_config}, + runtime_properties, security_properties, + }, }, validate::{DruidRoleGroupConfig, ValidatedCluster}, }, crd::{ - DruidRole, build_recommended_labels, build_string_list, env_var_reference, file_reference, - v1alpha1, + DruidConfigOverrides, DruidRole, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, + build_recommended_labels, build_string_list, env_var_reference, file_reference, v1alpha1, }, }; @@ -87,10 +91,56 @@ pub enum Error { GenerateAuthenticationRuntimeSettings { source: crate::authentication::Error, }, + + #[snafu(display("failed to derive Druid memory settings from resources"))] + DeriveMemorySettings { source: crate::crd::resource::Error }, + + #[snafu(display("failed to construct the jvm.config"))] + GetJvmConfig { + source: crate::controller::build::jvm::Error, + }, } type Result = std::result::Result; +const INDEXER_JAVA_OPTS: &str = "druid.indexer.runner.javaOptsArray"; + +/// The `druid.indexer.runner.javaOptsArray` entry that must be present in *every* rendered file +/// (runtime.properties and security.properties) for MiddleManagers. +fn middlemanager_indexer_java_opts() -> (String, String) { + ( + INDEXER_JAVA_OPTS.to_string(), + build_string_list(&[ + format!("-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}"), + format!("-Djavax.net.ssl.trustStorePassword={STACKABLE_TRUST_STORE_PASSWORD}"), + "-Djavax.net.ssl.trustStoreType=pkcs12".to_owned(), + ]), + ) +} + +/// Returns the user-supplied key/value overrides for the given config file from a +/// [`DruidConfigOverrides`]. +/// +/// The CRD override map allows a value-less key (`someKey:` / null in YAML), modelled as +/// `Option`. We flatten `None` to an empty string here, matching how the Java properties +/// writer rendered a missing value (`key=`), so the rest of the pipeline can work with plain +/// `String` values. +fn key_value_overrides( + overrides: &DruidConfigOverrides, + file: ConfigFileName, +) -> BTreeMap { + let raw = match file { + ConfigFileName::RuntimeProperties => overrides.runtime_properties.overrides.clone(), + ConfigFileName::SecurityProperties => overrides.security_properties.overrides.clone(), + // log4j2.properties is rendered by the logging framework, and jvm.config is rendered from + // JVM argument overrides; neither is assembled from key/value overrides here. + ConfigFileName::Log4j2Properties | ConfigFileName::JvmConfig => BTreeMap::new(), + }; + raw.into_iter() + .map(|(k, v)| (k, v.unwrap_or_default())) + .collect() +} + /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator pub fn build_rolegroup_config_map( cluster: &ValidatedCluster, @@ -117,7 +167,7 @@ pub fn build_rolegroup_config_map( // This has to be done here since there is no other suitable place for it. // Previously such properties were added in the compute_files() function, // but that code path is now incompatible with the design of fragment merging. - rg.merged_config + rg.config .resources .update_druid_config_file(&mut conf) .context(UpdateDruidConfigFromResourcesSnafu)?; @@ -217,8 +267,22 @@ pub fn build_rolegroup_config_map( ); }; - // extend the config to respect the precomputed defaults and overrides - conf.extend(rg.runtime_config.clone()); + // Role/rolegroup runtime.properties: the recommended cluster-config-derived properties, + // the MiddleManager indexer opts, the per-role defaults and finally the user overrides + // (each layer wins over the previous, and over the cluster-level properties above). + conf.extend(runtime_properties::cluster_runtime_properties( + &cluster_config.deep_storage, + cluster_config.opa_connection_string.is_some(), + )); + if *role == DruidRole::MiddleManager { + let (k, v) = middlemanager_indexer_java_opts(); + conf.insert(k, v); + } + conf.extend(runtime_properties::defaults(role)); + conf.extend(key_value_overrides( + &rg.config_overrides, + ConfigFileName::RuntimeProperties, + )); let runtime_properties = to_java_properties_string(conf.iter()).context(SerializeRuntimePropertiesSnafu)?; @@ -229,14 +293,37 @@ pub fn build_rolegroup_config_map( } // ----- jvm.config ----- - // Precomputed during validation; see `DruidRoleGroupConfig::jvm_config`. - cm_conf_data.insert(ConfigFileName::JvmConfig.to_string(), rg.jvm_config.clone()); + { + let (heap, direct) = rg + .config + .resources + .get_memory_sizes(role) + .context(DeriveMemorySettingsSnafu)?; + let jvm_config = construct_jvm_args( + role, + cluster.get_role(role), + &rolegroup.role_group, + heap, + direct, + ) + .context(GetJvmConfigSnafu)?; + cm_conf_data.insert(ConfigFileName::JvmConfig.to_string(), jvm_config); + } // ----- security.properties ----- { + let mut security_config: BTreeMap = BTreeMap::new(); + if *role == DruidRole::MiddleManager { + let (k, v) = middlemanager_indexer_java_opts(); + security_config.insert(k, v); + } + let security_overrides = + key_value_overrides(&rg.config_overrides, ConfigFileName::SecurityProperties); + security_config.extend(security_properties::build(&security_overrides)); + cm_conf_data.insert( ConfigFileName::SecurityProperties.to_string(), - to_java_properties_string(rg.security_config.iter()).with_context(|_| { + to_java_properties_string(security_config.iter()).with_context(|_| { JvmSecurityPropertiesSnafu { rolegroup: rolegroup.role_group.clone(), } @@ -265,11 +352,11 @@ pub fn build_rolegroup_config_map( config_map_builder.add_data(filename, file_content); } - if let Some(log4j2_config) = build_log4j2_config(&rg.merged_config.logging) { + if let Some(log4j2_config) = build_log4j2_config(&rg.config.logging) { config_map_builder.add_data(ConfigFileName::Log4j2Properties.to_string(), log4j2_config); } - if let Some(vector_config) = build_vector_config(rolegroup, &rg.merged_config.logging) { + if let Some(vector_config) = build_vector_config(rolegroup, &rg.config.logging) { config_map_builder.add_data(VECTOR_CONFIG_FILE, vector_config); } diff --git a/rust/operator-binary/src/controller/build/jvm.rs b/rust/operator-binary/src/controller/build/jvm.rs index 4026db9f..bb72f41b 100644 --- a/rust/operator-binary/src/controller/build/jvm.rs +++ b/rust/operator-binary/src/controller/build/jvm.rs @@ -294,10 +294,11 @@ mod tests { serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); let role = druid.get_role(druid_role); - let merged_config = druid.merged_config().unwrap(); - let (heap, direct) = merged_config - .common_config(druid_role, "default") + let merged_role = druid.merged_role(druid_role).unwrap(); + let (heap, direct) = merged_role + .get("default") .unwrap() + .config .resources .get_memory_sizes(druid_role) .unwrap(); diff --git a/rust/operator-binary/src/controller/build/properties/runtime_properties.rs b/rust/operator-binary/src/controller/build/properties/runtime_properties.rs index f2f91968..4f0c9343 100644 --- a/rust/operator-binary/src/controller/build/properties/runtime_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/runtime_properties.rs @@ -9,7 +9,63 @@ use std::collections::BTreeMap; -use crate::crd::DruidRole; +use crate::crd::{DeepStorageSpec, DruidRole, METRICS_PORT}; + +// deep storage +const DS_TYPE: &str = "druid.storage.type"; +const DS_DIRECTORY: &str = "druid.storage.storageDirectory"; +const DS_BASE_KEY: &str = "druid.storage.baseKey"; +// OPA +const AUTH_AUTHORIZERS: &str = "druid.auth.authorizers"; +const AUTH_AUTHORIZERS_VALUE: &str = "[\"OpaAuthorizer\"]"; +const AUTH_AUTHORIZER_OPA_TYPE: &str = "druid.auth.authorizer.OpaAuthorizer.type"; +const AUTH_AUTHORIZER_OPA_TYPE_VALUE: &str = "opa"; +// metrics +const PROMETHEUS_PORT: &str = "druid.emitter.prometheus.port"; + +/// The recommended cluster-level `runtime.properties` derived from the cluster config (deep +/// storage, OPA authorization and metrics). These are independent of role and role group. +/// +/// `opa_authorization_enabled` mirrors `authorization.opa` being configured (equivalently, the +/// OPA connection string having been resolved during dereferencing). +pub fn cluster_runtime_properties( + deep_storage: &DeepStorageSpec, + opa_authorization_enabled: bool, +) -> BTreeMap { + let mut result = BTreeMap::new(); + + // OPA + if opa_authorization_enabled { + result.insert( + AUTH_AUTHORIZERS.to_string(), + AUTH_AUTHORIZERS_VALUE.to_string(), + ); + result.insert( + AUTH_AUTHORIZER_OPA_TYPE.to_string(), + AUTH_AUTHORIZER_OPA_TYPE_VALUE.to_string(), + ); + // The opaUri still needs to be set, but that requires a discovery config map and is + // handled in the controller. + } + + // deep storage + result.insert(DS_TYPE.to_string(), deep_storage.to_string()); + match deep_storage { + DeepStorageSpec::Hdfs(hdfs) => { + result.insert(DS_DIRECTORY.to_string(), hdfs.directory.clone()); + } + DeepStorageSpec::S3(s3_spec) => { + if let Some(key) = &s3_spec.base_key { + result.insert(DS_BASE_KEY.to_string(), key.to_string()); + } + } + } + + // metrics + result.insert(PROMETHEUS_PORT.to_string(), METRICS_PORT.to_string()); + + result +} /// Defaults rendered for every role. const ALL_ROLES: &[(&str, &str)] = &[ diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 85e71e71..be0d06f3 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -3,23 +3,17 @@ //! Synchronously validates inputs that don't require a Kubernetes client. Produces //! [`ValidatedCluster`], consumed by the rest of `reconcile_druid`. -use std::{ - borrow::Cow, - collections::{BTreeMap, HashMap}, - str::FromStr, -}; +use std::{borrow::Cow, collections::BTreeMap}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, commons::product_image_selection::{self, ResolvedProductImage}, - config::merge::Merge, crd::s3, database_connections::drivers::jdbc::{JdbcDatabaseConnection, JdbcDatabaseConnectionDetails}, kube::{Resource, api::ObjectMeta}, v2::{ HasName, HasUid, - builder::pod::container::{self, EnvVarName, EnvVarSet}, controller_utils::{get_cluster_name, get_namespace, get_uid}, types::{ kubernetes::{NamespaceName, Uid}, @@ -31,17 +25,8 @@ use strum::IntoEnumIterator; use crate::{ authentication::DruidAuthenticationConfig, - controller::{ - build::{ - jvm::construct_jvm_args, - properties::{ConfigFileName, runtime_properties, security_properties}, - }, - dereference::DereferencedObjects, - }, - crd::{ - CommonRoleGroupConfig, DruidConfigOverrides, DruidRole, STACKABLE_TRUST_STORE, - STACKABLE_TRUST_STORE_PASSWORD, build_string_list, security::DruidTlsSecurity, v1alpha1, - }, + controller::dereference::DereferencedObjects, + crd::{DeepStorageSpec, DruidErasedRole, DruidRole, security::DruidTlsSecurity, v1alpha1}, extensions::get_extension_list, }; @@ -66,45 +51,26 @@ pub enum Error { source: stackable_operator::v2::controller_utils::Error, }, - #[snafu(display("failed to get JVM config"))] - GetJvmConfig { - source: crate::controller::build::jvm::Error, - }, - - #[snafu(display("failed to derive Druid memory settings from resources"))] - DeriveMemorySettings { source: crate::crd::resource::Error }, - #[snafu(display("invalid metadata database connection"))] InvalidMetadataDatabaseConnection { source: stackable_operator::database_connections::Error, }, - - #[snafu(display("invalid environment variable override name"))] - ParseEnvVarName { source: container::Error }, } type Result = std::result::Result; pub type RoleGroupName = String; -/// The merged config plus the per-file "validated config" maps that used to be produced by -/// product-config. These are computed from first principles so that rendered config stays -/// byte-identical. -#[derive(Clone)] -pub struct DruidRoleGroupConfig { - pub merged_config: CommonRoleGroupConfig, - /// The runtime.properties config: recommended defaults plus merged overrides. - pub runtime_config: BTreeMap, - /// The security.properties "validated config". - pub security_config: BTreeMap, - /// Merged env overrides (role <- rolegroup). Druid has no computed env vars. Names are - /// validated here so the build step can render them directly into the container. - pub env: EnvVarSet, - /// The fully rendered `jvm.config` (operator defaults merged with the role/rolegroup JVM - /// argument overrides). Precomputed here so the config-map builder no longer needs the raw - /// cluster's `get_role`. - pub jvm_config: String, -} +/// A validated, merged role-group config. +/// +/// This is the framework [`RoleGroupConfig`] (config plus the four merged override categories), +/// with the typed per-role config erased to [`CommonRoleGroupConfig`] so that all roles share a +/// single type. The rendered per-file configs (runtime.properties / security.properties / +/// jvm.config) are produced later, in the config-map build step. +/// +/// Defined in [`crate::crd`] (where it has access to the private typed config fields) and +/// re-exported here for the build step. +pub use crate::crd::DruidRoleGroupConfig; /// Cluster-wide resolved fields that are not role/rolegroup specific. pub struct ValidatedClusterConfig { @@ -121,6 +87,10 @@ pub struct ValidatedClusterConfig { pub metadata_storage_type: String, /// The JDBC connection details (URL plus credential env vars) for the metadata database. pub metadata_db_connection: JdbcDatabaseConnectionDetails, + /// The deep-storage spec, carried so the build step can derive the cluster-level + /// `runtime.properties` (deep storage type / directory / base key) without the raw + /// `DruidCluster`. + pub deep_storage: DeepStorageSpec, } /// Synchronous inputs the rest of `reconcile_druid` needs after dereferencing. @@ -137,9 +107,14 @@ pub struct ValidatedCluster { pub image: ResolvedProductImage, pub cluster_config: ValidatedClusterConfig, pub role_group_configs: BTreeMap>, + /// The erased roles (see [`DruidErasedRole`]), retained so the build step can render + /// `jvm.config` (which merges the role/rolegroup JVM argument overrides) without the raw + /// `DruidCluster`. + roles: BTreeMap, } impl ValidatedCluster { + #[allow(clippy::too_many_arguments)] pub(crate) fn new( name: ClusterName, namespace: NamespaceName, @@ -147,6 +122,7 @@ impl ValidatedCluster { image: ResolvedProductImage, cluster_config: ValidatedClusterConfig, role_group_configs: BTreeMap>, + roles: BTreeMap, ) -> Self { let metadata = ObjectMeta { name: Some(name.to_string()), @@ -162,8 +138,17 @@ impl ValidatedCluster { image, cluster_config, role_group_configs, + roles, } } + + /// The erased role (carrying the JVM argument overrides) for the given role. Used by the build + /// step to render `jvm.config`. + pub fn get_role(&self, role: &DruidRole) -> &DruidErasedRole { + self.roles + .get(role) + .expect("every DruidRole is populated during validation") + } } // Implementing `Resource` (plus `HasName`/`HasUid`) lets `ValidatedCluster` stand in for the raw @@ -211,129 +196,6 @@ impl HasUid for ValidatedCluster { } } -/// Returns the user-supplied key/value overrides for the given config file from a -/// [`DruidConfigOverrides`]. -/// -/// The CRD override map allows a value-less key (`someKey:` / null in YAML), modelled as -/// `Option`. We flatten `None` to an empty string here, matching how the Java -/// properties writer rendered a missing value (`key=`), so the rest of the pipeline can work -/// with plain `String` values. -fn key_value_overrides( - overrides: &DruidConfigOverrides, - file: ConfigFileName, -) -> BTreeMap { - let raw = match file { - ConfigFileName::RuntimeProperties => overrides.runtime_properties.overrides.clone(), - ConfigFileName::SecurityProperties => overrides.security_properties.overrides.clone(), - // log4j2.properties is rendered by the logging framework, and jvm.config is rendered from - // JVM argument overrides; neither is assembled from key/value overrides here. - ConfigFileName::Log4j2Properties | ConfigFileName::JvmConfig => BTreeMap::new(), - }; - raw.into_iter() - .map(|(k, v)| (k, v.unwrap_or_default())) - .collect() -} - -/// Merges the role-level and rolegroup-level env overrides into a validated [`EnvVarSet`]. -/// -/// The role is processed first, then the rolegroup, so that rolegroup overrides win on key -/// collisions ([`EnvVarSet::with_value`] overrides earlier entries with the same name). The -/// override names are validated here so the build step can render them directly. -fn merged_env_overrides( - role_env_overrides: &HashMap, - rg_env_overrides: &HashMap, -) -> Result { - let mut env = EnvVarSet::new(); - for (name, value) in role_env_overrides.iter().chain(rg_env_overrides.iter()) { - env = env.with_value( - &EnvVarName::from_str(name).context(ParseEnvVarNameSnafu)?, - value.clone(), - ); - } - Ok(env) -} - -/// Builds the precomputed per-file config for a single rolegroup. Pure assembly: combines the -/// role-level overrides with the rolegroup-level overrides (rolegroup wins) on top of the -/// computed defaults. No behavior change vs. the inline loop body it was extracted from. -#[allow(clippy::too_many_arguments)] -fn build_role_group_config( - druid: &v1alpha1::DruidCluster, - druid_role: &DruidRole, - rg_name: &str, - merged_config: CommonRoleGroupConfig, - role_config_overrides: &DruidConfigOverrides, - role_env_overrides: &HashMap, - rg_config_overrides: &DruidConfigOverrides, - rg_env_overrides: &HashMap, -) -> Result { - // Merge the role-level and rolegroup-level config overrides (rolegroup wins over role). - let mut config_overrides = rg_config_overrides.clone(); - config_overrides.merge(role_config_overrides); - - // ----- runtime.properties ----- - let mut runtime_config = druid.compute_runtime_properties(); - if *druid_role == DruidRole::MiddleManager { - let (k, v) = middlemanager_indexer_java_opts(); - runtime_config.insert(k, v); - } - runtime_config.extend(runtime_properties::defaults(druid_role)); - runtime_config.extend(key_value_overrides( - &config_overrides, - ConfigFileName::RuntimeProperties, - )); - - // ----- security.properties ----- - let mut security_config: BTreeMap = BTreeMap::new(); - if *druid_role == DruidRole::MiddleManager { - let (k, v) = middlemanager_indexer_java_opts(); - security_config.insert(k, v); - } - let security_overrides = - key_value_overrides(&config_overrides, ConfigFileName::SecurityProperties); - security_config.extend(security_properties::build(&security_overrides)); - - // ----- env ----- - let env = merged_env_overrides(role_env_overrides, rg_env_overrides)?; - - // ----- jvm.config ----- - let (heap, direct) = merged_config - .resources - .get_memory_sizes(druid_role) - .context(DeriveMemorySettingsSnafu)?; - let jvm_config = construct_jvm_args( - druid_role, - &druid.get_role(druid_role), - rg_name, - heap, - direct, - ) - .context(GetJvmConfigSnafu)?; - - Ok(DruidRoleGroupConfig { - merged_config, - runtime_config, - security_config, - env, - jvm_config, - }) -} - -const INDEXER_JAVA_OPTS: &str = "druid.indexer.runner.javaOptsArray"; - -/// The `druid.indexer.runner.javaOptsArray` entry that must be present in *every* rendered file -/// (runtime.properties and security.properties) for MiddleManagers. -fn middlemanager_indexer_java_opts() -> (String, String) { - ( - INDEXER_JAVA_OPTS.to_string(), - build_string_list(&[ - format!("-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}"), - format!("-Djavax.net.ssl.trustStorePassword={STACKABLE_TRUST_STORE_PASSWORD}"), - "-Djavax.net.ssl.trustStoreType=pkcs12".to_owned(), - ]), - ) -} - /// Validates the cluster spec and the dereferenced inputs. pub fn validate( druid: &v1alpha1::DruidCluster, @@ -360,44 +222,16 @@ pub fn validate( ) .context(InvalidDruidAuthenticationConfigSnafu)?; - let merged = druid.merged_config().context(FailedToResolveConfigSnafu)?; - let mut role_group_configs: BTreeMap> = BTreeMap::new(); + let mut roles = BTreeMap::new(); for druid_role in DruidRole::iter() { - // The role-level overrides (role <- rolegroup precedence starts here). - let role = druid.get_role(&druid_role); - let role_config_overrides = &role.config.config_overrides; - let role_env_overrides = role.config.env_overrides.clone(); - - let rolegroups = merged.role_group_names(&druid_role); - - let mut group_map: BTreeMap = BTreeMap::new(); - for rg_name in rolegroups { - let merged_config = merged - .common_config(&druid_role, &rg_name) - .context(FailedToResolveConfigSnafu)?; - // The rolegroup-level config/env overrides (rolegroup wins over role). - // The rolegroup is guaranteed to exist because `rg_name` comes from - // `role_group_names` and `common_config` above already resolved it. - let (rg_config_overrides, rg_env_overrides) = merged - .role_group_overrides(&druid_role, &rg_name) - .expect("role group resolved by common_config must exist"); - - let rg_config = build_role_group_config( - druid, - &druid_role, - &rg_name, - merged_config, - role_config_overrides, - &role_env_overrides, - rg_config_overrides, - rg_env_overrides, - )?; - group_map.insert(rg_name, rg_config); - } - role_group_configs.insert(druid_role, group_map); + let group_map = druid + .merged_role(&druid_role) + .context(FailedToResolveConfigSnafu)?; + role_group_configs.insert(druid_role.clone(), group_map); + roles.insert(druid_role.clone(), druid.get_role(&druid_role)); } let name = get_cluster_name(druid).context(ClusterIdentitySnafu)?; @@ -433,7 +267,9 @@ pub fn validate( extensions, metadata_storage_type, metadata_db_connection, + deep_storage: druid.spec.cluster_config.deep_storage.clone(), }, role_group_configs, + roles, )) } diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 23b61397..5e0bae1e 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -142,11 +142,8 @@ mod tests { let deserializer = serde_yaml::Deserializer::from_str(input); let druid: v1alpha1::DruidCluster = serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); - let merged_config = druid - .merged_config() - .unwrap() - .common_config(&role, "default") - .unwrap(); + let merged_role = druid.merged_role(&role).unwrap(); + let merged_config = merged_role.get("default").unwrap(); let mut expected_affinities = vec![]; @@ -257,7 +254,7 @@ mod tests { }; assert_eq!( - merged_config.affinity, + merged_config.config.affinity, StackableAffinity { pod_affinity: Some(PodAffinity { preferred_during_scheduling_ignored_during_execution: Some(expected_affinities), diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 794eba3c..b091eb2b 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeMap, HashSet}; use indoc::formatdoc; use security::add_cert_to_jvm_trust_store_cmd; @@ -13,7 +13,7 @@ use stackable_operator::{ resources::{NoRuntimeLimits, Resources}, }, config::{ - fragment::{self, Fragment, FromFragment, ValidationError}, + fragment::{Fragment, FromFragment, ValidationError}, merge::Merge, }, crd::{ @@ -21,7 +21,7 @@ use stackable_operator::{ s3, }, deep_merger::ObjectOverrides, - k8s_openapi::api::core::v1::{PodTemplateSpec, Volume}, + k8s_openapi::api::core::v1::Volume, kube::{CustomResource, ResourceExt}, kvp::ObjectLabels, product_logging::{ @@ -39,12 +39,15 @@ use stackable_operator::{ }; use strum::{Display, EnumDiscriminants, EnumIter, EnumString, IntoStaticStr}; -use crate::crd::{ - affinity::get_affinity, - authorization::DruidAuthorization, - database::MetadataDatabaseConnection, - resource::RoleResource, - tls::{DruidTls, default_druid_tls}, +use crate::{ + crd::{ + affinity::get_affinity, + authorization::DruidAuthorization, + database::MetadataDatabaseConnection, + resource::RoleResource, + tls::{DruidTls, default_druid_tls}, + }, + framework::role_utils::{RoleGroupConfig, with_validated_config}, }; pub mod affinity; @@ -76,17 +79,6 @@ pub const PROP_SEGMENT_CACHE_LOCATIONS: &str = "druid.segmentCache.locations"; ///////////////////////////// // CONFIG PROPERTIES // ///////////////////////////// -// deep storage -const DS_TYPE: &str = "druid.storage.type"; -const DS_DIRECTORY: &str = "druid.storage.storageDirectory"; -const DS_BASE_KEY: &str = "druid.storage.baseKey"; -// OPA -const AUTH_AUTHORIZERS: &str = "druid.auth.authorizers"; -const AUTH_AUTHORIZERS_VALUE: &str = "[\"OpaAuthorizer\"]"; -const AUTH_AUTHORIZER_OPA_TYPE: &str = "druid.auth.authorizer.OpaAuthorizer.type"; -const AUTH_AUTHORIZER_OPA_TYPE_VALUE: &str = "opa"; -// metrics -const PROMETHEUS_PORT: &str = "druid.emitter.prometheus.port"; pub const METRICS_PORT_NAME: &str = "metrics"; pub const METRICS_PORT: u16 = 9090; @@ -167,6 +159,12 @@ pub enum Error { #[snafu(display("fragment validation failure"))] FragmentValidationFailure { source: ValidationError }, + + #[snafu(display("failed to merge and validate config for role group {role_group:?}"))] + FailedToMergeRoleGroupConfig { + source: crate::framework::role_utils::Error, + role_group: String, + }, } #[versioned( @@ -329,44 +327,6 @@ impl HasStatusCondition for v1alpha1::DruidCluster { } impl v1alpha1::DruidCluster { - pub fn compute_runtime_properties(&self) -> BTreeMap { - let mut result = BTreeMap::new(); - // OPA - if let Some(DruidAuthorization { opa: _ }) = &self.spec.cluster_config.authorization { - result.insert( - AUTH_AUTHORIZERS.to_string(), - AUTH_AUTHORIZERS_VALUE.to_string(), - ); - result.insert( - AUTH_AUTHORIZER_OPA_TYPE.to_string(), - AUTH_AUTHORIZER_OPA_TYPE_VALUE.to_string(), - ); - // The opaUri still needs to be set, but that requires a discovery config map and is handled in the controller.rs - } - // deep storage - result.insert( - DS_TYPE.to_string(), - self.spec.cluster_config.deep_storage.to_string(), - ); - match self.spec.cluster_config.deep_storage.clone() { - DeepStorageSpec::Hdfs(hdfs) => { - result.insert(DS_DIRECTORY.to_string(), hdfs.directory); - } - DeepStorageSpec::S3(s3_spec) => { - if let Some(key) = &s3_spec.base_key { - result.insert(DS_BASE_KEY.to_string(), key.to_string()); - } - // bucket information (name, connection) needs to be resolved first, - // that is done directly in the controller - } - } - - // metrics - result.insert(PROMETHEUS_PORT.to_string(), METRICS_PORT.to_string()); - - result - } - /// If an s3 connection for ingestion is given, as well as an s3 connection for deep storage, they need to be the same. /// This function returns the resolved connection, or raises an Error if the connections are not identical. pub async fn get_s3_connection( @@ -451,98 +411,102 @@ impl v1alpha1::DruidCluster { s3_ingestion || s3_storage } - /// Returns the merged and validated configuration for all roles - pub fn merged_config(&self) -> Result { + /// Merges and validates all role groups of the given role. + /// + /// All four override categories (config / env / cli / pod) are merged by + /// [`with_validated_config`] (role group wins over role); the typed per-role config is then + /// erased to the shared [`CommonRoleGroupConfig`] view consumed by the build step. + pub fn merged_role( + &self, + role: &DruidRole, + ) -> Result, Error> { let deep_storage = &self.spec.cluster_config.deep_storage; - - Ok(MergedConfig { - brokers: v1alpha1::DruidCluster::merged_role( - &extract_role_from_role_config::(self.spec.brokers.clone()), - &BrokerConfig::default_config(&self.name_any(), &DruidRole::Broker, deep_storage), - )?, - coordinators: v1alpha1::DruidCluster::merged_role( - &extract_role_from_role_config::(self.spec.coordinators.clone()), - &CoordinatorConfig::default_config( - &self.name_any(), - &DruidRole::Coordinator, - deep_storage, - ), - )?, - historicals: v1alpha1::DruidCluster::merged_role( - &self.spec.historicals, - &HistoricalConfig::default_config( - &self.name_any(), - &DruidRole::Historical, - deep_storage, - ), - )?, - middle_managers: v1alpha1::DruidCluster::merged_role( - &self.spec.middle_managers, - &MiddleManagerConfig::default_config( - &self.name_any(), - &DruidRole::MiddleManager, - deep_storage, - ), - )?, - routers: v1alpha1::DruidCluster::merged_role( - &extract_role_from_role_config::(self.spec.routers.clone()), - &RouterConfig::default_config(&self.name_any(), &DruidRole::Router, deep_storage), - )?, - }) - } - - /// Merges and validates the role groups of the given role with the given default configuration - fn merged_role( - role: &Role, - default_config: &T::Fragment, - ) -> Result>, Error> - where - T: FromFragment, - T::Fragment: Clone + Merge, - { - let mut merged_role_config = HashMap::new(); - - for (rolegroup_name, rolegroup) in &role.role_groups { - let merged_rolegroup_config = v1alpha1::DruidCluster::merged_rolegroup( - rolegroup, - &role.config.config, - default_config, - )?; - merged_role_config.insert(rolegroup_name.to_owned(), merged_rolegroup_config); + let name = self.name_any(); + + // All roles erase to an identical `CommonRoleGroupConfig`; only the typed config, the role + // config type and the `RoleResource` variant (historicals carry typed storage) differ. + macro_rules! merged_role { + ($field:ident, $config:ty, $role_config:ty, $resource:path) => {{ + let typed_role = &self.spec.$field; + let default_config = <$config>::default_config(&name, role, deep_storage); + let mut groups = BTreeMap::new(); + for (rg_name, rg) in &typed_role.role_groups { + let validated = with_validated_config::< + $config, + JavaCommonConfig, + _, + $role_config, + DruidConfigOverrides, + >(rg, typed_role, &default_config) + .with_context(|_| FailedToMergeRoleGroupConfigSnafu { + role_group: rg_name.clone(), + })?; + let common = CommonRoleGroupConfig { + resources: $resource(validated.config.resources), + logging: validated.config.logging, + replicas: rg.replicas, + affinity: validated.config.affinity, + graceful_shutdown_timeout: validated.config.graceful_shutdown_timeout, + requested_secret_lifetime: validated + .config + .requested_secret_lifetime + .context(MissingSecretLifetimeSnafu)?, + }; + groups.insert( + rg_name.clone(), + DruidRoleGroupConfig { + replicas: validated.replicas, + config: common, + config_overrides: validated.config_overrides, + env_overrides: validated.env_overrides, + cli_overrides: validated.cli_overrides, + pod_overrides: validated.pod_overrides, + product_specific_common_config: validated + .product_specific_common_config, + }, + ); + } + groups + }}; } - Ok(merged_role_config) - } - - /// Merges and validates the given role group with the given role and default configurations - fn merged_rolegroup( - rolegroup: &RoleGroup, - role_config: &T::Fragment, - default_config: &T::Fragment, - ) -> Result, Error> - where - T: FromFragment, - T::Fragment: Clone + Merge, - { - let merged_config = v1alpha1::DruidCluster::merged_rolegroup_config( - &rolegroup.config.config, - role_config, - default_config, - )?; - Ok(RoleGroup { - config: CommonConfiguration { - config: merged_config, - config_overrides: rolegroup.config.config_overrides.to_owned(), - env_overrides: rolegroup.config.env_overrides.to_owned(), - cli_overrides: rolegroup.config.cli_overrides.to_owned(), - pod_overrides: rolegroup.config.pod_overrides.to_owned(), - product_specific_common_config: rolegroup - .config - .product_specific_common_config - .to_owned(), - }, - replicas: rolegroup.replicas, - }) + let groups = match role { + DruidRole::Broker => { + merged_role!( + brokers, + BrokerConfig, + v1alpha1::DruidRoleConfig, + RoleResource::Druid + ) + } + DruidRole::Coordinator => merged_role!( + coordinators, + CoordinatorConfig, + v1alpha1::DruidRoleConfig, + RoleResource::Druid + ), + DruidRole::Historical => merged_role!( + historicals, + HistoricalConfig, + GenericRoleConfig, + RoleResource::Historical + ), + DruidRole::MiddleManager => merged_role!( + middle_managers, + MiddleManagerConfig, + GenericRoleConfig, + RoleResource::Druid + ), + DruidRole::Router => { + merged_role!( + routers, + RouterConfig, + v1alpha1::DruidRoleConfig, + RoleResource::Druid + ) + } + }; + Ok(groups) } pub fn generic_role_config(&self, role: &DruidRole) -> &GenericRoleConfig { @@ -555,29 +519,7 @@ impl v1alpha1::DruidCluster { } } - /// Merges and validates the given role group, role, and default configurations - pub(crate) fn merged_rolegroup_config( - rolegroup_config: &T::Fragment, - role_config: &T::Fragment, - default_config: &T::Fragment, - ) -> Result - where - T: FromFragment, - T::Fragment: Clone + Merge, - { - let mut role_config = role_config.to_owned(); - let mut rolegroup_config = rolegroup_config.to_owned(); - - role_config.merge(default_config); - rolegroup_config.merge(&role_config); - - fragment::validate(rolegroup_config).context(FragmentValidationFailureSnafu) - } - - pub fn get_role( - &self, - druid_role: &DruidRole, - ) -> Role<(), DruidConfigOverrides, GenericRoleConfig, JavaCommonConfig> { + pub fn get_role(&self, druid_role: &DruidRole) -> DruidErasedRole { match druid_role { DruidRole::Broker => erase_config(extract_role_from_role_config::( self.spec.brokers.clone(), @@ -592,39 +534,6 @@ impl v1alpha1::DruidCluster { )), } } - - pub fn pod_overrides_for_role(&self, role: &DruidRole) -> &PodTemplateSpec { - match role { - DruidRole::Broker => &self.spec.brokers.config.pod_overrides, - DruidRole::Coordinator => &self.spec.coordinators.config.pod_overrides, - DruidRole::Historical => &self.spec.historicals.config.pod_overrides, - DruidRole::MiddleManager => &self.spec.middle_managers.config.pod_overrides, - DruidRole::Router => &self.spec.routers.config.pod_overrides, - } - } - - pub fn pod_overrides_for_role_group( - &self, - role: &DruidRole, - role_group: &str, - ) -> Option<&PodTemplateSpec> { - macro_rules! pod_overrides { - ($field:ident) => { - self.spec - .$field - .role_groups - .get(role_group) - .map(|rg| &rg.config.pod_overrides) - }; - } - match role { - DruidRole::Broker => pod_overrides!(brokers), - DruidRole::Coordinator => pod_overrides!(coordinators), - DruidRole::Historical => pod_overrides!(historicals), - DruidRole::MiddleManager => pod_overrides!(middle_managers), - DruidRole::Router => pod_overrides!(routers), - } - } } #[derive( @@ -659,98 +568,23 @@ pub struct CommonRoleGroupConfig { pub requested_secret_lifetime: Duration, } -/// Container for the merged and validated role group configurations +/// A validated, merged role-group config. /// -/// This structure contains for every role a map from the role group names to their configurations. -/// The role group configurations are merged with the role and default configurations. The product -/// configuration is not applied. -pub struct MergedConfig { - /// Merged configuration of the broker role - pub brokers: HashMap>, - /// Merged configuration of the coordinator role - pub coordinators: - HashMap>, - /// Merged configuration of the historical role - pub historicals: - HashMap>, - /// Merged configuration of the middle manager role - pub middle_managers: - HashMap>, - /// Merged configuration of the router role - pub routers: HashMap>, -} - -impl MergedConfig { - /// Returns the common configuration for the given role and rolegroup name - pub fn common_config( - &self, - role: &DruidRole, - rolegroup_name: &str, - ) -> Result { - // All roles build an identical `CommonRoleGroupConfig`; only the role-group map and the - // `RoleResource` variant (historicals carry typed storage) differ. - macro_rules! common_config { - ($field:ident, $resource:path) => {{ - let rolegroup = self - .$field - .get(rolegroup_name) - .context(CannotRetrieveRoleGroupSnafu { rolegroup_name })?; - Ok(CommonRoleGroupConfig { - resources: $resource(rolegroup.config.config.resources.to_owned()), - logging: rolegroup.config.config.logging.to_owned(), - replicas: rolegroup.replicas, - affinity: rolegroup.config.config.affinity.clone(), - graceful_shutdown_timeout: rolegroup.config.config.graceful_shutdown_timeout, - requested_secret_lifetime: rolegroup - .config - .config - .requested_secret_lifetime - .context(MissingSecretLifetimeSnafu)?, - }) - }}; - } - match role { - DruidRole::Broker => common_config!(brokers, RoleResource::Druid), - DruidRole::Coordinator => common_config!(coordinators, RoleResource::Druid), - DruidRole::Historical => common_config!(historicals, RoleResource::Historical), - DruidRole::MiddleManager => common_config!(middle_managers, RoleResource::Druid), - DruidRole::Router => common_config!(routers, RoleResource::Druid), - } - } - - /// Returns the (sorted) role group names defined for the given role. - pub fn role_group_names(&self, role: &DruidRole) -> BTreeSet { - match role { - DruidRole::Broker => self.brokers.keys().cloned().collect(), - DruidRole::Coordinator => self.coordinators.keys().cloned().collect(), - DruidRole::Historical => self.historicals.keys().cloned().collect(), - DruidRole::MiddleManager => self.middle_managers.keys().cloned().collect(), - DruidRole::Router => self.routers.keys().cloned().collect(), - } - } - - /// Returns the rolegroup-level config and env overrides for the given role and rolegroup. - pub fn role_group_overrides( - &self, - role: &DruidRole, - rolegroup_name: &str, - ) -> Option<(&DruidConfigOverrides, &HashMap)> { - macro_rules! get { - ($field:expr) => { - $field - .get(rolegroup_name) - .map(|rg| (&rg.config.config_overrides, &rg.config.env_overrides)) - }; - } - match role { - DruidRole::Broker => get!(self.brokers), - DruidRole::Coordinator => get!(self.coordinators), - DruidRole::Historical => get!(self.historicals), - DruidRole::MiddleManager => get!(self.middle_managers), - DruidRole::Router => get!(self.routers), - } - } -} +/// This is the framework [`RoleGroupConfig`] (config plus the four merged override categories, +/// role group winning over role), with the typed per-role config erased to the shared +/// [`CommonRoleGroupConfig`] view. The rendered per-file configs (runtime.properties / +/// security.properties / jvm.config) are produced later, in the config-map build step. +/// +/// Note: the StatefulSet replicas come from [`CommonRoleGroupConfig::replicas`] (an `Option`, so +/// an unspecified count is left unset and stays HPA-friendly), not from the framework +/// [`RoleGroupConfig::replicas`]. +pub type DruidRoleGroupConfig = + RoleGroupConfig; + +/// A role with its typed config erased to `()`, retaining the override layers (notably the JVM +/// argument overrides in `product_specific_common_config`). Carried on the validated cluster so +/// the build step can render `jvm.config` without reaching back to the raw `DruidCluster`. +pub type DruidErasedRole = Role<(), DruidConfigOverrides, GenericRoleConfig, JavaCommonConfig>; impl Default for v1alpha1::DruidRoleConfig { fn default() -> Self { diff --git a/rust/operator-binary/src/crd/resource.rs b/rust/operator-binary/src/crd/resource.rs index 71b5a357..7d17a947 100644 --- a/rust/operator-binary/src/crd/resource.rs +++ b/rust/operator-binary/src/crd/resource.rs @@ -254,14 +254,14 @@ mod test { CpuLimits, CpuLimitsFragment, MemoryLimits, MemoryLimitsFragment, NoRuntimeLimitsFragment, }, + config::{fragment, merge::Merge}, k8s_openapi::apimachinery::pkg::api::resource::Quantity, - role_utils::{CommonConfiguration, RoleGroup}, utils::yaml_from_str_singleton_map, }; use super::*; use crate::crd::{ - MiddleManagerConfig, + DruidRole, storage::{HistoricalStorage, default_free_percentage_empty_dir}, v1alpha1, }; @@ -399,11 +399,13 @@ mod test { #[case] third: Option>, #[case] expected: Resources, ) { - let got = v1alpha1::DruidCluster::merged_rolegroup_config( - &first.unwrap_or_default(), - &second.unwrap_or_default(), - &third.unwrap_or_default(), - ); + // Replicates the fragment merge done by `with_validated_config` / + // `RoleGroup::validate_config`: default <- role <- role group, then validate. + let mut role_config = second.unwrap_or_default(); + role_config.merge(&third.unwrap_or_default()); + let mut rolegroup_config = first.unwrap_or_default(); + rolegroup_config.merge(&role_config); + let got = fragment::validate::>(rolegroup_config); assert_eq!(expected, got.unwrap()); } @@ -415,19 +417,11 @@ mod test { )) .expect("failed to parse YAML"); - let config = cluster.merged_config().unwrap(); - if let Some(RoleGroup { - config: - CommonConfiguration { - config: - MiddleManagerConfig { - resources: middlemanager_resources_from_rg, - .. - }, - .. - }, - .. - }) = config.middle_managers.get("resources-from-role-group") + let middle_managers = cluster.merged_role(&DruidRole::MiddleManager).unwrap(); + + if let Some(RoleResource::Druid(middlemanager_resources_from_rg)) = middle_managers + .get("resources-from-role-group") + .map(|rg| &rg.config.resources) { let expected = Resources { cpu: CpuLimits { @@ -449,18 +443,9 @@ mod test { panic!("No role group named [resources-from-role-group] found"); } - if let Some(RoleGroup { - config: - CommonConfiguration { - config: - MiddleManagerConfig { - resources: middlemanager_resources_from_rg, - .. - }, - .. - }, - .. - }) = config.middle_managers.get("resources-from-role") + if let Some(RoleResource::Druid(middlemanager_resources_from_rg)) = middle_managers + .get("resources-from-role") + .map(|rg| &rg.config.resources) { let expected = Resources { cpu: CpuLimits { @@ -492,12 +477,10 @@ mod test { )) .expect("failed to parse YAML"); + let historicals = cluster.merged_role(&DruidRole::Historical).unwrap(); + // ---------- default role group - let config = cluster.merged_config().unwrap(); - let res = config - .common_config(&DruidRole::Historical, "default") - .unwrap() - .resources; + let res = &historicals.get("default").unwrap().config.resources; let mut got = BTreeMap::new(); assert!(res.update_druid_config_file(&mut got).is_ok()); @@ -508,10 +491,7 @@ mod test { assert_eq!(value, &expected, "primary"); // ---------- secondary role group - let res = config - .common_config(&DruidRole::Historical, "secondary") - .unwrap() - .resources; + let res = &historicals.get("secondary").unwrap().config.resources; let mut got = BTreeMap::new(); assert!(res.update_druid_config_file(&mut got).is_ok()); diff --git a/rust/operator-binary/src/framework.rs b/rust/operator-binary/src/framework.rs new file mode 100644 index 00000000..1edc49b4 --- /dev/null +++ b/rust/operator-binary/src/framework.rs @@ -0,0 +1,11 @@ +//! Local framework helpers that mirror the work-in-progress upstream +//! `stackable_operator::v2::*` modules. +//! +//! We vendor `role_utils` because the upstream `v2::role_utils` requires +//! `CommonConfig: Merge`. Druid (like hive and trino) uses `JavaCommonConfig`, whose +//! JVM-argument merge is fallible and so does not implement `Merge`. +//! +//! Follow-up: replace with `stackable_operator::v2::role_utils::*` once upstream +//! relaxes the `Merge` bound. + +pub mod role_utils; diff --git a/rust/operator-binary/src/framework/role_utils.rs b/rust/operator-binary/src/framework/role_utils.rs new file mode 100644 index 00000000..9cf4b3ba --- /dev/null +++ b/rust/operator-binary/src/framework/role_utils.rs @@ -0,0 +1,152 @@ +//! Vendored variant of `stackable_operator::v2::role_utils` from the +//! `smooth-operator` branch, with simplifications appropriate for druid-operator. +//! +//! Differences from upstream: +//! - No `cli_overrides_to_vec` helper, `ResourceNames`, or service-account helpers. +//! - The `CommonConfig` (a.k.a. `product_specific_common_config`) does NOT need to +//! implement `Merge`. Druid uses `JavaCommonConfig`, which intentionally does not +//! implement `Merge` because its inner `JvmArgumentOverrides::try_merge` is +//! fallible (regex validation). The `RoleGroupConfig::product_specific_common_config` +//! field here simply carries the role-group level value through. +//! +//! Replace with `stackable_operator::v2::role_utils::*` once upstream relaxes the +//! `Merge` bound. + +use std::{ + collections::{BTreeMap, HashMap}, + str::FromStr, +}; + +use serde::Serialize; +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + config::{ + fragment::{self, FromFragment}, + merge::{Merge, merge}, + }, + k8s_openapi::{DeepMerge, api::core::v1::PodTemplateSpec}, + role_utils::{Role, RoleGroup}, + schemars::JsonSchema, + v2::builder::pod::container::{self, EnvVarName, EnvVarSet}, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to validate the role group config"))] + ValidateConfig { source: fragment::ValidationError }, + + #[snafu(display("invalid environment variable override name"))] + ParseEnvVarName { source: container::Error }, +} + +/// Druid-friendly view of a validated, merged `RoleGroup`. +#[derive(Clone, Debug, PartialEq)] +pub struct RoleGroupConfig { + pub replicas: u16, + pub config: Config, + pub config_overrides: ConfigOverrides, + pub env_overrides: EnvVarSet, + pub cli_overrides: BTreeMap, + pub pod_overrides: PodTemplateSpec, + pub product_specific_common_config: CommonConfig, +} + +/// Merges and validates the `RoleGroup` with the given `role` and `default_config`. +pub fn with_validated_config( + role_group: &RoleGroup, + role: &Role, + default_config: &Config, +) -> Result, Error> +where + ValidatedConfig: FromFragment, + CommonConfig: Clone + Default + JsonSchema + Serialize, + Config: Clone + Merge, + RoleConfig: Default + JsonSchema + Serialize, + ConfigOverrides: Clone + Default + JsonSchema + Merge + Serialize, +{ + let validated_config = + validate_config(role_group, role, default_config).context(ValidateConfigSnafu)?; + Ok(RoleGroupConfig { + replicas: role_group.replicas.unwrap_or(1), + config: validated_config, + config_overrides: merged_config_overrides( + &role.config.config_overrides, + role_group.config.config_overrides.clone(), + ), + env_overrides: merged_env_overrides( + &role.config.env_overrides, + &role_group.config.env_overrides, + )?, + cli_overrides: merged_cli_overrides( + role.config.cli_overrides.clone(), + role_group.config.cli_overrides.clone(), + ), + pod_overrides: merged_pod_overrides( + role.config.pod_overrides.clone(), + role_group.config.pod_overrides.clone(), + ), + product_specific_common_config: role_group.config.product_specific_common_config.clone(), + }) +} + +fn validate_config( + role_group: &RoleGroup, + role: &Role, + default_config: &Config, +) -> Result +where + ValidatedConfig: FromFragment, + CommonConfig: Default + JsonSchema + Serialize, + Config: Clone + Merge, + RoleConfig: Default + JsonSchema + Serialize, + ConfigOverrides: Default + JsonSchema + Serialize, +{ + role_group.validate_config(role, default_config) +} + +fn merged_config_overrides( + role_config_overrides: &ConfigOverrides, + role_group_config_overrides: ConfigOverrides, +) -> ConfigOverrides +where + ConfigOverrides: Merge, +{ + merge(role_group_config_overrides, role_config_overrides) +} + +fn merged_env_overrides( + role_env_overrides: &HashMap, + role_group_env_overrides: &HashMap, +) -> Result { + // Process the role first, then the role group, so that role-group overrides win on key + // collisions (`EnvVarSet::with_value` overrides earlier entries with the same name). + let mut env_overrides = EnvVarSet::new(); + for (name, value) in role_env_overrides + .iter() + .chain(role_group_env_overrides.iter()) + { + env_overrides = env_overrides.with_value( + &EnvVarName::from_str(name).context(ParseEnvVarNameSnafu)?, + value.clone(), + ); + } + Ok(env_overrides) +} + +fn merged_cli_overrides( + role_cli_overrides: BTreeMap, + role_group_cli_overrides: BTreeMap, +) -> BTreeMap { + let mut merged = role_cli_overrides; + merged.extend(role_group_cli_overrides); + merged +} + +fn merged_pod_overrides( + role_pod_overrides: PodTemplateSpec, + role_group_pod_overrides: PodTemplateSpec, +) -> PodTemplateSpec { + let mut merged = role_pod_overrides; + merged.merge_from(role_group_pod_overrides); + merged +} diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 32d08118..ad7301be 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -41,6 +41,7 @@ mod authentication; mod controller; mod crd; mod extensions; +mod framework; mod internal_secret; mod listener; mod operations; From 9217498b30abcc6b2c0ec3a3d133f3e6d65de475 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 10 Jun 2026 10:54:46 +0200 Subject: [PATCH 29/48] fix: update stackable-operator for optionless KeyValueConfigOverrides --- Cargo.lock | 34 +++++++++---------- Cargo.nix | 34 +++++++++---------- extra/crds.yaml | 30 ---------------- .../src/controller/build/config_map.rs | 12 ++----- 4 files changed, 36 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fa91be38..bb2c2528 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1529,7 +1529,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "darling", "regex", @@ -2373,9 +2373,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.3" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ "aho-corasick", "memchr", @@ -2396,9 +2396,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "relative-path" @@ -2945,7 +2945,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "const-oid", "ecdsa", @@ -2995,7 +2995,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.111.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "base64", "clap", @@ -3039,7 +3039,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "darling", "proc-macro2", @@ -3050,7 +3050,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "jiff", "k8s-openapi", @@ -3067,7 +3067,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.4" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "axum", "clap", @@ -3091,7 +3091,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "kube", "schemars", @@ -3105,7 +3105,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "convert_case", "convert_case_extras", @@ -3123,7 +3123,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#de69410331ea51a37ec91e511d0d2f33056b6032" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" dependencies = [ "arc-swap", "async-trait", @@ -4049,18 +4049,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.50" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.50" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.nix b/Cargo.nix index 174ff591..ec4fb402 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4854,7 +4854,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "k8s_version"; @@ -7701,9 +7701,9 @@ rec { }; "regex" = rec { crateName = "regex"; - version = "1.12.3"; + version = "1.12.4"; edition = "2021"; - sha256 = "0xp2q0x7ybmpa5zlgaz00p8zswcirj9h8nry3rxxsdwi9fhm81z1"; + sha256 = "1fm6si2xpmhwqflabdqsakc0qkq718wx2ljl37nbj75fb5vjnagi"; authors = [ "The Rust Project Developers" "Andrew Gallant " @@ -7820,9 +7820,9 @@ rec { }; "regex-syntax" = rec { crateName = "regex-syntax"; - version = "0.8.10"; + version = "0.8.11"; edition = "2021"; - sha256 = "02jx311ka0daxxc7v45ikzhcl3iydjbbb0mdrpc1xgg8v7c7v2fw"; + sha256 = "1m25h5q2wp976fb9gc3dsc9l99svcvd5cri8lncb51c46ydgzxnn"; libName = "regex_syntax"; authors = [ "The Rust Project Developers" @@ -9647,7 +9647,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_certs"; @@ -9859,7 +9859,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_operator"; @@ -10053,7 +10053,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; procMacro = true; @@ -10088,7 +10088,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_shared"; @@ -10169,7 +10169,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_telemetry"; @@ -10279,7 +10279,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_versioned"; @@ -10329,7 +10329,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; procMacro = true; @@ -10397,7 +10397,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "de69410331ea51a37ec91e511d0d2f33056b6032"; + rev = "451088f77acee6c3d296754698260256c250ecb2"; sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; }; libName = "stackable_webhook"; @@ -14237,9 +14237,9 @@ rec { }; "zerocopy" = rec { crateName = "zerocopy"; - version = "0.8.50"; + version = "0.8.52"; edition = "2021"; - sha256 = "1laahnfxs4qyfb1fdf5nbb2qfshi72b1hbi0ffp2zy2m1r7ms1iv"; + sha256 = "0gv563swc1yn3k8w3wjj07a8q293rkx99nfp3a25vzzmbycj446f"; authors = [ "Joshua Liebow-Feeser " "Jack Wrenn " @@ -14273,9 +14273,9 @@ rec { }; "zerocopy-derive" = rec { crateName = "zerocopy-derive"; - version = "0.8.50"; + version = "0.8.52"; edition = "2021"; - sha256 = "0fdnr9qslx1hbn2i9rsvy9s95mychfy2vj90ajsjm2basccinqqb"; + sha256 = "0c3rhsh4sd9kdym4z55zprybjkydy9y2gvw75d72aapcfa5z7rqs"; procMacro = true; libName = "zerocopy_derive"; authors = [ diff --git a/extra/crds.yaml b/extra/crds.yaml index 063e1a98..b79058b2 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -412,21 +412,18 @@ spec: properties: jvm.config: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `jvm.config` file. type: object runtime.properties: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `runtime.properties` file. type: object security.properties: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `security.properties` file. @@ -906,21 +903,18 @@ spec: properties: jvm.config: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `jvm.config` file. type: object runtime.properties: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `runtime.properties` file. type: object security.properties: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `security.properties` file. @@ -2017,21 +2011,18 @@ spec: properties: jvm.config: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `jvm.config` file. type: object runtime.properties: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `runtime.properties` file. type: object security.properties: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `security.properties` file. @@ -2511,21 +2502,18 @@ spec: properties: jvm.config: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `jvm.config` file. type: object runtime.properties: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `runtime.properties` file. type: object security.properties: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `security.properties` file. @@ -3024,21 +3012,18 @@ spec: properties: jvm.config: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `jvm.config` file. type: object runtime.properties: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `runtime.properties` file. type: object security.properties: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `security.properties` file. @@ -3558,21 +3543,18 @@ spec: properties: jvm.config: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `jvm.config` file. type: object runtime.properties: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `runtime.properties` file. type: object security.properties: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `security.properties` file. @@ -4090,21 +4072,18 @@ spec: properties: jvm.config: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `jvm.config` file. type: object runtime.properties: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `runtime.properties` file. type: object security.properties: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `security.properties` file. @@ -4580,21 +4559,18 @@ spec: properties: jvm.config: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `jvm.config` file. type: object runtime.properties: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `runtime.properties` file. type: object security.properties: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `security.properties` file. @@ -5063,21 +5039,18 @@ spec: properties: jvm.config: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `jvm.config` file. type: object runtime.properties: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `runtime.properties` file. type: object security.properties: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `security.properties` file. @@ -5557,21 +5530,18 @@ spec: properties: jvm.config: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `jvm.config` file. type: object runtime.properties: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `runtime.properties` file. type: object security.properties: additionalProperties: - nullable: true type: string default: {} description: Overrides for the `security.properties` file. diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index b17dc841..9709be6f 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -120,25 +120,17 @@ fn middlemanager_indexer_java_opts() -> (String, String) { /// Returns the user-supplied key/value overrides for the given config file from a /// [`DruidConfigOverrides`]. -/// -/// The CRD override map allows a value-less key (`someKey:` / null in YAML), modelled as -/// `Option`. We flatten `None` to an empty string here, matching how the Java properties -/// writer rendered a missing value (`key=`), so the rest of the pipeline can work with plain -/// `String` values. fn key_value_overrides( overrides: &DruidConfigOverrides, file: ConfigFileName, ) -> BTreeMap { - let raw = match file { + match file { ConfigFileName::RuntimeProperties => overrides.runtime_properties.overrides.clone(), ConfigFileName::SecurityProperties => overrides.security_properties.overrides.clone(), // log4j2.properties is rendered by the logging framework, and jvm.config is rendered from // JVM argument overrides; neither is assembled from key/value overrides here. ConfigFileName::Log4j2Properties | ConfigFileName::JvmConfig => BTreeMap::new(), - }; - raw.into_iter() - .map(|(k, v)| (k, v.unwrap_or_default())) - .collect() + } } /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator From afbbbfb048c9b84612ea3bfefdac4d7d1a57568c Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 10 Jun 2026 11:34:00 +0200 Subject: [PATCH 30/48] fix: upate hashes --- Cargo.nix | 18 +++++++++--------- crate-hashes.json | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index ec4fb402..ff09eb4d 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4855,7 +4855,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "451088f77acee6c3d296754698260256c250ecb2"; - sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "k8s_version"; authors = [ @@ -9648,7 +9648,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "451088f77acee6c3d296754698260256c250ecb2"; - sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_certs"; authors = [ @@ -9860,7 +9860,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "451088f77acee6c3d296754698260256c250ecb2"; - sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_operator"; authors = [ @@ -10054,7 +10054,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "451088f77acee6c3d296754698260256c250ecb2"; - sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -10089,7 +10089,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "451088f77acee6c3d296754698260256c250ecb2"; - sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_shared"; authors = [ @@ -10170,7 +10170,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "451088f77acee6c3d296754698260256c250ecb2"; - sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_telemetry"; authors = [ @@ -10280,7 +10280,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "451088f77acee6c3d296754698260256c250ecb2"; - sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_versioned"; authors = [ @@ -10330,7 +10330,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "451088f77acee6c3d296754698260256c250ecb2"; - sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -10398,7 +10398,7 @@ rec { src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; rev = "451088f77acee6c3d296754698260256c250ecb2"; - sha256 = "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47"; + sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_webhook"; authors = [ diff --git a/crate-hashes.json b/crate-hashes.json index deac3bf4..c9a6e6a9 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "05zvdnnmjvpma9yds4kp8rwkna2d3kkws3yry2080jdy753npz47", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file From 46bfa1d2524f16780633304d2798f38113d4bbc2 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 11 Jun 2026 09:50:14 +0200 Subject: [PATCH 31/48] fix: remove product-config references in comments --- .../src/controller/build/config_map.rs | 2 +- .../build/properties/runtime_properties.rs | 13 +++---------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 9709be6f..378f13f7 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -3,7 +3,7 @@ //! The per-file configs (runtime.properties / security.properties / jvm.config) are rendered here //! from the merged [`DruidRoleGroupConfig`] (config plus the merged config overrides); the //! recommended cluster-level runtime properties and the erased roles needed for `jvm.config` are -//! carried on `ValidatedCluster`. Product-config is no longer involved. +//! carried on `ValidatedCluster`. //! //! Metadata, owner reference and recommended labels are derived entirely from `ValidatedCluster` //! (which carries the validated name/namespace/uid and implements `Resource`). diff --git a/rust/operator-binary/src/controller/build/properties/runtime_properties.rs b/rust/operator-binary/src/controller/build/properties/runtime_properties.rs index 4f0c9343..26d56c4b 100644 --- a/rust/operator-binary/src/controller/build/properties/runtime_properties.rs +++ b/rust/operator-binary/src/controller/build/properties/runtime_properties.rs @@ -1,11 +1,7 @@ -//! Builder for the static, role-specific `runtime.properties` defaults that were -//! previously injected by product-config's `recommendedValues`. +//! Builder for the static, role-specific `runtime.properties`. //! -//! Only includes a recommended value for a role where the property was -//! `required: true` for that role (verified against -//! `tests/templates/kuttl/smoke/53-assert.yaml.j2`). Dynamic, cluster-derived -//! values (ZooKeeper, extensions, metadata DB, deep storage, TLS, auth, ports, -//! resource-derived sizes) are added by the controller, not here. +//! Dynamic, cluster-derived values (ZooKeeper, extensions, metadata DB, deep storage, +//! TLS, auth, ports, resource-derived sizes) are added by the controller, not here. use std::collections::BTreeMap; @@ -25,9 +21,6 @@ const PROMETHEUS_PORT: &str = "druid.emitter.prometheus.port"; /// The recommended cluster-level `runtime.properties` derived from the cluster config (deep /// storage, OPA authorization and metrics). These are independent of role and role group. -/// -/// `opa_authorization_enabled` mirrors `authorization.opa` being configured (equivalently, the -/// OPA connection string having been resolved during dereferencing). pub fn cluster_runtime_properties( deep_storage: &DeepStorageSpec, opa_authorization_enabled: bool, From ece60ddbea545b1814db2c53d9931331a717759a Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 11 Jun 2026 10:01:36 +0200 Subject: [PATCH 32/48] fix: remove unused namespace in ValidatedCluster --- rust/operator-binary/src/controller/validate.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index be0d06f3..a18cb909 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -99,10 +99,6 @@ pub struct ValidatedCluster { /// [`Resource`] and be passed directly to the metadata/owner-reference builders. metadata: ObjectMeta, pub name: ClusterName, - // Read from the mirrored `metadata` in the configmap path (via `name_and_namespace`); the typed - // field is consumed directly when the service/statefulset builders move onto `ValidatedCluster`. - #[allow(dead_code)] - pub namespace: NamespaceName, pub uid: Uid, pub image: ResolvedProductImage, pub cluster_config: ValidatedClusterConfig, @@ -133,7 +129,6 @@ impl ValidatedCluster { Self { metadata, name, - namespace, uid, image, cluster_config, From 8ad2fe75a93391ddfbb1826a29b271eeb3e4841b Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 11 Jun 2026 21:12:23 +0200 Subject: [PATCH 33/48] refactor: switch to v2 JavaCommonConfig & remove framework module --- Cargo.lock | 18 +- Cargo.nix | 18 +- rust/operator-binary/src/controller.rs | 92 +-------- .../src/controller/build/config_map.rs | 9 +- .../src/controller/build/jvm.rs | 97 +++------ .../src/controller/validate.rs | 169 +++++++++++++--- rust/operator-binary/src/crd/mod.rs | 187 ++++++------------ rust/operator-binary/src/framework.rs | 11 -- .../src/framework/role_utils.rs | 152 -------------- rust/operator-binary/src/main.rs | 1 - 10 files changed, 258 insertions(+), 496 deletions(-) delete mode 100644 rust/operator-binary/src/framework.rs delete mode 100644 rust/operator-binary/src/framework/role_utils.rs diff --git a/Cargo.lock b/Cargo.lock index bb2c2528..6766c75a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1529,7 +1529,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "darling", "regex", @@ -2945,7 +2945,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "const-oid", "ecdsa", @@ -2995,7 +2995,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.111.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "base64", "clap", @@ -3039,7 +3039,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "darling", "proc-macro2", @@ -3050,7 +3050,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "jiff", "k8s-openapi", @@ -3067,7 +3067,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.4" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "axum", "clap", @@ -3091,7 +3091,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "kube", "schemars", @@ -3105,7 +3105,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "convert_case", "convert_case_extras", @@ -3123,7 +3123,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#451088f77acee6c3d296754698260256c250ecb2" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" dependencies = [ "arc-swap", "async-trait", diff --git a/Cargo.nix b/Cargo.nix index 5671fb0e..b2762459 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4854,7 +4854,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "k8s_version"; @@ -9647,7 +9647,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_certs"; @@ -9859,7 +9859,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_operator"; @@ -10053,7 +10053,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; procMacro = true; @@ -10088,7 +10088,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_shared"; @@ -10169,7 +10169,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_telemetry"; @@ -10279,7 +10279,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_versioned"; @@ -10329,7 +10329,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; procMacro = true; @@ -10397,7 +10397,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "451088f77acee6c3d296754698260256c250ecb2"; + rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; }; libName = "stackable_webhook"; diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index aa5f473e..347763b4 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -923,25 +923,12 @@ pub fn error_policy( #[cfg(test)] mod test { - use std::{collections::BTreeMap, str::FromStr}; - use rstest::*; - use stackable_operator::{ - database_connections::drivers::jdbc::JdbcDatabaseConnection, - v2::types::{ - kubernetes::{NamespaceName, Uid}, - operator::ClusterName, - }, - }; use super::*; use crate::{ - controller::{ - build::{config_map::build_rolegroup_config_map, properties::ConfigFileName}, - validate::{ValidatedCluster, ValidatedClusterConfig}, - }, - crd::{PROP_SEGMENT_CACHE_LOCATIONS, authentication::AuthenticationClassesResolved}, - extensions::get_extension_list, + controller::build::{config_map::build_rolegroup_config_map, properties::ConfigFileName}, + crd::PROP_SEGMENT_CACHE_LOCATIONS, }; #[rstest] @@ -960,82 +947,23 @@ mod test { #[case] tested_rolegroup_name: &str, #[case] expected_druid_segment_cache_property: &str, ) { - let cluster_cr = - std::fs::File::open(format!("test/resources/druid_controller/{druid_manifest}")) + let yaml = + std::fs::read_to_string(format!("test/resources/druid_controller/{druid_manifest}")) .unwrap(); - let deserializer = serde_yaml::Deserializer::from_reader(&cluster_cr); - let druid: v1alpha1::DruidCluster = - serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); - - let resolved_product_image: ResolvedProductImage = druid - .spec - .image - .resolve( - CONTAINER_IMAGE_BASE_NAME, - "oci.example.org", - crate::built_info::PKG_VERSION, - ) - .expect("test: resolved product image is always valid"); + let druid = crate::controller::validate::test_support::druid_from_yaml(&yaml); - let druid_tls_security = DruidTlsSecurity::new( - &AuthenticationClassesResolved { - auth_classes: vec![], - }, - Some("tls".to_string()), - ); + let cluster = crate::controller::validate::test_support::validated_cluster(&druid); // The segment cache property is injected dynamically by the config_map builder from the // merged resources of the validated role group config. - let rg = druid - .merged_role(&DruidRole::Historical) - .expect("merged historical role") + let rg = cluster + .role_group_configs + .get(&DruidRole::Historical) + .expect("historical role groups") .get(tested_rolegroup_name) .expect("tested rolegroup") .clone(); - // The build step renders jvm.config from the erased role; populate the one role the test - // exercises. - let mut roles = BTreeMap::new(); - roles.insert( - DruidRole::Historical, - druid.get_role(&DruidRole::Historical), - ); - - let extensions = get_extension_list(&druid, &druid_tls_security, &None); - let metadata_storage_type = druid - .spec - .cluster_config - .metadata_database - .as_metadata_storage_type() - .to_string(); - let metadata_db_connection = druid - .spec - .cluster_config - .metadata_database - .jdbc_connection_details("metadata") - .expect("test: valid metadata db connection"); - - let cluster = ValidatedCluster::new( - ClusterName::from_str(&druid.name_any()).expect("test: valid cluster name"), - NamespaceName::from_str("default").expect("test: valid namespace"), - Uid::from_str("c27b3971-ca72-42c1-80a4-abdfc1db0ddd").expect("test: valid uid"), - resolved_product_image.clone(), - ValidatedClusterConfig { - zookeeper_connection_string: "zookeeper-connection-string".to_string(), - opa_connection_string: None, - s3_connection: None, - deep_storage_bucket_name: None, - druid_tls_security, - druid_auth_config: None, - extensions, - metadata_storage_type, - metadata_db_connection, - deep_storage: druid.spec.cluster_config.deep_storage.clone(), - }, - BTreeMap::new(), - roles, - ); - let rolegroup_ref = RoleGroupRef { cluster: ObjectRef::from_obj(&druid), role: DruidRole::Historical.to_string(), diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index 378f13f7..88cacbd9 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -1,9 +1,9 @@ //! Builds the rolegroup [`ConfigMap`] from a [`ValidatedCluster`]. //! //! The per-file configs (runtime.properties / security.properties / jvm.config) are rendered here -//! from the merged [`DruidRoleGroupConfig`] (config plus the merged config overrides); the -//! recommended cluster-level runtime properties and the erased roles needed for `jvm.config` are -//! carried on `ValidatedCluster`. +//! from the merged [`DruidRoleGroupConfig`] (config plus the merged config overrides, including the +//! merged JVM argument overrides used for `jvm.config`); the recommended cluster-level runtime +//! properties are carried on `ValidatedCluster`. //! //! Metadata, owner reference and recommended labels are derived entirely from `ValidatedCluster` //! (which carries the validated name/namespace/uid and implements `Resource`). @@ -293,8 +293,7 @@ pub fn build_rolegroup_config_map( .context(DeriveMemorySettingsSnafu)?; let jvm_config = construct_jvm_args( role, - cluster.get_role(role), - &rolegroup.role_group, + &rg.product_specific_common_config.jvm_argument_overrides, heap, direct, ) diff --git a/rust/operator-binary/src/controller/build/jvm.rs b/rust/operator-binary/src/controller/build/jvm.rs index bb72f41b..64b114f6 100644 --- a/rust/operator-binary/src/controller/build/jvm.rs +++ b/rust/operator-binary/src/controller/build/jvm.rs @@ -1,14 +1,11 @@ use snafu::{ResultExt, Snafu}; use stackable_operator::{ - memory::MemoryQuantity, - role_utils, - role_utils::{GenericRoleConfig, JavaCommonConfig, JvmArgumentOverrides, Role}, + memory::MemoryQuantity, v2::jvm_argument_overrides::JvmArgumentOverrides, }; use super::properties::ConfigFileName; use crate::crd::{ - DruidConfigOverrides, DruidRole, RW_CONFIG_DIRECTORY, STACKABLE_TRUST_STORE, - STACKABLE_TRUST_STORE_PASSWORD, + DruidRole, RW_CONFIG_DIRECTORY, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, }; #[derive(Snafu, Debug)] @@ -18,17 +15,13 @@ pub enum Error { value: MemoryQuantity, source: stackable_operator::memory::Error, }, - - #[snafu(display("failed to merge jvm argument overrides"))] - MergeJvmArgumentOverrides { source: role_utils::Error }, } /// Please note that this function is slightly different than all other operators, because memory /// management is far more advanced in this operator. -pub fn construct_jvm_args( +pub fn construct_jvm_args( druid_role: &DruidRole, - role: &Role, - role_group: &str, + jvm_argument_overrides: &JvmArgumentOverrides, heap: MemoryQuantity, direct_memory: Option, ) -> Result { @@ -71,14 +64,7 @@ pub fn construct_jvm_args( jvm_args.push("-Dderby.stream.error.file=/stackable/var/druid/derby.log".to_owned()); } - let operator_generated = JvmArgumentOverrides::new_with_only_additions(jvm_args); - let merged_jvm_argument_overrides = role - .get_merged_jvm_argument_overrides(role_group, &operator_generated) - .context(MergeJvmArgumentOverridesSnafu)?; - - Ok(merged_jvm_argument_overrides - .effective_jvm_config_after_merging() - .join("\n")) + Ok(jvm_argument_overrides.apply_to(jvm_args).join("\n")) } #[cfg(test)] @@ -86,53 +72,15 @@ mod tests { use indoc::indoc; use super::*; - use crate::crd::v1alpha1::DruidCluster; #[test] fn test_construct_jvm_arguments_defaults() { - let input = r#" - apiVersion: druid.stackable.tech/v1alpha1 - kind: DruidCluster - metadata: - name: simple-druid - spec: - image: - productVersion: 30.0.0 - clusterConfig: - deepStorage: - hdfs: - configMapName: simple-hdfs - directory: /druid - metadataDatabase: - postgresql: - host: druid-postgresql - database: druid - credentialsSecretName: mySecret - zookeeperConfigMapName: simple-druid-znode - brokers: - roleGroups: - default: - replicas: 1 - coordinators: - roleGroups: - default: - replicas: 1 - historicals: - roleGroups: - default: - replicas: 1 - middleManagers: - roleGroups: - default: - replicas: 1 - routers: - roleGroups: - default: - replicas: 1 - "#; + use crate::controller::validate::test_support::MINIMAL_DRUID_YAML; - let coordinator_jvm_config = construct_jvm_config_for_test(input, &DruidRole::Coordinator); - let historical_jvm_config = construct_jvm_config_for_test(input, &DruidRole::Historical); + let coordinator_jvm_config = + construct_jvm_config_for_test(MINIMAL_DRUID_YAML, &DruidRole::Coordinator); + let historical_jvm_config = + construct_jvm_config_for_test(MINIMAL_DRUID_YAML, &DruidRole::Historical); assert_eq!( coordinator_jvm_config, @@ -289,20 +237,19 @@ mod tests { } fn construct_jvm_config_for_test(druid_cluster: &str, druid_role: &DruidRole) -> String { - let deserializer = serde_yaml::Deserializer::from_str(druid_cluster); - let druid: DruidCluster = - serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); + use crate::controller::validate::test_support::druid_from_yaml; - let role = druid.get_role(druid_role); + let druid = druid_from_yaml(druid_cluster); let merged_role = druid.merged_role(druid_role).unwrap(); - let (heap, direct) = merged_role - .get("default") - .unwrap() - .config - .resources - .get_memory_sizes(druid_role) - .unwrap(); - - construct_jvm_args(druid_role, &role, "default", heap, direct).unwrap() + let rg = merged_role.get("default").unwrap(); + let (heap, direct) = rg.config.resources.get_memory_sizes(druid_role).unwrap(); + + construct_jvm_args( + druid_role, + &rg.product_specific_common_config.jvm_argument_overrides, + heap, + direct, + ) + .unwrap() } } diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index a18cb909..8d33fe5f 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -26,7 +26,7 @@ use strum::IntoEnumIterator; use crate::{ authentication::DruidAuthenticationConfig, controller::dereference::DereferencedObjects, - crd::{DeepStorageSpec, DruidErasedRole, DruidRole, security::DruidTlsSecurity, v1alpha1}, + crd::{DeepStorageSpec, DruidRole, security::DruidTlsSecurity, v1alpha1}, extensions::get_extension_list, }; @@ -63,10 +63,11 @@ pub type RoleGroupName = String; /// A validated, merged role-group config. /// -/// This is the framework [`RoleGroupConfig`] (config plus the four merged override categories), -/// with the typed per-role config erased to [`CommonRoleGroupConfig`] so that all roles share a -/// single type. The rendered per-file configs (runtime.properties / security.properties / -/// jvm.config) are produced later, in the config-map build step. +/// This is the upstream [`stackable_operator::v2::role_utils::RoleGroupConfig`] (config plus the +/// four merged override categories), with the typed per-role config erased to +/// [`CommonRoleGroupConfig`] so that all roles share a single type. The rendered per-file configs +/// (runtime.properties / security.properties / jvm.config) are produced later, in the config-map +/// build step. /// /// Defined in [`crate::crd`] (where it has access to the private typed config fields) and /// re-exported here for the build step. @@ -103,10 +104,6 @@ pub struct ValidatedCluster { pub image: ResolvedProductImage, pub cluster_config: ValidatedClusterConfig, pub role_group_configs: BTreeMap>, - /// The erased roles (see [`DruidErasedRole`]), retained so the build step can render - /// `jvm.config` (which merges the role/rolegroup JVM argument overrides) without the raw - /// `DruidCluster`. - roles: BTreeMap, } impl ValidatedCluster { @@ -118,7 +115,6 @@ impl ValidatedCluster { image: ResolvedProductImage, cluster_config: ValidatedClusterConfig, role_group_configs: BTreeMap>, - roles: BTreeMap, ) -> Self { let metadata = ObjectMeta { name: Some(name.to_string()), @@ -133,17 +129,8 @@ impl ValidatedCluster { image, cluster_config, role_group_configs, - roles, } } - - /// The erased role (carrying the JVM argument overrides) for the given role. Used by the build - /// step to render `jvm.config`. - pub fn get_role(&self, role: &DruidRole) -> &DruidErasedRole { - self.roles - .get(role) - .expect("every DruidRole is populated during validation") - } } // Implementing `Resource` (plus `HasName`/`HasUid`) lets `ValidatedCluster` stand in for the raw @@ -219,14 +206,12 @@ pub fn validate( let mut role_group_configs: BTreeMap> = BTreeMap::new(); - let mut roles = BTreeMap::new(); for druid_role in DruidRole::iter() { let group_map = druid .merged_role(&druid_role) .context(FailedToResolveConfigSnafu)?; role_group_configs.insert(druid_role.clone(), group_map); - roles.insert(druid_role.clone(), druid.get_role(&druid_role)); } let name = get_cluster_name(druid).context(ClusterIdentitySnafu)?; @@ -265,6 +250,146 @@ pub fn validate( deep_storage: druid.spec.cluster_config.deep_storage.clone(), }, role_group_configs, - roles, )) } + +#[cfg(test)] +pub(crate) mod test_support { + use std::{collections::BTreeMap, str::FromStr}; + + use stackable_operator::{ + database_connections::drivers::jdbc::JdbcDatabaseConnection, + kube::ResourceExt, + v2::types::{ + kubernetes::{NamespaceName, Uid}, + operator::ClusterName, + }, + }; + use strum::IntoEnumIterator; + + use super::{RoleGroupName, ValidatedCluster, ValidatedClusterConfig}; + use crate::{ + controller::CONTAINER_IMAGE_BASE_NAME, + crd::{ + DruidRole, authentication::AuthenticationClassesResolved, security::DruidTlsSecurity, + v1alpha1, + }, + extensions::get_extension_list, + }; + + /// A minimal but fully-valid `DruidCluster` with one role group per role. + pub const MINIMAL_DRUID_YAML: &str = r#" +apiVersion: druid.stackable.tech/v1alpha1 +kind: DruidCluster +metadata: + name: simple-druid + namespace: default + uid: c27b3971-ca72-42c1-80a4-abdfc1db0ddd +spec: + image: + productVersion: 30.0.0 + clusterConfig: + deepStorage: + hdfs: + configMapName: simple-hdfs + directory: /druid + metadataDatabase: + postgresql: + host: druid-postgresql + database: druid + credentialsSecretName: mySecret + zookeeperConfigMapName: simple-druid-znode + brokers: + roleGroups: + default: + replicas: 1 + coordinators: + roleGroups: + default: + replicas: 1 + historicals: + roleGroups: + default: + replicas: 1 + middleManagers: + roleGroups: + default: + replicas: 1 + routers: + roleGroups: + default: + replicas: 1 +"#; + + /// Parses a `DruidCluster` from YAML the same way the operator does (singleton-map enums). + pub fn druid_from_yaml(yaml: &str) -> v1alpha1::DruidCluster { + let deserializer = serde_yaml::Deserializer::from_str(yaml); + serde_yaml::with::singleton_map_recursive::deserialize(deserializer) + .expect("invalid test DruidCluster YAML") + } + + /// Builds a [`ValidatedCluster`] from a `DruidCluster` using test defaults for the + /// dereferenced/cluster-wide fields (no S3, no auth, fixed zookeeper string). Runs the real + /// `merged_role` pipeline for every role. + pub fn validated_cluster(druid: &v1alpha1::DruidCluster) -> ValidatedCluster { + let image = druid + .spec + .image + .resolve( + CONTAINER_IMAGE_BASE_NAME, + "oci.example.org", + crate::built_info::PKG_VERSION, + ) + .expect("test: resolvable product image"); + + let druid_tls_security = DruidTlsSecurity::new( + &AuthenticationClassesResolved { + auth_classes: vec![], + }, + Some("tls".to_string()), + ); + + let mut role_group_configs: BTreeMap> = + BTreeMap::new(); + for role in DruidRole::iter() { + role_group_configs.insert( + role.clone(), + druid.merged_role(&role).expect("test: merged role"), + ); + } + + let extensions = get_extension_list(druid, &druid_tls_security, &None); + let metadata_storage_type = druid + .spec + .cluster_config + .metadata_database + .as_metadata_storage_type() + .to_string(); + let metadata_db_connection = druid + .spec + .cluster_config + .metadata_database + .jdbc_connection_details("metadata") + .expect("test: valid metadata db connection"); + + ValidatedCluster::new( + ClusterName::from_str(&druid.name_any()).expect("test: valid cluster name"), + NamespaceName::from_str("default").expect("test: valid namespace"), + Uid::from_str("c27b3971-ca72-42c1-80a4-abdfc1db0ddd").expect("test: valid uid"), + image, + ValidatedClusterConfig { + zookeeper_connection_string: "zookeeper-connection-string".to_string(), + opa_connection_string: None, + s3_connection: None, + deep_storage_bucket_name: None, + druid_tls_security, + druid_auth_config: None, + extensions, + metadata_storage_type, + metadata_db_connection, + deep_storage: druid.spec.cluster_config.deep_storage.clone(), + }, + role_group_configs, + ) + } +} diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 8e8351be..370fffa7 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,4 +1,7 @@ -use std::collections::{BTreeMap, HashSet}; +use std::{ + collections::{BTreeMap, HashSet}, + str::FromStr, +}; use indoc::formatdoc; use security::add_cert_to_jvm_trust_store_cmd; @@ -13,7 +16,7 @@ use stackable_operator::{ resources::{NoRuntimeLimits, Resources}, }, config::{ - fragment::{Fragment, FromFragment, ValidationError}, + fragment::{Fragment, ValidationError}, merge::Merge, }, crd::{ @@ -29,25 +32,26 @@ use stackable_operator::{ framework::{create_vector_shutdown_file_command, remove_vector_shutdown_file_command}, spec::Logging, }, - role_utils::{CommonConfiguration, GenericRoleConfig, JavaCommonConfig, Role, RoleGroup}, + role_utils::{GenericRoleConfig, Role}, schemars::{self, JsonSchema}, shared::time::Duration, status::condition::{ClusterCondition, HasStatusCondition}, utils::{COMMON_BASH_TRAP_FUNCTIONS, crds::raw_object_list_schema}, - v2::config_overrides::KeyValueConfigOverrides, + v2::{ + builder::pod::container::{EnvVarName, EnvVarSet}, + config_overrides::KeyValueConfigOverrides, + role_utils::{JavaCommonConfig, RoleGroupConfig, with_validated_config}, + }, versioned::versioned, }; use strum::{Display, EnumDiscriminants, EnumIter, EnumString, IntoStaticStr}; -use crate::{ - crd::{ - affinity::get_affinity, - authorization::DruidAuthorization, - database::MetadataDatabaseConnection, - resource::RoleResource, - tls::{DruidTls, default_druid_tls}, - }, - framework::role_utils::{RoleGroupConfig, with_validated_config}, +use crate::crd::{ + affinity::get_affinity, + authorization::DruidAuthorization, + database::MetadataDatabaseConnection, + resource::RoleResource, + tls::{DruidTls, default_druid_tls}, }; pub mod affinity; @@ -162,7 +166,13 @@ pub enum Error { #[snafu(display("failed to merge and validate config for role group {role_group:?}"))] FailedToMergeRoleGroupConfig { - source: crate::framework::role_utils::Error, + source: ValidationError, + role_group: String, + }, + + #[snafu(display("invalid environment variable override name in role group {role_group:?}"))] + ParseEnvVarName { + source: stackable_operator::v2::builder::pod::container::Error, role_group: String, }, } @@ -442,26 +452,45 @@ impl v1alpha1::DruidCluster { role_group: rg_name.clone(), })?; let common = CommonRoleGroupConfig { - resources: $resource(validated.config.resources), - logging: validated.config.logging, + resources: $resource(validated.config.config.resources), + logging: validated.config.config.logging, replicas: rg.replicas, - affinity: validated.config.affinity, - graceful_shutdown_timeout: validated.config.graceful_shutdown_timeout, + affinity: validated.config.config.affinity, + graceful_shutdown_timeout: validated + .config + .config + .graceful_shutdown_timeout, requested_secret_lifetime: validated + .config .config .requested_secret_lifetime .context(MissingSecretLifetimeSnafu)?, }; + // Upstream returns env overrides as a `HashMap`; the build step consumes an + // `EnvVarSet`. Convert here, validating each name. (Role/role-group precedence + // is already resolved by `with_validated_config`.) + let mut env_overrides = EnvVarSet::new(); + for (name, value) in validated.config.env_overrides { + env_overrides = env_overrides.with_value( + &EnvVarName::from_str(&name).with_context(|_| { + ParseEnvVarNameSnafu { + role_group: rg_name.clone(), + } + })?, + value, + ); + } groups.insert( rg_name.clone(), DruidRoleGroupConfig { - replicas: validated.replicas, + replicas: validated.replicas.unwrap_or(1), config: common, - config_overrides: validated.config_overrides, - env_overrides: validated.env_overrides, - cli_overrides: validated.cli_overrides, - pod_overrides: validated.pod_overrides, + config_overrides: validated.config.config_overrides, + env_overrides, + cli_overrides: validated.config.cli_overrides, + pod_overrides: validated.config.pod_overrides, product_specific_common_config: validated + .config .product_specific_common_config, }, ); @@ -518,22 +547,6 @@ impl v1alpha1::DruidCluster { DruidRole::Router => &self.spec.routers.role_config.generic, } } - - pub fn get_role(&self, druid_role: &DruidRole) -> DruidErasedRole { - match druid_role { - DruidRole::Broker => erase_config(extract_role_from_role_config::( - self.spec.brokers.clone(), - )), - DruidRole::Coordinator => erase_config(extract_role_from_role_config::< - CoordinatorConfig, - >(self.spec.coordinators.clone())), - DruidRole::Historical => erase_config(self.spec.historicals.clone()), - DruidRole::MiddleManager => erase_config(self.spec.middle_managers.clone()), - DruidRole::Router => erase_config(extract_role_from_role_config::( - self.spec.routers.clone(), - )), - } - } } #[derive( @@ -570,22 +583,18 @@ pub struct CommonRoleGroupConfig { /// A validated, merged role-group config. /// -/// This is the framework [`RoleGroupConfig`] (config plus the four merged override categories, -/// role group winning over role), with the typed per-role config erased to the shared -/// [`CommonRoleGroupConfig`] view. The rendered per-file configs (runtime.properties / +/// This is the upstream [`stackable_operator::v2::role_utils::RoleGroupConfig`] (config plus the +/// four merged override categories, role group winning over role), with the typed per-role config +/// erased to the shared [`CommonRoleGroupConfig`] view. The merged JVM argument overrides live in +/// `product_specific_common_config`; the rendered per-file configs (runtime.properties / /// security.properties / jvm.config) are produced later, in the config-map build step. /// /// Note: the StatefulSet replicas come from [`CommonRoleGroupConfig::replicas`] (an `Option`, so -/// an unspecified count is left unset and stays HPA-friendly), not from the framework +/// an unspecified count is left unset and stays HPA-friendly), not from /// [`RoleGroupConfig::replicas`]. pub type DruidRoleGroupConfig = RoleGroupConfig; -/// A role with its typed config erased to `()`, retaining the override layers (notably the JVM -/// argument overrides in `product_specific_common_config`). Carried on the validated cluster so -/// the build step can render `jvm.config` without reaching back to the raw `DruidCluster`. -pub type DruidErasedRole = Role<(), DruidConfigOverrides, GenericRoleConfig, JavaCommonConfig>; - impl Default for v1alpha1::DruidRoleConfig { fn default() -> Self { v1alpha1::DruidRoleConfig { @@ -936,88 +945,6 @@ pub fn build_recommended_labels<'a, T>( } } -fn extract_role_from_role_config( - fragment: Role, -) -> Role -where - T: FromFragment, - T::Fragment: Clone + Merge, -{ - Role { - config: CommonConfiguration { - config: fragment.config.config, - config_overrides: fragment.config.config_overrides, - env_overrides: fragment.config.env_overrides, - cli_overrides: fragment.config.cli_overrides, - pod_overrides: fragment.config.pod_overrides, - product_specific_common_config: fragment.config.product_specific_common_config, - }, - role_config: fragment.role_config.generic, - role_groups: fragment - .role_groups - .into_iter() - .map(|(k, v)| { - ( - k, - RoleGroup { - config: CommonConfiguration { - config: v.config.config, - config_overrides: v.config.config_overrides, - env_overrides: v.config.env_overrides, - cli_overrides: v.config.cli_overrides, - pod_overrides: v.config.pod_overrides, - product_specific_common_config: v.config.product_specific_common_config, - }, - replicas: v.replicas, - }, - ) - }) - .collect(), - } -} - -/// Discards the typed `config` of a [`Role`], replacing it with `()`. -/// -/// `get_role` needs to return a single concrete type across all roles, but each role has a -/// different config fragment. Callers only read the role/role-group level overrides -/// (`config_overrides`, `env_overrides`, `product_specific_common_config`), never the typed config -/// itself, so erasing it to `()` yields a uniform return type without needing a trait object. -fn erase_config( - role: Role, -) -> Role<(), DruidConfigOverrides, GenericRoleConfig, JavaCommonConfig> { - Role { - config: CommonConfiguration { - config: (), - config_overrides: role.config.config_overrides, - env_overrides: role.config.env_overrides, - cli_overrides: role.config.cli_overrides, - pod_overrides: role.config.pod_overrides, - product_specific_common_config: role.config.product_specific_common_config, - }, - role_config: role.role_config, - role_groups: role - .role_groups - .into_iter() - .map(|(k, v)| { - ( - k, - RoleGroup { - config: CommonConfiguration { - config: (), - config_overrides: v.config.config_overrides, - env_overrides: v.config.env_overrides, - cli_overrides: v.config.cli_overrides, - pod_overrides: v.config.pod_overrides, - product_specific_common_config: v.config.product_specific_common_config, - }, - replicas: v.replicas, - }, - ) - }) - .collect(), - } -} - #[cfg(test)] mod tests { use stackable_operator::versioned::test_utils::RoundtripTestData; diff --git a/rust/operator-binary/src/framework.rs b/rust/operator-binary/src/framework.rs deleted file mode 100644 index 1edc49b4..00000000 --- a/rust/operator-binary/src/framework.rs +++ /dev/null @@ -1,11 +0,0 @@ -//! Local framework helpers that mirror the work-in-progress upstream -//! `stackable_operator::v2::*` modules. -//! -//! We vendor `role_utils` because the upstream `v2::role_utils` requires -//! `CommonConfig: Merge`. Druid (like hive and trino) uses `JavaCommonConfig`, whose -//! JVM-argument merge is fallible and so does not implement `Merge`. -//! -//! Follow-up: replace with `stackable_operator::v2::role_utils::*` once upstream -//! relaxes the `Merge` bound. - -pub mod role_utils; diff --git a/rust/operator-binary/src/framework/role_utils.rs b/rust/operator-binary/src/framework/role_utils.rs deleted file mode 100644 index 9cf4b3ba..00000000 --- a/rust/operator-binary/src/framework/role_utils.rs +++ /dev/null @@ -1,152 +0,0 @@ -//! Vendored variant of `stackable_operator::v2::role_utils` from the -//! `smooth-operator` branch, with simplifications appropriate for druid-operator. -//! -//! Differences from upstream: -//! - No `cli_overrides_to_vec` helper, `ResourceNames`, or service-account helpers. -//! - The `CommonConfig` (a.k.a. `product_specific_common_config`) does NOT need to -//! implement `Merge`. Druid uses `JavaCommonConfig`, which intentionally does not -//! implement `Merge` because its inner `JvmArgumentOverrides::try_merge` is -//! fallible (regex validation). The `RoleGroupConfig::product_specific_common_config` -//! field here simply carries the role-group level value through. -//! -//! Replace with `stackable_operator::v2::role_utils::*` once upstream relaxes the -//! `Merge` bound. - -use std::{ - collections::{BTreeMap, HashMap}, - str::FromStr, -}; - -use serde::Serialize; -use snafu::{ResultExt, Snafu}; -use stackable_operator::{ - config::{ - fragment::{self, FromFragment}, - merge::{Merge, merge}, - }, - k8s_openapi::{DeepMerge, api::core::v1::PodTemplateSpec}, - role_utils::{Role, RoleGroup}, - schemars::JsonSchema, - v2::builder::pod::container::{self, EnvVarName, EnvVarSet}, -}; - -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("failed to validate the role group config"))] - ValidateConfig { source: fragment::ValidationError }, - - #[snafu(display("invalid environment variable override name"))] - ParseEnvVarName { source: container::Error }, -} - -/// Druid-friendly view of a validated, merged `RoleGroup`. -#[derive(Clone, Debug, PartialEq)] -pub struct RoleGroupConfig { - pub replicas: u16, - pub config: Config, - pub config_overrides: ConfigOverrides, - pub env_overrides: EnvVarSet, - pub cli_overrides: BTreeMap, - pub pod_overrides: PodTemplateSpec, - pub product_specific_common_config: CommonConfig, -} - -/// Merges and validates the `RoleGroup` with the given `role` and `default_config`. -pub fn with_validated_config( - role_group: &RoleGroup, - role: &Role, - default_config: &Config, -) -> Result, Error> -where - ValidatedConfig: FromFragment, - CommonConfig: Clone + Default + JsonSchema + Serialize, - Config: Clone + Merge, - RoleConfig: Default + JsonSchema + Serialize, - ConfigOverrides: Clone + Default + JsonSchema + Merge + Serialize, -{ - let validated_config = - validate_config(role_group, role, default_config).context(ValidateConfigSnafu)?; - Ok(RoleGroupConfig { - replicas: role_group.replicas.unwrap_or(1), - config: validated_config, - config_overrides: merged_config_overrides( - &role.config.config_overrides, - role_group.config.config_overrides.clone(), - ), - env_overrides: merged_env_overrides( - &role.config.env_overrides, - &role_group.config.env_overrides, - )?, - cli_overrides: merged_cli_overrides( - role.config.cli_overrides.clone(), - role_group.config.cli_overrides.clone(), - ), - pod_overrides: merged_pod_overrides( - role.config.pod_overrides.clone(), - role_group.config.pod_overrides.clone(), - ), - product_specific_common_config: role_group.config.product_specific_common_config.clone(), - }) -} - -fn validate_config( - role_group: &RoleGroup, - role: &Role, - default_config: &Config, -) -> Result -where - ValidatedConfig: FromFragment, - CommonConfig: Default + JsonSchema + Serialize, - Config: Clone + Merge, - RoleConfig: Default + JsonSchema + Serialize, - ConfigOverrides: Default + JsonSchema + Serialize, -{ - role_group.validate_config(role, default_config) -} - -fn merged_config_overrides( - role_config_overrides: &ConfigOverrides, - role_group_config_overrides: ConfigOverrides, -) -> ConfigOverrides -where - ConfigOverrides: Merge, -{ - merge(role_group_config_overrides, role_config_overrides) -} - -fn merged_env_overrides( - role_env_overrides: &HashMap, - role_group_env_overrides: &HashMap, -) -> Result { - // Process the role first, then the role group, so that role-group overrides win on key - // collisions (`EnvVarSet::with_value` overrides earlier entries with the same name). - let mut env_overrides = EnvVarSet::new(); - for (name, value) in role_env_overrides - .iter() - .chain(role_group_env_overrides.iter()) - { - env_overrides = env_overrides.with_value( - &EnvVarName::from_str(name).context(ParseEnvVarNameSnafu)?, - value.clone(), - ); - } - Ok(env_overrides) -} - -fn merged_cli_overrides( - role_cli_overrides: BTreeMap, - role_group_cli_overrides: BTreeMap, -) -> BTreeMap { - let mut merged = role_cli_overrides; - merged.extend(role_group_cli_overrides); - merged -} - -fn merged_pod_overrides( - role_pod_overrides: PodTemplateSpec, - role_group_pod_overrides: PodTemplateSpec, -) -> PodTemplateSpec { - let mut merged = role_pod_overrides; - merged.merge_from(role_group_pod_overrides); - merged -} diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index ad7301be..32d08118 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -41,7 +41,6 @@ mod authentication; mod controller; mod crd; mod extensions; -mod framework; mod internal_secret; mod listener; mod operations; From 3ebe45e4b5c142f009739a406941566dfdb04b8b Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 11:39:49 +0200 Subject: [PATCH 34/48] chore: bump stackable operator --- Cargo.lock | 95 ++++++++++++++++--------------- Cargo.nix | 138 ++++++++++++++++++++++------------------------ crate-hashes.json | 18 +++--- 3 files changed, 120 insertions(+), 131 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6766c75a..0ad739b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -302,9 +302,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "cc" -version = "1.2.63" +version = "1.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" +checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" dependencies = [ "find-msvc-tools", "jobserver", @@ -574,9 +574,6 @@ name = "deranged" version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" -dependencies = [ - "powerfmt", -] [[package]] name = "derive_more" @@ -1468,9 +1465,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.100" +version = "0.3.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2025f20d7a4fa7785846e7b63d10a76d3f1cee98ee5cb79ea59703f95e42162" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" dependencies = [ "cfg-if", "futures-util", @@ -1529,7 +1526,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "darling", "regex", @@ -1742,9 +1739,9 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "memchr" -version = "2.8.1" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "mime" @@ -1848,9 +1845,9 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "openssl" -version = "0.10.80" +version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ "bitflags", "cfg-if", @@ -1879,9 +1876,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.116" +version = "0.9.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" dependencies = [ "cc", "libc", @@ -2843,9 +2840,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.1" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "snafu" @@ -2945,7 +2942,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "const-oid", "ecdsa", @@ -2995,7 +2992,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.111.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "base64", "clap", @@ -3020,6 +3017,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "sha2", "snafu 0.9.1", "stackable-operator-derive", "stackable-shared", @@ -3039,7 +3037,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "darling", "proc-macro2", @@ -3050,7 +3048,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "jiff", "k8s-openapi", @@ -3067,7 +3065,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.4" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "axum", "clap", @@ -3091,7 +3089,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "kube", "schemars", @@ -3105,7 +3103,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "convert_case", "convert_case_extras", @@ -3123,7 +3121,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#1e8099fd157b06f27d93854b0838f67871448c4e" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "arc-swap", "async-trait", @@ -3284,12 +3282,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.47" +version = "0.3.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" dependencies = [ "deranged", - "itoa", "num-conv", "powerfmt", "serde_core", @@ -3299,15 +3296,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.27" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +checksum = "71c652a3727a9cbb9a02f707f530b618ce00d0ccd762009c8c23bd191df3c17d" dependencies = [ "num-conv", "time-core", @@ -3760,18 +3757,18 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.3+wasi-0.2.9" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ "wit-bindgen", ] [[package]] name = "wasm-bindgen" -version = "0.2.123" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a254a4b10c19a76f09a27640e7ffbf9bc30bf67e16a3bf28aaefa4920fe81563" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" dependencies = [ "cfg-if", "once_cell", @@ -3782,9 +3779,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.73" +version = "0.4.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54568702fabf5d4849ce2b90fadfa64168a097eaf4b351ce9df8b687a0086aaf" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" dependencies = [ "js-sys", "wasm-bindgen", @@ -3792,9 +3789,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.123" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24a40fc75b0ec6f3746ceb10d36f53a93dcd68a93b11b6445983945d79eba0dc" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3802,9 +3799,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.123" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "908f34bd9b9ce3d4caf07b72dfab63d61504d156856c6bd3cd87fa350cf3985b" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" dependencies = [ "bumpalo", "proc-macro2", @@ -3815,18 +3812,18 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.123" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7acbf7616c27b194bbb550bf77ed0c2c3e5b7fd1260a93082b95fb7f47959b92" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" -version = "0.3.100" +version = "0.3.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e0871acf327f283dc6da28a1696cdc64fb355ba9f935d052021fa77f35cce69" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" dependencies = [ "js-sys", "wasm-bindgen", @@ -4090,18 +4087,18 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.8.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" dependencies = [ "zeroize_derive", ] [[package]] name = "zeroize_derive" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.nix b/Cargo.nix index b2762459..48e9e657 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -962,9 +962,9 @@ rec { }; "cc" = rec { crateName = "cc"; - version = "1.2.63"; + version = "1.2.64"; edition = "2018"; - sha256 = "0zy2bqc4nvj6bv2cipx4h4bn65wf1zqf1fw1hsh64mmvg1hh2vjm"; + sha256 = "07shcd8faxw7csz13m3cg2mj6i8z07pqs960k181pscbjpyqgn6s"; authors = [ "Alex Crichton " ]; @@ -1740,14 +1740,6 @@ rec { authors = [ "Jacob Pratt " ]; - dependencies = [ - { - name = "powerfmt"; - packageId = "powerfmt"; - optional = true; - usesDefaultFeatures = false; - } - ]; features = { "macros" = [ "dep:deranged-macros" ]; "num" = [ "dep:num-traits" ]; @@ -1759,7 +1751,7 @@ rec { "rand09" = [ "dep:rand09" ]; "serde" = [ "dep:serde_core" ]; }; - resolvedDefaultFeatures = [ "default" "powerfmt" ]; + resolvedDefaultFeatures = [ "default" ]; }; "derive_more" = rec { crateName = "derive_more"; @@ -4646,9 +4638,9 @@ rec { }; "js-sys" = rec { crateName = "js-sys"; - version = "0.3.100"; + version = "0.3.102"; edition = "2021"; - sha256 = "0qi1wjakyw2rx9wwprcfx77g3lvn1b8n6yvfhj2pgym4swh5y0pj"; + sha256 = "0cgxklnyrfpzvf32cvdl3x5d070kfsv7ykdxfl3yizwdjqq4rl03"; libName = "js_sys"; authors = [ "The wasm-bindgen Developers" @@ -4854,8 +4846,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; libName = "k8s_version"; authors = [ @@ -5718,9 +5710,9 @@ rec { }; "memchr" = rec { crateName = "memchr"; - version = "2.8.1"; + version = "2.8.2"; edition = "2021"; - sha256 = "1n448jx01h5z2xknj6x2dhxgr8s8fb717cf6vfqj5lmhkpj7m53b"; + sha256 = "1i33wr49pcz2sbd12nds3n9fszay8kq5bk78gwciz462mcs49448"; authors = [ "Andrew Gallant " "bluss" @@ -6043,9 +6035,9 @@ rec { }; "openssl" = rec { crateName = "openssl"; - version = "0.10.80"; + version = "0.10.81"; edition = "2021"; - sha256 = "0ryrcbdd7hq0ydvassn4cr02agii1l54yd6sali7chkci2ma4px4"; + sha256 = "0ibsv2ppsjrp62jqyzprhay9vczk1bw9xvdr3h4h7fxsy0kkm0kp"; authors = [ "Steven Fackler " ]; @@ -6122,10 +6114,10 @@ rec { }; "openssl-sys" = rec { crateName = "openssl-sys"; - version = "0.9.116"; + version = "0.9.117"; edition = "2021"; links = "openssl"; - sha256 = "1i0qcgsimh8qkfgrglmzz2kq3jk2d5575rz5jvqabka0f7f252pj"; + sha256 = "159nf6jsqnmsynkh6gjzx088q1ifll7v88sss8qdk363n9mpwzml"; build = "build/main.rs"; libName = "openssl_sys"; authors = [ @@ -9338,9 +9330,9 @@ rec { }; "smallvec" = rec { crateName = "smallvec"; - version = "1.15.1"; + version = "1.15.2"; edition = "2018"; - sha256 = "00xxdxxpgyq5vjnpljvkmy99xij5rxgh913ii1v16kzynnivgcb7"; + sha256 = "143wzbqf6vgapdp2z4qpl0yvlqcn17s8cnk8m28rqly808zsdmlf"; authors = [ "The Servo Project Developers" ]; @@ -9647,8 +9639,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; libName = "stackable_certs"; authors = [ @@ -9859,8 +9851,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; libName = "stackable_operator"; authors = [ @@ -9969,6 +9961,11 @@ rec { name = "serde_yaml"; packageId = "serde_yaml"; } + { + name = "sha2"; + packageId = "sha2"; + features = [ "oid" ]; + } { name = "snafu"; packageId = "snafu 0.9.1"; @@ -10053,8 +10050,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -10088,8 +10085,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; libName = "stackable_shared"; authors = [ @@ -10169,8 +10166,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; libName = "stackable_telemetry"; authors = [ @@ -10279,8 +10276,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; libName = "stackable_versioned"; authors = [ @@ -10329,8 +10326,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -10397,8 +10394,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "1e8099fd157b06f27d93854b0838f67871448c4e"; - sha256 = "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; libName = "stackable_webhook"; authors = [ @@ -10837,9 +10834,9 @@ rec { }; "time" = rec { crateName = "time"; - version = "0.3.47"; + version = "0.3.49"; edition = "2024"; - sha256 = "0b7g9ly2iabrlgizliz6v5x23yq5d6bpp0mqz6407z1s526d8fvl"; + sha256 = "0sc4dgw6g187gvz5qj9iqqk2ashqzvdwi664b2183gbvsk1566ki"; authors = [ "Jacob Pratt " "Time contributors" @@ -10848,12 +10845,6 @@ rec { { name = "deranged"; packageId = "deranged"; - features = [ "powerfmt" ]; - } - { - name = "itoa"; - packageId = "itoa"; - optional = true; } { name = "num-conv"; @@ -10893,13 +10884,14 @@ rec { features = { "alloc" = [ "serde_core?/alloc" ]; "default" = [ "std" ]; - "formatting" = [ "dep:itoa" "std" "time-macros?/formatting" ]; + "formatting" = [ "std" "time-macros?/formatting" ]; "large-dates" = [ "time-core/large-dates" "time-macros?/large-dates" ]; "local-offset" = [ "std" "dep:libc" "dep:num_threads" ]; "macros" = [ "dep:time-macros" ]; "parsing" = [ "time-macros?/parsing" ]; "quickcheck" = [ "dep:quickcheck" "alloc" "deranged/quickcheck" ]; - "rand" = [ "rand08" "rand09" ]; + "rand" = [ "rand08" "rand09" "rand010" ]; + "rand010" = [ "dep:rand010" "deranged/rand010" ]; "rand08" = [ "dep:rand08" "deranged/rand08" ]; "rand09" = [ "dep:rand09" "deranged/rand09" ]; "serde" = [ "dep:serde_core" "time-macros?/serde" "deranged/serde" ]; @@ -10912,9 +10904,9 @@ rec { }; "time-core" = rec { crateName = "time-core"; - version = "0.1.8"; + version = "0.1.9"; edition = "2024"; - sha256 = "1jidl426mw48i7hjj4hs9vxgd9lwqq4vyalm4q8d7y4iwz7y353n"; + sha256 = "028ix0ax7ixp1h1k5zsqwgw85w6y1q32irslma7ci6ddd5kr074y"; libName = "time_core"; authors = [ "Jacob Pratt " @@ -10925,9 +10917,9 @@ rec { }; "time-macros" = rec { crateName = "time-macros"; - version = "0.2.27"; + version = "0.2.29"; edition = "2024"; - sha256 = "058ja265waq275wxvnfwavbz9r1hd4dgwpfn7a1a9a70l32y8w1f"; + sha256 = "0zf1ycfikg93ijf00qnprk801khqnqqga1zp0adbp73sfaim5iki"; procMacro = true; libName = "time_macros"; authors = [ @@ -12569,9 +12561,9 @@ rec { }; "wasip2" = rec { crateName = "wasip2"; - version = "1.0.3+wasi-0.2.9"; + version = "1.0.4+wasi-0.2.12"; edition = "2021"; - sha256 = "1mi3w855dz99xzjqc4aa8c9q5b6z1y5c963pkk4cvmr6vdr4c1i0"; + sha256 = "11wl7lqwq4pbmlmzr6n7bwz0hzy1z6sxc4554bkmrr86w4vznzmn"; dependencies = [ { name = "wit-bindgen"; @@ -12589,9 +12581,9 @@ rec { }; "wasm-bindgen" = rec { crateName = "wasm-bindgen"; - version = "0.2.123"; + version = "0.2.125"; edition = "2021"; - sha256 = "0qqmx07r597gm8lbz8qngvv0phwvpzzyfh3nl84nz9qr1jqs8m52"; + sha256 = "06nakz7nfy0ymyp7a27wfbjwx69659i12117hkgddkiv2iwkznwd"; libName = "wasm_bindgen"; authors = [ "The wasm-bindgen Developers" @@ -12640,9 +12632,9 @@ rec { }; "wasm-bindgen-futures" = rec { crateName = "wasm-bindgen-futures"; - version = "0.4.73"; + version = "0.4.75"; edition = "2021"; - sha256 = "1bva12h8gdpqkp753czlxabs0s21lvgzm41brr4lhpdzz818fmjl"; + sha256 = "104jssshr6cm5hmkn6c66mbkyxgaaphng6c17g0dmj7jhk918fsh"; libName = "wasm_bindgen_futures"; authors = [ "The wasm-bindgen Developers" @@ -12668,9 +12660,9 @@ rec { }; "wasm-bindgen-macro" = rec { crateName = "wasm-bindgen-macro"; - version = "0.2.123"; + version = "0.2.125"; edition = "2021"; - sha256 = "1p50xdwmv543b52bc49vm5lcsgd9adpx647bdisg7ihfbg3hz914"; + sha256 = "0g9w68dwcs4ylm5kxf7schi0kjdfarhc9qlnf8arxc9zn62a28af"; procMacro = true; libName = "wasm_bindgen_macro"; authors = [ @@ -12692,9 +12684,9 @@ rec { }; "wasm-bindgen-macro-support" = rec { crateName = "wasm-bindgen-macro-support"; - version = "0.2.123"; + version = "0.2.125"; edition = "2021"; - sha256 = "0nwqyc63byl7rp9nnv45av8h85fncfmxywkvy35d9qwwkfyk93wh"; + sha256 = "1gayzdx5iwl8gllh7ys79wg9cf4iyasl9hrzzhh5m4xx6nfgvkpy"; libName = "wasm_bindgen_macro_support"; authors = [ "The wasm-bindgen Developers" @@ -12728,10 +12720,10 @@ rec { }; "wasm-bindgen-shared" = rec { crateName = "wasm-bindgen-shared"; - version = "0.2.123"; + version = "0.2.125"; edition = "2021"; links = "wasm_bindgen"; - sha256 = "14lvjm3pzywm5c4962i6s5zmngic1knpggshnnxr9c97dihzgjvs"; + sha256 = "07w7fy5qa14ys3p8v2p84h98yqinw713smibz9v7apcspd29x4r3"; libName = "wasm_bindgen_shared"; authors = [ "The wasm-bindgen Developers" @@ -12746,9 +12738,9 @@ rec { }; "web-sys" = rec { crateName = "web-sys"; - version = "0.3.100"; + version = "0.3.102"; edition = "2021"; - sha256 = "0sffbkrpgyi1402mv4wzp9av6ky6rnb1d2m2dpf87wi7yfn7223f"; + sha256 = "0786aybrnwsgdmcynhc2k5ii291a02rq9zk054j35csyvxr0lhx6"; libName = "web_sys"; authors = [ "The wasm-bindgen Developers" @@ -14361,9 +14353,9 @@ rec { }; "zeroize" = rec { crateName = "zeroize"; - version = "1.8.2"; - edition = "2021"; - sha256 = "1l48zxgcv34d7kjskr610zqsm6j2b4fcr2vfh9jm9j1jgvk58wdr"; + version = "1.9.0"; + edition = "2024"; + sha256 = "0kpnij2v1ig6g2mhc0bnci0lrdfdhiq40afbc0fahajqc9jiag71"; authors = [ "The RustCrypto Project Developers" ]; @@ -14385,9 +14377,9 @@ rec { }; "zeroize_derive" = rec { crateName = "zeroize_derive"; - version = "1.4.3"; - edition = "2021"; - sha256 = "0bl5vd1lz27p4z336nximg5wrlw5j7jc8fxh7iv6r1wrhhav99c5"; + version = "1.5.0"; + edition = "2024"; + sha256 = "0a7kq8srk81pn23xqn7c9jw1jpnfy41ffn802x1zrqqgpdf6al1w"; procMacro = true; authors = [ "The RustCrypto Project Developers" diff --git a/crate-hashes.json b/crate-hashes.json index c9a6e6a9..bb8b51c7 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "1ifdpn0jvrf3xbgqldqxrq9ig1dc34d4fip7qxn38526k8004p4b", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file From 4ea8968ce63294bf8b27645e0ebfdc29620cbda8 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 11:51:14 +0200 Subject: [PATCH 35/48] refactor: move k8s resource builders under controller/build/resource --- rust/operator-binary/src/controller.rs | 37 ++++++++++++------- rust/operator-binary/src/controller/build.rs | 3 +- .../build/{ => resource}/config_map.rs | 0 .../build/{ => resource}/discovery.rs | 9 +++-- .../build/resource}/listener.rs | 0 .../src/controller/build/resource/mod.rs | 7 ++++ .../build/resource}/pdb.rs | 0 .../build/resource}/service.rs | 0 rust/operator-binary/src/main.rs | 2 - rust/operator-binary/src/operations/mod.rs | 1 - 10 files changed, 38 insertions(+), 21 deletions(-) rename rust/operator-binary/src/controller/build/{ => resource}/config_map.rs (100%) rename rust/operator-binary/src/controller/build/{ => resource}/discovery.rs (92%) rename rust/operator-binary/src/{ => controller/build/resource}/listener.rs (100%) create mode 100644 rust/operator-binary/src/controller/build/resource/mod.rs rename rust/operator-binary/src/{operations => controller/build/resource}/pdb.rs (100%) rename rust/operator-binary/src/{ => controller/build/resource}/service.rs (100%) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 347763b4..ac264217 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -54,6 +54,14 @@ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ authentication::DruidAuthenticationConfig, + controller::build::resource::{ + listener::{ + LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_group_listener, + build_group_listener_pvc, group_listener_name, secret_volume_listener_scope, + }, + pdb::add_pdbs, + service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, + }, crd::{ APP_NAME, CommonRoleGroupConfig, Container, DRUID_CONFIG_DIRECTORY, DeepStorageSpec, DruidClusterStatus, DruidRole, HDFS_CONFIG_DIRECTORY, LOG_CONFIG_DIRECTORY, METRICS_PORT, @@ -61,12 +69,7 @@ use crate::{ build_recommended_labels, security::DruidTlsSecurity, v1alpha1, }, internal_secret::create_shared_internal_secret, - listener::{ - LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_group_listener, build_group_listener_pvc, - group_listener_name, secret_volume_listener_scope, - }, - operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, - service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, + operations::graceful_shutdown::add_graceful_shutdown_config, }; mod build; @@ -74,8 +77,8 @@ mod dereference; mod validate; use build::{ - discovery::{self, build_discovery_configmaps}, properties::logging::MAX_DRUID_LOG_FILES_SIZE, + resource::discovery::{self, build_discovery_configmaps}, }; use validate::DruidRoleGroupConfig; @@ -196,7 +199,7 @@ pub enum Error { #[snafu(display("failed to create PodDisruptionBudget"))] FailedToCreatePdb { - source: crate::operations::pdb::Error, + source: crate::controller::build::resource::pdb::Error, }, #[snafu(display("failed to configure graceful shutdown"))] @@ -244,16 +247,22 @@ pub enum Error { }, #[snafu(display("failed to configure listener"))] - ListenerConfiguration { source: crate::listener::Error }, + ListenerConfiguration { + source: crate::controller::build::resource::listener::Error, + }, #[snafu(display("failed to configure service"))] - ServiceConfiguration { source: crate::service::Error }, + ServiceConfiguration { + source: crate::controller::build::resource::service::Error, + }, #[snafu(display("failed to validate cluster"))] ValidateCluster { source: validate::Error }, #[snafu(display("failed to build rolegroup ConfigMap"))] - BuildConfigMap { source: build::config_map::Error }, + BuildConfigMap { + source: build::resource::config_map::Error, + }, #[snafu(display("invalid metadata database connection"))] InvalidMetadataDatabaseConnection { @@ -367,7 +376,7 @@ pub async fn reconcile_druid( ) .context(ServiceConfigurationSnafu)?; - let rg_configmap = build::config_map::build_rolegroup_config_map( + let rg_configmap = build::resource::config_map::build_rolegroup_config_map( &validated_cluster, druid_role, &rolegroup, @@ -927,7 +936,9 @@ mod test { use super::*; use crate::{ - controller::build::{config_map::build_rolegroup_config_map, properties::ConfigFileName}, + controller::build::{ + properties::ConfigFileName, resource::config_map::build_rolegroup_config_map, + }, crd::PROP_SEGMENT_CACHE_LOCATIONS, }; diff --git a/rust/operator-binary/src/controller/build.rs b/rust/operator-binary/src/controller/build.rs index c090434d..398af03c 100644 --- a/rust/operator-binary/src/controller/build.rs +++ b/rust/operator-binary/src/controller/build.rs @@ -1,6 +1,5 @@ //! Build steps that turn a `ValidatedCluster` into Kubernetes resources. -pub mod config_map; -pub mod discovery; pub mod jvm; pub mod properties; +pub mod resource; diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/resource/config_map.rs similarity index 100% rename from rust/operator-binary/src/controller/build/config_map.rs rename to rust/operator-binary/src/controller/build/resource/config_map.rs diff --git a/rust/operator-binary/src/controller/build/discovery.rs b/rust/operator-binary/src/controller/build/resource/discovery.rs similarity index 92% rename from rust/operator-binary/src/controller/build/discovery.rs rename to rust/operator-binary/src/controller/build/resource/discovery.rs index 388684b8..ccda3887 100644 --- a/rust/operator-binary/src/controller/build/discovery.rs +++ b/rust/operator-binary/src/controller/build/resource/discovery.rs @@ -11,9 +11,10 @@ use stackable_operator::{ use crate::{ DRUID_CONTROLLER_NAME, - controller::validate::ValidatedCluster, + controller::{ + build::resource::listener::build_listener_connection_string, validate::ValidatedCluster, + }, crd::{DruidRole, build_recommended_labels}, - listener::build_listener_connection_string, }; #[derive(Snafu, Debug)] @@ -29,7 +30,9 @@ pub enum Error { }, #[snafu(display("failed to configure listener discovery configmap"))] - ListenerConfiguration { source: crate::listener::Error }, + ListenerConfiguration { + source: crate::controller::build::resource::listener::Error, + }, } /// Builds discovery [`ConfigMap`]s for connecting to a Druid cluster. diff --git a/rust/operator-binary/src/listener.rs b/rust/operator-binary/src/controller/build/resource/listener.rs similarity index 100% rename from rust/operator-binary/src/listener.rs rename to rust/operator-binary/src/controller/build/resource/listener.rs diff --git a/rust/operator-binary/src/controller/build/resource/mod.rs b/rust/operator-binary/src/controller/build/resource/mod.rs new file mode 100644 index 00000000..512cb299 --- /dev/null +++ b/rust/operator-binary/src/controller/build/resource/mod.rs @@ -0,0 +1,7 @@ +//! Builders for the individual Kubernetes resources of a Druid cluster. + +pub mod config_map; +pub mod discovery; +pub mod listener; +pub mod pdb; +pub mod service; diff --git a/rust/operator-binary/src/operations/pdb.rs b/rust/operator-binary/src/controller/build/resource/pdb.rs similarity index 100% rename from rust/operator-binary/src/operations/pdb.rs rename to rust/operator-binary/src/controller/build/resource/pdb.rs diff --git a/rust/operator-binary/src/service.rs b/rust/operator-binary/src/controller/build/resource/service.rs similarity index 100% rename from rust/operator-binary/src/service.rs rename to rust/operator-binary/src/controller/build/resource/service.rs diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 32d08118..9a984771 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -42,9 +42,7 @@ mod controller; mod crd; mod extensions; mod internal_secret; -mod listener; mod operations; -mod service; mod webhooks; mod built_info { diff --git a/rust/operator-binary/src/operations/mod.rs b/rust/operator-binary/src/operations/mod.rs index 92ca2ec7..590612ba 100644 --- a/rust/operator-binary/src/operations/mod.rs +++ b/rust/operator-binary/src/operations/mod.rs @@ -1,2 +1 @@ pub mod graceful_shutdown; -pub mod pdb; From fb1dac0ead64b3ef0d981ff097332427015341b0 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 12:32:30 +0200 Subject: [PATCH 36/48] refactor: typed RoleGroupName + ValidatedCluster::resource_names; adopt in services --- rust/operator-binary/src/controller.rs | 16 +++++--- .../src/controller/build/jvm.rs | 7 +++- .../src/controller/build/resource/service.rs | 38 ++++++++++++------- .../src/controller/validate.rs | 21 ++++++++-- rust/operator-binary/src/crd/affinity.rs | 7 +++- rust/operator-binary/src/crd/mod.rs | 16 +++++++- rust/operator-binary/src/crd/resource.rs | 19 ++++++++-- 7 files changed, 92 insertions(+), 32 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index ac264217..f1c6fdd0 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -340,7 +340,7 @@ pub async fn reconcile_druid( let rolegroup = RoleGroupRef { cluster: ObjectRef::from_obj(druid), role: role_name.clone(), - role_group: rolegroup_name.into(), + role_group: rolegroup_name.to_string(), }; let role_group_service_recommended_labels = build_recommended_labels( @@ -360,17 +360,18 @@ pub async fn reconcile_druid( .context(LabelBuildSnafu)?; let rg_headless_service = build_rolegroup_headless_service( - druid, + &validated_cluster, &validated_cluster.cluster_config.druid_tls_security, druid_role, - &rolegroup, + rolegroup_name, role_group_service_recommended_labels.clone(), role_group_service_selector.clone().into(), ) .context(ServiceConfigurationSnafu)?; let rg_metrics_service = build_rolegroup_metrics_service( - druid, - &rolegroup, + &validated_cluster, + druid_role, + rolegroup_name, role_group_service_recommended_labels, role_group_service_selector.into(), ) @@ -932,7 +933,10 @@ pub fn error_policy( #[cfg(test)] mod test { + use std::str::FromStr; + use rstest::*; + use stackable_operator::v2::types::operator::RoleGroupName; use super::*; use crate::{ @@ -971,7 +975,7 @@ mod test { .role_group_configs .get(&DruidRole::Historical) .expect("historical role groups") - .get(tested_rolegroup_name) + .get(&RoleGroupName::from_str(tested_rolegroup_name).unwrap()) .expect("tested rolegroup") .clone(); diff --git a/rust/operator-binary/src/controller/build/jvm.rs b/rust/operator-binary/src/controller/build/jvm.rs index 64b114f6..d456de33 100644 --- a/rust/operator-binary/src/controller/build/jvm.rs +++ b/rust/operator-binary/src/controller/build/jvm.rs @@ -69,7 +69,10 @@ pub fn construct_jvm_args( #[cfg(test)] mod tests { + use std::str::FromStr; + use indoc::indoc; + use stackable_operator::v2::types::operator::RoleGroupName; use super::*; @@ -241,7 +244,9 @@ mod tests { let druid = druid_from_yaml(druid_cluster); let merged_role = druid.merged_role(druid_role).unwrap(); - let rg = merged_role.get("default").unwrap(); + let rg = merged_role + .get(&RoleGroupName::from_str("default").unwrap()) + .unwrap(); let (heap, direct) = rg.config.resources.get_memory_sizes(druid_role).unwrap(); construct_jvm_args( diff --git a/rust/operator-binary/src/controller/build/resource/service.rs b/rust/operator-binary/src/controller/build/resource/service.rs index 1d9ec6cb..889a708e 100644 --- a/rust/operator-binary/src/controller/build/resource/service.rs +++ b/rust/operator-binary/src/controller/build/resource/service.rs @@ -5,11 +5,12 @@ use stackable_operator::{ builder::meta::ObjectMetaBuilder, k8s_openapi::api::core::v1::{Service, ServicePort, ServiceSpec}, kvp::{Annotations, Label, ObjectLabels}, - role_utils::RoleGroupRef, + v2::types::operator::RoleGroupName, }; -use crate::crd::{ - DruidRole, METRICS_PORT, METRICS_PORT_NAME, security::DruidTlsSecurity, v1alpha1, +use crate::{ + controller::validate::ValidatedCluster, + crd::{DruidRole, METRICS_PORT, METRICS_PORT_NAME, security::DruidTlsSecurity, v1alpha1}, }; #[derive(Snafu, Debug)] @@ -33,18 +34,23 @@ pub enum Error { /// The rolegroup headless [`Service`] is a service that allows direct access to the instances of a certain rolegroup /// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. pub fn build_rolegroup_headless_service( - druid: &v1alpha1::DruidCluster, + cluster: &ValidatedCluster, druid_tls_security: &DruidTlsSecurity, druid_role: &DruidRole, - role_group_ref: &RoleGroupRef, + role_group_name: &RoleGroupName, object_labels: ObjectLabels, selector: BTreeMap, ) -> Result { Ok(Service { metadata: ObjectMetaBuilder::new() - .name_and_namespace(druid) - .name(role_group_ref.rolegroup_headless_service_name()) - .ownerreference_from_resource(druid, None, Some(true)) + .name_and_namespace(cluster) + .name( + cluster + .resource_names(druid_role, role_group_name) + .headless_service_name() + .to_string(), + ) + .ownerreference_from_resource(cluster, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(&object_labels) .context(MetadataBuildSnafu)? @@ -64,16 +70,22 @@ pub fn build_rolegroup_headless_service( /// The rolegroup metrics [`Service`] is a service that exposes metrics and a prometheus scraping label. pub fn build_rolegroup_metrics_service( - druid: &v1alpha1::DruidCluster, - role_group_ref: &RoleGroupRef, + cluster: &ValidatedCluster, + druid_role: &DruidRole, + role_group_name: &RoleGroupName, object_labels: ObjectLabels, selector: BTreeMap, ) -> Result { Ok(Service { metadata: ObjectMetaBuilder::new() - .name_and_namespace(druid) - .name(role_group_ref.rolegroup_metrics_service_name()) - .ownerreference_from_resource(druid, None, Some(true)) + .name_and_namespace(cluster) + .name( + cluster + .resource_names(druid_role, role_group_name) + .metrics_service_name() + .to_string(), + ) + .ownerreference_from_resource(cluster, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(&object_labels) .context(MetadataBuildSnafu)? diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 8d33fe5f..7f4182d0 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -3,7 +3,7 @@ //! Synchronously validates inputs that don't require a Kubernetes client. Produces //! [`ValidatedCluster`], consumed by the rest of `reconcile_druid`. -use std::{borrow::Cow, collections::BTreeMap}; +use std::{borrow::Cow, collections::BTreeMap, str::FromStr}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ @@ -15,9 +15,10 @@ use stackable_operator::{ v2::{ HasName, HasUid, controller_utils::{get_cluster_name, get_namespace, get_uid}, + role_group_utils::ResourceNames, types::{ kubernetes::{NamespaceName, Uid}, - operator::ClusterName, + operator::{ClusterName, RoleGroupName, RoleName}, }, }, }; @@ -59,8 +60,6 @@ pub enum Error { type Result = std::result::Result; -pub type RoleGroupName = String; - /// A validated, merged role-group config. /// /// This is the upstream [`stackable_operator::v2::role_utils::RoleGroupConfig`] (config plus the @@ -131,6 +130,20 @@ impl ValidatedCluster { role_group_configs, } } + + /// Type-safe names for the resources of the given role's role group. + pub(crate) fn resource_names( + &self, + role: &DruidRole, + role_group_name: &RoleGroupName, + ) -> ResourceNames { + ResourceNames { + cluster_name: self.name.clone(), + role_name: RoleName::from_str(&role.to_string()) + .expect("a DruidRole is a valid role name"), + role_group_name: role_group_name.clone(), + } + } } // Implementing `Resource` (plus `HasName`/`HasUid`) lets `ValidatedCluster` stand in for the raw diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 5e0bae1e..2076d8c4 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -76,7 +76,7 @@ pub fn get_affinity( #[cfg(test)] mod tests { - use std::collections::BTreeMap; + use std::{collections::BTreeMap, str::FromStr}; use rstest::rstest; use stackable_operator::{ @@ -87,6 +87,7 @@ mod tests { }, apimachinery::pkg::apis::meta::v1::LabelSelector, }, + v2::types::operator::RoleGroupName, }; use super::*; @@ -143,7 +144,9 @@ mod tests { let druid: v1alpha1::DruidCluster = serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); let merged_role = druid.merged_role(&role).unwrap(); - let merged_config = merged_role.get("default").unwrap(); + let merged_config = merged_role + .get(&RoleGroupName::from_str("default").unwrap()) + .unwrap(); let mut expected_affinities = vec![]; diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 370fffa7..f926d210 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -41,6 +41,7 @@ use stackable_operator::{ builder::pod::container::{EnvVarName, EnvVarSet}, config_overrides::KeyValueConfigOverrides, role_utils::{JavaCommonConfig, RoleGroupConfig, with_validated_config}, + types::operator::RoleGroupName, }, versioned::versioned, }; @@ -175,6 +176,12 @@ pub enum Error { source: stackable_operator::v2::builder::pod::container::Error, role_group: String, }, + + #[snafu(display("invalid role group name {role_group:?}"))] + ParseRoleGroupName { + source: stackable_operator::v2::macros::attributed_string_type::Error, + role_group: String, + }, } #[versioned( @@ -429,7 +436,7 @@ impl v1alpha1::DruidCluster { pub fn merged_role( &self, role: &DruidRole, - ) -> Result, Error> { + ) -> Result, Error> { let deep_storage = &self.spec.cluster_config.deep_storage; let name = self.name_any(); @@ -480,8 +487,13 @@ impl v1alpha1::DruidCluster { value, ); } + let role_group_name = RoleGroupName::from_str(rg_name).with_context(|_| { + ParseRoleGroupNameSnafu { + role_group: rg_name.clone(), + } + })?; groups.insert( - rg_name.clone(), + role_group_name, DruidRoleGroupConfig { replicas: validated.replicas.unwrap_or(1), config: common, diff --git a/rust/operator-binary/src/crd/resource.rs b/rust/operator-binary/src/crd/resource.rs index 7d17a947..1a3f3693 100644 --- a/rust/operator-binary/src/crd/resource.rs +++ b/rust/operator-binary/src/crd/resource.rs @@ -248,6 +248,8 @@ pub(crate) static ROUTER_RESOURCES: LazyLock< #[cfg(test)] mod test { + use std::str::FromStr; + use rstest::*; use stackable_operator::{ commons::resources::{ @@ -257,6 +259,7 @@ mod test { config::{fragment, merge::Merge}, k8s_openapi::apimachinery::pkg::api::resource::Quantity, utils::yaml_from_str_singleton_map, + v2::types::operator::RoleGroupName, }; use super::*; @@ -420,7 +423,7 @@ mod test { let middle_managers = cluster.merged_role(&DruidRole::MiddleManager).unwrap(); if let Some(RoleResource::Druid(middlemanager_resources_from_rg)) = middle_managers - .get("resources-from-role-group") + .get(&RoleGroupName::from_str("resources-from-role-group").unwrap()) .map(|rg| &rg.config.resources) { let expected = Resources { @@ -444,7 +447,7 @@ mod test { } if let Some(RoleResource::Druid(middlemanager_resources_from_rg)) = middle_managers - .get("resources-from-role") + .get(&RoleGroupName::from_str("resources-from-role").unwrap()) .map(|rg| &rg.config.resources) { let expected = Resources { @@ -480,7 +483,11 @@ mod test { let historicals = cluster.merged_role(&DruidRole::Historical).unwrap(); // ---------- default role group - let res = &historicals.get("default").unwrap().config.resources; + let res = &historicals + .get(&RoleGroupName::from_str("default").unwrap()) + .unwrap() + .config + .resources; let mut got = BTreeMap::new(); assert!(res.update_druid_config_file(&mut got).is_ok()); @@ -491,7 +498,11 @@ mod test { assert_eq!(value, &expected, "primary"); // ---------- secondary role group - let res = &historicals.get("secondary").unwrap().config.resources; + let res = &historicals + .get(&RoleGroupName::from_str("secondary").unwrap()) + .unwrap() + .config + .resources; let mut got = BTreeMap::new(); assert!(res.update_druid_config_file(&mut got).is_ok()); From 7ed679ab3c4f7036b0126e8305b73b3d9247cb29 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 12:54:05 +0200 Subject: [PATCH 37/48] refactor: rename CommonRoleGroupConfig -> ValidatedDruidConfig --- rust/operator-binary/src/controller.rs | 10 +++---- .../src/controller/validate.rs | 2 +- rust/operator-binary/src/crd/mod.rs | 27 +++++++++---------- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index f1c6fdd0..a7b62df0 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -63,9 +63,9 @@ use crate::{ service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, }, crd::{ - APP_NAME, CommonRoleGroupConfig, Container, DRUID_CONFIG_DIRECTORY, DeepStorageSpec, - DruidClusterStatus, DruidRole, HDFS_CONFIG_DIRECTORY, LOG_CONFIG_DIRECTORY, METRICS_PORT, - METRICS_PORT_NAME, OPERATOR_NAME, RW_CONFIG_DIRECTORY, STACKABLE_LOG_DIR, + APP_NAME, Container, DRUID_CONFIG_DIRECTORY, DeepStorageSpec, DruidClusterStatus, + DruidRole, HDFS_CONFIG_DIRECTORY, LOG_CONFIG_DIRECTORY, METRICS_PORT, METRICS_PORT_NAME, + OPERATOR_NAME, RW_CONFIG_DIRECTORY, STACKABLE_LOG_DIR, ValidatedDruidConfig, build_recommended_labels, security::DruidTlsSecurity, v1alpha1, }, internal_secret::create_shared_internal_secret, @@ -788,7 +788,7 @@ fn build_rolegroup_statefulset( .build(), spec: Some(StatefulSetSpec { pod_management_policy: Some("Parallel".to_string()), - replicas: merged_rolegroup_config.replicas.map(i32::from), + replicas: Some(i32::from(rg.replicas)), selector: LabelSelector { match_labels: Some( Labels::role_group_selector( @@ -861,7 +861,7 @@ fn add_config_volume_and_volume_mounts( fn add_log_config_volume_and_volume_mounts( rolegroup_ref: &RoleGroupRef, - merged_rolegroup_config: &CommonRoleGroupConfig, + merged_rolegroup_config: &ValidatedDruidConfig, cb_druid: &mut ContainerBuilder, pb: &mut PodBuilder, ) -> Result<()> { diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 7f4182d0..46ad5726 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -64,7 +64,7 @@ type Result = std::result::Result; /// /// This is the upstream [`stackable_operator::v2::role_utils::RoleGroupConfig`] (config plus the /// four merged override categories), with the typed per-role config erased to -/// [`CommonRoleGroupConfig`] so that all roles share a single type. The rendered per-file configs +/// [`ValidatedDruidConfig`] so that all roles share a single type. The rendered per-file configs /// (runtime.properties / security.properties / jvm.config) are produced later, in the config-map /// build step. /// diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index f926d210..21dd09b9 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -432,7 +432,7 @@ impl v1alpha1::DruidCluster { /// /// All four override categories (config / env / cli / pod) are merged by /// [`with_validated_config`] (role group wins over role); the typed per-role config is then - /// erased to the shared [`CommonRoleGroupConfig`] view consumed by the build step. + /// erased to the shared [`ValidatedDruidConfig`] view consumed by the build step. pub fn merged_role( &self, role: &DruidRole, @@ -440,7 +440,7 @@ impl v1alpha1::DruidCluster { let deep_storage = &self.spec.cluster_config.deep_storage; let name = self.name_any(); - // All roles erase to an identical `CommonRoleGroupConfig`; only the typed config, the role + // All roles erase to an identical `ValidatedDruidConfig`; only the typed config, the role // config type and the `RoleResource` variant (historicals carry typed storage) differ. macro_rules! merged_role { ($field:ident, $config:ty, $role_config:ty, $resource:path) => {{ @@ -458,10 +458,9 @@ impl v1alpha1::DruidCluster { .with_context(|_| FailedToMergeRoleGroupConfigSnafu { role_group: rg_name.clone(), })?; - let common = CommonRoleGroupConfig { + let common = ValidatedDruidConfig { resources: $resource(validated.config.config.resources), logging: validated.config.config.logging, - replicas: rg.replicas, affinity: validated.config.config.affinity, graceful_shutdown_timeout: validated .config @@ -582,30 +581,30 @@ pub enum Container { Vector, } -/// Common configuration for all role groups +/// The validated, merged Druid config shared by all role groups (the typed per-role config erased +/// to a single view). Mirrors the opensearch-operator's `ValidatedOpenSearchConfig`. #[derive(Clone)] -pub struct CommonRoleGroupConfig { - pub resources: RoleResource, - pub logging: Logging, - pub replicas: Option, +pub struct ValidatedDruidConfig { pub affinity: StackableAffinity, pub graceful_shutdown_timeout: Option, + pub logging: Logging, pub requested_secret_lifetime: Duration, + pub resources: RoleResource, } /// A validated, merged role-group config. /// /// This is the upstream [`stackable_operator::v2::role_utils::RoleGroupConfig`] (config plus the /// four merged override categories, role group winning over role), with the typed per-role config -/// erased to the shared [`CommonRoleGroupConfig`] view. The merged JVM argument overrides live in +/// erased to the shared [`ValidatedDruidConfig`] view. The merged JVM argument overrides live in /// `product_specific_common_config`; the rendered per-file configs (runtime.properties / /// security.properties / jvm.config) are produced later, in the config-map build step. /// -/// Note: the StatefulSet replicas come from [`CommonRoleGroupConfig::replicas`] (an `Option`, so -/// an unspecified count is left unset and stays HPA-friendly), not from -/// [`RoleGroupConfig::replicas`]. +/// The StatefulSet replicas come from [`RoleGroupConfig::replicas`] (defaulted to 1 during the +/// merge), mirroring the hive-operator. Upstream `replicas` should be made optional again so an +/// unspecified count can stay HPA-friendly. pub type DruidRoleGroupConfig = - RoleGroupConfig; + RoleGroupConfig; impl Default for v1alpha1::DruidRoleConfig { fn default() -> Self { From 81d3af9cb78544180ae36d599ce704a487c92bd6 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 13:31:28 +0200 Subject: [PATCH 38/48] feat: migrate product logging to ValidatedLogging + static vector.yaml & v2 vector_container --- rust/operator-binary/src/controller.rs | 97 ++-- .../controller/build/properties/logging.rs | 77 ---- .../src/controller/build/properties/mod.rs | 2 +- .../build/properties/product_logging/mod.rs | 70 +++ .../properties/product_logging/test-vector.sh | 11 + .../product_logging/vector-test.yaml | 162 +++++++ .../properties/product_logging/vector.yaml | 231 ++++++++++ .../controller/build/resource/config_map.rs | 10 +- rust/operator-binary/src/crd/mod.rs | 87 +++- tests/templates/kuttl/smoke/52-assert.yaml.j2 | 200 +++++++++ tests/templates/kuttl/smoke/53-assert.yaml.j2 | 424 ++---------------- 11 files changed, 828 insertions(+), 543 deletions(-) delete mode 100644 rust/operator-binary/src/controller/build/properties/logging.rs create mode 100644 rust/operator-binary/src/controller/build/properties/product_logging/mod.rs create mode 100755 rust/operator-binary/src/controller/build/properties/product_logging/test-vector.sh create mode 100644 rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml create mode 100644 rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index a7b62df0..a11bd832 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -1,7 +1,7 @@ //! Ensures that `Pod`s are configured and running for each [`DruidCluster`][v1alpha1] //! //! [v1alpha1]: v1alpha1::DruidCluster -use std::sync::Arc; +use std::{str::FromStr, sync::Arc}; use const_format::concatcp; use snafu::{ResultExt, Snafu}; @@ -35,20 +35,21 @@ use stackable_operator::{ }, kvp::{KeyValuePairError, LabelError, LabelValueError, Labels}, logging::controller::ReconcilerError, - product_logging::{ - self, - framework::LoggingError, - spec::{ - ConfigMapLogConfig, ContainerLogConfig, ContainerLogConfigChoice, - CustomContainerLogConfig, - }, - }, + product_logging, role_utils::RoleGroupRef, shared::time::Duration, status::condition::{ compute_conditions, operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, }, + v2::{ + builder::pod::container::EnvVarSet, + product_logging::framework::{ValidatedContainerLogConfigChoice, vector_container}, + types::{ + kubernetes::{ContainerName, VolumeName}, + operator::RoleGroupName, + }, + }, }; use strum::{EnumDiscriminants, IntoStaticStr}; @@ -77,10 +78,10 @@ mod dereference; mod validate; use build::{ - properties::logging::MAX_DRUID_LOG_FILES_SIZE, + properties::product_logging::MAX_DRUID_LOG_FILES_SIZE, resource::discovery::{self, build_discovery_configmaps}, }; -use validate::DruidRoleGroupConfig; +use validate::{DruidRoleGroupConfig, ValidatedCluster}; pub const DRUID_CONTROLLER_NAME: &str = "druidcluster"; pub const FULL_CONTROLLER_NAME: &str = concatcp!(DRUID_CONTROLLER_NAME, '.', OPERATOR_NAME); @@ -179,9 +180,6 @@ pub enum Error { source: crate::internal_secret::Error, }, - #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] - VectorAggregatorConfigMapMissing, - #[snafu(display("failed to create RBAC service account"))] ApplyServiceAccount { source: stackable_operator::cluster_resources::Error, @@ -225,9 +223,6 @@ pub enum Error { source: KeyValuePairError, }, - #[snafu(display("failed to build vector container"))] - BuildVectorContainer { source: LoggingError }, - #[snafu(display("failed to add needed volume"))] AddVolume { source: builder::pod::Error }, @@ -386,8 +381,10 @@ pub async fn reconcile_druid( .context(BuildConfigMapSnafu)?; let rg_statefulset = build_rolegroup_statefulset( druid, + &validated_cluster, &validated_cluster.image, druid_role, + rolegroup_name, &rolegroup, rg, validated_cluster.cluster_config.s3_connection.as_ref(), @@ -505,8 +502,10 @@ pub async fn reconcile_druid( /// corresponding [`stackable_operator::k8s_openapi::api::core::v1::Service`] (from [`build_rolegroup_headless_service`]). fn build_rolegroup_statefulset( druid: &v1alpha1::DruidCluster, + cluster: &ValidatedCluster, resolved_product_image: &ResolvedProductImage, role: &DruidRole, + role_group_name: &RoleGroupName, rolegroup_ref: &RoleGroupRef, rg: &DruidRoleGroupConfig, s3_conn: Option<&s3::v1alpha1::ConnectionSpec>, @@ -550,12 +549,8 @@ fn build_rolegroup_statefulset( let mut main_container_commands = role.main_container_prepare_commands(s3_conn); let mut prepare_container_commands = vec![]; - if let Some(ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic(log_config)), - }) = merged_rolegroup_config - .logging - .containers - .get(&Container::Prepare) + if let ValidatedContainerLogConfigChoice::Automatic(log_config) = + &merged_rolegroup_config.logging.prepare_container { // This command needs to be added at the beginning of the shell commands, // otherwise the output of the following commands will not be captured! @@ -737,33 +732,19 @@ fn build_rolegroup_statefulset( .service_account_name(service_account.name_any()) .security_context(PodSecurityContextBuilder::new().fs_group(1000).build()); - if merged_rolegroup_config.logging.enable_vector_agent { - match &druid.spec.cluster_config.vector_aggregator_config_map_name { - Some(vector_aggregator_config_map_name) => { - pb.add_container( - product_logging::framework::vector_container( - resolved_product_image, - DRUID_CONFIG_VOLUME_NAME, - LOG_VOLUME_NAME, - merged_rolegroup_config - .logging - .containers - .get(&Container::Vector), - ResourceRequirementsBuilder::new() - .with_cpu_request("250m") - .with_cpu_limit("500m") - .with_memory_request("128Mi") - .with_memory_limit("128Mi") - .build(), - vector_aggregator_config_map_name, - ) - .context(BuildVectorContainerSnafu)?, - ); - } - None => { - VectorAggregatorConfigMapMissingSnafu.fail()?; - } - } + // The Vector agent reads the static `vector.yaml` (added to the rolegroup ConfigMap) from the + // config volume; the validated aggregator address comes from the up-front `ValidatedLogging`. + if let Some(vector_log_config) = &merged_rolegroup_config.logging.vector_container { + pb.add_container(vector_container( + &ContainerName::from_str(&Container::Vector.to_string()) + .expect("'vector' is a valid container name"), + resolved_product_image, + vector_log_config, + &cluster.resource_names(role, role_group_name), + &VolumeName::from_str(DRUID_CONFIG_VOLUME_NAME).expect("a valid volume name"), + &VolumeName::from_str(LOG_VOLUME_NAME).expect("a valid volume name"), + EnvVarSet::new(), + )); } let mut pod_template = pb.build_template(); @@ -869,19 +850,9 @@ fn add_log_config_volume_and_volume_mounts( .add_volume_mount(LOG_CONFIG_VOLUME_NAME, LOG_CONFIG_DIRECTORY) .context(AddVolumeMountSnafu)?; - let config_map = if let Some(ContainerLogConfig { - choice: - Some(ContainerLogConfigChoice::Custom(CustomContainerLogConfig { - custom: ConfigMapLogConfig { config_map }, - })), - }) = merged_rolegroup_config - .logging - .containers - .get(&Container::Druid) - { - config_map.into() - } else { - rolegroup_ref.object_name() + let config_map = match &merged_rolegroup_config.logging.druid_container { + ValidatedContainerLogConfigChoice::Custom(config_map_name) => config_map_name.to_string(), + ValidatedContainerLogConfigChoice::Automatic(_) => rolegroup_ref.object_name(), }; pb.add_volume( diff --git a/rust/operator-binary/src/controller/build/properties/logging.rs b/rust/operator-binary/src/controller/build/properties/logging.rs deleted file mode 100644 index adbaac20..00000000 --- a/rust/operator-binary/src/controller/build/properties/logging.rs +++ /dev/null @@ -1,77 +0,0 @@ -//! Builds the log4j2 and Vector logging configuration for the rolegroup `ConfigMap`. - -use stackable_operator::{ - memory::{BinaryMultiple, MemoryQuantity}, - product_logging::{ - self, - spec::{ContainerLogConfig, ContainerLogConfigChoice, Logging}, - }, - role_utils::RoleGroupRef, -}; - -use crate::crd::{Container, STACKABLE_LOG_DIR, v1alpha1}; - -const CONSOLE_CONVERSION_PATTERN: &str = "%d{ISO8601} %p [%t] %c - %m%n"; - -/// File that the Druid log4j2 config writes its rolling log output to. -const DRUID_LOG_FILE: &str = "druid.log4j2.xml"; - -/// Maximum size of all Druid log files combined, used both for the log4j2 rollover configuration -/// and to size the log volume in the controller. -pub const MAX_DRUID_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { - value: 10.0, - unit: BinaryMultiple::Mebi, -}; - -/// Renders the `log4j2.properties` content for the Druid container. -/// -/// Returns `None` when the container uses a custom log ConfigMap rather than the operator's -/// automatic logging configuration. -pub fn build_log4j2_config(logging: &Logging) -> Option { - if let Some(ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic(log_config)), - }) = logging.containers.get(&Container::Druid) - { - Some(product_logging::framework::create_log4j2_config( - &format!( - "{STACKABLE_LOG_DIR}/{container}", - container = Container::Druid - ), - DRUID_LOG_FILE, - MAX_DRUID_LOG_FILES_SIZE - .scale_to(BinaryMultiple::Mebi) - .floor() - .value as u32, - CONSOLE_CONVERSION_PATTERN, - log_config, - )) - } else { - None - } -} - -/// Renders the Vector agent config (`vector.yaml`). -/// -/// Returns `None` when the Vector agent is disabled for this role group. -pub fn build_vector_config( - rolegroup: &RoleGroupRef, - logging: &Logging, -) -> Option { - if !logging.enable_vector_agent { - return None; - } - - let vector_log_config = if let Some(ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic(log_config)), - }) = logging.containers.get(&Container::Vector) - { - Some(log_config) - } else { - None - }; - - Some(product_logging::framework::create_vector_config( - rolegroup, - vector_log_config, - )) -} diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index 1f3cf715..0ab11128 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -1,6 +1,6 @@ //! Per-file builders for Druid `.properties` files. -pub mod logging; +pub mod product_logging; pub mod runtime_properties; pub mod security_properties; diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs b/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs new file mode 100644 index 00000000..6a098fd7 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs @@ -0,0 +1,70 @@ +//! Renders the logging config files (`log4j2.properties` and the Vector agent config) assembled +//! into the rolegroup `ConfigMap`. + +use stackable_operator::{ + memory::{BinaryMultiple, MemoryQuantity}, + product_logging::{self, spec::AutomaticContainerLogConfig}, + v2::product_logging::framework::{STACKABLE_LOG_DIR, ValidatedContainerLogConfigChoice}, +}; + +use crate::crd::Container; + +/// Maximum size of all Druid log files combined, used both for the log4j2 rollover configuration +/// and to size the log volume in the controller. +pub const MAX_DRUID_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { + value: 10.0, + unit: BinaryMultiple::Mebi, +}; + +const CONSOLE_CONVERSION_PATTERN: &str = "%d{ISO8601} %p [%t] %c - %m%n"; + +/// File that the Druid log4j2 config writes its rolling log output to. +const DRUID_LOG_FILE: &str = "druid.log4j2.xml"; + +/// The Vector agent configuration (`vector.yaml`). +const VECTOR_CONFIG: &str = include_str!("vector.yaml"); + +/// Returns the Vector agent config (`vector.yaml`) content, added to the rolegroup `ConfigMap` +/// only when the Vector agent is enabled. +pub fn vector_config_file_content() -> String { + VECTOR_CONFIG.to_owned() +} + +/// Renders the `log4j2.properties` content for the Druid container. +/// +/// Returns `None` when the container uses a custom log ConfigMap rather than the operator's +/// automatic logging configuration (in which case no `log4j2.properties` is added). +pub fn build_log4j2(druid_container: &ValidatedContainerLogConfigChoice) -> Option { + match druid_container { + ValidatedContainerLogConfigChoice::Automatic(log_config) => Some(log4j2_config(log_config)), + ValidatedContainerLogConfigChoice::Custom(_) => None, + } +} + +fn log4j2_config(log_config: &AutomaticContainerLogConfig) -> String { + product_logging::framework::create_log4j2_config( + &format!( + "{STACKABLE_LOG_DIR}/{container}", + container = Container::Druid + ), + DRUID_LOG_FILE, + MAX_DRUID_LOG_FILES_SIZE + .scale_to(BinaryMultiple::Mebi) + .floor() + .value as u32, + CONSOLE_CONVERSION_PATTERN, + log_config, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn vector_config_file_content_is_non_empty_and_keeps_log4j2_source() { + let content = vector_config_file_content(); + assert!(!content.is_empty()); + assert!(content.contains("files_log4j2")); + } +} diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/test-vector.sh b/rust/operator-binary/src/controller/build/properties/product_logging/test-vector.sh new file mode 100755 index 00000000..f77dcf62 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/product_logging/test-vector.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env sh + +DATA_DIR=/stackable/log/_vector-state \ +LOG_DIR=/stackable/log \ +NAMESPACE=default \ +CLUSTER_NAME=druid \ +ROLE_NAME=broker \ +ROLE_GROUP_NAME=default \ +VECTOR_AGGREGATOR_ADDRESS=vector-aggregator \ +VECTOR_FILE_LOG_LEVEL=info \ +vector test vector.yaml vector-test.yaml diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml new file mode 100644 index 00000000..862ad3fa --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml @@ -0,0 +1,162 @@ +# Run tests with `./test-vector.sh` +# +# A downside of these test cases is that they compare the whole event and that the message can +# contain source code positions in vector.yaml, e.g. "function call error for \"parse_xml\" at +# (584:643)". Please adapt the tests if you change VRL code in vector.yaml. +--- +tests: + - name: Test stdout log entry + inputs: + - type: log + insert_at: processed_files_stdout + log_fields: + file: /stackable/log/druid/druid.stdout.log + message: Starting Apache Druid + pod: druid-broker-default-0 + source_type: file + timestamp: 2025-10-02T09:27:28.582Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "druid", + "container": "druid", + "file": "druid.stdout.log", + "level": "INFO", + "logger": "ROOT", + "message": "Starting Apache Druid", + "namespace": "default", + "pod": "druid-broker-default-0", + "role": "broker", + "roleGroup": "default", + "timestamp": "2025-10-02T09:27:28.582Z" + } + + assert_eq!(expected_log_event, .) + - name: Test stderr log entry + inputs: + - type: log + insert_at: processed_files_stderr + log_fields: + file: /stackable/log/druid/druid.stderr.log + message: "Exception in thread \"main\"" + pod: druid-broker-default-0 + source_type: file + timestamp: 2025-10-02T09:27:28.582Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "druid", + "container": "druid", + "file": "druid.stderr.log", + "level": "ERROR", + "logger": "ROOT", + "message": "Exception in thread \"main\"", + "namespace": "default", + "pod": "druid-broker-default-0", + "role": "broker", + "roleGroup": "default", + "timestamp": "2025-10-02T09:27:28.582Z" + } + + assert_eq!(expected_log_event, .) + - name: Test log4j2 XML log entry without stacktrace + inputs: + - type: log + insert_at: processed_files_log4j2 + log_fields: + file: /stackable/log/druid/druid.log4j2.xml + message: > + Starting Druid broker on port + [8082]... + pod: druid-broker-default-0 + source_type: file + timestamp: 2025-10-02T09:27:29.473487331Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "druid", + "container": "druid", + "file": "druid.log4j2.xml", + "level": "INFO", + "logger": "org.apache.druid.cli.CliBroker", + "message": "Starting Druid broker on port [8082]...", + "namespace": "default", + "pod": "druid-broker-default-0", + "role": "broker", + "roleGroup": "default", + "timestamp": t'2025-10-02T09:27:28.582Z' + } + + assert_eq!(expected_log_event, .) + - name: Test Vector internal logs + inputs: + - type: log + insert_at: filtered_logs_vector + log_fields: + arch: x86_64 + message: Vector has started. + metadata: + kind: event + level: INFO + module_path: vector::internal_events::process + target: vector + pid: 14 + pod: druid-broker-default-0 + source_type: internal_logs + timestamp: 2025-10-02T09:46:14.479381097Z + version: 0.49.0 + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "arch": "x86_64", + "cluster": "druid", + "container": "vector", + "level": "INFO", + "logger": "vector::internal_events::process", + "message": "Vector has started.", + "namespace": "default", + "pod": "druid-broker-default-0", + "role": "broker", + "roleGroup": "default", + "timestamp": "2025-10-02T09:46:14.479381097Z", + "version": "0.49.0" + } + + assert_eq!(expected_log_event, .) + - name: Test Vector internal log level filtering - INFO passes + inputs: + - type: log + insert_at: filtered_logs_vector + log_fields: + metadata: + level: INFO + outputs: + - extract_from: filtered_logs_vector + conditions: + - type: vrl + source: | + assert_eq!("INFO", .metadata.level) + - name: Test Vector internal log level filtering - DEBUG dropped + inputs: + - type: log + insert_at: filtered_logs_vector + log_fields: + metadata: + level: DEBUG + no_outputs_from: + - filtered_logs_vector diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml new file mode 100644 index 00000000..e4d40b6e --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml @@ -0,0 +1,231 @@ +--- +data_dir: ${DATA_DIR} + +log_schema: + host_key: pod + +sources: + # Reads the internal Vector logs + vector: + type: internal_logs + + files_stdout: + type: file + include: + - ${LOG_DIR}/*/*.stdout.log + + files_stderr: + type: file + include: + - ${LOG_DIR}/*/*.stderr.log + + files_log4j2: + type: file + include: + - ${LOG_DIR}/*/*.log4j2.xml + line_delimiter: "\r\n" + +transforms: + processed_files_stdout: + inputs: + - files_stdout + type: remap + source: | + .logger = "ROOT" + .level = "INFO" + + processed_files_stderr: + inputs: + - files_stderr + type: remap + source: | + .logger = "ROOT" + .level = "ERROR" + + processed_files_log4j2: + inputs: + - files_log4j2 + type: remap + source: | + raw_message = string!(.message) + + .timestamp = now() + .logger = "" + .level = "INFO" + .message = "" + .errors = [] + + event = {} + parsed_event, err = parse_xml(raw_message) + if err != null { + error = "XML not parsable: " + err + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + if !is_object(parsed_event.Event) { + error = "Parsed event contains no \"Event\" tag." + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + event = object!(parsed_event.Event) + + tag_instant_valid = false + instant, err = object(event.Instant) + if err == null { + epoch_nanoseconds, err = to_int(instant.@epochSecond) * 1_000_000_000 + to_int(instant.@nanoOfSecond) + if err == null && epoch_nanoseconds != 0 { + converted_timestamp, err = from_unix_timestamp(epoch_nanoseconds, "nanoseconds") + if err == null { + .timestamp = converted_timestamp + tag_instant_valid = true + } else { + .errors = push(.errors, "Instant invalid, trying property timeMillis instead: " + err) + } + } else { + .errors = push(.errors, "Instant invalid, trying property timeMillis instead: " + err) + } + } + if !tag_instant_valid { + epoch_milliseconds, err = to_int(event.@timeMillis) + if err == null && epoch_milliseconds != 0 { + converted_timestamp, err = from_unix_timestamp(epoch_milliseconds, "milliseconds") + if err == null { + .timestamp = converted_timestamp + } else { + .errors = push(.errors, "timeMillis not parsable, using current time instead: " + err) + } + } else { + .errors = push(.errors, "timeMillis not parsable, using current time instead: " + err) + } + } + + .logger, err = string(event.@loggerName) + if err != null || is_empty(.logger) { + .errors = push(.errors, "Logger not found.") + } + + level, err = string(event.@level) + if err != null { + .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") + } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], level) { + .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") + } else { + .level = level + } + + exception = null + thrown = event.Thrown + if is_object(thrown) { + exception = "Exception" + thread, err = string(event.@thread) + if err == null && !is_empty(thread) { + exception = exception + " in thread \"" + thread + "\"" + } + thrown_name, err = string(thrown.@name) + if err == null && !is_empty(exception) { + exception = exception + " " + thrown_name + } + message = string(thrown.@localizedMessage) ?? + string(thrown.@message) ?? + "" + if !is_empty(message) { + exception = exception + ": " + message + } + stacktrace_items = array(thrown.ExtendedStackTrace.ExtendedStackTraceItem) ?? [] + stacktrace = "" + for_each(stacktrace_items) -> |_index, value| { + stacktrace = stacktrace + " " + class = string(value.@class) ?? "" + method = string(value.@method) ?? "" + if !is_empty(class) && !is_empty(method) { + stacktrace = stacktrace + "at " + class + "." + method + } + file = string(value.@file) ?? "" + line = string(value.@line) ?? "" + if !is_empty(file) && !is_empty(line) { + stacktrace = stacktrace + "(" + file + ":" + line + ")" + } + exact = to_bool(value.@exact) ?? false + location = string(value.@location) ?? "" + version = string(value.@version) ?? "" + if !is_empty(location) && !is_empty(version) { + stacktrace = stacktrace + " " + if !exact { + stacktrace = stacktrace + "~" + } + stacktrace = stacktrace + "[" + location + ":" + version + "]" + } + stacktrace = stacktrace + "\n" + } + if stacktrace != "" { + exception = exception + "\n" + stacktrace + } + } + + message, err = string(event.Message) + if err != null || is_empty(message) { + message = null + .errors = push(.errors, "Message not found.") + } + .message = join!(compact([message, exception]), "\n") + } + } + + # Extends the processed files with the fields "container" and "file" + extended_logs_files: + inputs: + - processed_files_* + type: remap + source: | + del(.source_type) + if .errors == [] { + del(.errors) + } + . |= parse_regex!(.file, r'^${LOG_DIR}/(?P.*?)/(?P.*?)$') + + # Filters the logs of the Vector agent according to the defined log level + filtered_logs_vector: + inputs: + - vector + type: filter + condition: > + (.metadata.level == "TRACE" && "${VECTOR_FILE_LOG_LEVEL}" == "trace") || + (.metadata.level == "DEBUG" && includes(["trace", "debug"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "INFO" && includes(["trace", "debug", "info"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "WARN" && includes(["trace", "debug", "info", "warn"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "ERROR" && includes(["trace", "debug", "info", "warn", "error"], "${VECTOR_FILE_LOG_LEVEL}")) + + # Aligns the logs of the Vector agent with the common format + extended_logs_vector: + inputs: + - filtered_logs_vector + type: remap + source: | + .container = "vector" + .level = .metadata.level + .logger = .metadata.module_path + if exists(.file) { .processed_file = del(.file) } + del(.metadata) + del(.pid) + del(.source_type) + + # Add the fields "namespace", "cluster", "role" and "roleGroup" to all logs + extended_logs: + inputs: + - extended_logs_* + type: remap + source: | + .namespace = "${NAMESPACE}" + .cluster = "${CLUSTER_NAME}" + .role = "${ROLE_NAME}" + .roleGroup = "${ROLE_GROUP_NAME}" + +sinks: + # Forward the logs to the Vector aggregator + aggregator: + inputs: + - extended_logs + type: vector + address: ${VECTOR_AGGREGATOR_ADDRESS} diff --git a/rust/operator-binary/src/controller/build/resource/config_map.rs b/rust/operator-binary/src/controller/build/resource/config_map.rs index 88cacbd9..a12e78dc 100644 --- a/rust/operator-binary/src/controller/build/resource/config_map.rs +++ b/rust/operator-binary/src/controller/build/resource/config_map.rs @@ -32,7 +32,7 @@ use crate::{ jvm::construct_jvm_args, properties::{ ConfigFileName, - logging::{build_log4j2_config, build_vector_config}, + product_logging::{build_log4j2, vector_config_file_content}, runtime_properties, security_properties, }, }, @@ -343,12 +343,14 @@ pub fn build_rolegroup_config_map( config_map_builder.add_data(filename, file_content); } - if let Some(log4j2_config) = build_log4j2_config(&rg.config.logging) { + if let Some(log4j2_config) = build_log4j2(&rg.config.logging.druid_container) { config_map_builder.add_data(ConfigFileName::Log4j2Properties.to_string(), log4j2_config); } - if let Some(vector_config) = build_vector_config(rolegroup, &rg.config.logging) { - config_map_builder.add_data(VECTOR_CONFIG_FILE, vector_config); + // The Vector agent config (`vector.yaml`) is a static, env-var-parameterized file (mirroring + // the hive-/opensearch-operator). It is only added when the Vector agent is enabled. + if rg.config.logging.enable_vector_agent { + config_map_builder.add_data(VECTOR_CONFIG_FILE, vector_config_file_content()); } config_map_builder diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 21dd09b9..0308d39b 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -40,8 +40,12 @@ use stackable_operator::{ v2::{ builder::pod::container::{EnvVarName, EnvVarSet}, config_overrides::KeyValueConfigOverrides, + product_logging::framework::{ + ValidatedContainerLogConfigChoice, VectorContainerLogConfig, + validate_logging_configuration_for_container, + }, role_utils::{JavaCommonConfig, RoleGroupConfig, with_validated_config}, - types::operator::RoleGroupName, + types::{kubernetes::ConfigMapName, operator::RoleGroupName}, }, versioned::versioned, }; @@ -182,6 +186,21 @@ pub enum Error { source: stackable_operator::v2::macros::attributed_string_type::Error, role_group: String, }, + + #[snafu(display("failed to validate container log configuration"))] + ValidateLoggingConfig { + source: stackable_operator::v2::product_logging::framework::Error, + }, + + #[snafu(display( + "the Vector agent is enabled but the Vector aggregator ConfigMap name is missing" + ))] + MissingVectorAggregatorConfigMapName, + + #[snafu(display("invalid Vector aggregator ConfigMap name"))] + ParseVectorAggregatorConfigMapName { + source: stackable_operator::v2::macros::attributed_string_type::Error, + }, } #[versioned( @@ -440,6 +459,17 @@ impl v1alpha1::DruidCluster { let deep_storage = &self.spec.cluster_config.deep_storage; let name = self.name_any(); + // The Vector aggregator discovery ConfigMap name (validated here so an invalid name fails + // up-front). It is only required when the Vector agent is enabled for a role group. + let vector_aggregator_config_map_name = self + .spec + .cluster_config + .vector_aggregator_config_map_name + .as_deref() + .map(ConfigMapName::from_str) + .transpose() + .context(ParseVectorAggregatorConfigMapNameSnafu)?; + // All roles erase to an identical `ValidatedDruidConfig`; only the typed config, the role // config type and the `RoleResource` variant (historicals carry typed storage) differ. macro_rules! merged_role { @@ -460,7 +490,10 @@ impl v1alpha1::DruidCluster { })?; let common = ValidatedDruidConfig { resources: $resource(validated.config.config.resources), - logging: validated.config.config.logging, + logging: validate_logging( + &validated.config.config.logging, + &vector_aggregator_config_map_name, + )?, affinity: validated.config.config.affinity, graceful_shutdown_timeout: validated .config @@ -587,11 +620,59 @@ pub enum Container { pub struct ValidatedDruidConfig { pub affinity: StackableAffinity, pub graceful_shutdown_timeout: Option, - pub logging: Logging, + pub logging: ValidatedLogging, pub requested_secret_lifetime: Duration, pub resources: RoleResource, } +/// Validated logging configuration for the Druid, Prepare (init) and (optional) Vector containers. +/// +/// Produced up-front by [`validate_logging`] (mirroring the hive-/opensearch-operator) so that an +/// invalid custom log ConfigMap name or a missing Vector aggregator discovery ConfigMap name fails +/// reconciliation during validation rather than at resource-build time. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ValidatedLogging { + pub druid_container: ValidatedContainerLogConfigChoice, + pub prepare_container: ValidatedContainerLogConfigChoice, + pub vector_container: Option, + pub enable_vector_agent: bool, +} + +/// Validates the logging configuration for the Druid, Prepare and (optional) Vector containers. +/// +/// `vector_aggregator_config_map_name` is the discovery ConfigMap name of the Vector aggregator; +/// it is required (and was validated up-front) only when the Vector agent is enabled. +fn validate_logging( + logging: &Logging, + vector_aggregator_config_map_name: &Option, +) -> Result { + let druid_container = validate_logging_configuration_for_container(logging, &Container::Druid) + .context(ValidateLoggingConfigSnafu)?; + let prepare_container = + validate_logging_configuration_for_container(logging, &Container::Prepare) + .context(ValidateLoggingConfigSnafu)?; + + let vector_container = if logging.enable_vector_agent { + let vector_aggregator_config_map_name = vector_aggregator_config_map_name + .clone() + .context(MissingVectorAggregatorConfigMapNameSnafu)?; + Some(VectorContainerLogConfig { + log_config: validate_logging_configuration_for_container(logging, &Container::Vector) + .context(ValidateLoggingConfigSnafu)?, + vector_aggregator_config_map_name, + }) + } else { + None + }; + + Ok(ValidatedLogging { + druid_container, + prepare_container, + vector_container, + enable_vector_agent: logging.enable_vector_agent, + }) +} + /// A validated, merged role-group config. /// /// This is the upstream [`stackable_operator::v2::role_utils::RoleGroupConfig`] (config plus the diff --git a/tests/templates/kuttl/smoke/52-assert.yaml.j2 b/tests/templates/kuttl/smoke/52-assert.yaml.j2 index 52e24ac3..14ac2aa0 100644 --- a/tests/templates/kuttl/smoke/52-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/52-assert.yaml.j2 @@ -43,6 +43,46 @@ spec: spec: serviceAccount: druid-serviceaccount serviceAccountName: druid-serviceaccount +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + # The Vector agent sidecar (added by the v2 `vector_container`). kuttl matches list + # elements positionally, so the first (druid) container is named only to align indices. + containers: + - name: druid + - name: vector + env: + - name: CLUSTER_NAME + value: druid + - name: DATA_DIR + value: /stackable/log/_vector-state + - name: LOG_DIR + value: /stackable/log + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ROLE_GROUP_NAME + value: default + - name: ROLE_NAME + value: broker + - name: VECTOR_AGGREGATOR_ADDRESS + valueFrom: + configMapKeyRef: + key: ADDRESS + name: vector-aggregator-discovery + - name: VECTOR_CONFIG_YAML + value: /stackable/config/vector.yaml + - name: VECTOR_FILE_LOG_LEVEL + value: info + - name: VECTOR_LOG + value: info + volumeMounts: + - mountPath: /stackable/config/vector.yaml + name: config + readOnly: true + subPath: vector.yaml + - mountPath: /stackable/log + name: log +{% endif %} terminationGracePeriodSeconds: 1 status: readyReplicas: 1 @@ -78,6 +118,46 @@ spec: spec: serviceAccount: druid-serviceaccount serviceAccountName: druid-serviceaccount +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + # The Vector agent sidecar (added by the v2 `vector_container`). kuttl matches list + # elements positionally, so the first (druid) container is named only to align indices. + containers: + - name: druid + - name: vector + env: + - name: CLUSTER_NAME + value: druid + - name: DATA_DIR + value: /stackable/log/_vector-state + - name: LOG_DIR + value: /stackable/log + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ROLE_GROUP_NAME + value: default + - name: ROLE_NAME + value: coordinator + - name: VECTOR_AGGREGATOR_ADDRESS + valueFrom: + configMapKeyRef: + key: ADDRESS + name: vector-aggregator-discovery + - name: VECTOR_CONFIG_YAML + value: /stackable/config/vector.yaml + - name: VECTOR_FILE_LOG_LEVEL + value: info + - name: VECTOR_LOG + value: info + volumeMounts: + - mountPath: /stackable/config/vector.yaml + name: config + readOnly: true + subPath: vector.yaml + - mountPath: /stackable/log + name: log +{% endif %} terminationGracePeriodSeconds: 1 status: readyReplicas: 1 @@ -113,6 +193,46 @@ spec: spec: serviceAccount: druid-serviceaccount serviceAccountName: druid-serviceaccount +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + # The Vector agent sidecar (added by the v2 `vector_container`). kuttl matches list + # elements positionally, so the first (druid) container is named only to align indices. + containers: + - name: druid + - name: vector + env: + - name: CLUSTER_NAME + value: druid + - name: DATA_DIR + value: /stackable/log/_vector-state + - name: LOG_DIR + value: /stackable/log + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ROLE_GROUP_NAME + value: default + - name: ROLE_NAME + value: historical + - name: VECTOR_AGGREGATOR_ADDRESS + valueFrom: + configMapKeyRef: + key: ADDRESS + name: vector-aggregator-discovery + - name: VECTOR_CONFIG_YAML + value: /stackable/config/vector.yaml + - name: VECTOR_FILE_LOG_LEVEL + value: info + - name: VECTOR_LOG + value: info + volumeMounts: + - mountPath: /stackable/config/vector.yaml + name: config + readOnly: true + subPath: vector.yaml + - mountPath: /stackable/log + name: log +{% endif %} terminationGracePeriodSeconds: 1 volumes: - name: tls-mount @@ -176,6 +296,46 @@ spec: spec: serviceAccount: druid-serviceaccount serviceAccountName: druid-serviceaccount +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + # The Vector agent sidecar (added by the v2 `vector_container`). kuttl matches list + # elements positionally, so the first (druid) container is named only to align indices. + containers: + - name: druid + - name: vector + env: + - name: CLUSTER_NAME + value: druid + - name: DATA_DIR + value: /stackable/log/_vector-state + - name: LOG_DIR + value: /stackable/log + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ROLE_GROUP_NAME + value: default + - name: ROLE_NAME + value: middlemanager + - name: VECTOR_AGGREGATOR_ADDRESS + valueFrom: + configMapKeyRef: + key: ADDRESS + name: vector-aggregator-discovery + - name: VECTOR_CONFIG_YAML + value: /stackable/config/vector.yaml + - name: VECTOR_FILE_LOG_LEVEL + value: info + - name: VECTOR_LOG + value: info + volumeMounts: + - mountPath: /stackable/config/vector.yaml + name: config + readOnly: true + subPath: vector.yaml + - mountPath: /stackable/log + name: log +{% endif %} terminationGracePeriodSeconds: 1 status: readyReplicas: 1 @@ -211,6 +371,46 @@ spec: spec: serviceAccount: druid-serviceaccount serviceAccountName: druid-serviceaccount +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + # The Vector agent sidecar (added by the v2 `vector_container`). kuttl matches list + # elements positionally, so the first (druid) container is named only to align indices. + containers: + - name: druid + - name: vector + env: + - name: CLUSTER_NAME + value: druid + - name: DATA_DIR + value: /stackable/log/_vector-state + - name: LOG_DIR + value: /stackable/log + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ROLE_GROUP_NAME + value: default + - name: ROLE_NAME + value: router + - name: VECTOR_AGGREGATOR_ADDRESS + valueFrom: + configMapKeyRef: + key: ADDRESS + name: vector-aggregator-discovery + - name: VECTOR_CONFIG_YAML + value: /stackable/config/vector.yaml + - name: VECTOR_FILE_LOG_LEVEL + value: info + - name: VECTOR_LOG + value: info + volumeMounts: + - mountPath: /stackable/config/vector.yaml + name: config + readOnly: true + subPath: vector.yaml + - mountPath: /stackable/log + name: log +{% endif %} terminationGracePeriodSeconds: 1 status: readyReplicas: 1 diff --git a/tests/templates/kuttl/smoke/53-assert.yaml.j2 b/tests/templates/kuttl/smoke/53-assert.yaml.j2 index 53c7b9f8..55a89f96 100644 --- a/tests/templates/kuttl/smoke/53-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/53-assert.yaml.j2 @@ -1,210 +1,33 @@ {% set vector_enabled = lookup('env', 'VECTOR_AGGREGATOR') | length > 0 %} {% set vector_yaml -%} -data_dir: /stackable/log/_vector-state +--- +data_dir: ${DATA_DIR} log_schema: host_key: pod sources: + # Reads the internal Vector logs vector: type: internal_logs files_stdout: type: file include: - - /stackable/log/*/*.stdout.log + - ${LOG_DIR}/*/*.stdout.log files_stderr: type: file include: - - /stackable/log/*/*.stderr.log - - files_log4j: - type: file - include: - - /stackable/log/*/*.log4j.xml - line_delimiter: "\r\n" - multiline: - mode: halt_before - start_pattern: ^" + raw_message + "" - parsed_event, err = parse_xml(wrapped_xml_event) - if err != null { - error = "XML not parsable: " + err - .errors = push(.errors, error) - log(error, level: "warn") - .message = raw_message - } else { - root = object!(parsed_event.root) - if !is_object(root.event) { - error = "Parsed event contains no \"event\" tag." - .errors = push(.errors, error) - log(error, level: "warn") - .message = raw_message - } else { - if keys(root) != ["event"] { - .errors = push(.errors, "Parsed event contains multiple tags: " + join!(keys(root), ", ")) - } - event = object!(root.event) - - epoch_milliseconds, err = to_int(event.@timestamp) - if err == null && epoch_milliseconds != 0 { - converted_timestamp, err = from_unix_timestamp(epoch_milliseconds, "milliseconds") - if err == null { - .timestamp = converted_timestamp - } else { - .errors = push(.errors, "Time not parsable, using current time instead: " + err) - } - } else { - .errors = push(.errors, "Timestamp not found, using current time instead.") - } - - .logger, err = string(event.@logger) - if err != null || is_empty(.logger) { - .errors = push(.errors, "Logger not found.") - } - - level, err = string(event.@level) - if err != null { - .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") - } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], level) { - .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") - } else { - .level = level - } - - message, err = string(event.message) - if err != null || is_empty(message) { - .errors = push(.errors, "Message not found.") - } - throwable = string(event.throwable) ?? "" - .message = join!(compact([message, throwable]), "\n") - } - } - processed_files_log4j2: inputs: - files_log4j2 @@ -421,136 +175,7 @@ transforms: } } - processed_files_py: - inputs: - - files_py - type: remap - source: | - raw_message = string!(.message) - - .timestamp = now() - .logger = "" - .level = "INFO" - .message = "" - .errors = [] - - parsed_event, err = parse_json(raw_message) - if err != null { - error = "JSON not parsable: " + err - .errors = push(.errors, error) - log(error, level: "warn") - .message = raw_message - } else if !is_object(parsed_event) { - error = "Parsed event is not a JSON object." - .errors = push(.errors, error) - log(error, level: "warn") - .message = raw_message - } else { - event = object!(parsed_event) - - asctime, err = string(event.asctime) - if err == null { - parsed_timestamp, err = parse_timestamp(asctime, "%F %T,%3f") - if err == null { - .timestamp = parsed_timestamp - } else { - .errors = push(.errors, "Timestamp not parsable, using current time instead: "+ err) - } - } else { - .errors = push(.errors, "Timestamp not found, using current time instead.") - } - - .logger, err = string(event.name) - if err != null || is_empty(.logger) { - .errors = push(.errors, "Logger not found.") - } - - level, err = string(event.levelname) - if err != null { - .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") - } else if level == "DEBUG" { - .level = "DEBUG" - } else if level == "INFO" { - .level = "INFO" - } else if level == "WARNING" { - .level = "WARN" - } else if level == "ERROR" { - .level = "ERROR" - } else if level == "CRITICAL" { - .level = "FATAL" - } else { - .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") - } - - .message, err = string(event.message) - if err != null || is_empty(.message) { - .errors = push(.errors, "Message not found.") - } - } - - processed_files_airlift: - inputs: - - files_airlift - type: remap - source: | - raw_message = string!(.message) - - .timestamp = now() - .logger = "" - .level = "INFO" - .message = "" - .errors = [] - - parsed_event, err = parse_json(raw_message) - if err != null { - error = "JSON not parsable: " + err - .errors = push(.errors, error) - log(error, level: "warn") - .message = raw_message - } else if !is_object(parsed_event) { - error = "Parsed event is not a JSON object." - .errors = push(.errors, error) - log(error, level: "warn") - .message = raw_message - } else { - event = object!(parsed_event) - - timestamp_string, err = string(event.timestamp) - if err == null { - parsed_timestamp, err = parse_timestamp(timestamp_string, "%Y-%m-%dT%H:%M:%S.%fZ") - if err == null { - .timestamp = parsed_timestamp - } else { - .errors = push(.errors, "Timestamp not parsable, using current time instead: " + err) - } - } else { - .errors = push(.errors, "Timestamp not found, using current time instead.") - } - - .logger, err = string(event.logger) - if err != null || is_empty(.logger) { - .errors = push(.errors, "Logger not found.") - } - - level, err = string(event.level) - if err != null { - .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") - } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], level) { - .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") - } else { - .level = level - } - - .thread = string(parsed_event.thread) ?? null - - .message, err = string(event.message) - if err != null || is_empty(.message) { - .errors = push(.errors, "Message not found.") - } - stacktrace = string(event.stackTrace) ?? "" - .message = join!(compact([.message, stacktrace]), "\n\n") - } - + # Extends the processed files with the fields "container" and "file" extended_logs_files: inputs: - processed_files_* @@ -560,14 +185,21 @@ transforms: if .errors == [] { del(.errors) } - . |= parse_regex!(.file, r'^/stackable/log/(?P.*?)/(?P.*?)$') + . |= parse_regex!(.file, r'^${LOG_DIR}/(?P.*?)/(?P.*?)$') + # Filters the logs of the Vector agent according to the defined log level filtered_logs_vector: inputs: - vector type: filter - condition: '!includes(["TRACE", "DEBUG"], .metadata.level)' - + condition: > + (.metadata.level == "TRACE" && "${VECTOR_FILE_LOG_LEVEL}" == "trace") || + (.metadata.level == "DEBUG" && includes(["trace", "debug"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "INFO" && includes(["trace", "debug", "info"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "WARN" && includes(["trace", "debug", "info", "warn"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "ERROR" && includes(["trace", "debug", "info", "warn", "error"], "${VECTOR_FILE_LOG_LEVEL}")) + + # Aligns the logs of the Vector agent with the common format extended_logs_vector: inputs: - filtered_logs_vector @@ -581,22 +213,24 @@ transforms: del(.pid) del(.source_type) + # Add the fields "namespace", "cluster", "role" and "roleGroup" to all logs extended_logs: inputs: - extended_logs_* type: remap source: | - .namespace = "__NAMESPACE__" - .cluster = "druid" - .role = "__ROLE__" - .roleGroup = "default" + .namespace = "${NAMESPACE}" + .cluster = "${CLUSTER_NAME}" + .role = "${ROLE_NAME}" + .roleGroup = "${ROLE_GROUP_NAME}" sinks: + # Forward the logs to the Vector aggregator aggregator: inputs: - extended_logs type: vector - address: $VECTOR_AGGREGATOR_ADDRESS + address: ${VECTOR_AGGREGATOR_ADDRESS} {% endset %} --- # Snapshot the full `.data` of each operator-managed ConfigMap. @@ -728,7 +362,7 @@ commands: networkaddress.cache.ttl=30 {% if vector_enabled %} vector.yaml: | -{{ vector_yaml | replace("__ROLE__", "broker") | indent(8, true) }} +{{ vector_yaml | indent(8, true) }} {% endif %} YAMLEOF ) @@ -836,7 +470,7 @@ commands: networkaddress.cache.ttl=30 {% if vector_enabled %} vector.yaml: | -{{ vector_yaml | replace("__ROLE__", "coordinator") | indent(8, true) }} +{{ vector_yaml | indent(8, true) }} {% endif %} YAMLEOF ) @@ -944,7 +578,7 @@ commands: networkaddress.cache.ttl=30 {% if vector_enabled %} vector.yaml: | -{{ vector_yaml | replace("__ROLE__", "historical") | indent(8, true) }} +{{ vector_yaml | indent(8, true) }} {% endif %} YAMLEOF ) @@ -1049,7 +683,7 @@ commands: networkaddress.cache.ttl=30 {% if vector_enabled %} vector.yaml: | -{{ vector_yaml | replace("__ROLE__", "middlemanager") | indent(8, true) }} +{{ vector_yaml | indent(8, true) }} {% endif %} YAMLEOF ) @@ -1152,7 +786,7 @@ commands: networkaddress.cache.ttl=30 {% if vector_enabled %} vector.yaml: | -{{ vector_yaml | replace("__ROLE__", "router") | indent(8, true) }} +{{ vector_yaml | indent(8, true) }} {% endif %} YAMLEOF ) From 86807e671428ba85fcd6282cfa4c9a88092e46eb Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 13:52:24 +0200 Subject: [PATCH 39/48] refactor: remove RoleGroupRef; build steps use ValidatedCluster + ResourceNames --- rust/operator-binary/src/controller.rs | 102 ++++++++---------- .../controller/build/resource/config_map.rs | 21 ++-- rust/operator-binary/src/crd/resource.rs | 11 +- 3 files changed, 59 insertions(+), 75 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index a11bd832..f31d16cc 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -31,12 +31,11 @@ use stackable_operator::{ kube::{ Resource, ResourceExt, core::{DeserializeGuard, error_boundary}, - runtime::{controller::Action, reflector::ObjectRef}, + runtime::controller::Action, }, kvp::{KeyValuePairError, LabelError, LabelValueError, Labels}, logging::controller::ReconcilerError, product_logging, - role_utils::RoleGroupRef, shared::time::Duration, status::condition::{ compute_conditions, operations::ClusterOperationsConditionBuilder, @@ -45,6 +44,7 @@ use stackable_operator::{ v2::{ builder::pod::container::EnvVarSet, product_logging::framework::{ValidatedContainerLogConfigChoice, vector_container}, + role_group_utils::ResourceNames, types::{ kubernetes::{ContainerName, VolumeName}, operator::RoleGroupName, @@ -104,22 +104,22 @@ pub struct Ctx { #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] pub enum Error { - #[snafu(display("failed to apply Service for {}", rolegroup))] + #[snafu(display("failed to apply Service for role group {role_group}"))] ApplyRoleGroupService { source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, + role_group: String, }, - #[snafu(display("failed to apply ConfigMap for {}", rolegroup))] + #[snafu(display("failed to apply ConfigMap for role group {role_group}"))] ApplyRoleGroupConfig { source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, + role_group: String, }, - #[snafu(display("failed to apply StatefulSet for {}", rolegroup))] + #[snafu(display("failed to apply StatefulSet for role group {role_group}"))] ApplyRoleGroupStatefulSet { source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, + role_group: String, }, #[snafu(display("object is missing metadata to build owner reference"))] @@ -332,27 +332,17 @@ pub async fn reconcile_druid( .context(FailedInternalSecretCreationSnafu)?; for (rolegroup_name, rg) in groups.iter() { - let rolegroup = RoleGroupRef { - cluster: ObjectRef::from_obj(druid), - role: role_name.clone(), - role_group: rolegroup_name.to_string(), - }; - let role_group_service_recommended_labels = build_recommended_labels( druid, DRUID_CONTROLLER_NAME, &validated_cluster.image.app_version_label_value, - &rolegroup.role, - &rolegroup.role_group, + &role_name, + rolegroup_name.as_ref(), ); - let role_group_service_selector = Labels::role_group_selector( - druid, - APP_NAME, - &rolegroup.role, - &rolegroup.role_group, - ) - .context(LabelBuildSnafu)?; + let role_group_service_selector = + Labels::role_group_selector(druid, APP_NAME, &role_name, rolegroup_name.as_ref()) + .context(LabelBuildSnafu)?; let rg_headless_service = build_rolegroup_headless_service( &validated_cluster, @@ -375,7 +365,7 @@ pub async fn reconcile_druid( let rg_configmap = build::resource::config_map::build_rolegroup_config_map( &validated_cluster, druid_role, - &rolegroup, + rolegroup_name, rg, ) .context(BuildConfigMapSnafu)?; @@ -385,7 +375,6 @@ pub async fn reconcile_druid( &validated_cluster.image, druid_role, rolegroup_name, - &rolegroup, rg, validated_cluster.cluster_config.s3_connection.as_ref(), &validated_cluster.cluster_config.druid_tls_security, @@ -397,19 +386,19 @@ pub async fn reconcile_druid( .add(client, rg_headless_service) .await .with_context(|_| ApplyRoleGroupServiceSnafu { - rolegroup: rolegroup.clone(), + role_group: rolegroup_name.to_string(), })?; cluster_resources .add(client, rg_metrics_service) .await .with_context(|_| ApplyRoleGroupServiceSnafu { - rolegroup: rolegroup.clone(), + role_group: rolegroup_name.to_string(), })?; cluster_resources .add(client, rg_configmap) .await .with_context(|_| ApplyRoleGroupConfigSnafu { - rolegroup: rolegroup.clone(), + role_group: rolegroup_name.to_string(), })?; // Note: The StatefulSet needs to be applied after all ConfigMaps and Secrets it mounts @@ -420,7 +409,7 @@ pub async fn reconcile_druid( .add(client, rg_statefulset) .await .with_context(|_| ApplyRoleGroupStatefulSetSnafu { - rolegroup: rolegroup.clone(), + role_group: rolegroup_name.to_string(), })?, ); } @@ -506,7 +495,6 @@ fn build_rolegroup_statefulset( resolved_product_image: &ResolvedProductImage, role: &DruidRole, role_group_name: &RoleGroupName, - rolegroup_ref: &RoleGroupRef, rg: &DruidRoleGroupConfig, s3_conn: Option<&s3::v1alpha1::ConnectionSpec>, druid_tls_security: &DruidTlsSecurity, @@ -514,6 +502,8 @@ fn build_rolegroup_statefulset( service_account: &ServiceAccount, ) -> Result { let merged_rolegroup_config = &rg.config; + let role_name = role.to_string(); + let resource_names = cluster.resource_names(role, role_group_name); // prepare container builder let prepare_container_name = Container::Prepare.to_string(); let mut cb_prepare = ContainerBuilder::new(&prepare_container_name).context( @@ -590,9 +580,9 @@ fn build_rolegroup_statefulset( .context(ConfigureS3Snafu)?; } - add_config_volume_and_volume_mounts(rolegroup_ref, &mut cb_druid, &mut pb)?; + add_config_volume_and_volume_mounts(&resource_names, &mut cb_druid, &mut pb)?; add_log_config_volume_and_volume_mounts( - rolegroup_ref, + &resource_names, merged_rolegroup_config, &mut cb_druid, &mut pb, @@ -703,8 +693,8 @@ fn build_rolegroup_statefulset( DRUID_CONTROLLER_NAME, // A version value is required, and we do want to use the "recommended" format for the other desired labels "none", - &rolegroup_ref.role, - &rolegroup_ref.role_group, + &role_name, + role_group_name.as_ref(), )) .context(LabelBuildSnafu)?; @@ -719,8 +709,8 @@ fn build_rolegroup_statefulset( druid, DRUID_CONTROLLER_NAME, &resolved_product_image.app_version_label_value, - &rolegroup_ref.role, - &rolegroup_ref.role_group, + &role_name, + role_group_name.as_ref(), )) .context(MetadataBuildSnafu)? .build(); @@ -740,7 +730,7 @@ fn build_rolegroup_statefulset( .expect("'vector' is a valid container name"), resolved_product_image, vector_log_config, - &cluster.resource_names(role, role_group_name), + &resource_names, &VolumeName::from_str(DRUID_CONFIG_VOLUME_NAME).expect("a valid volume name"), &VolumeName::from_str(LOG_VOLUME_NAME).expect("a valid volume name"), EnvVarSet::new(), @@ -754,15 +744,15 @@ fn build_rolegroup_statefulset( Ok(StatefulSet { metadata: ObjectMetaBuilder::new() .name_and_namespace(druid) - .name(rolegroup_ref.object_name()) + .name(resource_names.stateful_set_name().to_string()) .ownerreference_from_resource(druid, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(&build_recommended_labels( druid, DRUID_CONTROLLER_NAME, &resolved_product_image.app_version_label_value, - &rolegroup_ref.role, - &rolegroup_ref.role_group, + &role_name, + role_group_name.as_ref(), )) .context(MetadataBuildSnafu)? .with_label(RESTART_CONTROLLER_ENABLED_LABEL.to_owned()) @@ -775,15 +765,15 @@ fn build_rolegroup_statefulset( Labels::role_group_selector( druid, APP_NAME, - &rolegroup_ref.role, - &rolegroup_ref.role_group, + &role_name, + role_group_name.as_ref(), ) .context(LabelBuildSnafu)? .into(), ), ..LabelSelector::default() }, - service_name: Some(rolegroup_ref.rolegroup_headless_service_name()), + service_name: Some(resource_names.headless_service_name().to_string()), template: pod_template, volume_claim_templates: pvcs, ..StatefulSetSpec::default() @@ -814,7 +804,7 @@ fn add_hdfs_cm_volume_and_volume_mounts( } fn add_config_volume_and_volume_mounts( - rolegroup_ref: &RoleGroupRef, + resource_names: &ResourceNames, cb_druid: &mut ContainerBuilder, pb: &mut PodBuilder, ) -> Result<()> { @@ -823,7 +813,7 @@ fn add_config_volume_and_volume_mounts( .context(AddVolumeMountSnafu)?; pb.add_volume( VolumeBuilder::new(DRUID_CONFIG_VOLUME_NAME) - .with_config_map(rolegroup_ref.object_name()) + .with_config_map(resource_names.role_group_config_map().to_string()) .build(), ) .context(AddVolumeSnafu)?; @@ -841,7 +831,7 @@ fn add_config_volume_and_volume_mounts( } fn add_log_config_volume_and_volume_mounts( - rolegroup_ref: &RoleGroupRef, + resource_names: &ResourceNames, merged_rolegroup_config: &ValidatedDruidConfig, cb_druid: &mut ContainerBuilder, pb: &mut PodBuilder, @@ -852,7 +842,9 @@ fn add_log_config_volume_and_volume_mounts( let config_map = match &merged_rolegroup_config.logging.druid_container { ValidatedContainerLogConfigChoice::Custom(config_map_name) => config_map_name.to_string(), - ValidatedContainerLogConfigChoice::Automatic(_) => rolegroup_ref.object_name(), + ValidatedContainerLogConfigChoice::Automatic(_) => { + resource_names.role_group_config_map().to_string() + } }; pb.add_volume( @@ -950,15 +942,13 @@ mod test { .expect("tested rolegroup") .clone(); - let rolegroup_ref = RoleGroupRef { - cluster: ObjectRef::from_obj(&druid), - role: DruidRole::Historical.to_string(), - role_group: tested_rolegroup_name.to_string(), - }; - - let rg_configmap = - build_rolegroup_config_map(&cluster, &DruidRole::Historical, &rolegroup_ref, &rg) - .expect("build rolegroup config map"); + let rg_configmap = build_rolegroup_config_map( + &cluster, + &DruidRole::Historical, + &RoleGroupName::from_str(tested_rolegroup_name).unwrap(), + &rg, + ) + .expect("build rolegroup config map"); let druid_segment_cache_property = rg_configmap .data diff --git a/rust/operator-binary/src/controller/build/resource/config_map.rs b/rust/operator-binary/src/controller/build/resource/config_map.rs index a12e78dc..67237c9c 100644 --- a/rust/operator-binary/src/controller/build/resource/config_map.rs +++ b/rust/operator-binary/src/controller/build/resource/config_map.rs @@ -19,9 +19,9 @@ use stackable_operator::{ crd::s3, k8s_openapi::api::core::v1::{ConfigMap, EnvVar}, product_logging::framework::VECTOR_CONFIG_FILE, - role_utils::RoleGroupRef, v2::{ builder::meta::ownerreference_from_resource, config_file_writer::to_java_properties_string, + types::operator::RoleGroupName, }, }; @@ -40,7 +40,7 @@ use crate::{ }, crd::{ DruidConfigOverrides, DruidRole, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, - build_recommended_labels, build_string_list, env_var_reference, file_reference, v1alpha1, + build_recommended_labels, build_string_list, env_var_reference, file_reference, }, }; @@ -57,10 +57,10 @@ const AUTH_AUTHORIZER_OPA_URI: &str = "druid.auth.authorizer.OpaAuthorizer.opaUr #[derive(Snafu, Debug)] #[allow(clippy::enum_variant_names)] pub enum Error { - #[snafu(display("failed to build ConfigMap for {}", rolegroup))] + #[snafu(display("failed to build ConfigMap for role group {role_group}"))] BuildRoleGroupConfig { source: stackable_operator::builder::configmap::Error, - rolegroup: RoleGroupRef, + role_group: String, }, #[snafu(display("failed to configure S3 connection"))] @@ -137,9 +137,10 @@ fn key_value_overrides( pub fn build_rolegroup_config_map( cluster: &ValidatedCluster, role: &DruidRole, - rolegroup: &RoleGroupRef, + role_group_name: &RoleGroupName, rg: &DruidRoleGroupConfig, ) -> Result { + let resource_names = cluster.resource_names(role, role_group_name); let cluster_config = &cluster.cluster_config; let druid_tls_security = &cluster_config.druid_tls_security; let druid_auth_config = &cluster_config.druid_auth_config; @@ -316,7 +317,7 @@ pub fn build_rolegroup_config_map( ConfigFileName::SecurityProperties.to_string(), to_java_properties_string(security_config.iter()).with_context(|_| { JvmSecurityPropertiesSnafu { - rolegroup: rolegroup.role_group.clone(), + rolegroup: role_group_name.to_string(), } })?, ); @@ -326,14 +327,14 @@ pub fn build_rolegroup_config_map( config_map_builder.metadata( ObjectMetaBuilder::new() .name_and_namespace(cluster) - .name(rolegroup.object_name()) + .name(resource_names.role_group_config_map().to_string()) .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) .with_recommended_labels(&build_recommended_labels( cluster, DRUID_CONTROLLER_NAME, &cluster.image.app_version_label_value, - &rolegroup.role, - &rolegroup.role_group, + &role.to_string(), + role_group_name.as_ref(), )) .context(MetadataBuildSnafu)? .build(), @@ -356,6 +357,6 @@ pub fn build_rolegroup_config_map( config_map_builder .build() .with_context(|_| BuildRoleGroupConfigSnafu { - rolegroup: rolegroup.clone(), + role_group: role_group_name.to_string(), }) } diff --git a/rust/operator-binary/src/crd/resource.rs b/rust/operator-binary/src/crd/resource.rs index 1a3f3693..92cc546b 100644 --- a/rust/operator-binary/src/crd/resource.rs +++ b/rust/operator-binary/src/crd/resource.rs @@ -5,8 +5,8 @@ use stackable_operator::{ builder, builder::pod::{PodBuilder, container::ContainerBuilder, volume::VolumeBuilder}, commons::resources::{ - CpuLimitsFragment, MemoryLimits, MemoryLimitsFragment, NoRuntimeLimits, - NoRuntimeLimitsFragment, Resources, ResourcesFragment, + CpuLimitsFragment, MemoryLimitsFragment, NoRuntimeLimits, NoRuntimeLimitsFragment, + Resources, ResourcesFragment, }, k8s_openapi::{ api::core::v1::{EmptyDirVolumeSource, ResourceRequirements}, @@ -69,13 +69,6 @@ impl RoleResource { } } - pub fn as_memory_limits(&self) -> MemoryLimits { - match self { - Self::Druid(r) => r.memory.clone(), - Self::Historical(r) => r.memory.clone(), - } - } - /// Update the given configuration file with resource properties. /// Currently it only adds historical-specific configs for direct memory buffers, thread counts and segment cache. pub fn update_druid_config_file( From 09c8d1a0c1c7583e42571cefedad4bcb05869d27 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 14:29:20 +0200 Subject: [PATCH 40/48] refactor: build step consumes ValidatedCluster; drop redundant builder params & raw druid --- .../operator-binary/src/authentication/mod.rs | 10 +- rust/operator-binary/src/controller.rs | 100 +++++------------- .../src/controller/build/resource/listener.rs | 34 +++--- .../src/controller/build/resource/service.rs | 46 +++++--- .../src/controller/validate.rs | 6 ++ rust/operator-binary/src/internal_secret.rs | 4 +- 6 files changed, 91 insertions(+), 109 deletions(-) diff --git a/rust/operator-binary/src/authentication/mod.rs b/rust/operator-binary/src/authentication/mod.rs index 73dc67ce..5baf81ac 100644 --- a/rust/operator-binary/src/authentication/mod.rs +++ b/rust/operator-binary/src/authentication/mod.rs @@ -8,12 +8,12 @@ use stackable_operator::{ }; use crate::{ + controller::validate::ValidatedCluster, crd::{ DruidRole, authentication::{AuthenticationClassResolved, AuthenticationClassesResolved}, env_var_reference, security::INTERNAL_INITIAL_CLIENT_PASSWORD_ENV, - v1alpha1, }, internal_secret::{build_shared_internal_secret_name, env_var_from_secret}, }; @@ -166,13 +166,9 @@ impl DruidAuthenticationConfig { command } - pub fn get_env_var_mounts( - &self, - druid: &v1alpha1::DruidCluster, - role: &DruidRole, - ) -> Vec { + pub fn get_env_var_mounts(&self, cluster: &ValidatedCluster, role: &DruidRole) -> Vec { let mut envs = vec![]; - let internal_secret_name = build_shared_internal_secret_name(druid); + let internal_secret_name = build_shared_internal_secret_name(cluster); envs.push(env_var_from_secret( &internal_secret_name, None, diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index f31d16cc..abed3eeb 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -16,10 +16,8 @@ use stackable_operator::{ }, cli::OperatorEnvironmentOptions, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, - commons::{product_image_selection::ResolvedProductImage, rbac::build_rbac_resources}, + commons::rbac::build_rbac_resources, constants::RESTART_CONTROLLER_ENABLED_LABEL, - crd::s3, - database_connections::drivers::jdbc::JdbcDatabaseConnection as _, k8s_openapi::{ DeepMerge, api::{ @@ -54,7 +52,6 @@ use stackable_operator::{ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ - authentication::DruidAuthenticationConfig, controller::build::resource::{ listener::{ LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_group_listener, @@ -67,7 +64,7 @@ use crate::{ APP_NAME, Container, DRUID_CONFIG_DIRECTORY, DeepStorageSpec, DruidClusterStatus, DruidRole, HDFS_CONFIG_DIRECTORY, LOG_CONFIG_DIRECTORY, METRICS_PORT, METRICS_PORT_NAME, OPERATOR_NAME, RW_CONFIG_DIRECTORY, STACKABLE_LOG_DIR, ValidatedDruidConfig, - build_recommended_labels, security::DruidTlsSecurity, v1alpha1, + build_recommended_labels, v1alpha1, }, internal_secret::create_shared_internal_secret, operations::graceful_shutdown::add_graceful_shutdown_config, @@ -75,7 +72,7 @@ use crate::{ mod build; mod dereference; -mod validate; +pub(crate) mod validate; use build::{ properties::product_logging::MAX_DRUID_LOG_FILES_SIZE, @@ -258,11 +255,6 @@ pub enum Error { BuildConfigMap { source: build::resource::config_map::Error, }, - - #[snafu(display("invalid metadata database connection"))] - InvalidMetadataDatabaseConnection { - source: stackable_operator::database_connections::Error, - }, } type Result = std::result::Result; @@ -332,35 +324,12 @@ pub async fn reconcile_druid( .context(FailedInternalSecretCreationSnafu)?; for (rolegroup_name, rg) in groups.iter() { - let role_group_service_recommended_labels = build_recommended_labels( - druid, - DRUID_CONTROLLER_NAME, - &validated_cluster.image.app_version_label_value, - &role_name, - rolegroup_name.as_ref(), - ); - - let role_group_service_selector = - Labels::role_group_selector(druid, APP_NAME, &role_name, rolegroup_name.as_ref()) - .context(LabelBuildSnafu)?; - - let rg_headless_service = build_rolegroup_headless_service( - &validated_cluster, - &validated_cluster.cluster_config.druid_tls_security, - druid_role, - rolegroup_name, - role_group_service_recommended_labels.clone(), - role_group_service_selector.clone().into(), - ) - .context(ServiceConfigurationSnafu)?; - let rg_metrics_service = build_rolegroup_metrics_service( - &validated_cluster, - druid_role, - rolegroup_name, - role_group_service_recommended_labels, - role_group_service_selector.into(), - ) - .context(ServiceConfigurationSnafu)?; + let rg_headless_service = + build_rolegroup_headless_service(&validated_cluster, druid_role, rolegroup_name) + .context(ServiceConfigurationSnafu)?; + let rg_metrics_service = + build_rolegroup_metrics_service(&validated_cluster, druid_role, rolegroup_name) + .context(ServiceConfigurationSnafu)?; let rg_configmap = build::resource::config_map::build_rolegroup_config_map( &validated_cluster, @@ -370,15 +339,10 @@ pub async fn reconcile_druid( ) .context(BuildConfigMapSnafu)?; let rg_statefulset = build_rolegroup_statefulset( - druid, &validated_cluster, - &validated_cluster.image, druid_role, rolegroup_name, rg, - validated_cluster.cluster_config.s3_connection.as_ref(), - &validated_cluster.cluster_config.druid_tls_security, - &validated_cluster.cluster_config.druid_auth_config, &rbac_sa, )?; @@ -415,12 +379,12 @@ pub async fn reconcile_druid( } if let Some(listener_class) = druid_role.listener_class_name(druid) - && let Some(listener_group_name) = group_listener_name(druid, druid_role) + && let Some(listener_group_name) = group_listener_name(&validated_cluster, druid_role) { let role_group_listener = build_group_listener( - druid, + &validated_cluster, build_recommended_labels( - druid, + &validated_cluster, DRUID_CONTROLLER_NAME, &validated_cluster.image.app_version_label_value, &role_name, @@ -429,7 +393,6 @@ pub async fn reconcile_druid( listener_class.to_string(), listener_group_name, druid_role, - &validated_cluster.cluster_config.druid_tls_security, ) .context(ListenerConfigurationSnafu)?; @@ -484,26 +447,26 @@ pub async fn reconcile_druid( Ok(Action::await_change()) } -#[allow(clippy::too_many_arguments)] /// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. /// /// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the /// corresponding [`stackable_operator::k8s_openapi::api::core::v1::Service`] (from [`build_rolegroup_headless_service`]). fn build_rolegroup_statefulset( - druid: &v1alpha1::DruidCluster, cluster: &ValidatedCluster, - resolved_product_image: &ResolvedProductImage, role: &DruidRole, role_group_name: &RoleGroupName, rg: &DruidRoleGroupConfig, - s3_conn: Option<&s3::v1alpha1::ConnectionSpec>, - druid_tls_security: &DruidTlsSecurity, - druid_auth_config: &Option, service_account: &ServiceAccount, ) -> Result { let merged_rolegroup_config = &rg.config; let role_name = role.to_string(); let resource_names = cluster.resource_names(role, role_group_name); + // Everything below used to be threaded in as separate parameters; it all lives on the + // `ValidatedCluster` now. + let resolved_product_image = &cluster.image; + let s3_conn = cluster.cluster_config.s3_connection.as_ref(); + let druid_tls_security = &cluster.cluster_config.druid_tls_security; + let druid_auth_config = &cluster.cluster_config.druid_auth_config; // prepare container builder let prepare_container_name = Container::Prepare.to_string(); let mut cb_prepare = ContainerBuilder::new(&prepare_container_name).context( @@ -530,12 +493,7 @@ fn build_rolegroup_statefulset( ) .context(GracefulShutdownSnafu)?; - let metadata_database_connection_details = druid - .spec - .cluster_config - .metadata_database - .jdbc_connection_details("metadata") - .context(InvalidMetadataDatabaseConnectionSnafu)?; + let metadata_database_connection_details = &cluster.cluster_config.metadata_db_connection; let mut main_container_commands = role.main_container_prepare_commands(s3_conn); let mut prepare_container_commands = vec![]; @@ -589,7 +547,7 @@ fn build_rolegroup_statefulset( )?; add_log_volume_and_volume_mounts(&mut cb_druid, &mut cb_prepare, &mut pb)?; add_hdfs_cm_volume_and_volume_mounts( - &druid.spec.cluster_config.deep_storage, + &cluster.cluster_config.deep_storage, &mut cb_druid, &mut pb, )?; @@ -623,7 +581,7 @@ fn build_rolegroup_statefulset( let mut rest_env: Vec = rg.env_overrides.clone().into(); if let Some(auth_config) = druid_auth_config { - rest_env.extend(auth_config.get_env_var_mounts(druid, role)) + rest_env.extend(auth_config.get_env_var_mounts(cluster, role)) } // Needed for the `containerdebug` process to log it's tracing information to. @@ -662,7 +620,7 @@ fn build_rolegroup_statefulset( // Known roles are MiddleManagers for ingestion and Historicals for deep storage (GCS plugin) // We may at some time in the future revisit this and limit it again to avoid needlessly // propagating potentially confidential files throughout the cluster - for volume in &druid.spec.cluster_config.extra_volumes { + for volume in &cluster.cluster_config.extra_volumes { // Extract values into vars so we make it impossible to log something other than // what we actually use to create the mounts - maybe paranoid, but hey .. let volume_name = &volume.name; @@ -682,14 +640,14 @@ fn build_rolegroup_statefulset( let mut pvcs: Option> = None; - if let Some(group_listener_name) = group_listener_name(druid, role) { + if let Some(group_listener_name) = group_listener_name(cluster, role) { cb_druid .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) .context(AddVolumeMountSnafu)?; // Used for PVC templates that cannot be modified once they are deployed let unversioned_recommended_labels = Labels::recommended(&build_recommended_labels( - druid, + cluster, DRUID_CONTROLLER_NAME, // A version value is required, and we do want to use the "recommended" format for the other desired labels "none", @@ -706,7 +664,7 @@ fn build_rolegroup_statefulset( let metadata = ObjectMetaBuilder::new() .with_recommended_labels(&build_recommended_labels( - druid, + cluster, DRUID_CONTROLLER_NAME, &resolved_product_image.app_version_label_value, &role_name, @@ -743,12 +701,12 @@ fn build_rolegroup_statefulset( Ok(StatefulSet { metadata: ObjectMetaBuilder::new() - .name_and_namespace(druid) + .name_and_namespace(cluster) .name(resource_names.stateful_set_name().to_string()) - .ownerreference_from_resource(druid, None, Some(true)) + .ownerreference_from_resource(cluster, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(&build_recommended_labels( - druid, + cluster, DRUID_CONTROLLER_NAME, &resolved_product_image.app_version_label_value, &role_name, @@ -763,7 +721,7 @@ fn build_rolegroup_statefulset( selector: LabelSelector { match_labels: Some( Labels::role_group_selector( - druid, + cluster, APP_NAME, &role_name, role_group_name.as_ref(), diff --git a/rust/operator-binary/src/controller/build/resource/listener.rs b/rust/operator-binary/src/controller/build/resource/listener.rs index 6bc9247a..c2025e9c 100644 --- a/rust/operator-binary/src/controller/build/resource/listener.rs +++ b/rust/operator-binary/src/controller/build/resource/listener.rs @@ -6,14 +6,15 @@ use stackable_operator::{ }, crd::listener::{self, v1alpha1::Listener}, k8s_openapi::api::core::v1::PersistentVolumeClaim, - kube::ResourceExt, kvp::{Labels, ObjectLabels}, }; -use crate::crd::{ - DruidRole, - security::{DruidTlsSecurity, PLAINTEXT_PORT_NAME, TLS_PORT_NAME}, - v1alpha1, +use crate::{ + controller::validate::ValidatedCluster, + crd::{ + DruidRole, + security::{DruidTlsSecurity, PLAINTEXT_PORT_NAME, TLS_PORT_NAME}, + }, }; pub const LISTENER_VOLUME_NAME: &str = "listener"; @@ -47,25 +48,29 @@ pub enum Error { } pub fn build_group_listener( - druid: &v1alpha1::DruidCluster, - object_labels: ObjectLabels, + cluster: &ValidatedCluster, + object_labels: ObjectLabels, listener_class: String, listener_group_name: String, druid_role: &DruidRole, - druid_tls_security: &DruidTlsSecurity, ) -> Result { Ok(Listener { metadata: ObjectMetaBuilder::new() - .name_and_namespace(druid) + .name_and_namespace(cluster) .name(listener_group_name) - .ownerreference_from_resource(druid, None, Some(true)) + .ownerreference_from_resource(cluster, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(&object_labels) .context(BuildObjectMetaSnafu)? .build(), spec: listener::v1alpha1::ListenerSpec { class_name: Some(listener_class), - ports: Some(druid_tls_security.listener_ports(druid_role)), + ports: Some( + cluster + .cluster_config + .druid_tls_security + .listener_ports(druid_role), + ), ..listener::v1alpha1::ListenerSpec::default() }, status: None, @@ -84,14 +89,11 @@ pub fn build_group_listener_pvc( .context(BuildListenerPersistentVolumeSnafu) } -pub fn group_listener_name( - druid: &v1alpha1::DruidCluster, - druid_role: &DruidRole, -) -> Option { +pub fn group_listener_name(cluster: &ValidatedCluster, druid_role: &DruidRole) -> Option { match druid_role { DruidRole::Coordinator | DruidRole::Broker | DruidRole::Router => Some(format!( "{cluster_name}-{druid_role}", - cluster_name = druid.name_any(), + cluster_name = cluster.name, )), DruidRole::Historical | DruidRole::MiddleManager => None, } diff --git a/rust/operator-binary/src/controller/build/resource/service.rs b/rust/operator-binary/src/controller/build/resource/service.rs index 889a708e..388bf9e8 100644 --- a/rust/operator-binary/src/controller/build/resource/service.rs +++ b/rust/operator-binary/src/controller/build/resource/service.rs @@ -1,16 +1,14 @@ -use std::collections::BTreeMap; - use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::meta::ObjectMetaBuilder, k8s_openapi::api::core::v1::{Service, ServicePort, ServiceSpec}, - kvp::{Annotations, Label, ObjectLabels}, + kvp::{Annotations, Label, Labels}, v2::types::operator::RoleGroupName, }; use crate::{ - controller::validate::ValidatedCluster, - crd::{DruidRole, METRICS_PORT, METRICS_PORT_NAME, security::DruidTlsSecurity, v1alpha1}, + controller::{DRUID_CONTROLLER_NAME, validate::ValidatedCluster}, + crd::{APP_NAME, DruidRole, METRICS_PORT, METRICS_PORT_NAME, build_recommended_labels}, }; #[derive(Snafu, Debug)] @@ -35,12 +33,20 @@ pub enum Error { /// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. pub fn build_rolegroup_headless_service( cluster: &ValidatedCluster, - druid_tls_security: &DruidTlsSecurity, druid_role: &DruidRole, role_group_name: &RoleGroupName, - object_labels: ObjectLabels, - selector: BTreeMap, ) -> Result { + let role_name = druid_role.to_string(); + let object_labels = build_recommended_labels( + cluster, + DRUID_CONTROLLER_NAME, + &cluster.image.app_version_label_value, + &role_name, + role_group_name.as_ref(), + ); + let selector = + Labels::role_group_selector(cluster, APP_NAME, &role_name, role_group_name.as_ref()) + .context(LabelBuildSnafu)?; Ok(Service { metadata: ObjectMetaBuilder::new() .name_and_namespace(cluster) @@ -59,8 +65,13 @@ pub fn build_rolegroup_headless_service( // Internal communication does not need to be exposed type_: Some("ClusterIP".to_string()), cluster_ip: Some("None".to_string()), - ports: Some(druid_tls_security.service_ports(druid_role)), - selector: Some(selector), + ports: Some( + cluster + .cluster_config + .druid_tls_security + .service_ports(druid_role), + ), + selector: Some(selector.into()), publish_not_ready_addresses: Some(true), ..ServiceSpec::default() }), @@ -73,9 +84,18 @@ pub fn build_rolegroup_metrics_service( cluster: &ValidatedCluster, druid_role: &DruidRole, role_group_name: &RoleGroupName, - object_labels: ObjectLabels, - selector: BTreeMap, ) -> Result { + let role_name = druid_role.to_string(); + let object_labels = build_recommended_labels( + cluster, + DRUID_CONTROLLER_NAME, + &cluster.image.app_version_label_value, + &role_name, + role_group_name.as_ref(), + ); + let selector = + Labels::role_group_selector(cluster, APP_NAME, &role_name, role_group_name.as_ref()) + .context(LabelBuildSnafu)?; Ok(Service { metadata: ObjectMetaBuilder::new() .name_and_namespace(cluster) @@ -97,7 +117,7 @@ pub fn build_rolegroup_metrics_service( type_: Some("ClusterIP".to_string()), cluster_ip: Some("None".to_string()), ports: Some(metrics_service_ports()), - selector: Some(selector), + selector: Some(selector.into()), publish_not_ready_addresses: Some(true), ..ServiceSpec::default() }), diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 46ad5726..047928e8 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -11,6 +11,7 @@ use stackable_operator::{ commons::product_image_selection::{self, ResolvedProductImage}, crd::s3, database_connections::drivers::jdbc::{JdbcDatabaseConnection, JdbcDatabaseConnectionDetails}, + k8s_openapi::api::core::v1::Volume, kube::{Resource, api::ObjectMeta}, v2::{ HasName, HasUid, @@ -91,6 +92,9 @@ pub struct ValidatedClusterConfig { /// `runtime.properties` (deep storage type / directory / base key) without the raw /// `DruidCluster`. pub deep_storage: DeepStorageSpec, + /// User-supplied extra volumes, mounted into every container, carried so the build step does + /// not read the raw `DruidCluster`. + pub extra_volumes: Vec, } /// Synchronous inputs the rest of `reconcile_druid` needs after dereferencing. @@ -261,6 +265,7 @@ pub fn validate( metadata_storage_type, metadata_db_connection, deep_storage: druid.spec.cluster_config.deep_storage.clone(), + extra_volumes: druid.spec.cluster_config.extra_volumes.clone(), }, role_group_configs, )) @@ -401,6 +406,7 @@ spec: metadata_storage_type, metadata_db_connection, deep_storage: druid.spec.cluster_config.deep_storage.clone(), + extra_volumes: druid.spec.cluster_config.extra_volumes.clone(), }, role_group_configs, ) diff --git a/rust/operator-binary/src/internal_secret.rs b/rust/operator-binary/src/internal_secret.rs index 629c3ae6..6529ee64 100644 --- a/rust/operator-binary/src/internal_secret.rs +++ b/rust/operator-binary/src/internal_secret.rs @@ -168,8 +168,8 @@ fn build_immutable_shared_internal_secret_name(druid: &v1alpha1::DruidCluster) - format!("{}-internal-secret", druid.name_any()) } -pub fn build_shared_internal_secret_name(druid: &v1alpha1::DruidCluster) -> String { - format!("{}-shared-internal-secret", druid.name_any()) +pub fn build_shared_internal_secret_name(owner: &T) -> String { + format!("{}-shared-internal-secret", owner.name_any()) } fn get_random_base64() -> String { From e6a91973b54ec0a4a8ca5859326f0c25953aa9e9 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 14:55:11 +0200 Subject: [PATCH 41/48] refactor: build_pdb returns PDB via infallible v2 builder; add typed name helpers --- rust/operator-binary/src/controller.rs | 41 +++++++++---- .../src/controller/build/resource/pdb.rs | 59 ++++++------------- .../src/controller/validate.rs | 8 ++- rust/operator-binary/src/crd/mod.rs | 8 ++- 4 files changed, 62 insertions(+), 54 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index abed3eeb..c8231a69 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -45,7 +45,7 @@ use stackable_operator::{ role_group_utils::ResourceNames, types::{ kubernetes::{ContainerName, VolumeName}, - operator::RoleGroupName, + operator::{ControllerName, OperatorName, ProductName, RoleGroupName}, }, }, }; @@ -57,7 +57,7 @@ use crate::{ LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_group_listener, build_group_listener_pvc, group_listener_name, secret_volume_listener_scope, }, - pdb::add_pdbs, + pdb::build_pdb, service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, }, crd::{ @@ -85,6 +85,22 @@ pub const FULL_CONTROLLER_NAME: &str = concatcp!(DRUID_CONTROLLER_NAME, '.', OPE pub(super) const CONTAINER_IMAGE_BASE_NAME: &str = "druid"; +/// The product name (`druid`) as a type-safe label value. +pub(crate) fn product_name() -> ProductName { + ProductName::from_str(APP_NAME).expect("'druid' is a valid product name") +} + +/// The operator name as a type-safe label value. +pub(crate) fn operator_name() -> OperatorName { + OperatorName::from_str(OPERATOR_NAME).expect("the operator name is a valid label value") +} + +/// The controller name as a type-safe label value. +pub(crate) fn controller_name() -> ControllerName { + ControllerName::from_str(DRUID_CONTROLLER_NAME) + .expect("the controller name is a valid label value") +} + // volume names const DRUID_CONFIG_VOLUME_NAME: &str = "config"; const HDFS_CONFIG_VOLUME_NAME: &str = "hdfs"; @@ -192,9 +208,9 @@ pub enum Error { source: stackable_operator::commons::rbac::Error, }, - #[snafu(display("failed to create PodDisruptionBudget"))] - FailedToCreatePdb { - source: crate::controller::build::resource::pdb::Error, + #[snafu(display("failed to apply PodDisruptionBudget"))] + ApplyPdb { + source: stackable_operator::cluster_resources::Error, }, #[snafu(display("failed to configure graceful shutdown"))] @@ -417,15 +433,16 @@ pub async fn reconcile_druid( let role_config = druid.generic_role_config(druid_role); - add_pdbs( + if let Some(pdb) = build_pdb( &role_config.pod_disruption_budget, - druid, + &validated_cluster, druid_role, - client, - &mut cluster_resources, - ) - .await - .context(FailedToCreatePdbSnafu)?; + ) { + cluster_resources + .add(client, pdb) + .await + .context(ApplyPdbSnafu)?; + } } let cluster_operation_cond_builder = diff --git a/rust/operator-binary/src/controller/build/resource/pdb.rs b/rust/operator-binary/src/controller/build/resource/pdb.rs index da25223c..76fab894 100644 --- a/rust/operator-binary/src/controller/build/resource/pdb.rs +++ b/rust/operator-binary/src/controller/build/resource/pdb.rs @@ -1,37 +1,24 @@ -use snafu::{ResultExt, Snafu}; +use std::str::FromStr; + use stackable_operator::{ - builder::pdb::PodDisruptionBudgetBuilder, client::Client, cluster_resources::ClusterResources, - commons::pdb::PdbConfig, kube::ResourceExt, + commons::pdb::PdbConfig, + k8s_openapi::api::policy::v1::PodDisruptionBudget, + v2::{builder::pdb::pod_disruption_budget_builder_with_role, types::operator::RoleName}, }; use crate::{ - controller::DRUID_CONTROLLER_NAME, - crd::{APP_NAME, DruidRole, OPERATOR_NAME, v1alpha1}, + controller::{controller_name, operator_name, product_name, validate::ValidatedCluster}, + crd::DruidRole, }; -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("Cannot create PodDisruptionBudget for role [{role}]"))] - CreatePdb { - source: stackable_operator::builder::pdb::Error, - role: String, - }, - #[snafu(display("Cannot apply PodDisruptionBudget [{name}]"))] - ApplyPdb { - source: stackable_operator::cluster_resources::Error, - name: String, - }, -} - -pub async fn add_pdbs( +/// Builds the [`PodDisruptionBudget`] for the given `role`, or `None` if PDBs are disabled. +pub fn build_pdb( pdb: &PdbConfig, - druid: &v1alpha1::DruidCluster, + cluster: &ValidatedCluster, role: &DruidRole, - client: &Client, - cluster_resources: &mut ClusterResources<'_>, -) -> Result<(), Error> { +) -> Option { if !pdb.enabled { - return Ok(()); + return None; } let max_unavailable = pdb.max_unavailable.unwrap_or(match role { DruidRole::Broker => max_unavailable_brokers(), @@ -40,25 +27,17 @@ pub async fn add_pdbs( DruidRole::MiddleManager => max_unavailable_middle_managers(), DruidRole::Router => max_unavailable_routers(), }); - let pdb = PodDisruptionBudgetBuilder::new_with_role( - druid, - APP_NAME, - &role.to_string(), - OPERATOR_NAME, - DRUID_CONTROLLER_NAME, + let pdb = pod_disruption_budget_builder_with_role( + cluster, + &product_name(), + &RoleName::from_str(&role.to_string()).expect("a DruidRole is a valid role name"), + &operator_name(), + &controller_name(), ) - .with_context(|_| CreatePdbSnafu { - role: role.to_string(), - })? .with_max_unavailable(max_unavailable) .build(); - let pdb_name = pdb.name_any(); - cluster_resources - .add(client, pdb) - .await - .with_context(|_| ApplyPdbSnafu { name: pdb_name })?; - Ok(()) + Some(pdb) } fn max_unavailable_brokers() -> u16 { diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 047928e8..8f257439 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -14,7 +14,7 @@ use stackable_operator::{ k8s_openapi::api::core::v1::Volume, kube::{Resource, api::ObjectMeta}, v2::{ - HasName, HasUid, + HasName, HasUid, NameIsValidLabelValue, controller_utils::{get_cluster_name, get_namespace, get_uid}, role_group_utils::ResourceNames, types::{ @@ -195,6 +195,12 @@ impl HasUid for ValidatedCluster { } } +impl NameIsValidLabelValue for ValidatedCluster { + fn to_label_value(&self) -> String { + self.name.to_label_value() + } +} + /// Validates the cluster spec and the dereferenced inputs. pub fn validate( druid: &v1alpha1::DruidCluster, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 0308d39b..1929c043 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -447,11 +447,17 @@ impl v1alpha1::DruidCluster { s3_ingestion || s3_storage } - /// Merges and validates all role groups of the given role. + /// Merges and validates all role groups of the given role. Invoked from the validate step + /// (`controller::validate`). /// /// All four override categories (config / env / cli / pod) are merged by /// [`with_validated_config`] (role group wins over role); the typed per-role config is then /// erased to the shared [`ValidatedDruidConfig`] view consumed by the build step. + /// + /// This lives in `crate::crd` and is a `macro_rules!` rather than a generic function on purpose: the erasure + /// reads the per-role configs' private `resources` field and calls their private + /// `default_config`, and the five roles wrap *different* `resources` types into different + /// [`RoleResource`] variants. pub fn merged_role( &self, role: &DruidRole, From b83c371ecb8149a401e9234de9b5985b326c834d Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 15:28:29 +0200 Subject: [PATCH 42/48] efactor: extract StatefulSet builder into controller/build/resource/statefulset --- rust/operator-binary/src/controller.rs | 514 +---------------- .../src/controller/build/resource/mod.rs | 1 + .../controller/build/resource/statefulset.rs | 520 ++++++++++++++++++ 3 files changed, 535 insertions(+), 500 deletions(-) create mode 100644 rust/operator-binary/src/controller/build/resource/statefulset.rs diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index c8231a69..509275b0 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -6,79 +6,42 @@ use std::{str::FromStr, sync::Arc}; use const_format::concatcp; use snafu::{ResultExt, Snafu}; use stackable_operator::{ - builder::{ - self, - meta::ObjectMetaBuilder, - pod::{ - PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, - security::PodSecurityContextBuilder, volume::VolumeBuilder, - }, - }, cli::OperatorEnvironmentOptions, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, commons::rbac::build_rbac_resources, - constants::RESTART_CONTROLLER_ENABLED_LABEL, - k8s_openapi::{ - DeepMerge, - api::{ - apps::v1::{StatefulSet, StatefulSetSpec}, - core::v1::{EnvVar, PersistentVolumeClaim, ServiceAccount}, - }, - apimachinery::pkg::apis::meta::v1::LabelSelector, - }, kube::{ - Resource, ResourceExt, + Resource, core::{DeserializeGuard, error_boundary}, runtime::controller::Action, }, - kvp::{KeyValuePairError, LabelError, LabelValueError, Labels}, + kvp::{KeyValuePairError, LabelValueError}, logging::controller::ReconcilerError, - product_logging, shared::time::Duration, status::condition::{ compute_conditions, operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, }, - v2::{ - builder::pod::container::EnvVarSet, - product_logging::framework::{ValidatedContainerLogConfigChoice, vector_container}, - role_group_utils::ResourceNames, - types::{ - kubernetes::{ContainerName, VolumeName}, - operator::{ControllerName, OperatorName, ProductName, RoleGroupName}, - }, - }, + v2::types::operator::{ControllerName, OperatorName, ProductName}, }; use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ controller::build::resource::{ - listener::{ - LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_group_listener, - build_group_listener_pvc, group_listener_name, secret_volume_listener_scope, - }, + listener::{build_group_listener, group_listener_name}, pdb::build_pdb, service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, }, crd::{ - APP_NAME, Container, DRUID_CONFIG_DIRECTORY, DeepStorageSpec, DruidClusterStatus, - DruidRole, HDFS_CONFIG_DIRECTORY, LOG_CONFIG_DIRECTORY, METRICS_PORT, METRICS_PORT_NAME, - OPERATOR_NAME, RW_CONFIG_DIRECTORY, STACKABLE_LOG_DIR, ValidatedDruidConfig, - build_recommended_labels, v1alpha1, + APP_NAME, DruidClusterStatus, DruidRole, OPERATOR_NAME, build_recommended_labels, v1alpha1, }, internal_secret::create_shared_internal_secret, - operations::graceful_shutdown::add_graceful_shutdown_config, }; mod build; mod dereference; pub(crate) mod validate; -use build::{ - properties::product_logging::MAX_DRUID_LOG_FILES_SIZE, - resource::discovery::{self, build_discovery_configmaps}, -}; -use validate::{DruidRoleGroupConfig, ValidatedCluster}; +use build::resource::discovery::{self, build_discovery_configmaps}; pub const DRUID_CONTROLLER_NAME: &str = "druidcluster"; pub const FULL_CONTROLLER_NAME: &str = concatcp!(DRUID_CONTROLLER_NAME, '.', OPERATOR_NAME); @@ -101,14 +64,6 @@ pub(crate) fn controller_name() -> ControllerName { .expect("the controller name is a valid label value") } -// volume names -const DRUID_CONFIG_VOLUME_NAME: &str = "config"; -const HDFS_CONFIG_VOLUME_NAME: &str = "hdfs"; -const LOG_CONFIG_VOLUME_NAME: &str = "log-config"; -const LOG_VOLUME_NAME: &str = "log"; -const RW_CONFIG_VOLUME_NAME: &str = "rwconfig"; -const USERDATA_MOUNTPOINT: &str = "/stackable/userdata"; - pub struct Ctx { pub client: stackable_operator::client::Client, pub operator_environment: OperatorEnvironmentOptions, @@ -129,25 +84,20 @@ pub enum Error { role_group: String, }, + #[snafu(display("failed to build StatefulSet"))] + BuildRoleGroupStatefulSet { + source: build::resource::statefulset::Error, + }, + #[snafu(display("failed to apply StatefulSet for role group {role_group}"))] ApplyRoleGroupStatefulSet { source: stackable_operator::cluster_resources::Error, role_group: String, }, - #[snafu(display("object is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, - }, - #[snafu(display("failed to dereference cluster objects"))] Dereference { source: dereference::Error }, - #[snafu(display("failed to configure S3 connection"))] - ConfigureS3 { - source: stackable_operator::crd::s3::v1alpha1::ConnectionError, - }, - #[snafu(display("failed to build discovery ConfigMap"))] BuildDiscoveryConfig { source: discovery::Error }, @@ -161,11 +111,6 @@ pub enum Error { source: stackable_operator::client::Error, }, - #[snafu(display( - "Druid does not support skipping the verification of the tls enabled S3 server" - ))] - S3TlsNoVerificationNotSupported, - #[snafu(display("failed to create cluster resources"))] CreateClusterResources { source: stackable_operator::cluster_resources::Error, @@ -176,18 +121,6 @@ pub enum Error { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("failed to create container builder with name [{name}]"))] - FailedContainerBuilderCreation { - source: stackable_operator::builder::pod::container::Error, - name: String, - }, - - #[snafu(display("failed to initialize security context"))] - FailedToInitializeSecurityContext { source: crate::crd::security::Error }, - - #[snafu(display("failed to update Druid config from resources"))] - UpdateDruidConfigFromResources { source: crate::crd::resource::Error }, - #[snafu(display("failed to retrieve secret for internal communications"))] FailedInternalSecretCreation { source: crate::internal_secret::Error, @@ -213,37 +146,11 @@ pub enum Error { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("failed to configure graceful shutdown"))] - GracefulShutdown { - source: crate::operations::graceful_shutdown::Error, - }, - - #[snafu(display("failed to add OIDC Volumes and VolumeMounts to the Pod and containers"))] - AuthVolumesBuild { - source: crate::authentication::Error, - }, - - #[snafu(display("failed to build labels"))] - LabelBuild { source: LabelError }, - - #[snafu(display("failed to build metadata"))] - MetadataBuild { - source: stackable_operator::builder::meta::Error, - }, - #[snafu(display("failed to get required labels"))] GetRequiredLabels { source: KeyValuePairError, }, - #[snafu(display("failed to add needed volume"))] - AddVolume { source: builder::pod::Error }, - - #[snafu(display("failed to add needed volumeMount"))] - AddVolumeMount { - source: builder::pod::container::Error, - }, - #[snafu(display("DruidCluster object is invalid"))] InvalidDruidCluster { source: error_boundary::InvalidObject, @@ -354,13 +261,14 @@ pub async fn reconcile_druid( rg, ) .context(BuildConfigMapSnafu)?; - let rg_statefulset = build_rolegroup_statefulset( + let rg_statefulset = build::resource::statefulset::build_rolegroup_statefulset( &validated_cluster, druid_role, rolegroup_name, rg, &rbac_sa, - )?; + ) + .context(BuildRoleGroupStatefulSetSnafu)?; cluster_resources .add(client, rg_headless_service) @@ -464,400 +372,6 @@ pub async fn reconcile_druid( Ok(Action::await_change()) } -/// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. -/// -/// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the -/// corresponding [`stackable_operator::k8s_openapi::api::core::v1::Service`] (from [`build_rolegroup_headless_service`]). -fn build_rolegroup_statefulset( - cluster: &ValidatedCluster, - role: &DruidRole, - role_group_name: &RoleGroupName, - rg: &DruidRoleGroupConfig, - service_account: &ServiceAccount, -) -> Result { - let merged_rolegroup_config = &rg.config; - let role_name = role.to_string(); - let resource_names = cluster.resource_names(role, role_group_name); - // Everything below used to be threaded in as separate parameters; it all lives on the - // `ValidatedCluster` now. - let resolved_product_image = &cluster.image; - let s3_conn = cluster.cluster_config.s3_connection.as_ref(); - let druid_tls_security = &cluster.cluster_config.druid_tls_security; - let druid_auth_config = &cluster.cluster_config.druid_auth_config; - // prepare container builder - let prepare_container_name = Container::Prepare.to_string(); - let mut cb_prepare = ContainerBuilder::new(&prepare_container_name).context( - FailedContainerBuilderCreationSnafu { - name: &prepare_container_name, - }, - )?; - // druid container builder - let druid_container_name = Container::Druid.to_string(); - let mut cb_druid = ContainerBuilder::new(&druid_container_name).context( - FailedContainerBuilderCreationSnafu { - name: &druid_container_name, - }, - )?; - // init pod builder - let mut pb = PodBuilder::new(); - pb.affinity(&merged_rolegroup_config.affinity); - add_graceful_shutdown_config( - role, - druid_tls_security, - merged_rolegroup_config.graceful_shutdown_timeout, - &mut pb, - &mut cb_druid, - ) - .context(GracefulShutdownSnafu)?; - - let metadata_database_connection_details = &cluster.cluster_config.metadata_db_connection; - - let mut main_container_commands = role.main_container_prepare_commands(s3_conn); - let mut prepare_container_commands = vec![]; - if let ValidatedContainerLogConfigChoice::Automatic(log_config) = - &merged_rolegroup_config.logging.prepare_container - { - // This command needs to be added at the beginning of the shell commands, - // otherwise the output of the following commands will not be captured! - prepare_container_commands.push(product_logging::framework::capture_shell_output( - STACKABLE_LOG_DIR, - &prepare_container_name, - log_config, - )); - } - prepare_container_commands.extend(druid_tls_security.build_tls_key_stores_cmd()); - - if let Some(auth_config) = druid_auth_config { - auth_config - .add_volumes_and_mounts(&mut pb, &mut cb_druid, &mut cb_prepare) - .context(AuthVolumesBuildSnafu)?; - prepare_container_commands.extend(auth_config.prepare_container_commands()); - main_container_commands.extend(auth_config.main_container_commands()) - } - - // volume and volume mounts - druid_tls_security - .add_tls_volume_and_volume_mounts( - &mut cb_prepare, - &mut cb_druid, - &mut pb, - &merged_rolegroup_config.requested_secret_lifetime, - // add listener - secret_volume_listener_scope(role), - ) - .context(FailedToInitializeSecurityContextSnafu)?; - - if let Some(s3) = s3_conn { - if s3.tls.uses_tls() && !s3.tls.uses_tls_verification() { - S3TlsNoVerificationNotSupportedSnafu.fail()?; - } - s3.add_volumes_and_mounts(&mut pb, vec![&mut cb_druid]) - .context(ConfigureS3Snafu)?; - } - - add_config_volume_and_volume_mounts(&resource_names, &mut cb_druid, &mut pb)?; - add_log_config_volume_and_volume_mounts( - &resource_names, - merged_rolegroup_config, - &mut cb_druid, - &mut pb, - )?; - add_log_volume_and_volume_mounts(&mut cb_druid, &mut cb_prepare, &mut pb)?; - add_hdfs_cm_volume_and_volume_mounts( - &cluster.cluster_config.deep_storage, - &mut cb_druid, - &mut pb, - )?; - merged_rolegroup_config - .resources - .update_volumes_and_volume_mounts(&mut cb_druid, &mut pb) - .context(UpdateDruidConfigFromResourcesSnafu)?; - - cb_prepare - .image_from_product_image(resolved_product_image) - .command(vec![ - "/bin/bash".to_string(), - "-x".to_string(), - "-euo".to_string(), - "pipefail".to_string(), - "-c".to_string(), - ]) - .args(vec![prepare_container_commands.join("\n")]) - .resources( - ResourceRequirementsBuilder::new() - .with_cpu_request("100m") - .with_cpu_limit("400m") - .with_memory_request("512Mi") - .with_memory_limit("512Mi") - .build(), - ); - - metadata_database_connection_details.add_to_container(&mut cb_druid); - - // rest of env: the validated env overrides, rendered in sorted-by-name order. - let mut rest_env: Vec = rg.env_overrides.clone().into(); - - if let Some(auth_config) = druid_auth_config { - rest_env.extend(auth_config.get_env_var_mounts(cluster, role)) - } - - // Needed for the `containerdebug` process to log it's tracing information to. - rest_env.push(EnvVar { - name: "CONTAINERDEBUG_LOG_DIRECTORY".to_string(), - value: Some(format!("{STACKABLE_LOG_DIR}/containerdebug")), - value_from: None, - }); - - main_container_commands.push(role.main_container_start_command()); - cb_druid - .image_from_product_image(resolved_product_image) - .command(vec![ - "/bin/bash".to_string(), - "-x".to_string(), - "-euo".to_string(), - "pipefail".to_string(), - "-c".to_string(), - ]) - .args(vec![main_container_commands.join("\n")]) - .add_env_vars(rest_env) - .add_container_ports(druid_tls_security.container_ports(role)) - .add_container_port(METRICS_PORT_NAME, METRICS_PORT.into()) - // 10s * 30 = 300s to come up - .startup_probe(druid_tls_security.get_tcp_socket_probe(30, 10, 30, 3)) - // 10s * 1 = 10s to get removed from service - .readiness_probe(druid_tls_security.get_tcp_socket_probe(10, 10, 1, 3)) - // 10s * 3 = 30s to be restarted - .liveness_probe(druid_tls_security.get_tcp_socket_probe(10, 10, 3, 3)) - .resources(merged_rolegroup_config.resources.as_resource_requirements()); - - // Add extra mounts if any are specified and the current role is MiddleManager - // Extra mounts may be needed for ingestion to add required certificates, truststores or similar - // files. - // Mounts are added to all roles, as we are currently unsure where they may be needed - // Known roles are MiddleManagers for ingestion and Historicals for deep storage (GCS plugin) - // We may at some time in the future revisit this and limit it again to avoid needlessly - // propagating potentially confidential files throughout the cluster - for volume in &cluster.cluster_config.extra_volumes { - // Extract values into vars so we make it impossible to log something other than - // what we actually use to create the mounts - maybe paranoid, but hey .. - let volume_name = &volume.name; - let mount_point = format!("{USERDATA_MOUNTPOINT}/{}", volume.name); - - tracing::info!( - ?volume_name, - ?mount_point, - ?role, - "Adding user specified extra volume", - ); - pb.add_volume(volume.clone()).context(AddVolumeSnafu)?; - cb_druid - .add_volume_mount(volume_name, mount_point) - .context(AddVolumeMountSnafu)?; - } - - let mut pvcs: Option> = None; - - if let Some(group_listener_name) = group_listener_name(cluster, role) { - cb_druid - .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) - .context(AddVolumeMountSnafu)?; - - // Used for PVC templates that cannot be modified once they are deployed - let unversioned_recommended_labels = Labels::recommended(&build_recommended_labels( - cluster, - DRUID_CONTROLLER_NAME, - // A version value is required, and we do want to use the "recommended" format for the other desired labels - "none", - &role_name, - role_group_name.as_ref(), - )) - .context(LabelBuildSnafu)?; - - pvcs = Some(vec![ - build_group_listener_pvc(&group_listener_name, &unversioned_recommended_labels) - .context(ListenerConfigurationSnafu)?, - ]); - } - - let metadata = ObjectMetaBuilder::new() - .with_recommended_labels(&build_recommended_labels( - cluster, - DRUID_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - &role_name, - role_group_name.as_ref(), - )) - .context(MetadataBuildSnafu)? - .build(); - - pb.image_pull_secrets_from_product_image(resolved_product_image) - .add_init_container(cb_prepare.build()) - .add_container(cb_druid.build()) - .metadata(metadata) - .service_account_name(service_account.name_any()) - .security_context(PodSecurityContextBuilder::new().fs_group(1000).build()); - - // The Vector agent reads the static `vector.yaml` (added to the rolegroup ConfigMap) from the - // config volume; the validated aggregator address comes from the up-front `ValidatedLogging`. - if let Some(vector_log_config) = &merged_rolegroup_config.logging.vector_container { - pb.add_container(vector_container( - &ContainerName::from_str(&Container::Vector.to_string()) - .expect("'vector' is a valid container name"), - resolved_product_image, - vector_log_config, - &resource_names, - &VolumeName::from_str(DRUID_CONFIG_VOLUME_NAME).expect("a valid volume name"), - &VolumeName::from_str(LOG_VOLUME_NAME).expect("a valid volume name"), - EnvVarSet::new(), - )); - } - - let mut pod_template = pb.build_template(); - // The role and rolegroup pod overrides were already merged (rolegroup wins) during validation. - pod_template.merge_from(rg.pod_overrides.clone()); - - Ok(StatefulSet { - metadata: ObjectMetaBuilder::new() - .name_and_namespace(cluster) - .name(resource_names.stateful_set_name().to_string()) - .ownerreference_from_resource(cluster, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&build_recommended_labels( - cluster, - DRUID_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - &role_name, - role_group_name.as_ref(), - )) - .context(MetadataBuildSnafu)? - .with_label(RESTART_CONTROLLER_ENABLED_LABEL.to_owned()) - .build(), - spec: Some(StatefulSetSpec { - pod_management_policy: Some("Parallel".to_string()), - replicas: Some(i32::from(rg.replicas)), - selector: LabelSelector { - match_labels: Some( - Labels::role_group_selector( - cluster, - APP_NAME, - &role_name, - role_group_name.as_ref(), - ) - .context(LabelBuildSnafu)? - .into(), - ), - ..LabelSelector::default() - }, - service_name: Some(resource_names.headless_service_name().to_string()), - template: pod_template, - volume_claim_templates: pvcs, - ..StatefulSetSpec::default() - }), - status: None, - }) -} - -fn add_hdfs_cm_volume_and_volume_mounts( - deep_storage_spec: &DeepStorageSpec, - cb_druid: &mut ContainerBuilder, - pb: &mut PodBuilder, -) -> Result<()> { - // hdfs deep storage mount - if let DeepStorageSpec::Hdfs(hdfs) = deep_storage_spec { - cb_druid - .add_volume_mount(HDFS_CONFIG_VOLUME_NAME, HDFS_CONFIG_DIRECTORY) - .context(AddVolumeMountSnafu)?; - pb.add_volume( - VolumeBuilder::new(HDFS_CONFIG_VOLUME_NAME) - .with_config_map(&hdfs.config_map_name) - .build(), - ) - .context(AddVolumeSnafu)?; - } - - Ok(()) -} - -fn add_config_volume_and_volume_mounts( - resource_names: &ResourceNames, - cb_druid: &mut ContainerBuilder, - pb: &mut PodBuilder, -) -> Result<()> { - cb_druid - .add_volume_mount(DRUID_CONFIG_VOLUME_NAME, DRUID_CONFIG_DIRECTORY) - .context(AddVolumeMountSnafu)?; - pb.add_volume( - VolumeBuilder::new(DRUID_CONFIG_VOLUME_NAME) - .with_config_map(resource_names.role_group_config_map().to_string()) - .build(), - ) - .context(AddVolumeSnafu)?; - cb_druid - .add_volume_mount(RW_CONFIG_VOLUME_NAME, RW_CONFIG_DIRECTORY) - .context(AddVolumeMountSnafu)?; - pb.add_volume( - VolumeBuilder::new(RW_CONFIG_VOLUME_NAME) - .with_empty_dir(Some(""), None) - .build(), - ) - .context(AddVolumeSnafu)?; - - Ok(()) -} - -fn add_log_config_volume_and_volume_mounts( - resource_names: &ResourceNames, - merged_rolegroup_config: &ValidatedDruidConfig, - cb_druid: &mut ContainerBuilder, - pb: &mut PodBuilder, -) -> Result<()> { - cb_druid - .add_volume_mount(LOG_CONFIG_VOLUME_NAME, LOG_CONFIG_DIRECTORY) - .context(AddVolumeMountSnafu)?; - - let config_map = match &merged_rolegroup_config.logging.druid_container { - ValidatedContainerLogConfigChoice::Custom(config_map_name) => config_map_name.to_string(), - ValidatedContainerLogConfigChoice::Automatic(_) => { - resource_names.role_group_config_map().to_string() - } - }; - - pb.add_volume( - VolumeBuilder::new(LOG_CONFIG_VOLUME_NAME) - .with_config_map(config_map) - .build(), - ) - .context(AddVolumeSnafu)?; - - Ok(()) -} - -fn add_log_volume_and_volume_mounts( - cb_druid: &mut ContainerBuilder, - cb_prepare: &mut ContainerBuilder, - pb: &mut PodBuilder, -) -> Result<()> { - cb_druid - .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) - .context(AddVolumeMountSnafu)?; - cb_prepare - .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) - .context(AddVolumeMountSnafu)?; - pb.add_volume( - VolumeBuilder::new(LOG_VOLUME_NAME) - .with_empty_dir( - Some(""), - Some(product_logging::framework::calculate_log_volume_size_limit( - &[MAX_DRUID_LOG_FILES_SIZE], - )), - ) - .build(), - ) - .context(AddVolumeSnafu)?; - - Ok(()) -} - pub fn error_policy( _obj: Arc>, error: &Error, diff --git a/rust/operator-binary/src/controller/build/resource/mod.rs b/rust/operator-binary/src/controller/build/resource/mod.rs index 512cb299..d7fba99e 100644 --- a/rust/operator-binary/src/controller/build/resource/mod.rs +++ b/rust/operator-binary/src/controller/build/resource/mod.rs @@ -5,3 +5,4 @@ pub mod discovery; pub mod listener; pub mod pdb; pub mod service; +pub mod statefulset; diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs new file mode 100644 index 00000000..d99e3b0f --- /dev/null +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -0,0 +1,520 @@ +//! Builds the rolegroup [`StatefulSet`] from a [`ValidatedCluster`]. + +use std::str::FromStr; + +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::{ + meta::ObjectMetaBuilder, + pod::{ + PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, + security::PodSecurityContextBuilder, volume::VolumeBuilder, + }, + }, + constants::RESTART_CONTROLLER_ENABLED_LABEL, + k8s_openapi::{ + DeepMerge, + api::{ + apps::v1::{StatefulSet, StatefulSetSpec}, + core::v1::{EnvVar, PersistentVolumeClaim, ServiceAccount}, + }, + apimachinery::pkg::apis::meta::v1::LabelSelector, + }, + kube::ResourceExt, + kvp::Labels, + product_logging, + v2::{ + builder::pod::container::EnvVarSet, + product_logging::framework::{ValidatedContainerLogConfigChoice, vector_container}, + role_group_utils::ResourceNames, + types::{ + kubernetes::{ContainerName, VolumeName}, + operator::RoleGroupName, + }, + }, +}; + +use crate::{ + controller::{ + DRUID_CONTROLLER_NAME, + build::{ + properties::product_logging::MAX_DRUID_LOG_FILES_SIZE, + resource::listener::{ + LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_group_listener_pvc, + group_listener_name, secret_volume_listener_scope, + }, + }, + validate::{DruidRoleGroupConfig, ValidatedCluster}, + }, + crd::{ + APP_NAME, Container, DRUID_CONFIG_DIRECTORY, DeepStorageSpec, DruidRole, + HDFS_CONFIG_DIRECTORY, LOG_CONFIG_DIRECTORY, METRICS_PORT, METRICS_PORT_NAME, + RW_CONFIG_DIRECTORY, STACKABLE_LOG_DIR, ValidatedDruidConfig, build_recommended_labels, + }, + operations::graceful_shutdown::add_graceful_shutdown_config, +}; + +// volume names +const DRUID_CONFIG_VOLUME_NAME: &str = "config"; +const HDFS_CONFIG_VOLUME_NAME: &str = "hdfs"; +const LOG_CONFIG_VOLUME_NAME: &str = "log-config"; +const LOG_VOLUME_NAME: &str = "log"; +const RW_CONFIG_VOLUME_NAME: &str = "rwconfig"; +const USERDATA_MOUNTPOINT: &str = "/stackable/userdata"; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to create container builder with name [{name}]"))] + FailedContainerBuilderCreation { + source: stackable_operator::builder::pod::container::Error, + name: String, + }, + + #[snafu(display("failed to configure graceful shutdown"))] + GracefulShutdown { + source: crate::operations::graceful_shutdown::Error, + }, + + #[snafu(display("failed to add OIDC Volumes and VolumeMounts to the Pod and containers"))] + AuthVolumesBuild { + source: crate::authentication::Error, + }, + + #[snafu(display("failed to initialize security context"))] + FailedToInitializeSecurityContext { source: crate::crd::security::Error }, + + #[snafu(display( + "Druid does not support skipping the verification of the tls enabled S3 server" + ))] + S3TlsNoVerificationNotSupported, + + #[snafu(display("failed to configure S3 connection"))] + ConfigureS3 { + source: stackable_operator::crd::s3::v1alpha1::ConnectionError, + }, + + #[snafu(display("failed to update Druid config from resources"))] + UpdateDruidConfigFromResources { source: crate::crd::resource::Error }, + + #[snafu(display("failed to add needed volume"))] + AddVolume { + source: stackable_operator::builder::pod::Error, + }, + + #[snafu(display("failed to add needed volumeMount"))] + AddVolumeMount { + source: stackable_operator::builder::pod::container::Error, + }, + + #[snafu(display("failed to configure listener"))] + ListenerConfiguration { + source: crate::controller::build::resource::listener::Error, + }, + + #[snafu(display("failed to build labels"))] + LabelBuild { + source: stackable_operator::kvp::LabelError, + }, + + #[snafu(display("failed to build metadata"))] + MetadataBuild { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("object is missing metadata to build owner reference"))] + ObjectMissingMetadataForOwnerRef { + source: stackable_operator::builder::meta::Error, + }, +} + +type Result = std::result::Result; + +pub fn build_rolegroup_statefulset( + cluster: &ValidatedCluster, + role: &DruidRole, + role_group_name: &RoleGroupName, + rg: &DruidRoleGroupConfig, + service_account: &ServiceAccount, +) -> Result { + let merged_rolegroup_config = &rg.config; + let role_name = role.to_string(); + let resource_names = cluster.resource_names(role, role_group_name); + // Everything below used to be threaded in as separate parameters; it all lives on the + // `ValidatedCluster` now. + let resolved_product_image = &cluster.image; + let s3_conn = cluster.cluster_config.s3_connection.as_ref(); + let druid_tls_security = &cluster.cluster_config.druid_tls_security; + let druid_auth_config = &cluster.cluster_config.druid_auth_config; + // prepare container builder + let prepare_container_name = Container::Prepare.to_string(); + let mut cb_prepare = ContainerBuilder::new(&prepare_container_name).context( + FailedContainerBuilderCreationSnafu { + name: &prepare_container_name, + }, + )?; + // druid container builder + let druid_container_name = Container::Druid.to_string(); + let mut cb_druid = ContainerBuilder::new(&druid_container_name).context( + FailedContainerBuilderCreationSnafu { + name: &druid_container_name, + }, + )?; + // init pod builder + let mut pb = PodBuilder::new(); + pb.affinity(&merged_rolegroup_config.affinity); + add_graceful_shutdown_config( + role, + druid_tls_security, + merged_rolegroup_config.graceful_shutdown_timeout, + &mut pb, + &mut cb_druid, + ) + .context(GracefulShutdownSnafu)?; + + let metadata_database_connection_details = &cluster.cluster_config.metadata_db_connection; + + let mut main_container_commands = role.main_container_prepare_commands(s3_conn); + let mut prepare_container_commands = vec![]; + if let ValidatedContainerLogConfigChoice::Automatic(log_config) = + &merged_rolegroup_config.logging.prepare_container + { + // This command needs to be added at the beginning of the shell commands, + // otherwise the output of the following commands will not be captured! + prepare_container_commands.push(product_logging::framework::capture_shell_output( + STACKABLE_LOG_DIR, + &prepare_container_name, + log_config, + )); + } + prepare_container_commands.extend(druid_tls_security.build_tls_key_stores_cmd()); + + if let Some(auth_config) = druid_auth_config { + auth_config + .add_volumes_and_mounts(&mut pb, &mut cb_druid, &mut cb_prepare) + .context(AuthVolumesBuildSnafu)?; + prepare_container_commands.extend(auth_config.prepare_container_commands()); + main_container_commands.extend(auth_config.main_container_commands()) + } + + // volume and volume mounts + druid_tls_security + .add_tls_volume_and_volume_mounts( + &mut cb_prepare, + &mut cb_druid, + &mut pb, + &merged_rolegroup_config.requested_secret_lifetime, + // add listener + secret_volume_listener_scope(role), + ) + .context(FailedToInitializeSecurityContextSnafu)?; + + if let Some(s3) = s3_conn { + if s3.tls.uses_tls() && !s3.tls.uses_tls_verification() { + S3TlsNoVerificationNotSupportedSnafu.fail()?; + } + s3.add_volumes_and_mounts(&mut pb, vec![&mut cb_druid]) + .context(ConfigureS3Snafu)?; + } + + add_config_volume_and_volume_mounts(&resource_names, &mut cb_druid, &mut pb)?; + add_log_config_volume_and_volume_mounts( + &resource_names, + merged_rolegroup_config, + &mut cb_druid, + &mut pb, + )?; + add_log_volume_and_volume_mounts(&mut cb_druid, &mut cb_prepare, &mut pb)?; + add_hdfs_cm_volume_and_volume_mounts( + &cluster.cluster_config.deep_storage, + &mut cb_druid, + &mut pb, + )?; + merged_rolegroup_config + .resources + .update_volumes_and_volume_mounts(&mut cb_druid, &mut pb) + .context(UpdateDruidConfigFromResourcesSnafu)?; + + cb_prepare + .image_from_product_image(resolved_product_image) + .command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]) + .args(vec![prepare_container_commands.join("\n")]) + .resources( + ResourceRequirementsBuilder::new() + .with_cpu_request("100m") + .with_cpu_limit("400m") + .with_memory_request("512Mi") + .with_memory_limit("512Mi") + .build(), + ); + + metadata_database_connection_details.add_to_container(&mut cb_druid); + + // rest of env: the validated env overrides, rendered in sorted-by-name order. + let mut rest_env: Vec = rg.env_overrides.clone().into(); + + if let Some(auth_config) = druid_auth_config { + rest_env.extend(auth_config.get_env_var_mounts(cluster, role)) + } + + // Needed for the `containerdebug` process to log it's tracing information to. + rest_env.push(EnvVar { + name: "CONTAINERDEBUG_LOG_DIRECTORY".to_string(), + value: Some(format!("{STACKABLE_LOG_DIR}/containerdebug")), + value_from: None, + }); + + main_container_commands.push(role.main_container_start_command()); + cb_druid + .image_from_product_image(resolved_product_image) + .command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]) + .args(vec![main_container_commands.join("\n")]) + .add_env_vars(rest_env) + .add_container_ports(druid_tls_security.container_ports(role)) + .add_container_port(METRICS_PORT_NAME, METRICS_PORT.into()) + // 10s * 30 = 300s to come up + .startup_probe(druid_tls_security.get_tcp_socket_probe(30, 10, 30, 3)) + // 10s * 1 = 10s to get removed from service + .readiness_probe(druid_tls_security.get_tcp_socket_probe(10, 10, 1, 3)) + // 10s * 3 = 30s to be restarted + .liveness_probe(druid_tls_security.get_tcp_socket_probe(10, 10, 3, 3)) + .resources(merged_rolegroup_config.resources.as_resource_requirements()); + + // Add extra mounts if any are specified and the current role is MiddleManager + // Extra mounts may be needed for ingestion to add required certificates, truststores or similar + // files. + // Mounts are added to all roles, as we are currently unsure where they may be needed + // Known roles are MiddleManagers for ingestion and Historicals for deep storage (GCS plugin) + // We may at some time in the future revisit this and limit it again to avoid needlessly + // propagating potentially confidential files throughout the cluster + for volume in &cluster.cluster_config.extra_volumes { + // Extract values into vars so we make it impossible to log something other than + // what we actually use to create the mounts - maybe paranoid, but hey .. + let volume_name = &volume.name; + let mount_point = format!("{USERDATA_MOUNTPOINT}/{}", volume.name); + + tracing::info!( + ?volume_name, + ?mount_point, + ?role, + "Adding user specified extra volume", + ); + pb.add_volume(volume.clone()).context(AddVolumeSnafu)?; + cb_druid + .add_volume_mount(volume_name, mount_point) + .context(AddVolumeMountSnafu)?; + } + + let mut pvcs: Option> = None; + + if let Some(group_listener_name) = group_listener_name(cluster, role) { + cb_druid + .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) + .context(AddVolumeMountSnafu)?; + + // Used for PVC templates that cannot be modified once they are deployed + let unversioned_recommended_labels = Labels::recommended(&build_recommended_labels( + cluster, + DRUID_CONTROLLER_NAME, + // A version value is required, and we do want to use the "recommended" format for the other desired labels + "none", + &role_name, + role_group_name.as_ref(), + )) + .context(LabelBuildSnafu)?; + + pvcs = Some(vec![ + build_group_listener_pvc(&group_listener_name, &unversioned_recommended_labels) + .context(ListenerConfigurationSnafu)?, + ]); + } + + let metadata = ObjectMetaBuilder::new() + .with_recommended_labels(&build_recommended_labels( + cluster, + DRUID_CONTROLLER_NAME, + &resolved_product_image.app_version_label_value, + &role_name, + role_group_name.as_ref(), + )) + .context(MetadataBuildSnafu)? + .build(); + + pb.image_pull_secrets_from_product_image(resolved_product_image) + .add_init_container(cb_prepare.build()) + .add_container(cb_druid.build()) + .metadata(metadata) + .service_account_name(service_account.name_any()) + .security_context(PodSecurityContextBuilder::new().fs_group(1000).build()); + + // The Vector agent reads the static `vector.yaml` (added to the rolegroup ConfigMap) from the + // config volume; the validated aggregator address comes from the up-front `ValidatedLogging`. + if let Some(vector_log_config) = &merged_rolegroup_config.logging.vector_container { + pb.add_container(vector_container( + &ContainerName::from_str(&Container::Vector.to_string()) + .expect("'vector' is a valid container name"), + resolved_product_image, + vector_log_config, + &resource_names, + &VolumeName::from_str(DRUID_CONFIG_VOLUME_NAME).expect("a valid volume name"), + &VolumeName::from_str(LOG_VOLUME_NAME).expect("a valid volume name"), + EnvVarSet::new(), + )); + } + + let mut pod_template = pb.build_template(); + // The role and rolegroup pod overrides were already merged (rolegroup wins) during validation. + pod_template.merge_from(rg.pod_overrides.clone()); + + Ok(StatefulSet { + metadata: ObjectMetaBuilder::new() + .name_and_namespace(cluster) + .name(resource_names.stateful_set_name().to_string()) + .ownerreference_from_resource(cluster, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(&build_recommended_labels( + cluster, + DRUID_CONTROLLER_NAME, + &resolved_product_image.app_version_label_value, + &role_name, + role_group_name.as_ref(), + )) + .context(MetadataBuildSnafu)? + .with_label(RESTART_CONTROLLER_ENABLED_LABEL.to_owned()) + .build(), + spec: Some(StatefulSetSpec { + pod_management_policy: Some("Parallel".to_string()), + replicas: Some(i32::from(rg.replicas)), + selector: LabelSelector { + match_labels: Some( + Labels::role_group_selector( + cluster, + APP_NAME, + &role_name, + role_group_name.as_ref(), + ) + .context(LabelBuildSnafu)? + .into(), + ), + ..LabelSelector::default() + }, + service_name: Some(resource_names.headless_service_name().to_string()), + template: pod_template, + volume_claim_templates: pvcs, + ..StatefulSetSpec::default() + }), + status: None, + }) +} + +fn add_hdfs_cm_volume_and_volume_mounts( + deep_storage_spec: &DeepStorageSpec, + cb_druid: &mut ContainerBuilder, + pb: &mut PodBuilder, +) -> Result<()> { + // hdfs deep storage mount + if let DeepStorageSpec::Hdfs(hdfs) = deep_storage_spec { + cb_druid + .add_volume_mount(HDFS_CONFIG_VOLUME_NAME, HDFS_CONFIG_DIRECTORY) + .context(AddVolumeMountSnafu)?; + pb.add_volume( + VolumeBuilder::new(HDFS_CONFIG_VOLUME_NAME) + .with_config_map(&hdfs.config_map_name) + .build(), + ) + .context(AddVolumeSnafu)?; + } + + Ok(()) +} + +fn add_config_volume_and_volume_mounts( + resource_names: &ResourceNames, + cb_druid: &mut ContainerBuilder, + pb: &mut PodBuilder, +) -> Result<()> { + cb_druid + .add_volume_mount(DRUID_CONFIG_VOLUME_NAME, DRUID_CONFIG_DIRECTORY) + .context(AddVolumeMountSnafu)?; + pb.add_volume( + VolumeBuilder::new(DRUID_CONFIG_VOLUME_NAME) + .with_config_map(resource_names.role_group_config_map().to_string()) + .build(), + ) + .context(AddVolumeSnafu)?; + cb_druid + .add_volume_mount(RW_CONFIG_VOLUME_NAME, RW_CONFIG_DIRECTORY) + .context(AddVolumeMountSnafu)?; + pb.add_volume( + VolumeBuilder::new(RW_CONFIG_VOLUME_NAME) + .with_empty_dir(Some(""), None) + .build(), + ) + .context(AddVolumeSnafu)?; + + Ok(()) +} + +fn add_log_config_volume_and_volume_mounts( + resource_names: &ResourceNames, + merged_rolegroup_config: &ValidatedDruidConfig, + cb_druid: &mut ContainerBuilder, + pb: &mut PodBuilder, +) -> Result<()> { + cb_druid + .add_volume_mount(LOG_CONFIG_VOLUME_NAME, LOG_CONFIG_DIRECTORY) + .context(AddVolumeMountSnafu)?; + + let config_map = match &merged_rolegroup_config.logging.druid_container { + ValidatedContainerLogConfigChoice::Custom(config_map_name) => config_map_name.to_string(), + ValidatedContainerLogConfigChoice::Automatic(_) => { + resource_names.role_group_config_map().to_string() + } + }; + + pb.add_volume( + VolumeBuilder::new(LOG_CONFIG_VOLUME_NAME) + .with_config_map(config_map) + .build(), + ) + .context(AddVolumeSnafu)?; + + Ok(()) +} + +fn add_log_volume_and_volume_mounts( + cb_druid: &mut ContainerBuilder, + cb_prepare: &mut ContainerBuilder, + pb: &mut PodBuilder, +) -> Result<()> { + cb_druid + .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) + .context(AddVolumeMountSnafu)?; + cb_prepare + .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) + .context(AddVolumeMountSnafu)?; + pb.add_volume( + VolumeBuilder::new(LOG_VOLUME_NAME) + .with_empty_dir( + Some(""), + Some(product_logging::framework::calculate_log_volume_size_limit( + &[MAX_DRUID_LOG_FILES_SIZE], + )), + ) + .build(), + ) + .context(AddVolumeSnafu)?; + + Ok(()) +} From 1d92b97dc5a5ce63c384feb878b3a6988af5eb66 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 15:40:41 +0200 Subject: [PATCH 43/48] test: add config_map unit tests; dedupe trust-store type / derby.log into consts --- .../src/controller/build/jvm.rs | 8 +- .../controller/build/resource/config_map.rs | 79 ++++++++++++++++++- rust/operator-binary/src/crd/mod.rs | 1 + 3 files changed, 84 insertions(+), 4 deletions(-) diff --git a/rust/operator-binary/src/controller/build/jvm.rs b/rust/operator-binary/src/controller/build/jvm.rs index d456de33..995bcdb8 100644 --- a/rust/operator-binary/src/controller/build/jvm.rs +++ b/rust/operator-binary/src/controller/build/jvm.rs @@ -6,8 +6,12 @@ use stackable_operator::{ use super::properties::ConfigFileName; use crate::crd::{ DruidRole, RW_CONFIG_DIRECTORY, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, + STACKABLE_TRUST_STORE_TYPE, }; +/// The Derby error log file, written by the Coordinator's embedded Derby (default metadata store). +const DERBY_LOG_FILE: &str = "/stackable/var/druid/derby.log"; + #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("failed to format memory quantity {value:?} for Java"))] @@ -58,10 +62,10 @@ pub fn construct_jvm_args( format!("-Dlog4j.configurationFile={RW_CONFIG_DIRECTORY}/{log4j2_config_file}"), format!("-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}"), format!("-Djavax.net.ssl.trustStorePassword={STACKABLE_TRUST_STORE_PASSWORD}"), - "-Djavax.net.ssl.trustStoreType=pkcs12".to_owned(), + format!("-Djavax.net.ssl.trustStoreType={STACKABLE_TRUST_STORE_TYPE}"), ]); if druid_role == &DruidRole::Coordinator { - jvm_args.push("-Dderby.stream.error.file=/stackable/var/druid/derby.log".to_owned()); + jvm_args.push(format!("-Dderby.stream.error.file={DERBY_LOG_FILE}")); } Ok(jvm_argument_overrides.apply_to(jvm_args).join("\n")) diff --git a/rust/operator-binary/src/controller/build/resource/config_map.rs b/rust/operator-binary/src/controller/build/resource/config_map.rs index 67237c9c..8c9ee2b6 100644 --- a/rust/operator-binary/src/controller/build/resource/config_map.rs +++ b/rust/operator-binary/src/controller/build/resource/config_map.rs @@ -40,7 +40,8 @@ use crate::{ }, crd::{ DruidConfigOverrides, DruidRole, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, - build_recommended_labels, build_string_list, env_var_reference, file_reference, + STACKABLE_TRUST_STORE_TYPE, build_recommended_labels, build_string_list, env_var_reference, + file_reference, }, }; @@ -113,7 +114,7 @@ fn middlemanager_indexer_java_opts() -> (String, String) { build_string_list(&[ format!("-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}"), format!("-Djavax.net.ssl.trustStorePassword={STACKABLE_TRUST_STORE_PASSWORD}"), - "-Djavax.net.ssl.trustStoreType=pkcs12".to_owned(), + format!("-Djavax.net.ssl.trustStoreType={STACKABLE_TRUST_STORE_TYPE}"), ]), ) } @@ -360,3 +361,77 @@ pub fn build_rolegroup_config_map( role_group: role_group_name.to_string(), }) } + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use stackable_operator::{ + k8s_openapi::api::core::v1::ConfigMap, v2::types::operator::RoleGroupName, + }; + + use super::*; + use crate::controller::validate::test_support::{ + MINIMAL_DRUID_YAML, druid_from_yaml, validated_cluster, + }; + + /// Builds the rolegroup `ConfigMap` for `/` from the minimal test fixture. + fn build_cm(role: DruidRole, role_group: &str) -> ConfigMap { + let druid = druid_from_yaml(MINIMAL_DRUID_YAML); + let cluster = validated_cluster(&druid); + let role_group_name = RoleGroupName::from_str(role_group).unwrap(); + let rg = cluster + .role_group_configs + .get(&role) + .expect("role present") + .get(&role_group_name) + .expect("role group present") + .clone(); + build_rolegroup_config_map(&cluster, &role, &role_group_name, &rg).expect("config map") + } + + #[test] + fn contains_the_operator_written_config_files() { + let cm = build_cm(DruidRole::Broker, "default"); + let data = cm.data.expect("config map has data"); + + // Always-present operator-written files. + assert!(data.contains_key(&ConfigFileName::RuntimeProperties.to_string())); + assert!(data.contains_key(&ConfigFileName::SecurityProperties.to_string())); + assert!(data.contains_key(&ConfigFileName::JvmConfig.to_string())); + // Automatic logging is the default, so log4j2 is rendered. + assert!(data.contains_key(&ConfigFileName::Log4j2Properties.to_string())); + // The Vector agent is disabled by default, so no `vector.yaml` is added. + assert!(!data.contains_key(VECTOR_CONFIG_FILE)); + } + + #[test] + fn has_the_expected_name_and_recommended_labels() { + let cm = build_cm(DruidRole::Broker, "default"); + let meta = cm.metadata; + + assert_eq!(meta.name.as_deref(), Some("simple-druid-broker-default")); + + let labels = meta.labels.expect("recommended labels"); + assert_eq!( + labels.get("app.kubernetes.io/name").map(String::as_str), + Some("druid") + ); + assert_eq!( + labels.get("app.kubernetes.io/instance").map(String::as_str), + Some("simple-druid") + ); + assert_eq!( + labels + .get("app.kubernetes.io/component") + .map(String::as_str), + Some("broker") + ); + assert_eq!( + labels + .get("app.kubernetes.io/role-group") + .map(String::as_str), + Some("default") + ); + } +} diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 1929c043..9a9d3589 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -81,6 +81,7 @@ pub const RW_CONFIG_DIRECTORY: &str = "/stackable/rwconfig"; // store directories pub const STACKABLE_TRUST_STORE: &str = "/stackable/truststore.p12"; pub const STACKABLE_TRUST_STORE_PASSWORD: &str = "changeit"; +pub const STACKABLE_TRUST_STORE_TYPE: &str = "pkcs12"; pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; pub const PROP_SEGMENT_CACHE_LOCATIONS: &str = "druid.segmentCache.locations"; From 52104230de97fc9799343f955732e9d1507e2767 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 19:57:13 +0200 Subject: [PATCH 44/48] refactor: use v2 container builder --- .../controller/build/resource/statefulset.rs | 28 ++++++------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index d99e3b0f..21581ac5 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -24,7 +24,7 @@ use stackable_operator::{ kvp::Labels, product_logging, v2::{ - builder::pod::container::EnvVarSet, + builder::pod::container::{EnvVarSet, new_container_builder}, product_logging::framework::{ValidatedContainerLogConfigChoice, vector_container}, role_group_utils::ResourceNames, types::{ @@ -64,12 +64,6 @@ const USERDATA_MOUNTPOINT: &str = "/stackable/userdata"; #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("failed to create container builder with name [{name}]"))] - FailedContainerBuilderCreation { - source: stackable_operator::builder::pod::container::Error, - name: String, - }, - #[snafu(display("failed to configure graceful shutdown"))] GracefulShutdown { source: crate::operations::graceful_shutdown::Error, @@ -146,19 +140,13 @@ pub fn build_rolegroup_statefulset( let druid_tls_security = &cluster.cluster_config.druid_tls_security; let druid_auth_config = &cluster.cluster_config.druid_auth_config; // prepare container builder - let prepare_container_name = Container::Prepare.to_string(); - let mut cb_prepare = ContainerBuilder::new(&prepare_container_name).context( - FailedContainerBuilderCreationSnafu { - name: &prepare_container_name, - }, - )?; + let prepare_container_name = ContainerName::from_str(&Container::Prepare.to_string()) + .expect("'prepare' is a valid container name"); + let mut cb_prepare = new_container_builder(&prepare_container_name); // druid container builder - let druid_container_name = Container::Druid.to_string(); - let mut cb_druid = ContainerBuilder::new(&druid_container_name).context( - FailedContainerBuilderCreationSnafu { - name: &druid_container_name, - }, - )?; + let druid_container_name = ContainerName::from_str(&Container::Druid.to_string()) + .expect("'druid' is a valid container name"); + let mut cb_druid = new_container_builder(&druid_container_name); // init pod builder let mut pb = PodBuilder::new(); pb.affinity(&merged_rolegroup_config.affinity); @@ -182,7 +170,7 @@ pub fn build_rolegroup_statefulset( // otherwise the output of the following commands will not be captured! prepare_container_commands.push(product_logging::framework::capture_shell_output( STACKABLE_LOG_DIR, - &prepare_container_name, + prepare_container_name.as_ref(), log_config, )); } From fa9c56d90c26297fdc7340b95b493cf1230a7e43 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 20:01:34 +0200 Subject: [PATCH 45/48] refactor: use v2 cluster resources --- rust/operator-binary/src/controller.rs | 28 +++++++++---------- .../src/controller/validate.rs | 2 ++ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 509275b0..ac2135bd 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -7,10 +7,9 @@ use const_format::concatcp; use snafu::{ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, - cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, + cluster_resources::ClusterResourceApplyStrategy, commons::rbac::build_rbac_resources, kube::{ - Resource, core::{DeserializeGuard, error_boundary}, runtime::controller::Action, }, @@ -21,7 +20,10 @@ use stackable_operator::{ compute_conditions, operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, }, - v2::types::operator::{ControllerName, OperatorName, ProductName}, + v2::{ + cluster_resources::cluster_resources_new, + types::operator::{ControllerName, OperatorName, ProductName}, + }, }; use strum::{EnumDiscriminants, IntoStaticStr}; @@ -111,11 +113,6 @@ pub enum Error { source: stackable_operator::client::Error, }, - #[snafu(display("failed to create cluster resources"))] - CreateClusterResources { - source: stackable_operator::cluster_resources::Error, - }, - #[snafu(display("failed to delete orphaned resources"))] DeleteOrphanedResources { source: stackable_operator::cluster_resources::Error, @@ -209,15 +206,16 @@ pub async fn reconcile_druid( validate::validate(druid, &dereferenced_objects, &ctx.operator_environment) .context(ValidateClusterSnafu)?; - let mut cluster_resources = ClusterResources::new( - APP_NAME, - OPERATOR_NAME, - DRUID_CONTROLLER_NAME, - &druid.object_ref(&()), + let mut cluster_resources = cluster_resources_new( + &product_name(), + &operator_name(), + &controller_name(), + &validated_cluster.name, + &validated_cluster.namespace, + &validated_cluster.uid, ClusterResourceApplyStrategy::from(&druid.spec.cluster_operation), &druid.spec.object_overrides, - ) - .context(CreateClusterResourcesSnafu)?; + ); let (rbac_sa, rbac_rolebinding) = build_rbac_resources( druid, diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 8f257439..fdbadced 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -103,6 +103,7 @@ pub struct ValidatedCluster { /// [`Resource`] and be passed directly to the metadata/owner-reference builders. metadata: ObjectMeta, pub name: ClusterName, + pub namespace: NamespaceName, pub uid: Uid, pub image: ResolvedProductImage, pub cluster_config: ValidatedClusterConfig, @@ -128,6 +129,7 @@ impl ValidatedCluster { Self { metadata, name, + namespace, uid, image, cluster_config, From 9cea43cc735ab5c6bbc891433f504b39498d72a3 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 20:05:38 +0200 Subject: [PATCH 46/48] refactor: use v2 ListenerName --- .../src/controller/build/resource/listener.rs | 50 +++++++++++-------- .../controller/build/resource/statefulset.rs | 13 ++--- 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/rust/operator-binary/src/controller/build/resource/listener.rs b/rust/operator-binary/src/controller/build/resource/listener.rs index c2025e9c..d1149293 100644 --- a/rust/operator-binary/src/controller/build/resource/listener.rs +++ b/rust/operator-binary/src/controller/build/resource/listener.rs @@ -1,12 +1,17 @@ +use std::str::FromStr; + use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ - builder::{ - meta::ObjectMetaBuilder, - pod::volume::{ListenerOperatorVolumeSourceBuilder, ListenerReference}, - }, + builder::meta::ObjectMetaBuilder, crd::listener::{self, v1alpha1::Listener}, k8s_openapi::api::core::v1::PersistentVolumeClaim, kvp::{Labels, ObjectLabels}, + v2::{ + builder::pod::volume::{ + ListenerReference, listener_operator_volume_source_builder_build_pvc, + }, + types::kubernetes::{ListenerName, PersistentVolumeClaimName}, + }, }; use crate::{ @@ -32,11 +37,6 @@ pub enum Error { source: stackable_operator::builder::meta::Error, }, - #[snafu(display("failed to build listener volume"))] - BuildListenerPersistentVolume { - source: stackable_operator::builder::pod::volume::ListenerOperatorVolumeSourceBuilderError, - }, - #[snafu(display("{role_name} listener has no adress"))] RoleListenerHasNoAddress { role_name: String }, @@ -51,13 +51,13 @@ pub fn build_group_listener( cluster: &ValidatedCluster, object_labels: ObjectLabels, listener_class: String, - listener_group_name: String, + listener_group_name: ListenerName, druid_role: &DruidRole, ) -> Result { Ok(Listener { metadata: ObjectMetaBuilder::new() .name_and_namespace(cluster) - .name(listener_group_name) + .name(listener_group_name.to_string()) .ownerreference_from_resource(cluster, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(&object_labels) @@ -78,23 +78,29 @@ pub fn build_group_listener( } pub fn build_group_listener_pvc( - group_listener_name: &String, + group_listener_name: &ListenerName, unversioned_recommended_labels: &Labels, -) -> Result { - ListenerOperatorVolumeSourceBuilder::new( - &ListenerReference::ListenerName(group_listener_name.to_string()), +) -> PersistentVolumeClaim { + listener_operator_volume_source_builder_build_pvc( + &ListenerReference::Listener(group_listener_name.clone()), unversioned_recommended_labels, + &PersistentVolumeClaimName::from_str(LISTENER_VOLUME_NAME) + .expect("a valid persistent volume claim name"), ) - .build_pvc(LISTENER_VOLUME_NAME.to_string()) - .context(BuildListenerPersistentVolumeSnafu) } -pub fn group_listener_name(cluster: &ValidatedCluster, druid_role: &DruidRole) -> Option { +pub fn group_listener_name( + cluster: &ValidatedCluster, + druid_role: &DruidRole, +) -> Option { match druid_role { - DruidRole::Coordinator | DruidRole::Broker | DruidRole::Router => Some(format!( - "{cluster_name}-{druid_role}", - cluster_name = cluster.name, - )), + DruidRole::Coordinator | DruidRole::Broker | DruidRole::Router => Some( + ListenerName::from_str(&format!( + "{cluster_name}-{druid_role}", + cluster_name = cluster.name, + )) + .expect("a valid listener name"), + ), DruidRole::Historical | DruidRole::MiddleManager => None, } } diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 21581ac5..b341d3ad 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -100,11 +100,6 @@ pub enum Error { source: stackable_operator::builder::pod::container::Error, }, - #[snafu(display("failed to configure listener"))] - ListenerConfiguration { - source: crate::controller::build::resource::listener::Error, - }, - #[snafu(display("failed to build labels"))] LabelBuild { source: stackable_operator::kvp::LabelError, @@ -322,10 +317,10 @@ pub fn build_rolegroup_statefulset( )) .context(LabelBuildSnafu)?; - pvcs = Some(vec![ - build_group_listener_pvc(&group_listener_name, &unversioned_recommended_labels) - .context(ListenerConfigurationSnafu)?, - ]); + pvcs = Some(vec![build_group_listener_pvc( + &group_listener_name, + &unversioned_recommended_labels, + )]); } let metadata = ObjectMetaBuilder::new() From ff9b9b5035ba73443377888a35664183c20c2c88 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 20:29:50 +0200 Subject: [PATCH 47/48] refactor: use v2 ownerref, labes --- rust/operator-binary/src/controller.rs | 43 +++---- .../controller/build/resource/config_map.rs | 32 +++-- .../controller/build/resource/discovery.rs | 33 +++--- .../src/controller/build/resource/listener.rs | 33 ++---- .../src/controller/build/resource/service.rs | 110 +++++++++--------- .../controller/build/resource/statefulset.rs | 91 +++++++-------- rust/operator-binary/src/crd/mod.rs | 31 ++--- rust/operator-binary/src/main.rs | 2 +- 8 files changed, 166 insertions(+), 209 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index ac2135bd..42b69d3a 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -22,7 +22,10 @@ use stackable_operator::{ }, v2::{ cluster_resources::cluster_resources_new, - types::operator::{ControllerName, OperatorName, ProductName}, + kvp::label::recommended_labels, + types::operator::{ + ControllerName, OperatorName, ProductName, ProductVersion, RoleGroupName, + }, }, }; use strum::{EnumDiscriminants, IntoStaticStr}; @@ -33,9 +36,7 @@ use crate::{ pdb::build_pdb, service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, }, - crd::{ - APP_NAME, DruidClusterStatus, DruidRole, OPERATOR_NAME, build_recommended_labels, v1alpha1, - }, + crd::{APP_NAME, DruidClusterStatus, DruidRole, OPERATOR_NAME, v1alpha1}, internal_secret::create_shared_internal_secret, }; @@ -158,16 +159,6 @@ pub enum Error { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("failed to configure listener"))] - ListenerConfiguration { - source: crate::controller::build::resource::listener::Error, - }, - - #[snafu(display("failed to configure service"))] - ServiceConfiguration { - source: crate::controller::build::resource::service::Error, - }, - #[snafu(display("failed to validate cluster"))] ValidateCluster { source: validate::Error }, @@ -238,19 +229,15 @@ pub async fn reconcile_druid( let mut ss_cond_builder = StatefulSetConditionBuilder::default(); for (druid_role, groups) in validated_cluster.role_group_configs.iter() { - let role_name = druid_role.to_string(); - create_shared_internal_secret(druid, client, DRUID_CONTROLLER_NAME) .await .context(FailedInternalSecretCreationSnafu)?; for (rolegroup_name, rg) in groups.iter() { let rg_headless_service = - build_rolegroup_headless_service(&validated_cluster, druid_role, rolegroup_name) - .context(ServiceConfigurationSnafu)?; + build_rolegroup_headless_service(&validated_cluster, druid_role, rolegroup_name); let rg_metrics_service = - build_rolegroup_metrics_service(&validated_cluster, druid_role, rolegroup_name) - .context(ServiceConfigurationSnafu)?; + build_rolegroup_metrics_service(&validated_cluster, druid_role, rolegroup_name); let rg_configmap = build::resource::config_map::build_rolegroup_config_map( &validated_cluster, @@ -305,18 +292,20 @@ pub async fn reconcile_druid( { let role_group_listener = build_group_listener( &validated_cluster, - build_recommended_labels( + recommended_labels( &validated_cluster, - DRUID_CONTROLLER_NAME, - &validated_cluster.image.app_version_label_value, - &role_name, - "none", + &product_name(), + &ProductVersion::from_str(&validated_cluster.image.app_version_label_value) + .expect("a valid product version"), + &operator_name(), + &controller_name(), + &druid_role.to_role_name(), + &RoleGroupName::from_str("none").expect("a valid role group name"), ), listener_class.to_string(), listener_group_name, druid_role, - ) - .context(ListenerConfigurationSnafu)?; + ); let listener = cluster_resources .add(client, role_group_listener) diff --git a/rust/operator-binary/src/controller/build/resource/config_map.rs b/rust/operator-binary/src/controller/build/resource/config_map.rs index 8c9ee2b6..00a8a383 100644 --- a/rust/operator-binary/src/controller/build/resource/config_map.rs +++ b/rust/operator-binary/src/controller/build/resource/config_map.rs @@ -11,7 +11,7 @@ //! The builder does not read the raw [`v1alpha1::DruidCluster`] at all: everything it needs is //! carried on `ValidatedCluster` (resolved during the validate step). -use std::collections::BTreeMap; +use std::{collections::BTreeMap, str::FromStr}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ @@ -20,14 +20,15 @@ use stackable_operator::{ k8s_openapi::api::core::v1::{ConfigMap, EnvVar}, product_logging::framework::VECTOR_CONFIG_FILE, v2::{ - builder::meta::ownerreference_from_resource, config_file_writer::to_java_properties_string, - types::operator::RoleGroupName, + builder::meta::ownerreference_from_resource, + config_file_writer::to_java_properties_string, + kvp::label::recommended_labels, + types::operator::{ProductVersion, RoleGroupName}, }, }; use crate::{ controller::{ - DRUID_CONTROLLER_NAME, build::{ jvm::construct_jvm_args, properties::{ @@ -36,12 +37,12 @@ use crate::{ runtime_properties, security_properties, }, }, + controller_name, operator_name, product_name, validate::{DruidRoleGroupConfig, ValidatedCluster}, }, crd::{ DruidConfigOverrides, DruidRole, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, - STACKABLE_TRUST_STORE_TYPE, build_recommended_labels, build_string_list, env_var_reference, - file_reference, + STACKABLE_TRUST_STORE_TYPE, build_string_list, env_var_reference, file_reference, }, }; @@ -83,11 +84,6 @@ pub enum Error { #[snafu(display("failed to update Druid config from resources"))] UpdateDruidConfigFromResources { source: crate::crd::resource::Error }, - #[snafu(display("failed to build metadata"))] - MetadataBuild { - source: stackable_operator::builder::meta::Error, - }, - #[snafu(display("there was an error generating the authentication runtime settings"))] GenerateAuthenticationRuntimeSettings { source: crate::authentication::Error, @@ -330,14 +326,16 @@ pub fn build_rolegroup_config_map( .name_and_namespace(cluster) .name(resource_names.role_group_config_map().to_string()) .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) - .with_recommended_labels(&build_recommended_labels( + .with_labels(recommended_labels( cluster, - DRUID_CONTROLLER_NAME, - &cluster.image.app_version_label_value, - &role.to_string(), - role_group_name.as_ref(), + &product_name(), + &ProductVersion::from_str(&cluster.image.app_version_label_value) + .expect("a valid product version"), + &operator_name(), + &controller_name(), + &role.to_role_name(), + role_group_name, )) - .context(MetadataBuildSnafu)? .build(), ); diff --git a/rust/operator-binary/src/controller/build/resource/discovery.rs b/rust/operator-binary/src/controller/build/resource/discovery.rs index ccda3887..512922a8 100644 --- a/rust/operator-binary/src/controller/build/resource/discovery.rs +++ b/rust/operator-binary/src/controller/build/resource/discovery.rs @@ -1,20 +1,26 @@ //! Discovery for Druid. We make Druid discoverable by putting a connection string to the router service //! inside a config map. We only provide a connection string to the router service, since it serves as //! a gateway to the cluster for client queries. +use std::str::FromStr; + use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, crd::listener::v1alpha1::Listener, k8s_openapi::api::core::v1::ConfigMap, - v2::builder::meta::ownerreference_from_resource, + v2::{ + builder::meta::ownerreference_from_resource, + kvp::label::recommended_labels, + types::operator::{ProductVersion, RoleGroupName}, + }, }; use crate::{ - DRUID_CONTROLLER_NAME, controller::{ - build::resource::listener::build_listener_connection_string, validate::ValidatedCluster, + build::resource::listener::build_listener_connection_string, controller_name, + operator_name, product_name, validate::ValidatedCluster, }, - crd::{DruidRole, build_recommended_labels}, + crd::DruidRole, }; #[derive(Snafu, Debug)] @@ -24,11 +30,6 @@ pub enum Error { source: stackable_operator::builder::configmap::Error, }, - #[snafu(display("failed to add recommended labels"))] - AddRecommendedLabels { - source: stackable_operator::builder::meta::Error, - }, - #[snafu(display("failed to configure listener discovery configmap"))] ListenerConfiguration { source: crate::controller::build::resource::listener::Error, @@ -65,14 +66,16 @@ fn build_discovery_configmap( ObjectMetaBuilder::new() .name_and_namespace(cluster) .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) - .with_recommended_labels(&build_recommended_labels( + .with_labels(recommended_labels( cluster, - DRUID_CONTROLLER_NAME, - &cluster.image.app_version_label_value, - &DruidRole::Router.to_string(), - "discovery", + &product_name(), + &ProductVersion::from_str(&cluster.image.app_version_label_value) + .expect("a valid product version"), + &operator_name(), + &controller_name(), + &DruidRole::Router.to_role_name(), + &RoleGroupName::from_str("discovery").expect("a valid role group name"), )) - .context(AddRecommendedLabelsSnafu)? .build(), ) .add_data("DRUID_ROUTER", router_host) diff --git a/rust/operator-binary/src/controller/build/resource/listener.rs b/rust/operator-binary/src/controller/build/resource/listener.rs index d1149293..cd33aa6d 100644 --- a/rust/operator-binary/src/controller/build/resource/listener.rs +++ b/rust/operator-binary/src/controller/build/resource/listener.rs @@ -1,14 +1,15 @@ use std::str::FromStr; -use snafu::{OptionExt, ResultExt, Snafu}; +use snafu::{OptionExt, Snafu}; use stackable_operator::{ builder::meta::ObjectMetaBuilder, crd::listener::{self, v1alpha1::Listener}, k8s_openapi::api::core::v1::PersistentVolumeClaim, - kvp::{Labels, ObjectLabels}, + kvp::Labels, v2::{ - builder::pod::volume::{ - ListenerReference, listener_operator_volume_source_builder_build_pvc, + builder::{ + meta::ownerreference_from_resource, + pod::volume::{ListenerReference, listener_operator_volume_source_builder_build_pvc}, }, types::kubernetes::{ListenerName, PersistentVolumeClaimName}, }, @@ -27,16 +28,6 @@ pub const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("listener object is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, - }, - - #[snafu(display("failed to build listener object meta data"))] - BuildObjectMeta { - source: stackable_operator::builder::meta::Error, - }, - #[snafu(display("{role_name} listener has no adress"))] RoleListenerHasNoAddress { role_name: String }, @@ -49,19 +40,17 @@ pub enum Error { pub fn build_group_listener( cluster: &ValidatedCluster, - object_labels: ObjectLabels, + object_labels: Labels, listener_class: String, listener_group_name: ListenerName, druid_role: &DruidRole, -) -> Result { - Ok(Listener { +) -> Listener { + Listener { metadata: ObjectMetaBuilder::new() .name_and_namespace(cluster) .name(listener_group_name.to_string()) - .ownerreference_from_resource(cluster, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&object_labels) - .context(BuildObjectMetaSnafu)? + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(object_labels) .build(), spec: listener::v1alpha1::ListenerSpec { class_name: Some(listener_class), @@ -74,7 +63,7 @@ pub fn build_group_listener( ..listener::v1alpha1::ListenerSpec::default() }, status: None, - }) + } } pub fn build_group_listener_pvc( diff --git a/rust/operator-binary/src/controller/build/resource/service.rs b/rust/operator-binary/src/controller/build/resource/service.rs index 388bf9e8..a103e737 100644 --- a/rust/operator-binary/src/controller/build/resource/service.rs +++ b/rust/operator-binary/src/controller/build/resource/service.rs @@ -1,53 +1,45 @@ -use snafu::{ResultExt, Snafu}; +use std::str::FromStr; + use stackable_operator::{ builder::meta::ObjectMetaBuilder, k8s_openapi::api::core::v1::{Service, ServicePort, ServiceSpec}, - kvp::{Annotations, Label, Labels}, - v2::types::operator::RoleGroupName, + kvp::{Annotations, Labels}, + v2::{ + builder::meta::ownerreference_from_resource, + kvp::label::{recommended_labels, role_group_selector}, + types::operator::{ProductVersion, RoleGroupName}, + }, }; use crate::{ - controller::{DRUID_CONTROLLER_NAME, validate::ValidatedCluster}, - crd::{APP_NAME, DruidRole, METRICS_PORT, METRICS_PORT_NAME, build_recommended_labels}, + controller::{controller_name, operator_name, product_name, validate::ValidatedCluster}, + crd::{DruidRole, METRICS_PORT, METRICS_PORT_NAME}, }; -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("object is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, - }, - - #[snafu(display("failed to build Metadata"))] - MetadataBuild { - source: stackable_operator::builder::meta::Error, - }, - - #[snafu(display("failed to build Labels"))] - LabelBuild { - source: stackable_operator::kvp::LabelError, - }, -} - /// The rolegroup headless [`Service`] is a service that allows direct access to the instances of a certain rolegroup /// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. pub fn build_rolegroup_headless_service( cluster: &ValidatedCluster, druid_role: &DruidRole, role_group_name: &RoleGroupName, -) -> Result { - let role_name = druid_role.to_string(); - let object_labels = build_recommended_labels( +) -> Service { + let object_labels = recommended_labels( + cluster, + &product_name(), + &ProductVersion::from_str(&cluster.image.app_version_label_value) + .expect("a valid product version"), + &operator_name(), + &controller_name(), + &druid_role.to_role_name(), + role_group_name, + ); + let selector = role_group_selector( cluster, - DRUID_CONTROLLER_NAME, - &cluster.image.app_version_label_value, - &role_name, - role_group_name.as_ref(), + &product_name(), + &druid_role.to_role_name(), + role_group_name, ); - let selector = - Labels::role_group_selector(cluster, APP_NAME, &role_name, role_group_name.as_ref()) - .context(LabelBuildSnafu)?; - Ok(Service { + Service { metadata: ObjectMetaBuilder::new() .name_and_namespace(cluster) .name( @@ -56,10 +48,8 @@ pub fn build_rolegroup_headless_service( .headless_service_name() .to_string(), ) - .ownerreference_from_resource(cluster, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&object_labels) - .context(MetadataBuildSnafu)? + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(object_labels) .build(), spec: Some(ServiceSpec { // Internal communication does not need to be exposed @@ -76,7 +66,7 @@ pub fn build_rolegroup_headless_service( ..ServiceSpec::default() }), status: None, - }) + } } /// The rolegroup metrics [`Service`] is a service that exposes metrics and a prometheus scraping label. @@ -84,19 +74,24 @@ pub fn build_rolegroup_metrics_service( cluster: &ValidatedCluster, druid_role: &DruidRole, role_group_name: &RoleGroupName, -) -> Result { - let role_name = druid_role.to_string(); - let object_labels = build_recommended_labels( +) -> Service { + let object_labels = recommended_labels( cluster, - DRUID_CONTROLLER_NAME, - &cluster.image.app_version_label_value, - &role_name, - role_group_name.as_ref(), + &product_name(), + &ProductVersion::from_str(&cluster.image.app_version_label_value) + .expect("a valid product version"), + &operator_name(), + &controller_name(), + &druid_role.to_role_name(), + role_group_name, ); - let selector = - Labels::role_group_selector(cluster, APP_NAME, &role_name, role_group_name.as_ref()) - .context(LabelBuildSnafu)?; - Ok(Service { + let selector = role_group_selector( + cluster, + &product_name(), + &druid_role.to_role_name(), + role_group_name, + ); + Service { metadata: ObjectMetaBuilder::new() .name_and_namespace(cluster) .name( @@ -105,11 +100,9 @@ pub fn build_rolegroup_metrics_service( .metrics_service_name() .to_string(), ) - .ownerreference_from_resource(cluster, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&object_labels) - .context(MetadataBuildSnafu)? - .with_label(Label::try_from(("prometheus.io/scrape", "true")).context(LabelBuildSnafu)?) + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(object_labels) + .with_labels(prometheus_labels()) .with_annotations(prometheus_annotations()) .build(), spec: Some(ServiceSpec { @@ -122,7 +115,7 @@ pub fn build_rolegroup_metrics_service( ..ServiceSpec::default() }), status: None, - }) + } } fn metrics_service_ports() -> Vec { @@ -134,6 +127,11 @@ fn metrics_service_ports() -> Vec { }] } +/// The Prometheus scraping label added to the metrics [`Service`]. +fn prometheus_labels() -> Labels { + Labels::try_from([("prometheus.io/scrape", "true")]).expect("should be a valid label") +} + /// Common annotations for Prometheus /// /// These annotations can be used in a ServiceMonitor. diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index b341d3ad..79bd3329 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -21,22 +21,24 @@ use stackable_operator::{ apimachinery::pkg::apis::meta::v1::LabelSelector, }, kube::ResourceExt, - kvp::Labels, product_logging, v2::{ - builder::pod::container::{EnvVarSet, new_container_builder}, + builder::{ + meta::ownerreference_from_resource, + pod::container::{EnvVarSet, new_container_builder}, + }, + kvp::label::{recommended_labels, role_group_selector}, product_logging::framework::{ValidatedContainerLogConfigChoice, vector_container}, role_group_utils::ResourceNames, types::{ kubernetes::{ContainerName, VolumeName}, - operator::RoleGroupName, + operator::{ProductVersion, RoleGroupName}, }, }, }; use crate::{ controller::{ - DRUID_CONTROLLER_NAME, build::{ properties::product_logging::MAX_DRUID_LOG_FILES_SIZE, resource::listener::{ @@ -44,12 +46,13 @@ use crate::{ group_listener_name, secret_volume_listener_scope, }, }, + controller_name, operator_name, product_name, validate::{DruidRoleGroupConfig, ValidatedCluster}, }, crd::{ - APP_NAME, Container, DRUID_CONFIG_DIRECTORY, DeepStorageSpec, DruidRole, - HDFS_CONFIG_DIRECTORY, LOG_CONFIG_DIRECTORY, METRICS_PORT, METRICS_PORT_NAME, - RW_CONFIG_DIRECTORY, STACKABLE_LOG_DIR, ValidatedDruidConfig, build_recommended_labels, + Container, DRUID_CONFIG_DIRECTORY, DeepStorageSpec, DruidRole, HDFS_CONFIG_DIRECTORY, + LOG_CONFIG_DIRECTORY, METRICS_PORT, METRICS_PORT_NAME, RW_CONFIG_DIRECTORY, + STACKABLE_LOG_DIR, ValidatedDruidConfig, }, operations::graceful_shutdown::add_graceful_shutdown_config, }; @@ -99,21 +102,6 @@ pub enum Error { AddVolumeMount { source: stackable_operator::builder::pod::container::Error, }, - - #[snafu(display("failed to build labels"))] - LabelBuild { - source: stackable_operator::kvp::LabelError, - }, - - #[snafu(display("failed to build metadata"))] - MetadataBuild { - source: stackable_operator::builder::meta::Error, - }, - - #[snafu(display("object is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, - }, } type Result = std::result::Result; @@ -126,7 +114,6 @@ pub fn build_rolegroup_statefulset( service_account: &ServiceAccount, ) -> Result { let merged_rolegroup_config = &rg.config; - let role_name = role.to_string(); let resource_names = cluster.resource_names(role, role_group_name); // Everything below used to be threaded in as separate parameters; it all lives on the // `ValidatedCluster` now. @@ -307,15 +294,17 @@ pub fn build_rolegroup_statefulset( .context(AddVolumeMountSnafu)?; // Used for PVC templates that cannot be modified once they are deployed - let unversioned_recommended_labels = Labels::recommended(&build_recommended_labels( + // A version value is required, and we do want to use the "recommended" format for the + // other desired labels, hence the "none" product version. + let unversioned_recommended_labels = recommended_labels( cluster, - DRUID_CONTROLLER_NAME, - // A version value is required, and we do want to use the "recommended" format for the other desired labels - "none", - &role_name, - role_group_name.as_ref(), - )) - .context(LabelBuildSnafu)?; + &product_name(), + &ProductVersion::from_str("none").expect("a valid product version"), + &operator_name(), + &controller_name(), + &role.to_role_name(), + role_group_name, + ); pvcs = Some(vec![build_group_listener_pvc( &group_listener_name, @@ -324,14 +313,16 @@ pub fn build_rolegroup_statefulset( } let metadata = ObjectMetaBuilder::new() - .with_recommended_labels(&build_recommended_labels( + .with_labels(recommended_labels( cluster, - DRUID_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - &role_name, - role_group_name.as_ref(), + &product_name(), + &ProductVersion::from_str(&resolved_product_image.app_version_label_value) + .expect("a valid product version"), + &operator_name(), + &controller_name(), + &role.to_role_name(), + role_group_name, )) - .context(MetadataBuildSnafu)? .build(); pb.image_pull_secrets_from_product_image(resolved_product_image) @@ -364,16 +355,17 @@ pub fn build_rolegroup_statefulset( metadata: ObjectMetaBuilder::new() .name_and_namespace(cluster) .name(resource_names.stateful_set_name().to_string()) - .ownerreference_from_resource(cluster, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&build_recommended_labels( + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(recommended_labels( cluster, - DRUID_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - &role_name, - role_group_name.as_ref(), + &product_name(), + &ProductVersion::from_str(&resolved_product_image.app_version_label_value) + .expect("a valid product version"), + &operator_name(), + &controller_name(), + &role.to_role_name(), + role_group_name, )) - .context(MetadataBuildSnafu)? .with_label(RESTART_CONTROLLER_ENABLED_LABEL.to_owned()) .build(), spec: Some(StatefulSetSpec { @@ -381,13 +373,12 @@ pub fn build_rolegroup_statefulset( replicas: Some(i32::from(rg.replicas)), selector: LabelSelector { match_labels: Some( - Labels::role_group_selector( + role_group_selector( cluster, - APP_NAME, - &role_name, - role_group_name.as_ref(), + &product_name(), + &role.to_role_name(), + role_group_name, ) - .context(LabelBuildSnafu)? .into(), ), ..LabelSelector::default() diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 9a9d3589..e0a4198e 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -26,7 +26,6 @@ use stackable_operator::{ deep_merger::ObjectOverrides, k8s_openapi::api::core::v1::Volume, kube::{CustomResource, ResourceExt}, - kvp::ObjectLabels, product_logging::{ self, framework::{create_vector_shutdown_file_command, remove_vector_shutdown_file_command}, @@ -45,7 +44,10 @@ use stackable_operator::{ validate_logging_configuration_for_container, }, role_utils::{JavaCommonConfig, RoleGroupConfig, with_validated_config}, - types::{kubernetes::ConfigMapName, operator::RoleGroupName}, + types::{ + kubernetes::ConfigMapName, + operator::{RoleGroupName, RoleName}, + }, }, versioned::versioned, }; @@ -736,6 +738,12 @@ pub enum DruidRole { } impl DruidRole { + /// Returns the typed role name used for Kubernetes labels and selectors. + pub fn to_role_name(&self) -> RoleName { + RoleName::from_str(&self.to_string()) + .expect("a DruidRole always serializes to a valid role name") + } + /// Returns the name of the internal druid process name associated with the role. /// These strings are used by druid internally to identify processes. fn get_process_name(&self) -> &str { @@ -1025,25 +1033,6 @@ pub fn build_string_list(strings: &[String]) -> String { format!("[{}]", comma_list) } -/// Creates recommended `ObjectLabels` to be used in deployed resources -pub fn build_recommended_labels<'a, T>( - owner: &'a T, - controller_name: &'a str, - app_version: &'a str, - role: &'a str, - role_group: &'a str, -) -> ObjectLabels<'a, T> { - ObjectLabels { - owner, - app_name: APP_NAME, - app_version, - operator_name: OPERATOR_NAME, - controller_name, - role, - role_group, - } -} - #[cfg(test)] mod tests { use stackable_operator::versioned::test_utils::RoundtripTestData; diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 9a984771..b8712db9 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use anyhow::anyhow; use clap::Parser; -use controller::{DRUID_CONTROLLER_NAME, FULL_CONTROLLER_NAME}; +use controller::FULL_CONTROLLER_NAME; use futures::{FutureExt, StreamExt, TryFutureExt}; use stackable_operator::{ YamlSchema, From 76ba31c2f4729ed6c7d6e98b19f0ad0c7342c62e Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 15 Jun 2026 20:41:34 +0200 Subject: [PATCH 48/48] refactor: use ConfigMapName, SecretName, ListenerName in CRD --- extra/crds.yaml | 21 ++++++++++++ .../controller/build/resource/statefulset.rs | 2 +- .../src/controller/dereference.rs | 6 +++- rust/operator-binary/src/crd/affinity.rs | 2 +- .../operator-binary/src/crd/authentication.rs | 9 ++--- rust/operator-binary/src/crd/mod.rs | 31 +++++++---------- rust/operator-binary/src/crd/security.rs | 13 ++++---- rust/operator-binary/src/crd/tls.rs | 33 ++++++++++++++----- rust/operator-binary/src/main.rs | 9 +++-- 9 files changed, 84 insertions(+), 42 deletions(-) diff --git a/extra/crds.yaml b/extra/crds.yaml index b79058b2..3bad58df 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -491,6 +491,9 @@ spec: properties: listenerClass: default: cluster-internal + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string podDisruptionBudget: default: @@ -1091,6 +1094,9 @@ spec: The [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) for the HDFS instance. When running an HDFS cluster with the Stackable operator, the operator will create this ConfigMap for you. It has the same name as your HDFSCluster resource. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string directory: description: The directory inside of HDFS where Druid should store its data. @@ -1574,7 +1580,10 @@ spec: - If TLS encryption is used at all - Which cert the servers should use to authenticate themselves against the clients - Which cert the servers should use to authenticate themselves among each other + maxLength: 253 + minLength: 1 nullable: true + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string type: object vectorAggregatorConfigMapName: @@ -1583,7 +1592,10 @@ spec: It must contain the key `ADDRESS` with the address of the Vector aggregator. Follow the [logging tutorial](https://docs.stackable.tech/home/nightly/tutorials/logging-vector-aggregator) to learn how to configure log aggregation with Vector. + maxLength: 253 + minLength: 1 nullable: true + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string zookeeperConfigMapName: description: |- @@ -1591,6 +1603,9 @@ spec: Provide the name of the ZooKeeper [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) here. When using the [Stackable operator for Apache ZooKeeper](https://docs.stackable.tech/home/nightly/zookeeper/) to deploy a ZooKeeper cluster, this will simply be the name of your ZookeeperCluster resource. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string required: - deepStorage @@ -2090,6 +2105,9 @@ spec: properties: listenerClass: default: cluster-internal + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string podDisruptionBudget: default: @@ -5118,6 +5136,9 @@ spec: properties: listenerClass: default: cluster-internal + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string podDisruptionBudget: default: diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 79bd3329..eb1545fd 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -404,7 +404,7 @@ fn add_hdfs_cm_volume_and_volume_mounts( .context(AddVolumeMountSnafu)?; pb.add_volume( VolumeBuilder::new(HDFS_CONFIG_VOLUME_NAME) - .with_config_map(&hdfs.config_map_name) + .with_config_map(hdfs.config_map_name.to_string()) .build(), ) .context(AddVolumeSnafu)?; diff --git a/rust/operator-binary/src/controller/dereference.rs b/rust/operator-binary/src/controller/dereference.rs index bd4098a8..9406f311 100644 --- a/rust/operator-binary/src/controller/dereference.rs +++ b/rust/operator-binary/src/controller/dereference.rs @@ -84,7 +84,11 @@ pub async fn dereference( .as_deref() .context(ObjectHasNoNamespaceSnafu)?; - let zk_confmap = druid.spec.cluster_config.zookeeper_config_map_name.clone(); + let zk_confmap = druid + .spec + .cluster_config + .zookeeper_config_map_name + .to_string(); let zookeeper_connection_string = client .get::(&zk_confmap, namespace) .await diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 2076d8c4..52bdfdcc 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -39,7 +39,7 @@ pub fn get_affinity( { vec![affinity_between_role_pods( "hdfs", - hdfs_discovery_cm_name, // The discovery cm has the same name as the HdfsCluster itself + hdfs_discovery_cm_name.as_ref(), // The discovery cm has the same name as the HdfsCluster itself "datanode", 50, )] diff --git a/rust/operator-binary/src/crd/authentication.rs b/rust/operator-binary/src/crd/authentication.rs index 358bb9f4..3a166371 100644 --- a/rust/operator-binary/src/crd/authentication.rs +++ b/rust/operator-binary/src/crd/authentication.rs @@ -128,10 +128,11 @@ impl AuthenticationClassesResolved { .context(AuthenticationClassRetrievalFailedSnafu)?; let auth_class_name = auth_class.name_any(); - let server_and_internal_secret_class = cluster_config - .tls - .as_ref() - .and_then(|tls| tls.server_and_internal_secret_class.to_owned()); + let server_and_internal_secret_class = cluster_config.tls.as_ref().and_then(|tls| { + tls.server_and_internal_secret_class + .as_ref() + .map(|secret_class| secret_class.to_string()) + }); match &auth_class.spec.provider { core::v1alpha1::AuthenticationClassProvider::Tls(provider) => { diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index e0a4198e..dc5a05e7 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -45,7 +45,7 @@ use stackable_operator::{ }, role_utils::{JavaCommonConfig, RoleGroupConfig, with_validated_config}, types::{ - kubernetes::ConfigMapName, + kubernetes::{ConfigMapName, ListenerClassName}, operator::{RoleGroupName, RoleName}, }, }, @@ -199,11 +199,6 @@ pub enum Error { "the Vector agent is enabled but the Vector aggregator ConfigMap name is missing" ))] MissingVectorAggregatorConfigMapName, - - #[snafu(display("invalid Vector aggregator ConfigMap name"))] - ParseVectorAggregatorConfigMapName { - source: stackable_operator::v2::macros::attributed_string_type::Error, - }, } #[versioned( @@ -328,14 +323,14 @@ pub mod versioned { /// Provide the name of the ZooKeeper [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery) /// here. When using the [Stackable operator for Apache ZooKeeper](DOCS_BASE_URL_PLACEHOLDER/zookeeper/) /// to deploy a ZooKeeper cluster, this will simply be the name of your ZookeeperCluster resource. - pub zookeeper_config_map_name: String, + pub zookeeper_config_map_name: ConfigMapName, /// Name of the Vector aggregator [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery). /// It must contain the key `ADDRESS` with the address of the Vector aggregator. /// Follow the [logging tutorial](DOCS_BASE_URL_PLACEHOLDER/tutorials/logging-vector-aggregator) /// to learn how to configure log aggregation with Vector. #[serde(skip_serializing_if = "Option::is_none")] - pub vector_aggregator_config_map_name: Option, + pub vector_aggregator_config_map_name: Option, /// Extra volumes similar to `.spec.volumes` on a Pod to mount into every container, this can be useful to for /// example make client certificates, keytabs or similar things available to processors. These volumes will be @@ -351,7 +346,7 @@ pub mod versioned { pub generic: GenericRoleConfig, #[serde(default = "druid_default_listener_class")] - pub listener_class: String, + pub listener_class: ListenerClassName, } } @@ -468,16 +463,14 @@ impl v1alpha1::DruidCluster { let deep_storage = &self.spec.cluster_config.deep_storage; let name = self.name_any(); - // The Vector aggregator discovery ConfigMap name (validated here so an invalid name fails - // up-front). It is only required when the Vector agent is enabled for a role group. + // The Vector aggregator discovery ConfigMap name (a typed `ConfigMapName`, so an invalid + // name is already rejected at deserialization). It is only required when the Vector agent + // is enabled for a role group. let vector_aggregator_config_map_name = self .spec .cluster_config .vector_aggregator_config_map_name - .as_deref() - .map(ConfigMapName::from_str) - .transpose() - .context(ParseVectorAggregatorConfigMapNameSnafu)?; + .clone(); // All roles erase to an identical `ValidatedDruidConfig`; only the typed config, the role // config type and the `RoleResource` variant (historicals carry typed storage) differ. @@ -705,8 +698,8 @@ impl Default for v1alpha1::DruidRoleConfig { } } -fn druid_default_listener_class() -> String { - "cluster-internal".to_string() +fn druid_default_listener_class() -> ListenerClassName { + ListenerClassName::from_str("cluster-internal").expect("a valid listener class name") } #[derive( @@ -851,7 +844,7 @@ impl DruidRole { } } - pub fn listener_class_name(&self, druid: &v1alpha1::DruidCluster) -> Option { + pub fn listener_class_name(&self, druid: &v1alpha1::DruidCluster) -> Option { match self { Self::Broker => Some(druid.spec.brokers.role_config.listener_class.to_owned()), Self::Coordinator => Some( @@ -893,7 +886,7 @@ pub struct HdfsDeepStorageSpec { /// The [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery) /// for the HDFS instance. When running an HDFS cluster with the Stackable operator, the operator /// will create this ConfigMap for you. It has the same name as your HDFSCluster resource. - pub config_map_name: String, + pub config_map_name: ConfigMapName, /// The directory inside of HDFS where Druid should store its data. pub directory: String, } diff --git a/rust/operator-binary/src/crd/security.rs b/rust/operator-binary/src/crd/security.rs index 9e67b8dd..5568e8db 100644 --- a/rust/operator-binary/src/crd/security.rs +++ b/rust/operator-binary/src/crd/security.rs @@ -119,12 +119,13 @@ impl DruidTlsSecurity { ) -> Self { DruidTlsSecurity { auth_classes: auth_classes.clone(), - server_and_internal_secret_class: druid - .spec - .cluster_config - .tls - .as_ref() - .and_then(|tls| tls.server_and_internal_secret_class.clone()), + server_and_internal_secret_class: druid.spec.cluster_config.tls.as_ref().and_then( + |tls| { + tls.server_and_internal_secret_class + .as_ref() + .map(|secret_class| secret_class.to_string()) + }, + ), } } diff --git a/rust/operator-binary/src/crd/tls.rs b/rust/operator-binary/src/crd/tls.rs index 558b1a5a..5e24cea2 100644 --- a/rust/operator-binary/src/crd/tls.rs +++ b/rust/operator-binary/src/crd/tls.rs @@ -1,5 +1,10 @@ +use std::str::FromStr; + use serde::{Deserialize, Serialize}; -use stackable_operator::schemars::{self, JsonSchema}; +use stackable_operator::{ + schemars::{self, JsonSchema}, + v2::types::kubernetes::SecretClassName, +}; const TLS_DEFAULT_SECRET_CLASS: &str = "tls"; @@ -14,7 +19,7 @@ pub struct DruidTls { // happens via the HTTPS port. Even if both HTTPS and HTTP port are enabled, Druid clients // will default to using TLS. #[serde(default = "tls_default", skip_serializing_if = "Option::is_none")] - pub server_and_internal_secret_class: Option, + pub server_and_internal_secret_class: Option, } /// Default TLS settings. Internal and server communication default to "tls" secret class. @@ -25,14 +30,18 @@ pub fn default_druid_tls() -> Option { } /// Helper methods to provide defaults in the CRDs and tests -pub fn tls_default() -> Option { - Some(TLS_DEFAULT_SECRET_CLASS.to_string()) +pub fn tls_default() -> Option { + Some(SecretClassName::from_str(TLS_DEFAULT_SECRET_CLASS).expect("a valid secret class name")) } #[cfg(test)] mod tests { + use std::str::FromStr; + use indoc::formatdoc; - use stackable_operator::utils::yaml_from_str_singleton_map; + use stackable_operator::{ + utils::yaml_from_str_singleton_map, v2::types::kubernetes::SecretClassName, + }; use crate::crd::{tls::DruidTls, v1alpha1::DruidClusterConfig}; @@ -54,7 +63,9 @@ zookeeperConfigMapName: zk-config-map assert_eq!( druid_cluster_config.tls, Some(DruidTls { - server_and_internal_secret_class: Some("tls".to_string()) + server_and_internal_secret_class: Some( + SecretClassName::from_str("tls").expect("a valid secret class name") + ) }), ); assert_eq!(druid_cluster_config.authentication, vec![]); @@ -73,7 +84,10 @@ zookeeperConfigMapName: zk-config-map assert_eq!( druid_cluster_config.tls, Some(DruidTls { - server_and_internal_secret_class: Some("druid-secret-class".to_string()) + server_and_internal_secret_class: Some( + SecretClassName::from_str("druid-secret-class") + .expect("a valid secret class name") + ) }), ); assert_eq!(druid_cluster_config.authentication, vec![]); @@ -126,7 +140,10 @@ zookeeperConfigMapName: zk-config-map assert_eq!( druid_cluster_config.tls, Some(DruidTls { - server_and_internal_secret_class: Some("druid-secret-class".to_string()) + server_and_internal_secret_class: Some( + SecretClassName::from_str("druid-secret-class") + .expect("a valid secret class name") + ) }), ); /* diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index b8712db9..39a49af5 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -197,7 +197,12 @@ fn references_config_map( return false; }; - druid.spec.cluster_config.zookeeper_config_map_name == config_map.name_any() + druid + .spec + .cluster_config + .zookeeper_config_map_name + .to_string() + == config_map.name_any() || match &druid.spec.cluster_config.authorization { Some(druid_authorization) => { druid_authorization.opa.config_map_name == config_map.name_any() @@ -206,7 +211,7 @@ fn references_config_map( } || match &druid.spec.cluster_config.deep_storage { crd::DeepStorageSpec::Hdfs(hdfs_spec) => { - hdfs_spec.config_map_name == config_map.name_any() + hdfs_spec.config_map_name.to_string() == config_map.name_any() } _ => false, }