diff --git a/CHANGELOG.md b/CHANGELOG.md index a66e0195..736a7351 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ All notable changes to this project will be documented in this file. - Document Helm deployed RBAC permissions and remove unnecessary permissions ([#810]). - Internal operator refactoring: introduce dereference() and validate() steps in the reconciler ([#824]). - test: Bump vector-aggregator to 0.55.0, replace /graphql call with gRPC calls ([#826]). +- BREAKING: Removed product-config machinery. This is a breaking change in terms of configuration. + Users relying on the product-config `properties.yaml` file have to set these properties via the CRD ([#830]). ### Deleted @@ -29,6 +31,7 @@ All notable changes to this project will be documented in this file. [#818]: https://github.com/stackabletech/druid-operator/pull/818 [#824]: https://github.com/stackabletech/druid-operator/pull/824 [#826]: https://github.com/stackabletech/druid-operator/pull/826 +[#830]: https://github.com/stackabletech/druid-operator/pull/830 ## [26.3.0] - 2026-03-16 diff --git a/Cargo.lock b/Cargo.lock index 39a18400..0ad739b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -163,9 +163,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "axum" @@ -265,9 +265,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "2.11.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" [[package]] name = "block-buffer" @@ -280,9 +280,9 @@ dependencies = [ [[package]] name = "built" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4ad8f11f288f48ca24471bbd51ac257aaeaaa07adae295591266b792902ae64" +checksum = "5c0e531d93d39c34eef561e929e8a7f86d77a5af08aac4f6d6e39976c51858e9" dependencies = [ "chrono", "git2", @@ -290,9 +290,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.20.2" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytes" @@ -302,9 +302,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "cc" -version = "1.2.61" +version = "1.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" +checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" dependencies = [ "find-msvc-tools", "jobserver", @@ -320,9 +320,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "num-traits", @@ -574,9 +574,6 @@ name = "deranged" version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" -dependencies = [ - "powerfmt", -] [[package]] name = "derive_more" @@ -613,9 +610,9 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", @@ -676,9 +673,9 @@ dependencies = [ [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "elliptic-curve" @@ -939,9 +936,9 @@ checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-timer" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" [[package]] name = "futures-util" @@ -998,15 +995,14 @@ dependencies = [ [[package]] name = "git2" -version = "0.20.4" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" +checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" dependencies = [ "bitflags", "libc", "libgit2-sys", "log", - "url", ] [[package]] @@ -1040,9 +1036,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" dependencies = [ "atomic-waker", "bytes", @@ -1070,9 +1066,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.17.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "heck" @@ -1102,9 +1098,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes", "itoa", @@ -1153,9 +1149,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hyper" -version = "1.9.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" dependencies = [ "atomic-waker", "bytes", @@ -1366,7 +1362,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.17.0", + "hashbrown 0.17.1", ] [[package]] @@ -1384,16 +1380,6 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" -[[package]] -name = "iri-string" -version = "0.7.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -1428,9 +1414,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.24" +version = "0.2.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" +checksum = "4603d3033e49e2b0e31229fcab20a5d40089c607d975cd9c80551dc69eed9102" dependencies = [ "jiff-static", "jiff-tzdb-platform", @@ -1438,14 +1424,14 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.61.2", + "windows-link", ] [[package]] name = "jiff-static" -version = "0.2.24" +version = "0.2.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" +checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47" dependencies = [ "proc-macro2", "quote", @@ -1479,26 +1465,26 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.97" +version = "0.3.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1840c94c045fbcf8ba2812c95db44499f7c64910a912551aaaa541decebcacf" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" dependencies = [ "cfg-if", "futures-util", - "once_cell", "wasm-bindgen", ] [[package]] name = "json-patch" -version = "4.1.0" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f300e415e2134745ef75f04562dd0145405c2f7fd92065db029ac4b16b57fe90" +checksum = "7421438de105a0827e44fadd05377727847d717c80ce29a229f85fd04c427b72" dependencies = [ "jsonptr", + "schemars", "serde", "serde_json", - "thiserror 1.0.69", + "thiserror 2.0.18", ] [[package]] @@ -1540,11 +1526,11 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "darling", "regex", - "snafu 0.9.0", + "snafu 0.9.1", ] [[package]] @@ -1687,9 +1673,9 @@ checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libgit2-sys" -version = "0.18.3+1.9.2" +version = "0.18.5+1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487" +checksum = "005d6ae6eac1912906073e069f7db60b1fa98e052a68227824afe3e3a1c59ca2" dependencies = [ "cc", "libc", @@ -1705,9 +1691,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libz-sys" -version = "1.1.28" +version = "1.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc3a226e576f50782b3305c5ccf458698f92798987f551c6a02efe8276721e22" +checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" dependencies = [ "cc", "libc", @@ -1732,9 +1718,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" [[package]] name = "matchers" @@ -1753,9 +1739,9 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "mime" @@ -1775,9 +1761,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi", @@ -1811,9 +1797,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" @@ -1859,15 +1845,14 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "openssl" -version = "0.10.78" +version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38c4372413cdaaf3cc79dd92d29d7d9f5ab09b51b10dded508fb90bb70b9222" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ "bitflags", "cfg-if", "foreign-types", "libc", - "once_cell", "openssl-macros", "openssl-sys", ] @@ -1891,9 +1876,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.114" +version = "0.9.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13ce1245cd07fcc4cfdb438f7507b0c7e4f3849a69fd84d52374c66d83741bb6" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" dependencies = [ "cc", "libc", @@ -1903,9 +1888,9 @@ dependencies = [ [[package]] name = "opentelemetry" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" dependencies = [ "futures-core", "futures-sink", @@ -1917,9 +1902,9 @@ dependencies = [ [[package]] name = "opentelemetry-appender-tracing" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef6a1ac5ca3accf562b8c306fa8483c85f4390f768185ab775f242f7fe8fdcc2" +checksum = "2c0080f0dc1d7c786f467cd85a4e395fcab11ee852004f39a29a18ab7c25d837" dependencies = [ "opentelemetry", "tracing", @@ -1929,9 +1914,9 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" +checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331" dependencies = [ "async-trait", "bytes", @@ -1942,9 +1927,9 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" +checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" dependencies = [ "http", "opentelemetry", @@ -1956,14 +1941,14 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tonic", - "tracing", + "tonic-types", ] [[package]] name = "opentelemetry-proto" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" +checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -1974,21 +1959,22 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e62e29dfe041afb8ed2a6c9737ab57db4907285d999ef8ad3a59092a36bdc846" +checksum = "6ca2f98a0437b427b4b08f19f1caa3c44db885a202bc12cfea13d6c702243d68" [[package]] name = "opentelemetry_sdk" -version = "0.31.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" +checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" dependencies = [ "futures-channel", "futures-executor", "futures-util", "opentelemetry", "percent-encoding", + "portable-atomic", "rand 0.9.4", "thiserror 2.0.18", "tokio", @@ -2115,18 +2101,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.11" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.11" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", @@ -2250,9 +2236,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ "bytes", "prost-derive", @@ -2260,9 +2246,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", "itertools", @@ -2271,6 +2257,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.45" @@ -2375,9 +2370,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.3" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ "aho-corasick", "memchr", @@ -2398,9 +2393,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "relative-path" @@ -2410,9 +2405,9 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" -version = "0.12.28" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64", "bytes", @@ -2428,9 +2423,6 @@ dependencies = [ "log", "percent-encoding", "pin-project-lite", - "serde", - "serde_json", - "serde_urlencoded", "sync_wrapper", "tokio", "tower", @@ -2527,9 +2519,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.39" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c2c118cb077cca2822033836dfb1b975355dfb784b5e8da48f7b6c5db74e60e" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "log", "once_cell", @@ -2542,9 +2534,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -2730,9 +2722,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -2810,9 +2802,9 @@ dependencies = [ [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "signal-hook-registry" @@ -2848,9 +2840,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.1" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "snafu" @@ -2873,11 +2865,11 @@ dependencies = [ [[package]] name = "snafu" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1d4bced6a69f90b2056c03dcff2c4737f98d6fb9e0853493996e1d253ca29c6" +checksum = "d1a012328be2e3f5d5f6f3218147ca02588cea4cb865e876849ab6debcf36522" dependencies = [ - "snafu-derive 0.9.0", + "snafu-derive 0.9.1", ] [[package]] @@ -2905,9 +2897,9 @@ dependencies = [ [[package]] name = "snafu-derive" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" +checksum = "5f103c50866b8743da9429b8a581d81a27c2d3a9c4ac7df8f8571c1dd7896eda" dependencies = [ "heck", "proc-macro2", @@ -2917,9 +2909,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", "windows-sys 0.61.2", @@ -2950,7 +2942,7 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "const-oid", "ecdsa", @@ -2962,7 +2954,7 @@ dependencies = [ "rsa", "sha2", "signature", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-shared", "tokio", "tokio-rustls", @@ -2984,23 +2976,23 @@ dependencies = [ "indoc", "openssl", "pin-project", - "product-config", "rstest", "semver", "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-operator", "strum", "tokio", "tracing", + "uuid", ] [[package]] name = "stackable-operator" -version = "0.111.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +version = "0.111.1" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "base64", "clap", @@ -3012,6 +3004,7 @@ dependencies = [ "futures 0.3.32", "http", "indexmap", + "java-properties", "jiff", "json-patch", "k8s-openapi", @@ -3024,7 +3017,8 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "sha2", + "snafu 0.9.1", "stackable-operator-derive", "stackable-shared", "stackable-telemetry", @@ -3036,12 +3030,14 @@ dependencies = [ "tracing-appender", "tracing-subscriber", "url", + "uuid", + "xml", ] [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "darling", "proc-macro2", @@ -3051,8 +3047,8 @@ dependencies = [ [[package]] name = "stackable-shared" -version = "0.1.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +version = "0.1.1" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "jiff", "k8s-openapi", @@ -3061,15 +3057,15 @@ dependencies = [ "semver", "serde", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "strum", "time", ] [[package]] name = "stackable-telemetry" -version = "0.6.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +version = "0.6.4" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "axum", "clap", @@ -3080,7 +3076,7 @@ dependencies = [ "opentelemetry-semantic-conventions", "opentelemetry_sdk", "pin-project", - "snafu 0.9.0", + "snafu 0.9.1", "strum", "tokio", "tower", @@ -3093,21 +3089,21 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "kube", "schemars", "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-versioned-macros", ] [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "convert_case", "convert_case_extras", @@ -3125,7 +3121,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c" dependencies = [ "arc-swap", "async-trait", @@ -3141,7 +3137,7 @@ dependencies = [ "rand 0.9.4", "serde", "serde_json", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-certs", "stackable-shared", "stackable-telemetry", @@ -3286,12 +3282,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.47" +version = "0.3.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" dependencies = [ "deranged", - "itoa", "num-conv", "powerfmt", "serde_core", @@ -3301,15 +3296,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.27" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +checksum = "71c652a3727a9cbb9a02f707f530b618ce00d0ccd762009c8c23bd191df3c17d" dependencies = [ "num-conv", "time-core", @@ -3348,9 +3343,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.1" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -3420,9 +3415,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.11+spec-1.1.0" +version = "0.25.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" +checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" dependencies = [ "indexmap", "toml_datetime", @@ -3441,9 +3436,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "base64", @@ -3468,15 +3463,26 @@ dependencies = [ [[package]] name = "tonic-prost" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" dependencies = [ "bytes", "prost", "tonic", ] +[[package]] +name = "tonic-types" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab1b02061f83d519bba3caa167f88f261ef05720ab8ebc954ade70de3348e8" +dependencies = [ + "prost", + "prost-types", + "tonic", +] + [[package]] name = "tower" version = "0.5.3" @@ -3498,9 +3504,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "base64", "bitflags", @@ -3508,13 +3514,13 @@ dependencies = [ "futures-util", "http", "http-body", - "iri-string", "mime", "pin-project-lite", "tower", "tower-layer", "tower-service", "tracing", + "url", ] [[package]] @@ -3588,9 +3594,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.32.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac28f2d093c6c477eaa76b23525478f38de514fa9aeb1285738d4b97a9552fc" +checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" dependencies = [ "js-sys", "opentelemetry", @@ -3641,9 +3647,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" -version = "1.20.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "ucd-trie" @@ -3659,9 +3665,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-segmentation" -version = "1.13.2" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] name = "unicode-xid" @@ -3706,6 +3712,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -3741,18 +3757,18 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.3+wasi-0.2.9" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ "wit-bindgen", ] [[package]] name = "wasm-bindgen" -version = "0.2.120" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df52b6d9b87e0c74c9edfa1eb2d9bf85e5d63515474513aa50fa181b3c4f5db1" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" dependencies = [ "cfg-if", "once_cell", @@ -3763,9 +3779,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.70" +version = "0.4.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af934872acec734c2d80e6617bbb5ff4f12b052dd8e6332b0817bce889516084" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" dependencies = [ "js-sys", "wasm-bindgen", @@ -3773,9 +3789,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.120" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b1041f495fb322e64aca85f5756b2172e35cd459376e67f2a6c9dffcedb103" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3783,9 +3799,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.120" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dcd0ff20416988a18ac686d4d4d0f6aae9ebf08a389ff5d29012b05af2a1b41" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" dependencies = [ "bumpalo", "proc-macro2", @@ -3796,18 +3812,18 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.120" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49757b3c82ebf16c57d69365a142940b384176c24df52a087fb748e2085359ea" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" -version = "0.3.97" +version = "0.3.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eadbac71025cd7b0834f20d1fe8472e8495821b4e9801eb0a60bd1f19827602" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" dependencies = [ "js-sys", "wasm-bindgen", @@ -3966,9 +3982,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" dependencies = [ "memchr", ] @@ -4001,15 +4017,15 @@ dependencies = [ [[package]] name = "xml" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8aa498d22c9bbaf482329839bc5620c46be275a19a812e9a22a2b07529a642a" +checksum = "636f85e5ca6488e96401b61eb7de54f4e44755c988af0f52cf90230c312a1a89" [[package]] name = "yoke" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -4030,18 +4046,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.48" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.48" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", @@ -4050,9 +4066,9 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ "zerofrom-derive", ] @@ -4071,18 +4087,18 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.8.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" dependencies = [ "zeroize_derive", ] [[package]] name = "zeroize_derive" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.nix b/Cargo.nix index e0d5e27f..48e9e657 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -489,9 +489,9 @@ rec { }; "autocfg" = rec { crateName = "autocfg"; - version = "1.5.0"; + version = "1.5.1"; edition = "2015"; - sha256 = "1s77f98id9l4af4alklmzq46f21c980v13z2r1pcxx6bqgw0d1n0"; + sha256 = "0lqasy5i30flcgih1b50kvsk6z32g09r1q4ql7q81pj6228jy0zj"; authors = [ "Josh Stone " ]; @@ -866,9 +866,9 @@ rec { }; "bitflags" = rec { crateName = "bitflags"; - version = "2.11.1"; + version = "2.13.0"; edition = "2021"; - sha256 = "1cvqijg3rvwgis20a66vfdxannjsxfy5fgjqkaq3l13gyfcj4lf4"; + sha256 = "1y239gpvl061rfvav7jds8mjs42kmwi39is7yx5d1qw3hvp8nf5l"; authors = [ "The Rust Project Developers" ]; @@ -898,9 +898,9 @@ rec { }; "built" = rec { crateName = "built"; - version = "0.8.0"; - edition = "2021"; - sha256 = "0r5f08lpjsr6j5ajkbmd0ymfmajpq8ddbfvi8ji8rx48y88qzbgl"; + version = "0.8.1"; + edition = "2024"; + sha256 = "1saq332pd6g3svvc9ah8myjpfvgqlzl2ksb1ypp3976kjcfm63jw"; authors = [ "Lukas Lueg " ]; @@ -924,15 +924,16 @@ rec { "chrono" = [ "dep:chrono" ]; "dependency-tree" = [ "cargo-lock/dependency-tree" ]; "git2" = [ "dep:git2" ]; + "gix" = [ "dep:gix" ]; "semver" = [ "dep:semver" ]; }; resolvedDefaultFeatures = [ "chrono" "git2" ]; }; "bumpalo" = rec { crateName = "bumpalo"; - version = "3.20.2"; + version = "3.20.3"; edition = "2021"; - sha256 = "1jrgxlff76k9glam0akhwpil2fr1w32gbjdf5hpipc7ld2c7h82x"; + sha256 = "0jc6va3nwcqikm7chnpdv1s87my3gs2j7g1sc7g3k91brg3arxbj"; authors = [ "Nick Fitzgerald " ]; @@ -961,9 +962,9 @@ rec { }; "cc" = rec { crateName = "cc"; - version = "1.2.61"; + version = "1.2.64"; edition = "2018"; - sha256 = "0vawvnrrsmi8dygavq3wx085cmlp10sp3fhld5842rlqkqsr0vfi"; + sha256 = "07shcd8faxw7csz13m3cg2mj6i8z07pqs960k181pscbjpyqgn6s"; authors = [ "Alex Crichton " ]; @@ -1011,9 +1012,9 @@ rec { }; "chrono" = rec { crateName = "chrono"; - version = "0.4.44"; + version = "0.4.45"; edition = "2021"; - sha256 = "1c64mk9a235271j5g3v4zrzqqmd43vp9vki7vqfllpqf5rd0fwy6"; + sha256 = "09rkcgk6is2sdhqs9142zv8xqnj8ryx8m9hknllqwyv9wxi9x9qs"; dependencies = [ { name = "iana-time-zone"; @@ -1739,14 +1740,6 @@ rec { authors = [ "Jacob Pratt " ]; - dependencies = [ - { - name = "powerfmt"; - packageId = "powerfmt"; - optional = true; - usesDefaultFeatures = false; - } - ]; features = { "macros" = [ "dep:deranged-macros" ]; "num" = [ "dep:num-traits" ]; @@ -1758,7 +1751,7 @@ rec { "rand09" = [ "dep:rand09" ]; "serde" = [ "dep:serde_core" ]; }; - resolvedDefaultFeatures = [ "default" "powerfmt" ]; + resolvedDefaultFeatures = [ "default" ]; }; "derive_more" = rec { crateName = "derive_more"; @@ -1906,9 +1899,9 @@ rec { }; "displaydoc" = rec { crateName = "displaydoc"; - version = "0.2.5"; + version = "0.2.6"; edition = "2021"; - sha256 = "1q0alair462j21iiqwrr21iabkfnb13d6x5w95lkdg21q2xrqdlp"; + sha256 = "0kyxwfbdmagd8afzb2pzja7wj8dhah7smxdsgw00iq8pa2jhmiqs"; procMacro = true; authors = [ "Jane Lusby " @@ -2108,12 +2101,9 @@ rec { }; "either" = rec { crateName = "either"; - version = "1.15.0"; + version = "1.16.0"; edition = "2021"; - sha256 = "069p1fknsmzn9llaizh77kip0pqmcwpdsykv2x30xpjyija5gis8"; - authors = [ - "bluss" - ]; + sha256 = "17k7jfbdz7k440h6lws9baz8p9zlxgb41sig3w81h80nwzsjyqli"; features = { "default" = [ "std" ]; "serde" = [ "dep:serde" ]; @@ -2858,9 +2848,9 @@ rec { }; "futures-timer" = rec { crateName = "futures-timer"; - version = "3.0.3"; + version = "3.0.4"; edition = "2018"; - sha256 = "094vw8k37djpbwv74bwf2qb7n6v6ghif4myss6smd6hgyajb127j"; + sha256 = "0s39in8ivw7g4d37pf31q02y44zd1hpfkd1pgra2slcqibdzlhxg"; libName = "futures_timer"; authors = [ "Alex Crichton " @@ -3118,9 +3108,9 @@ rec { }; "git2" = rec { crateName = "git2"; - version = "0.20.4"; - edition = "2018"; - sha256 = "0azykjpk3j6s354z23jkyq3r3pbmlw9ha1zsxkw5cnnpi1h2b23v"; + version = "0.21.0"; + edition = "2021"; + sha256 = "0bmqga9vlyx5sdlr0i28z0362s89xv9i4qcv20vvx9j54y9vzpfx"; authors = [ "Josh Triplett " "Alex Crichton " @@ -3142,17 +3132,14 @@ rec { name = "log"; packageId = "log"; } - { - name = "url"; - packageId = "url"; - } ]; features = { - "default" = [ "ssh" "https" ]; - "https" = [ "libgit2-sys/https" "openssl-sys" "openssl-probe" ]; + "cred" = [ "dep:url" ]; + "https" = [ "libgit2-sys/https" "openssl-sys" "openssl-probe" "cred" ]; "openssl-probe" = [ "dep:openssl-probe" ]; "openssl-sys" = [ "dep:openssl-sys" ]; - "ssh" = [ "libgit2-sys/ssh" ]; + "ssh" = [ "libgit2-sys/ssh" "cred" ]; + "unstable-sha256" = [ "libgit2-sys/unstable-sha256" ]; "vendored-libgit2" = [ "libgit2-sys/vendored" ]; "vendored-openssl" = [ "openssl-sys/vendored" "libgit2-sys/vendored-openssl" ]; "zlib-ng-compat" = [ "libgit2-sys/zlib-ng-compat" ]; @@ -3242,9 +3229,9 @@ rec { }; "h2" = rec { crateName = "h2"; - version = "0.4.13"; + version = "0.4.14"; edition = "2021"; - sha256 = "0m6w5gg0n0m1m5915bxrv8n4rlazhx5icknkslz719jhh4xdli1g"; + sha256 = "0cw7jk7kn2vn6f8w8ssh6gis1mljnfjxd606gvi4sjpyjayfy7qp"; authors = [ "Carl Lerche " "Sean McArthur " @@ -3355,14 +3342,11 @@ rec { }; resolvedDefaultFeatures = [ "allocator-api2" "default" "default-hasher" "equivalent" "inline-more" "raw-entry" ]; }; - "hashbrown 0.17.0" = rec { + "hashbrown 0.17.1" = rec { crateName = "hashbrown"; - version = "0.17.0"; + version = "0.17.1"; edition = "2024"; - sha256 = "0l8gvcz80lvinb7x22h53cqbi2y1fm603y2jhhh9qwygvkb7sijg"; - authors = [ - "Amanieu d'Antras " - ]; + sha256 = "0jmqz7i4yl6cm7rbn0i2ffkfrmwi6xkmzkaldr2v8bcsx2v0jngd"; features = { "alloc" = [ "dep:alloc" ]; "allocator-api2" = [ "dep:allocator-api2" ]; @@ -3437,9 +3421,9 @@ rec { }; "http" = rec { crateName = "http"; - version = "1.4.0"; + version = "1.4.2"; edition = "2021"; - sha256 = "06iind4cwsj1d6q8c2xgq8i2wka4ps74kmws24gsi1bzdlw2mfp3"; + sha256 = "09b4p8fiivkg7wm0b59fyrn1jkm7px298ci7zb9igz6n647gaw39"; authors = [ "Alex Crichton " "Carl Lerche " @@ -3556,9 +3540,9 @@ rec { }; "hyper" = rec { crateName = "hyper"; - version = "1.9.0"; + version = "1.10.1"; edition = "2021"; - sha256 = "1jmwbwqcaficskg76kq402gbymbnh2z4v99xwq3l5aa6n8bg16b2"; + sha256 = "1624nwrh1ci34psqcl3q8q266kha8kd6fmqjj14qck49l59iqa2m"; authors = [ "Sean McArthur " ]; @@ -4365,7 +4349,7 @@ rec { } { name = "hashbrown"; - packageId = "hashbrown 0.17.0"; + packageId = "hashbrown 0.17.1"; usesDefaultFeatures = false; } ]; @@ -4423,39 +4407,6 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; - "iri-string" = rec { - crateName = "iri-string"; - version = "0.7.12"; - edition = "2021"; - sha256 = "082fpx6c5ghvmqpwxaf2b268m47z2ic3prajqbmi1s1qpfj5kri5"; - libName = "iri_string"; - authors = [ - "YOSHIOKA Takuma " - ]; - dependencies = [ - { - name = "memchr"; - packageId = "memchr"; - optional = true; - usesDefaultFeatures = false; - } - { - name = "serde"; - packageId = "serde"; - optional = true; - usesDefaultFeatures = false; - features = [ "derive" ]; - } - ]; - features = { - "alloc" = [ "serde?/alloc" ]; - "default" = [ "std" ]; - "memchr" = [ "dep:memchr" ]; - "serde" = [ "dep:serde" ]; - "std" = [ "alloc" "memchr?/std" "serde?/std" ]; - }; - resolvedDefaultFeatures = [ "alloc" "default" "std" ]; - }; "is_terminal_polyfill" = rec { crateName = "is_terminal_polyfill"; version = "1.70.2"; @@ -4525,9 +4476,9 @@ rec { }; "jiff" = rec { crateName = "jiff"; - version = "0.2.24"; + version = "0.2.28"; edition = "2021"; - sha256 = "0g87al8yqp05m63dhqzi359xgsslc0grqz00nvfdyq8dcayms2zh"; + sha256 = "00lixngcc7amh2fcsxfr0z38j06lllhapz192biv1qj97q1x60s6"; authors = [ "Andrew Gallant " ]; @@ -4573,12 +4524,10 @@ rec { usesDefaultFeatures = false; } { - name = "windows-sys"; - packageId = "windows-sys 0.61.2"; + name = "windows-link"; + packageId = "windows-link"; optional = true; - usesDefaultFeatures = false; target = { target, features }: (target."windows" or false); - features = [ "Win32_Foundation" "Win32_System_Time" ]; } ]; devDependencies = [ @@ -4597,7 +4546,7 @@ rec { "static-tz" = [ "dep:jiff-static" ]; "std" = [ "alloc" "log?/std" "serde_core?/std" ]; "tz-fat" = [ "jiff-static?/tz-fat" ]; - "tz-system" = [ "std" "dep:windows-sys" ]; + "tz-system" = [ "std" "dep:windows-link" ]; "tzdb-bundle-always" = [ "dep:jiff-tzdb" "alloc" ]; "tzdb-bundle-platform" = [ "dep:jiff-tzdb-platform" "alloc" ]; "tzdb-concatenated" = [ "std" ]; @@ -4607,9 +4556,9 @@ rec { }; "jiff-static" = rec { crateName = "jiff-static"; - version = "0.2.24"; + version = "0.2.28"; edition = "2021"; - sha256 = "1mz6v0d1hd8wjgfzccgda5g9z01s1yxnyiizvahjw0pq1w1xw070"; + sha256 = "0irbhfh2f4i9w5l53jcmh6ssnhdd92wfy76978chgwnxilvk4bbq"; procMacro = true; libName = "jiff_static"; authors = [ @@ -4689,9 +4638,9 @@ rec { }; "js-sys" = rec { crateName = "js-sys"; - version = "0.3.97"; + version = "0.3.102"; edition = "2021"; - sha256 = "1kyaxgn1sm5am98jb48aj5j7r7s98kdrab41la5wzys5q2a0r151"; + sha256 = "0cgxklnyrfpzvf32cvdl3x5d070kfsv7ykdxfl3yizwdjqq4rl03"; libName = "js_sys"; authors = [ "The wasm-bindgen Developers" @@ -4708,11 +4657,6 @@ rec { usesDefaultFeatures = false; features = [ "std" ]; } - { - name = "once_cell"; - packageId = "once_cell"; - usesDefaultFeatures = false; - } { name = "wasm-bindgen"; packageId = "wasm-bindgen"; @@ -4728,9 +4672,9 @@ rec { }; "json-patch" = rec { crateName = "json-patch"; - version = "4.1.0"; + version = "4.2.0"; edition = "2021"; - sha256 = "147yaxmv3i4s0bdna86rgwpmqh2507fn4ighfpplaiqkw8ay807k"; + sha256 = "0wkv896d0pzq56i2kkl0giqpv117fwvhbpgs8iz85805w66l68bl"; libName = "json_patch"; authors = [ "Ivan Dubrov " @@ -4740,6 +4684,11 @@ rec { name = "jsonptr"; packageId = "jsonptr"; } + { + name = "schemars"; + packageId = "schemars"; + optional = true; + } { name = "serde"; packageId = "serde"; @@ -4751,10 +4700,14 @@ rec { } { name = "thiserror"; - packageId = "thiserror 1.0.69"; + packageId = "thiserror 2.0.18"; } ]; devDependencies = [ + { + name = "schemars"; + packageId = "schemars"; + } { name = "serde_json"; packageId = "serde_json"; @@ -4766,7 +4719,7 @@ rec { "schemars" = [ "dep:schemars" ]; "utoipa" = [ "dep:utoipa" ]; }; - resolvedDefaultFeatures = [ "default" "diff" ]; + resolvedDefaultFeatures = [ "default" "diff" "schemars" ]; }; "jsonpath-rust" = rec { crateName = "jsonpath-rust"; @@ -4892,9 +4845,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; libName = "k8s_version"; authors = [ @@ -4912,7 +4865,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } ]; features = { @@ -5560,10 +5513,10 @@ rec { }; "libgit2-sys" = rec { crateName = "libgit2-sys"; - version = "0.18.3+1.9.2"; + version = "0.18.5+1.9.4"; edition = "2021"; links = "git2"; - sha256 = "11rlbyihj3k35mnkxxz4yvsnlx33a4r9srl66c5vp08pp72arcy9"; + sha256 = "18lwqnhy7qxg4iw24s1a0n7aj7qbnryry1iy0w32k4f1xbk6lp80"; libName = "libgit2_sys"; libPath = "lib.rs"; authors = [ @@ -5621,10 +5574,10 @@ rec { }; "libz-sys" = rec { crateName = "libz-sys"; - version = "1.1.28"; + version = "1.1.29"; edition = "2018"; links = "z"; - sha256 = "08hyf9v85zifl3353xc7i5wr53v9b3scri856cmphl3gaxp24fpw"; + sha256 = "1n98kqya7a7a0cxf5n5z3g13rj7a1vqxynk2xc7bja1qfxbrdg45"; libName = "libz_sys"; authors = [ "Alex Crichton " @@ -5701,9 +5654,9 @@ rec { }; "log" = rec { crateName = "log"; - version = "0.4.29"; + version = "0.4.32"; edition = "2021"; - sha256 = "15q8j9c8g5zpkcw0hnd6cf2z7fxqnvsjh3rw5mv5q10r83i34l2y"; + sha256 = "0fmdg0cxig7i4fwf1sw7fmg4d1gdbfzniawcfpwydy1q7320fgwm"; authors = [ "The Rust Project Developers" ]; @@ -5757,9 +5710,9 @@ rec { }; "memchr" = rec { crateName = "memchr"; - version = "2.8.0"; + version = "2.8.2"; edition = "2021"; - sha256 = "0y9zzxcqxvdqg6wyag7vc3h0blhdn7hkq164bxyx2vph8zs5ijpq"; + sha256 = "1i33wr49pcz2sbd12nds3n9fszay8kq5bk78gwciz462mcs49448"; authors = [ "Andrew Gallant " "bluss" @@ -5820,9 +5773,9 @@ rec { }; "mio" = rec { crateName = "mio"; - version = "1.2.0"; + version = "1.2.1"; edition = "2021"; - sha256 = "1hanrh4fwsfkdqdaqfidz48zz1wdix23zwn3r2x78am0garfbdsh"; + sha256 = "1nkggmrlnjs93w8rja4lvjj4aml1xqahgimv1h0p7d373kvhmg82"; authors = [ "Carl Lerche " "Thomas de Zeeuw " @@ -5958,9 +5911,9 @@ rec { }; "num-conv" = rec { crateName = "num-conv"; - version = "0.2.1"; + version = "0.2.2"; edition = "2021"; - sha256 = "0rqrr29brafaa2za352pbmhkk556n7f8z9rrkgmjp1idvdl3fry6"; + sha256 = "0hg4f9bwmy7cwpxdkm165dmkfc8jhkkayci234jsmi5ssb33j5sj"; libName = "num_conv"; authors = [ "Jacob Pratt " @@ -6082,9 +6035,9 @@ rec { }; "openssl" = rec { crateName = "openssl"; - version = "0.10.78"; + version = "0.10.81"; edition = "2021"; - sha256 = "08lj1fvhpfcga3nxs40vnl4spxfrswljvncxqwyazniw85r4737k"; + sha256 = "0ibsv2ppsjrp62jqyzprhay9vczk1bw9xvdr3h4h7fxsy0kkm0kp"; authors = [ "Steven Fackler " ]; @@ -6105,10 +6058,6 @@ rec { name = "libc"; packageId = "libc"; } - { - name = "once_cell"; - packageId = "once_cell"; - } { name = "openssl-macros"; packageId = "openssl-macros"; @@ -6165,10 +6114,10 @@ rec { }; "openssl-sys" = rec { crateName = "openssl-sys"; - version = "0.9.114"; + version = "0.9.117"; edition = "2021"; links = "openssl"; - sha256 = "1dhvfj1nvikl4gaq9zb9ka2g7r67n03pb3s3vg7w9z07rm2i5khk"; + sha256 = "159nf6jsqnmsynkh6gjzx088q1ifll7v88sss8qdk363n9mpwzml"; build = "build/main.rs"; libName = "openssl_sys"; authors = [ @@ -6207,9 +6156,9 @@ rec { }; "opentelemetry" = rec { crateName = "opentelemetry"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "18629xsj4rsyiby9aj511q6wcw6s9m09gx3ymw1yjcvix1mcsjxq"; + sha256 = "10ln14d1jgc8rvw97mblc9blzcgpg1bimim4d170b7ia4mijq55h"; dependencies = [ { name = "futures-core"; @@ -6246,24 +6195,24 @@ rec { ]; features = { "default" = [ "trace" "metrics" "logs" "internal-logs" "futures" ]; + "experimental_metrics_bound_instruments" = [ "metrics" ]; "futures" = [ "futures-core" "futures-sink" "pin-project-lite" ]; "futures-core" = [ "dep:futures-core" ]; "futures-sink" = [ "dep:futures-sink" ]; "internal-logs" = [ "tracing" ]; "pin-project-lite" = [ "dep:pin-project-lite" ]; - "spec_unstable_logs_enabled" = [ "logs" ]; "testing" = [ "trace" ]; "thiserror" = [ "dep:thiserror" ]; "trace" = [ "futures" "thiserror" ]; "tracing" = [ "dep:tracing" ]; }; - resolvedDefaultFeatures = [ "default" "futures" "futures-core" "futures-sink" "internal-logs" "logs" "metrics" "pin-project-lite" "spec_unstable_logs_enabled" "thiserror" "trace" "tracing" ]; + resolvedDefaultFeatures = [ "default" "futures" "futures-core" "futures-sink" "internal-logs" "logs" "metrics" "pin-project-lite" "thiserror" "trace" "tracing" ]; }; "opentelemetry-appender-tracing" = rec { crateName = "opentelemetry-appender-tracing"; - version = "0.31.1"; + version = "0.32.0"; edition = "2021"; - sha256 = "1hnwizzgfhpjfnvml638yy846py8hf2gl1n3p1igbk1srb2ilspg"; + sha256 = "0dyq4myan64sl8wly02jx0gb3jjz7575mn3w8rpphz0xvkq8001c"; libName = "opentelemetry_appender_tracing"; dependencies = [ { @@ -6306,18 +6255,15 @@ rec { ]; features = { "experimental_metadata_attributes" = [ "dep:tracing-log" ]; - "experimental_use_tracing_span_context" = [ "tracing-opentelemetry" ]; "log" = [ "dep:log" ]; - "spec_unstable_logs_enabled" = [ "opentelemetry/spec_unstable_logs_enabled" ]; - "tracing-opentelemetry" = [ "dep:tracing-opentelemetry" ]; }; resolvedDefaultFeatures = [ "default" ]; }; "opentelemetry-http" = rec { crateName = "opentelemetry-http"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "0pc5nw1ds8v8w0nvyall39m92v8m1xl1p3vwvxk6nkhrffdd19np"; + sha256 = "0ca3drvm4fx5nskl7yn42dimy3bg35ppzc85y1p27pz215fh30sn"; libName = "opentelemetry_http"; dependencies = [ { @@ -6353,16 +6299,16 @@ rec { "internal-logs" = [ "opentelemetry/internal-logs" ]; "reqwest" = [ "dep:reqwest" ]; "reqwest-blocking" = [ "dep:reqwest" "reqwest/blocking" ]; - "reqwest-rustls" = [ "dep:reqwest" "reqwest/rustls-tls-native-roots" ]; - "reqwest-rustls-webpki-roots" = [ "dep:reqwest" "reqwest/rustls-tls-webpki-roots" ]; + "reqwest-rustls" = [ "dep:reqwest" "reqwest/default-tls" ]; + "reqwest-rustls-webpki-roots" = [ "dep:reqwest" "reqwest/default-tls" "reqwest/webpki-roots" ]; }; - resolvedDefaultFeatures = [ "internal-logs" "reqwest" "reqwest-blocking" ]; + resolvedDefaultFeatures = [ "reqwest" "reqwest-blocking" ]; }; "opentelemetry-otlp" = rec { crateName = "opentelemetry-otlp"; - version = "0.31.1"; + version = "0.32.0"; edition = "2021"; - sha256 = "07zp0b62b9dajnvvcd6j2ppw5zg7wp4ixka9z6fr3bxrrdmcss8z"; + sha256 = "0d9cys2flpidfxbr6h1103hjc633cax47ihnqgbj0xnicscr4rlr"; libName = "opentelemetry_otlp"; dependencies = [ { @@ -6423,10 +6369,9 @@ rec { usesDefaultFeatures = false; } { - name = "tracing"; - packageId = "tracing"; + name = "tonic-types"; + packageId = "tonic-types"; optional = true; - usesDefaultFeatures = false; } ]; devDependencies = [ @@ -6451,16 +6396,19 @@ rec { ]; features = { "default" = [ "http-proto" "reqwest-blocking-client" "trace" "metrics" "logs" "internal-logs" ]; + "experimental-grpc-retry" = [ "grpc-tonic" "opentelemetry_sdk/experimental_async_runtime" "opentelemetry_sdk/rt-tokio" ]; + "experimental-http-retry" = [ "opentelemetry_sdk/experimental_async_runtime" "opentelemetry_sdk/rt-tokio" "tokio" "httpdate" ]; "flate2" = [ "dep:flate2" ]; - "grpc-tonic" = [ "tonic" "prost" "http" "tokio" "opentelemetry-proto/gen-tonic" ]; + "grpc-tonic" = [ "tonic" "tonic-types" "prost" "http" "tokio" "opentelemetry-proto/gen-tonic" ]; "gzip-http" = [ "flate2" ]; "gzip-tonic" = [ "tonic/gzip" ]; "http" = [ "dep:http" ]; "http-json" = [ "serde_json" "prost" "opentelemetry-http" "opentelemetry-proto/gen-tonic-messages" "opentelemetry-proto/with-serde" "http" "trace" "metrics" ]; "http-proto" = [ "prost" "opentelemetry-http" "opentelemetry-proto/gen-tonic-messages" "http" "trace" "metrics" ]; + "httpdate" = [ "dep:httpdate" ]; "hyper-client" = [ "opentelemetry-http/hyper" ]; "integration-testing" = [ "tonic" "prost" "tokio/full" "trace" "logs" ]; - "internal-logs" = [ "tracing" "opentelemetry_sdk/internal-logs" "opentelemetry-http/internal-logs" ]; + "internal-logs" = [ "opentelemetry_sdk/internal-logs" "opentelemetry/internal-logs" ]; "logs" = [ "opentelemetry/logs" "opentelemetry_sdk/logs" "opentelemetry-proto/logs" ]; "metrics" = [ "opentelemetry/metrics" "opentelemetry_sdk/metrics" "opentelemetry-proto/metrics" ]; "opentelemetry-http" = [ "dep:opentelemetry-http" ]; @@ -6473,27 +6421,27 @@ rec { "serde" = [ "dep:serde" ]; "serde_json" = [ "dep:serde_json" ]; "serialize" = [ "serde" "serde_json" ]; - "tls" = [ "tonic/tls-ring" ]; + "tls" = [ "tls-ring" ]; "tls-aws-lc" = [ "tonic/tls-aws-lc" ]; "tls-provider-agnostic" = [ "tonic/_tls-any" ]; "tls-ring" = [ "tonic/tls-ring" ]; - "tls-roots" = [ "tls" "tonic/tls-native-roots" ]; - "tls-webpki-roots" = [ "tls" "tonic/tls-webpki-roots" ]; + "tls-roots" = [ "tonic/tls-native-roots" ]; + "tls-webpki-roots" = [ "tonic/tls-webpki-roots" ]; "tokio" = [ "dep:tokio" ]; "tonic" = [ "dep:tonic" ]; + "tonic-types" = [ "dep:tonic-types" ]; "trace" = [ "opentelemetry/trace" "opentelemetry_sdk/trace" "opentelemetry-proto/trace" ]; - "tracing" = [ "dep:tracing" ]; "zstd" = [ "dep:zstd" ]; "zstd-http" = [ "zstd" ]; "zstd-tonic" = [ "tonic/zstd" ]; }; - resolvedDefaultFeatures = [ "default" "grpc-tonic" "gzip-tonic" "http" "http-proto" "internal-logs" "logs" "metrics" "opentelemetry-http" "prost" "reqwest" "reqwest-blocking-client" "tokio" "tonic" "trace" "tracing" ]; + resolvedDefaultFeatures = [ "default" "grpc-tonic" "gzip-tonic" "http" "http-proto" "internal-logs" "logs" "metrics" "opentelemetry-http" "prost" "reqwest" "reqwest-blocking-client" "tokio" "tonic" "tonic-types" "trace" ]; }; "opentelemetry-proto" = rec { crateName = "opentelemetry-proto"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "03xkjsjrsm7zkkx5gascqd9bg2z20wymm06l16cyxsp5dpq5s5x7"; + sha256 = "0f5ny4rpnpq6q5q34b8k2q548rf31rpbxkwjqjwzfqxg3yx5imjn"; libName = "opentelemetry_proto"; dependencies = [ { @@ -6537,30 +6485,29 @@ rec { "const-hex" = [ "dep:const-hex" ]; "default" = [ "full" ]; "full" = [ "gen-tonic" "trace" "logs" "metrics" "zpages" "with-serde" "internal-logs" ]; - "gen-tonic" = [ "gen-tonic-messages" "tonic/channel" ]; - "gen-tonic-messages" = [ "tonic" "tonic-prost" "prost" ]; + "gen-tonic" = [ "gen-tonic-messages" "tonic" "tonic-prost" "tonic/channel" ]; + "gen-tonic-messages" = [ "prost" ]; "internal-logs" = [ "opentelemetry/internal-logs" ]; "logs" = [ "opentelemetry/logs" "opentelemetry_sdk/logs" ]; "metrics" = [ "opentelemetry/metrics" "opentelemetry_sdk/metrics" ]; "prost" = [ "dep:prost" ]; "schemars" = [ "dep:schemars" ]; "serde" = [ "dep:serde" ]; - "serde_json" = [ "dep:serde_json" ]; "testing" = [ "opentelemetry/testing" ]; "tonic" = [ "dep:tonic" ]; "tonic-prost" = [ "dep:tonic-prost" ]; "trace" = [ "opentelemetry/trace" "opentelemetry_sdk/trace" ]; "with-schemars" = [ "schemars" ]; - "with-serde" = [ "serde" "const-hex" "base64" "serde_json" ]; + "with-serde" = [ "serde" "const-hex" "base64" ]; "zpages" = [ "trace" ]; }; resolvedDefaultFeatures = [ "gen-tonic" "gen-tonic-messages" "logs" "metrics" "prost" "tonic" "tonic-prost" "trace" ]; }; "opentelemetry-semantic-conventions" = rec { crateName = "opentelemetry-semantic-conventions"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "0in8plv2l2ar7anzi7lrbll0fjfvaymkg5vc5bnvibs1w3gjjbp6"; + sha256 = "0s1x4h1cgmhkxb7i5g02la2vhkf4lg5g26cgn2s2gd1p0j5gk8kc"; libName = "opentelemetry_semantic_conventions"; features = { }; @@ -6568,9 +6515,9 @@ rec { }; "opentelemetry_sdk" = rec { crateName = "opentelemetry_sdk"; - version = "0.31.0"; + version = "0.32.1"; edition = "2021"; - sha256 = "1gbjsggdxfpjbanjvaxa3nq32vfa37i3v13dvx4gsxhrk7sy8jp1"; + sha256 = "1ycl11syranrinhgn4c2hlzhyzyvpa06ryxq5mxgzmf4387ghncv"; dependencies = [ { name = "futures-channel"; @@ -6596,6 +6543,13 @@ rec { packageId = "percent-encoding"; optional = true; } + { + name = "portable-atomic"; + packageId = "portable-atomic"; + usesDefaultFeatures = false; + target = { target, features }: (!("64" == target."has_atomic" or null)); + features = [ "fallback" ]; + } { name = "rand"; packageId = "rand 0.9.4"; @@ -6620,10 +6574,18 @@ rec { optional = true; } ]; + devDependencies = [ + { + name = "tokio"; + packageId = "tokio"; + usesDefaultFeatures = false; + features = [ "macros" "rt-multi-thread" ]; + } + ]; features = { "default" = [ "trace" "metrics" "logs" "internal-logs" ]; "experimental_logs_batch_log_processor_with_async_runtime" = [ "logs" "experimental_async_runtime" ]; - "experimental_logs_concurrent_log_processor" = [ "logs" ]; + "experimental_metrics_bound_instruments" = [ "metrics" "opentelemetry/experimental_metrics_bound_instruments" ]; "experimental_metrics_custom_reader" = [ "metrics" ]; "experimental_metrics_disable_name_validation" = [ "metrics" ]; "experimental_metrics_periodicreader_with_async_runtime" = [ "metrics" "experimental_async_runtime" ]; @@ -6640,15 +6602,14 @@ rec { "rt-tokio-current-thread" = [ "tokio/rt" "tokio/time" "tokio-stream" "experimental_async_runtime" ]; "serde" = [ "dep:serde" ]; "serde_json" = [ "dep:serde_json" ]; - "spec_unstable_logs_enabled" = [ "logs" "opentelemetry/spec_unstable_logs_enabled" ]; "spec_unstable_metrics_views" = [ "metrics" ]; - "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "rt-tokio" "rt-tokio-current-thread" "tokio/macros" "tokio/rt-multi-thread" ]; + "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "tokio/sync" ]; "tokio" = [ "dep:tokio" ]; "tokio-stream" = [ "dep:tokio-stream" ]; "trace" = [ "opentelemetry/trace" "rand" "percent-encoding" ]; "url" = [ "dep:url" ]; }; - resolvedDefaultFeatures = [ "default" "experimental_async_runtime" "internal-logs" "logs" "metrics" "percent-encoding" "rand" "rt-tokio" "spec_unstable_logs_enabled" "tokio" "tokio-stream" "trace" ]; + resolvedDefaultFeatures = [ "default" "experimental_async_runtime" "internal-logs" "logs" "metrics" "percent-encoding" "rand" "rt-tokio" "tokio" "tokio-stream" "trace" ]; }; "ordered-float" = rec { crateName = "ordered-float"; @@ -7022,9 +6983,9 @@ rec { }; "pin-project" = rec { crateName = "pin-project"; - version = "1.1.11"; + version = "1.1.13"; edition = "2021"; - sha256 = "05zm3y3bl83ypsr6favxvny2kys4i19jiz1y18ylrbxwsiz9qx7i"; + sha256 = "09091qp946lpmjz4yp0xil1r5v4hgc91fi19dg5csayhdqrv4ri4"; libName = "pin_project"; dependencies = [ { @@ -7036,9 +6997,9 @@ rec { }; "pin-project-internal" = rec { crateName = "pin-project-internal"; - version = "1.1.11"; + version = "1.1.13"; edition = "2021"; - sha256 = "1ik4mpb92da75inmjvxf2qm61vrnwml3x24wddvrjlqh1z9hxcnr"; + sha256 = "12rzlh07i1sdgrvzj6wgkka5bjqyvbfsl8knq6qi7g16m7q9aqy9"; procMacro = true; libName = "pin_project_internal"; dependencies = [ @@ -7157,7 +7118,7 @@ rec { "default" = [ "fallback" ]; "serde" = [ "dep:serde" ]; }; - resolvedDefaultFeatures = [ "require-cas" ]; + resolvedDefaultFeatures = [ "fallback" "require-cas" ]; }; "portable-atomic-util" = rec { crateName = "portable-atomic-util"; @@ -7360,9 +7321,9 @@ rec { }; "prost" = rec { crateName = "prost"; - version = "0.14.3"; + version = "0.14.4"; edition = "2021"; - sha256 = "0s057z9nzggzy7x4bbsiar852hg7zb81f4z4phcdb0ig99971snj"; + sha256 = "1qas5v5rap45f43v3ja0jngxrrafrkcwl0iw5a3ld1pz2rscd2jj"; authors = [ "Dan Burkert " "Lucio Franco " @@ -7389,9 +7350,9 @@ rec { }; "prost-derive" = rec { crateName = "prost-derive"; - version = "0.14.3"; + version = "0.14.4"; edition = "2021"; - sha256 = "02zvva6kb0pfvlyc4nac6gd37ncjrs8jq5scxcq4nbqkc8wh5ii7"; + sha256 = "1pqa77d7da5pf6ba3kjj7510m5cynz6902ax01ckvr0pfrgv4w5m"; procMacro = true; libName = "prost_derive"; authors = [ @@ -7425,6 +7386,34 @@ rec { ]; }; + "prost-types" = rec { + crateName = "prost-types"; + version = "0.14.4"; + edition = "2021"; + sha256 = "02ivjvc4cwl5bfgjs3l00hwlrk74z8zlg1xcgx60bww8fvf6fjgr"; + libName = "prost_types"; + authors = [ + "Dan Burkert " + "Lucio Franco " + "Casper Meijn " + "Tokio Contributors " + ]; + dependencies = [ + { + name = "prost"; + packageId = "prost"; + usesDefaultFeatures = false; + features = [ "derive" ]; + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + "chrono" = [ "dep:chrono" ]; + "default" = [ "std" ]; + "std" = [ "prost/std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "quote" = rec { crateName = "quote"; version = "1.0.45"; @@ -7704,9 +7693,9 @@ rec { }; "regex" = rec { crateName = "regex"; - version = "1.12.3"; + version = "1.12.4"; edition = "2021"; - sha256 = "0xp2q0x7ybmpa5zlgaz00p8zswcirj9h8nry3rxxsdwi9fhm81z1"; + sha256 = "1fm6si2xpmhwqflabdqsakc0qkq718wx2ljl37nbj75fb5vjnagi"; authors = [ "The Rust Project Developers" "Andrew Gallant " @@ -7823,9 +7812,9 @@ rec { }; "regex-syntax" = rec { crateName = "regex-syntax"; - version = "0.8.10"; + version = "0.8.11"; edition = "2021"; - sha256 = "02jx311ka0daxxc7v45ikzhcl3iydjbbb0mdrpc1xgg8v7c7v2fw"; + sha256 = "1m25h5q2wp976fb9gc3dsc9l99svcvd5cri8lncb51c46ydgzxnn"; libName = "regex_syntax"; authors = [ "The Rust Project Developers" @@ -7854,9 +7843,9 @@ rec { }; "reqwest" = rec { crateName = "reqwest"; - version = "0.12.28"; + version = "0.13.4"; edition = "2021"; - sha256 = "0iqidijghgqbzl3bjg5hb4zmigwa4r612bgi0yiq0c90b6jkrpgd"; + sha256 = "1hy1plns9krbh3h1dy2sdjygsfkdcnxm6pbxdi0ya9b5vq8mi711"; authors = [ "Sean McArthur " ]; @@ -7873,7 +7862,7 @@ rec { name = "futures-channel"; packageId = "futures-channel"; optional = true; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "futures-core"; @@ -7893,62 +7882,44 @@ rec { { name = "http-body"; packageId = "http-body"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "http-body-util"; packageId = "http-body-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "hyper"; packageId = "hyper"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "client" ]; } { name = "hyper-util"; packageId = "hyper-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "client" "client-legacy" "client-proxy" "tokio" ]; } { name = "js-sys"; packageId = "js-sys"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "log"; packageId = "log"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "percent-encoding"; packageId = "percent-encoding"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "pin-project-lite"; packageId = "pin-project-lite"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - } - { - name = "serde"; - packageId = "serde"; - } - { - name = "serde_json"; - packageId = "serde_json"; - optional = true; - } - { - name = "serde_json"; - packageId = "serde_json"; - target = { target, features }: ("wasm32" == target."arch" or null); - } - { - name = "serde_urlencoded"; - packageId = "serde_urlencoded"; + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "sync_wrapper"; @@ -7959,27 +7930,27 @@ rec { name = "tokio"; packageId = "tokio"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "net" "time" ]; } { name = "tower"; packageId = "tower"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "retry" "timeout" "util" ]; } { name = "tower-http"; packageId = "tower-http"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "follow-redirect" ]; } { name = "tower-service"; packageId = "tower-service"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "url"; @@ -7988,17 +7959,17 @@ rec { { name = "wasm-bindgen"; packageId = "wasm-bindgen"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "wasm-bindgen-futures"; packageId = "wasm-bindgen-futures"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "web-sys"; packageId = "web-sys"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); features = [ "AbortController" "AbortSignal" "Headers" "Request" "RequestInit" "RequestMode" "Response" "Window" "FormData" "Blob" "BlobPropertyBag" "ServiceWorkerGlobalScope" "RequestCredentials" "File" "ReadableStream" "RequestCache" ]; } ]; @@ -8007,33 +7978,27 @@ rec { name = "futures-util"; packageId = "futures-util"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "std" "alloc" ]; } { name = "hyper"; packageId = "hyper"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "http2" "client" "server" ]; } { name = "hyper-util"; packageId = "hyper-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "http2" "client" "client-legacy" "server-auto" "server-graceful" "tokio" ]; } - { - name = "serde"; - packageId = "serde"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - features = [ "derive" ]; - } { name = "tokio"; packageId = "tokio"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "macros" "rt-multi-thread" ]; } { @@ -8045,40 +8010,37 @@ rec { { name = "wasm-bindgen"; packageId = "wasm-bindgen"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); features = [ "serde-serialize" ]; } ]; features = { + "__native-tls" = [ "dep:hyper-tls" "dep:native-tls-crate" "__tls" "dep:tokio-native-tls" ]; + "__native-tls-alpn" = [ "native-tls-crate?/alpn" "hyper-tls?/alpn" ]; "__rustls" = [ "dep:hyper-rustls" "dep:tokio-rustls" "dep:rustls" "__tls" ]; - "__rustls-ring" = [ "hyper-rustls?/ring" "tokio-rustls?/ring" "rustls?/ring" "quinn?/ring" ]; + "__rustls-aws-lc-rs" = [ "hyper-rustls?/aws-lc-rs" "tokio-rustls?/aws-lc-rs" "rustls?/aws-lc-rs" "quinn?/rustls-aws-lc-rs" ]; "__tls" = [ "dep:rustls-pki-types" "tokio/io-util" ]; "blocking" = [ "dep:futures-channel" "futures-channel?/sink" "dep:futures-util" "futures-util?/io" "futures-util?/sink" "tokio/sync" ]; "brotli" = [ "tower-http/decompression-br" ]; "charset" = [ "dep:encoding_rs" "dep:mime" ]; "cookies" = [ "dep:cookie_crate" "dep:cookie_store" ]; "default" = [ "default-tls" "charset" "http2" "system-proxy" ]; - "default-tls" = [ "dep:hyper-tls" "dep:native-tls-crate" "__tls" "dep:tokio-native-tls" ]; + "default-tls" = [ "rustls" ]; "deflate" = [ "tower-http/decompression-deflate" ]; + "form" = [ "dep:serde" "dep:serde_urlencoded" ]; "gzip" = [ "tower-http/decompression-gzip" ]; - "h2" = [ "dep:h2" ]; "hickory-dns" = [ "dep:hickory-resolver" "dep:once_cell" ]; - "http2" = [ "h2" "hyper/http2" "hyper-util/http2" "hyper-rustls?/http2" ]; - "http3" = [ "rustls-tls-manual-roots" "dep:h3" "dep:h3-quinn" "dep:quinn" "tokio/macros" ]; - "json" = [ "dep:serde_json" ]; - "macos-system-configuration" = [ "system-proxy" ]; + "http2" = [ "dep:h2" "hyper/http2" "hyper-util/http2" "hyper-rustls?/http2" ]; + "http3" = [ "rustls" "dep:h3" "dep:h3-quinn" "dep:quinn" "tokio/macros" ]; + "json" = [ "dep:serde" "dep:serde_json" ]; "multipart" = [ "dep:mime_guess" "dep:futures-util" ]; - "native-tls" = [ "default-tls" ]; - "native-tls-alpn" = [ "native-tls" "native-tls-crate?/alpn" "hyper-tls?/alpn" ]; - "native-tls-vendored" = [ "native-tls" "native-tls-crate?/vendored" ]; - "rustls-tls" = [ "rustls-tls-webpki-roots" ]; - "rustls-tls-manual-roots" = [ "rustls-tls-manual-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-manual-roots-no-provider" = [ "__rustls" ]; - "rustls-tls-native-roots" = [ "rustls-tls-native-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-native-roots-no-provider" = [ "dep:rustls-native-certs" "hyper-rustls?/native-tokio" "__rustls" ]; - "rustls-tls-no-provider" = [ "rustls-tls-manual-roots-no-provider" ]; - "rustls-tls-webpki-roots" = [ "rustls-tls-webpki-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-webpki-roots-no-provider" = [ "dep:webpki-roots" "hyper-rustls?/webpki-tokio" "__rustls" ]; + "native-tls" = [ "__native-tls" "__native-tls-alpn" ]; + "native-tls-no-alpn" = [ "__native-tls" ]; + "native-tls-vendored" = [ "__native-tls" "native-tls-crate?/vendored" "__native-tls-alpn" ]; + "native-tls-vendored-no-alpn" = [ "__native-tls" "native-tls-crate?/vendored" ]; + "query" = [ "dep:serde" "dep:serde_urlencoded" ]; + "rustls" = [ "__rustls-aws-lc-rs" "dep:rustls-platform-verifier" "__rustls" ]; + "rustls-no-provider" = [ "dep:rustls-platform-verifier" "__rustls" ]; "stream" = [ "tokio/fs" "dep:futures-util" "dep:tokio-util" "dep:wasm-streams" ]; "system-proxy" = [ "hyper-util/client-proxy-system" ]; "zstd" = [ "tower-http/decompression-zstd" ]; @@ -8390,9 +8352,9 @@ rec { }; "rustls" = rec { crateName = "rustls"; - version = "0.23.39"; + version = "0.23.40"; edition = "2021"; - sha256 = "03p6fkdwbdpp93dfidc4nzgmalwp3gxnv0rk421a5k3pn2612b3w"; + sha256 = "12qnv3ag4wrw7aj8jng74kgrilpjm2b1rfcjaac8h691frccv1pg"; dependencies = [ { name = "log"; @@ -8459,9 +8421,9 @@ rec { }; "rustls-native-certs" = rec { crateName = "rustls-native-certs"; - version = "0.8.3"; + version = "0.8.4"; edition = "2021"; - sha256 = "0qrajg2n90bcr3bcq6j95gjm7a9lirfkkdmjj32419dyyzan0931"; + sha256 = "0kgazl8zc1sv63qg179bz96ilzh56lzfa5k92ji7d265f4kibdfs"; libName = "rustls_native_certs"; dependencies = [ { @@ -9028,9 +8990,9 @@ rec { }; "serde_json" = rec { crateName = "serde_json"; - version = "1.0.149"; + version = "1.0.150"; edition = "2021"; - sha256 = "11jdx4vilzrjjd1dpgy67x5lgzr0laplz30dhv75lnf5ffa07z43"; + sha256 = "1ffgfhy9kndjnrz8lmy95pr758p2zk8dxv6yi99x0vkkni24w0g8"; authors = [ "Erick Tryzelaar " "David Tolnay " @@ -9271,9 +9233,9 @@ rec { }; "shlex" = rec { crateName = "shlex"; - version = "1.3.0"; - edition = "2015"; - sha256 = "0r1y6bv26c1scpxvhg2cabimrmwgbp4p3wy6syj9n0c4s3q2znhg"; + version = "2.0.1"; + edition = "2018"; + sha256 = "1fjsll1cd7d2bcpdij9kd6w62rpbc7qqzvydvs021vsmr1cxvypq"; authors = [ "comex " "Fenhl " @@ -9368,9 +9330,9 @@ rec { }; "smallvec" = rec { crateName = "smallvec"; - version = "1.15.1"; + version = "1.15.2"; edition = "2018"; - sha256 = "00xxdxxpgyq5vjnpljvkmy99xij5rxgh913ii1v16kzynnivgcb7"; + sha256 = "143wzbqf6vgapdp2z4qpl0yvlqcn17s8cnk8m28rqly808zsdmlf"; authors = [ "The Servo Project Developers" ]; @@ -9452,29 +9414,25 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "rust_1_61" "rust_1_65" "std" ]; }; - "snafu 0.9.0" = rec { + "snafu 0.9.1" = rec { crateName = "snafu"; - version = "0.9.0"; + version = "0.9.1"; edition = "2018"; - sha256 = "1ii9r99x5qcn754m624yzgb9hzvkqkrcygf0aqh0pyb9dbnvrm6i"; + sha256 = "08k5yfydxdlshivfhrdq9km8qn02r93q28gkyvazbqz2icr1586i"; authors = [ "Jake Goulding " ]; dependencies = [ { name = "snafu-derive"; - packageId = "snafu-derive 0.9.0"; + packageId = "snafu-derive 0.9.1"; } ]; features = { - "backtrace" = [ "dep:backtrace" ]; - "backtraces-impl-backtrace-crate" = [ "backtrace" ]; + "backtraces-impl-backtrace-crate" = [ "dep:backtrace" ]; "default" = [ "std" "rust_1_81" ]; - "futures" = [ "futures-core-crate" "pin-project" ]; - "futures-core-crate" = [ "dep:futures-core-crate" ]; - "futures-crate" = [ "dep:futures-crate" ]; - "internal-dev-dependencies" = [ "futures-crate" ]; - "pin-project" = [ "dep:pin-project" ]; + "futures" = [ "dep:futures-core" "dep:pin-project" ]; + "internal-dev-dependencies" = [ "dep:futures" ]; "std" = [ "alloc" ]; "unstable-provider-api" = [ "snafu-derive/unstable-provider-api" ]; }; @@ -9542,11 +9500,11 @@ rec { }; resolvedDefaultFeatures = [ "rust_1_61" ]; }; - "snafu-derive 0.9.0" = rec { + "snafu-derive 0.9.1" = rec { crateName = "snafu-derive"; - version = "0.9.0"; + version = "0.9.1"; edition = "2018"; - sha256 = "0h0x61kyj4fvilcr2nj02l85shw1ika64vq9brf2gyna662ln9al"; + sha256 = "1nkfi7bis72pz3w7vb64m79w49qsv20sbf19jkd471vbhr83q42z"; procMacro = true; libName = "snafu_derive"; authors = [ @@ -9572,7 +9530,7 @@ rec { name = "syn"; packageId = "syn 2.0.117"; usesDefaultFeatures = false; - features = [ "clone-impls" "derive" "full" "parsing" "printing" "proc-macro" ]; + features = [ "clone-impls" "derive" "full" "parsing" "printing" "proc-macro" "visit-mut" ]; } ]; features = { @@ -9580,9 +9538,9 @@ rec { }; "socket2" = rec { crateName = "socket2"; - version = "0.6.3"; + version = "0.6.4"; edition = "2021"; - sha256 = "0gkjjcyn69hqhhlh5kl8byk5m0d7hyrp2aqwzbs3d33q208nwxis"; + sha256 = "0ldyp5rhba15spwxj1n94xh7sjks1398c3vwpwkxkd1087nwzlaj"; authors = [ "Alex Crichton " "Thomas de Zeeuw " @@ -9680,9 +9638,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; libName = "stackable_certs"; authors = [ @@ -9740,7 +9698,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-shared"; @@ -9826,10 +9784,6 @@ rec { name = "pin-project"; packageId = "pin-project"; } - { - name = "product-config"; - packageId = "product-config"; - } { name = "semver"; packageId = "semver"; @@ -9845,7 +9799,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-operator"; @@ -9866,6 +9820,10 @@ rec { name = "tracing"; packageId = "tracing"; } + { + name = "uuid"; + packageId = "uuid"; + } ]; buildDependencies = [ { @@ -9888,13 +9846,13 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.111.0"; + version = "0.111.1"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; libName = "stackable_operator"; authors = [ @@ -9944,6 +9902,10 @@ rec { name = "indexmap"; packageId = "indexmap"; } + { + name = "java-properties"; + packageId = "java-properties"; + } { name = "jiff"; packageId = "jiff"; @@ -9951,6 +9913,7 @@ rec { { name = "json-patch"; packageId = "json-patch"; + features = [ "schemars" ]; } { name = "k8s-openapi"; @@ -9998,9 +9961,14 @@ rec { name = "serde_yaml"; packageId = "serde_yaml"; } + { + name = "sha2"; + packageId = "sha2"; + features = [ "oid" ]; + } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-operator-derive"; @@ -10054,12 +10022,21 @@ rec { packageId = "url"; features = [ "serde" ]; } + { + name = "uuid"; + packageId = "uuid"; + } + { + name = "xml"; + packageId = "xml"; + } ]; features = { "certs" = [ "dep:stackable-certs" ]; + "client-feature-gates" = [ "dep:winnow" ]; "crds" = [ "dep:stackable-versioned" ]; "default" = [ "crds" ]; - "full" = [ "crds" "certs" "time" "webhook" "kube-ws" ]; + "full" = [ "client-feature-gates" "crds" "certs" "time" "webhook" "kube-ws" ]; "kube-ws" = [ "kube/ws" ]; "time" = [ "stackable-shared/time" ]; "webhook" = [ "dep:stackable-webhook" ]; @@ -10072,9 +10049,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -10103,13 +10080,13 @@ rec { }; "stackable-shared" = rec { crateName = "stackable-shared"; - version = "0.1.0"; + version = "0.1.1"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; libName = "stackable_shared"; authors = [ @@ -10153,7 +10130,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "strum"; @@ -10184,13 +10161,13 @@ rec { }; "stackable-telemetry" = rec { crateName = "stackable-telemetry"; - version = "0.6.3"; + version = "0.6.4"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; libName = "stackable_telemetry"; authors = [ @@ -10233,7 +10210,7 @@ rec { { name = "opentelemetry_sdk"; packageId = "opentelemetry_sdk"; - features = [ "rt-tokio" "logs" "rt-tokio" "spec_unstable_logs_enabled" ]; + features = [ "rt-tokio" "logs" "rt-tokio" ]; } { name = "pin-project"; @@ -10241,7 +10218,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "strum"; @@ -10298,9 +10275,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; libName = "stackable_versioned"; authors = [ @@ -10333,7 +10310,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-versioned-macros"; @@ -10348,9 +10325,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; procMacro = true; libName = "stackable_versioned_macros"; @@ -10416,9 +10393,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; - sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a9fbbc8d1f0dc0387f26ae443a8b7d5a6e24323c"; + sha256 = "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr"; }; libName = "stackable_webhook"; authors = [ @@ -10490,7 +10467,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-certs"; @@ -10857,9 +10834,9 @@ rec { }; "time" = rec { crateName = "time"; - version = "0.3.47"; + version = "0.3.49"; edition = "2024"; - sha256 = "0b7g9ly2iabrlgizliz6v5x23yq5d6bpp0mqz6407z1s526d8fvl"; + sha256 = "0sc4dgw6g187gvz5qj9iqqk2ashqzvdwi664b2183gbvsk1566ki"; authors = [ "Jacob Pratt " "Time contributors" @@ -10868,12 +10845,6 @@ rec { { name = "deranged"; packageId = "deranged"; - features = [ "powerfmt" ]; - } - { - name = "itoa"; - packageId = "itoa"; - optional = true; } { name = "num-conv"; @@ -10913,13 +10884,14 @@ rec { features = { "alloc" = [ "serde_core?/alloc" ]; "default" = [ "std" ]; - "formatting" = [ "dep:itoa" "std" "time-macros?/formatting" ]; + "formatting" = [ "std" "time-macros?/formatting" ]; "large-dates" = [ "time-core/large-dates" "time-macros?/large-dates" ]; "local-offset" = [ "std" "dep:libc" "dep:num_threads" ]; "macros" = [ "dep:time-macros" ]; "parsing" = [ "time-macros?/parsing" ]; "quickcheck" = [ "dep:quickcheck" "alloc" "deranged/quickcheck" ]; - "rand" = [ "rand08" "rand09" ]; + "rand" = [ "rand08" "rand09" "rand010" ]; + "rand010" = [ "dep:rand010" "deranged/rand010" ]; "rand08" = [ "dep:rand08" "deranged/rand08" ]; "rand09" = [ "dep:rand09" "deranged/rand09" ]; "serde" = [ "dep:serde_core" "time-macros?/serde" "deranged/serde" ]; @@ -10932,9 +10904,9 @@ rec { }; "time-core" = rec { crateName = "time-core"; - version = "0.1.8"; + version = "0.1.9"; edition = "2024"; - sha256 = "1jidl426mw48i7hjj4hs9vxgd9lwqq4vyalm4q8d7y4iwz7y353n"; + sha256 = "028ix0ax7ixp1h1k5zsqwgw85w6y1q32irslma7ci6ddd5kr074y"; libName = "time_core"; authors = [ "Jacob Pratt " @@ -10945,9 +10917,9 @@ rec { }; "time-macros" = rec { crateName = "time-macros"; - version = "0.2.27"; + version = "0.2.29"; edition = "2024"; - sha256 = "058ja265waq275wxvnfwavbz9r1hd4dgwpfn7a1a9a70l32y8w1f"; + sha256 = "0zf1ycfikg93ijf00qnprk801khqnqqga1zp0adbp73sfaim5iki"; procMacro = true; libName = "time_macros"; authors = [ @@ -11062,9 +11034,9 @@ rec { }; "tokio" = rec { crateName = "tokio"; - version = "1.52.1"; + version = "1.52.3"; edition = "2021"; - sha256 = "1imw1dkkv38p66i33m5hsyk3d6prsbyrayjvqhndjvz89ybywzdn"; + sha256 = "1zpzazypkg61sw91na1m85x5s4rsjym335fwwhwm1hcs70dz1iwg"; authors = [ "Tokio Contributors " ]; @@ -11374,9 +11346,9 @@ rec { }; "toml_edit" = rec { crateName = "toml_edit"; - version = "0.25.11+spec-1.1.0"; + version = "0.25.12+spec-1.1.0"; edition = "2024"; - sha256 = "0awzffbkx33v9x4h19b5mfrwp3sn4ifr16y58sbk6j6l5v9c8n8b"; + sha256 = "1mx5paq837rjw7w51zprrjynk1vaig9yzxfqz9ac79jmd7f3w5fj"; dependencies = [ { name = "indexmap"; @@ -11429,9 +11401,9 @@ rec { }; "tonic" = rec { crateName = "tonic"; - version = "0.14.5"; - edition = "2021"; - sha256 = "1v4k7aa28m7722gz9qak2jiy7lis1ycm4fdmq63iip4m0qdcdizy"; + version = "0.14.6"; + edition = "2024"; + sha256 = "1vs5ci6z6b9xhfsnx4s8qx6bqi1zzcrxncjp71147a0gqwc5aamc"; authors = [ "Lucio Franco " ]; @@ -11558,9 +11530,9 @@ rec { }; "tonic-prost" = rec { crateName = "tonic-prost"; - version = "0.14.5"; - edition = "2021"; - sha256 = "02fkg2bv87q0yds2wz3w0s7i1x6qcgbrl00dy6ipajdapfh7clx5"; + version = "0.14.6"; + edition = "2024"; + sha256 = "184y40nf0iyzc5rg32ivgd88snv68sqy1kchynn55r1vhml9z12h"; libName = "tonic_prost"; authors = [ "Lucio Franco " @@ -11581,6 +11553,33 @@ rec { } ]; + }; + "tonic-types" = rec { + crateName = "tonic-types"; + version = "0.14.6"; + edition = "2024"; + sha256 = "1s286gg71pjajny8xar0azq1w9lgz1ks3jm3pccxb0qz0q11pavk"; + libName = "tonic_types"; + authors = [ + "Lucio Franco " + "Rafael Lemos " + ]; + dependencies = [ + { + name = "prost"; + packageId = "prost"; + } + { + name = "prost-types"; + packageId = "prost-types"; + } + { + name = "tonic"; + packageId = "tonic"; + usesDefaultFeatures = false; + } + ]; + }; "tower" = rec { crateName = "tower"; @@ -11702,9 +11701,9 @@ rec { }; "tower-http" = rec { crateName = "tower-http"; - version = "0.6.8"; + version = "0.6.11"; edition = "2018"; - sha256 = "1y514jwzbyrmrkbaajpwmss4rg0mak82k16d6588w9ncaffmbrnl"; + sha256 = "0h08wjgs3hwnq11iwwzlmnabn1h4cl0fzd48svaccvqffkiggz2c"; libName = "tower_http"; authors = [ "Tower Maintainers " @@ -11738,11 +11737,6 @@ rec { packageId = "http-body"; optional = true; } - { - name = "iri-string"; - packageId = "iri-string"; - optional = true; - } { name = "mime"; packageId = "mime"; @@ -11772,6 +11766,11 @@ rec { optional = true; usesDefaultFeatures = false; } + { + name = "url"; + packageId = "url"; + optional = true; + } ]; devDependencies = [ { @@ -11793,35 +11792,33 @@ rec { } ]; features = { - "async-compression" = [ "dep:async-compression" ]; "auth" = [ "base64" "validate-request" ]; "base64" = [ "dep:base64" ]; "catch-panic" = [ "tracing" "futures-util/std" "dep:http-body" "dep:http-body-util" ]; - "compression-br" = [ "async-compression/brotli" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; - "compression-deflate" = [ "async-compression/zlib" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; + "compression-br" = [ "dep:async-compression" "async-compression?/brotli" "futures-core" "dep:http-body" "tokio-util" "dep:tokio" ]; + "compression-deflate" = [ "dep:async-compression" "async-compression?/zlib" "futures-core" "dep:http-body" "tokio-util" "dep:tokio" ]; "compression-full" = [ "compression-br" "compression-deflate" "compression-gzip" "compression-zstd" ]; - "compression-gzip" = [ "async-compression/gzip" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; - "compression-zstd" = [ "async-compression/zstd" "futures-core" "dep:http-body" "tokio-util" "tokio" ]; - "decompression-br" = [ "async-compression/brotli" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; - "decompression-deflate" = [ "async-compression/zlib" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; + "compression-gzip" = [ "dep:async-compression" "async-compression?/gzip" "futures-core" "dep:http-body" "tokio-util" "dep:tokio" ]; + "compression-zstd" = [ "dep:async-compression" "async-compression?/zstd" "futures-core" "dep:http-body" "tokio-util" "dep:tokio" ]; + "decompression-br" = [ "dep:async-compression" "async-compression?/brotli" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "dep:tokio" ]; + "decompression-deflate" = [ "dep:async-compression" "async-compression?/zlib" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "dep:tokio" ]; "decompression-full" = [ "decompression-br" "decompression-deflate" "decompression-gzip" "decompression-zstd" ]; - "decompression-gzip" = [ "async-compression/gzip" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; - "decompression-zstd" = [ "async-compression/zstd" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "tokio" ]; - "follow-redirect" = [ "futures-util" "dep:http-body" "iri-string" "tower/util" ]; - "fs" = [ "futures-core" "futures-util" "dep:http-body" "dep:http-body-util" "tokio/fs" "tokio-util/io" "tokio/io-util" "dep:http-range-header" "mime_guess" "mime" "percent-encoding" "httpdate" "set-status" "futures-util/alloc" "tracing" ]; - "full" = [ "add-extension" "auth" "catch-panic" "compression-full" "cors" "decompression-full" "follow-redirect" "fs" "limit" "map-request-body" "map-response-body" "metrics" "normalize-path" "propagate-header" "redirect" "request-id" "sensitive-headers" "set-header" "set-status" "timeout" "trace" "util" "validate-request" ]; + "decompression-gzip" = [ "dep:async-compression" "async-compression?/gzip" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "dep:tokio" ]; + "decompression-zstd" = [ "dep:async-compression" "async-compression?/zstd" "futures-core" "dep:http-body" "dep:http-body-util" "tokio-util" "dep:tokio" ]; + "follow-redirect" = [ "futures-util" "dep:http-body" "dep:url" "tower/util" ]; + "fs" = [ "dep:tokio" "tokio?/fs" "tokio?/io-util" "futures-core" "futures-util" "dep:http-body" "dep:http-body-util" "tokio-util/io" "dep:http-range-header" "mime_guess" "mime" "percent-encoding" "httpdate" "set-status" "futures-util/alloc" ]; + "full" = [ "add-extension" "auth" "catch-panic" "compression-full" "cors" "decompression-full" "follow-redirect" "fs" "limit" "map-request-body" "map-response-body" "metrics" "normalize-path" "on-early-drop" "propagate-header" "redirect" "request-id" "sensitive-headers" "set-header" "set-status" "timeout" "trace" "util" "validate-request" ]; "futures-core" = [ "dep:futures-core" ]; "futures-util" = [ "dep:futures-util" ]; "httpdate" = [ "dep:httpdate" ]; - "iri-string" = [ "dep:iri-string" ]; "limit" = [ "dep:http-body" "dep:http-body-util" ]; - "metrics" = [ "dep:http-body" "tokio/time" ]; + "metrics" = [ "dep:http-body" "dep:tokio" "tokio?/time" ]; "mime" = [ "dep:mime" ]; "mime_guess" = [ "dep:mime_guess" ]; + "on-early-drop" = [ "dep:http-body" ]; "percent-encoding" = [ "dep:percent-encoding" ]; "request-id" = [ "uuid" ]; - "timeout" = [ "dep:http-body" "tokio/time" ]; - "tokio" = [ "dep:tokio" ]; + "timeout" = [ "dep:http-body" "dep:tokio" "tokio?/time" ]; "tokio-util" = [ "dep:tokio-util" ]; "tower" = [ "dep:tower" ]; "trace" = [ "dep:http-body" "tracing" ]; @@ -11830,7 +11827,7 @@ rec { "uuid" = [ "dep:uuid" ]; "validate-request" = [ "mime" ]; }; - resolvedDefaultFeatures = [ "auth" "base64" "default" "follow-redirect" "futures-util" "iri-string" "map-response-body" "mime" "tower" "trace" "tracing" "util" "validate-request" ]; + resolvedDefaultFeatures = [ "auth" "base64" "default" "follow-redirect" "futures-util" "map-response-body" "mime" "tower" "trace" "tracing" "util" "validate-request" ]; }; "tower-layer" = rec { crateName = "tower-layer"; @@ -12037,9 +12034,9 @@ rec { }; "tracing-opentelemetry" = rec { crateName = "tracing-opentelemetry"; - version = "0.32.1"; + version = "0.33.0"; edition = "2021"; - sha256 = "1z2jjmxbkm1qawlb3bm99x8xwf4g8wjkbcknm9z4fv1w14nqzhhs"; + sha256 = "09nvxy5m7nxmifz4b6szdcyczapp2jcgxcac0jw4ax8klz5n9g5d"; libName = "tracing_opentelemetry"; dependencies = [ { @@ -12274,13 +12271,9 @@ rec { }; "typenum" = rec { crateName = "typenum"; - version = "1.20.0"; + version = "1.20.1"; edition = "2018"; - sha256 = "1pj35y6q11d3y55gdl6g1h2dfhmybjming0jdi9bh0bpnqm11kj0"; - authors = [ - "Paho Lurie-Gregg " - "Andre Bogus " - ]; + sha256 = "086s9ly0906kw5yw41249fba97w5zfxf03pyfwdkffvcprqfixdn"; features = { "scale-info" = [ "dep:scale-info" ]; "scale_info" = [ "scale-info/derive" ]; @@ -12313,9 +12306,9 @@ rec { }; "unicode-segmentation" = rec { crateName = "unicode-segmentation"; - version = "1.13.2"; + version = "1.13.3"; edition = "2018"; - sha256 = "135a26m4a0wj319gcw28j6a5aqvz00jmgwgmcs6szgxjf942facn"; + sha256 = "1a47zaq83p386r3baq4m018xd5q4q0grdg56i1x042dzn71x7xf6"; libName = "unicode_segmentation"; authors = [ "kwantam " @@ -12441,6 +12434,66 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; + "uuid" = rec { + crateName = "uuid"; + version = "1.23.3"; + edition = "2021"; + sha256 = "1drddl03gi12vl1s3l2h371dw39plhn9wappp00v707g7h96nk8l"; + authors = [ + "Ashley Mannix" + "Dylan DPC" + "Hunar Roop Kahlon" + ]; + dependencies = [ + { + name = "js-sys"; + packageId = "js-sys"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)) && (builtins.elem "atomics" targetFeatures)); + } + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); + } + ]; + devDependencies = [ + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + "atomic" = [ "dep:atomic" ]; + "borsh" = [ "dep:borsh" "dep:borsh-derive" ]; + "bytemuck" = [ "dep:bytemuck" ]; + "default" = [ "std" ]; + "fast-rng" = [ "rng" "dep:rand" ]; + "js" = [ "dep:wasm-bindgen" "dep:js-sys" ]; + "md5" = [ "dep:md-5" ]; + "rng" = [ "dep:getrandom" ]; + "rng-getrandom" = [ "rng" "dep:getrandom" "uuid-rng-internal-lib" "uuid-rng-internal-lib/getrandom" ]; + "rng-rand" = [ "rng" "dep:rand" "uuid-rng-internal-lib" "uuid-rng-internal-lib/rand" ]; + "serde" = [ "dep:serde_core" ]; + "sha1" = [ "dep:sha1_smol" ]; + "slog" = [ "dep:slog" ]; + "std" = [ "wasm-bindgen?/std" "js-sys?/std" ]; + "uuid-rng-internal-lib" = [ "dep:uuid-rng-internal-lib" ]; + "v1" = [ "atomic" ]; + "v3" = [ "md5" ]; + "v4" = [ "rng" ]; + "v5" = [ "sha1" ]; + "v6" = [ "atomic" ]; + "v7" = [ "rng" ]; + "zerocopy" = [ "dep:zerocopy" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "valuable" = rec { crateName = "valuable"; version = "0.1.1"; @@ -12508,9 +12561,9 @@ rec { }; "wasip2" = rec { crateName = "wasip2"; - version = "1.0.3+wasi-0.2.9"; + version = "1.0.4+wasi-0.2.12"; edition = "2021"; - sha256 = "1mi3w855dz99xzjqc4aa8c9q5b6z1y5c963pkk4cvmr6vdr4c1i0"; + sha256 = "11wl7lqwq4pbmlmzr6n7bwz0hzy1z6sxc4554bkmrr86w4vznzmn"; dependencies = [ { name = "wit-bindgen"; @@ -12528,9 +12581,9 @@ rec { }; "wasm-bindgen" = rec { crateName = "wasm-bindgen"; - version = "0.2.120"; + version = "0.2.125"; edition = "2021"; - sha256 = "1cax9wy1n67sa2m16ia72lsxdrc5pzcv47psxp4p833yp3cvclnz"; + sha256 = "06nakz7nfy0ymyp7a27wfbjwx69659i12117hkgddkiv2iwkznwd"; libName = "wasm_bindgen"; authors = [ "The wasm-bindgen Developers" @@ -12579,9 +12632,9 @@ rec { }; "wasm-bindgen-futures" = rec { crateName = "wasm-bindgen-futures"; - version = "0.4.70"; + version = "0.4.75"; edition = "2021"; - sha256 = "1130a64yig0p10mk7rnq5l2jpwglbyxpnqg6h0nlqwzcmir4i4xg"; + sha256 = "104jssshr6cm5hmkn6c66mbkyxgaaphng6c17g0dmj7jhk918fsh"; libName = "wasm_bindgen_futures"; authors = [ "The wasm-bindgen Developers" @@ -12607,9 +12660,9 @@ rec { }; "wasm-bindgen-macro" = rec { crateName = "wasm-bindgen-macro"; - version = "0.2.120"; + version = "0.2.125"; edition = "2021"; - sha256 = "00xixpydzjd6y9knwdsrsiff6wi1ddszb1fa9bk25csz94gh9cbq"; + sha256 = "0g9w68dwcs4ylm5kxf7schi0kjdfarhc9qlnf8arxc9zn62a28af"; procMacro = true; libName = "wasm_bindgen_macro"; authors = [ @@ -12631,9 +12684,9 @@ rec { }; "wasm-bindgen-macro-support" = rec { crateName = "wasm-bindgen-macro-support"; - version = "0.2.120"; + version = "0.2.125"; edition = "2021"; - sha256 = "0h8v5aphaaq155fzz2d312zrxbka1x6lsvb8mhc8m60n0kr0zkcx"; + sha256 = "1gayzdx5iwl8gllh7ys79wg9cf4iyasl9hrzzhh5m4xx6nfgvkpy"; libName = "wasm_bindgen_macro_support"; authors = [ "The wasm-bindgen Developers" @@ -12667,10 +12720,10 @@ rec { }; "wasm-bindgen-shared" = rec { crateName = "wasm-bindgen-shared"; - version = "0.2.120"; + version = "0.2.125"; edition = "2021"; links = "wasm_bindgen"; - sha256 = "1sjrac4f4j5pgw42mxadq9v42f0bji1a2rcksrbnrwgbh8y7nxa9"; + sha256 = "07w7fy5qa14ys3p8v2p84h98yqinw713smibz9v7apcspd29x4r3"; libName = "wasm_bindgen_shared"; authors = [ "The wasm-bindgen Developers" @@ -12685,9 +12738,9 @@ rec { }; "web-sys" = rec { crateName = "web-sys"; - version = "0.3.97"; + version = "0.3.102"; edition = "2021"; - sha256 = "00knh8cizgb01bmh362f3f19b11f8zl1y3gj6h47pk95233vmb9f"; + sha256 = "0786aybrnwsgdmcynhc2k5ii291a02rq9zk054j35csyvxr0lhx6"; libName = "web_sys"; authors = [ "The wasm-bindgen Developers" @@ -12771,6 +12824,7 @@ rec { "CssStyleSheet" = [ "StyleSheet" ]; "CssSupportsRule" = [ "CssConditionRule" "CssGroupingRule" "CssRule" ]; "CssTransition" = [ "Animation" "EventTarget" ]; + "CssViewTransitionRule" = [ "CssRule" ]; "CustomEvent" = [ "Event" ]; "DedicatedWorkerGlobalScope" = [ "EventTarget" "WorkerGlobalScope" ]; "DelayNode" = [ "AudioNode" "EventTarget" ]; @@ -13847,7 +13901,7 @@ rec { "Win32_Web" = [ "Win32" ]; "Win32_Web_InternetExplorer" = [ "Win32_Web" ]; }; - resolvedDefaultFeatures = [ "Wdk" "Wdk_Foundation" "Wdk_Storage" "Wdk_Storage_FileSystem" "Wdk_System" "Wdk_System_IO" "Win32" "Win32_Foundation" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_IO" "Win32_System_LibraryLoader" "Win32_System_Memory" "Win32_System_Pipes" "Win32_System_SystemInformation" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_Time" "Win32_System_WindowsProgramming" "default" ]; + resolvedDefaultFeatures = [ "Wdk" "Wdk_Foundation" "Wdk_Storage" "Wdk_Storage_FileSystem" "Wdk_System" "Wdk_System_IO" "Win32" "Win32_Foundation" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_IO" "Win32_System_LibraryLoader" "Win32_System_Memory" "Win32_System_Pipes" "Win32_System_SystemInformation" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_WindowsProgramming" "default" ]; }; "windows-targets" = rec { crateName = "windows-targets"; @@ -13984,9 +14038,9 @@ rec { }; "winnow" = rec { crateName = "winnow"; - version = "1.0.2"; + version = "1.0.3"; edition = "2021"; - sha256 = "1l7xnfvlgy4da6gq5ip2bgcm8i9d0rwzaxg1p88nlw8lxy5p1q9f"; + sha256 = "1wajycd3krn6h699vydjv7hm0ll5l31p899qzpk59y2is74y34h5"; dependencies = [ { name = "memchr"; @@ -14098,9 +14152,9 @@ rec { }; "xml" = rec { crateName = "xml"; - version = "1.2.1"; + version = "1.3.0"; edition = "2021"; - sha256 = "0ak4k990faralbli5a0rb8kvwihccb2rp0r94d4azfy94a6lkamq"; + sha256 = "128s58qhq8whrx90zbw8r5algr7lakgbf7mn05jfk234rbjqavv3"; authors = [ "Vladimir Matveev " "Kornel (https://github.com/kornelski)" @@ -14109,9 +14163,9 @@ rec { }; "yoke" = rec { crateName = "yoke"; - version = "0.8.2"; + version = "0.8.3"; edition = "2021"; - sha256 = "1jprcs7a98a5whvfs6r3jvfh1nnfp6zyijl7y4ywmn88lzywbs5b"; + sha256 = "1xgyj6c2lxj2bp891ynmhws87c6z7yyv2li1v0ss9di40hxf57vh"; authors = [ "Manish Goregaokar " ]; @@ -14175,9 +14229,9 @@ rec { }; "zerocopy" = rec { crateName = "zerocopy"; - version = "0.8.48"; + version = "0.8.52"; edition = "2021"; - sha256 = "1sb8plax8jbrsng1jdval7bdhk7hhrx40dz3hwh074k6knzkgm7f"; + sha256 = "0gv563swc1yn3k8w3wjj07a8q293rkx99nfp3a25vzzmbycj446f"; authors = [ "Joshua Liebow-Feeser " "Jack Wrenn " @@ -14211,9 +14265,9 @@ rec { }; "zerocopy-derive" = rec { crateName = "zerocopy-derive"; - version = "0.8.48"; + version = "0.8.52"; edition = "2021"; - sha256 = "1m5s0g92cxggqc74j83k1priz24k3z93sj5gadppd20p9c4cvqvh"; + sha256 = "0c3rhsh4sd9kdym4z55zprybjkydy9y2gvw75d72aapcfa5z7rqs"; procMacro = true; libName = "zerocopy_derive"; authors = [ @@ -14246,11 +14300,11 @@ rec { }; "zerofrom" = rec { crateName = "zerofrom"; - version = "0.1.7"; + version = "0.1.8"; edition = "2021"; - sha256 = "1py40in4rirc9q8w36q67pld0zk8ssg024xhh0cncxgal7ra3yk9"; + sha256 = "0wjjdj7gdmd0iq91gzkxl7dlv0nhkk80l4bmdpzh3a1yh48mmh0f"; authors = [ - "Manish Goregaokar " + "The ICU4X Project Developers" ]; dependencies = [ { @@ -14299,9 +14353,9 @@ rec { }; "zeroize" = rec { crateName = "zeroize"; - version = "1.8.2"; - edition = "2021"; - sha256 = "1l48zxgcv34d7kjskr610zqsm6j2b4fcr2vfh9jm9j1jgvk58wdr"; + version = "1.9.0"; + edition = "2024"; + sha256 = "0kpnij2v1ig6g2mhc0bnci0lrdfdhiq40afbc0fahajqc9jiag71"; authors = [ "The RustCrypto Project Developers" ]; @@ -14323,9 +14377,9 @@ rec { }; "zeroize_derive" = rec { crateName = "zeroize_derive"; - version = "1.4.3"; - edition = "2021"; - sha256 = "0bl5vd1lz27p4z336nximg5wrlw5j7jc8fxh7iv6r1wrhhav99c5"; + version = "1.5.0"; + edition = "2024"; + sha256 = "0a7kq8srk81pn23xqn7c9jw1jpnfy41ffn802x1zrqqgpdf6al1w"; procMacro = true; authors = [ "The RustCrypto Project Developers" diff --git a/Cargo.toml b/Cargo.toml index 99226bcf..f08479ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,12 +10,11 @@ edition = "2024" repository = "https://github.com/stackabletech/druid-operator" [workspace.dependencies] -product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.8.0" } stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.111.0", features = ["crds", "webhook"] } anyhow = "1.0" built = { version = "0.8", features = ["chrono", "git2"] } -clap = "4.5" +clap = "4.6" const_format = "0.2" fnv = "1.0" futures = { version = "0.3", features = ["compat"] } @@ -29,9 +28,11 @@ serde_json = "1.0" serde_yaml = "0.9" snafu = "0.9" strum = { version = "0.28", features = ["derive"] } -tokio = { version = "1.40", features = ["full"] } +tokio = { version = "1.52", features = ["full"] } tracing = "0.1" +uuid = "1.23" [patch."https://github.com/stackabletech/operator-rs.git"] +stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "smooth-operator" } # stackable-operator = { path = "../operator-rs/crates/stackable-operator" } # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } diff --git a/crate-hashes.json b/crate-hashes.json index 71fbc1c3..bb8b51c7 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#k8s-version@0.1.3": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-certs@0.4.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-operator-derive@0.3.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-operator@0.111.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-shared@0.1.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-telemetry@0.6.3": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-versioned-macros@0.10.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-versioned@0.10.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-webhook@0.9.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "07r6kla2132l9dckn0bdxfy22n9x27xxjxmjydxcypdvwh5lk4gr", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file diff --git a/deploy/config-spec/properties.yaml b/deploy/config-spec/properties.yaml index e4ede43b..9bd8c3b2 100644 --- a/deploy/config-spec/properties.yaml +++ b/deploy/config-spec/properties.yaml @@ -1,827 +1,5 @@ --- version: 0.1.0 spec: - units: - - unit: &unitDirectory - name: "directory" - regex: "^/|(/[\\w-]+)+$" - examples: - - "/tmp/xyz" - - unit: &unitPort - name: "port" - regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]\ - |6553[0-5])$" - - unit: &unitPrometheusNamespace - name: "prometheusNamespace" - regex: "^[a-zA-Z_:][a-zA-Z0-9_:]*$" - - unit: &unitDuration - name: "duration" - regex: "^P(?!$)(\\d+Y)?(\\d+M)?(\\d+W)?(\\d+D)?(T(?=\\d)(\\d+H)?(\\d+M)?(\\d+S)\ - ?)?$" - examples: - - "PT300S" - -################################################################################################### -# runtime.properties -# For information on the properties, see: https://druid.apache.org/docs/latest/configuration/index.html -################################################################################################### - -properties: - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "broker" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "coordinator" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "historical" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "middlemanager" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "router" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: true - asOfVersion: "0.0.0" - comment: "TTL for host names that cannot be resolved." - description: "TTL for host names that cannot be resolved." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "coordinator" - required: true - asOfVersion: "0.0.0" - comment: "TTL for host names that cannot be resolved." - description: "TTL for host names that cannot be resolved." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "historical" - required: true - asOfVersion: "0.0.0" - comment: "TTL for host names that cannot be resolved." - description: "TTL for host names that cannot be resolved." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "middlemanager" - required: true - asOfVersion: "0.0.0" - comment: "TTL for host names that cannot be resolved." - description: "TTL for host names that cannot be resolved." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "router" - required: true - asOfVersion: "0.0.0" - comment: "TTL for host names that cannot be resolved." - description: "TTL for host names that cannot be resolved." - - - property: &plaintextPort - propertyNames: - - name: "druid.plaintextPort" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - min: "1" - max: "65535" - unit: *unitPort - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &tlsPort - propertyNames: - - name: "druid.tlsPort" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - min: "1" - max: "65535" - unit: *unitPort - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &startupLoggingLogProperties - propertyNames: - - name: "druid.startup.logging.logProperties" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &monitoringMonitors - propertyNames: - - name: "druid.monitoring.monitors" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "[\"org.apache.druid.java.util.metrics.JvmMonitor\"]" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitter - propertyNames: - - name: "druid.emitter" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "prometheus" - allowedValues: - - "noop" - - "logging" - - "http" - - "parametrized" - - "composing" - - "prometheus" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitterPrometheusStrategy - propertyNames: - - name: "druid.emitter.prometheus.strategy" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "exporter" - allowedValues: - - "exporter" - - "pushgateway" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitterPrometheusNamespace - propertyNames: - - name: "druid.emitter.prometheus.namespace" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitPrometheusNamespace - defaultValues: - - fromVersion: "0.0.0" - value: "druid" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitterPrometheusPort - propertyNames: - - name: "druid.emitter.prometheus.port" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - min: "1" - max: "65535" - unit: *unitPort - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &indexerLogsDirectory - propertyNames: - - name: "druid.indexer.logs.directory" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/indexing-logs" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &processingTmpDir - propertyNames: - - name: "druid.processing.tmpDir" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/processing" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: false - - name: "historical" - required: true - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerTaskHadoopWorkingPath - propertyNames: - - name: "druid.indexer.task.hadoopWorkingPath" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/hadoop-tmp" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: true - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerTaskBaseTaskDir - propertyNames: - - name: "druid.indexer.task.baseTaskDir" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/task" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: true - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerRunnerJavaOpts - propertyNames: - - name: "druid.indexer.runner.javaOpts" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "-server -Xms256m -Xmx256m -XX:MaxDirectMemorySize=300m - -Duser.timezone=UTC -Dfile.encoding=UTF-8 - -XX:+ExitOnOutOfMemoryError - -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: true - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &routerManagementProxyEnabled - propertyNames: - # Management proxy to coordinator / overlord: required for unified web console. - - name: "druid.router.managementProxy.enabled" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &routerHttpNumConnections - propertyNames: - - name: "druid.router.http.numConnections" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - defaultValues: - - fromVersion: "0.0.0" - value: "25" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &historicalCacheUseCache - propertyNames: - - name: "druid.historical.cache.useCache" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: true - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &historicalCachePopulateCache - propertyNames: - - name: "druid.historical.cache.populateCache" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: true - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorStartDelay - propertyNames: - - name: "druid.coordinator.startDelay" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDuration - defaultValues: - - fromVersion: "0.0.0" - value: "PT20S" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerQueueStartDelay - propertyNames: - - name: "druid.indexer.queue.startDelay" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDuration - defaultValues: - - fromVersion: "0.0.0" - value: "PT20S" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerRunnerType - propertyNames: - - name: "druid.indexer.runner.type" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - allowedValues: - - "local" - - "remote" - - "httpRemote" - defaultValues: - - fromVersion: "0.0.0" - value: "remote" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerStorageType - propertyNames: - - name: "druid.indexer.storage.type" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - allowedValues: - - "local" - - "metadata" - defaultValues: - - fromVersion: "0.0.0" - value: "metadata" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorPeriod - propertyNames: - - name: "druid.coordinator.period" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDuration - defaultValues: - - fromVersion: "0.0.0" - value: "PT20S" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorAsOverlordEnabled - propertyNames: - - name: "druid.coordinator.asOverlord.enabled" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorAsOverlordOverlordService - propertyNames: - - name: "druid.coordinator.asOverlord.overlordService" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "druid/overlord" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - -################################################################################################### -# jvm.config -################################################################################################### + units: [] +properties: [] diff --git a/deploy/helm/druid-operator/configs/properties.yaml b/deploy/helm/druid-operator/configs/properties.yaml index e4ede43b..9bd8c3b2 100644 --- a/deploy/helm/druid-operator/configs/properties.yaml +++ b/deploy/helm/druid-operator/configs/properties.yaml @@ -1,827 +1,5 @@ --- version: 0.1.0 spec: - units: - - unit: &unitDirectory - name: "directory" - regex: "^/|(/[\\w-]+)+$" - examples: - - "/tmp/xyz" - - unit: &unitPort - name: "port" - regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]\ - |6553[0-5])$" - - unit: &unitPrometheusNamespace - name: "prometheusNamespace" - regex: "^[a-zA-Z_:][a-zA-Z0-9_:]*$" - - unit: &unitDuration - name: "duration" - regex: "^P(?!$)(\\d+Y)?(\\d+M)?(\\d+W)?(\\d+D)?(T(?=\\d)(\\d+H)?(\\d+M)?(\\d+S)\ - ?)?$" - examples: - - "PT300S" - -################################################################################################### -# runtime.properties -# For information on the properties, see: https://druid.apache.org/docs/latest/configuration/index.html -################################################################################################### - -properties: - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "broker" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "coordinator" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "historical" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "middlemanager" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "30" - roles: - - name: "router" - required: true - asOfVersion: "0.0.0" - comment: "TTL for successfully resolved domain names." - description: "TTL for successfully resolved domain names." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "broker" - required: true - asOfVersion: "0.0.0" - comment: "TTL for host names that cannot be resolved." - description: "TTL for host names that cannot be resolved." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "coordinator" - required: true - asOfVersion: "0.0.0" - comment: "TTL for host names that cannot be resolved." - description: "TTL for host names that cannot be resolved." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "historical" - required: true - asOfVersion: "0.0.0" - comment: "TTL for host names that cannot be resolved." - description: "TTL for host names that cannot be resolved." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "middlemanager" - required: true - asOfVersion: "0.0.0" - comment: "TTL for host names that cannot be resolved." - description: "TTL for host names that cannot be resolved." - - - property: - propertyNames: - - name: "networkaddress.cache.negative.ttl" - kind: - type: "file" - file: "security.properties" - datatype: - type: "integer" - min: "0" - recommendedValues: - - fromVersion: "0.0.0" - value: "0" - roles: - - name: "router" - required: true - asOfVersion: "0.0.0" - comment: "TTL for host names that cannot be resolved." - description: "TTL for host names that cannot be resolved." - - - property: &plaintextPort - propertyNames: - - name: "druid.plaintextPort" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - min: "1" - max: "65535" - unit: *unitPort - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &tlsPort - propertyNames: - - name: "druid.tlsPort" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - min: "1" - max: "65535" - unit: *unitPort - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &startupLoggingLogProperties - propertyNames: - - name: "druid.startup.logging.logProperties" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &monitoringMonitors - propertyNames: - - name: "druid.monitoring.monitors" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "[\"org.apache.druid.java.util.metrics.JvmMonitor\"]" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitter - propertyNames: - - name: "druid.emitter" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "prometheus" - allowedValues: - - "noop" - - "logging" - - "http" - - "parametrized" - - "composing" - - "prometheus" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitterPrometheusStrategy - propertyNames: - - name: "druid.emitter.prometheus.strategy" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "exporter" - allowedValues: - - "exporter" - - "pushgateway" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitterPrometheusNamespace - propertyNames: - - name: "druid.emitter.prometheus.namespace" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitPrometheusNamespace - defaultValues: - - fromVersion: "0.0.0" - value: "druid" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitterPrometheusPort - propertyNames: - - name: "druid.emitter.prometheus.port" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - min: "1" - max: "65535" - unit: *unitPort - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &indexerLogsDirectory - propertyNames: - - name: "druid.indexer.logs.directory" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/indexing-logs" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &processingTmpDir - propertyNames: - - name: "druid.processing.tmpDir" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/processing" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: false - - name: "historical" - required: true - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerTaskHadoopWorkingPath - propertyNames: - - name: "druid.indexer.task.hadoopWorkingPath" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/hadoop-tmp" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: true - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerTaskBaseTaskDir - propertyNames: - - name: "druid.indexer.task.baseTaskDir" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/task" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: true - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerRunnerJavaOpts - propertyNames: - - name: "druid.indexer.runner.javaOpts" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "-server -Xms256m -Xmx256m -XX:MaxDirectMemorySize=300m - -Duser.timezone=UTC -Dfile.encoding=UTF-8 - -XX:+ExitOnOutOfMemoryError - -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: true - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &routerManagementProxyEnabled - propertyNames: - # Management proxy to coordinator / overlord: required for unified web console. - - name: "druid.router.managementProxy.enabled" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &routerHttpNumConnections - propertyNames: - - name: "druid.router.http.numConnections" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - defaultValues: - - fromVersion: "0.0.0" - value: "25" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &historicalCacheUseCache - propertyNames: - - name: "druid.historical.cache.useCache" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: true - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &historicalCachePopulateCache - propertyNames: - - name: "druid.historical.cache.populateCache" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: true - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorStartDelay - propertyNames: - - name: "druid.coordinator.startDelay" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDuration - defaultValues: - - fromVersion: "0.0.0" - value: "PT20S" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerQueueStartDelay - propertyNames: - - name: "druid.indexer.queue.startDelay" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDuration - defaultValues: - - fromVersion: "0.0.0" - value: "PT20S" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerRunnerType - propertyNames: - - name: "druid.indexer.runner.type" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - allowedValues: - - "local" - - "remote" - - "httpRemote" - defaultValues: - - fromVersion: "0.0.0" - value: "remote" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerStorageType - propertyNames: - - name: "druid.indexer.storage.type" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - allowedValues: - - "local" - - "metadata" - defaultValues: - - fromVersion: "0.0.0" - value: "metadata" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorPeriod - propertyNames: - - name: "druid.coordinator.period" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDuration - defaultValues: - - fromVersion: "0.0.0" - value: "PT20S" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorAsOverlordEnabled - propertyNames: - - name: "druid.coordinator.asOverlord.enabled" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorAsOverlordOverlordService - propertyNames: - - name: "druid.coordinator.asOverlord.overlordService" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "druid/overlord" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - -################################################################################################### -# jvm.config -################################################################################################### + units: [] +properties: [] diff --git a/docs/modules/druid/pages/reference/commandline-parameters.adoc b/docs/modules/druid/pages/reference/commandline-parameters.adoc index 2aeb7591..3278c897 100644 --- a/docs/modules/druid/pages/reference/commandline-parameters.adoc +++ b/docs/modules/druid/pages/reference/commandline-parameters.adoc @@ -2,19 +2,6 @@ This operator accepts the following command line parameters: -== product-config - -*Default value*: `/etc/stackable/druid-operator/config-spec/properties.yaml` - -*Required*: false - -*Multiple values:* false - -[source] ----- -stackable-druid-operator run --product-config /foo/bar/properties.yaml ----- - == watch-namespace *Default value*: All namespaces diff --git a/docs/modules/druid/pages/reference/environment-variables.adoc b/docs/modules/druid/pages/reference/environment-variables.adoc index 2083b351..1f603488 100644 --- a/docs/modules/druid/pages/reference/environment-variables.adoc +++ b/docs/modules/druid/pages/reference/environment-variables.adoc @@ -33,32 +33,6 @@ docker run \ oci.stackable.tech/sdp/druid-operator:0.0.0-dev ---- -== PRODUCT_CONFIG - -*Default value*: `/etc/stackable/druid-operator/config-spec/properties.yaml` - -*Required*: false - -*Multiple values*: false - -[source] ----- -export PRODUCT_CONFIG=/foo/bar/properties.yaml -stackable-druid-operator run ----- - -or via docker: - ----- -docker run \ - --name druid-operator \ - --network host \ - --env KUBECONFIG=/home/stackable/.kube/config \ - --env PRODUCT_CONFIG=/my/product/config.yaml \ - --mount type=bind,source="$HOME/.kube/config",target="/home/stackable/.kube/config" \ - oci.stackable.tech/sdp/druid-operator:0.0.0-dev ----- - == WATCH_NAMESPACE *Default value*: All namespaces diff --git a/extra/crds.yaml b/extra/crds.yaml index 73ad0b48..3bad58df 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -413,20 +413,20 @@ spec: jvm.config: additionalProperties: type: string + default: {} description: Overrides for the `jvm.config` file. - nullable: true type: object runtime.properties: additionalProperties: type: string + default: {} description: Overrides for the `runtime.properties` file. - nullable: true type: object security.properties: additionalProperties: type: string + default: {} description: Overrides for the `security.properties` file. - nullable: true type: object type: object envOverrides: @@ -491,6 +491,9 @@ spec: properties: listenerClass: default: cluster-internal + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string podDisruptionBudget: default: @@ -904,20 +907,20 @@ spec: jvm.config: additionalProperties: type: string + default: {} description: Overrides for the `jvm.config` file. - nullable: true type: object runtime.properties: additionalProperties: type: string + default: {} description: Overrides for the `runtime.properties` file. - nullable: true type: object security.properties: additionalProperties: type: string + default: {} description: Overrides for the `security.properties` file. - nullable: true type: object type: object envOverrides: @@ -1091,6 +1094,9 @@ spec: The [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) for the HDFS instance. When running an HDFS cluster with the Stackable operator, the operator will create this ConfigMap for you. It has the same name as your HDFSCluster resource. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string directory: description: The directory inside of HDFS where Druid should store its data. @@ -1574,7 +1580,10 @@ spec: - If TLS encryption is used at all - Which cert the servers should use to authenticate themselves against the clients - Which cert the servers should use to authenticate themselves among each other + maxLength: 253 + minLength: 1 nullable: true + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string type: object vectorAggregatorConfigMapName: @@ -1583,7 +1592,10 @@ spec: It must contain the key `ADDRESS` with the address of the Vector aggregator. Follow the [logging tutorial](https://docs.stackable.tech/home/nightly/tutorials/logging-vector-aggregator) to learn how to configure log aggregation with Vector. + maxLength: 253 + minLength: 1 nullable: true + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string zookeeperConfigMapName: description: |- @@ -1591,6 +1603,9 @@ spec: Provide the name of the ZooKeeper [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) here. When using the [Stackable operator for Apache ZooKeeper](https://docs.stackable.tech/home/nightly/zookeeper/) to deploy a ZooKeeper cluster, this will simply be the name of your ZookeeperCluster resource. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string required: - deepStorage @@ -2012,20 +2027,20 @@ spec: jvm.config: additionalProperties: type: string + default: {} description: Overrides for the `jvm.config` file. - nullable: true type: object runtime.properties: additionalProperties: type: string + default: {} description: Overrides for the `runtime.properties` file. - nullable: true type: object security.properties: additionalProperties: type: string + default: {} description: Overrides for the `security.properties` file. - nullable: true type: object type: object envOverrides: @@ -2090,6 +2105,9 @@ spec: properties: listenerClass: default: cluster-internal + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string podDisruptionBudget: default: @@ -2503,20 +2521,20 @@ spec: jvm.config: additionalProperties: type: string + default: {} description: Overrides for the `jvm.config` file. - nullable: true type: object runtime.properties: additionalProperties: type: string + default: {} description: Overrides for the `runtime.properties` file. - nullable: true type: object security.properties: additionalProperties: type: string + default: {} description: Overrides for the `security.properties` file. - nullable: true type: object type: object envOverrides: @@ -3013,20 +3031,20 @@ spec: jvm.config: additionalProperties: type: string + default: {} description: Overrides for the `jvm.config` file. - nullable: true type: object runtime.properties: additionalProperties: type: string + default: {} description: Overrides for the `runtime.properties` file. - nullable: true type: object security.properties: additionalProperties: type: string + default: {} description: Overrides for the `security.properties` file. - nullable: true type: object type: object envOverrides: @@ -3544,20 +3562,20 @@ spec: jvm.config: additionalProperties: type: string + default: {} description: Overrides for the `jvm.config` file. - nullable: true type: object runtime.properties: additionalProperties: type: string + default: {} description: Overrides for the `runtime.properties` file. - nullable: true type: object security.properties: additionalProperties: type: string + default: {} description: Overrides for the `security.properties` file. - nullable: true type: object type: object envOverrides: @@ -4073,20 +4091,20 @@ spec: jvm.config: additionalProperties: type: string + default: {} description: Overrides for the `jvm.config` file. - nullable: true type: object runtime.properties: additionalProperties: type: string + default: {} description: Overrides for the `runtime.properties` file. - nullable: true type: object security.properties: additionalProperties: type: string + default: {} description: Overrides for the `security.properties` file. - nullable: true type: object type: object envOverrides: @@ -4560,20 +4578,20 @@ spec: jvm.config: additionalProperties: type: string + default: {} description: Overrides for the `jvm.config` file. - nullable: true type: object runtime.properties: additionalProperties: type: string + default: {} description: Overrides for the `runtime.properties` file. - nullable: true type: object security.properties: additionalProperties: type: string + default: {} description: Overrides for the `security.properties` file. - nullable: true type: object type: object envOverrides: @@ -5040,20 +5058,20 @@ spec: jvm.config: additionalProperties: type: string + default: {} description: Overrides for the `jvm.config` file. - nullable: true type: object runtime.properties: additionalProperties: type: string + default: {} description: Overrides for the `runtime.properties` file. - nullable: true type: object security.properties: additionalProperties: type: string + default: {} description: Overrides for the `security.properties` file. - nullable: true type: object type: object envOverrides: @@ -5118,6 +5136,9 @@ spec: properties: listenerClass: default: cluster-internal + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string podDisruptionBudget: default: @@ -5531,20 +5552,20 @@ spec: jvm.config: additionalProperties: type: string + default: {} description: Overrides for the `jvm.config` file. - nullable: true type: object runtime.properties: additionalProperties: type: string + default: {} description: Overrides for the `runtime.properties` file. - nullable: true type: object security.properties: additionalProperties: type: string + default: {} description: Overrides for the `security.properties` file. - nullable: true type: object type: object envOverrides: diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 6bce1b1f..225c5d44 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -9,7 +9,6 @@ repository.workspace = true publish = false [dependencies] -product-config.workspace = true stackable-operator.workspace = true anyhow.workspace = true @@ -27,6 +26,7 @@ snafu.workspace = true strum.workspace = true tokio.workspace = true tracing.workspace = true +uuid.workspace = true [build-dependencies] built.workspace = true diff --git a/rust/operator-binary/src/authentication/ldap.rs b/rust/operator-binary/src/authentication/ldap.rs index efc99471..730de69b 100644 --- a/rust/operator-binary/src/authentication/ldap.rs +++ b/rust/operator-binary/src/authentication/ldap.rs @@ -9,32 +9,33 @@ use stackable_operator::{ use super::{ AddLdapVolumesSnafu, ConstructLdapEndpointUrlSnafu, Error, MissingLdapBindCredentialsSnafu, }; -use crate::crd::security::{STACKABLE_TLS_DIR, TLS_STORE_PASSWORD, add_cert_to_trust_store_cmd}; +use crate::crd::{ + file_reference, + security::{STACKABLE_TLS_DIR, TLS_STORE_PASSWORD, add_cert_to_trust_store_cmd}, +}; fn add_authenticator_config( provider: &ldap::v1alpha1::AuthenticationProvider, - config: &mut BTreeMap>, + config: &mut BTreeMap, ) -> Result<(), Error> { config.insert( "druid.auth.authenticator.Ldap.type".to_string(), - Some("basic".to_string()), + "basic".to_string(), ); config.insert( "druid.auth.authenticator.Ldap.enableCacheNotifications".to_string(), - Some("true".to_string()), + "true".to_string(), ); config.insert( "druid.auth.authenticator.Ldap.credentialsValidator.type".to_string(), - Some("ldap".to_string()), + "ldap".to_string(), ); config.insert( "druid.auth.authenticator.Ldap.credentialsValidator.url".to_string(), - Some( - provider - .endpoint_url() - .context(ConstructLdapEndpointUrlSnafu)? - .into(), - ), + provider + .endpoint_url() + .context(ConstructLdapEndpointUrlSnafu)? + .into(), ); if let Some((ldap_bind_user_path, ldap_bind_password_path)) = @@ -42,60 +43,46 @@ fn add_authenticator_config( { config.insert( "druid.auth.authenticator.Ldap.credentialsValidator.bindUser".to_string(), - Some(format!("${{file:UTF-8:{ldap_bind_user_path}}}").to_string()), + file_reference(ldap_bind_user_path), ); config.insert( "druid.auth.authenticator.Ldap.credentialsValidator.bindPassword".to_string(), - Some(format!("${{file:UTF-8:{ldap_bind_password_path}}}").to_string()), + file_reference(ldap_bind_password_path), ); } config.insert( "druid.auth.authenticator.Ldap.credentialsValidator.baseDn".to_string(), - Some(provider.search_base.to_string()), + provider.search_base.to_string(), ); config.insert( "druid.auth.authenticator.Ldap.credentialsValidator.userAttribute".to_string(), - Some(provider.ldap_field_names.uid.to_string()), + provider.ldap_field_names.uid.to_string(), ); config.insert( "druid.auth.authenticator.Ldap.credentialsValidator.userSearch".to_string(), - Some(provider.search_filter.to_string()), + provider.search_filter.to_string(), ); config.insert( "druid.auth.authenticator.Ldap.authorizerName".to_string(), - Some("LdapAuthorizer".to_string()), - ); - config.insert( - "druid.auth.authenticatorChain".to_string(), - Some(r#"["DruidSystemAuthenticator", "Ldap"]"#.to_string()), + "LdapAuthorizer".to_string(), ); + super::set_authenticator_chain(config, &["Ldap"]); Ok(()) } -fn add_authorizer_config(config: &mut BTreeMap>) { - config.insert( - "druid.auth.authorizers".to_string(), - Some(r#"["LdapAuthorizer", "DruidSystemAuthorizer"]"#.to_string()), - ); - config.insert( - "druid.auth.authorizer.LdapAuthorizer.type".to_string(), - Some(r#"allowAll"#.to_string()), - ); -} - -pub fn generate_runtime_properties_config( +pub(super) fn generate_runtime_properties_config( provider: &ldap::v1alpha1::AuthenticationProvider, - config: &mut BTreeMap>, + config: &mut BTreeMap, ) -> Result<(), Error> { add_authenticator_config(provider, config)?; - add_authorizer_config(config); + super::add_authorizer_config(config, "LdapAuthorizer"); Ok(()) } -pub fn prepare_container_commands( +pub(super) fn prepare_container_commands( provider: &ldap::v1alpha1::AuthenticationProvider, command: &mut Vec, ) { @@ -108,7 +95,7 @@ pub fn prepare_container_commands( } } -pub fn add_volumes_and_mounts( +pub(super) fn add_volumes_and_mounts( provider: &ldap::v1alpha1::AuthenticationProvider, pb: &mut PodBuilder, cb_druid: &mut ContainerBuilder, diff --git a/rust/operator-binary/src/authentication/mod.rs b/rust/operator-binary/src/authentication/mod.rs index 548c9213..5baf81ac 100644 --- a/rust/operator-binary/src/authentication/mod.rs +++ b/rust/operator-binary/src/authentication/mod.rs @@ -8,11 +8,12 @@ use stackable_operator::{ }; use crate::{ + controller::validate::ValidatedCluster, crd::{ DruidRole, authentication::{AuthenticationClassResolved, AuthenticationClassesResolved}, - security::{ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV, INTERNAL_INITIAL_CLIENT_PASSWORD_ENV}, - v1alpha1, + env_var_reference, + security::INTERNAL_INITIAL_CLIENT_PASSWORD_ENV, }, internal_secret::{build_shared_internal_secret_name, env_var_from_secret}, }; @@ -20,6 +21,38 @@ use crate::{ pub mod ldap; pub mod oidc; +// It seems this needs to be the same password for Druid to work, so we re-use the existing env variable. +const ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV: &str = INTERNAL_INITIAL_CLIENT_PASSWORD_ENV; + +/// Configures the given provider authorizer (e.g. `"LdapAuthorizer"`) alongside the Druid system +/// authorizer, both with the `allowAll` authorizer type. Shared by the LDAP and OIDC providers. +fn add_authorizer_config(config: &mut BTreeMap, authorizer_name: &str) { + config.insert( + "druid.auth.authorizers".to_string(), + format!(r#"["{authorizer_name}", "DruidSystemAuthorizer"]"#), + ); + config.insert( + format!("druid.auth.authorizer.{authorizer_name}.type"), + "allowAll".to_string(), + ); +} + +/// Sets `druid.auth.authenticatorChain` to the Druid system authenticator followed by the given +/// provider authenticators (e.g. `["Ldap"]`). Shared by the LDAP and OIDC providers. +fn set_authenticator_chain( + config: &mut BTreeMap, + provider_authenticators: &[&str], +) { + let authenticators: Vec = std::iter::once("DruidSystemAuthenticator") + .chain(provider_authenticators.iter().copied()) + .map(|name| format!("\"{name}\"")) + .collect(); + config.insert( + "druid.auth.authenticatorChain".to_string(), + format!("[{}]", authenticators.join(", ")), + ); +} + type Result = std::result::Result; #[derive(Snafu, Debug)] @@ -89,8 +122,8 @@ impl DruidAuthenticationConfig { pub fn generate_runtime_properties_config( &self, role: &DruidRole, - ) -> Result>, Error> { - let mut config: BTreeMap> = BTreeMap::new(); + ) -> Result, Error> { + let mut config: BTreeMap = BTreeMap::new(); match self { DruidAuthenticationConfig::Ldap { provider, .. } => { @@ -107,16 +140,13 @@ impl DruidAuthenticationConfig { } /// Creates authentication config that is required by LDAP and OIDC and doesn't depend on user input. - fn generate_common_runtime_properties_config( - &self, - config: &mut BTreeMap>, - ) { + fn generate_common_runtime_properties_config(&self, config: &mut BTreeMap) { self.add_druid_system_authenticator_config(config); self.add_escalator_config(config); config.insert( "druid.auth.authorizer.DruidSystemAuthorizer.type".to_string(), - Some(r#"allowAll"#.to_string()), + r#"allowAll"#.to_string(), ); } @@ -136,13 +166,9 @@ impl DruidAuthenticationConfig { command } - pub fn get_env_var_mounts( - &self, - druid: &v1alpha1::DruidCluster, - role: &DruidRole, - ) -> Vec { + pub fn get_env_var_mounts(&self, cluster: &ValidatedCluster, role: &DruidRole) -> Vec { let mut envs = vec![]; - let internal_secret_name = build_shared_internal_secret_name(druid); + let internal_secret_name = build_shared_internal_secret_name(cluster); envs.push(env_var_from_secret( &internal_secret_name, None, @@ -176,50 +202,47 @@ impl DruidAuthenticationConfig { /// When using LDAP or OIDC the DruidSystemAuthenticator is always tried first and skipped if no basic auth credentials were supplied. /// We don't want to create an admin user for the internal authentication, so this line is left out of the config: /// # druid.auth.authenticator.DruidSystemAuthenticator.initialAdminPassword: XXX - fn add_druid_system_authenticator_config(&self, config: &mut BTreeMap>) { + fn add_druid_system_authenticator_config(&self, config: &mut BTreeMap) { config.insert( "druid.auth.authenticator.DruidSystemAuthenticator.type".to_string(), - Some("basic".to_string()), + "basic".to_string(), ); config.insert( "druid.auth.authenticator.DruidSystemAuthenticator.credentialsValidator.type" .to_string(), - Some("metadata".to_string()), + "metadata".to_string(), ); config.insert( "druid.auth.authenticator.DruidSystemAuthenticator.initialInternalClientPassword" .to_string(), - Some(format!("${{env:{INTERNAL_INITIAL_CLIENT_PASSWORD_ENV}}}").to_string()), + env_var_reference(INTERNAL_INITIAL_CLIENT_PASSWORD_ENV), ); config.insert( "druid.auth.authenticator.DruidSystemAuthenticator.authorizerName".to_string(), - Some("DruidSystemAuthorizer".to_string()), + "DruidSystemAuthorizer".to_string(), ); config.insert( "druid.auth.authenticator.DruidSystemAuthenticator.skipOnFailure".to_string(), - Some("true".to_string()), + "true".to_string(), ); } /// Creates the escalator config: . /// This configures Druid processes to use the basic auth authentication added in `add_druid_system_authenticator_config` for internal communication. - fn add_escalator_config(&self, config: &mut BTreeMap>) { - config.insert( - "druid.escalator.type".to_string(), - Some("basic".to_string()), - ); + fn add_escalator_config(&self, config: &mut BTreeMap) { + config.insert("druid.escalator.type".to_string(), "basic".to_string()); config.insert( "druid.escalator.internalClientUsername".to_string(), - Some("druid_system".to_string()), + "druid_system".to_string(), ); config.insert( "druid.escalator.internalClientPassword".to_string(), - Some(format!("${{env:{ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV}}}").to_string()), + env_var_reference(ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV), ); config.insert( "druid.escalator.authorizerName".to_string(), - Some("DruidSystemAuthorizer".to_string()), + "DruidSystemAuthorizer".to_string(), ); } } diff --git a/rust/operator-binary/src/authentication/oidc.rs b/rust/operator-binary/src/authentication/oidc.rs index e37e45a8..7540531f 100644 --- a/rust/operator-binary/src/authentication/oidc.rs +++ b/rust/operator-binary/src/authentication/oidc.rs @@ -9,7 +9,10 @@ use stackable_operator::{ use super::{AddOidcVolumesSnafu, ConstructOidcWellKnownUrlSnafu, Error}; use crate::{ - crd::{COOKIE_PASSPHRASE_ENV, DruidRole, security::add_cert_to_jvm_trust_store_cmd}, + crd::{ + COOKIE_PASSPHRASE_ENV, DruidRole, env_var_reference, + security::add_cert_to_jvm_trust_store_cmd, + }, internal_secret::env_var_from_secret, }; @@ -22,7 +25,7 @@ pub type DruidClientAuthenticationOptions = fn add_authenticator_config( provider: &oidc::v1alpha1::AuthenticationProvider, oidc: &DruidClientAuthenticationOptions, - config: &mut BTreeMap>, + config: &mut BTreeMap, ) -> Result<(), Error> { let well_known_url = &provider .well_known_config_url() @@ -38,36 +41,33 @@ fn add_authenticator_config( config.insert( "druid.auth.authenticator.Oidc.type".to_string(), - Some(r#"pac4j"#.to_string()), + r#"pac4j"#.to_string(), ); config.insert( "druid.auth.authenticator.Oidc.authorizerName".to_string(), - Some(r#"OidcAuthorizer"#.to_string()), + r#"OidcAuthorizer"#.to_string(), ); config.insert( "druid.auth.pac4j.cookiePassphrase".to_string(), - Some(format!("${{env:{COOKIE_PASSPHRASE_ENV}}}").to_string()), + env_var_reference(COOKIE_PASSPHRASE_ENV), ); config.insert( "druid.auth.pac4j.oidc.clientID".to_string(), - Some(format!("${{env:{oidc_client_id_env}}}").to_string()), + env_var_reference(oidc_client_id_env), ); config.insert( "druid.auth.pac4j.oidc.clientSecret".to_string(), - Some(format!("${{env:{oidc_client_secret_env}}}").to_string()), + env_var_reference(oidc_client_secret_env), ); config.insert( "druid.auth.pac4j.oidc.discoveryURI".to_string(), - Some(well_known_url.to_string()), + well_known_url.to_string(), ); config.insert( "druid.auth.pac4j.oidc.oidcClaim".to_string(), - Some(provider.principal_claim.to_string()), - ); - config.insert( - "druid.auth.pac4j.oidc.scope".to_string(), - Some(scopes.join(" ")), + provider.principal_claim.to_string(), ); + config.insert("druid.auth.pac4j.oidc.scope".to_string(), scopes.join(" ")); let method_string = oidc .product_specific_fields @@ -75,53 +75,36 @@ fn add_authenticator_config( .as_ref(); config.insert( "druid.auth.pac4j.oidc.clientAuthenticationMethod".to_string(), - Some(method_string.to_string()), + method_string.to_string(), ); - config.insert( - "druid.auth.authenticatorChain".to_string(), - Some(r#"["DruidSystemAuthenticator", "Oidc"]"#.to_string()), - ); + super::set_authenticator_chain(config, &["Oidc"]); Ok(()) } -fn add_authorizer_config(config: &mut BTreeMap>) { - config.insert( - "druid.auth.authorizers".to_string(), - Some(r#"["OidcAuthorizer", "DruidSystemAuthorizer"]"#.to_string()), - ); - config.insert( - "druid.auth.authorizer.OidcAuthorizer.type".to_string(), - Some(r#"allowAll"#.to_string()), - ); -} - /// Creates the OIDC parts of the runtime.properties config file. /// OIDC authentication is not configured on middlemanagers, because end users don't interact with them directly using the web console and /// turning on OIDC will lead to problems with the communication with coordinators during data ingest. -pub fn generate_runtime_properties_config( +pub(super) fn generate_runtime_properties_config( provider: &oidc::v1alpha1::AuthenticationProvider, oidc: &DruidClientAuthenticationOptions, role: &DruidRole, - config: &mut BTreeMap>, + config: &mut BTreeMap, ) -> Result<(), Error> { match role { DruidRole::MiddleManager => { - config.insert( - "druid.auth.authenticatorChain".to_string(), - Some(r#"["DruidSystemAuthenticator"]"#.to_string()), - ); + super::set_authenticator_chain(config, &[]); } _ => { add_authenticator_config(provider, oidc, config)?; - add_authorizer_config(config) + super::add_authorizer_config(config, "OidcAuthorizer"); } } Ok(()) } -pub fn main_container_commands( +pub(super) fn main_container_commands( provider: &oidc::v1alpha1::AuthenticationProvider, command: &mut Vec, ) { @@ -132,7 +115,7 @@ pub fn main_container_commands( /// Mounts the OIDC credentials secret and the auto-generated internal secret containing the cookie passphrase. /// Not necessary on middlemanagers, because OIDC is not configured on them. -pub fn get_env_var_mounts( +pub(super) fn get_env_var_mounts( role: &DruidRole, oidc: &DruidClientAuthenticationOptions, internal_secret_name: &str, @@ -156,7 +139,7 @@ pub fn get_env_var_mounts( envs } -pub fn add_volumes_and_mounts( +pub(super) fn add_volumes_and_mounts( provider: &oidc::v1alpha1::AuthenticationProvider, pb: &mut PodBuilder, cb_druid: &mut ContainerBuilder, @@ -215,35 +198,35 @@ mod tests { assert_eq!( properties.get("druid.auth.authenticator.Oidc.type"), - Some(&Some("pac4j".to_owned())) + Some(&"pac4j".to_owned()) ); assert_eq!( properties.get("druid.auth.pac4j.oidc.oidcClaim"), - Some(&Some("preferred_username".to_owned())) + Some(&"preferred_username".to_owned()) ); assert_eq!( properties.get("druid.auth.pac4j.oidc.scope"), - Some(&Some("openid".to_owned())) + Some(&"openid".to_owned()) ); assert_eq!( properties.get("druid.auth.authenticator.Oidc.authorizerName"), - Some(&Some("OidcAuthorizer".to_owned())) + Some(&"OidcAuthorizer".to_owned()) ); assert_eq!( properties.get("druid.auth.authenticatorChain"), - Some(&Some("[\"DruidSystemAuthenticator\", \"Oidc\"]".to_owned())) + Some(&"[\"DruidSystemAuthenticator\", \"Oidc\"]".to_owned()) ); assert_eq!( properties.get("druid.auth.pac4j.oidc.discoveryURI"), - Some(&Some( - "https://keycloak.mycorp.org/realms/sdp/.well-known/openid-configuration" + Some( + &"https://keycloak.mycorp.org/realms/sdp/.well-known/openid-configuration" .to_owned() - )) + ) ); assert_eq!( properties.get("druid.auth.pac4j.oidc.clientAuthenticationMethod"), - Some(&Some("client_secret_post".to_owned())) + Some(&"client_secret_post".to_owned()) ); assert!(properties.contains_key("druid.auth.pac4j.oidc.clientID")); assert!(properties.contains_key("druid.auth.pac4j.oidc.clientSecret")); diff --git a/rust/operator-binary/src/config/mod.rs b/rust/operator-binary/src/config/mod.rs deleted file mode 100644 index 271c6d99..00000000 --- a/rust/operator-binary/src/config/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod jvm; diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index b3c705f4..42b69d3a 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -1,157 +1,106 @@ //! Ensures that `Pod`s are configured and running for each [`DruidCluster`][v1alpha1] //! //! [v1alpha1]: v1alpha1::DruidCluster -use std::{ - collections::{BTreeMap, HashMap}, - str::FromStr, - sync::Arc, -}; +use std::{str::FromStr, sync::Arc}; use const_format::concatcp; -use product_config::{ - ProductConfigManager, - types::PropertyNameKind, - writer::{PropertiesWriterError, to_java_properties_string}, -}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ - builder::{ - self, - configmap::ConfigMapBuilder, - meta::ObjectMetaBuilder, - pod::{ - PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, - security::PodSecurityContextBuilder, volume::VolumeBuilder, - }, - }, cli::OperatorEnvironmentOptions, - cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, - commons::{product_image_selection::ResolvedProductImage, rbac::build_rbac_resources}, - constants::RESTART_CONTROLLER_ENABLED_LABEL, - crd::s3, - database_connections::drivers::jdbc::JdbcDatabaseConnection as _, - k8s_openapi::{ - DeepMerge, - api::{ - apps::v1::{StatefulSet, StatefulSetSpec}, - core::v1::{ConfigMap, EnvVar, PersistentVolumeClaim, ServiceAccount}, - }, - apimachinery::pkg::apis::meta::v1::LabelSelector, - }, + cluster_resources::ClusterResourceApplyStrategy, + commons::rbac::build_rbac_resources, kube::{ - Resource, ResourceExt, core::{DeserializeGuard, error_boundary}, - runtime::{controller::Action, reflector::ObjectRef}, + runtime::controller::Action, }, - kvp::{KeyValuePairError, LabelError, LabelValueError, Labels}, + kvp::{KeyValuePairError, LabelValueError}, logging::controller::ReconcilerError, - product_logging::{ - self, - framework::LoggingError, - spec::{ - ConfigMapLogConfig, ContainerLogConfig, ContainerLogConfigChoice, - CustomContainerLogConfig, - }, - }, - role_utils::RoleGroupRef, shared::time::Duration, status::condition::{ compute_conditions, operations::ClusterOperationsConditionBuilder, statefulset::StatefulSetConditionBuilder, }, + v2::{ + cluster_resources::cluster_resources_new, + kvp::label::recommended_labels, + types::operator::{ + ControllerName, OperatorName, ProductName, ProductVersion, RoleGroupName, + }, + }, }; use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ - authentication::DruidAuthenticationConfig, - config::jvm::construct_jvm_args, - crd::{ - APP_NAME, AUTH_AUTHORIZER_OPA_URI, CommonRoleGroupConfig, Container, - DRUID_CONFIG_DIRECTORY, DS_BUCKET, DeepStorageSpec, DruidClusterStatus, DruidRole, - EXTENSIONS_LOADLIST, HDFS_CONFIG_DIRECTORY, JVM_CONFIG, JVM_SECURITY_PROPERTIES_FILE, - LOG_CONFIG_DIRECTORY, MAX_DRUID_LOG_FILES_SIZE, METRICS_PORT, METRICS_PORT_NAME, - OPERATOR_NAME, RUNTIME_PROPS, RW_CONFIG_DIRECTORY, S3_ACCESS_KEY, S3_ENDPOINT_URL, - S3_PATH_STYLE_ACCESS, S3_SECRET_KEY, STACKABLE_LOG_DIR, ZOOKEEPER_CONNECTION_STRING, - build_recommended_labels, build_string_list, security::DruidTlsSecurity, v1alpha1, + controller::build::resource::{ + listener::{build_group_listener, group_listener_name}, + pdb::build_pdb, + service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, }, - discovery::{self, build_discovery_configmaps}, - extensions::get_extension_list, + crd::{APP_NAME, DruidClusterStatus, DruidRole, OPERATOR_NAME, v1alpha1}, internal_secret::create_shared_internal_secret, - listener::{ - LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_group_listener, build_group_listener_pvc, - group_listener_name, secret_volume_listener_scope, - }, - operations::{graceful_shutdown::add_graceful_shutdown_config, pdb::add_pdbs}, - product_logging::extend_role_group_config_map, - service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, }; +mod build; mod dereference; -mod validate; +pub(crate) mod validate; + +use build::resource::discovery::{self, build_discovery_configmaps}; pub const DRUID_CONTROLLER_NAME: &str = "druidcluster"; pub const FULL_CONTROLLER_NAME: &str = concatcp!(DRUID_CONTROLLER_NAME, '.', OPERATOR_NAME); pub(super) const CONTAINER_IMAGE_BASE_NAME: &str = "druid"; -// volume names -const DRUID_CONFIG_VOLUME_NAME: &str = "config"; -const HDFS_CONFIG_VOLUME_NAME: &str = "hdfs"; -const LOG_CONFIG_VOLUME_NAME: &str = "log-config"; -const LOG_VOLUME_NAME: &str = "log"; -const RW_CONFIG_VOLUME_NAME: &str = "rwconfig"; -const USERDATA_MOUNTPOINT: &str = "/stackable/userdata"; +/// The product name (`druid`) as a type-safe label value. +pub(crate) fn product_name() -> ProductName { + ProductName::from_str(APP_NAME).expect("'druid' is a valid product name") +} + +/// The operator name as a type-safe label value. +pub(crate) fn operator_name() -> OperatorName { + OperatorName::from_str(OPERATOR_NAME).expect("the operator name is a valid label value") +} + +/// The controller name as a type-safe label value. +pub(crate) fn controller_name() -> ControllerName { + ControllerName::from_str(DRUID_CONTROLLER_NAME) + .expect("the controller name is a valid label value") +} pub struct Ctx { pub client: stackable_operator::client::Client, - pub product_config: ProductConfigManager, pub operator_environment: OperatorEnvironmentOptions, } #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] -#[allow(clippy::enum_variant_names)] pub enum Error { - #[snafu(display("failed to apply Service for {}", rolegroup))] + #[snafu(display("failed to apply Service for role group {role_group}"))] ApplyRoleGroupService { source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, - }, - - #[snafu(display("failed to build ConfigMap for {}", rolegroup))] - BuildRoleGroupConfig { - source: stackable_operator::builder::configmap::Error, - rolegroup: RoleGroupRef, + role_group: String, }, - #[snafu(display("failed to apply ConfigMap for {}", rolegroup))] + #[snafu(display("failed to apply ConfigMap for role group {role_group}"))] ApplyRoleGroupConfig { source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, + role_group: String, }, - #[snafu(display("failed to apply StatefulSet for {}", rolegroup))] - ApplyRoleGroupStatefulSet { - source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, + #[snafu(display("failed to build StatefulSet"))] + BuildRoleGroupStatefulSet { + source: build::resource::statefulset::Error, }, - #[snafu(display("object is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, + #[snafu(display("failed to apply StatefulSet for role group {role_group}"))] + ApplyRoleGroupStatefulSet { + source: stackable_operator::cluster_resources::Error, + role_group: String, }, #[snafu(display("failed to dereference cluster objects"))] Dereference { source: dereference::Error }, - #[snafu(display("failed to configure S3 connection"))] - ConfigureS3 { - source: stackable_operator::crd::s3::v1alpha1::ConnectionError, - }, - - #[snafu(display("failed to format runtime properties"))] - PropertiesWriteError { source: PropertiesWriterError }, - #[snafu(display("failed to build discovery ConfigMap"))] BuildDiscoveryConfig { source: discovery::Error }, @@ -165,65 +114,16 @@ pub enum Error { source: stackable_operator::client::Error, }, - #[snafu(display( - "Druid does not support skipping the verification of the tls enabled S3 server" - ))] - S3TlsNoVerificationNotSupported, - - #[snafu(display("could not parse Druid role [{role}]"))] - UnidentifiedDruidRole { - source: strum::ParseError, - role: String, - }, - - #[snafu(display("failed to resolve and merge config for role and role group"))] - FailedToResolveConfig { source: crate::crd::Error }, - - #[snafu(display("invalid configuration"))] - InvalidConfiguration { source: crate::crd::Error }, - - #[snafu(display("failed to create cluster resources"))] - CreateClusterResources { - source: stackable_operator::cluster_resources::Error, - }, - #[snafu(display("failed to delete orphaned resources"))] DeleteOrphanedResources { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("failed to create container builder with name [{name}]"))] - FailedContainerBuilderCreation { - source: stackable_operator::builder::pod::container::Error, - name: String, - }, - - #[snafu(display("failed to initialize security context"))] - FailedToInitializeSecurityContext { source: crate::crd::security::Error }, - - #[snafu(display("failed to get JVM config"))] - GetJvmConfig { source: crate::config::jvm::Error }, - - #[snafu(display("failed to derive Druid memory settings from resources"))] - DeriveMemorySettings { source: crate::crd::resource::Error }, - - #[snafu(display("failed to update Druid config from resources"))] - UpdateDruidConfigFromResources { source: crate::crd::resource::Error }, - #[snafu(display("failed to retrieve secret for internal communications"))] FailedInternalSecretCreation { source: crate::internal_secret::Error, }, - #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] - VectorAggregatorConfigMapMissing, - - #[snafu(display("failed to add the logging configuration to the ConfigMap [{cm_name}]"))] - InvalidLoggingConfig { - source: crate::product_logging::Error, - cm_name: String, - }, - #[snafu(display("failed to create RBAC service account"))] ApplyServiceAccount { source: stackable_operator::cluster_resources::Error, @@ -239,36 +139,9 @@ pub enum Error { source: stackable_operator::commons::rbac::Error, }, - #[snafu(display( - "failed to serialize [{JVM_SECURITY_PROPERTIES_FILE}] for {}", - rolegroup - ))] - JvmSecurityProperties { - source: PropertiesWriterError, - rolegroup: String, - }, - - #[snafu(display("failed to create PodDisruptionBudget"))] - FailedToCreatePdb { - source: crate::operations::pdb::Error, - }, - - #[snafu(display("failed to configure graceful shutdown"))] - GracefulShutdown { - source: crate::operations::graceful_shutdown::Error, - }, - - #[snafu(display("failed to add OIDC Volumes and VolumeMounts to the Pod and containers"))] - AuthVolumesBuild { - source: crate::authentication::Error, - }, - - #[snafu(display("failed to build labels"))] - LabelBuild { source: LabelError }, - - #[snafu(display("failed to build metadata"))] - MetadataBuild { - source: stackable_operator::builder::meta::Error, + #[snafu(display("failed to apply PodDisruptionBudget"))] + ApplyPdb { + source: stackable_operator::cluster_resources::Error, }, #[snafu(display("failed to get required labels"))] @@ -276,22 +149,6 @@ pub enum Error { source: KeyValuePairError, }, - #[snafu(display("there was an error generating the authentication runtime settings"))] - GenerateAuthenticationRuntimeSettings { - source: crate::authentication::Error, - }, - - #[snafu(display("failed to build vector container"))] - BuildVectorContainer { source: LoggingError }, - - #[snafu(display("failed to add needed volume"))] - AddVolume { source: builder::pod::Error }, - - #[snafu(display("failed to add needed volumeMount"))] - AddVolumeMount { - source: builder::pod::container::Error, - }, - #[snafu(display("DruidCluster object is invalid"))] InvalidDruidCluster { source: error_boundary::InvalidObject, @@ -302,18 +159,12 @@ pub enum Error { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("failed to configure listener"))] - ListenerConfiguration { source: crate::listener::Error }, - - #[snafu(display("failed to configure service"))] - ServiceConfiguration { source: crate::service::Error }, - #[snafu(display("failed to validate cluster"))] ValidateCluster { source: validate::Error }, - #[snafu(display("invalid metadata database connection"))] - InvalidMetadataDatabaseConnection { - source: stackable_operator::database_connections::Error, + #[snafu(display("failed to build rolegroup ConfigMap"))] + BuildConfigMap { + source: build::resource::config_map::Error, }, } @@ -342,25 +193,20 @@ pub async fn reconcile_druid( .await .context(DereferenceSnafu)?; - let validated = validate::validate( - druid, - &dereferenced_objects, - &ctx.operator_environment, - &ctx.product_config, - ) - .context(ValidateClusterSnafu)?; - - let mut cluster_resources = ClusterResources::new( - APP_NAME, - OPERATOR_NAME, - DRUID_CONTROLLER_NAME, - &druid.object_ref(&()), + let validated_cluster = + validate::validate(druid, &dereferenced_objects, &ctx.operator_environment) + .context(ValidateClusterSnafu)?; + + let mut cluster_resources = cluster_resources_new( + &product_name(), + &operator_name(), + &controller_name(), + &validated_cluster.name, + &validated_cluster.namespace, + &validated_cluster.uid, ClusterResourceApplyStrategy::from(&druid.spec.cluster_operation), &druid.spec.object_overrides, - ) - .context(CreateClusterResourcesSnafu)?; - - let merged_config = druid.merged_config().context(FailedToResolveConfigSnafu)?; + ); let (rbac_sa, rbac_rolebinding) = build_rbac_resources( druid, @@ -382,102 +228,50 @@ pub async fn reconcile_druid( let mut ss_cond_builder = StatefulSetConditionBuilder::default(); - for (role_name, role_config) in validated.validated_role_config.iter() { - let druid_role = DruidRole::from_str(role_name).context(UnidentifiedDruidRoleSnafu { - role: role_name.to_string(), - })?; - + for (druid_role, groups) in validated_cluster.role_group_configs.iter() { create_shared_internal_secret(druid, client, DRUID_CONTROLLER_NAME) .await .context(FailedInternalSecretCreationSnafu)?; - for (rolegroup_name, rolegroup_config) in role_config.iter() { - let rolegroup = RoleGroupRef { - cluster: ObjectRef::from_obj(druid), - role: role_name.into(), - role_group: rolegroup_name.into(), - }; - - let merged_rolegroup_config = merged_config - .common_config(&druid_role, rolegroup_name) - .context(FailedToResolveConfigSnafu)?; - - let role_group_service_recommended_labels = build_recommended_labels( - druid, - DRUID_CONTROLLER_NAME, - &validated.resolved_product_image.app_version_label_value, - &rolegroup.role, - &rolegroup.role_group, - ); - - let role_group_service_selector = Labels::role_group_selector( - druid, - APP_NAME, - &rolegroup.role, - &rolegroup.role_group, - ) - .context(LabelBuildSnafu)?; - - let rg_headless_service = build_rolegroup_headless_service( - druid, - &validated.druid_tls_security, - &druid_role, - &rolegroup, - role_group_service_recommended_labels.clone(), - role_group_service_selector.clone().into(), + for (rolegroup_name, rg) in groups.iter() { + let rg_headless_service = + build_rolegroup_headless_service(&validated_cluster, druid_role, rolegroup_name); + let rg_metrics_service = + build_rolegroup_metrics_service(&validated_cluster, druid_role, rolegroup_name); + + let rg_configmap = build::resource::config_map::build_rolegroup_config_map( + &validated_cluster, + druid_role, + rolegroup_name, + rg, ) - .context(ServiceConfigurationSnafu)?; - let rg_metrics_service = build_rolegroup_metrics_service( - druid, - &rolegroup, - role_group_service_recommended_labels, - role_group_service_selector.into(), - ) - .context(ServiceConfigurationSnafu)?; - - let rg_configmap = build_rolegroup_config_map( - druid, - &validated.resolved_product_image, - &rolegroup, - rolegroup_config, - &merged_rolegroup_config, - &validated.zookeeper_connection_string, - validated.opa_connection_string.as_deref(), - validated.s3_connection.as_ref(), - validated.deep_storage_bucket_name.as_deref(), - &validated.druid_tls_security, - &validated.druid_auth_config, - )?; - let rg_statefulset = build_rolegroup_statefulset( - druid, - &validated.resolved_product_image, - &druid_role, - &rolegroup, - rolegroup_config, - &merged_rolegroup_config, - validated.s3_connection.as_ref(), - &validated.druid_tls_security, - &validated.druid_auth_config, + .context(BuildConfigMapSnafu)?; + let rg_statefulset = build::resource::statefulset::build_rolegroup_statefulset( + &validated_cluster, + druid_role, + rolegroup_name, + rg, &rbac_sa, - )?; + ) + .context(BuildRoleGroupStatefulSetSnafu)?; cluster_resources .add(client, rg_headless_service) .await .with_context(|_| ApplyRoleGroupServiceSnafu { - rolegroup: rolegroup.clone(), + role_group: rolegroup_name.to_string(), })?; cluster_resources .add(client, rg_metrics_service) .await .with_context(|_| ApplyRoleGroupServiceSnafu { - rolegroup: rolegroup.clone(), + role_group: rolegroup_name.to_string(), })?; cluster_resources .add(client, rg_configmap) .await .with_context(|_| ApplyRoleGroupConfigSnafu { - rolegroup: rolegroup.clone(), + role_group: rolegroup_name.to_string(), })?; // Note: The StatefulSet needs to be applied after all ConfigMaps and Secrets it mounts @@ -488,46 +282,41 @@ pub async fn reconcile_druid( .add(client, rg_statefulset) .await .with_context(|_| ApplyRoleGroupStatefulSetSnafu { - rolegroup: rolegroup.clone(), + role_group: rolegroup_name.to_string(), })?, ); } if let Some(listener_class) = druid_role.listener_class_name(druid) - && let Some(listener_group_name) = group_listener_name(druid, &druid_role) + && let Some(listener_group_name) = group_listener_name(&validated_cluster, druid_role) { let role_group_listener = build_group_listener( - druid, - build_recommended_labels( - druid, - DRUID_CONTROLLER_NAME, - &validated.resolved_product_image.app_version_label_value, - role_name, - "none", + &validated_cluster, + recommended_labels( + &validated_cluster, + &product_name(), + &ProductVersion::from_str(&validated_cluster.image.app_version_label_value) + .expect("a valid product version"), + &operator_name(), + &controller_name(), + &druid_role.to_role_name(), + &RoleGroupName::from_str("none").expect("a valid role group name"), ), listener_class.to_string(), listener_group_name, - &druid_role, - &validated.druid_tls_security, - ) - .context(ListenerConfigurationSnafu)?; + druid_role, + ); let listener = cluster_resources .add(client, role_group_listener) .await .context(ApplyGroupListenerSnafu)?; - if druid_role == DruidRole::Router { + if *druid_role == DruidRole::Router { // discovery - for discovery_cm in build_discovery_configmaps( - druid, - druid, - &validated.resolved_product_image, - &validated.druid_tls_security, - listener, - ) - .await - .context(BuildDiscoveryConfigSnafu)? + for discovery_cm in build_discovery_configmaps(&validated_cluster, listener) + .await + .context(BuildDiscoveryConfigSnafu)? { cluster_resources .add(client, discovery_cm) @@ -537,17 +326,18 @@ pub async fn reconcile_druid( } } - let role_config = druid.generic_role_config(&druid_role); + let role_config = druid.generic_role_config(druid_role); - add_pdbs( + if let Some(pdb) = build_pdb( &role_config.pod_disruption_budget, - druid, - &druid_role, - client, - &mut cluster_resources, - ) - .await - .context(FailedToCreatePdbSnafu)?; + &validated_cluster, + druid_role, + ) { + cluster_resources + .add(client, pdb) + .await + .context(ApplyPdbSnafu)?; + } } let cluster_operation_cond_builder = @@ -569,681 +359,6 @@ pub async fn reconcile_druid( Ok(Action::await_change()) } -#[allow(clippy::too_many_arguments)] -/// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator -fn build_rolegroup_config_map( - druid: &v1alpha1::DruidCluster, - resolved_product_image: &ResolvedProductImage, - rolegroup: &RoleGroupRef, - rolegroup_config: &HashMap>, - merged_rolegroup_config: &CommonRoleGroupConfig, - zk_connstr: &str, - opa_connstr: Option<&str>, - s3_conn: Option<&s3::v1alpha1::ConnectionSpec>, - deep_storage_bucket_name: Option<&str>, - druid_tls_security: &DruidTlsSecurity, - druid_auth_config: &Option, -) -> Result { - let druid_role = - DruidRole::from_str(&rolegroup.role).with_context(|_| UnidentifiedDruidRoleSnafu { - role: &rolegroup.role, - })?; - let role = druid.get_role(&druid_role); - let mut cm_conf_data = BTreeMap::new(); // filename -> filecontent - let metadata_database_connection_details = druid - .spec - .cluster_config - .metadata_database - .jdbc_connection_details("metadata") - .context(InvalidMetadataDatabaseConnectionSnafu)?; - - for (property_name_kind, config) in rolegroup_config { - let mut conf: BTreeMap> = Default::default(); - - match property_name_kind { - PropertyNameKind::File(file_name) if file_name == RUNTIME_PROPS => { - // Add any properties derived from storage manifests, such as segment cache locations. - // This has to be done here since there is no other suitable place for it. - // Previously such properties were added in the compute_files() function, - // but that code path is now incompatible with the design of fragment merging. - merged_rolegroup_config - .resources - .update_druid_config_file(&mut conf) - .context(UpdateDruidConfigFromResourcesSnafu)?; - // NOTE: druid.host can be set manually - if it isn't, the canonical host name of - // the local host is used. This should work with the agent and k8s host networking - // but might need to be revisited in the future - conf.insert( - ZOOKEEPER_CONNECTION_STRING.to_string(), - Some(zk_connstr.to_string()), - ); - - conf.insert( - EXTENSIONS_LOADLIST.to_string(), - Some(build_string_list(&get_extension_list( - druid, - druid_tls_security, - druid_auth_config, - ))), - ); - - if let Some(opa_str) = opa_connstr { - conf.insert( - AUTH_AUTHORIZER_OPA_URI.to_string(), - Some(opa_str.to_string()), - ); - }; - - conf.insert( - crate::crd::database::METADATA_STORAGE_TYPE.to_string(), - Some( - druid - .spec - .cluster_config - .metadata_database - .as_metadata_storage_type() - .to_string(), - ), - ); - - conf.insert( - crate::crd::database::METADATA_STORAGE_CONNECTOR_CONNECT_URI.to_string(), - Some( - metadata_database_connection_details - .connection_url - .to_string(), - ), - ); - - if let Some(EnvVar { - name: username_env_name, - .. - }) = &metadata_database_connection_details.username_env - { - conf.insert( - crate::crd::database::METADATA_STORAGE_USER.to_string(), - Some(format!("${{env:{username_env_name}}}",)), - ); - } - - if let Some(EnvVar { - name: password_env_name, - .. - }) = &metadata_database_connection_details.password_env - { - conf.insert( - crate::crd::database::METADATA_STORAGE_PASSWORD.to_string(), - Some(format!("${{env:{password_env_name}}}",)), - ); - } - - if let Some(s3) = s3_conn { - if !s3.region.is_default_config() { - // Raising this as warning instead of returning an error, better safe than sorry. - // It might still work out for the user. - tracing::warn!( - region = ?s3.region, - "You configured a non-default region on the S3Connection. - The S3Connection region field is ignored because Druid uses the AWS SDK v1, which ignores the region if the endpoint is set. \ - The host is a required field, therefore the endpoint will always be set." - ) - } - - conf.insert( - S3_ENDPOINT_URL.to_string(), - Some(s3.endpoint().context(ConfigureS3Snafu)?.to_string()), - ); - - if let Some((access_key_file, secret_key_file)) = s3.credentials_mount_paths() { - conf.insert( - S3_ACCESS_KEY.to_string(), - Some(format!("${{file:UTF-8:{access_key_file}}}")), - ); - conf.insert( - S3_SECRET_KEY.to_string(), - Some(format!("${{file:UTF-8:{secret_key_file}}}")), - ); - } - - conf.insert( - S3_PATH_STYLE_ACCESS.to_string(), - Some((s3.access_style == s3::v1alpha1::S3AccessStyle::Path).to_string()), - ); - } - conf.insert( - DS_BUCKET.to_string(), - deep_storage_bucket_name.map(str::to_string), - ); - - // add tls encryption / auth properties - druid_tls_security.add_tls_config_properties(&mut conf, &druid_role); - - if let Some(auth_config) = druid_auth_config { - conf.extend( - auth_config - .generate_runtime_properties_config(&druid_role) - .context(GenerateAuthenticationRuntimeSettingsSnafu)?, - ); - }; - - let transformed_config: BTreeMap> = config - .iter() - .map(|(k, v)| (k.clone(), Some(v.clone()))) - .collect(); - // extend the config to respect overrides - conf.extend(transformed_config); - - let runtime_properties = - to_java_properties_string(conf.iter()).context(PropertiesWriteSnafu)?; - cm_conf_data.insert(RUNTIME_PROPS.to_string(), runtime_properties); - } - - PropertyNameKind::File(file_name) if file_name == JVM_CONFIG => { - let (heap, direct) = merged_rolegroup_config - .resources - .get_memory_sizes(&druid_role) - .context(DeriveMemorySettingsSnafu)?; - let jvm_config = - construct_jvm_args(&druid_role, &role, &rolegroup.role_group, heap, direct) - .context(GetJvmConfigSnafu)?; - cm_conf_data.insert(JVM_CONFIG.to_string(), jvm_config); - } - - PropertyNameKind::File(file_name) if file_name == JVM_SECURITY_PROPERTIES_FILE => { - let jvm_sec_props: BTreeMap> = rolegroup_config - .get(&PropertyNameKind::File( - JVM_SECURITY_PROPERTIES_FILE.to_string(), - )) - .cloned() - .unwrap_or_default() - .into_iter() - .map(|(k, v)| (k, Some(v))) - .collect(); - cm_conf_data.insert( - JVM_SECURITY_PROPERTIES_FILE.to_string(), - to_java_properties_string(jvm_sec_props.iter()).with_context(|_| { - JvmSecurityPropertiesSnafu { - rolegroup: rolegroup.role_group.clone(), - } - })?, - ); - } - _ => {} - } - } - - let mut config_map_builder = ConfigMapBuilder::new(); - config_map_builder.metadata( - ObjectMetaBuilder::new() - .name_and_namespace(druid) - .name(rolegroup.object_name()) - .ownerreference_from_resource(druid, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&build_recommended_labels( - druid, - DRUID_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - &rolegroup.role, - &rolegroup.role_group, - )) - .context(MetadataBuildSnafu)? - .build(), - ); - - for (filename, file_content) in cm_conf_data.iter() { - config_map_builder.add_data(filename, file_content); - } - - extend_role_group_config_map( - rolegroup, - &merged_rolegroup_config.logging, - &mut config_map_builder, - ) - .context(InvalidLoggingConfigSnafu { - cm_name: rolegroup.object_name(), - })?; - - config_map_builder - .build() - .with_context(|_| BuildRoleGroupConfigSnafu { - rolegroup: rolegroup.clone(), - }) -} - -#[allow(clippy::too_many_arguments)] -/// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. -/// -/// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the -/// corresponding [`stackable_operator::k8s_openapi::api::core::v1::Service`] (from [`build_rolegroup_headless_service`]). -fn build_rolegroup_statefulset( - druid: &v1alpha1::DruidCluster, - resolved_product_image: &ResolvedProductImage, - role: &DruidRole, - rolegroup_ref: &RoleGroupRef, - rolegroup_config: &HashMap>, - merged_rolegroup_config: &CommonRoleGroupConfig, - s3_conn: Option<&s3::v1alpha1::ConnectionSpec>, - druid_tls_security: &DruidTlsSecurity, - druid_auth_config: &Option, - service_account: &ServiceAccount, -) -> Result { - // prepare container builder - let prepare_container_name = Container::Prepare.to_string(); - let mut cb_prepare = ContainerBuilder::new(&prepare_container_name).context( - FailedContainerBuilderCreationSnafu { - name: &prepare_container_name, - }, - )?; - // druid container builder - let druid_container_name = Container::Druid.to_string(); - let mut cb_druid = ContainerBuilder::new(&druid_container_name).context( - FailedContainerBuilderCreationSnafu { - name: &druid_container_name, - }, - )?; - // init pod builder - let mut pb = PodBuilder::new(); - pb.affinity(&merged_rolegroup_config.affinity); - add_graceful_shutdown_config( - role, - druid_tls_security, - merged_rolegroup_config.graceful_shutdown_timeout, - &mut pb, - &mut cb_druid, - ) - .context(GracefulShutdownSnafu)?; - - let metadata_database_connection_details = druid - .spec - .cluster_config - .metadata_database - .jdbc_connection_details("metadata") - .context(InvalidMetadataDatabaseConnectionSnafu)?; - - let mut main_container_commands = role.main_container_prepare_commands(s3_conn); - let mut prepare_container_commands = vec![]; - if let Some(ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic(log_config)), - }) = merged_rolegroup_config - .logging - .containers - .get(&Container::Prepare) - { - // This command needs to be added at the beginning of the shell commands, - // otherwise the output of the following commands will not be captured! - prepare_container_commands.push(product_logging::framework::capture_shell_output( - STACKABLE_LOG_DIR, - &prepare_container_name, - log_config, - )); - } - prepare_container_commands.extend(druid_tls_security.build_tls_key_stores_cmd()); - - if let Some(auth_config) = druid_auth_config { - auth_config - .add_volumes_and_mounts(&mut pb, &mut cb_druid, &mut cb_prepare) - .context(AuthVolumesBuildSnafu)?; - prepare_container_commands.extend(auth_config.prepare_container_commands()); - main_container_commands.extend(auth_config.main_container_commands()) - } - - // volume and volume mounts - druid_tls_security - .add_tls_volume_and_volume_mounts( - &mut cb_prepare, - &mut cb_druid, - &mut pb, - &merged_rolegroup_config.requested_secret_lifetime, - // add listener - secret_volume_listener_scope(role), - ) - .context(FailedToInitializeSecurityContextSnafu)?; - - if let Some(s3) = s3_conn { - if s3.tls.uses_tls() && !s3.tls.uses_tls_verification() { - S3TlsNoVerificationNotSupportedSnafu.fail()?; - } - s3.add_volumes_and_mounts(&mut pb, vec![&mut cb_druid]) - .context(ConfigureS3Snafu)?; - } - - add_config_volume_and_volume_mounts(rolegroup_ref, &mut cb_druid, &mut pb)?; - add_log_config_volume_and_volume_mounts( - rolegroup_ref, - merged_rolegroup_config, - &mut cb_druid, - &mut pb, - )?; - add_log_volume_and_volume_mounts(&mut cb_druid, &mut cb_prepare, &mut pb)?; - add_hdfs_cm_volume_and_volume_mounts( - &druid.spec.cluster_config.deep_storage, - &mut cb_druid, - &mut pb, - )?; - merged_rolegroup_config - .resources - .update_volumes_and_volume_mounts(&mut cb_druid, &mut pb) - .context(UpdateDruidConfigFromResourcesSnafu)?; - - cb_prepare - .image_from_product_image(resolved_product_image) - .command(vec![ - "/bin/bash".to_string(), - "-x".to_string(), - "-euo".to_string(), - "pipefail".to_string(), - "-c".to_string(), - ]) - .args(vec![prepare_container_commands.join("\n")]) - .resources( - ResourceRequirementsBuilder::new() - .with_cpu_request("100m") - .with_cpu_limit("400m") - .with_memory_request("512Mi") - .with_memory_limit("512Mi") - .build(), - ); - - metadata_database_connection_details.add_to_container(&mut cb_druid); - - // rest of env - let mut rest_env = rolegroup_config - .get(&PropertyNameKind::Env) - .iter() - .flat_map(|env_vars| env_vars.iter()) - .map(|(k, v)| EnvVar { - name: k.clone(), - value: Some(v.clone()), - ..EnvVar::default() - }) - .collect::>(); - - if let Some(auth_config) = druid_auth_config { - rest_env.extend(auth_config.get_env_var_mounts(druid, role)) - } - - // Needed for the `containerdebug` process to log it's tracing information to. - rest_env.push(EnvVar { - name: "CONTAINERDEBUG_LOG_DIRECTORY".to_string(), - value: Some(format!("{STACKABLE_LOG_DIR}/containerdebug")), - value_from: None, - }); - - main_container_commands.push(role.main_container_start_command()); - cb_druid - .image_from_product_image(resolved_product_image) - .command(vec![ - "/bin/bash".to_string(), - "-x".to_string(), - "-euo".to_string(), - "pipefail".to_string(), - "-c".to_string(), - ]) - .args(vec![main_container_commands.join("\n")]) - .add_env_vars(rest_env) - .add_container_ports(druid_tls_security.container_ports(role)) - .add_container_port(METRICS_PORT_NAME, METRICS_PORT.into()) - // 10s * 30 = 300s to come up - .startup_probe(druid_tls_security.get_tcp_socket_probe(30, 10, 30, 3)) - // 10s * 1 = 10s to get removed from service - .readiness_probe(druid_tls_security.get_tcp_socket_probe(10, 10, 1, 3)) - // 10s * 3 = 30s to be restarted - .liveness_probe(druid_tls_security.get_tcp_socket_probe(10, 10, 3, 3)) - .resources(merged_rolegroup_config.resources.as_resource_requirements()); - - // Add extra mounts if any are specified and the current role is MiddleManager - // Extra mounts may be needed for ingestion to add required certificates, truststores or similar - // files. - // Mounts are added to all roles, as we are currently unsure where they may be needed - // Known roles are MiddleManagers for ingestion and Historicals for deep storage (GCS plugin) - // We may at some time in the future revisit this and limit it again to avoid needlessly - // propagating potentially confidential files throughout the cluster - for volume in &druid.spec.cluster_config.extra_volumes { - // Extract values into vars so we make it impossible to log something other than - // what we actually use to create the mounts - maybe paranoid, but hey .. - let volume_name = &volume.name; - let mount_point = format!("{USERDATA_MOUNTPOINT}/{}", volume.name); - - tracing::info!( - ?volume_name, - ?mount_point, - ?role, - "Adding user specified extra volume", - ); - pb.add_volume(volume.clone()).context(AddVolumeSnafu)?; - cb_druid - .add_volume_mount(volume_name, mount_point) - .context(AddVolumeMountSnafu)?; - } - - let mut pvcs: Option> = None; - - if let Some(group_listener_name) = group_listener_name(druid, role) { - cb_druid - .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) - .context(AddVolumeMountSnafu)?; - - // Used for PVC templates that cannot be modified once they are deployed - let unversioned_recommended_labels = Labels::recommended(&build_recommended_labels( - druid, - DRUID_CONTROLLER_NAME, - // A version value is required, and we do want to use the "recommended" format for the other desired labels - "none", - &rolegroup_ref.role, - &rolegroup_ref.role_group, - )) - .context(LabelBuildSnafu)?; - - pvcs = Some(vec![ - build_group_listener_pvc(&group_listener_name, &unversioned_recommended_labels) - .context(ListenerConfigurationSnafu)?, - ]); - } - - let metadata = ObjectMetaBuilder::new() - .with_recommended_labels(&build_recommended_labels( - druid, - DRUID_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - )) - .context(MetadataBuildSnafu)? - .build(); - - pb.image_pull_secrets_from_product_image(resolved_product_image) - .add_init_container(cb_prepare.build()) - .add_container(cb_druid.build()) - .metadata(metadata) - .service_account_name(service_account.name_any()) - .security_context(PodSecurityContextBuilder::new().fs_group(1000).build()); - - if merged_rolegroup_config.logging.enable_vector_agent { - match &druid.spec.cluster_config.vector_aggregator_config_map_name { - Some(vector_aggregator_config_map_name) => { - pb.add_container( - product_logging::framework::vector_container( - resolved_product_image, - DRUID_CONFIG_VOLUME_NAME, - LOG_VOLUME_NAME, - merged_rolegroup_config - .logging - .containers - .get(&Container::Vector), - ResourceRequirementsBuilder::new() - .with_cpu_request("250m") - .with_cpu_limit("500m") - .with_memory_request("128Mi") - .with_memory_limit("128Mi") - .build(), - vector_aggregator_config_map_name, - ) - .context(BuildVectorContainerSnafu)?, - ); - } - None => { - VectorAggregatorConfigMapMissingSnafu.fail()?; - } - } - } - - let mut pod_template = pb.build_template(); - pod_template.merge_from(druid.pod_overrides_for_role(role).clone()); - if let Some(pod_overrides) = druid.pod_overrides_for_role_group(role, &rolegroup_ref.role_group) - { - pod_template.merge_from(pod_overrides.clone()); - } - - Ok(StatefulSet { - metadata: ObjectMetaBuilder::new() - .name_and_namespace(druid) - .name(rolegroup_ref.object_name()) - .ownerreference_from_resource(druid, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&build_recommended_labels( - druid, - DRUID_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - )) - .context(MetadataBuildSnafu)? - .with_label(RESTART_CONTROLLER_ENABLED_LABEL.to_owned()) - .build(), - spec: Some(StatefulSetSpec { - pod_management_policy: Some("Parallel".to_string()), - replicas: merged_rolegroup_config.replicas.map(i32::from), - selector: LabelSelector { - match_labels: Some( - Labels::role_group_selector( - druid, - APP_NAME, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - ) - .context(LabelBuildSnafu)? - .into(), - ), - ..LabelSelector::default() - }, - service_name: Some(rolegroup_ref.rolegroup_headless_service_name()), - template: pod_template, - volume_claim_templates: pvcs, - ..StatefulSetSpec::default() - }), - status: None, - }) -} - -fn add_hdfs_cm_volume_and_volume_mounts( - deep_storage_spec: &DeepStorageSpec, - cb_druid: &mut ContainerBuilder, - pb: &mut PodBuilder, -) -> Result<()> { - // hdfs deep storage mount - if let DeepStorageSpec::Hdfs(hdfs) = deep_storage_spec { - cb_druid - .add_volume_mount(HDFS_CONFIG_VOLUME_NAME, HDFS_CONFIG_DIRECTORY) - .context(AddVolumeMountSnafu)?; - pb.add_volume( - VolumeBuilder::new(HDFS_CONFIG_VOLUME_NAME) - .with_config_map(&hdfs.config_map_name) - .build(), - ) - .context(AddVolumeSnafu)?; - } - - Ok(()) -} - -fn add_config_volume_and_volume_mounts( - rolegroup_ref: &RoleGroupRef, - cb_druid: &mut ContainerBuilder, - pb: &mut PodBuilder, -) -> Result<()> { - cb_druid - .add_volume_mount(DRUID_CONFIG_VOLUME_NAME, DRUID_CONFIG_DIRECTORY) - .context(AddVolumeMountSnafu)?; - pb.add_volume( - VolumeBuilder::new(DRUID_CONFIG_VOLUME_NAME) - .with_config_map(rolegroup_ref.object_name()) - .build(), - ) - .context(AddVolumeSnafu)?; - cb_druid - .add_volume_mount(RW_CONFIG_VOLUME_NAME, RW_CONFIG_DIRECTORY) - .context(AddVolumeMountSnafu)?; - pb.add_volume( - VolumeBuilder::new(RW_CONFIG_VOLUME_NAME) - .with_empty_dir(Some(""), None) - .build(), - ) - .context(AddVolumeSnafu)?; - - Ok(()) -} - -fn add_log_config_volume_and_volume_mounts( - rolegroup_ref: &RoleGroupRef, - merged_rolegroup_config: &CommonRoleGroupConfig, - cb_druid: &mut ContainerBuilder, - pb: &mut PodBuilder, -) -> Result<()> { - cb_druid - .add_volume_mount(LOG_CONFIG_VOLUME_NAME, LOG_CONFIG_DIRECTORY) - .context(AddVolumeMountSnafu)?; - - let config_map = if let Some(ContainerLogConfig { - choice: - Some(ContainerLogConfigChoice::Custom(CustomContainerLogConfig { - custom: ConfigMapLogConfig { config_map }, - })), - }) = merged_rolegroup_config - .logging - .containers - .get(&Container::Druid) - { - config_map.into() - } else { - rolegroup_ref.object_name() - }; - - pb.add_volume( - VolumeBuilder::new(LOG_CONFIG_VOLUME_NAME) - .with_config_map(config_map) - .build(), - ) - .context(AddVolumeSnafu)?; - - Ok(()) -} - -fn add_log_volume_and_volume_mounts( - cb_druid: &mut ContainerBuilder, - cb_prepare: &mut ContainerBuilder, - pb: &mut PodBuilder, -) -> Result<()> { - cb_druid - .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) - .context(AddVolumeMountSnafu)?; - cb_prepare - .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) - .context(AddVolumeMountSnafu)?; - pb.add_volume( - VolumeBuilder::new(LOG_VOLUME_NAME) - .with_empty_dir( - Some(""), - Some(product_logging::framework::calculate_log_volume_size_limit( - &[MAX_DRUID_LOG_FILES_SIZE], - )), - ) - .build(), - ) - .context(AddVolumeSnafu)?; - - Ok(()) -} - pub fn error_policy( _obj: Arc>, error: &Error, @@ -1257,46 +372,18 @@ pub fn error_policy( #[cfg(test)] mod test { - use product_config::{ProductConfigManager, writer}; + use std::str::FromStr; + use rstest::*; - use stackable_operator::product_config_utils::{ - transform_all_roles_to_config, validate_all_roles_and_groups_config, - }; + use stackable_operator::v2::types::operator::RoleGroupName; use super::*; - use crate::crd::{PROP_SEGMENT_CACHE_LOCATIONS, authentication::AuthenticationClassesResolved}; - - #[derive(Snafu, Debug, EnumDiscriminants)] - #[strum_discriminants(derive(IntoStaticStr))] - #[allow(clippy::enum_variant_names)] - pub enum Error { - #[snafu(display("controller error"))] - Controller { - #[snafu(source(from(super::Error, Box::new)))] - source: Box, - }, - - #[snafu(display("product config error"))] - ProductConfig { - source: product_config::error::Error, - }, - - #[snafu(display("product config utils error"))] - ProductConfigUtils { - source: stackable_operator::product_config_utils::Error, - }, - - #[snafu(display("operator framework error"))] - OperatorFramework { - source: stackable_operator::product_config_utils::Error, + use crate::{ + controller::build::{ + properties::ConfigFileName, resource::config_map::build_rolegroup_config_map, }, - - #[snafu(display("failed to resolve and merge config for role and role group"))] - FailedToResolveConfig { source: crate::crd::Error }, - - #[snafu(display("invalid configuration"))] - InvalidConfiguration { source: crate::crd::Error }, - } + crd::PROP_SEGMENT_CACHE_LOCATIONS, + }; #[rstest] #[case( @@ -1313,107 +400,52 @@ mod test { #[case] druid_manifest: &str, #[case] tested_rolegroup_name: &str, #[case] expected_druid_segment_cache_property: &str, - ) -> Result<(), Box> { - let cluster_cr = - std::fs::File::open(format!("test/resources/druid_controller/{druid_manifest}")) + ) { + let yaml = + std::fs::read_to_string(format!("test/resources/druid_controller/{druid_manifest}")) .unwrap(); - let deserializer = serde_yaml::Deserializer::from_reader(&cluster_cr); - let druid: v1alpha1::DruidCluster = - serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); - - let resolved_product_image: ResolvedProductImage = druid - .spec - .image - .resolve( - CONTAINER_IMAGE_BASE_NAME, - "oci.example.org", - crate::built_info::PKG_VERSION, - ) - .expect("test: resolved product image is always valid"); - let role_config = transform_all_roles_to_config(&druid, &druid.build_role_properties()); - - let product_config_manager = - ProductConfigManager::from_yaml_file("test/resources/druid_controller/properties.yaml") - .context(ProductConfigSnafu)?; - - let validated_role_config = validate_all_roles_and_groups_config( - &resolved_product_image.product_version, - &role_config.context(ProductConfigUtilsSnafu)?, - &product_config_manager, - false, - false, - ) - .context(OperatorFrameworkSnafu)?; - - let druid_tls_security = DruidTlsSecurity::new( - &AuthenticationClassesResolved { - auth_classes: vec![], - }, - Some("tls".to_string()), - ); - - let mut druid_segment_cache_property = "invalid".to_string(); - - let config = druid.merged_config().context(FailedToResolveConfigSnafu)?; - - for (role_name, role_config) in validated_role_config.iter() { - for (rolegroup_name, rolegroup_config) in role_config.iter() { - if rolegroup_name == tested_rolegroup_name - && role_name == &DruidRole::Historical.to_string() - { - let rolegroup_ref = RoleGroupRef { - cluster: ObjectRef::from_obj(&druid), - role: role_name.into(), - role_group: rolegroup_name.clone(), - }; - - let merged_rolegroup_config = config - .common_config(&DruidRole::Historical, rolegroup_name) - .context(InvalidConfigurationSnafu)?; - - let auth_settings: Option = None; - - let rg_configmap = build_rolegroup_config_map( - &druid, - &resolved_product_image, - &rolegroup_ref, - rolegroup_config, - &merged_rolegroup_config, - "zookeeper-connection-string", - None, - None, - None, - &druid_tls_security, - &auth_settings, - ) - .context(ControllerSnafu)?; - - druid_segment_cache_property = rg_configmap - .data - .unwrap() - .get(RUNTIME_PROPS) - .unwrap() - .to_string(); - - break; - } - } - } - let escaped_segment_cache_property = writer::to_java_properties_string( - vec![( - &PROP_SEGMENT_CACHE_LOCATIONS.to_string(), - &Some(expected_druid_segment_cache_property.to_string()), - )] - .into_iter(), + let druid = crate::controller::validate::test_support::druid_from_yaml(&yaml); + + let cluster = crate::controller::validate::test_support::validated_cluster(&druid); + + // The segment cache property is injected dynamically by the config_map builder from the + // merged resources of the validated role group config. + let rg = cluster + .role_group_configs + .get(&DruidRole::Historical) + .expect("historical role groups") + .get(&RoleGroupName::from_str(tested_rolegroup_name).unwrap()) + .expect("tested rolegroup") + .clone(); + + let rg_configmap = build_rolegroup_config_map( + &cluster, + &DruidRole::Historical, + &RoleGroupName::from_str(tested_rolegroup_name).unwrap(), + &rg, ) - .unwrap(); + .expect("build rolegroup config map"); + + let druid_segment_cache_property = rg_configmap + .data + .unwrap() + .get(&ConfigFileName::RuntimeProperties.to_string()) + .unwrap() + .to_string(); + + let escaped_segment_cache_property = + stackable_operator::v2::config_file_writer::to_java_properties_string( + vec![( + &PROP_SEGMENT_CACHE_LOCATIONS.to_string(), + &expected_druid_segment_cache_property.to_string(), + )] + .into_iter(), + ) + .unwrap(); assert!( druid_segment_cache_property.contains(&escaped_segment_cache_property), - "role group {}", - tested_rolegroup_name + "role group {tested_rolegroup_name}" ); - - Ok(()) } } diff --git a/rust/operator-binary/src/controller/build.rs b/rust/operator-binary/src/controller/build.rs new file mode 100644 index 00000000..398af03c --- /dev/null +++ b/rust/operator-binary/src/controller/build.rs @@ -0,0 +1,5 @@ +//! Build steps that turn a `ValidatedCluster` into Kubernetes resources. + +pub mod jvm; +pub mod properties; +pub mod resource; diff --git a/rust/operator-binary/src/config/jvm.rs b/rust/operator-binary/src/controller/build/jvm.rs similarity index 71% rename from rust/operator-binary/src/config/jvm.rs rename to rust/operator-binary/src/controller/build/jvm.rs index 90f8ffa0..995bcdb8 100644 --- a/rust/operator-binary/src/config/jvm.rs +++ b/rust/operator-binary/src/controller/build/jvm.rs @@ -1,15 +1,17 @@ use snafu::{ResultExt, Snafu}; use stackable_operator::{ - memory::MemoryQuantity, - role_utils, - role_utils::{GenericRoleConfig, JavaCommonConfig, JvmArgumentOverrides, Role}, + memory::MemoryQuantity, v2::jvm_argument_overrides::JvmArgumentOverrides, }; +use super::properties::ConfigFileName; use crate::crd::{ - DruidConfigOverrides, DruidRole, JVM_SECURITY_PROPERTIES_FILE, LOG4J2_CONFIG, - RW_CONFIG_DIRECTORY, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, + DruidRole, RW_CONFIG_DIRECTORY, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, + STACKABLE_TRUST_STORE_TYPE, }; +/// The Derby error log file, written by the Coordinator's embedded Derby (default metadata store). +const DERBY_LOG_FILE: &str = "/stackable/var/druid/derby.log"; + #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("failed to format memory quantity {value:?} for Java"))] @@ -17,17 +19,13 @@ pub enum Error { value: MemoryQuantity, source: stackable_operator::memory::Error, }, - - #[snafu(display("failed to merge jvm argument overrides"))] - MergeJvmArgumentOverrides { source: role_utils::Error }, } /// Please note that this function is slightly different than all other operators, because memory /// management is far more advanced in this operator. -pub fn construct_jvm_args( +pub fn construct_jvm_args( druid_role: &DruidRole, - role: &Role, - role_group: &str, + jvm_argument_overrides: &JvmArgumentOverrides, heap: MemoryQuantity, direct_memory: Option, ) -> Result { @@ -51,85 +49,45 @@ pub fn construct_jvm_args( if let Some(direct_memory) = direct_memory_str { jvm_args.push(format!("-XX:MaxDirectMemorySize={direct_memory}")); } + let security_properties_file = ConfigFileName::SecurityProperties; + let log4j2_config_file = ConfigFileName::Log4j2Properties; jvm_args.extend([ "-XX:+ExitOnOutOfMemoryError".to_owned(), "-XX:+UseG1GC".to_owned(), - format!("-Djava.security.properties={RW_CONFIG_DIRECTORY}/{JVM_SECURITY_PROPERTIES_FILE}"), + format!("-Djava.security.properties={RW_CONFIG_DIRECTORY}/{security_properties_file}"), "-Duser.timezone=UTC".to_owned(), "-Dfile.encoding=UTF-8".to_owned(), "-Djava.io.tmpdir=/tmp".to_owned(), "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager".to_owned(), - format!("-Dlog4j.configurationFile={RW_CONFIG_DIRECTORY}/{LOG4J2_CONFIG}"), + format!("-Dlog4j.configurationFile={RW_CONFIG_DIRECTORY}/{log4j2_config_file}"), format!("-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}"), format!("-Djavax.net.ssl.trustStorePassword={STACKABLE_TRUST_STORE_PASSWORD}"), - "-Djavax.net.ssl.trustStoreType=pkcs12".to_owned(), + format!("-Djavax.net.ssl.trustStoreType={STACKABLE_TRUST_STORE_TYPE}"), ]); if druid_role == &DruidRole::Coordinator { - jvm_args.push("-Dderby.stream.error.file=/stackable/var/druid/derby.log".to_owned()); + jvm_args.push(format!("-Dderby.stream.error.file={DERBY_LOG_FILE}")); } - let operator_generated = JvmArgumentOverrides::new_with_only_additions(jvm_args); - let merged_jvm_argument_overrides = role - .get_merged_jvm_argument_overrides(role_group, &operator_generated) - .context(MergeJvmArgumentOverridesSnafu)?; - - Ok(merged_jvm_argument_overrides - .effective_jvm_config_after_merging() - .join("\n")) + Ok(jvm_argument_overrides.apply_to(jvm_args).join("\n")) } #[cfg(test)] mod tests { + use std::str::FromStr; + use indoc::indoc; + use stackable_operator::v2::types::operator::RoleGroupName; use super::*; - use crate::crd::v1alpha1::DruidCluster; #[test] fn test_construct_jvm_arguments_defaults() { - let input = r#" - apiVersion: druid.stackable.tech/v1alpha1 - kind: DruidCluster - metadata: - name: simple-druid - spec: - image: - productVersion: 30.0.0 - clusterConfig: - deepStorage: - hdfs: - configMapName: simple-hdfs - directory: /druid - metadataDatabase: - postgresql: - host: druid-postgresql - database: druid - credentialsSecretName: mySecret - zookeeperConfigMapName: simple-druid-znode - brokers: - roleGroups: - default: - replicas: 1 - coordinators: - roleGroups: - default: - replicas: 1 - historicals: - roleGroups: - default: - replicas: 1 - middleManagers: - roleGroups: - default: - replicas: 1 - routers: - roleGroups: - default: - replicas: 1 - "#; + use crate::controller::validate::test_support::MINIMAL_DRUID_YAML; - let coordinator_jvm_config = construct_jvm_config_for_test(input, &DruidRole::Coordinator); - let historical_jvm_config = construct_jvm_config_for_test(input, &DruidRole::Historical); + let coordinator_jvm_config = + construct_jvm_config_for_test(MINIMAL_DRUID_YAML, &DruidRole::Coordinator); + let historical_jvm_config = + construct_jvm_config_for_test(MINIMAL_DRUID_YAML, &DruidRole::Historical); assert_eq!( coordinator_jvm_config, @@ -286,19 +244,21 @@ mod tests { } fn construct_jvm_config_for_test(druid_cluster: &str, druid_role: &DruidRole) -> String { - let deserializer = serde_yaml::Deserializer::from_str(druid_cluster); - let druid: DruidCluster = - serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); + use crate::controller::validate::test_support::druid_from_yaml; - let role = druid.get_role(druid_role); - let merged_config = druid.merged_config().unwrap(); - let (heap, direct) = merged_config - .common_config(druid_role, "default") - .unwrap() - .resources - .get_memory_sizes(druid_role) + let druid = druid_from_yaml(druid_cluster); + let merged_role = druid.merged_role(druid_role).unwrap(); + let rg = merged_role + .get(&RoleGroupName::from_str("default").unwrap()) .unwrap(); + let (heap, direct) = rg.config.resources.get_memory_sizes(druid_role).unwrap(); - construct_jvm_args(druid_role, &role, "default", heap, direct).unwrap() + construct_jvm_args( + druid_role, + &rg.product_specific_common_config.jvm_argument_overrides, + heap, + direct, + ) + .unwrap() } } diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs new file mode 100644 index 00000000..0ab11128 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -0,0 +1,23 @@ +//! Per-file builders for Druid `.properties` files. + +pub mod product_logging; +pub mod runtime_properties; +pub mod security_properties; + +/// The names of the operator-written Druid config files assembled into the rolegroup ConfigMap. +#[derive(Clone, Copy, Debug, strum::Display)] +pub enum ConfigFileName { + #[strum(serialize = "runtime.properties")] + RuntimeProperties, + #[strum(serialize = "security.properties")] + SecurityProperties, + /// `log4j2.properties` is rendered by the logging framework rather than a properties builder, + /// but it is still an operator-written file assembled into the rolegroup ConfigMap. + #[strum(serialize = "log4j2.properties")] + Log4j2Properties, + /// `jvm.config` is rendered from JVM argument overrides by [`super::jvm`] rather than a + /// properties builder, but it is still an operator-written file assembled into the rolegroup + /// ConfigMap. + #[strum(serialize = "jvm.config")] + JvmConfig, +} diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs b/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs new file mode 100644 index 00000000..6a098fd7 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs @@ -0,0 +1,70 @@ +//! Renders the logging config files (`log4j2.properties` and the Vector agent config) assembled +//! into the rolegroup `ConfigMap`. + +use stackable_operator::{ + memory::{BinaryMultiple, MemoryQuantity}, + product_logging::{self, spec::AutomaticContainerLogConfig}, + v2::product_logging::framework::{STACKABLE_LOG_DIR, ValidatedContainerLogConfigChoice}, +}; + +use crate::crd::Container; + +/// Maximum size of all Druid log files combined, used both for the log4j2 rollover configuration +/// and to size the log volume in the controller. +pub const MAX_DRUID_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { + value: 10.0, + unit: BinaryMultiple::Mebi, +}; + +const CONSOLE_CONVERSION_PATTERN: &str = "%d{ISO8601} %p [%t] %c - %m%n"; + +/// File that the Druid log4j2 config writes its rolling log output to. +const DRUID_LOG_FILE: &str = "druid.log4j2.xml"; + +/// The Vector agent configuration (`vector.yaml`). +const VECTOR_CONFIG: &str = include_str!("vector.yaml"); + +/// Returns the Vector agent config (`vector.yaml`) content, added to the rolegroup `ConfigMap` +/// only when the Vector agent is enabled. +pub fn vector_config_file_content() -> String { + VECTOR_CONFIG.to_owned() +} + +/// Renders the `log4j2.properties` content for the Druid container. +/// +/// Returns `None` when the container uses a custom log ConfigMap rather than the operator's +/// automatic logging configuration (in which case no `log4j2.properties` is added). +pub fn build_log4j2(druid_container: &ValidatedContainerLogConfigChoice) -> Option { + match druid_container { + ValidatedContainerLogConfigChoice::Automatic(log_config) => Some(log4j2_config(log_config)), + ValidatedContainerLogConfigChoice::Custom(_) => None, + } +} + +fn log4j2_config(log_config: &AutomaticContainerLogConfig) -> String { + product_logging::framework::create_log4j2_config( + &format!( + "{STACKABLE_LOG_DIR}/{container}", + container = Container::Druid + ), + DRUID_LOG_FILE, + MAX_DRUID_LOG_FILES_SIZE + .scale_to(BinaryMultiple::Mebi) + .floor() + .value as u32, + CONSOLE_CONVERSION_PATTERN, + log_config, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn vector_config_file_content_is_non_empty_and_keeps_log4j2_source() { + let content = vector_config_file_content(); + assert!(!content.is_empty()); + assert!(content.contains("files_log4j2")); + } +} diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/test-vector.sh b/rust/operator-binary/src/controller/build/properties/product_logging/test-vector.sh new file mode 100755 index 00000000..f77dcf62 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/product_logging/test-vector.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env sh + +DATA_DIR=/stackable/log/_vector-state \ +LOG_DIR=/stackable/log \ +NAMESPACE=default \ +CLUSTER_NAME=druid \ +ROLE_NAME=broker \ +ROLE_GROUP_NAME=default \ +VECTOR_AGGREGATOR_ADDRESS=vector-aggregator \ +VECTOR_FILE_LOG_LEVEL=info \ +vector test vector.yaml vector-test.yaml diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml new file mode 100644 index 00000000..862ad3fa --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml @@ -0,0 +1,162 @@ +# Run tests with `./test-vector.sh` +# +# A downside of these test cases is that they compare the whole event and that the message can +# contain source code positions in vector.yaml, e.g. "function call error for \"parse_xml\" at +# (584:643)". Please adapt the tests if you change VRL code in vector.yaml. +--- +tests: + - name: Test stdout log entry + inputs: + - type: log + insert_at: processed_files_stdout + log_fields: + file: /stackable/log/druid/druid.stdout.log + message: Starting Apache Druid + pod: druid-broker-default-0 + source_type: file + timestamp: 2025-10-02T09:27:28.582Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "druid", + "container": "druid", + "file": "druid.stdout.log", + "level": "INFO", + "logger": "ROOT", + "message": "Starting Apache Druid", + "namespace": "default", + "pod": "druid-broker-default-0", + "role": "broker", + "roleGroup": "default", + "timestamp": "2025-10-02T09:27:28.582Z" + } + + assert_eq!(expected_log_event, .) + - name: Test stderr log entry + inputs: + - type: log + insert_at: processed_files_stderr + log_fields: + file: /stackable/log/druid/druid.stderr.log + message: "Exception in thread \"main\"" + pod: druid-broker-default-0 + source_type: file + timestamp: 2025-10-02T09:27:28.582Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "druid", + "container": "druid", + "file": "druid.stderr.log", + "level": "ERROR", + "logger": "ROOT", + "message": "Exception in thread \"main\"", + "namespace": "default", + "pod": "druid-broker-default-0", + "role": "broker", + "roleGroup": "default", + "timestamp": "2025-10-02T09:27:28.582Z" + } + + assert_eq!(expected_log_event, .) + - name: Test log4j2 XML log entry without stacktrace + inputs: + - type: log + insert_at: processed_files_log4j2 + log_fields: + file: /stackable/log/druid/druid.log4j2.xml + message: > + Starting Druid broker on port + [8082]... + pod: druid-broker-default-0 + source_type: file + timestamp: 2025-10-02T09:27:29.473487331Z + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "druid", + "container": "druid", + "file": "druid.log4j2.xml", + "level": "INFO", + "logger": "org.apache.druid.cli.CliBroker", + "message": "Starting Druid broker on port [8082]...", + "namespace": "default", + "pod": "druid-broker-default-0", + "role": "broker", + "roleGroup": "default", + "timestamp": t'2025-10-02T09:27:28.582Z' + } + + assert_eq!(expected_log_event, .) + - name: Test Vector internal logs + inputs: + - type: log + insert_at: filtered_logs_vector + log_fields: + arch: x86_64 + message: Vector has started. + metadata: + kind: event + level: INFO + module_path: vector::internal_events::process + target: vector + pid: 14 + pod: druid-broker-default-0 + source_type: internal_logs + timestamp: 2025-10-02T09:46:14.479381097Z + version: 0.49.0 + outputs: + - extract_from: extended_logs + conditions: + - type: vrl + source: | + expected_log_event = { + "arch": "x86_64", + "cluster": "druid", + "container": "vector", + "level": "INFO", + "logger": "vector::internal_events::process", + "message": "Vector has started.", + "namespace": "default", + "pod": "druid-broker-default-0", + "role": "broker", + "roleGroup": "default", + "timestamp": "2025-10-02T09:46:14.479381097Z", + "version": "0.49.0" + } + + assert_eq!(expected_log_event, .) + - name: Test Vector internal log level filtering - INFO passes + inputs: + - type: log + insert_at: filtered_logs_vector + log_fields: + metadata: + level: INFO + outputs: + - extract_from: filtered_logs_vector + conditions: + - type: vrl + source: | + assert_eq!("INFO", .metadata.level) + - name: Test Vector internal log level filtering - DEBUG dropped + inputs: + - type: log + insert_at: filtered_logs_vector + log_fields: + metadata: + level: DEBUG + no_outputs_from: + - filtered_logs_vector diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml new file mode 100644 index 00000000..e4d40b6e --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml @@ -0,0 +1,231 @@ +--- +data_dir: ${DATA_DIR} + +log_schema: + host_key: pod + +sources: + # Reads the internal Vector logs + vector: + type: internal_logs + + files_stdout: + type: file + include: + - ${LOG_DIR}/*/*.stdout.log + + files_stderr: + type: file + include: + - ${LOG_DIR}/*/*.stderr.log + + files_log4j2: + type: file + include: + - ${LOG_DIR}/*/*.log4j2.xml + line_delimiter: "\r\n" + +transforms: + processed_files_stdout: + inputs: + - files_stdout + type: remap + source: | + .logger = "ROOT" + .level = "INFO" + + processed_files_stderr: + inputs: + - files_stderr + type: remap + source: | + .logger = "ROOT" + .level = "ERROR" + + processed_files_log4j2: + inputs: + - files_log4j2 + type: remap + source: | + raw_message = string!(.message) + + .timestamp = now() + .logger = "" + .level = "INFO" + .message = "" + .errors = [] + + event = {} + parsed_event, err = parse_xml(raw_message) + if err != null { + error = "XML not parsable: " + err + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + if !is_object(parsed_event.Event) { + error = "Parsed event contains no \"Event\" tag." + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + event = object!(parsed_event.Event) + + tag_instant_valid = false + instant, err = object(event.Instant) + if err == null { + epoch_nanoseconds, err = to_int(instant.@epochSecond) * 1_000_000_000 + to_int(instant.@nanoOfSecond) + if err == null && epoch_nanoseconds != 0 { + converted_timestamp, err = from_unix_timestamp(epoch_nanoseconds, "nanoseconds") + if err == null { + .timestamp = converted_timestamp + tag_instant_valid = true + } else { + .errors = push(.errors, "Instant invalid, trying property timeMillis instead: " + err) + } + } else { + .errors = push(.errors, "Instant invalid, trying property timeMillis instead: " + err) + } + } + if !tag_instant_valid { + epoch_milliseconds, err = to_int(event.@timeMillis) + if err == null && epoch_milliseconds != 0 { + converted_timestamp, err = from_unix_timestamp(epoch_milliseconds, "milliseconds") + if err == null { + .timestamp = converted_timestamp + } else { + .errors = push(.errors, "timeMillis not parsable, using current time instead: " + err) + } + } else { + .errors = push(.errors, "timeMillis not parsable, using current time instead: " + err) + } + } + + .logger, err = string(event.@loggerName) + if err != null || is_empty(.logger) { + .errors = push(.errors, "Logger not found.") + } + + level, err = string(event.@level) + if err != null { + .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") + } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], level) { + .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") + } else { + .level = level + } + + exception = null + thrown = event.Thrown + if is_object(thrown) { + exception = "Exception" + thread, err = string(event.@thread) + if err == null && !is_empty(thread) { + exception = exception + " in thread \"" + thread + "\"" + } + thrown_name, err = string(thrown.@name) + if err == null && !is_empty(exception) { + exception = exception + " " + thrown_name + } + message = string(thrown.@localizedMessage) ?? + string(thrown.@message) ?? + "" + if !is_empty(message) { + exception = exception + ": " + message + } + stacktrace_items = array(thrown.ExtendedStackTrace.ExtendedStackTraceItem) ?? [] + stacktrace = "" + for_each(stacktrace_items) -> |_index, value| { + stacktrace = stacktrace + " " + class = string(value.@class) ?? "" + method = string(value.@method) ?? "" + if !is_empty(class) && !is_empty(method) { + stacktrace = stacktrace + "at " + class + "." + method + } + file = string(value.@file) ?? "" + line = string(value.@line) ?? "" + if !is_empty(file) && !is_empty(line) { + stacktrace = stacktrace + "(" + file + ":" + line + ")" + } + exact = to_bool(value.@exact) ?? false + location = string(value.@location) ?? "" + version = string(value.@version) ?? "" + if !is_empty(location) && !is_empty(version) { + stacktrace = stacktrace + " " + if !exact { + stacktrace = stacktrace + "~" + } + stacktrace = stacktrace + "[" + location + ":" + version + "]" + } + stacktrace = stacktrace + "\n" + } + if stacktrace != "" { + exception = exception + "\n" + stacktrace + } + } + + message, err = string(event.Message) + if err != null || is_empty(message) { + message = null + .errors = push(.errors, "Message not found.") + } + .message = join!(compact([message, exception]), "\n") + } + } + + # Extends the processed files with the fields "container" and "file" + extended_logs_files: + inputs: + - processed_files_* + type: remap + source: | + del(.source_type) + if .errors == [] { + del(.errors) + } + . |= parse_regex!(.file, r'^${LOG_DIR}/(?P.*?)/(?P.*?)$') + + # Filters the logs of the Vector agent according to the defined log level + filtered_logs_vector: + inputs: + - vector + type: filter + condition: > + (.metadata.level == "TRACE" && "${VECTOR_FILE_LOG_LEVEL}" == "trace") || + (.metadata.level == "DEBUG" && includes(["trace", "debug"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "INFO" && includes(["trace", "debug", "info"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "WARN" && includes(["trace", "debug", "info", "warn"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "ERROR" && includes(["trace", "debug", "info", "warn", "error"], "${VECTOR_FILE_LOG_LEVEL}")) + + # Aligns the logs of the Vector agent with the common format + extended_logs_vector: + inputs: + - filtered_logs_vector + type: remap + source: | + .container = "vector" + .level = .metadata.level + .logger = .metadata.module_path + if exists(.file) { .processed_file = del(.file) } + del(.metadata) + del(.pid) + del(.source_type) + + # Add the fields "namespace", "cluster", "role" and "roleGroup" to all logs + extended_logs: + inputs: + - extended_logs_* + type: remap + source: | + .namespace = "${NAMESPACE}" + .cluster = "${CLUSTER_NAME}" + .role = "${ROLE_NAME}" + .roleGroup = "${ROLE_GROUP_NAME}" + +sinks: + # Forward the logs to the Vector aggregator + aggregator: + inputs: + - extended_logs + type: vector + address: ${VECTOR_AGGREGATOR_ADDRESS} diff --git a/rust/operator-binary/src/controller/build/properties/runtime_properties.rs b/rust/operator-binary/src/controller/build/properties/runtime_properties.rs new file mode 100644 index 00000000..26d56c4b --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/runtime_properties.rs @@ -0,0 +1,235 @@ +//! Builder for the static, role-specific `runtime.properties`. +//! +//! Dynamic, cluster-derived values (ZooKeeper, extensions, metadata DB, deep storage, +//! TLS, auth, ports, resource-derived sizes) are added by the controller, not here. + +use std::collections::BTreeMap; + +use crate::crd::{DeepStorageSpec, DruidRole, METRICS_PORT}; + +// deep storage +const DS_TYPE: &str = "druid.storage.type"; +const DS_DIRECTORY: &str = "druid.storage.storageDirectory"; +const DS_BASE_KEY: &str = "druid.storage.baseKey"; +// OPA +const AUTH_AUTHORIZERS: &str = "druid.auth.authorizers"; +const AUTH_AUTHORIZERS_VALUE: &str = "[\"OpaAuthorizer\"]"; +const AUTH_AUTHORIZER_OPA_TYPE: &str = "druid.auth.authorizer.OpaAuthorizer.type"; +const AUTH_AUTHORIZER_OPA_TYPE_VALUE: &str = "opa"; +// metrics +const PROMETHEUS_PORT: &str = "druid.emitter.prometheus.port"; + +/// The recommended cluster-level `runtime.properties` derived from the cluster config (deep +/// storage, OPA authorization and metrics). These are independent of role and role group. +pub fn cluster_runtime_properties( + deep_storage: &DeepStorageSpec, + opa_authorization_enabled: bool, +) -> BTreeMap { + let mut result = BTreeMap::new(); + + // OPA + if opa_authorization_enabled { + result.insert( + AUTH_AUTHORIZERS.to_string(), + AUTH_AUTHORIZERS_VALUE.to_string(), + ); + result.insert( + AUTH_AUTHORIZER_OPA_TYPE.to_string(), + AUTH_AUTHORIZER_OPA_TYPE_VALUE.to_string(), + ); + // The opaUri still needs to be set, but that requires a discovery config map and is + // handled in the controller. + } + + // deep storage + result.insert(DS_TYPE.to_string(), deep_storage.to_string()); + match deep_storage { + DeepStorageSpec::Hdfs(hdfs) => { + result.insert(DS_DIRECTORY.to_string(), hdfs.directory.clone()); + } + DeepStorageSpec::S3(s3_spec) => { + if let Some(key) = &s3_spec.base_key { + result.insert(DS_BASE_KEY.to_string(), key.to_string()); + } + } + } + + // metrics + result.insert(PROMETHEUS_PORT.to_string(), METRICS_PORT.to_string()); + + result +} + +/// Defaults rendered for every role. +const ALL_ROLES: &[(&str, &str)] = &[ + ("druid.startup.logging.logProperties", "true"), + ( + "druid.monitoring.monitors", + "[\"org.apache.druid.java.util.metrics.JvmMonitor\"]", + ), + ("druid.emitter", "prometheus"), + ("druid.emitter.prometheus.strategy", "exporter"), + ("druid.emitter.prometheus.namespace", "druid"), + ( + "druid.indexer.logs.directory", + "/stackable/var/druid/indexing-logs", + ), +]; + +const BROKER: &[(&str, &str)] = &[("druid.processing.tmpDir", "/stackable/var/druid/processing")]; + +const COORDINATOR: &[(&str, &str)] = &[ + ("druid.coordinator.startDelay", "PT20S"), + ("druid.coordinator.period", "PT20S"), + ("druid.coordinator.asOverlord.enabled", "true"), + ( + "druid.coordinator.asOverlord.overlordService", + "druid/overlord", + ), + ("druid.indexer.queue.startDelay", "PT20S"), + ("druid.indexer.runner.type", "remote"), + ("druid.indexer.storage.type", "metadata"), +]; + +const HISTORICAL: &[(&str, &str)] = &[ + ("druid.historical.cache.useCache", "true"), + ("druid.historical.cache.populateCache", "true"), + ("druid.processing.tmpDir", "/stackable/var/druid/processing"), +]; + +const MIDDLEMANAGER: &[(&str, &str)] = &[ + ( + "druid.indexer.task.hadoopWorkingPath", + "/stackable/var/druid/hadoop-tmp", + ), + ( + "druid.indexer.task.baseTaskDir", + "/stackable/var/druid/task", + ), + ( + "druid.indexer.runner.javaOpts", + "-server -Xms256m -Xmx256m -XX:MaxDirectMemorySize=300m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager", + ), +]; + +const ROUTER: &[(&str, &str)] = &[ + ("druid.router.managementProxy.enabled", "true"), + ("druid.router.http.numConnections", "25"), +]; + +/// Static `recommendedValues` defaults for a role, as `key -> value`. +pub fn defaults(role: &DruidRole) -> BTreeMap { + let role_specific: &[(&str, &str)] = match role { + DruidRole::Broker => BROKER, + DruidRole::Coordinator => COORDINATOR, + DruidRole::Historical => HISTORICAL, + DruidRole::MiddleManager => MIDDLEMANAGER, + DruidRole::Router => ROUTER, + }; + ALL_ROLES + .iter() + .chain(role_specific.iter()) + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + // All expected values below are copied verbatim from the runtime.properties + // blocks of tests/templates/kuttl/smoke/53-assert.yaml.j2. + + #[test] + fn common_defaults_present_for_every_role() { + for role in [ + DruidRole::Broker, + DruidRole::Coordinator, + DruidRole::Historical, + DruidRole::MiddleManager, + DruidRole::Router, + ] { + let p = defaults(&role); + assert_eq!(p["druid.emitter"], "prometheus".to_string()); + assert_eq!(p["druid.emitter.prometheus.namespace"], "druid".to_string()); + assert_eq!( + p["druid.emitter.prometheus.strategy"], + "exporter".to_string() + ); + assert_eq!( + p["druid.monitoring.monitors"], + "[\"org.apache.druid.java.util.metrics.JvmMonitor\"]".to_string() + ); + assert_eq!( + p["druid.indexer.logs.directory"], + "/stackable/var/druid/indexing-logs".to_string() + ); + assert_eq!(p["druid.startup.logging.logProperties"], "true".to_string()); + } + } + + #[test] + fn processing_tmpdir_only_for_broker_and_historical() { + // 53-assert: present for broker + historical, ABSENT for the other three. + assert!(defaults(&DruidRole::Broker).contains_key("druid.processing.tmpDir")); + assert!(defaults(&DruidRole::Historical).contains_key("druid.processing.tmpDir")); + assert!(!defaults(&DruidRole::Coordinator).contains_key("druid.processing.tmpDir")); + assert!(!defaults(&DruidRole::MiddleManager).contains_key("druid.processing.tmpDir")); + assert!(!defaults(&DruidRole::Router).contains_key("druid.processing.tmpDir")); + } + + #[test] + fn coordinator_defaults_match_snapshot() { + let p = defaults(&DruidRole::Coordinator); + assert_eq!( + p["druid.coordinator.asOverlord.enabled"], + "true".to_string() + ); + assert_eq!( + p["druid.coordinator.asOverlord.overlordService"], + "druid/overlord".to_string() + ); + assert_eq!(p["druid.coordinator.period"], "PT20S".to_string()); + assert_eq!(p["druid.coordinator.startDelay"], "PT20S".to_string()); + assert_eq!(p["druid.indexer.queue.startDelay"], "PT20S".to_string()); + assert_eq!(p["druid.indexer.runner.type"], "remote".to_string()); + assert_eq!(p["druid.indexer.storage.type"], "metadata".to_string()); + } + + #[test] + fn historical_defaults_match_snapshot() { + let p = defaults(&DruidRole::Historical); + assert_eq!(p["druid.historical.cache.useCache"], "true".to_string()); + assert_eq!( + p["druid.historical.cache.populateCache"], + "true".to_string() + ); + } + + #[test] + fn middlemanager_javaopts_exact() { + let p = defaults(&DruidRole::MiddleManager); + assert_eq!( + p["druid.indexer.runner.javaOpts"], + "-server -Xms256m -Xmx256m -XX:MaxDirectMemorySize=300m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager" + ); + assert_eq!( + p["druid.indexer.task.baseTaskDir"], + "/stackable/var/druid/task".to_string() + ); + assert_eq!( + p["druid.indexer.task.hadoopWorkingPath"], + "/stackable/var/druid/hadoop-tmp".to_string() + ); + } + + #[test] + fn router_defaults_match_snapshot() { + let p = defaults(&DruidRole::Router); + assert_eq!( + p["druid.router.managementProxy.enabled"], + "true".to_string() + ); + assert_eq!(p["druid.router.http.numConnections"], "25".to_string()); + } +} diff --git a/rust/operator-binary/src/controller/build/properties/security_properties.rs b/rust/operator-binary/src/controller/build/properties/security_properties.rs new file mode 100644 index 00000000..1bed42a2 --- /dev/null +++ b/rust/operator-binary/src/controller/build/properties/security_properties.rs @@ -0,0 +1,52 @@ +//! Builder for `security.properties` (Druid's JVM security properties file). + +use std::collections::BTreeMap; + +const NETWORKADDRESS_CACHE_TTL: &str = "networkaddress.cache.ttl"; +const NETWORKADDRESS_CACHE_NEGATIVE_TTL: &str = "networkaddress.cache.negative.ttl"; + +const DEFAULT_NETWORKADDRESS_CACHE_TTL: &str = "30"; +const DEFAULT_NETWORKADDRESS_CACHE_NEGATIVE_TTL: &str = "0"; + +/// Build the `security.properties` key/value pairs (same defaults for all roles). +/// +/// `overrides` are the user's merged (role <- rolegroup) `security.properties` +/// config overrides, as returned by `DruidConfigOverrides::get_key_value_overrides`. +/// Override values win. +pub fn build(overrides: &BTreeMap) -> BTreeMap { + let mut props: BTreeMap = BTreeMap::new(); + props.insert( + NETWORKADDRESS_CACHE_TTL.to_string(), + DEFAULT_NETWORKADDRESS_CACHE_TTL.to_string(), + ); + props.insert( + NETWORKADDRESS_CACHE_NEGATIVE_TTL.to_string(), + DEFAULT_NETWORKADDRESS_CACHE_NEGATIVE_TTL.to_string(), + ); + props.extend(overrides.iter().map(|(k, v)| (k.clone(), v.clone()))); + props +} + +#[cfg(test)] +mod tests { + use super::*; + + // Expected values copied verbatim from tests/templates/kuttl/smoke/53-assert.yaml.j2 + // (security.properties block, identical for every role): + // networkaddress.cache.negative.ttl=0 + // networkaddress.cache.ttl=30 + #[test] + fn defaults_match_snapshot() { + let props = build(&BTreeMap::new()); + assert_eq!(props["networkaddress.cache.ttl"], "30".to_string()); + assert_eq!(props["networkaddress.cache.negative.ttl"], "0".to_string()); + assert_eq!(props.len(), 2); + } + + #[test] + fn override_wins() { + let ov = BTreeMap::from([("networkaddress.cache.ttl".to_string(), "60".to_string())]); + let props = build(&ov); + assert_eq!(props["networkaddress.cache.ttl"], "60".to_string()); + } +} diff --git a/rust/operator-binary/src/controller/build/resource/config_map.rs b/rust/operator-binary/src/controller/build/resource/config_map.rs new file mode 100644 index 00000000..00a8a383 --- /dev/null +++ b/rust/operator-binary/src/controller/build/resource/config_map.rs @@ -0,0 +1,435 @@ +//! Builds the rolegroup [`ConfigMap`] from a [`ValidatedCluster`]. +//! +//! The per-file configs (runtime.properties / security.properties / jvm.config) are rendered here +//! from the merged [`DruidRoleGroupConfig`] (config plus the merged config overrides, including the +//! merged JVM argument overrides used for `jvm.config`); the recommended cluster-level runtime +//! properties are carried on `ValidatedCluster`. +//! +//! Metadata, owner reference and recommended labels are derived entirely from `ValidatedCluster` +//! (which carries the validated name/namespace/uid and implements `Resource`). +//! +//! The builder does not read the raw [`v1alpha1::DruidCluster`] at all: everything it needs is +//! carried on `ValidatedCluster` (resolved during the validate step). + +use std::{collections::BTreeMap, str::FromStr}; + +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, + crd::s3, + k8s_openapi::api::core::v1::{ConfigMap, EnvVar}, + product_logging::framework::VECTOR_CONFIG_FILE, + v2::{ + builder::meta::ownerreference_from_resource, + config_file_writer::to_java_properties_string, + kvp::label::recommended_labels, + types::operator::{ProductVersion, RoleGroupName}, + }, +}; + +use crate::{ + controller::{ + build::{ + jvm::construct_jvm_args, + properties::{ + ConfigFileName, + product_logging::{build_log4j2, vector_config_file_content}, + runtime_properties, security_properties, + }, + }, + controller_name, operator_name, product_name, + validate::{DruidRoleGroupConfig, ValidatedCluster}, + }, + crd::{ + DruidConfigOverrides, DruidRole, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, + STACKABLE_TRUST_STORE_TYPE, build_string_list, env_var_reference, file_reference, + }, +}; + +// Druid `runtime.properties` config-property keys assembled into the rolegroup ConfigMap here. +const EXTENSIONS_LOADLIST: &str = "druid.extensions.loadList"; +const ZOOKEEPER_CONNECTION_STRING: &str = "druid.zk.service.host"; +const DS_BUCKET: &str = "druid.storage.bucket"; +const S3_ENDPOINT_URL: &str = "druid.s3.endpoint.url"; +const S3_ACCESS_KEY: &str = "druid.s3.accessKey"; +const S3_SECRET_KEY: &str = "druid.s3.secretKey"; +const S3_PATH_STYLE_ACCESS: &str = "druid.s3.enablePathStyleAccess"; +const AUTH_AUTHORIZER_OPA_URI: &str = "druid.auth.authorizer.OpaAuthorizer.opaUri"; + +#[derive(Snafu, Debug)] +#[allow(clippy::enum_variant_names)] +pub enum Error { + #[snafu(display("failed to build ConfigMap for role group {role_group}"))] + BuildRoleGroupConfig { + source: stackable_operator::builder::configmap::Error, + role_group: String, + }, + + #[snafu(display("failed to configure S3 connection"))] + ConfigureS3 { + source: stackable_operator::crd::s3::v1alpha1::ConnectionError, + }, + + #[snafu(display("failed to serialize [runtime.properties]"))] + SerializeRuntimeProperties { + source: stackable_operator::v2::config_file_writer::PropertiesWriterError, + }, + + #[snafu(display("failed to serialize [security.properties] for {rolegroup}"))] + JvmSecurityProperties { + source: stackable_operator::v2::config_file_writer::PropertiesWriterError, + rolegroup: String, + }, + + #[snafu(display("failed to update Druid config from resources"))] + UpdateDruidConfigFromResources { source: crate::crd::resource::Error }, + + #[snafu(display("there was an error generating the authentication runtime settings"))] + GenerateAuthenticationRuntimeSettings { + source: crate::authentication::Error, + }, + + #[snafu(display("failed to derive Druid memory settings from resources"))] + DeriveMemorySettings { source: crate::crd::resource::Error }, + + #[snafu(display("failed to construct the jvm.config"))] + GetJvmConfig { + source: crate::controller::build::jvm::Error, + }, +} + +type Result = std::result::Result; + +const INDEXER_JAVA_OPTS: &str = "druid.indexer.runner.javaOptsArray"; + +/// The `druid.indexer.runner.javaOptsArray` entry that must be present in *every* rendered file +/// (runtime.properties and security.properties) for MiddleManagers. +fn middlemanager_indexer_java_opts() -> (String, String) { + ( + INDEXER_JAVA_OPTS.to_string(), + build_string_list(&[ + format!("-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}"), + format!("-Djavax.net.ssl.trustStorePassword={STACKABLE_TRUST_STORE_PASSWORD}"), + format!("-Djavax.net.ssl.trustStoreType={STACKABLE_TRUST_STORE_TYPE}"), + ]), + ) +} + +/// Returns the user-supplied key/value overrides for the given config file from a +/// [`DruidConfigOverrides`]. +fn key_value_overrides( + overrides: &DruidConfigOverrides, + file: ConfigFileName, +) -> BTreeMap { + match file { + ConfigFileName::RuntimeProperties => overrides.runtime_properties.overrides.clone(), + ConfigFileName::SecurityProperties => overrides.security_properties.overrides.clone(), + // log4j2.properties is rendered by the logging framework, and jvm.config is rendered from + // JVM argument overrides; neither is assembled from key/value overrides here. + ConfigFileName::Log4j2Properties | ConfigFileName::JvmConfig => BTreeMap::new(), + } +} + +/// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator +pub fn build_rolegroup_config_map( + cluster: &ValidatedCluster, + role: &DruidRole, + role_group_name: &RoleGroupName, + rg: &DruidRoleGroupConfig, +) -> Result { + let resource_names = cluster.resource_names(role, role_group_name); + let cluster_config = &cluster.cluster_config; + let druid_tls_security = &cluster_config.druid_tls_security; + let druid_auth_config = &cluster_config.druid_auth_config; + let zk_connstr = cluster_config.zookeeper_connection_string.as_str(); + let opa_connstr = cluster_config.opa_connection_string.as_deref(); + let s3_conn = cluster_config.s3_connection.as_ref(); + let deep_storage_bucket_name = cluster_config.deep_storage_bucket_name.as_deref(); + + let mut cm_conf_data = BTreeMap::new(); // filename -> filecontent + let metadata_database_connection_details = &cluster_config.metadata_db_connection; + + // ----- runtime.properties ----- + { + let mut conf: BTreeMap = Default::default(); + + // Add any properties derived from storage manifests, such as segment cache locations. + // This has to be done here since there is no other suitable place for it. + // Previously such properties were added in the compute_files() function, + // but that code path is now incompatible with the design of fragment merging. + rg.config + .resources + .update_druid_config_file(&mut conf) + .context(UpdateDruidConfigFromResourcesSnafu)?; + // NOTE: druid.host can be set manually - if it isn't, the canonical host name of + // the local host is used. This should work with the agent and k8s host networking + // but might need to be revisited in the future + conf.insert( + ZOOKEEPER_CONNECTION_STRING.to_string(), + zk_connstr.to_string(), + ); + + conf.insert( + EXTENSIONS_LOADLIST.to_string(), + build_string_list(&cluster_config.extensions), + ); + + if let Some(opa_str) = opa_connstr { + conf.insert(AUTH_AUTHORIZER_OPA_URI.to_string(), opa_str.to_string()); + }; + + conf.insert( + crate::crd::database::METADATA_STORAGE_TYPE.to_string(), + cluster_config.metadata_storage_type.clone(), + ); + + conf.insert( + crate::crd::database::METADATA_STORAGE_CONNECTOR_CONNECT_URI.to_string(), + metadata_database_connection_details + .connection_url + .to_string(), + ); + + if let Some(EnvVar { + name: username_env_name, + .. + }) = &metadata_database_connection_details.username_env + { + conf.insert( + crate::crd::database::METADATA_STORAGE_USER.to_string(), + env_var_reference(username_env_name), + ); + } + + if let Some(EnvVar { + name: password_env_name, + .. + }) = &metadata_database_connection_details.password_env + { + conf.insert( + crate::crd::database::METADATA_STORAGE_PASSWORD.to_string(), + env_var_reference(password_env_name), + ); + } + + if let Some(s3) = s3_conn { + if !s3.region.is_default_config() { + // Raising this as warning instead of returning an error, better safe than sorry. + // It might still work out for the user. + tracing::warn!( + region = ?s3.region, + "You configured a non-default region on the S3Connection. + The S3Connection region field is ignored because Druid uses the AWS SDK v1, which ignores the region if the endpoint is set. \ + The host is a required field, therefore the endpoint will always be set." + ) + } + + conf.insert( + S3_ENDPOINT_URL.to_string(), + s3.endpoint().context(ConfigureS3Snafu)?.to_string(), + ); + + if let Some((access_key_file, secret_key_file)) = s3.credentials_mount_paths() { + conf.insert(S3_ACCESS_KEY.to_string(), file_reference(access_key_file)); + conf.insert(S3_SECRET_KEY.to_string(), file_reference(secret_key_file)); + } + + conf.insert( + S3_PATH_STYLE_ACCESS.to_string(), + (s3.access_style == s3::v1alpha1::S3AccessStyle::Path).to_string(), + ); + } + // When no deep-storage bucket is set (e.g. HDFS deep storage) this renders an empty + // `druid.storage.bucket=`, matching the previous `None` -> empty-value behavior. + conf.insert( + DS_BUCKET.to_string(), + deep_storage_bucket_name.unwrap_or_default().to_string(), + ); + + // add tls encryption / auth properties + druid_tls_security.add_tls_config_properties(&mut conf, role); + + if let Some(auth_config) = druid_auth_config { + conf.extend( + auth_config + .generate_runtime_properties_config(role) + .context(GenerateAuthenticationRuntimeSettingsSnafu)?, + ); + }; + + // Role/rolegroup runtime.properties: the recommended cluster-config-derived properties, + // the MiddleManager indexer opts, the per-role defaults and finally the user overrides + // (each layer wins over the previous, and over the cluster-level properties above). + conf.extend(runtime_properties::cluster_runtime_properties( + &cluster_config.deep_storage, + cluster_config.opa_connection_string.is_some(), + )); + if *role == DruidRole::MiddleManager { + let (k, v) = middlemanager_indexer_java_opts(); + conf.insert(k, v); + } + conf.extend(runtime_properties::defaults(role)); + conf.extend(key_value_overrides( + &rg.config_overrides, + ConfigFileName::RuntimeProperties, + )); + + let runtime_properties = + to_java_properties_string(conf.iter()).context(SerializeRuntimePropertiesSnafu)?; + cm_conf_data.insert( + ConfigFileName::RuntimeProperties.to_string(), + runtime_properties, + ); + } + + // ----- jvm.config ----- + { + let (heap, direct) = rg + .config + .resources + .get_memory_sizes(role) + .context(DeriveMemorySettingsSnafu)?; + let jvm_config = construct_jvm_args( + role, + &rg.product_specific_common_config.jvm_argument_overrides, + heap, + direct, + ) + .context(GetJvmConfigSnafu)?; + cm_conf_data.insert(ConfigFileName::JvmConfig.to_string(), jvm_config); + } + + // ----- security.properties ----- + { + let mut security_config: BTreeMap = BTreeMap::new(); + if *role == DruidRole::MiddleManager { + let (k, v) = middlemanager_indexer_java_opts(); + security_config.insert(k, v); + } + let security_overrides = + key_value_overrides(&rg.config_overrides, ConfigFileName::SecurityProperties); + security_config.extend(security_properties::build(&security_overrides)); + + cm_conf_data.insert( + ConfigFileName::SecurityProperties.to_string(), + to_java_properties_string(security_config.iter()).with_context(|_| { + JvmSecurityPropertiesSnafu { + rolegroup: role_group_name.to_string(), + } + })?, + ); + } + + let mut config_map_builder = ConfigMapBuilder::new(); + config_map_builder.metadata( + ObjectMetaBuilder::new() + .name_and_namespace(cluster) + .name(resource_names.role_group_config_map().to_string()) + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(recommended_labels( + cluster, + &product_name(), + &ProductVersion::from_str(&cluster.image.app_version_label_value) + .expect("a valid product version"), + &operator_name(), + &controller_name(), + &role.to_role_name(), + role_group_name, + )) + .build(), + ); + + for (filename, file_content) in cm_conf_data.iter() { + config_map_builder.add_data(filename, file_content); + } + + if let Some(log4j2_config) = build_log4j2(&rg.config.logging.druid_container) { + config_map_builder.add_data(ConfigFileName::Log4j2Properties.to_string(), log4j2_config); + } + + // The Vector agent config (`vector.yaml`) is a static, env-var-parameterized file (mirroring + // the hive-/opensearch-operator). It is only added when the Vector agent is enabled. + if rg.config.logging.enable_vector_agent { + config_map_builder.add_data(VECTOR_CONFIG_FILE, vector_config_file_content()); + } + + config_map_builder + .build() + .with_context(|_| BuildRoleGroupConfigSnafu { + role_group: role_group_name.to_string(), + }) +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use stackable_operator::{ + k8s_openapi::api::core::v1::ConfigMap, v2::types::operator::RoleGroupName, + }; + + use super::*; + use crate::controller::validate::test_support::{ + MINIMAL_DRUID_YAML, druid_from_yaml, validated_cluster, + }; + + /// Builds the rolegroup `ConfigMap` for `/` from the minimal test fixture. + fn build_cm(role: DruidRole, role_group: &str) -> ConfigMap { + let druid = druid_from_yaml(MINIMAL_DRUID_YAML); + let cluster = validated_cluster(&druid); + let role_group_name = RoleGroupName::from_str(role_group).unwrap(); + let rg = cluster + .role_group_configs + .get(&role) + .expect("role present") + .get(&role_group_name) + .expect("role group present") + .clone(); + build_rolegroup_config_map(&cluster, &role, &role_group_name, &rg).expect("config map") + } + + #[test] + fn contains_the_operator_written_config_files() { + let cm = build_cm(DruidRole::Broker, "default"); + let data = cm.data.expect("config map has data"); + + // Always-present operator-written files. + assert!(data.contains_key(&ConfigFileName::RuntimeProperties.to_string())); + assert!(data.contains_key(&ConfigFileName::SecurityProperties.to_string())); + assert!(data.contains_key(&ConfigFileName::JvmConfig.to_string())); + // Automatic logging is the default, so log4j2 is rendered. + assert!(data.contains_key(&ConfigFileName::Log4j2Properties.to_string())); + // The Vector agent is disabled by default, so no `vector.yaml` is added. + assert!(!data.contains_key(VECTOR_CONFIG_FILE)); + } + + #[test] + fn has_the_expected_name_and_recommended_labels() { + let cm = build_cm(DruidRole::Broker, "default"); + let meta = cm.metadata; + + assert_eq!(meta.name.as_deref(), Some("simple-druid-broker-default")); + + let labels = meta.labels.expect("recommended labels"); + assert_eq!( + labels.get("app.kubernetes.io/name").map(String::as_str), + Some("druid") + ); + assert_eq!( + labels.get("app.kubernetes.io/instance").map(String::as_str), + Some("simple-druid") + ); + assert_eq!( + labels + .get("app.kubernetes.io/component") + .map(String::as_str), + Some("broker") + ); + assert_eq!( + labels + .get("app.kubernetes.io/role-group") + .map(String::as_str), + Some("default") + ); + } +} diff --git a/rust/operator-binary/src/controller/build/resource/discovery.rs b/rust/operator-binary/src/controller/build/resource/discovery.rs new file mode 100644 index 00000000..512922a8 --- /dev/null +++ b/rust/operator-binary/src/controller/build/resource/discovery.rs @@ -0,0 +1,86 @@ +//! Discovery for Druid. We make Druid discoverable by putting a connection string to the router service +//! inside a config map. We only provide a connection string to the router service, since it serves as +//! a gateway to the cluster for client queries. +use std::str::FromStr; + +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, + crd::listener::v1alpha1::Listener, + k8s_openapi::api::core::v1::ConfigMap, + v2::{ + builder::meta::ownerreference_from_resource, + kvp::label::recommended_labels, + types::operator::{ProductVersion, RoleGroupName}, + }, +}; + +use crate::{ + controller::{ + build::resource::listener::build_listener_connection_string, controller_name, + operator_name, product_name, validate::ValidatedCluster, + }, + crd::DruidRole, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to build ConfigMap"))] + BuildConfigMap { + source: stackable_operator::builder::configmap::Error, + }, + + #[snafu(display("failed to configure listener discovery configmap"))] + ListenerConfiguration { + source: crate::controller::build::resource::listener::Error, + }, +} + +/// Builds discovery [`ConfigMap`]s for connecting to a Druid cluster. +pub async fn build_discovery_configmaps( + cluster: &ValidatedCluster, + listener: Listener, +) -> Result, Error> { + Ok(vec![build_discovery_configmap(cluster, listener)?]) +} + +/// Build a discovery [`ConfigMap`] containing information about how to connect to a certain Druid cluster. +fn build_discovery_configmap( + cluster: &ValidatedCluster, + listener: Listener, +) -> Result { + let router_host = build_listener_connection_string( + listener, + &cluster.cluster_config.druid_tls_security, + &DruidRole::Router.to_string(), + ) + .context(ListenerConfigurationSnafu)?; + let sqlalchemy_conn_str = format!("druid://{}/druid/v2/sql", router_host); + let avatica_conn_str = format!( + "jdbc:avatica:remote:url=http://{}/druid/v2/sql/avatica/", + router_host + ); + + ConfigMapBuilder::new() + .metadata( + ObjectMetaBuilder::new() + .name_and_namespace(cluster) + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(recommended_labels( + cluster, + &product_name(), + &ProductVersion::from_str(&cluster.image.app_version_label_value) + .expect("a valid product version"), + &operator_name(), + &controller_name(), + &DruidRole::Router.to_role_name(), + &RoleGroupName::from_str("discovery").expect("a valid role group name"), + )) + .build(), + ) + .add_data("DRUID_ROUTER", router_host) + .add_data("DRUID_SQLALCHEMY", sqlalchemy_conn_str) + .add_data("DRUID_AVATICA_JDBC", avatica_conn_str) + .build() + .context(BuildConfigMapSnafu) +} diff --git a/rust/operator-binary/src/listener.rs b/rust/operator-binary/src/controller/build/resource/listener.rs similarity index 56% rename from rust/operator-binary/src/listener.rs rename to rust/operator-binary/src/controller/build/resource/listener.rs index 2283cd76..cd33aa6d 100644 --- a/rust/operator-binary/src/listener.rs +++ b/rust/operator-binary/src/controller/build/resource/listener.rs @@ -1,19 +1,26 @@ -use snafu::{OptionExt, ResultExt, Snafu}; +use std::str::FromStr; + +use snafu::{OptionExt, Snafu}; use stackable_operator::{ - builder::{ - meta::ObjectMetaBuilder, - pod::volume::{ListenerOperatorVolumeSourceBuilder, ListenerReference}, - }, + builder::meta::ObjectMetaBuilder, crd::listener::{self, v1alpha1::Listener}, k8s_openapi::api::core::v1::PersistentVolumeClaim, - kube::ResourceExt, - kvp::{Labels, ObjectLabels}, + kvp::Labels, + v2::{ + builder::{ + meta::ownerreference_from_resource, + pod::volume::{ListenerReference, listener_operator_volume_source_builder_build_pvc}, + }, + types::kubernetes::{ListenerName, PersistentVolumeClaimName}, + }, }; -use crate::crd::{ - DruidRole, - security::{DruidTlsSecurity, PLAINTEXT_PORT_NAME, TLS_PORT_NAME}, - v1alpha1, +use crate::{ + controller::validate::ValidatedCluster, + crd::{ + DruidRole, + security::{DruidTlsSecurity, PLAINTEXT_PORT_NAME, TLS_PORT_NAME}, + }, }; pub const LISTENER_VOLUME_NAME: &str = "listener"; @@ -21,21 +28,6 @@ pub const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("listener object is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, - }, - - #[snafu(display("failed to build listener object meta data"))] - BuildObjectMeta { - source: stackable_operator::builder::meta::Error, - }, - - #[snafu(display("failed to build listener volume"))] - BuildListenerPersistentVolume { - source: stackable_operator::builder::pod::volume::ListenerOperatorVolumeSourceBuilderError, - }, - #[snafu(display("{role_name} listener has no adress"))] RoleListenerHasNoAddress { role_name: String }, @@ -47,52 +39,57 @@ pub enum Error { } pub fn build_group_listener( - druid: &v1alpha1::DruidCluster, - object_labels: ObjectLabels, + cluster: &ValidatedCluster, + object_labels: Labels, listener_class: String, - listener_group_name: String, + listener_group_name: ListenerName, druid_role: &DruidRole, - druid_tls_security: &DruidTlsSecurity, -) -> Result { - Ok(Listener { +) -> Listener { + Listener { metadata: ObjectMetaBuilder::new() - .name_and_namespace(druid) - .name(listener_group_name) - .ownerreference_from_resource(druid, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&object_labels) - .context(BuildObjectMetaSnafu)? + .name_and_namespace(cluster) + .name(listener_group_name.to_string()) + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(object_labels) .build(), spec: listener::v1alpha1::ListenerSpec { class_name: Some(listener_class), - ports: druid_tls_security.listener_ports(druid_role), + ports: Some( + cluster + .cluster_config + .druid_tls_security + .listener_ports(druid_role), + ), ..listener::v1alpha1::ListenerSpec::default() }, status: None, - }) + } } pub fn build_group_listener_pvc( - group_listener_name: &String, + group_listener_name: &ListenerName, unversioned_recommended_labels: &Labels, -) -> Result { - ListenerOperatorVolumeSourceBuilder::new( - &ListenerReference::ListenerName(group_listener_name.to_string()), +) -> PersistentVolumeClaim { + listener_operator_volume_source_builder_build_pvc( + &ListenerReference::Listener(group_listener_name.clone()), unversioned_recommended_labels, + &PersistentVolumeClaimName::from_str(LISTENER_VOLUME_NAME) + .expect("a valid persistent volume claim name"), ) - .build_pvc(LISTENER_VOLUME_NAME.to_string()) - .context(BuildListenerPersistentVolumeSnafu) } pub fn group_listener_name( - druid: &v1alpha1::DruidCluster, + cluster: &ValidatedCluster, druid_role: &DruidRole, -) -> Option { +) -> Option { match druid_role { - DruidRole::Coordinator | DruidRole::Broker | DruidRole::Router => Some(format!( - "{cluster_name}-{druid_role}", - cluster_name = druid.name_any(), - )), + DruidRole::Coordinator | DruidRole::Broker | DruidRole::Router => Some( + ListenerName::from_str(&format!( + "{cluster_name}-{druid_role}", + cluster_name = cluster.name, + )) + .expect("a valid listener name"), + ), DruidRole::Historical | DruidRole::MiddleManager => None, } } diff --git a/rust/operator-binary/src/controller/build/resource/mod.rs b/rust/operator-binary/src/controller/build/resource/mod.rs new file mode 100644 index 00000000..d7fba99e --- /dev/null +++ b/rust/operator-binary/src/controller/build/resource/mod.rs @@ -0,0 +1,8 @@ +//! Builders for the individual Kubernetes resources of a Druid cluster. + +pub mod config_map; +pub mod discovery; +pub mod listener; +pub mod pdb; +pub mod service; +pub mod statefulset; diff --git a/rust/operator-binary/src/controller/build/resource/pdb.rs b/rust/operator-binary/src/controller/build/resource/pdb.rs new file mode 100644 index 00000000..76fab894 --- /dev/null +++ b/rust/operator-binary/src/controller/build/resource/pdb.rs @@ -0,0 +1,61 @@ +use std::str::FromStr; + +use stackable_operator::{ + commons::pdb::PdbConfig, + k8s_openapi::api::policy::v1::PodDisruptionBudget, + v2::{builder::pdb::pod_disruption_budget_builder_with_role, types::operator::RoleName}, +}; + +use crate::{ + controller::{controller_name, operator_name, product_name, validate::ValidatedCluster}, + crd::DruidRole, +}; + +/// Builds the [`PodDisruptionBudget`] for the given `role`, or `None` if PDBs are disabled. +pub fn build_pdb( + pdb: &PdbConfig, + cluster: &ValidatedCluster, + role: &DruidRole, +) -> Option { + if !pdb.enabled { + return None; + } + let max_unavailable = pdb.max_unavailable.unwrap_or(match role { + DruidRole::Broker => max_unavailable_brokers(), + DruidRole::Coordinator => max_unavailable_coordinators(), + DruidRole::Historical => max_unavailable_historicals(), + DruidRole::MiddleManager => max_unavailable_middle_managers(), + DruidRole::Router => max_unavailable_routers(), + }); + let pdb = pod_disruption_budget_builder_with_role( + cluster, + &product_name(), + &RoleName::from_str(&role.to_string()).expect("a DruidRole is a valid role name"), + &operator_name(), + &controller_name(), + ) + .with_max_unavailable(max_unavailable) + .build(); + + Some(pdb) +} + +fn max_unavailable_brokers() -> u16 { + 1 +} + +fn max_unavailable_coordinators() -> u16 { + 1 +} + +fn max_unavailable_historicals() -> u16 { + 1 +} + +fn max_unavailable_middle_managers() -> u16 { + 1 +} + +fn max_unavailable_routers() -> u16 { + 1 +} diff --git a/rust/operator-binary/src/controller/build/resource/service.rs b/rust/operator-binary/src/controller/build/resource/service.rs new file mode 100644 index 00000000..a103e737 --- /dev/null +++ b/rust/operator-binary/src/controller/build/resource/service.rs @@ -0,0 +1,148 @@ +use std::str::FromStr; + +use stackable_operator::{ + builder::meta::ObjectMetaBuilder, + k8s_openapi::api::core::v1::{Service, ServicePort, ServiceSpec}, + kvp::{Annotations, Labels}, + v2::{ + builder::meta::ownerreference_from_resource, + kvp::label::{recommended_labels, role_group_selector}, + types::operator::{ProductVersion, RoleGroupName}, + }, +}; + +use crate::{ + controller::{controller_name, operator_name, product_name, validate::ValidatedCluster}, + crd::{DruidRole, METRICS_PORT, METRICS_PORT_NAME}, +}; + +/// The rolegroup headless [`Service`] is a service that allows direct access to the instances of a certain rolegroup +/// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. +pub fn build_rolegroup_headless_service( + cluster: &ValidatedCluster, + druid_role: &DruidRole, + role_group_name: &RoleGroupName, +) -> Service { + let object_labels = recommended_labels( + cluster, + &product_name(), + &ProductVersion::from_str(&cluster.image.app_version_label_value) + .expect("a valid product version"), + &operator_name(), + &controller_name(), + &druid_role.to_role_name(), + role_group_name, + ); + let selector = role_group_selector( + cluster, + &product_name(), + &druid_role.to_role_name(), + role_group_name, + ); + Service { + metadata: ObjectMetaBuilder::new() + .name_and_namespace(cluster) + .name( + cluster + .resource_names(druid_role, role_group_name) + .headless_service_name() + .to_string(), + ) + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(object_labels) + .build(), + spec: Some(ServiceSpec { + // Internal communication does not need to be exposed + type_: Some("ClusterIP".to_string()), + cluster_ip: Some("None".to_string()), + ports: Some( + cluster + .cluster_config + .druid_tls_security + .service_ports(druid_role), + ), + selector: Some(selector.into()), + publish_not_ready_addresses: Some(true), + ..ServiceSpec::default() + }), + status: None, + } +} + +/// The rolegroup metrics [`Service`] is a service that exposes metrics and a prometheus scraping label. +pub fn build_rolegroup_metrics_service( + cluster: &ValidatedCluster, + druid_role: &DruidRole, + role_group_name: &RoleGroupName, +) -> Service { + let object_labels = recommended_labels( + cluster, + &product_name(), + &ProductVersion::from_str(&cluster.image.app_version_label_value) + .expect("a valid product version"), + &operator_name(), + &controller_name(), + &druid_role.to_role_name(), + role_group_name, + ); + let selector = role_group_selector( + cluster, + &product_name(), + &druid_role.to_role_name(), + role_group_name, + ); + Service { + metadata: ObjectMetaBuilder::new() + .name_and_namespace(cluster) + .name( + cluster + .resource_names(druid_role, role_group_name) + .metrics_service_name() + .to_string(), + ) + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(object_labels) + .with_labels(prometheus_labels()) + .with_annotations(prometheus_annotations()) + .build(), + spec: Some(ServiceSpec { + // Internal communication does not need to be exposed + type_: Some("ClusterIP".to_string()), + cluster_ip: Some("None".to_string()), + ports: Some(metrics_service_ports()), + selector: Some(selector.into()), + publish_not_ready_addresses: Some(true), + ..ServiceSpec::default() + }), + status: None, + } +} + +fn metrics_service_ports() -> Vec { + vec![ServicePort { + name: Some(METRICS_PORT_NAME.to_string()), + port: METRICS_PORT.into(), + protocol: Some("TCP".to_string()), + ..ServicePort::default() + }] +} + +/// The Prometheus scraping label added to the metrics [`Service`]. +fn prometheus_labels() -> Labels { + Labels::try_from([("prometheus.io/scrape", "true")]).expect("should be a valid label") +} + +/// Common annotations for Prometheus +/// +/// These annotations can be used in a ServiceMonitor. +/// +/// see also +fn prometheus_annotations() -> Annotations { + Annotations::try_from([ + ("prometheus.io/path".to_owned(), "/metrics".to_owned()), + ("prometheus.io/port".to_owned(), METRICS_PORT.to_string()), + ("prometheus.io/scheme".to_owned(), "http".to_owned()), + ("prometheus.io/scrape".to_owned(), "true".to_owned()), + ]) + .expect("should be valid annotations") +} diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs new file mode 100644 index 00000000..eb1545fd --- /dev/null +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -0,0 +1,494 @@ +//! Builds the rolegroup [`StatefulSet`] from a [`ValidatedCluster`]. + +use std::str::FromStr; + +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::{ + meta::ObjectMetaBuilder, + pod::{ + PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, + security::PodSecurityContextBuilder, volume::VolumeBuilder, + }, + }, + constants::RESTART_CONTROLLER_ENABLED_LABEL, + k8s_openapi::{ + DeepMerge, + api::{ + apps::v1::{StatefulSet, StatefulSetSpec}, + core::v1::{EnvVar, PersistentVolumeClaim, ServiceAccount}, + }, + apimachinery::pkg::apis::meta::v1::LabelSelector, + }, + kube::ResourceExt, + product_logging, + v2::{ + builder::{ + meta::ownerreference_from_resource, + pod::container::{EnvVarSet, new_container_builder}, + }, + kvp::label::{recommended_labels, role_group_selector}, + product_logging::framework::{ValidatedContainerLogConfigChoice, vector_container}, + role_group_utils::ResourceNames, + types::{ + kubernetes::{ContainerName, VolumeName}, + operator::{ProductVersion, RoleGroupName}, + }, + }, +}; + +use crate::{ + controller::{ + build::{ + properties::product_logging::MAX_DRUID_LOG_FILES_SIZE, + resource::listener::{ + LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_group_listener_pvc, + group_listener_name, secret_volume_listener_scope, + }, + }, + controller_name, operator_name, product_name, + validate::{DruidRoleGroupConfig, ValidatedCluster}, + }, + crd::{ + Container, DRUID_CONFIG_DIRECTORY, DeepStorageSpec, DruidRole, HDFS_CONFIG_DIRECTORY, + LOG_CONFIG_DIRECTORY, METRICS_PORT, METRICS_PORT_NAME, RW_CONFIG_DIRECTORY, + STACKABLE_LOG_DIR, ValidatedDruidConfig, + }, + operations::graceful_shutdown::add_graceful_shutdown_config, +}; + +// volume names +const DRUID_CONFIG_VOLUME_NAME: &str = "config"; +const HDFS_CONFIG_VOLUME_NAME: &str = "hdfs"; +const LOG_CONFIG_VOLUME_NAME: &str = "log-config"; +const LOG_VOLUME_NAME: &str = "log"; +const RW_CONFIG_VOLUME_NAME: &str = "rwconfig"; +const USERDATA_MOUNTPOINT: &str = "/stackable/userdata"; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to configure graceful shutdown"))] + GracefulShutdown { + source: crate::operations::graceful_shutdown::Error, + }, + + #[snafu(display("failed to add OIDC Volumes and VolumeMounts to the Pod and containers"))] + AuthVolumesBuild { + source: crate::authentication::Error, + }, + + #[snafu(display("failed to initialize security context"))] + FailedToInitializeSecurityContext { source: crate::crd::security::Error }, + + #[snafu(display( + "Druid does not support skipping the verification of the tls enabled S3 server" + ))] + S3TlsNoVerificationNotSupported, + + #[snafu(display("failed to configure S3 connection"))] + ConfigureS3 { + source: stackable_operator::crd::s3::v1alpha1::ConnectionError, + }, + + #[snafu(display("failed to update Druid config from resources"))] + UpdateDruidConfigFromResources { source: crate::crd::resource::Error }, + + #[snafu(display("failed to add needed volume"))] + AddVolume { + source: stackable_operator::builder::pod::Error, + }, + + #[snafu(display("failed to add needed volumeMount"))] + AddVolumeMount { + source: stackable_operator::builder::pod::container::Error, + }, +} + +type Result = std::result::Result; + +pub fn build_rolegroup_statefulset( + cluster: &ValidatedCluster, + role: &DruidRole, + role_group_name: &RoleGroupName, + rg: &DruidRoleGroupConfig, + service_account: &ServiceAccount, +) -> Result { + let merged_rolegroup_config = &rg.config; + let resource_names = cluster.resource_names(role, role_group_name); + // Everything below used to be threaded in as separate parameters; it all lives on the + // `ValidatedCluster` now. + let resolved_product_image = &cluster.image; + let s3_conn = cluster.cluster_config.s3_connection.as_ref(); + let druid_tls_security = &cluster.cluster_config.druid_tls_security; + let druid_auth_config = &cluster.cluster_config.druid_auth_config; + // prepare container builder + let prepare_container_name = ContainerName::from_str(&Container::Prepare.to_string()) + .expect("'prepare' is a valid container name"); + let mut cb_prepare = new_container_builder(&prepare_container_name); + // druid container builder + let druid_container_name = ContainerName::from_str(&Container::Druid.to_string()) + .expect("'druid' is a valid container name"); + let mut cb_druid = new_container_builder(&druid_container_name); + // init pod builder + let mut pb = PodBuilder::new(); + pb.affinity(&merged_rolegroup_config.affinity); + add_graceful_shutdown_config( + role, + druid_tls_security, + merged_rolegroup_config.graceful_shutdown_timeout, + &mut pb, + &mut cb_druid, + ) + .context(GracefulShutdownSnafu)?; + + let metadata_database_connection_details = &cluster.cluster_config.metadata_db_connection; + + let mut main_container_commands = role.main_container_prepare_commands(s3_conn); + let mut prepare_container_commands = vec![]; + if let ValidatedContainerLogConfigChoice::Automatic(log_config) = + &merged_rolegroup_config.logging.prepare_container + { + // This command needs to be added at the beginning of the shell commands, + // otherwise the output of the following commands will not be captured! + prepare_container_commands.push(product_logging::framework::capture_shell_output( + STACKABLE_LOG_DIR, + prepare_container_name.as_ref(), + log_config, + )); + } + prepare_container_commands.extend(druid_tls_security.build_tls_key_stores_cmd()); + + if let Some(auth_config) = druid_auth_config { + auth_config + .add_volumes_and_mounts(&mut pb, &mut cb_druid, &mut cb_prepare) + .context(AuthVolumesBuildSnafu)?; + prepare_container_commands.extend(auth_config.prepare_container_commands()); + main_container_commands.extend(auth_config.main_container_commands()) + } + + // volume and volume mounts + druid_tls_security + .add_tls_volume_and_volume_mounts( + &mut cb_prepare, + &mut cb_druid, + &mut pb, + &merged_rolegroup_config.requested_secret_lifetime, + // add listener + secret_volume_listener_scope(role), + ) + .context(FailedToInitializeSecurityContextSnafu)?; + + if let Some(s3) = s3_conn { + if s3.tls.uses_tls() && !s3.tls.uses_tls_verification() { + S3TlsNoVerificationNotSupportedSnafu.fail()?; + } + s3.add_volumes_and_mounts(&mut pb, vec![&mut cb_druid]) + .context(ConfigureS3Snafu)?; + } + + add_config_volume_and_volume_mounts(&resource_names, &mut cb_druid, &mut pb)?; + add_log_config_volume_and_volume_mounts( + &resource_names, + merged_rolegroup_config, + &mut cb_druid, + &mut pb, + )?; + add_log_volume_and_volume_mounts(&mut cb_druid, &mut cb_prepare, &mut pb)?; + add_hdfs_cm_volume_and_volume_mounts( + &cluster.cluster_config.deep_storage, + &mut cb_druid, + &mut pb, + )?; + merged_rolegroup_config + .resources + .update_volumes_and_volume_mounts(&mut cb_druid, &mut pb) + .context(UpdateDruidConfigFromResourcesSnafu)?; + + cb_prepare + .image_from_product_image(resolved_product_image) + .command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]) + .args(vec![prepare_container_commands.join("\n")]) + .resources( + ResourceRequirementsBuilder::new() + .with_cpu_request("100m") + .with_cpu_limit("400m") + .with_memory_request("512Mi") + .with_memory_limit("512Mi") + .build(), + ); + + metadata_database_connection_details.add_to_container(&mut cb_druid); + + // rest of env: the validated env overrides, rendered in sorted-by-name order. + let mut rest_env: Vec = rg.env_overrides.clone().into(); + + if let Some(auth_config) = druid_auth_config { + rest_env.extend(auth_config.get_env_var_mounts(cluster, role)) + } + + // Needed for the `containerdebug` process to log it's tracing information to. + rest_env.push(EnvVar { + name: "CONTAINERDEBUG_LOG_DIRECTORY".to_string(), + value: Some(format!("{STACKABLE_LOG_DIR}/containerdebug")), + value_from: None, + }); + + main_container_commands.push(role.main_container_start_command()); + cb_druid + .image_from_product_image(resolved_product_image) + .command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]) + .args(vec![main_container_commands.join("\n")]) + .add_env_vars(rest_env) + .add_container_ports(druid_tls_security.container_ports(role)) + .add_container_port(METRICS_PORT_NAME, METRICS_PORT.into()) + // 10s * 30 = 300s to come up + .startup_probe(druid_tls_security.get_tcp_socket_probe(30, 10, 30, 3)) + // 10s * 1 = 10s to get removed from service + .readiness_probe(druid_tls_security.get_tcp_socket_probe(10, 10, 1, 3)) + // 10s * 3 = 30s to be restarted + .liveness_probe(druid_tls_security.get_tcp_socket_probe(10, 10, 3, 3)) + .resources(merged_rolegroup_config.resources.as_resource_requirements()); + + // Add extra mounts if any are specified and the current role is MiddleManager + // Extra mounts may be needed for ingestion to add required certificates, truststores or similar + // files. + // Mounts are added to all roles, as we are currently unsure where they may be needed + // Known roles are MiddleManagers for ingestion and Historicals for deep storage (GCS plugin) + // We may at some time in the future revisit this and limit it again to avoid needlessly + // propagating potentially confidential files throughout the cluster + for volume in &cluster.cluster_config.extra_volumes { + // Extract values into vars so we make it impossible to log something other than + // what we actually use to create the mounts - maybe paranoid, but hey .. + let volume_name = &volume.name; + let mount_point = format!("{USERDATA_MOUNTPOINT}/{}", volume.name); + + tracing::info!( + ?volume_name, + ?mount_point, + ?role, + "Adding user specified extra volume", + ); + pb.add_volume(volume.clone()).context(AddVolumeSnafu)?; + cb_druid + .add_volume_mount(volume_name, mount_point) + .context(AddVolumeMountSnafu)?; + } + + let mut pvcs: Option> = None; + + if let Some(group_listener_name) = group_listener_name(cluster, role) { + cb_druid + .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) + .context(AddVolumeMountSnafu)?; + + // Used for PVC templates that cannot be modified once they are deployed + // A version value is required, and we do want to use the "recommended" format for the + // other desired labels, hence the "none" product version. + let unversioned_recommended_labels = recommended_labels( + cluster, + &product_name(), + &ProductVersion::from_str("none").expect("a valid product version"), + &operator_name(), + &controller_name(), + &role.to_role_name(), + role_group_name, + ); + + pvcs = Some(vec![build_group_listener_pvc( + &group_listener_name, + &unversioned_recommended_labels, + )]); + } + + let metadata = ObjectMetaBuilder::new() + .with_labels(recommended_labels( + cluster, + &product_name(), + &ProductVersion::from_str(&resolved_product_image.app_version_label_value) + .expect("a valid product version"), + &operator_name(), + &controller_name(), + &role.to_role_name(), + role_group_name, + )) + .build(); + + pb.image_pull_secrets_from_product_image(resolved_product_image) + .add_init_container(cb_prepare.build()) + .add_container(cb_druid.build()) + .metadata(metadata) + .service_account_name(service_account.name_any()) + .security_context(PodSecurityContextBuilder::new().fs_group(1000).build()); + + // The Vector agent reads the static `vector.yaml` (added to the rolegroup ConfigMap) from the + // config volume; the validated aggregator address comes from the up-front `ValidatedLogging`. + if let Some(vector_log_config) = &merged_rolegroup_config.logging.vector_container { + pb.add_container(vector_container( + &ContainerName::from_str(&Container::Vector.to_string()) + .expect("'vector' is a valid container name"), + resolved_product_image, + vector_log_config, + &resource_names, + &VolumeName::from_str(DRUID_CONFIG_VOLUME_NAME).expect("a valid volume name"), + &VolumeName::from_str(LOG_VOLUME_NAME).expect("a valid volume name"), + EnvVarSet::new(), + )); + } + + let mut pod_template = pb.build_template(); + // The role and rolegroup pod overrides were already merged (rolegroup wins) during validation. + pod_template.merge_from(rg.pod_overrides.clone()); + + Ok(StatefulSet { + metadata: ObjectMetaBuilder::new() + .name_and_namespace(cluster) + .name(resource_names.stateful_set_name().to_string()) + .ownerreference(ownerreference_from_resource(cluster, None, Some(true))) + .with_labels(recommended_labels( + cluster, + &product_name(), + &ProductVersion::from_str(&resolved_product_image.app_version_label_value) + .expect("a valid product version"), + &operator_name(), + &controller_name(), + &role.to_role_name(), + role_group_name, + )) + .with_label(RESTART_CONTROLLER_ENABLED_LABEL.to_owned()) + .build(), + spec: Some(StatefulSetSpec { + pod_management_policy: Some("Parallel".to_string()), + replicas: Some(i32::from(rg.replicas)), + selector: LabelSelector { + match_labels: Some( + role_group_selector( + cluster, + &product_name(), + &role.to_role_name(), + role_group_name, + ) + .into(), + ), + ..LabelSelector::default() + }, + service_name: Some(resource_names.headless_service_name().to_string()), + template: pod_template, + volume_claim_templates: pvcs, + ..StatefulSetSpec::default() + }), + status: None, + }) +} + +fn add_hdfs_cm_volume_and_volume_mounts( + deep_storage_spec: &DeepStorageSpec, + cb_druid: &mut ContainerBuilder, + pb: &mut PodBuilder, +) -> Result<()> { + // hdfs deep storage mount + if let DeepStorageSpec::Hdfs(hdfs) = deep_storage_spec { + cb_druid + .add_volume_mount(HDFS_CONFIG_VOLUME_NAME, HDFS_CONFIG_DIRECTORY) + .context(AddVolumeMountSnafu)?; + pb.add_volume( + VolumeBuilder::new(HDFS_CONFIG_VOLUME_NAME) + .with_config_map(hdfs.config_map_name.to_string()) + .build(), + ) + .context(AddVolumeSnafu)?; + } + + Ok(()) +} + +fn add_config_volume_and_volume_mounts( + resource_names: &ResourceNames, + cb_druid: &mut ContainerBuilder, + pb: &mut PodBuilder, +) -> Result<()> { + cb_druid + .add_volume_mount(DRUID_CONFIG_VOLUME_NAME, DRUID_CONFIG_DIRECTORY) + .context(AddVolumeMountSnafu)?; + pb.add_volume( + VolumeBuilder::new(DRUID_CONFIG_VOLUME_NAME) + .with_config_map(resource_names.role_group_config_map().to_string()) + .build(), + ) + .context(AddVolumeSnafu)?; + cb_druid + .add_volume_mount(RW_CONFIG_VOLUME_NAME, RW_CONFIG_DIRECTORY) + .context(AddVolumeMountSnafu)?; + pb.add_volume( + VolumeBuilder::new(RW_CONFIG_VOLUME_NAME) + .with_empty_dir(Some(""), None) + .build(), + ) + .context(AddVolumeSnafu)?; + + Ok(()) +} + +fn add_log_config_volume_and_volume_mounts( + resource_names: &ResourceNames, + merged_rolegroup_config: &ValidatedDruidConfig, + cb_druid: &mut ContainerBuilder, + pb: &mut PodBuilder, +) -> Result<()> { + cb_druid + .add_volume_mount(LOG_CONFIG_VOLUME_NAME, LOG_CONFIG_DIRECTORY) + .context(AddVolumeMountSnafu)?; + + let config_map = match &merged_rolegroup_config.logging.druid_container { + ValidatedContainerLogConfigChoice::Custom(config_map_name) => config_map_name.to_string(), + ValidatedContainerLogConfigChoice::Automatic(_) => { + resource_names.role_group_config_map().to_string() + } + }; + + pb.add_volume( + VolumeBuilder::new(LOG_CONFIG_VOLUME_NAME) + .with_config_map(config_map) + .build(), + ) + .context(AddVolumeSnafu)?; + + Ok(()) +} + +fn add_log_volume_and_volume_mounts( + cb_druid: &mut ContainerBuilder, + cb_prepare: &mut ContainerBuilder, + pb: &mut PodBuilder, +) -> Result<()> { + cb_druid + .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) + .context(AddVolumeMountSnafu)?; + cb_prepare + .add_volume_mount(LOG_VOLUME_NAME, STACKABLE_LOG_DIR) + .context(AddVolumeMountSnafu)?; + pb.add_volume( + VolumeBuilder::new(LOG_VOLUME_NAME) + .with_empty_dir( + Some(""), + Some(product_logging::framework::calculate_log_volume_size_limit( + &[MAX_DRUID_LOG_FILES_SIZE], + )), + ) + .build(), + ) + .context(AddVolumeSnafu)?; + + Ok(()) +} diff --git a/rust/operator-binary/src/controller/dereference.rs b/rust/operator-binary/src/controller/dereference.rs index bd4098a8..9406f311 100644 --- a/rust/operator-binary/src/controller/dereference.rs +++ b/rust/operator-binary/src/controller/dereference.rs @@ -84,7 +84,11 @@ pub async fn dereference( .as_deref() .context(ObjectHasNoNamespaceSnafu)?; - let zk_confmap = druid.spec.cluster_config.zookeeper_config_map_name.clone(); + let zk_confmap = druid + .spec + .cluster_config + .zookeeper_config_map_name + .to_string(); let zookeeper_connection_string = client .get::(&zk_confmap, namespace) .await diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 2d55d0e6..fdbadced 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -1,24 +1,35 @@ //! The validate step in the DruidCluster controller //! //! Synchronously validates inputs that don't require a Kubernetes client. Produces -//! [`ValidatedInputs`], consumed by the rest of `reconcile_druid`. +//! [`ValidatedCluster`], consumed by the rest of `reconcile_druid`. + +use std::{borrow::Cow, collections::BTreeMap, str::FromStr}; -use product_config::ProductConfigManager; use snafu::{ResultExt, Snafu}; use stackable_operator::{ cli::OperatorEnvironmentOptions, commons::product_image_selection::{self, ResolvedProductImage}, crd::s3, - product_config_utils::{ - ValidatedRoleConfigByPropertyKind, transform_all_roles_to_config, - validate_all_roles_and_groups_config, + database_connections::drivers::jdbc::{JdbcDatabaseConnection, JdbcDatabaseConnectionDetails}, + k8s_openapi::api::core::v1::Volume, + kube::{Resource, api::ObjectMeta}, + v2::{ + HasName, HasUid, NameIsValidLabelValue, + controller_utils::{get_cluster_name, get_namespace, get_uid}, + role_group_utils::ResourceNames, + types::{ + kubernetes::{NamespaceName, Uid}, + operator::{ClusterName, RoleGroupName, RoleName}, + }, }, }; +use strum::IntoEnumIterator; use crate::{ authentication::DruidAuthenticationConfig, controller::dereference::DereferencedObjects, - crd::{security::DruidTlsSecurity, v1alpha1}, + crd::{DeepStorageSpec, DruidRole, security::DruidTlsSecurity, v1alpha1}, + extensions::get_extension_list, }; #[derive(Snafu, Debug)] @@ -34,29 +45,162 @@ pub enum Error { source: crate::authentication::Error, }, - #[snafu(display("failed to transform configs"))] - ProductConfigTransform { - source: stackable_operator::product_config_utils::Error, + #[snafu(display("failed to resolve and merge config for role and role group"))] + FailedToResolveConfig { source: crate::crd::Error }, + + #[snafu(display("failed to determine the cluster's name, namespace, or uid"))] + ClusterIdentity { + source: stackable_operator::v2::controller_utils::Error, }, - #[snafu(display("invalid product configuration"))] - InvalidProductConfig { - source: stackable_operator::product_config_utils::Error, + #[snafu(display("invalid metadata database connection"))] + InvalidMetadataDatabaseConnection { + source: stackable_operator::database_connections::Error, }, } type Result = std::result::Result; -/// Synchronous inputs the rest of `reconcile_druid` needs after dereferencing. -pub struct ValidatedInputs { +/// A validated, merged role-group config. +/// +/// This is the upstream [`stackable_operator::v2::role_utils::RoleGroupConfig`] (config plus the +/// four merged override categories), with the typed per-role config erased to +/// [`ValidatedDruidConfig`] so that all roles share a single type. The rendered per-file configs +/// (runtime.properties / security.properties / jvm.config) are produced later, in the config-map +/// build step. +/// +/// Defined in [`crate::crd`] (where it has access to the private typed config fields) and +/// re-exported here for the build step. +pub use crate::crd::DruidRoleGroupConfig; + +/// Cluster-wide resolved fields that are not role/rolegroup specific. +pub struct ValidatedClusterConfig { pub zookeeper_connection_string: String, pub opa_connection_string: Option, pub s3_connection: Option, pub deep_storage_bucket_name: Option, - pub resolved_product_image: ResolvedProductImage, pub druid_tls_security: DruidTlsSecurity, pub druid_auth_config: Option, - pub validated_role_config: ValidatedRoleConfigByPropertyKind, + /// The `druid.extensions.loadList` entries, resolved from the metadata database, TLS, S3 and + /// authentication settings during validation. + pub extensions: Vec, + /// The `druid.metadata.storage.type` value derived from the configured metadata database. + pub metadata_storage_type: String, + /// The JDBC connection details (URL plus credential env vars) for the metadata database. + pub metadata_db_connection: JdbcDatabaseConnectionDetails, + /// The deep-storage spec, carried so the build step can derive the cluster-level + /// `runtime.properties` (deep storage type / directory / base key) without the raw + /// `DruidCluster`. + pub deep_storage: DeepStorageSpec, + /// User-supplied extra volumes, mounted into every container, carried so the build step does + /// not read the raw `DruidCluster`. + pub extra_volumes: Vec, +} + +/// Synchronous inputs the rest of `reconcile_druid` needs after dereferencing. +pub struct ValidatedCluster { + /// Mirrors `name`/`namespace`/`uid` below so that `ValidatedCluster` can implement + /// [`Resource`] and be passed directly to the metadata/owner-reference builders. + metadata: ObjectMeta, + pub name: ClusterName, + pub namespace: NamespaceName, + pub uid: Uid, + pub image: ResolvedProductImage, + pub cluster_config: ValidatedClusterConfig, + pub role_group_configs: BTreeMap>, +} + +impl ValidatedCluster { + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + name: ClusterName, + namespace: NamespaceName, + uid: Uid, + image: ResolvedProductImage, + cluster_config: ValidatedClusterConfig, + role_group_configs: BTreeMap>, + ) -> Self { + let metadata = ObjectMeta { + name: Some(name.to_string()), + namespace: Some(namespace.to_string()), + uid: Some(uid.to_string()), + ..ObjectMeta::default() + }; + Self { + metadata, + name, + namespace, + uid, + image, + cluster_config, + role_group_configs, + } + } + + /// Type-safe names for the resources of the given role's role group. + pub(crate) fn resource_names( + &self, + role: &DruidRole, + role_group_name: &RoleGroupName, + ) -> ResourceNames { + ResourceNames { + cluster_name: self.name.clone(), + role_name: RoleName::from_str(&role.to_string()) + .expect("a DruidRole is a valid role name"), + role_group_name: role_group_name.clone(), + } + } +} + +// Implementing `Resource` (plus `HasName`/`HasUid`) lets `ValidatedCluster` stand in for the raw +// `DruidCluster` when building child-object metadata and owner references. The identity-bearing +// methods are backed by the `metadata` built in `new`, while kind/group/version/plural delegate to +// the CRD so produced owner references are byte-identical to the previous raw-cluster ones. +impl Resource for ValidatedCluster { + type DynamicType = ::DynamicType; + type Scope = ::Scope; + + fn kind(dt: &Self::DynamicType) -> Cow<'_, str> { + v1alpha1::DruidCluster::kind(dt) + } + + fn group(dt: &Self::DynamicType) -> Cow<'_, str> { + v1alpha1::DruidCluster::group(dt) + } + + fn version(dt: &Self::DynamicType) -> Cow<'_, str> { + v1alpha1::DruidCluster::version(dt) + } + + fn plural(dt: &Self::DynamicType) -> Cow<'_, str> { + v1alpha1::DruidCluster::plural(dt) + } + + fn meta(&self) -> &ObjectMeta { + &self.metadata + } + + fn meta_mut(&mut self) -> &mut ObjectMeta { + &mut self.metadata + } +} + +impl HasName for ValidatedCluster { + fn to_name(&self) -> String { + self.name.to_string() + } +} + +impl HasUid for ValidatedCluster { + fn to_uid(&self) -> Uid { + self.uid.clone() + } +} + +impl NameIsValidLabelValue for ValidatedCluster { + fn to_label_value(&self) -> String { + self.name.to_label_value() + } } /// Validates the cluster spec and the dereferenced inputs. @@ -64,9 +208,8 @@ pub fn validate( druid: &v1alpha1::DruidCluster, dereferenced_objects: &DereferencedObjects, operator_environment: &OperatorEnvironmentOptions, - product_config: &ProductConfigManager, -) -> Result { - let resolved_product_image = druid +) -> Result { + let image = druid .spec .image .resolve( @@ -86,25 +229,194 @@ pub fn validate( ) .context(InvalidDruidAuthenticationConfigSnafu)?; - let role_config = transform_all_roles_to_config(druid, &druid.build_role_properties()) - .context(ProductConfigTransformSnafu)?; - let validated_role_config = validate_all_roles_and_groups_config( - &resolved_product_image.product_version, - &role_config, - product_config, - false, - false, - ) - .context(InvalidProductConfigSnafu)?; - - Ok(ValidatedInputs { - zookeeper_connection_string: dereferenced_objects.zookeeper_connection_string.clone(), - opa_connection_string: dereferenced_objects.opa_connection_string.clone(), - s3_connection: dereferenced_objects.s3_connection.clone(), - deep_storage_bucket_name: dereferenced_objects.deep_storage_bucket_name.clone(), - resolved_product_image, - druid_tls_security, - druid_auth_config, - validated_role_config, - }) + let mut role_group_configs: BTreeMap> = + BTreeMap::new(); + + for druid_role in DruidRole::iter() { + let group_map = druid + .merged_role(&druid_role) + .context(FailedToResolveConfigSnafu)?; + role_group_configs.insert(druid_role.clone(), group_map); + } + + let name = get_cluster_name(druid).context(ClusterIdentitySnafu)?; + let namespace = get_namespace(druid).context(ClusterIdentitySnafu)?; + let uid = get_uid(druid).context(ClusterIdentitySnafu)?; + + let extensions = get_extension_list(druid, &druid_tls_security, &druid_auth_config); + let metadata_storage_type = druid + .spec + .cluster_config + .metadata_database + .as_metadata_storage_type() + .to_string(); + let metadata_db_connection = druid + .spec + .cluster_config + .metadata_database + .jdbc_connection_details("metadata") + .context(InvalidMetadataDatabaseConnectionSnafu)?; + + Ok(ValidatedCluster::new( + name, + namespace, + uid, + image, + ValidatedClusterConfig { + zookeeper_connection_string: dereferenced_objects.zookeeper_connection_string.clone(), + opa_connection_string: dereferenced_objects.opa_connection_string.clone(), + s3_connection: dereferenced_objects.s3_connection.clone(), + deep_storage_bucket_name: dereferenced_objects.deep_storage_bucket_name.clone(), + druid_tls_security, + druid_auth_config, + extensions, + metadata_storage_type, + metadata_db_connection, + deep_storage: druid.spec.cluster_config.deep_storage.clone(), + extra_volumes: druid.spec.cluster_config.extra_volumes.clone(), + }, + role_group_configs, + )) +} + +#[cfg(test)] +pub(crate) mod test_support { + use std::{collections::BTreeMap, str::FromStr}; + + use stackable_operator::{ + database_connections::drivers::jdbc::JdbcDatabaseConnection, + kube::ResourceExt, + v2::types::{ + kubernetes::{NamespaceName, Uid}, + operator::ClusterName, + }, + }; + use strum::IntoEnumIterator; + + use super::{RoleGroupName, ValidatedCluster, ValidatedClusterConfig}; + use crate::{ + controller::CONTAINER_IMAGE_BASE_NAME, + crd::{ + DruidRole, authentication::AuthenticationClassesResolved, security::DruidTlsSecurity, + v1alpha1, + }, + extensions::get_extension_list, + }; + + /// A minimal but fully-valid `DruidCluster` with one role group per role. + pub const MINIMAL_DRUID_YAML: &str = r#" +apiVersion: druid.stackable.tech/v1alpha1 +kind: DruidCluster +metadata: + name: simple-druid + namespace: default + uid: c27b3971-ca72-42c1-80a4-abdfc1db0ddd +spec: + image: + productVersion: 30.0.0 + clusterConfig: + deepStorage: + hdfs: + configMapName: simple-hdfs + directory: /druid + metadataDatabase: + postgresql: + host: druid-postgresql + database: druid + credentialsSecretName: mySecret + zookeeperConfigMapName: simple-druid-znode + brokers: + roleGroups: + default: + replicas: 1 + coordinators: + roleGroups: + default: + replicas: 1 + historicals: + roleGroups: + default: + replicas: 1 + middleManagers: + roleGroups: + default: + replicas: 1 + routers: + roleGroups: + default: + replicas: 1 +"#; + + /// Parses a `DruidCluster` from YAML the same way the operator does (singleton-map enums). + pub fn druid_from_yaml(yaml: &str) -> v1alpha1::DruidCluster { + let deserializer = serde_yaml::Deserializer::from_str(yaml); + serde_yaml::with::singleton_map_recursive::deserialize(deserializer) + .expect("invalid test DruidCluster YAML") + } + + /// Builds a [`ValidatedCluster`] from a `DruidCluster` using test defaults for the + /// dereferenced/cluster-wide fields (no S3, no auth, fixed zookeeper string). Runs the real + /// `merged_role` pipeline for every role. + pub fn validated_cluster(druid: &v1alpha1::DruidCluster) -> ValidatedCluster { + let image = druid + .spec + .image + .resolve( + CONTAINER_IMAGE_BASE_NAME, + "oci.example.org", + crate::built_info::PKG_VERSION, + ) + .expect("test: resolvable product image"); + + let druid_tls_security = DruidTlsSecurity::new( + &AuthenticationClassesResolved { + auth_classes: vec![], + }, + Some("tls".to_string()), + ); + + let mut role_group_configs: BTreeMap> = + BTreeMap::new(); + for role in DruidRole::iter() { + role_group_configs.insert( + role.clone(), + druid.merged_role(&role).expect("test: merged role"), + ); + } + + let extensions = get_extension_list(druid, &druid_tls_security, &None); + let metadata_storage_type = druid + .spec + .cluster_config + .metadata_database + .as_metadata_storage_type() + .to_string(); + let metadata_db_connection = druid + .spec + .cluster_config + .metadata_database + .jdbc_connection_details("metadata") + .expect("test: valid metadata db connection"); + + ValidatedCluster::new( + ClusterName::from_str(&druid.name_any()).expect("test: valid cluster name"), + NamespaceName::from_str("default").expect("test: valid namespace"), + Uid::from_str("c27b3971-ca72-42c1-80a4-abdfc1db0ddd").expect("test: valid uid"), + image, + ValidatedClusterConfig { + zookeeper_connection_string: "zookeeper-connection-string".to_string(), + opa_connection_string: None, + s3_connection: None, + deep_storage_bucket_name: None, + druid_tls_security, + druid_auth_config: None, + extensions, + metadata_storage_type, + metadata_db_connection, + deep_storage: druid.spec.cluster_config.deep_storage.clone(), + extra_volumes: druid.spec.cluster_config.extra_volumes.clone(), + }, + role_group_configs, + ) + } } diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 23b61397..52bdfdcc 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -39,7 +39,7 @@ pub fn get_affinity( { vec![affinity_between_role_pods( "hdfs", - hdfs_discovery_cm_name, // The discovery cm has the same name as the HdfsCluster itself + hdfs_discovery_cm_name.as_ref(), // The discovery cm has the same name as the HdfsCluster itself "datanode", 50, )] @@ -76,7 +76,7 @@ pub fn get_affinity( #[cfg(test)] mod tests { - use std::collections::BTreeMap; + use std::{collections::BTreeMap, str::FromStr}; use rstest::rstest; use stackable_operator::{ @@ -87,6 +87,7 @@ mod tests { }, apimachinery::pkg::apis::meta::v1::LabelSelector, }, + v2::types::operator::RoleGroupName, }; use super::*; @@ -142,10 +143,9 @@ mod tests { let deserializer = serde_yaml::Deserializer::from_str(input); let druid: v1alpha1::DruidCluster = serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); - let merged_config = druid - .merged_config() - .unwrap() - .common_config(&role, "default") + let merged_role = druid.merged_role(&role).unwrap(); + let merged_config = merged_role + .get(&RoleGroupName::from_str("default").unwrap()) .unwrap(); let mut expected_affinities = vec![]; @@ -257,7 +257,7 @@ mod tests { }; assert_eq!( - merged_config.affinity, + merged_config.config.affinity, StackableAffinity { pod_affinity: Some(PodAffinity { preferred_during_scheduling_ignored_during_execution: Some(expected_affinities), diff --git a/rust/operator-binary/src/crd/authentication.rs b/rust/operator-binary/src/crd/authentication.rs index 081f0619..3a166371 100644 --- a/rust/operator-binary/src/crd/authentication.rs +++ b/rust/operator-binary/src/crd/authentication.rs @@ -13,7 +13,7 @@ use crate::crd::v1alpha1::DruidClusterConfig; type Result = std::result::Result; // The assumed OIDC provider if no hint is given in the AuthClass -pub const DEFAULT_OIDC_PROVIDER: oidc::v1alpha1::IdentityProviderHint = +const DEFAULT_OIDC_PROVIDER: oidc::v1alpha1::IdentityProviderHint = oidc::v1alpha1::IdentityProviderHint::Keycloak; const SUPPORTED_OIDC_PROVIDERS: &[oidc::v1alpha1::IdentityProviderHint] = @@ -101,7 +101,7 @@ impl AuthenticationClassesResolved { } /// Retrieves all provided `AuthenticationClass` references and checks if the configuration (TLS settings, secret class, OIDC config, etc.) is valid. - pub async fn resolve( + async fn resolve( cluster_config: &DruidClusterConfig, resolve_auth_class: impl Fn( core::v1alpha1::ClientAuthenticationDetails< @@ -128,10 +128,11 @@ impl AuthenticationClassesResolved { .context(AuthenticationClassRetrievalFailedSnafu)?; let auth_class_name = auth_class.name_any(); - let server_and_internal_secret_class = cluster_config - .tls - .as_ref() - .and_then(|tls| tls.server_and_internal_secret_class.to_owned()); + let server_and_internal_secret_class = cluster_config.tls.as_ref().and_then(|tls| { + tls.server_and_internal_secret_class + .as_ref() + .map(|secret_class| secret_class.to_string()) + }); match &auth_class.spec.provider { core::v1alpha1::AuthenticationClassProvider::Tls(provider) => { diff --git a/rust/operator-binary/src/crd/database.rs b/rust/operator-binary/src/crd/database.rs index 91116627..f24fc92c 100644 --- a/rust/operator-binary/src/crd/database.rs +++ b/rust/operator-binary/src/crd/database.rs @@ -11,11 +11,11 @@ use stackable_operator::{ }; // metadata storage config properties -pub const METADATA_STORAGE_TYPE: &str = "druid.metadata.storage.type"; -pub const METADATA_STORAGE_CONNECTOR_CONNECT_URI: &str = +pub(crate) const METADATA_STORAGE_TYPE: &str = "druid.metadata.storage.type"; +pub(crate) const METADATA_STORAGE_CONNECTOR_CONNECT_URI: &str = "druid.metadata.storage.connector.connectURI"; -pub const METADATA_STORAGE_USER: &str = "druid.metadata.storage.connector.user"; -pub const METADATA_STORAGE_PASSWORD: &str = "druid.metadata.storage.connector.password"; +pub(crate) const METADATA_STORAGE_USER: &str = "druid.metadata.storage.connector.user"; +pub(crate) const METADATA_STORAGE_PASSWORD: &str = "druid.metadata.storage.connector.password"; #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] diff --git a/rust/operator-binary/src/crd/memory.rs b/rust/operator-binary/src/crd/memory.rs index 218c4261..88c21b5c 100644 --- a/rust/operator-binary/src/crd/memory.rs +++ b/rust/operator-binary/src/crd/memory.rs @@ -7,10 +7,12 @@ use stackable_operator::{ memory::{BinaryMultiple, MemoryQuantity}, }; -use crate::crd::{ - PROCESSING_BUFFER_SIZE_BYTES, PROCESSING_NUM_MERGE_BUFFERS, PROCESSING_NUM_THREADS, - storage::HistoricalStorage, -}; +use crate::crd::storage::HistoricalStorage; + +// Druid historical processing config-property keys, only used for the memory calculations here. +const PROCESSING_BUFFER_SIZE_BYTES: &str = "druid.processing.buffer.sizeBytes"; +const PROCESSING_NUM_MERGE_BUFFERS: &str = "druid.processing.numMergeBuffers"; +const PROCESSING_NUM_THREADS: &str = "druid.processing.numThreads"; static MIN_HEAP_RATIO: f32 = 0.75; @@ -19,7 +21,7 @@ pub static RESERVED_OS_MEMORY: LazyLock = /// Max size for direct access buffers. This is defined in Druid to be 2GB: /// -pub static MAX_DIRECT_BUFFER_SIZE: LazyLock = +static MAX_DIRECT_BUFFER_SIZE: LazyLock = LazyLock::new(|| MemoryQuantity::from_gibi(2.)); #[derive(Snafu, Debug)] @@ -53,7 +55,7 @@ pub struct HistoricalDerivedSettings { } impl HistoricalDerivedSettings { - pub fn new(total_memory: MemoryQuantity, cpu_millis: CpuQuantity) -> Self { + fn new(total_memory: MemoryQuantity, cpu_millis: CpuQuantity) -> Self { Self { total_memory, cpu_millis, @@ -120,18 +122,18 @@ impl HistoricalDerivedSettings { } /// Adds derived runtime settings to the given config - pub fn add_settings(&self, config: &mut BTreeMap>) { + pub fn add_settings(&self, config: &mut BTreeMap) { config.insert( PROCESSING_NUM_THREADS.to_owned(), - Some(self.num_threads().to_string()), + self.num_threads().to_string(), ); config.insert( PROCESSING_NUM_MERGE_BUFFERS.to_owned(), - Some(self.num_merge_buffers().to_string()), + self.num_merge_buffers().to_string(), ); config.insert( PROCESSING_BUFFER_SIZE_BYTES.to_owned(), - Some(format_for_druid(&self.buffer_size())), + format_for_druid(&self.buffer_size()), ); } } diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 223ba5b8..dc5a05e7 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,12 +1,13 @@ -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::{ + collections::{BTreeMap, HashSet}, + str::FromStr, +}; use indoc::formatdoc; -use product_config::types::PropertyNameKind; use security::add_cert_to_jvm_trust_store_cmd; use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ - builder::pod::volume::ListenerOperatorVolumeSourceBuilderError, client::Client, commons::{ affinity::StackableAffinity, @@ -15,30 +16,39 @@ use stackable_operator::{ resources::{NoRuntimeLimits, Resources}, }, config::{ - fragment::{self, Fragment, FromFragment, ValidationError}, + fragment::{Fragment, ValidationError}, merge::Merge, }, - config_overrides::{KeyValueConfigOverrides, KeyValueOverridesProvider}, crd::{ authentication::{core, oidc}, s3, }, deep_merger::ObjectOverrides, - k8s_openapi::api::core::v1::{PodTemplateSpec, Volume}, + k8s_openapi::api::core::v1::Volume, kube::{CustomResource, ResourceExt}, - kvp::ObjectLabels, - memory::{BinaryMultiple, MemoryQuantity}, - product_config_utils::{Configuration, Error as ConfigError}, product_logging::{ self, framework::{create_vector_shutdown_file_command, remove_vector_shutdown_file_command}, spec::Logging, }, - role_utils::{CommonConfiguration, GenericRoleConfig, JavaCommonConfig, Role, RoleGroup}, + role_utils::{GenericRoleConfig, Role}, schemars::{self, JsonSchema}, shared::time::Duration, status::condition::{ClusterCondition, HasStatusCondition}, utils::{COMMON_BASH_TRAP_FUNCTIONS, crds::raw_object_list_schema}, + v2::{ + builder::pod::container::{EnvVarName, EnvVarSet}, + config_overrides::KeyValueConfigOverrides, + product_logging::framework::{ + ValidatedContainerLogConfigChoice, VectorContainerLogConfig, + validate_logging_configuration_for_container, + }, + role_utils::{JavaCommonConfig, RoleGroupConfig, with_validated_config}, + types::{ + kubernetes::{ConfigMapName, ListenerClassName}, + operator::{RoleGroupName, RoleName}, + }, + }, versioned::versioned, }; use strum::{Display, EnumDiscriminants, EnumIter, EnumString, IntoStaticStr}; @@ -63,7 +73,6 @@ pub mod tls; pub const APP_NAME: &str = "druid"; pub const OPERATOR_NAME: &str = "druid.stackable.tech"; -pub const FIELD_MANAGER: &str = "druid-operator"; // config directories pub const DRUID_CONFIG_DIRECTORY: &str = "/stackable/config"; @@ -71,64 +80,34 @@ pub const HDFS_CONFIG_DIRECTORY: &str = "/stackable/hdfs"; pub const LOG_CONFIG_DIRECTORY: &str = "/stackable/log_config"; pub const RW_CONFIG_DIRECTORY: &str = "/stackable/rwconfig"; -// config file names -pub const JVM_CONFIG: &str = "jvm.config"; -pub const RUNTIME_PROPS: &str = "runtime.properties"; -pub const LOG4J2_CONFIG: &str = "log4j2.properties"; -pub const JVM_SECURITY_PROPERTIES_FILE: &str = "security.properties"; - // store directories pub const STACKABLE_TRUST_STORE: &str = "/stackable/truststore.p12"; pub const STACKABLE_TRUST_STORE_PASSWORD: &str = "changeit"; +pub const STACKABLE_TRUST_STORE_TYPE: &str = "pkcs12"; pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; -// store file names -pub const DRUID_LOG_FILE: &str = "druid.log4j2.xml"; - pub const PROP_SEGMENT_CACHE_LOCATIONS: &str = "druid.segmentCache.locations"; -pub const PATH_SEGMENT_CACHE: &str = "/stackable/var/druid/segment-cache"; ///////////////////////////// // CONFIG PROPERTIES // ///////////////////////////// -// extensions -pub const EXTENSIONS_LOADLIST: &str = "druid.extensions.loadList"; -// zookeeper -pub const ZOOKEEPER_CONNECTION_STRING: &str = "druid.zk.service.host"; -// deep storage -pub const DS_TYPE: &str = "druid.storage.type"; -pub const DS_DIRECTORY: &str = "druid.storage.storageDirectory"; -// S3 -pub const DS_BUCKET: &str = "druid.storage.bucket"; -pub const DS_BASE_KEY: &str = "druid.storage.baseKey"; -pub const S3_ENDPOINT_URL: &str = "druid.s3.endpoint.url"; -pub const S3_ACCESS_KEY: &str = "druid.s3.accessKey"; -pub const S3_SECRET_KEY: &str = "druid.s3.secretKey"; -pub const S3_PATH_STYLE_ACCESS: &str = "druid.s3.enablePathStyleAccess"; -// OPA -pub const AUTH_AUTHORIZERS: &str = "druid.auth.authorizers"; -pub const AUTH_AUTHORIZERS_VALUE: &str = "[\"OpaAuthorizer\"]"; -pub const AUTH_AUTHORIZER_OPA_TYPE: &str = "druid.auth.authorizer.OpaAuthorizer.type"; -pub const AUTH_AUTHORIZER_OPA_TYPE_VALUE: &str = "opa"; -pub const AUTH_AUTHORIZER_OPA_URI: &str = "druid.auth.authorizer.OpaAuthorizer.opaUri"; -// indexer properties -pub const INDEXER_JAVA_OPTS: &str = "druid.indexer.runner.javaOptsArray"; -// historical settings -pub const PROCESSING_BUFFER_SIZE_BYTES: &str = "druid.processing.buffer.sizeBytes"; -pub const PROCESSING_NUM_MERGE_BUFFERS: &str = "druid.processing.numMergeBuffers"; -pub const PROCESSING_NUM_THREADS: &str = "druid.processing.numThreads"; -// logs -pub const MAX_DRUID_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { - value: 10.0, - unit: BinaryMultiple::Mebi, -}; -// metrics -pub const PROMETHEUS_PORT: &str = "druid.emitter.prometheus.port"; pub const METRICS_PORT_NAME: &str = "metrics"; pub const METRICS_PORT: u16 = 9090; pub const COOKIE_PASSPHRASE_ENV: &str = "OIDC_COOKIE_PASSPHRASE"; +/// Formats a Druid [dynamic config](https://druid.apache.org/docs/latest/operations/dynamic-config-provider) +/// reference to an environment variable, i.e. `${env:NAME}`. +pub(crate) fn env_var_reference(name: impl std::fmt::Display) -> String { + format!("${{env:{name}}}") +} + +/// Formats a Druid [dynamic config](https://druid.apache.org/docs/latest/operations/dynamic-config-provider) +/// reference to the UTF-8 contents of a file, i.e. `${file:UTF-8:PATH}`. +pub(crate) fn file_reference(path: impl std::fmt::Display) -> String { + format!("${{file:UTF-8:{path}}}") +} + // Graceful shutdown timeouts const DEFAULT_BROKER_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(5); const DEFAULT_COORDINATOR_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(5); @@ -145,55 +124,23 @@ const DEFAULT_ROUTER_SECRET_LIFETIME: Duration = Duration::from_days_unchecked(1 const DEFAULT_HISTORICAL_SECRET_LIFETIME: Duration = Duration::from_days_unchecked(1); /// Typed config override strategies for Druid config files. -#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, Merge, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct DruidConfigOverrides { /// Overrides for the `runtime.properties` file. - #[serde( - default, - rename = "runtime.properties", - skip_serializing_if = "Option::is_none" - )] - pub runtime_properties: Option, + // File name defined in [`crate::controller::build::properties::ConfigFileName`] + #[serde(default, rename = "runtime.properties")] + pub runtime_properties: KeyValueConfigOverrides, /// Overrides for the `jvm.config` file. - #[serde( - default, - rename = "jvm.config", - skip_serializing_if = "Option::is_none" - )] - pub jvm_config: Option, + // File name defined in [`crate::controller::build::properties::ConfigFileName`] + #[serde(default, rename = "jvm.config")] + pub jvm_config: KeyValueConfigOverrides, /// Overrides for the `security.properties` file. - #[serde( - default, - rename = "security.properties", - skip_serializing_if = "Option::is_none" - )] - pub security_properties: Option, -} - -impl KeyValueOverridesProvider for DruidConfigOverrides { - fn get_key_value_overrides(&self, file: &str) -> BTreeMap> { - match file { - RUNTIME_PROPS => self - .runtime_properties - .as_ref() - .map(KeyValueConfigOverrides::as_product_config_overrides) - .unwrap_or_default(), - JVM_CONFIG => self - .jvm_config - .as_ref() - .map(KeyValueConfigOverrides::as_product_config_overrides) - .unwrap_or_default(), - JVM_SECURITY_PROPERTIES_FILE => self - .security_properties - .as_ref() - .map(KeyValueConfigOverrides::as_product_config_overrides) - .unwrap_or_default(), - _ => BTreeMap::new(), - } - } + // File name defined in [`crate::controller::build::properties::ConfigFileName`] + #[serde(default, rename = "security.properties")] + pub security_properties: KeyValueConfigOverrides, } #[derive(Snafu, Debug, EnumDiscriminants)] @@ -225,20 +172,33 @@ pub enum Error { #[snafu(display("fragment validation failure"))] FragmentValidationFailure { source: ValidationError }, - #[snafu(display("failed to build Labels"))] - LabelBuild { - source: stackable_operator::kvp::LabelError, + #[snafu(display("failed to merge and validate config for role group {role_group:?}"))] + FailedToMergeRoleGroupConfig { + source: ValidationError, + role_group: String, }, - #[snafu(display("failed to build listener volume"))] - BuildListenerVolume { - source: ListenerOperatorVolumeSourceBuilderError, + #[snafu(display("invalid environment variable override name in role group {role_group:?}"))] + ParseEnvVarName { + source: stackable_operator::v2::builder::pod::container::Error, + role_group: String, }, - #[snafu(display("failed to apply group listener"))] - ApplyGroupListener { - source: stackable_operator::cluster_resources::Error, + #[snafu(display("invalid role group name {role_group:?}"))] + ParseRoleGroupName { + source: stackable_operator::v2::macros::attributed_string_type::Error, + role_group: String, }, + + #[snafu(display("failed to validate container log configuration"))] + ValidateLoggingConfig { + source: stackable_operator::v2::product_logging::framework::Error, + }, + + #[snafu(display( + "the Vector agent is enabled but the Vector aggregator ConfigMap name is missing" + ))] + MissingVectorAggregatorConfigMapName, } #[versioned( @@ -363,14 +323,14 @@ pub mod versioned { /// Provide the name of the ZooKeeper [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery) /// here. When using the [Stackable operator for Apache ZooKeeper](DOCS_BASE_URL_PLACEHOLDER/zookeeper/) /// to deploy a ZooKeeper cluster, this will simply be the name of your ZookeeperCluster resource. - pub zookeeper_config_map_name: String, + pub zookeeper_config_map_name: ConfigMapName, /// Name of the Vector aggregator [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery). /// It must contain the key `ADDRESS` with the address of the Vector aggregator. /// Follow the [logging tutorial](DOCS_BASE_URL_PLACEHOLDER/tutorials/logging-vector-aggregator) /// to learn how to configure log aggregation with Vector. #[serde(skip_serializing_if = "Option::is_none")] - pub vector_aggregator_config_map_name: Option, + pub vector_aggregator_config_map_name: Option, /// Extra volumes similar to `.spec.volumes` on a Pod to mount into every container, this can be useful to for /// example make client certificates, keytabs or similar things available to processors. These volumes will be @@ -386,7 +346,7 @@ pub mod versioned { pub generic: GenericRoleConfig, #[serde(default = "druid_default_listener_class")] - pub listener_class: String, + pub listener_class: ListenerClassName, } } @@ -401,119 +361,6 @@ impl HasStatusCondition for v1alpha1::DruidCluster { } impl v1alpha1::DruidCluster { - pub fn common_compute_files( - &self, - file: &str, - ) -> Result>, ConfigError> { - let mut result = BTreeMap::new(); - match file { - JVM_CONFIG => {} - RUNTIME_PROPS => { - // OPA - if let Some(DruidAuthorization { opa: _ }) = &self.spec.cluster_config.authorization - { - result.insert( - AUTH_AUTHORIZERS.to_string(), - Some(AUTH_AUTHORIZERS_VALUE.to_string()), - ); - result.insert( - AUTH_AUTHORIZER_OPA_TYPE.to_string(), - Some(AUTH_AUTHORIZER_OPA_TYPE_VALUE.to_string()), - ); - // The opaUri still needs to be set, but that requires a discovery config map and is handled in the controller.rs - } - // deep storage - result.insert( - DS_TYPE.to_string(), - Some(self.spec.cluster_config.deep_storage.to_string()), - ); - match self.spec.cluster_config.deep_storage.clone() { - DeepStorageSpec::Hdfs(hdfs) => { - result.insert(DS_DIRECTORY.to_string(), Some(hdfs.directory)); - } - DeepStorageSpec::S3(s3_spec) => { - if let Some(key) = &s3_spec.base_key { - result.insert(DS_BASE_KEY.to_string(), Some(key.to_string())); - } - // bucket information (name, connection) needs to be resolved first, - // that is done directly in the controller - } - } - - // metrics - result.insert(PROMETHEUS_PORT.to_string(), Some(METRICS_PORT.to_string())); - } - _ => {} - } - - Ok(result) - } - - #[allow(clippy::type_complexity)] - pub fn build_role_properties( - &self, - ) -> HashMap< - String, - ( - Vec, - Role< - impl Configuration + use<>, - DruidConfigOverrides, - GenericRoleConfig, - JavaCommonConfig, - >, - ), - > { - let config_files = vec![ - PropertyNameKind::Env, - PropertyNameKind::File(JVM_CONFIG.to_string()), - PropertyNameKind::File(RUNTIME_PROPS.to_string()), - PropertyNameKind::File(JVM_SECURITY_PROPERTIES_FILE.to_string()), - ]; - - vec![ - ( - DruidRole::Broker.to_string(), - ( - config_files.clone(), - extract_role_from_role_config::(self.spec.brokers.clone()) - .erase(), - ), - ), - ( - DruidRole::Coordinator.to_string(), - ( - config_files.clone(), - extract_role_from_role_config::( - self.spec.coordinators.clone(), - ) - .erase(), - ), - ), - ( - DruidRole::Historical.to_string(), - (config_files.clone(), self.spec.historicals.clone().erase()), - ), - ( - DruidRole::MiddleManager.to_string(), - ( - config_files.clone(), - self.spec.middle_managers.clone().erase(), - ), - ), - ( - DruidRole::Router.to_string(), - ( - config_files, - extract_role_from_role_config::(self.spec.routers.clone()) - .erase(), - ), - ), - ] - .into_iter() - .collect() - } - /// If an s3 connection for ingestion is given, as well as an s3 connection for deep storage, they need to be the same. /// This function returns the resolved connection, or raises an Error if the connections are not identical. pub async fn get_s3_connection( @@ -598,98 +445,143 @@ impl v1alpha1::DruidCluster { s3_ingestion || s3_storage } - /// Returns the merged and validated configuration for all roles - pub fn merged_config(&self) -> Result { + /// Merges and validates all role groups of the given role. Invoked from the validate step + /// (`controller::validate`). + /// + /// All four override categories (config / env / cli / pod) are merged by + /// [`with_validated_config`] (role group wins over role); the typed per-role config is then + /// erased to the shared [`ValidatedDruidConfig`] view consumed by the build step. + /// + /// This lives in `crate::crd` and is a `macro_rules!` rather than a generic function on purpose: the erasure + /// reads the per-role configs' private `resources` field and calls their private + /// `default_config`, and the five roles wrap *different* `resources` types into different + /// [`RoleResource`] variants. + pub fn merged_role( + &self, + role: &DruidRole, + ) -> Result, Error> { let deep_storage = &self.spec.cluster_config.deep_storage; + let name = self.name_any(); - Ok(MergedConfig { - brokers: v1alpha1::DruidCluster::merged_role( - &extract_role_from_role_config::(self.spec.brokers.clone()), - &BrokerConfig::default_config(&self.name_any(), &DruidRole::Broker, deep_storage), - )?, - coordinators: v1alpha1::DruidCluster::merged_role( - &extract_role_from_role_config::(self.spec.coordinators.clone()), - &CoordinatorConfig::default_config( - &self.name_any(), - &DruidRole::Coordinator, - deep_storage, - ), - )?, - historicals: v1alpha1::DruidCluster::merged_role( - &self.spec.historicals, - &HistoricalConfig::default_config( - &self.name_any(), - &DruidRole::Historical, - deep_storage, - ), - )?, - middle_managers: v1alpha1::DruidCluster::merged_role( - &self.spec.middle_managers, - &MiddleManagerConfig::default_config( - &self.name_any(), - &DruidRole::MiddleManager, - deep_storage, - ), - )?, - routers: v1alpha1::DruidCluster::merged_role( - &extract_role_from_role_config::(self.spec.routers.clone()), - &RouterConfig::default_config(&self.name_any(), &DruidRole::Router, deep_storage), - )?, - }) - } - - /// Merges and validates the role groups of the given role with the given default configuration - fn merged_role( - role: &Role, - default_config: &T::Fragment, - ) -> Result>, Error> - where - T: FromFragment, - T::Fragment: Clone + Merge, - { - let mut merged_role_config = HashMap::new(); - - for (rolegroup_name, rolegroup) in &role.role_groups { - let merged_rolegroup_config = v1alpha1::DruidCluster::merged_rolegroup( - rolegroup, - &role.config.config, - default_config, - )?; - merged_role_config.insert(rolegroup_name.to_owned(), merged_rolegroup_config); + // The Vector aggregator discovery ConfigMap name (a typed `ConfigMapName`, so an invalid + // name is already rejected at deserialization). It is only required when the Vector agent + // is enabled for a role group. + let vector_aggregator_config_map_name = self + .spec + .cluster_config + .vector_aggregator_config_map_name + .clone(); + + // All roles erase to an identical `ValidatedDruidConfig`; only the typed config, the role + // config type and the `RoleResource` variant (historicals carry typed storage) differ. + macro_rules! merged_role { + ($field:ident, $config:ty, $role_config:ty, $resource:path) => {{ + let typed_role = &self.spec.$field; + let default_config = <$config>::default_config(&name, role, deep_storage); + let mut groups = BTreeMap::new(); + for (rg_name, rg) in &typed_role.role_groups { + let validated = with_validated_config::< + $config, + JavaCommonConfig, + _, + $role_config, + DruidConfigOverrides, + >(rg, typed_role, &default_config) + .with_context(|_| FailedToMergeRoleGroupConfigSnafu { + role_group: rg_name.clone(), + })?; + let common = ValidatedDruidConfig { + resources: $resource(validated.config.config.resources), + logging: validate_logging( + &validated.config.config.logging, + &vector_aggregator_config_map_name, + )?, + affinity: validated.config.config.affinity, + graceful_shutdown_timeout: validated + .config + .config + .graceful_shutdown_timeout, + requested_secret_lifetime: validated + .config + .config + .requested_secret_lifetime + .context(MissingSecretLifetimeSnafu)?, + }; + // Upstream returns env overrides as a `HashMap`; the build step consumes an + // `EnvVarSet`. Convert here, validating each name. (Role/role-group precedence + // is already resolved by `with_validated_config`.) + let mut env_overrides = EnvVarSet::new(); + for (name, value) in validated.config.env_overrides { + env_overrides = env_overrides.with_value( + &EnvVarName::from_str(&name).with_context(|_| { + ParseEnvVarNameSnafu { + role_group: rg_name.clone(), + } + })?, + value, + ); + } + let role_group_name = RoleGroupName::from_str(rg_name).with_context(|_| { + ParseRoleGroupNameSnafu { + role_group: rg_name.clone(), + } + })?; + groups.insert( + role_group_name, + DruidRoleGroupConfig { + replicas: validated.replicas.unwrap_or(1), + config: common, + config_overrides: validated.config.config_overrides, + env_overrides, + cli_overrides: validated.config.cli_overrides, + pod_overrides: validated.config.pod_overrides, + product_specific_common_config: validated + .config + .product_specific_common_config, + }, + ); + } + groups + }}; } - Ok(merged_role_config) - } - - /// Merges and validates the given role group with the given role and default configurations - fn merged_rolegroup( - rolegroup: &RoleGroup, - role_config: &T::Fragment, - default_config: &T::Fragment, - ) -> Result, Error> - where - T: FromFragment, - T::Fragment: Clone + Merge, - { - let merged_config = v1alpha1::DruidCluster::merged_rolegroup_config( - &rolegroup.config.config, - role_config, - default_config, - )?; - Ok(RoleGroup { - config: CommonConfiguration { - config: merged_config, - config_overrides: rolegroup.config.config_overrides.to_owned(), - env_overrides: rolegroup.config.env_overrides.to_owned(), - cli_overrides: rolegroup.config.cli_overrides.to_owned(), - pod_overrides: rolegroup.config.pod_overrides.to_owned(), - product_specific_common_config: rolegroup - .config - .product_specific_common_config - .to_owned(), - }, - replicas: rolegroup.replicas, - }) + let groups = match role { + DruidRole::Broker => { + merged_role!( + brokers, + BrokerConfig, + v1alpha1::DruidRoleConfig, + RoleResource::Druid + ) + } + DruidRole::Coordinator => merged_role!( + coordinators, + CoordinatorConfig, + v1alpha1::DruidRoleConfig, + RoleResource::Druid + ), + DruidRole::Historical => merged_role!( + historicals, + HistoricalConfig, + GenericRoleConfig, + RoleResource::Historical + ), + DruidRole::MiddleManager => merged_role!( + middle_managers, + MiddleManagerConfig, + GenericRoleConfig, + RoleResource::Druid + ), + DruidRole::Router => { + merged_role!( + routers, + RouterConfig, + v1alpha1::DruidRoleConfig, + RoleResource::Druid + ) + } + }; + Ok(groups) } pub fn generic_role_config(&self, role: &DruidRole) -> &GenericRoleConfig { @@ -701,99 +593,6 @@ impl v1alpha1::DruidCluster { DruidRole::Router => &self.spec.routers.role_config.generic, } } - - /// Merges and validates the given role group, role, and default configurations - pub fn merged_rolegroup_config( - rolegroup_config: &T::Fragment, - role_config: &T::Fragment, - default_config: &T::Fragment, - ) -> Result - where - T: FromFragment, - T::Fragment: Clone + Merge, - { - let mut role_config = role_config.to_owned(); - let mut rolegroup_config = rolegroup_config.to_owned(); - - role_config.merge(default_config); - rolegroup_config.merge(&role_config); - - fragment::validate(rolegroup_config).context(FragmentValidationFailureSnafu) - } - - pub fn get_role( - &self, - druid_role: &DruidRole, - ) -> Role< - impl Configuration + use<>, - DruidConfigOverrides, - GenericRoleConfig, - JavaCommonConfig, - > { - match druid_role { - DruidRole::Broker => { - extract_role_from_role_config::(self.spec.brokers.clone()).erase() - } - DruidRole::Coordinator => { - extract_role_from_role_config::(self.spec.coordinators.clone()) - .erase() - } - DruidRole::Historical => self.spec.historicals.clone().erase(), - DruidRole::MiddleManager => self.spec.middle_managers.clone().erase(), - DruidRole::Router => { - extract_role_from_role_config::(self.spec.routers.clone()).erase() - } - } - } - - pub fn pod_overrides_for_role(&self, role: &DruidRole) -> &PodTemplateSpec { - match role { - DruidRole::Broker => &self.spec.brokers.config.pod_overrides, - DruidRole::Coordinator => &self.spec.coordinators.config.pod_overrides, - DruidRole::Historical => &self.spec.historicals.config.pod_overrides, - DruidRole::MiddleManager => &self.spec.middle_managers.config.pod_overrides, - DruidRole::Router => &self.spec.routers.config.pod_overrides, - } - } - - pub fn pod_overrides_for_role_group( - &self, - role: &DruidRole, - role_group: &str, - ) -> Option<&PodTemplateSpec> { - match role { - DruidRole::Broker => self - .spec - .brokers - .role_groups - .get(role_group) - .map(|rg| &rg.config.pod_overrides), - DruidRole::Coordinator => self - .spec - .coordinators - .role_groups - .get(role_group) - .map(|rg| &rg.config.pod_overrides), - DruidRole::Historical => self - .spec - .historicals - .role_groups - .get(role_group) - .map(|rg| &rg.config.pod_overrides), - DruidRole::MiddleManager => self - .spec - .middle_managers - .role_groups - .get(role_group) - .map(|rg| &rg.config.pod_overrides), - DruidRole::Router => self - .spec - .routers - .role_groups - .get(role_group) - .map(|rg| &rg.config.pod_overrides), - } - } } #[derive( @@ -817,141 +616,79 @@ pub enum Container { Vector, } -/// Common configuration for all role groups -pub struct CommonRoleGroupConfig { - pub resources: RoleResource, - pub logging: Logging, - pub replicas: Option, +/// The validated, merged Druid config shared by all role groups (the typed per-role config erased +/// to a single view). Mirrors the opensearch-operator's `ValidatedOpenSearchConfig`. +#[derive(Clone)] +pub struct ValidatedDruidConfig { pub affinity: StackableAffinity, pub graceful_shutdown_timeout: Option, + pub logging: ValidatedLogging, pub requested_secret_lifetime: Duration, + pub resources: RoleResource, } -/// Container for the merged and validated role group configurations +/// Validated logging configuration for the Druid, Prepare (init) and (optional) Vector containers. /// -/// This structure contains for every role a map from the role group names to their configurations. -/// The role group configurations are merged with the role and default configurations. The product -/// configuration is not applied. -pub struct MergedConfig { - /// Merged configuration of the broker role - pub brokers: HashMap>, - /// Merged configuration of the coordinator role - pub coordinators: - HashMap>, - /// Merged configuration of the historical role - pub historicals: - HashMap>, - /// Merged configuration of the middle manager role - pub middle_managers: - HashMap>, - /// Merged configuration of the router role - pub routers: HashMap>, +/// Produced up-front by [`validate_logging`] (mirroring the hive-/opensearch-operator) so that an +/// invalid custom log ConfigMap name or a missing Vector aggregator discovery ConfigMap name fails +/// reconciliation during validation rather than at resource-build time. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ValidatedLogging { + pub druid_container: ValidatedContainerLogConfigChoice, + pub prepare_container: ValidatedContainerLogConfigChoice, + pub vector_container: Option, + pub enable_vector_agent: bool, } -impl MergedConfig { - /// Returns the common configuration for the given role and rolegroup name - pub fn common_config( - &self, - role: &DruidRole, - rolegroup_name: &str, - ) -> Result { - match role { - DruidRole::Broker => { - let rolegroup = self - .brokers - .get(rolegroup_name) - .context(CannotRetrieveRoleGroupSnafu { rolegroup_name })?; - Ok(CommonRoleGroupConfig { - resources: RoleResource::Druid(rolegroup.config.config.resources.to_owned()), - logging: rolegroup.config.config.logging.to_owned(), - replicas: rolegroup.replicas, - affinity: rolegroup.config.config.affinity.clone(), - graceful_shutdown_timeout: rolegroup.config.config.graceful_shutdown_timeout, - requested_secret_lifetime: rolegroup - .config - .config - .requested_secret_lifetime - .context(MissingSecretLifetimeSnafu)?, - }) - } - DruidRole::Coordinator => { - let rolegroup = self - .coordinators - .get(rolegroup_name) - .context(CannotRetrieveRoleGroupSnafu { rolegroup_name })?; - Ok(CommonRoleGroupConfig { - resources: RoleResource::Druid(rolegroup.config.config.resources.to_owned()), - logging: rolegroup.config.config.logging.to_owned(), - replicas: rolegroup.replicas, - affinity: rolegroup.config.config.affinity.clone(), - graceful_shutdown_timeout: rolegroup.config.config.graceful_shutdown_timeout, - requested_secret_lifetime: rolegroup - .config - .config - .requested_secret_lifetime - .context(MissingSecretLifetimeSnafu)?, - }) - } - DruidRole::Historical => { - let rolegroup = self - .historicals - .get(rolegroup_name) - .context(CannotRetrieveRoleGroupSnafu { rolegroup_name })?; - Ok(CommonRoleGroupConfig { - resources: RoleResource::Historical( - rolegroup.config.config.resources.to_owned(), - ), - logging: rolegroup.config.config.logging.to_owned(), - replicas: rolegroup.replicas, - affinity: rolegroup.config.config.affinity.clone(), - graceful_shutdown_timeout: rolegroup.config.config.graceful_shutdown_timeout, - requested_secret_lifetime: rolegroup - .config - .config - .requested_secret_lifetime - .context(MissingSecretLifetimeSnafu)?, - }) - } - DruidRole::MiddleManager => { - let rolegroup = self - .middle_managers - .get(rolegroup_name) - .context(CannotRetrieveRoleGroupSnafu { rolegroup_name })?; - Ok(CommonRoleGroupConfig { - resources: RoleResource::Druid(rolegroup.config.config.resources.to_owned()), - logging: rolegroup.config.config.logging.to_owned(), - replicas: rolegroup.replicas, - affinity: rolegroup.config.config.affinity.clone(), - graceful_shutdown_timeout: rolegroup.config.config.graceful_shutdown_timeout, - requested_secret_lifetime: rolegroup - .config - .config - .requested_secret_lifetime - .context(MissingSecretLifetimeSnafu)?, - }) - } - DruidRole::Router => { - let rolegroup = self - .routers - .get(rolegroup_name) - .context(CannotRetrieveRoleGroupSnafu { rolegroup_name })?; - Ok(CommonRoleGroupConfig { - resources: RoleResource::Druid(rolegroup.config.config.resources.to_owned()), - logging: rolegroup.config.config.logging.to_owned(), - replicas: rolegroup.replicas, - affinity: rolegroup.config.config.affinity.clone(), - graceful_shutdown_timeout: rolegroup.config.config.graceful_shutdown_timeout, - requested_secret_lifetime: rolegroup - .config - .config - .requested_secret_lifetime - .context(MissingSecretLifetimeSnafu)?, - }) - } - } - } +/// Validates the logging configuration for the Druid, Prepare and (optional) Vector containers. +/// +/// `vector_aggregator_config_map_name` is the discovery ConfigMap name of the Vector aggregator; +/// it is required (and was validated up-front) only when the Vector agent is enabled. +fn validate_logging( + logging: &Logging, + vector_aggregator_config_map_name: &Option, +) -> Result { + let druid_container = validate_logging_configuration_for_container(logging, &Container::Druid) + .context(ValidateLoggingConfigSnafu)?; + let prepare_container = + validate_logging_configuration_for_container(logging, &Container::Prepare) + .context(ValidateLoggingConfigSnafu)?; + + let vector_container = if logging.enable_vector_agent { + let vector_aggregator_config_map_name = vector_aggregator_config_map_name + .clone() + .context(MissingVectorAggregatorConfigMapNameSnafu)?; + Some(VectorContainerLogConfig { + log_config: validate_logging_configuration_for_container(logging, &Container::Vector) + .context(ValidateLoggingConfigSnafu)?, + vector_aggregator_config_map_name, + }) + } else { + None + }; + + Ok(ValidatedLogging { + druid_container, + prepare_container, + vector_container, + enable_vector_agent: logging.enable_vector_agent, + }) } +/// A validated, merged role-group config. +/// +/// This is the upstream [`stackable_operator::v2::role_utils::RoleGroupConfig`] (config plus the +/// four merged override categories, role group winning over role), with the typed per-role config +/// erased to the shared [`ValidatedDruidConfig`] view. The merged JVM argument overrides live in +/// `product_specific_common_config`; the rendered per-file configs (runtime.properties / +/// security.properties / jvm.config) are produced later, in the config-map build step. +/// +/// The StatefulSet replicas come from [`RoleGroupConfig::replicas`] (defaulted to 1 during the +/// merge), mirroring the hive-operator. Upstream `replicas` should be made optional again so an +/// unspecified count can stay HPA-friendly. +pub type DruidRoleGroupConfig = + RoleGroupConfig; + impl Default for v1alpha1::DruidRoleConfig { fn default() -> Self { v1alpha1::DruidRoleConfig { @@ -961,8 +698,8 @@ impl Default for v1alpha1::DruidRoleConfig { } } -fn druid_default_listener_class() -> String { - "cluster-internal".to_string() +fn druid_default_listener_class() -> ListenerClassName { + ListenerClassName::from_str("cluster-internal").expect("a valid listener class name") } #[derive( @@ -974,7 +711,9 @@ fn druid_default_listener_class() -> String { Eq, Hash, JsonSchema, + Ord, PartialEq, + PartialOrd, Serialize, EnumString, )] @@ -992,6 +731,12 @@ pub enum DruidRole { } impl DruidRole { + /// Returns the typed role name used for Kubernetes labels and selectors. + pub fn to_role_name(&self) -> RoleName { + RoleName::from_str(&self.to_string()) + .expect("a DruidRole always serializes to a valid role name") + } + /// Returns the name of the internal druid process name associated with the role. /// These strings are used by druid internally to identify processes. fn get_process_name(&self) -> &str { @@ -1027,7 +772,7 @@ impl DruidRole { } /// Return the default graceful shutdown timeout - pub fn default_graceful_shutdown_timeout(&self) -> Duration { + fn default_graceful_shutdown_timeout(&self) -> Duration { match &self { DruidRole::Coordinator => DEFAULT_COORDINATOR_GRACEFUL_SHUTDOWN_TIMEOUT, DruidRole::Broker => DEFAULT_BROKER_GRACEFUL_SHUTDOWN_TIMEOUT, @@ -1099,7 +844,7 @@ impl DruidRole { } } - pub fn listener_class_name(&self, druid: &v1alpha1::DruidCluster) -> Option { + pub fn listener_class_name(&self, druid: &v1alpha1::DruidCluster) -> Option { match self { Self::Broker => Some(druid.spec.brokers.role_config.listener_class.to_owned()), Self::Coordinator => Some( @@ -1130,11 +875,7 @@ pub enum DeepStorageSpec { } impl DeepStorageSpec { - pub fn is_hdfs(&self) -> bool { - matches!(self, DeepStorageSpec::Hdfs(_)) - } - - pub fn is_s3(&self) -> bool { + fn is_s3(&self) -> bool { matches!(self, DeepStorageSpec::S3(_)) } } @@ -1145,7 +886,7 @@ pub struct HdfsDeepStorageSpec { /// The [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery) /// for the HDFS instance. When running an HDFS cluster with the Stackable operator, the operator /// will create this ConfigMap for you. It has the same name as your HDFSCluster resource. - pub config_map_name: String, + pub config_map_name: ConfigMapName, /// The directory inside of HDFS where Druid should store its data. pub directory: String, } @@ -1173,409 +914,102 @@ pub struct IngestionSpec { pub s3connection: Option, } -#[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] -#[fragment_attrs( - derive( - Clone, - Debug, - Default, - Deserialize, - Merge, - JsonSchema, - PartialEq, - Serialize - ), - serde(rename_all = "camelCase") -)] -pub struct BrokerConfig { - #[fragment_attrs(serde(default))] - resources: Resources, - #[fragment_attrs(serde(default))] - pub logging: Logging, - #[fragment_attrs(serde(default))] - pub affinity: StackableAffinity, - /// The time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. - /// Read more about graceful shutdown in the - /// [graceful shutdown documentation](DOCS_BASE_URL_PLACEHOLDER/druid/usage-guide/operations/graceful-shutdown). - #[fragment_attrs(serde(default))] - pub graceful_shutdown_timeout: Option, - - /// Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. - /// This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. - #[fragment_attrs(serde(default))] - pub requested_secret_lifetime: Option, -} - -impl BrokerConfig { - fn default_config( - cluster_name: &str, - role: &DruidRole, - deep_storage: &DeepStorageSpec, - ) -> BrokerConfigFragment { - BrokerConfigFragment { - resources: resource::BROKER_RESOURCES.to_owned(), - logging: product_logging::spec::default_logging(), - affinity: get_affinity(cluster_name, role, deep_storage), - graceful_shutdown_timeout: Some(role.default_graceful_shutdown_timeout()), - requested_secret_lifetime: Some(DEFAULT_BROKER_SECRET_LIFETIME), - } - } -} - -#[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] -#[fragment_attrs( - derive( - Clone, - Debug, - Default, - Deserialize, - Merge, - JsonSchema, - PartialEq, - Serialize - ), - serde(rename_all = "camelCase") -)] -pub struct CoordinatorConfig { - #[fragment_attrs(serde(default))] - resources: Resources, - #[fragment_attrs(serde(default))] - pub logging: Logging, - #[fragment_attrs(serde(default))] - pub affinity: StackableAffinity, - /// The time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. - /// Read more about graceful shutdown in the - /// [graceful shutdown documentation](DOCS_BASE_URL_PLACEHOLDER/druid/usage-guide/operations/graceful-shutdown). - #[fragment_attrs(serde(default))] - pub graceful_shutdown_timeout: Option, - - /// Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. - /// This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. - #[fragment_attrs(serde(default))] - pub requested_secret_lifetime: Option, -} - -impl CoordinatorConfig { - fn default_config( - cluster_name: &str, - role: &DruidRole, - deep_storage: &DeepStorageSpec, - ) -> CoordinatorConfigFragment { - CoordinatorConfigFragment { - resources: resource::COORDINATOR_RESOURCES.to_owned(), - logging: product_logging::spec::default_logging(), - affinity: get_affinity(cluster_name, role, deep_storage), - graceful_shutdown_timeout: Some(role.default_graceful_shutdown_timeout()), - requested_secret_lifetime: Some(DEFAULT_COORDINATOR_SECRET_LIFETIME), - } - } -} - -#[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] -#[fragment_attrs( - derive( - Clone, - Debug, - Default, - Deserialize, - Merge, - JsonSchema, - PartialEq, - Serialize - ), - serde(rename_all = "camelCase") -)] -pub struct MiddleManagerConfig { - #[fragment_attrs(serde(default))] - resources: Resources, - #[fragment_attrs(serde(default))] - pub logging: Logging, - #[fragment_attrs(serde(default))] - pub affinity: StackableAffinity, - /// The time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. - /// Read more about graceful shutdown in the - /// [graceful shutdown documentation](DOCS_BASE_URL_PLACEHOLDER/druid/usage-guide/operations/graceful-shutdown). - #[fragment_attrs(serde(default))] - pub graceful_shutdown_timeout: Option, - - /// Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. - /// This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. - #[fragment_attrs(serde(default))] - pub requested_secret_lifetime: Option, -} - -impl MiddleManagerConfig { - fn default_config( - cluster_name: &str, - role: &DruidRole, - deep_storage: &DeepStorageSpec, - ) -> MiddleManagerConfigFragment { - MiddleManagerConfigFragment { - resources: resource::MIDDLE_MANAGER_RESOURCES.to_owned(), - logging: product_logging::spec::default_logging(), - affinity: get_affinity(cluster_name, role, deep_storage), - graceful_shutdown_timeout: Some(role.default_graceful_shutdown_timeout()), - requested_secret_lifetime: Some(DEFAULT_MIDDLE_SECRET_LIFETIME), - } - } -} - -#[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] -#[fragment_attrs( - derive( - Clone, - Debug, - Default, - Deserialize, - Merge, - JsonSchema, - PartialEq, - Serialize - ), - serde(rename_all = "camelCase") -)] -pub struct RouterConfig { - #[fragment_attrs(serde(default))] - resources: Resources, - #[fragment_attrs(serde(default))] - pub logging: Logging, - #[fragment_attrs(serde(default))] - pub affinity: StackableAffinity, - /// The time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. - /// Read more about graceful shutdown in the - /// [graceful shutdown documentation](DOCS_BASE_URL_PLACEHOLDER/druid/usage-guide/operations/graceful-shutdown). - #[fragment_attrs(serde(default))] - pub graceful_shutdown_timeout: Option, - - /// Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. - /// This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. - #[fragment_attrs(serde(default))] - pub requested_secret_lifetime: Option, -} - -impl RouterConfig { - fn default_config( - cluster_name: &str, - role: &DruidRole, - deep_storage: &DeepStorageSpec, - ) -> RouterConfigFragment { - RouterConfigFragment { - resources: resource::ROUTER_RESOURCES.to_owned(), - logging: product_logging::spec::default_logging(), - affinity: get_affinity(cluster_name, role, deep_storage), - graceful_shutdown_timeout: Some(role.default_graceful_shutdown_timeout()), - requested_secret_lifetime: Some(DEFAULT_ROUTER_SECRET_LIFETIME), +/// Generates a per-role config struct and its `default_config`. The four non-historical roles share +/// an identical shape; only the storage type, default [`resource`] profile and default secret +/// lifetime differ. The `Fragment` type (named `Fragment` by the [`Fragment`] derive) is +/// passed explicitly because `macro_rules!` cannot synthesize identifiers. +macro_rules! role_group_config { + ($name:ident, $fragment:ident, $storage:ty, $resources:expr, $secret_lifetime:expr $(,)?) => { + #[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] + #[fragment_attrs( + derive( + Clone, + Debug, + Default, + Deserialize, + Merge, + JsonSchema, + PartialEq, + Serialize + ), + serde(rename_all = "camelCase") + )] + pub struct $name { + #[fragment_attrs(serde(default))] + resources: Resources<$storage, NoRuntimeLimits>, + #[fragment_attrs(serde(default))] + pub logging: Logging, + #[fragment_attrs(serde(default))] + pub affinity: StackableAffinity, + /// The time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. + /// Read more about graceful shutdown in the + /// [graceful shutdown documentation](DOCS_BASE_URL_PLACEHOLDER/druid/usage-guide/operations/graceful-shutdown). + #[fragment_attrs(serde(default))] + pub graceful_shutdown_timeout: Option, + + /// Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. + /// This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. + #[fragment_attrs(serde(default))] + pub requested_secret_lifetime: Option, } - } -} -#[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] -#[fragment_attrs( - derive( - Clone, - Debug, - Default, - Deserialize, - Merge, - JsonSchema, - PartialEq, - Serialize - ), - serde(rename_all = "camelCase") -)] -pub struct HistoricalConfig { - #[fragment_attrs(serde(default))] - resources: Resources, - #[fragment_attrs(serde(default))] - pub logging: Logging, - #[fragment_attrs(serde(default))] - pub affinity: StackableAffinity, - /// The time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. - /// Read more about graceful shutdown in the - /// [graceful shutdown documentation](DOCS_BASE_URL_PLACEHOLDER/druid/usage-guide/operations/graceful-shutdown). - #[fragment_attrs(serde(default))] - pub graceful_shutdown_timeout: Option, - - /// Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. - /// This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. - #[fragment_attrs(serde(default))] - pub requested_secret_lifetime: Option, -} - -impl HistoricalConfig { - fn default_config( - cluster_name: &str, - role: &DruidRole, - deep_storage: &DeepStorageSpec, - ) -> HistoricalConfigFragment { - HistoricalConfigFragment { - resources: resource::HISTORICAL_RESOURCES.to_owned(), - logging: product_logging::spec::default_logging(), - affinity: get_affinity(cluster_name, role, deep_storage), - graceful_shutdown_timeout: Some(role.default_graceful_shutdown_timeout()), - requested_secret_lifetime: Some(DEFAULT_HISTORICAL_SECRET_LIFETIME), + impl $name { + fn default_config( + cluster_name: &str, + role: &DruidRole, + deep_storage: &DeepStorageSpec, + ) -> $fragment { + $fragment { + resources: $resources.to_owned(), + logging: product_logging::spec::default_logging(), + affinity: get_affinity(cluster_name, role, deep_storage), + graceful_shutdown_timeout: Some(role.default_graceful_shutdown_timeout()), + requested_secret_lifetime: Some($secret_lifetime), + } + } } - } + }; } -impl Configuration for BrokerConfigFragment { - type Configurable = v1alpha1::DruidCluster; - - fn compute_env( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, ConfigError> { - let mut _result = BTreeMap::new(); - Ok(_result) - } - - fn compute_cli( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, ConfigError> { - Ok(BTreeMap::new()) - } - - fn compute_files( - &self, - resource: &Self::Configurable, - _role_name: &str, - file: &str, - ) -> Result>, ConfigError> { - resource.common_compute_files(file) - } -} - -impl Configuration for HistoricalConfigFragment { - type Configurable = v1alpha1::DruidCluster; - - fn compute_env( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, ConfigError> { - let mut _result = BTreeMap::new(); - Ok(_result) - } - - fn compute_cli( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, ConfigError> { - Ok(BTreeMap::new()) - } - - fn compute_files( - &self, - resource: &Self::Configurable, - _role_name: &str, - file: &str, - ) -> Result>, ConfigError> { - resource.common_compute_files(file) - } -} - -impl Configuration for RouterConfigFragment { - type Configurable = v1alpha1::DruidCluster; - - fn compute_env( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, ConfigError> { - let mut _result = BTreeMap::new(); - Ok(_result) - } - - fn compute_cli( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, ConfigError> { - Ok(BTreeMap::new()) - } - - fn compute_files( - &self, - resource: &Self::Configurable, - _role_name: &str, - file: &str, - ) -> Result>, ConfigError> { - resource.common_compute_files(file) - } -} - -impl Configuration for MiddleManagerConfigFragment { - type Configurable = v1alpha1::DruidCluster; - - fn compute_env( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, ConfigError> { - let mut _result = BTreeMap::new(); - Ok(_result) - } - - fn compute_cli( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, ConfigError> { - Ok(BTreeMap::new()) - } - - fn compute_files( - &self, - resource: &Self::Configurable, - _role_name: &str, - file: &str, - ) -> Result>, ConfigError> { - let mut result = resource.common_compute_files(file)?; - result.insert( - INDEXER_JAVA_OPTS.to_string(), - Some(build_string_list(&[ - format!("-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}"), - format!("-Djavax.net.ssl.trustStorePassword={STACKABLE_TRUST_STORE_PASSWORD}"), - "-Djavax.net.ssl.trustStoreType=pkcs12".to_owned(), - ])), - ); - Ok(result) - } -} - -impl Configuration for CoordinatorConfigFragment { - type Configurable = v1alpha1::DruidCluster; - - fn compute_env( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, ConfigError> { - let mut _result = BTreeMap::new(); - Ok(_result) - } - - fn compute_cli( - &self, - _resource: &Self::Configurable, - _role_name: &str, - ) -> Result>, ConfigError> { - Ok(BTreeMap::new()) - } - - fn compute_files( - &self, - resource: &Self::Configurable, - _role_name: &str, - file: &str, - ) -> Result>, ConfigError> { - resource.common_compute_files(file) - } -} +role_group_config!( + BrokerConfig, + BrokerConfigFragment, + storage::DruidStorage, + resource::BROKER_RESOURCES, + DEFAULT_BROKER_SECRET_LIFETIME, +); + +role_group_config!( + CoordinatorConfig, + CoordinatorConfigFragment, + storage::DruidStorage, + resource::COORDINATOR_RESOURCES, + DEFAULT_COORDINATOR_SECRET_LIFETIME, +); + +role_group_config!( + MiddleManagerConfig, + MiddleManagerConfigFragment, + storage::DruidStorage, + resource::MIDDLE_MANAGER_RESOURCES, + DEFAULT_MIDDLE_SECRET_LIFETIME, +); + +role_group_config!( + RouterConfig, + RouterConfigFragment, + storage::DruidStorage, + resource::ROUTER_RESOURCES, + DEFAULT_ROUTER_SECRET_LIFETIME, +); + +role_group_config!( + HistoricalConfig, + HistoricalConfigFragment, + storage::HistoricalStorage, + resource::HISTORICAL_RESOURCES, + DEFAULT_HISTORICAL_SECRET_LIFETIME, +); #[derive(Clone, Debug, Default, Deserialize, JsonSchema, Serialize)] #[serde(rename_all = "camelCase")] @@ -1592,65 +1026,6 @@ pub fn build_string_list(strings: &[String]) -> String { format!("[{}]", comma_list) } -/// Creates recommended `ObjectLabels` to be used in deployed resources -pub fn build_recommended_labels<'a, T>( - owner: &'a T, - controller_name: &'a str, - app_version: &'a str, - role: &'a str, - role_group: &'a str, -) -> ObjectLabels<'a, T> { - ObjectLabels { - owner, - app_name: APP_NAME, - app_version, - operator_name: OPERATOR_NAME, - controller_name, - role, - role_group, - } -} - -fn extract_role_from_role_config( - fragment: Role, -) -> Role -where - T: FromFragment, - T::Fragment: Clone + Merge, -{ - Role { - config: CommonConfiguration { - config: fragment.config.config, - config_overrides: fragment.config.config_overrides, - env_overrides: fragment.config.env_overrides, - cli_overrides: fragment.config.cli_overrides, - pod_overrides: fragment.config.pod_overrides, - product_specific_common_config: fragment.config.product_specific_common_config, - }, - role_config: fragment.role_config.generic, - role_groups: fragment - .role_groups - .into_iter() - .map(|(k, v)| { - ( - k, - RoleGroup { - config: CommonConfiguration { - config: v.config.config, - config_overrides: v.config.config_overrides, - env_overrides: v.config.env_overrides, - cli_overrides: v.config.cli_overrides, - pod_overrides: v.config.pod_overrides, - product_specific_common_config: v.config.product_specific_common_config, - }, - replicas: v.replicas, - }, - ) - }) - .collect(), - } -} - #[cfg(test)] mod tests { use stackable_operator::versioned::test_utils::RoundtripTestData; diff --git a/rust/operator-binary/src/crd/resource.rs b/rust/operator-binary/src/crd/resource.rs index e2d6bfb2..92cc546b 100644 --- a/rust/operator-binary/src/crd/resource.rs +++ b/rust/operator-binary/src/crd/resource.rs @@ -5,8 +5,8 @@ use stackable_operator::{ builder, builder::pod::{PodBuilder, container::ContainerBuilder, volume::VolumeBuilder}, commons::resources::{ - CpuLimitsFragment, MemoryLimits, MemoryLimitsFragment, NoRuntimeLimits, - NoRuntimeLimitsFragment, Resources, ResourcesFragment, + CpuLimitsFragment, MemoryLimitsFragment, NoRuntimeLimits, NoRuntimeLimitsFragment, + Resources, ResourcesFragment, }, k8s_openapi::{ api::core::v1::{EmptyDirVolumeSource, ResourceRequirements}, @@ -17,11 +17,13 @@ use stackable_operator::{ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::crd::{ - DruidRole, PATH_SEGMENT_CACHE, PROP_SEGMENT_CACHE_LOCATIONS, + DruidRole, PROP_SEGMENT_CACHE_LOCATIONS, memory::{HistoricalDerivedSettings, RESERVED_OS_MEMORY}, storage::{self, default_free_percentage_empty_dir_fragment}, }; +const PATH_SEGMENT_CACHE: &str = "/stackable/var/druid/segment-cache"; + // volume names const SEGMENT_CACHE_VOLUME_NAME: &str = "segment-cache"; @@ -67,18 +69,11 @@ impl RoleResource { } } - pub fn as_memory_limits(&self) -> MemoryLimits { - match self { - Self::Druid(r) => r.memory.clone(), - Self::Historical(r) => r.memory.clone(), - } - } - /// Update the given configuration file with resource properties. /// Currently it only adds historical-specific configs for direct memory buffers, thread counts and segment cache. pub fn update_druid_config_file( &self, - config: &mut BTreeMap>, + config: &mut BTreeMap, ) -> Result<(), Error> { match self { Self::Historical(r) => { @@ -87,10 +82,10 @@ impl RoleResource { config .entry(PROP_SEGMENT_CACHE_LOCATIONS.to_string()) .or_insert_with(|| { - Some(format!( + format!( r#"[{{"path":"{}","maxSize":"{}","freeSpacePercent":"{}"}}]"#, PATH_SEGMENT_CACHE, capacity.0, free_percentage - )) + ) }); let settings = @@ -172,7 +167,7 @@ impl RoleResource { } } -pub static MIDDLE_MANAGER_RESOURCES: LazyLock< +pub(crate) static MIDDLE_MANAGER_RESOURCES: LazyLock< ResourcesFragment, > = LazyLock::new(|| ResourcesFragment { cpu: CpuLimitsFragment { @@ -186,20 +181,21 @@ pub static MIDDLE_MANAGER_RESOURCES: LazyLock< storage: storage::DruidStorageFragment {}, }); -pub static BROKER_RESOURCES: LazyLock> = - LazyLock::new(|| ResourcesFragment { - cpu: CpuLimitsFragment { - min: Some(Quantity("300m".to_owned())), - max: Some(Quantity("1200m".to_owned())), - }, - memory: MemoryLimitsFragment { - limit: Some(Quantity("1500Mi".to_owned())), - runtime_limits: NoRuntimeLimitsFragment {}, - }, - storage: storage::DruidStorageFragment {}, - }); +pub(crate) static BROKER_RESOURCES: LazyLock< + ResourcesFragment, +> = LazyLock::new(|| ResourcesFragment { + cpu: CpuLimitsFragment { + min: Some(Quantity("300m".to_owned())), + max: Some(Quantity("1200m".to_owned())), + }, + memory: MemoryLimitsFragment { + limit: Some(Quantity("1500Mi".to_owned())), + runtime_limits: NoRuntimeLimitsFragment {}, + }, + storage: storage::DruidStorageFragment {}, +}); -pub static HISTORICAL_RESOURCES: LazyLock< +pub(crate) static HISTORICAL_RESOURCES: LazyLock< ResourcesFragment, > = LazyLock::new(|| ResourcesFragment { cpu: CpuLimitsFragment { @@ -215,7 +211,7 @@ pub static HISTORICAL_RESOURCES: LazyLock< }, }); -pub static COORDINATOR_RESOURCES: LazyLock< +pub(crate) static COORDINATOR_RESOURCES: LazyLock< ResourcesFragment, > = LazyLock::new(|| ResourcesFragment { cpu: CpuLimitsFragment { @@ -229,35 +225,39 @@ pub static COORDINATOR_RESOURCES: LazyLock< storage: storage::DruidStorageFragment {}, }); -pub static ROUTER_RESOURCES: LazyLock> = - LazyLock::new(|| ResourcesFragment { - cpu: CpuLimitsFragment { - min: Some(Quantity("300m".to_owned())), - max: Some(Quantity("1200m".to_owned())), - }, - memory: MemoryLimitsFragment { - limit: Some(Quantity("512Mi".to_owned())), - runtime_limits: NoRuntimeLimitsFragment {}, - }, - storage: storage::DruidStorageFragment {}, - }); +pub(crate) static ROUTER_RESOURCES: LazyLock< + ResourcesFragment, +> = LazyLock::new(|| ResourcesFragment { + cpu: CpuLimitsFragment { + min: Some(Quantity("300m".to_owned())), + max: Some(Quantity("1200m".to_owned())), + }, + memory: MemoryLimitsFragment { + limit: Some(Quantity("512Mi".to_owned())), + runtime_limits: NoRuntimeLimitsFragment {}, + }, + storage: storage::DruidStorageFragment {}, +}); #[cfg(test)] mod test { + use std::str::FromStr; + use rstest::*; use stackable_operator::{ commons::resources::{ CpuLimits, CpuLimitsFragment, MemoryLimits, MemoryLimitsFragment, NoRuntimeLimitsFragment, }, + config::{fragment, merge::Merge}, k8s_openapi::apimachinery::pkg::api::resource::Quantity, - role_utils::{CommonConfiguration, RoleGroup}, utils::yaml_from_str_singleton_map, + v2::types::operator::RoleGroupName, }; use super::*; use crate::crd::{ - MiddleManagerConfig, + DruidRole, storage::{HistoricalStorage, default_free_percentage_empty_dir}, v1alpha1, }; @@ -395,11 +395,13 @@ mod test { #[case] third: Option>, #[case] expected: Resources, ) { - let got = v1alpha1::DruidCluster::merged_rolegroup_config( - &first.unwrap_or_default(), - &second.unwrap_or_default(), - &third.unwrap_or_default(), - ); + // Replicates the fragment merge done by `with_validated_config` / + // `RoleGroup::validate_config`: default <- role <- role group, then validate. + let mut role_config = second.unwrap_or_default(); + role_config.merge(&third.unwrap_or_default()); + let mut rolegroup_config = first.unwrap_or_default(); + rolegroup_config.merge(&role_config); + let got = fragment::validate::>(rolegroup_config); assert_eq!(expected, got.unwrap()); } @@ -411,19 +413,11 @@ mod test { )) .expect("failed to parse YAML"); - let config = cluster.merged_config().unwrap(); - if let Some(RoleGroup { - config: - CommonConfiguration { - config: - MiddleManagerConfig { - resources: middlemanager_resources_from_rg, - .. - }, - .. - }, - .. - }) = config.middle_managers.get("resources-from-role-group") + let middle_managers = cluster.merged_role(&DruidRole::MiddleManager).unwrap(); + + if let Some(RoleResource::Druid(middlemanager_resources_from_rg)) = middle_managers + .get(&RoleGroupName::from_str("resources-from-role-group").unwrap()) + .map(|rg| &rg.config.resources) { let expected = Resources { cpu: CpuLimits { @@ -445,18 +439,9 @@ mod test { panic!("No role group named [resources-from-role-group] found"); } - if let Some(RoleGroup { - config: - CommonConfiguration { - config: - MiddleManagerConfig { - resources: middlemanager_resources_from_rg, - .. - }, - .. - }, - .. - }) = config.middle_managers.get("resources-from-role") + if let Some(RoleResource::Druid(middlemanager_resources_from_rg)) = middle_managers + .get(&RoleGroupName::from_str("resources-from-role").unwrap()) + .map(|rg| &rg.config.resources) { let expected = Resources { cpu: CpuLimits { @@ -488,11 +473,13 @@ mod test { )) .expect("failed to parse YAML"); + let historicals = cluster.merged_role(&DruidRole::Historical).unwrap(); + // ---------- default role group - let config = cluster.merged_config().unwrap(); - let res = config - .common_config(&DruidRole::Historical, "default") + let res = &historicals + .get(&RoleGroupName::from_str("default").unwrap()) .unwrap() + .config .resources; let mut got = BTreeMap::new(); @@ -500,13 +487,14 @@ mod test { assert!(got.contains_key(PROP_SEGMENT_CACHE_LOCATIONS)); let value = got.get(PROP_SEGMENT_CACHE_LOCATIONS).unwrap(); - let expected = Some(r#"[{"path":"/stackable/var/druid/segment-cache","maxSize":"5g","freeSpacePercent":"3"}]"#.to_string()); + let expected = r#"[{"path":"/stackable/var/druid/segment-cache","maxSize":"5g","freeSpacePercent":"3"}]"#.to_string(); assert_eq!(value, &expected, "primary"); // ---------- secondary role group - let res = config - .common_config(&DruidRole::Historical, "secondary") + let res = &historicals + .get(&RoleGroupName::from_str("secondary").unwrap()) .unwrap() + .config .resources; let mut got = BTreeMap::new(); @@ -514,7 +502,7 @@ mod test { assert!(got.contains_key(PROP_SEGMENT_CACHE_LOCATIONS)); let value = got.get(PROP_SEGMENT_CACHE_LOCATIONS).unwrap(); - let expected = Some(r#"[{"path":"/stackable/var/druid/segment-cache","maxSize":"2g","freeSpacePercent":"7"}]"#.to_string()); + let expected = r#"[{"path":"/stackable/var/druid/segment-cache","maxSize":"2g","freeSpacePercent":"7"}]"#.to_string(); assert_eq!(value, &expected, "secondary"); Ok(()) diff --git a/rust/operator-binary/src/crd/security.rs b/rust/operator-binary/src/crd/security.rs index bb610b30..5568e8db 100644 --- a/rust/operator-binary/src/crd/security.rs +++ b/rust/operator-binary/src/crd/security.rs @@ -23,16 +23,12 @@ use stackable_operator::{ }; use crate::crd::{ - DruidRole, STACKABLE_TRUST_STORE_PASSWORD, - authentication::{self, AuthenticationClassesResolved}, + DruidRole, STACKABLE_TRUST_STORE_PASSWORD, authentication::AuthenticationClassesResolved, v1alpha1, }; #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("failed to process authentication class"))] - InvalidAuthenticationClassConfiguration { source: authentication::Error }, - #[snafu(display("failed to build the Secret operator Volume"))] SecretVolumeBuild { source: SecretOperatorVolumeSourceBuilderError, @@ -86,12 +82,12 @@ const SERVER_HTTPS_REQUIRE_CLIENT_CERTIFICATE: &str = "druid.server.https.requir /// All secret-op generated keystores have one entry with the alias "1". /// (side node: I think technically they don't have an alias and the JVm counts them, but not sure) const TLS_ALIAS_NAME: &str = "1"; -pub const AUTH_TRUST_STORE_PATH: &str = "druid.auth.basic.ssl.trustStorePath"; -pub const AUTH_TRUST_STORE_TYPE: &str = "druid.auth.basic.ssl.trustStoreType"; -pub const AUTH_TRUST_STORE_PASSWORD: &str = "druid.auth.basic.ssl.trustStorePassword"; +const AUTH_TRUST_STORE_PATH: &str = "druid.auth.basic.ssl.trustStorePath"; +const AUTH_TRUST_STORE_TYPE: &str = "druid.auth.basic.ssl.trustStoreType"; +const AUTH_TRUST_STORE_PASSWORD: &str = "druid.auth.basic.ssl.trustStorePassword"; // Misc TLS pub const TLS_STORE_PASSWORD: &str = "changeit"; -pub const TLS_STORE_TYPE: &str = "pkcs12"; +const TLS_STORE_TYPE: &str = "pkcs12"; // directories const STACKABLE_MOUNT_TLS_DIR: &str = "/stackable/mount_tls"; @@ -102,12 +98,10 @@ const TLS_VOLUME_NAME: &str = "tls"; const TLS_MOUNT_VOLUME_NAME: &str = "tls-mount"; pub const INTERNAL_INITIAL_CLIENT_PASSWORD_ENV: &str = "INTERNAL_INITIAL_CLIENT_PASSWORD"; -// It seems this needs to be the same password for Druid to work, so we re-use the existing env variable from above. -pub const ESCALATOR_INTERNAL_CLIENT_PASSWORD_ENV: &str = INTERNAL_INITIAL_CLIENT_PASSWORD_ENV; impl DruidTlsSecurity { #[cfg(test)] - pub fn new( + pub(crate) fn new( auth_classes: &AuthenticationClassesResolved, server_and_internal_secret_class: Option, ) -> Self { @@ -125,12 +119,13 @@ impl DruidTlsSecurity { ) -> Self { DruidTlsSecurity { auth_classes: auth_classes.clone(), - server_and_internal_secret_class: druid - .spec - .cluster_config - .tls - .as_ref() - .and_then(|tls| tls.server_and_internal_secret_class.clone()), + server_and_internal_secret_class: druid.spec.cluster_config.tls.as_ref().and_then( + |tls| { + tls.server_and_internal_secret_class + .as_ref() + .map(|secret_class| secret_class.to_string()) + }, + ), } } @@ -148,7 +143,7 @@ impl DruidTlsSecurity { } /// Retrieve an optional TLS secret class for external client -> server and server <-> server communications. - pub fn tls_server_and_internal_secret_class(&self) -> Option<&str> { + fn tls_server_and_internal_secret_class(&self) -> Option<&str> { self.server_and_internal_secret_class.as_deref() } @@ -176,21 +171,15 @@ impl DruidTlsSecurity { .collect() } - pub fn listener_ports( - &self, - role: &DruidRole, - ) -> Option> { - let listener_ports = self - .exposed_ports(role) + pub fn listener_ports(&self, role: &DruidRole) -> Vec { + self.exposed_ports(role) .into_iter() .map(|(name, val)| listener::v1alpha1::ListenerPort { name, port: val.into(), protocol: Some("TCP".to_string()), }) - .collect(); - - Some(listener_ports) + .collect() } fn exposed_ports(&self, role: &DruidRole) -> Vec<(String, u16)> { @@ -265,33 +254,27 @@ impl DruidTlsSecurity { fn add_tls_port_config_properties( &self, - config: &mut BTreeMap>, + config: &mut BTreeMap, role: &DruidRole, ) { // no secure communication if !self.tls_enabled() { - config.insert(ENABLE_PLAINTEXT_PORT.to_string(), Some("true".to_string())); - config.insert(ENABLE_TLS_PORT.to_string(), Some("false".to_string())); - config.insert( - PLAINTEXT_PORT.to_string(), - Some(role.get_http_port().to_string()), - ); + config.insert(ENABLE_PLAINTEXT_PORT.to_string(), "true".to_string()); + config.insert(ENABLE_TLS_PORT.to_string(), "false".to_string()); + config.insert(PLAINTEXT_PORT.to_string(), role.get_http_port().to_string()); } // only allow secure communication else { - config.insert(ENABLE_PLAINTEXT_PORT.to_string(), Some("false".to_string())); - config.insert(ENABLE_TLS_PORT.to_string(), Some("true".to_string())); - config.insert( - TLS_PORT.to_string(), - Some(role.get_https_port().to_string()), - ); + config.insert(ENABLE_PLAINTEXT_PORT.to_string(), "false".to_string()); + config.insert(ENABLE_TLS_PORT.to_string(), "true".to_string()); + config.insert(TLS_PORT.to_string(), role.get_https_port().to_string()); } } /// Add required TLS ports, trust/key store properties pub fn add_tls_config_properties( &self, - config: &mut BTreeMap>, + config: &mut BTreeMap, role: &DruidRole, ) { self.add_tls_port_config_properties(config, role); @@ -305,75 +288,78 @@ impl DruidTlsSecurity { } } - fn add_tls_encryption_config_properties( - config: &mut BTreeMap>, + /// Inserts the path/type/password triple describing a single PKCS12 keystore or truststore + /// (`/`), keeping the store type and password consistent across + /// all of Druid's HTTPS store settings. + fn add_pkcs12_store_properties( + config: &mut BTreeMap, store_directory: &str, - store_alias: &str, + store_file: &str, + path_property: &str, + type_property: &str, + password_property: &str, ) { - // We need a truststore in addition to a keystore here, because server and internal tls - // can only be enabled/disabled together - config.insert( - CLIENT_HTTPS_TRUST_STORE_PATH.to_string(), - Some(format!("{}/truststore.p12", store_directory)), - ); config.insert( - CLIENT_HTTPS_TRUST_STORE_TYPE.to_string(), - Some(TLS_STORE_TYPE.to_string()), + path_property.to_string(), + format!("{store_directory}/{store_file}"), ); + config.insert(type_property.to_string(), TLS_STORE_TYPE.to_string()); config.insert( - CLIENT_HTTPS_TRUST_STORE_PASSWORD.to_string(), - Some(TLS_STORE_PASSWORD.to_string()), + password_property.to_string(), + TLS_STORE_PASSWORD.to_string(), ); + } - config.insert( - SERVER_HTTPS_KEY_STORE_PATH.to_string(), - Some(format!("{}/keystore.p12", store_directory)), - ); - config.insert( - SERVER_HTTPS_KEY_STORE_TYPE.to_string(), - Some(TLS_STORE_TYPE.to_string()), - ); - config.insert( - SERVER_HTTPS_KEY_STORE_PASSWORD.to_string(), - Some(TLS_STORE_PASSWORD.to_string()), + fn add_tls_encryption_config_properties( + config: &mut BTreeMap, + store_directory: &str, + store_alias: &str, + ) { + // We need a truststore in addition to a keystore here, because server and internal tls + // can only be enabled/disabled together + Self::add_pkcs12_store_properties( + config, + store_directory, + "truststore.p12", + CLIENT_HTTPS_TRUST_STORE_PATH, + CLIENT_HTTPS_TRUST_STORE_TYPE, + CLIENT_HTTPS_TRUST_STORE_PASSWORD, ); - config.insert( - SERVER_HTTPS_CERT_ALIAS.to_string(), - Some(store_alias.to_string()), + + Self::add_pkcs12_store_properties( + config, + store_directory, + "keystore.p12", + SERVER_HTTPS_KEY_STORE_PATH, + SERVER_HTTPS_KEY_STORE_TYPE, + SERVER_HTTPS_KEY_STORE_PASSWORD, ); + config.insert(SERVER_HTTPS_CERT_ALIAS.to_string(), store_alias.to_string()); // We also need to configure the truststore for authentication related stuff, // such as verifying the LDAP server - config.insert( - AUTH_TRUST_STORE_PATH.to_string(), - Some(format!("{}/truststore.p12", store_directory)), - ); - config.insert( - AUTH_TRUST_STORE_TYPE.to_string(), - Some(TLS_STORE_TYPE.to_string()), - ); - config.insert( - AUTH_TRUST_STORE_PASSWORD.to_string(), - Some(TLS_STORE_PASSWORD.to_string()), + Self::add_pkcs12_store_properties( + config, + store_directory, + "truststore.p12", + AUTH_TRUST_STORE_PATH, + AUTH_TRUST_STORE_TYPE, + AUTH_TRUST_STORE_PASSWORD, ); } fn add_tls_auth_config_properties( - config: &mut BTreeMap>, + config: &mut BTreeMap, store_directory: &str, store_alias: &str, ) { - config.insert( - CLIENT_HTTPS_KEY_STORE_PATH.to_string(), - Some(format!("{store_directory}/keystore.p12")), - ); - config.insert( - CLIENT_HTTPS_KEY_STORE_TYPE.to_string(), - Some(TLS_STORE_TYPE.to_string()), - ); - config.insert( - CLIENT_HTTPS_KEY_STORE_PASSWORD.to_string(), - Some(TLS_STORE_PASSWORD.to_string()), + Self::add_pkcs12_store_properties( + config, + store_directory, + "keystore.p12", + CLIENT_HTTPS_KEY_STORE_PATH, + CLIENT_HTTPS_KEY_STORE_TYPE, + CLIENT_HTTPS_KEY_STORE_PASSWORD, ); // This is required because PKCS12 does not use any key passwords but it will // be checked and would lead to an exception: @@ -382,36 +368,29 @@ impl DruidTlsSecurity { // javax.crypto.BadPaddingException: Given final block not properly padded. Such issues can arise if a bad key is used during decryption. config.insert( CLIENT_HTTPS_KEY_MANAGER_PASSWORD.to_string(), - Some(TLS_STORE_PASSWORD.to_string()), - ); - config.insert( - CLIENT_HTTPS_CERT_ALIAS.to_string(), - Some(store_alias.to_string()), + TLS_STORE_PASSWORD.to_string(), ); + config.insert(CLIENT_HTTPS_CERT_ALIAS.to_string(), store_alias.to_string()); // FIXME: https://github.com/stackabletech/druid-operator/issues/372 // This is required because the server will send its pod ip which is not in the SANs of the certificates config.insert( CLIENT_HTTPS_VALIDATE_HOST_NAMES.to_string(), - Some("false".to_string()), + "false".to_string(), ); // This will enforce the client to authenticate itself config.insert( SERVER_HTTPS_REQUIRE_CLIENT_CERTIFICATE.to_string(), - Some("true".to_string()), + "true".to_string(), ); - config.insert( - SERVER_HTTPS_TRUST_STORE_PATH.to_string(), - Some(format!("{store_directory}/truststore.p12")), - ); - config.insert( - SERVER_HTTPS_TRUST_STORE_TYPE.to_string(), - Some(TLS_STORE_TYPE.to_string()), - ); - config.insert( - SERVER_HTTPS_TRUST_STORE_PASSWORD.to_string(), - Some(TLS_STORE_PASSWORD.to_string()), + Self::add_pkcs12_store_properties( + config, + store_directory, + "truststore.p12", + SERVER_HTTPS_TRUST_STORE_PATH, + SERVER_HTTPS_TRUST_STORE_TYPE, + SERVER_HTTPS_TRUST_STORE_PASSWORD, ); // This is required because PKCS12 does not use any key passwords but it will // be checked and would lead to an exception: @@ -420,13 +399,13 @@ impl DruidTlsSecurity { // javax.crypto.BadPaddingException: Given final block not properly padded. Such issues can arise if a bad key is used during decryption. config.insert( SERVER_HTTPS_KEY_MANAGER_PASSWORD.to_string(), - Some(TLS_STORE_PASSWORD.to_string()), + TLS_STORE_PASSWORD.to_string(), ); // FIXME: https://github.com/stackabletech/druid-operator/issues/372 // This is required because the client will send its pod ip which is not in the SANs of the certificates config.insert( SERVER_HTTPS_VALIDATE_HOST_NAMES.to_string(), - Some("false".to_string()), + "false".to_string(), ); } diff --git a/rust/operator-binary/src/crd/tls.rs b/rust/operator-binary/src/crd/tls.rs index 558b1a5a..5e24cea2 100644 --- a/rust/operator-binary/src/crd/tls.rs +++ b/rust/operator-binary/src/crd/tls.rs @@ -1,5 +1,10 @@ +use std::str::FromStr; + use serde::{Deserialize, Serialize}; -use stackable_operator::schemars::{self, JsonSchema}; +use stackable_operator::{ + schemars::{self, JsonSchema}, + v2::types::kubernetes::SecretClassName, +}; const TLS_DEFAULT_SECRET_CLASS: &str = "tls"; @@ -14,7 +19,7 @@ pub struct DruidTls { // happens via the HTTPS port. Even if both HTTPS and HTTP port are enabled, Druid clients // will default to using TLS. #[serde(default = "tls_default", skip_serializing_if = "Option::is_none")] - pub server_and_internal_secret_class: Option, + pub server_and_internal_secret_class: Option, } /// Default TLS settings. Internal and server communication default to "tls" secret class. @@ -25,14 +30,18 @@ pub fn default_druid_tls() -> Option { } /// Helper methods to provide defaults in the CRDs and tests -pub fn tls_default() -> Option { - Some(TLS_DEFAULT_SECRET_CLASS.to_string()) +pub fn tls_default() -> Option { + Some(SecretClassName::from_str(TLS_DEFAULT_SECRET_CLASS).expect("a valid secret class name")) } #[cfg(test)] mod tests { + use std::str::FromStr; + use indoc::formatdoc; - use stackable_operator::utils::yaml_from_str_singleton_map; + use stackable_operator::{ + utils::yaml_from_str_singleton_map, v2::types::kubernetes::SecretClassName, + }; use crate::crd::{tls::DruidTls, v1alpha1::DruidClusterConfig}; @@ -54,7 +63,9 @@ zookeeperConfigMapName: zk-config-map assert_eq!( druid_cluster_config.tls, Some(DruidTls { - server_and_internal_secret_class: Some("tls".to_string()) + server_and_internal_secret_class: Some( + SecretClassName::from_str("tls").expect("a valid secret class name") + ) }), ); assert_eq!(druid_cluster_config.authentication, vec![]); @@ -73,7 +84,10 @@ zookeeperConfigMapName: zk-config-map assert_eq!( druid_cluster_config.tls, Some(DruidTls { - server_and_internal_secret_class: Some("druid-secret-class".to_string()) + server_and_internal_secret_class: Some( + SecretClassName::from_str("druid-secret-class") + .expect("a valid secret class name") + ) }), ); assert_eq!(druid_cluster_config.authentication, vec![]); @@ -126,7 +140,10 @@ zookeeperConfigMapName: zk-config-map assert_eq!( druid_cluster_config.tls, Some(DruidTls { - server_and_internal_secret_class: Some("druid-secret-class".to_string()) + server_and_internal_secret_class: Some( + SecretClassName::from_str("druid-secret-class") + .expect("a valid secret class name") + ) }), ); /* diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs deleted file mode 100644 index 95220b22..00000000 --- a/rust/operator-binary/src/discovery.rs +++ /dev/null @@ -1,108 +0,0 @@ -//! Discovery for Druid. We make Druid discoverable by putting a connection string to the router service -//! inside a config map. We only provide a connection string to the router service, since it serves as -//! a gateway to the cluster for client queries. -use snafu::{ResultExt, Snafu}; -use stackable_operator::{ - builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, - commons::product_image_selection::ResolvedProductImage, - crd::listener::v1alpha1::Listener, - k8s_openapi::api::core::v1::ConfigMap, - kube::{Resource, ResourceExt, runtime::reflector::ObjectRef}, -}; - -use crate::{ - DRUID_CONTROLLER_NAME, - crd::{DruidRole, build_recommended_labels, security::DruidTlsSecurity, v1alpha1}, - listener::build_listener_connection_string, -}; - -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("object {} is missing metadata to build owner reference", druid))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, - druid: ObjectRef, - }, - - #[snafu(display("failed to get service FQDN"))] - NoServiceFqdn, - - #[snafu(display("failed to build ConfigMap"))] - BuildConfigMap { - source: stackable_operator::builder::configmap::Error, - }, - - #[snafu(display("failed to add recommended labels"))] - AddRecommendedLabels { - source: stackable_operator::builder::meta::Error, - }, - - #[snafu(display("failed to configure listener discovery configmap"))] - ListenerConfiguration { source: crate::listener::Error }, -} - -/// Builds discovery [`ConfigMap`]s for connecting to a [`v1alpha1::DruidCluster`]. -pub async fn build_discovery_configmaps( - druid: &v1alpha1::DruidCluster, - owner: &impl Resource, - resolved_product_image: &ResolvedProductImage, - druid_tls_security: &DruidTlsSecurity, - listener: Listener, -) -> Result, Error> { - let name = owner.name_unchecked(); - Ok(vec![build_discovery_configmap( - druid, - owner, - resolved_product_image, - druid_tls_security, - &name, - listener, - )?]) -} - -/// Build a discovery [`ConfigMap`] containing information about how to connect to a certain [`v1alpha1::DruidCluster`]. -fn build_discovery_configmap( - druid: &v1alpha1::DruidCluster, - owner: &impl Resource, - resolved_product_image: &ResolvedProductImage, - druid_tls_security: &DruidTlsSecurity, - name: &str, - listener: Listener, -) -> Result { - let router_host = build_listener_connection_string( - listener, - druid_tls_security, - &DruidRole::Router.to_string(), - ) - .context(ListenerConfigurationSnafu)?; - let sqlalchemy_conn_str = format!("druid://{}/druid/v2/sql", router_host); - let avatica_conn_str = format!( - "jdbc:avatica:remote:url=http://{}/druid/v2/sql/avatica/", - router_host - ); - - ConfigMapBuilder::new() - .metadata( - ObjectMetaBuilder::new() - .name_and_namespace(druid) - .name(name) - .ownerreference_from_resource(owner, None, Some(true)) - .with_context(|_| ObjectMissingMetadataForOwnerRefSnafu { - druid: ObjectRef::from_obj(druid), - })? - .with_recommended_labels(&build_recommended_labels( - druid, - DRUID_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - &DruidRole::Router.to_string(), - "discovery", - )) - .context(AddRecommendedLabelsSnafu)? - .build(), - ) - .add_data("DRUID_ROUTER", router_host) - .add_data("DRUID_SQLALCHEMY", sqlalchemy_conn_str) - .add_data("DRUID_AVATICA_JDBC", avatica_conn_str) - .build() - .context(BuildConfigMapSnafu) -} diff --git a/rust/operator-binary/src/internal_secret.rs b/rust/operator-binary/src/internal_secret.rs index 0f398312..6529ee64 100644 --- a/rust/operator-binary/src/internal_secret.rs +++ b/rust/operator-binary/src/internal_secret.rs @@ -144,7 +144,7 @@ pub async fn create_shared_internal_secret( Ok(()) } -pub fn build_shared_internal_secret(druid: &v1alpha1::DruidCluster) -> Result { +fn build_shared_internal_secret(druid: &v1alpha1::DruidCluster) -> Result { let mut internal_secret = BTreeMap::new(); internal_secret.insert( INTERNAL_INITIAL_CLIENT_PASSWORD_ENV.to_string(), @@ -164,12 +164,12 @@ pub fn build_shared_internal_secret(druid: &v1alpha1::DruidCluster) -> Result String { +fn build_immutable_shared_internal_secret_name(druid: &v1alpha1::DruidCluster) -> String { format!("{}-internal-secret", druid.name_any()) } -pub fn build_shared_internal_secret_name(druid: &v1alpha1::DruidCluster) -> String { - format!("{}-shared-internal-secret", druid.name_any()) +pub fn build_shared_internal_secret_name(owner: &T) -> String { + format!("{}-shared-internal-secret", owner.name_any()) } fn get_random_base64() -> String { diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 1176b910..39a49af5 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use anyhow::anyhow; use clap::Parser; -use controller::{DRUID_CONTROLLER_NAME, FULL_CONTROLLER_NAME}; +use controller::FULL_CONTROLLER_NAME; use futures::{FutureExt, StreamExt, TryFutureExt}; use stackable_operator::{ YamlSchema, @@ -38,16 +38,11 @@ use crate::{ }; mod authentication; -mod config; mod controller; mod crd; -mod discovery; mod extensions; mod internal_secret; -mod listener; mod operations; -mod product_logging; -mod service; mod webhooks; mod built_info { @@ -70,7 +65,7 @@ async fn main() -> anyhow::Result<()> { Command::Run(RunArguments { operator_environment, watch_namespace, - product_config, + product_config: _, maintenance, common, }) => { @@ -117,11 +112,6 @@ async fn main() -> anyhow::Result<()> { .run(sigterm_watcher.handle()) .map_err(|err| anyhow!(err).context("failed to run webhook server")); - let product_config = product_config.load(&[ - "deploy/config-spec/properties.yaml", - "/etc/stackable/druid-operator/config-spec/properties.yaml", - ])?; - let event_recorder = Arc::new(Recorder::new( client.as_kube_client(), Reporter { @@ -166,7 +156,6 @@ async fn main() -> anyhow::Result<()> { Arc::new(controller::Ctx { client: client.clone(), operator_environment, - product_config, }), ) // We can let the reporting happen in the background @@ -208,7 +197,12 @@ fn references_config_map( return false; }; - druid.spec.cluster_config.zookeeper_config_map_name == config_map.name_any() + druid + .spec + .cluster_config + .zookeeper_config_map_name + .to_string() + == config_map.name_any() || match &druid.spec.cluster_config.authorization { Some(druid_authorization) => { druid_authorization.opa.config_map_name == config_map.name_any() @@ -217,7 +211,7 @@ fn references_config_map( } || match &druid.spec.cluster_config.deep_storage { crd::DeepStorageSpec::Hdfs(hdfs_spec) => { - hdfs_spec.config_map_name == config_map.name_any() + hdfs_spec.config_map_name.to_string() == config_map.name_any() } _ => false, } diff --git a/rust/operator-binary/src/operations/mod.rs b/rust/operator-binary/src/operations/mod.rs index 92ca2ec7..590612ba 100644 --- a/rust/operator-binary/src/operations/mod.rs +++ b/rust/operator-binary/src/operations/mod.rs @@ -1,2 +1 @@ pub mod graceful_shutdown; -pub mod pdb; diff --git a/rust/operator-binary/src/operations/pdb.rs b/rust/operator-binary/src/operations/pdb.rs deleted file mode 100644 index da25223c..00000000 --- a/rust/operator-binary/src/operations/pdb.rs +++ /dev/null @@ -1,82 +0,0 @@ -use snafu::{ResultExt, Snafu}; -use stackable_operator::{ - builder::pdb::PodDisruptionBudgetBuilder, client::Client, cluster_resources::ClusterResources, - commons::pdb::PdbConfig, kube::ResourceExt, -}; - -use crate::{ - controller::DRUID_CONTROLLER_NAME, - crd::{APP_NAME, DruidRole, OPERATOR_NAME, v1alpha1}, -}; - -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("Cannot create PodDisruptionBudget for role [{role}]"))] - CreatePdb { - source: stackable_operator::builder::pdb::Error, - role: String, - }, - #[snafu(display("Cannot apply PodDisruptionBudget [{name}]"))] - ApplyPdb { - source: stackable_operator::cluster_resources::Error, - name: String, - }, -} - -pub async fn add_pdbs( - pdb: &PdbConfig, - druid: &v1alpha1::DruidCluster, - role: &DruidRole, - client: &Client, - cluster_resources: &mut ClusterResources<'_>, -) -> Result<(), Error> { - if !pdb.enabled { - return Ok(()); - } - let max_unavailable = pdb.max_unavailable.unwrap_or(match role { - DruidRole::Broker => max_unavailable_brokers(), - DruidRole::Coordinator => max_unavailable_coordinators(), - DruidRole::Historical => max_unavailable_historicals(), - DruidRole::MiddleManager => max_unavailable_middle_managers(), - DruidRole::Router => max_unavailable_routers(), - }); - let pdb = PodDisruptionBudgetBuilder::new_with_role( - druid, - APP_NAME, - &role.to_string(), - OPERATOR_NAME, - DRUID_CONTROLLER_NAME, - ) - .with_context(|_| CreatePdbSnafu { - role: role.to_string(), - })? - .with_max_unavailable(max_unavailable) - .build(); - let pdb_name = pdb.name_any(); - cluster_resources - .add(client, pdb) - .await - .with_context(|_| ApplyPdbSnafu { name: pdb_name })?; - - Ok(()) -} - -fn max_unavailable_brokers() -> u16 { - 1 -} - -fn max_unavailable_coordinators() -> u16 { - 1 -} - -fn max_unavailable_historicals() -> u16 { - 1 -} - -fn max_unavailable_middle_managers() -> u16 { - 1 -} - -fn max_unavailable_routers() -> u16 { - 1 -} diff --git a/rust/operator-binary/src/product_logging.rs b/rust/operator-binary/src/product_logging.rs deleted file mode 100644 index b2923da1..00000000 --- a/rust/operator-binary/src/product_logging.rs +++ /dev/null @@ -1,81 +0,0 @@ -use snafu::Snafu; -use stackable_operator::{ - builder::configmap::ConfigMapBuilder, - memory::BinaryMultiple, - product_logging::{ - self, - spec::{ContainerLogConfig, ContainerLogConfigChoice, Logging}, - }, - role_utils::RoleGroupRef, -}; - -use crate::crd::{ - Container, DRUID_LOG_FILE, LOG4J2_CONFIG, MAX_DRUID_LOG_FILES_SIZE, STACKABLE_LOG_DIR, v1alpha1, -}; - -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("object has no namespace"))] - ObjectHasNoNamespace, - #[snafu(display("failed to retrieve the ConfigMap {cm_name}"))] - ConfigMapNotFound { - source: stackable_operator::client::Error, - cm_name: String, - }, - #[snafu(display("failed to retrieve the entry {entry} for ConfigMap {cm_name}"))] - MissingConfigMapEntry { - entry: &'static str, - cm_name: String, - }, -} - -type Result = std::result::Result; - -const CONSOLE_CONVERSION_PATTERN: &str = "%d{ISO8601} %p [%t] %c - %m%n"; - -/// Extend the role group ConfigMap with logging and Vector configurations -pub fn extend_role_group_config_map( - rolegroup: &RoleGroupRef, - logging: &Logging, - cm_builder: &mut ConfigMapBuilder, -) -> Result<()> { - if let Some(ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic(log_config)), - }) = logging.containers.get(&Container::Druid) - { - cm_builder.add_data( - LOG4J2_CONFIG, - product_logging::framework::create_log4j2_config( - &format!( - "{STACKABLE_LOG_DIR}/{container}", - container = Container::Druid - ), - DRUID_LOG_FILE, - MAX_DRUID_LOG_FILES_SIZE - .scale_to(BinaryMultiple::Mebi) - .floor() - .value as u32, - CONSOLE_CONVERSION_PATTERN, - log_config, - ), - ); - } - - let vector_log_config = if let Some(ContainerLogConfig { - choice: Some(ContainerLogConfigChoice::Automatic(log_config)), - }) = logging.containers.get(&Container::Vector) - { - Some(log_config) - } else { - None - }; - - if logging.enable_vector_agent { - cm_builder.add_data( - product_logging::framework::VECTOR_CONFIG_FILE, - product_logging::framework::create_vector_config(rolegroup, vector_log_config), - ); - } - - Ok(()) -} diff --git a/rust/operator-binary/src/service.rs b/rust/operator-binary/src/service.rs deleted file mode 100644 index 1d9ec6cb..00000000 --- a/rust/operator-binary/src/service.rs +++ /dev/null @@ -1,118 +0,0 @@ -use std::collections::BTreeMap; - -use snafu::{ResultExt, Snafu}; -use stackable_operator::{ - builder::meta::ObjectMetaBuilder, - k8s_openapi::api::core::v1::{Service, ServicePort, ServiceSpec}, - kvp::{Annotations, Label, ObjectLabels}, - role_utils::RoleGroupRef, -}; - -use crate::crd::{ - DruidRole, METRICS_PORT, METRICS_PORT_NAME, security::DruidTlsSecurity, v1alpha1, -}; - -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("object is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, - }, - - #[snafu(display("failed to build Metadata"))] - MetadataBuild { - source: stackable_operator::builder::meta::Error, - }, - - #[snafu(display("failed to build Labels"))] - LabelBuild { - source: stackable_operator::kvp::LabelError, - }, -} - -/// The rolegroup headless [`Service`] is a service that allows direct access to the instances of a certain rolegroup -/// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. -pub fn build_rolegroup_headless_service( - druid: &v1alpha1::DruidCluster, - druid_tls_security: &DruidTlsSecurity, - druid_role: &DruidRole, - role_group_ref: &RoleGroupRef, - object_labels: ObjectLabels, - selector: BTreeMap, -) -> Result { - Ok(Service { - metadata: ObjectMetaBuilder::new() - .name_and_namespace(druid) - .name(role_group_ref.rolegroup_headless_service_name()) - .ownerreference_from_resource(druid, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&object_labels) - .context(MetadataBuildSnafu)? - .build(), - spec: Some(ServiceSpec { - // Internal communication does not need to be exposed - type_: Some("ClusterIP".to_string()), - cluster_ip: Some("None".to_string()), - ports: Some(druid_tls_security.service_ports(druid_role)), - selector: Some(selector), - publish_not_ready_addresses: Some(true), - ..ServiceSpec::default() - }), - status: None, - }) -} - -/// The rolegroup metrics [`Service`] is a service that exposes metrics and a prometheus scraping label. -pub fn build_rolegroup_metrics_service( - druid: &v1alpha1::DruidCluster, - role_group_ref: &RoleGroupRef, - object_labels: ObjectLabels, - selector: BTreeMap, -) -> Result { - Ok(Service { - metadata: ObjectMetaBuilder::new() - .name_and_namespace(druid) - .name(role_group_ref.rolegroup_metrics_service_name()) - .ownerreference_from_resource(druid, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&object_labels) - .context(MetadataBuildSnafu)? - .with_label(Label::try_from(("prometheus.io/scrape", "true")).context(LabelBuildSnafu)?) - .with_annotations(prometheus_annotations()) - .build(), - spec: Some(ServiceSpec { - // Internal communication does not need to be exposed - type_: Some("ClusterIP".to_string()), - cluster_ip: Some("None".to_string()), - ports: Some(metrics_service_ports()), - selector: Some(selector), - publish_not_ready_addresses: Some(true), - ..ServiceSpec::default() - }), - status: None, - }) -} - -fn metrics_service_ports() -> Vec { - vec![ServicePort { - name: Some(METRICS_PORT_NAME.to_string()), - port: METRICS_PORT.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }] -} - -/// Common annotations for Prometheus -/// -/// These annotations can be used in a ServiceMonitor. -/// -/// see also -fn prometheus_annotations() -> Annotations { - Annotations::try_from([ - ("prometheus.io/path".to_owned(), "/metrics".to_owned()), - ("prometheus.io/port".to_owned(), METRICS_PORT.to_string()), - ("prometheus.io/scheme".to_owned(), "http".to_owned()), - ("prometheus.io/scrape".to_owned(), "true".to_owned()), - ]) - .expect("should be valid annotations") -} diff --git a/rust/operator-binary/src/webhooks/conversion.rs b/rust/operator-binary/src/webhooks/conversion.rs index 4b1a608c..6fc5b5bc 100644 --- a/rust/operator-binary/src/webhooks/conversion.rs +++ b/rust/operator-binary/src/webhooks/conversion.rs @@ -8,7 +8,9 @@ use stackable_operator::{ }, }; -use crate::crd::{DruidCluster, DruidClusterVersion, FIELD_MANAGER}; +use crate::crd::{DruidCluster, DruidClusterVersion}; + +const FIELD_MANAGER: &str = "druid-operator"; /// Contains errors which can be encountered when creating the conversion webhook server and the /// CRD maintainer. diff --git a/rust/operator-binary/test/resources/druid_controller/properties.yaml b/rust/operator-binary/test/resources/druid_controller/properties.yaml deleted file mode 100644 index e1e167d9..00000000 --- a/rust/operator-binary/test/resources/druid_controller/properties.yaml +++ /dev/null @@ -1,638 +0,0 @@ ---- -version: 0.1.0 -spec: - units: - - unit: &unitDirectory - name: "directory" - regex: "^/|(/[\\w-]+)+$" - examples: - - "/tmp/xyz" - - unit: &unitPort - name: "port" - regex: "^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]\ - |6553[0-5])$" - - unit: &unitPrometheusNamespace - name: "prometheusNamespace" - regex: "^[a-zA-Z_:][a-zA-Z0-9_:]*$" - - unit: &unitDuration - name: "duration" - regex: "^P(?!$)(\\d+Y)?(\\d+M)?(\\d+W)?(\\d+D)?(T(?=\\d)(\\d+H)?(\\d+M)?(\\d+S)\ - ?)?$" - examples: - - "PT300S" - -################################################################################################### -# runtime.properties -# For information on the properties, see: https://druid.apache.org/docs/latest/configuration/index.html -################################################################################################### - -properties: - - - property: &plaintextPort - propertyNames: - - name: "druid.plaintextPort" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - min: "1" - max: "65535" - unit: *unitPort - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &tlsPort - propertyNames: - - name: "druid.tlsPort" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - min: "1" - max: "65535" - unit: *unitPort - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &startupLoggingLogProperties - propertyNames: - - name: "druid.startup.logging.logProperties" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &monitoringMonitors - propertyNames: - - name: "druid.monitoring.monitors" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "[\"org.apache.druid.java.util.metrics.JvmMonitor\"]" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitter - propertyNames: - - name: "druid.emitter" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "prometheus" - allowedValues: - - "noop" - - "logging" - - "http" - - "parametrized" - - "composing" - - "prometheus" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitterPrometheusStrategy - propertyNames: - - name: "druid.emitter.prometheus.strategy" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "exporter" - allowedValues: - - "exporter" - - "pushgateway" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitterPrometheusNamespace - propertyNames: - - name: "druid.emitter.prometheus.namespace" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitPrometheusNamespace - defaultValues: - - fromVersion: "0.0.0" - value: "druid" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &emitterPrometheusPort - propertyNames: - - name: "druid.emitter.prometheus.port" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - min: "1" - max: "65535" - unit: *unitPort - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &indexerLogsDirectory - propertyNames: - - name: "druid.indexer.logs.directory" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/indexing-logs" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: true - - name: "historical" - required: true - - name: "middlemanager" - required: true - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &processingTmpDir - propertyNames: - - name: "druid.processing.tmpDir" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/processing" - roles: - - name: "broker" - required: true - - name: "coordinator" - required: false - - name: "historical" - required: true - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerTaskHadoopWorkingPath - propertyNames: - - name: "druid.indexer.task.hadoopWorkingPath" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/hadoop-tmp" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: true - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerTaskBaseTaskDir - propertyNames: - - name: "druid.indexer.task.baseTaskDir" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDirectory - defaultValues: - - fromVersion: "0.0.0" - value: "/stackable/var/druid/task" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: true - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerRunnerJavaOpts - propertyNames: - - name: "druid.indexer.runner.javaOpts" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "-server -Xms256m -Xmx256m -XX:MaxDirectMemorySize=300m - -Duser.timezone=UTC -Dfile.encoding=UTF-8 - -XX:+ExitOnOutOfMemoryError - -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: true - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &routerManagementProxyEnabled - propertyNames: - # Management proxy to coordinator / overlord: required for unified web console. - - name: "druid.router.managementProxy.enabled" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &routerHttpNumConnections - propertyNames: - - name: "druid.router.http.numConnections" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "integer" - defaultValues: - - fromVersion: "0.0.0" - value: "25" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: true - asOfVersion: "0.0.0" - - - property: &historicalCacheUseCache - propertyNames: - - name: "druid.historical.cache.useCache" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: true - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &historicalCachePopulateCache - propertyNames: - - name: "druid.historical.cache.populateCache" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: false - - name: "historical" - required: true - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorStartDelay - propertyNames: - - name: "druid.coordinator.startDelay" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDuration - defaultValues: - - fromVersion: "0.0.0" - value: "PT20S" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerQueueStartDelay - propertyNames: - - name: "druid.indexer.queue.startDelay" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDuration - defaultValues: - - fromVersion: "0.0.0" - value: "PT20S" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerRunnerType - propertyNames: - - name: "druid.indexer.runner.type" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - allowedValues: - - "local" - - "remote" - - "httpRemote" - defaultValues: - - fromVersion: "0.0.0" - value: "remote" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &indexerStorageType - propertyNames: - - name: "druid.indexer.storage.type" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - allowedValues: - - "local" - - "metadata" - defaultValues: - - fromVersion: "0.0.0" - value: "metadata" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorPeriod - propertyNames: - - name: "druid.coordinator.period" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - unit: *unitDuration - defaultValues: - - fromVersion: "0.0.0" - value: "PT20S" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorAsOverlordEnabled - propertyNames: - - name: "druid.coordinator.asOverlord.enabled" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "bool" - defaultValues: - - fromVersion: "0.0.0" - value: "true" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - - - property: &coordinatorAsOverlordOverlordService - propertyNames: - - name: "druid.coordinator.asOverlord.overlordService" - kind: - type: "file" - file: "runtime.properties" - datatype: - type: "string" - defaultValues: - - fromVersion: "0.0.0" - value: "druid/overlord" - roles: - - name: "broker" - required: false - - name: "coordinator" - required: true - - name: "historical" - required: false - - name: "middlemanager" - required: false - - name: "router" - required: false - asOfVersion: "0.0.0" - -################################################################################################### -# jvm.config -################################################################################################### diff --git a/tests/templates/kuttl/smoke/52-assert.yaml.j2 b/tests/templates/kuttl/smoke/52-assert.yaml.j2 index 52e24ac3..14ac2aa0 100644 --- a/tests/templates/kuttl/smoke/52-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/52-assert.yaml.j2 @@ -43,6 +43,46 @@ spec: spec: serviceAccount: druid-serviceaccount serviceAccountName: druid-serviceaccount +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + # The Vector agent sidecar (added by the v2 `vector_container`). kuttl matches list + # elements positionally, so the first (druid) container is named only to align indices. + containers: + - name: druid + - name: vector + env: + - name: CLUSTER_NAME + value: druid + - name: DATA_DIR + value: /stackable/log/_vector-state + - name: LOG_DIR + value: /stackable/log + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ROLE_GROUP_NAME + value: default + - name: ROLE_NAME + value: broker + - name: VECTOR_AGGREGATOR_ADDRESS + valueFrom: + configMapKeyRef: + key: ADDRESS + name: vector-aggregator-discovery + - name: VECTOR_CONFIG_YAML + value: /stackable/config/vector.yaml + - name: VECTOR_FILE_LOG_LEVEL + value: info + - name: VECTOR_LOG + value: info + volumeMounts: + - mountPath: /stackable/config/vector.yaml + name: config + readOnly: true + subPath: vector.yaml + - mountPath: /stackable/log + name: log +{% endif %} terminationGracePeriodSeconds: 1 status: readyReplicas: 1 @@ -78,6 +118,46 @@ spec: spec: serviceAccount: druid-serviceaccount serviceAccountName: druid-serviceaccount +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + # The Vector agent sidecar (added by the v2 `vector_container`). kuttl matches list + # elements positionally, so the first (druid) container is named only to align indices. + containers: + - name: druid + - name: vector + env: + - name: CLUSTER_NAME + value: druid + - name: DATA_DIR + value: /stackable/log/_vector-state + - name: LOG_DIR + value: /stackable/log + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ROLE_GROUP_NAME + value: default + - name: ROLE_NAME + value: coordinator + - name: VECTOR_AGGREGATOR_ADDRESS + valueFrom: + configMapKeyRef: + key: ADDRESS + name: vector-aggregator-discovery + - name: VECTOR_CONFIG_YAML + value: /stackable/config/vector.yaml + - name: VECTOR_FILE_LOG_LEVEL + value: info + - name: VECTOR_LOG + value: info + volumeMounts: + - mountPath: /stackable/config/vector.yaml + name: config + readOnly: true + subPath: vector.yaml + - mountPath: /stackable/log + name: log +{% endif %} terminationGracePeriodSeconds: 1 status: readyReplicas: 1 @@ -113,6 +193,46 @@ spec: spec: serviceAccount: druid-serviceaccount serviceAccountName: druid-serviceaccount +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + # The Vector agent sidecar (added by the v2 `vector_container`). kuttl matches list + # elements positionally, so the first (druid) container is named only to align indices. + containers: + - name: druid + - name: vector + env: + - name: CLUSTER_NAME + value: druid + - name: DATA_DIR + value: /stackable/log/_vector-state + - name: LOG_DIR + value: /stackable/log + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ROLE_GROUP_NAME + value: default + - name: ROLE_NAME + value: historical + - name: VECTOR_AGGREGATOR_ADDRESS + valueFrom: + configMapKeyRef: + key: ADDRESS + name: vector-aggregator-discovery + - name: VECTOR_CONFIG_YAML + value: /stackable/config/vector.yaml + - name: VECTOR_FILE_LOG_LEVEL + value: info + - name: VECTOR_LOG + value: info + volumeMounts: + - mountPath: /stackable/config/vector.yaml + name: config + readOnly: true + subPath: vector.yaml + - mountPath: /stackable/log + name: log +{% endif %} terminationGracePeriodSeconds: 1 volumes: - name: tls-mount @@ -176,6 +296,46 @@ spec: spec: serviceAccount: druid-serviceaccount serviceAccountName: druid-serviceaccount +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + # The Vector agent sidecar (added by the v2 `vector_container`). kuttl matches list + # elements positionally, so the first (druid) container is named only to align indices. + containers: + - name: druid + - name: vector + env: + - name: CLUSTER_NAME + value: druid + - name: DATA_DIR + value: /stackable/log/_vector-state + - name: LOG_DIR + value: /stackable/log + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ROLE_GROUP_NAME + value: default + - name: ROLE_NAME + value: middlemanager + - name: VECTOR_AGGREGATOR_ADDRESS + valueFrom: + configMapKeyRef: + key: ADDRESS + name: vector-aggregator-discovery + - name: VECTOR_CONFIG_YAML + value: /stackable/config/vector.yaml + - name: VECTOR_FILE_LOG_LEVEL + value: info + - name: VECTOR_LOG + value: info + volumeMounts: + - mountPath: /stackable/config/vector.yaml + name: config + readOnly: true + subPath: vector.yaml + - mountPath: /stackable/log + name: log +{% endif %} terminationGracePeriodSeconds: 1 status: readyReplicas: 1 @@ -211,6 +371,46 @@ spec: spec: serviceAccount: druid-serviceaccount serviceAccountName: druid-serviceaccount +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + # The Vector agent sidecar (added by the v2 `vector_container`). kuttl matches list + # elements positionally, so the first (druid) container is named only to align indices. + containers: + - name: druid + - name: vector + env: + - name: CLUSTER_NAME + value: druid + - name: DATA_DIR + value: /stackable/log/_vector-state + - name: LOG_DIR + value: /stackable/log + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ROLE_GROUP_NAME + value: default + - name: ROLE_NAME + value: router + - name: VECTOR_AGGREGATOR_ADDRESS + valueFrom: + configMapKeyRef: + key: ADDRESS + name: vector-aggregator-discovery + - name: VECTOR_CONFIG_YAML + value: /stackable/config/vector.yaml + - name: VECTOR_FILE_LOG_LEVEL + value: info + - name: VECTOR_LOG + value: info + volumeMounts: + - mountPath: /stackable/config/vector.yaml + name: config + readOnly: true + subPath: vector.yaml + - mountPath: /stackable/log + name: log +{% endif %} terminationGracePeriodSeconds: 1 status: readyReplicas: 1 diff --git a/tests/templates/kuttl/smoke/53-assert.yaml.j2 b/tests/templates/kuttl/smoke/53-assert.yaml.j2 index 53c7b9f8..55a89f96 100644 --- a/tests/templates/kuttl/smoke/53-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/53-assert.yaml.j2 @@ -1,210 +1,33 @@ {% set vector_enabled = lookup('env', 'VECTOR_AGGREGATOR') | length > 0 %} {% set vector_yaml -%} -data_dir: /stackable/log/_vector-state +--- +data_dir: ${DATA_DIR} log_schema: host_key: pod sources: + # Reads the internal Vector logs vector: type: internal_logs files_stdout: type: file include: - - /stackable/log/*/*.stdout.log + - ${LOG_DIR}/*/*.stdout.log files_stderr: type: file include: - - /stackable/log/*/*.stderr.log - - files_log4j: - type: file - include: - - /stackable/log/*/*.log4j.xml - line_delimiter: "\r\n" - multiline: - mode: halt_before - start_pattern: ^" + raw_message + "" - parsed_event, err = parse_xml(wrapped_xml_event) - if err != null { - error = "XML not parsable: " + err - .errors = push(.errors, error) - log(error, level: "warn") - .message = raw_message - } else { - root = object!(parsed_event.root) - if !is_object(root.event) { - error = "Parsed event contains no \"event\" tag." - .errors = push(.errors, error) - log(error, level: "warn") - .message = raw_message - } else { - if keys(root) != ["event"] { - .errors = push(.errors, "Parsed event contains multiple tags: " + join!(keys(root), ", ")) - } - event = object!(root.event) - - epoch_milliseconds, err = to_int(event.@timestamp) - if err == null && epoch_milliseconds != 0 { - converted_timestamp, err = from_unix_timestamp(epoch_milliseconds, "milliseconds") - if err == null { - .timestamp = converted_timestamp - } else { - .errors = push(.errors, "Time not parsable, using current time instead: " + err) - } - } else { - .errors = push(.errors, "Timestamp not found, using current time instead.") - } - - .logger, err = string(event.@logger) - if err != null || is_empty(.logger) { - .errors = push(.errors, "Logger not found.") - } - - level, err = string(event.@level) - if err != null { - .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") - } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], level) { - .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") - } else { - .level = level - } - - message, err = string(event.message) - if err != null || is_empty(message) { - .errors = push(.errors, "Message not found.") - } - throwable = string(event.throwable) ?? "" - .message = join!(compact([message, throwable]), "\n") - } - } - processed_files_log4j2: inputs: - files_log4j2 @@ -421,136 +175,7 @@ transforms: } } - processed_files_py: - inputs: - - files_py - type: remap - source: | - raw_message = string!(.message) - - .timestamp = now() - .logger = "" - .level = "INFO" - .message = "" - .errors = [] - - parsed_event, err = parse_json(raw_message) - if err != null { - error = "JSON not parsable: " + err - .errors = push(.errors, error) - log(error, level: "warn") - .message = raw_message - } else if !is_object(parsed_event) { - error = "Parsed event is not a JSON object." - .errors = push(.errors, error) - log(error, level: "warn") - .message = raw_message - } else { - event = object!(parsed_event) - - asctime, err = string(event.asctime) - if err == null { - parsed_timestamp, err = parse_timestamp(asctime, "%F %T,%3f") - if err == null { - .timestamp = parsed_timestamp - } else { - .errors = push(.errors, "Timestamp not parsable, using current time instead: "+ err) - } - } else { - .errors = push(.errors, "Timestamp not found, using current time instead.") - } - - .logger, err = string(event.name) - if err != null || is_empty(.logger) { - .errors = push(.errors, "Logger not found.") - } - - level, err = string(event.levelname) - if err != null { - .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") - } else if level == "DEBUG" { - .level = "DEBUG" - } else if level == "INFO" { - .level = "INFO" - } else if level == "WARNING" { - .level = "WARN" - } else if level == "ERROR" { - .level = "ERROR" - } else if level == "CRITICAL" { - .level = "FATAL" - } else { - .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") - } - - .message, err = string(event.message) - if err != null || is_empty(.message) { - .errors = push(.errors, "Message not found.") - } - } - - processed_files_airlift: - inputs: - - files_airlift - type: remap - source: | - raw_message = string!(.message) - - .timestamp = now() - .logger = "" - .level = "INFO" - .message = "" - .errors = [] - - parsed_event, err = parse_json(raw_message) - if err != null { - error = "JSON not parsable: " + err - .errors = push(.errors, error) - log(error, level: "warn") - .message = raw_message - } else if !is_object(parsed_event) { - error = "Parsed event is not a JSON object." - .errors = push(.errors, error) - log(error, level: "warn") - .message = raw_message - } else { - event = object!(parsed_event) - - timestamp_string, err = string(event.timestamp) - if err == null { - parsed_timestamp, err = parse_timestamp(timestamp_string, "%Y-%m-%dT%H:%M:%S.%fZ") - if err == null { - .timestamp = parsed_timestamp - } else { - .errors = push(.errors, "Timestamp not parsable, using current time instead: " + err) - } - } else { - .errors = push(.errors, "Timestamp not found, using current time instead.") - } - - .logger, err = string(event.logger) - if err != null || is_empty(.logger) { - .errors = push(.errors, "Logger not found.") - } - - level, err = string(event.level) - if err != null { - .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") - } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], level) { - .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") - } else { - .level = level - } - - .thread = string(parsed_event.thread) ?? null - - .message, err = string(event.message) - if err != null || is_empty(.message) { - .errors = push(.errors, "Message not found.") - } - stacktrace = string(event.stackTrace) ?? "" - .message = join!(compact([.message, stacktrace]), "\n\n") - } - + # Extends the processed files with the fields "container" and "file" extended_logs_files: inputs: - processed_files_* @@ -560,14 +185,21 @@ transforms: if .errors == [] { del(.errors) } - . |= parse_regex!(.file, r'^/stackable/log/(?P.*?)/(?P.*?)$') + . |= parse_regex!(.file, r'^${LOG_DIR}/(?P.*?)/(?P.*?)$') + # Filters the logs of the Vector agent according to the defined log level filtered_logs_vector: inputs: - vector type: filter - condition: '!includes(["TRACE", "DEBUG"], .metadata.level)' - + condition: > + (.metadata.level == "TRACE" && "${VECTOR_FILE_LOG_LEVEL}" == "trace") || + (.metadata.level == "DEBUG" && includes(["trace", "debug"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "INFO" && includes(["trace", "debug", "info"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "WARN" && includes(["trace", "debug", "info", "warn"], "${VECTOR_FILE_LOG_LEVEL}")) || + (.metadata.level == "ERROR" && includes(["trace", "debug", "info", "warn", "error"], "${VECTOR_FILE_LOG_LEVEL}")) + + # Aligns the logs of the Vector agent with the common format extended_logs_vector: inputs: - filtered_logs_vector @@ -581,22 +213,24 @@ transforms: del(.pid) del(.source_type) + # Add the fields "namespace", "cluster", "role" and "roleGroup" to all logs extended_logs: inputs: - extended_logs_* type: remap source: | - .namespace = "__NAMESPACE__" - .cluster = "druid" - .role = "__ROLE__" - .roleGroup = "default" + .namespace = "${NAMESPACE}" + .cluster = "${CLUSTER_NAME}" + .role = "${ROLE_NAME}" + .roleGroup = "${ROLE_GROUP_NAME}" sinks: + # Forward the logs to the Vector aggregator aggregator: inputs: - extended_logs type: vector - address: $VECTOR_AGGREGATOR_ADDRESS + address: ${VECTOR_AGGREGATOR_ADDRESS} {% endset %} --- # Snapshot the full `.data` of each operator-managed ConfigMap. @@ -728,7 +362,7 @@ commands: networkaddress.cache.ttl=30 {% if vector_enabled %} vector.yaml: | -{{ vector_yaml | replace("__ROLE__", "broker") | indent(8, true) }} +{{ vector_yaml | indent(8, true) }} {% endif %} YAMLEOF ) @@ -836,7 +470,7 @@ commands: networkaddress.cache.ttl=30 {% if vector_enabled %} vector.yaml: | -{{ vector_yaml | replace("__ROLE__", "coordinator") | indent(8, true) }} +{{ vector_yaml | indent(8, true) }} {% endif %} YAMLEOF ) @@ -944,7 +578,7 @@ commands: networkaddress.cache.ttl=30 {% if vector_enabled %} vector.yaml: | -{{ vector_yaml | replace("__ROLE__", "historical") | indent(8, true) }} +{{ vector_yaml | indent(8, true) }} {% endif %} YAMLEOF ) @@ -1049,7 +683,7 @@ commands: networkaddress.cache.ttl=30 {% if vector_enabled %} vector.yaml: | -{{ vector_yaml | replace("__ROLE__", "middlemanager") | indent(8, true) }} +{{ vector_yaml | indent(8, true) }} {% endif %} YAMLEOF ) @@ -1152,7 +786,7 @@ commands: networkaddress.cache.ttl=30 {% if vector_enabled %} vector.yaml: | -{{ vector_yaml | replace("__ROLE__", "router") | indent(8, true) }} +{{ vector_yaml | indent(8, true) }} {% endif %} YAMLEOF )