diff --git a/.env.example b/.env.example index eb0a2a9..754be54 100644 --- a/.env.example +++ b/.env.example @@ -1,19 +1,11 @@ -# Storage Configuration -STORAGE_DATA_DIR=./.storage-data -STORAGE_QUOTA_GB=100 -STORAGE_DISCOVERY_PORT=8089 -STORAGE_MAX_PEERS=50 - -# Network Discoverability Configuration -# NAT traversal method: any, none, upnp, pmp, or extip: -STORAGE_NAT=any - -# Listen addresses (multi-address format, comma-separated) -# Default listens on all interfaces with random TCP port -STORAGE_LISTEN_ADDRS=/ip4/0.0.0.0/tcp/0 - -# Bootstrap nodes - comma-separated SPR URIs -STORAGE_BOOTSTRAP_NODES= +# Kubo IPFS Configuration +KUBO_API_URL=http://127.0.0.1:5001 +KUBO_API_TIMEOUT_SECS=300 +KUBO_PIN_ON_UPLOAD=true + +# Kubo API Authentication (optional, for remote nodes with Basic Auth) +KUBO_API_USERNAME= +KUBO_API_PASSWORD= # Database Paths WHOSONFIRST_DB_PATH=./assets/whosonfirst-data-admin-latest.db diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..8dfcbb2 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,96 @@ +name: CI + +on: + pull_request: + branches: + - "*" + +# Cancel in-progress runs when a new commit is pushed +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write + statuses: write + +jobs: + setup: + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + cache-hit: ${{ steps.cache-deps.outputs.cache-hit }} + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Set up Rust using mise + uses: jdx/mise-action@v2 + with: + cache: true + + - name: Cache dependencies + id: cache-deps + uses: actions/cache@v4 + with: + path: | + ~/.cargo + ./target + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo- + + lint: + needs: setup + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Set up Rust using mise + uses: jdx/mise-action@v4 + with: + cache: true + + - name: Restore dependencies + uses: actions/cache/restore@v4 + with: + path: | + ~/.cargo + ./target + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Lint + run: | + cargo fmt -- --check + cargo clippy --all-targets -- -D warnings + cargo check --quiet 2>&1 + + test: + needs: setup + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Set up Rust using mise + uses: jdx/mise-action@v2 + with: + cache: true + + - name: Restore dependencies + uses: actions/cache/restore@v4 + with: + path: | + ~/.cargo + ./target + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Test + run: cargo test --lib --quiet diff --git a/Cargo.lock b/Cargo.lock index 51b147f..c3ff0bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - [[package]] name = "aho-corasick" version = "1.1.4" @@ -77,27 +71,26 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" name = "anynode" version = "0.1.0" dependencies = [ - "anyhow", "assert_matches", + "async-trait", "clap", - "dirs", "dotenvy", "futures", "indicatif", "mockall", "proptest", "reqwest", + "rstest", "rusqlite", "serde", "serde_json", - "storage-bindings", "tempfile", "thiserror", "tokio", "tracing", "tracing-indicatif", "tracing-subscriber", - "uuid", + "wiremock", ] [[package]] @@ -106,12 +99,33 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "assert_matches" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -130,26 +144,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "bindgen" -version = "0.72.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" -dependencies = [ - "bitflags", - "cexpr", - "clang-sys", - "itertools", - "log", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "syn", -] - [[package]] name = "bit-set" version = "0.8.0" @@ -171,15 +165,6 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - [[package]] name = "bumpalo" version = "3.20.2" @@ -192,12 +177,6 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" -[[package]] -name = "bytesize" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bd91ee7b2422bcb158d90ef4d14f75ef67f340943fc4149891dcce8f8b972a3" - [[package]] name = "cc" version = "1.2.56" @@ -208,15 +187,6 @@ dependencies = [ "shlex", ] -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom", -] - [[package]] name = "cfg-if" version = "1.0.4" @@ -229,17 +199,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" -[[package]] -name = "clang-sys" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" -dependencies = [ - "glob", - "libc", - "libloading", -] - [[package]] name = "clap" version = "4.5.60" @@ -300,89 +259,22 @@ dependencies = [ ] [[package]] -name = "core-foundation" -version = "0.9.4" +name = "deadpool" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - -[[package]] -name = "crc32fast" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "crypto-common" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", -] - -[[package]] -name = "dirs" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" -dependencies = [ - "dirs-sys", + "deadpool-runtime", + "lazy_static", + "num_cpus", + "tokio", ] [[package]] -name = "dirs-sys" -version = "0.5.0" +name = "deadpool-runtime" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" -dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.61.2", -] +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" [[package]] name = "displaydoc" @@ -407,27 +299,12 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" -[[package]] -name = "either" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" - [[package]] name = "encode_unicode" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" -[[package]] -name = "encoding_rs" -version = "0.8.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" -dependencies = [ - "cfg-if", -] - [[package]] name = "equivalent" version = "1.0.2" @@ -462,33 +339,12 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" -[[package]] -name = "filetime" -version = "0.2.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" -dependencies = [ - "cfg-if", - "libc", - "libredox", -] - [[package]] name = "find-msvc-tools" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" -[[package]] -name = "flate2" -version = "1.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" -dependencies = [ - "crc32fast", - "miniz_oxide", -] - [[package]] name = "fnv" version = "1.0.7" @@ -507,21 +363,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -608,6 +449,12 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +[[package]] +name = "futures-timer" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" + [[package]] name = "futures-util" version = "0.3.32" @@ -625,16 +472,6 @@ dependencies = [ "slab", ] -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "getrandom" version = "0.2.17" @@ -683,9 +520,9 @@ checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "h2" -version = "0.4.13" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" dependencies = [ "atomic-waker", "bytes", @@ -734,10 +571,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] -name = "hex" -version = "0.4.3" +name = "hermit-abi" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] name = "http" @@ -778,6 +615,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "hyper" version = "1.8.1" @@ -792,6 +635,7 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "pin-utils", @@ -817,22 +661,6 @@ dependencies = [ "webpki-roots", ] -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - [[package]] name = "hyper-util" version = "0.1.20" @@ -851,11 +679,9 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2", - "system-configuration", "tokio", "tower-service", "tracing", - "windows-registry", ] [[package]] @@ -1014,15 +840,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.17" @@ -1057,27 +874,6 @@ version = "0.2.182" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" -[[package]] -name = "libloading" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" -dependencies = [ - "cfg-if", - "windows-link", -] - -[[package]] -name = "libredox" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" -dependencies = [ - "bitflags", - "libc", - "redox_syscall 0.7.1", -] - [[package]] name = "libsqlite3-sys" version = "0.36.0" @@ -1144,19 +940,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "miniz_oxide" -version = "0.8.9" +name = "mime_guess" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" dependencies = [ - "adler2", - "simd-adler32", + "mime", + "unicase", ] [[package]] @@ -1196,33 +986,6 @@ dependencies = [ "syn", ] -[[package]] -name = "native-tls" -version = "0.2.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1242,66 +1005,26 @@ dependencies = [ ] [[package]] -name = "once_cell" -version = "1.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" - -[[package]] -name = "once_cell_polyfill" -version = "1.70.2" +name = "num_cpus" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" - -[[package]] -name = "openssl" -version = "0.10.75" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", + "hermit-abi", "libc", - "once_cell", - "openssl-macros", - "openssl-sys", ] [[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "openssl-probe" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" - -[[package]] -name = "openssl-sys" -version = "0.9.111" +name = "once_cell" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] -name = "option-ext" -version = "0.2.0" +name = "once_cell_polyfill" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "parking_lot" @@ -1321,7 +1044,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.18", + "redox_syscall", "smallvec", "windows-link", ] @@ -1410,6 +1133,15 @@ dependencies = [ "syn", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -1561,31 +1293,11 @@ dependencies = [ "bitflags", ] -[[package]] -name = "redox_syscall" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35985aa610addc02e24fc232012c86fd11f14111180f902b67e2d5331f8ebf2b" -dependencies = [ - "bitflags", -] - -[[package]] -name = "redox_users" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" -dependencies = [ - "getrandom 0.2.17", - "libredox", - "thiserror", -] - [[package]] name = "regex" -version = "1.12.3" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ "aho-corasick", "memchr", @@ -1606,9 +1318,15 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.9" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "relative-path" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" @@ -1618,22 +1336,17 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64", "bytes", - "encoding_rs", - "futures-channel", "futures-core", "futures-util", - "h2", "http", "http-body", "http-body-util", "hyper", "hyper-rustls", - "hyper-tls", "hyper-util", "js-sys", "log", - "mime", - "native-tls", + "mime_guess", "percent-encoding", "pin-project-lite", "quinn", @@ -1644,7 +1357,6 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-native-tls", "tokio-rustls", "tokio-util", "tower", @@ -1682,6 +1394,36 @@ dependencies = [ "thiserror", ] +[[package]] +name = "rstest" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fc39292f8613e913f7df8fa892b8944ceb47c247b78e1b1ae2f09e019be789d" +dependencies = [ + "futures-timer", + "futures-util", + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f168d99749d307be9de54d23fd226628d99768225ef08f6ffb52e0182a27746" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn", + "unicode-ident", +] + [[package]] name = "rusqlite" version = "0.38.0" @@ -1703,6 +1445,15 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "1.1.3" @@ -1775,44 +1526,12 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" -[[package]] -name = "schannel" -version = "0.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" -dependencies = [ - "windows-sys 0.61.2", -] - [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "security-framework" -version = "3.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" -dependencies = [ - "bitflags", - "core-foundation 0.10.1", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "semver" version = "1.0.27" @@ -1874,17 +1593,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sharded-slab" version = "0.1.7" @@ -1910,12 +1618,6 @@ dependencies = [ "libc", ] -[[package]] -name = "simd-adler32" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" - [[package]] name = "slab" version = "0.4.12" @@ -1956,29 +1658,6 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" -[[package]] -name = "storage-bindings" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c860b5a3b9c2ab7b32f752a13c92051740b9a18b441edfca2b491a32eb02440" -dependencies = [ - "bindgen", - "bytesize", - "cc", - "dirs", - "flate2", - "futures", - "hex", - "libc", - "reqwest", - "serde", - "serde_json", - "sha2", - "tar", - "thiserror", - "tokio", -] - [[package]] name = "strsim" version = "0.11.1" @@ -2022,38 +1701,6 @@ dependencies = [ "syn", ] -[[package]] -name = "system-configuration" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" -dependencies = [ - "bitflags", - "core-foundation 0.9.4", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "tar" -version = "0.4.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" -dependencies = [ - "filetime", - "libc", - "xattr", -] - [[package]] name = "tempfile" version = "3.25.0" @@ -2155,16 +1802,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.4" @@ -2188,6 +1825,36 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + [[package]] name = "tower" version = "0.5.3" @@ -2312,18 +1979,18 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "typenum" -version = "1.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" - [[package]] name = "unarray" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -2378,17 +2045,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "uuid" -version = "1.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb" -dependencies = [ - "getrandom 0.4.1", - "js-sys", - "wasm-bindgen", -] - [[package]] name = "valuable" version = "0.1.1" @@ -2401,12 +2057,6 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - [[package]] name = "vt100" version = "0.16.2" @@ -2611,35 +2261,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-registry" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" -dependencies = [ - "windows-link", - "windows-result", - "windows-strings", -] - -[[package]] -name = "windows-result" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" -dependencies = [ - "windows-link", -] - [[package]] name = "windows-sys" version = "0.52.0" @@ -2796,6 +2417,38 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "winnow" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" +dependencies = [ + "memchr", +] + +[[package]] +name = "wiremock" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" +dependencies = [ + "assert-json-diff", + "base64", + "deadpool", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "once_cell", + "regex", + "serde", + "serde_json", + "tokio", + "url", +] + [[package]] name = "wit-bindgen" version = "0.51.0" @@ -2890,16 +2543,6 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" -[[package]] -name = "xattr" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" -dependencies = [ - "libc", - "rustix", -] - [[package]] name = "yoke" version = "0.8.1" diff --git a/Cargo.toml b/Cargo.toml index e16b725..516ff16 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,11 +2,11 @@ name = "anynode" version = "0.1.0" edition = "2021" -description = "CLI tool for extracting PMTiles map data and uploading to Logos Storage" +description = "CLI tool for extracting PMTiles map data and uploading to Kubo (IPFS)" license = "GPL-3.0-or-later" [dependencies] -storage-bindings = "0.2" +async-trait = "0.1" rusqlite = { version = "0.38", features = ["bundled"] } tokio = { version = "1", features = ["full"] } serde = { version = "1", features = ["derive"] } @@ -16,11 +16,8 @@ dotenvy = "0.15" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } thiserror = "2" -anyhow = "1" futures = "0.3" -reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "stream"] } -uuid = { version = "1", features = ["v4"] } -dirs = "6" +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "stream", "multipart"] } indicatif = "0.18" tracing-indicatif = "0.3" @@ -29,3 +26,25 @@ mockall = "0.14" proptest = "1.10" tempfile = "3.25" assert_matches = "1.5" +wiremock = "0.6" +rstest = "0.25" +tokio = { version = "1", features = ["test-util"] } + +[lints.clippy] +# Prevent panics +unwrap_used = "warn" +expect_used = "warn" +panic = "warn" +todo = "warn" +unimplemented = "warn" + +# Don't fail silently +let_underscore_must_use = "warn" +unused_result_ok = "warn" + +# Async safety +await_holding_lock = "warn" +large_futures = "warn" + +# Hygiene +dbg_macro = "warn" diff --git a/clippy.toml b/clippy.toml new file mode 100644 index 0000000..76f91ce --- /dev/null +++ b/clippy.toml @@ -0,0 +1,4 @@ +allow-unwrap-in-tests = true +allow-expect-in-tests = true +allow-dbg-in-tests = true +allow-indexing-slicing-in-tests = true diff --git a/lefthook.yml b/lefthook.yml new file mode 100644 index 0000000..e606a42 --- /dev/null +++ b/lefthook.yml @@ -0,0 +1,20 @@ +# Pre-commit: Fast checks (< 5s) +pre-commit: + parallel: true + commands: + fmt: + glob: "*.rs" + run: cargo fmt -- --check + check: + glob: "*.rs" + run: cargo check --quiet 2>&1 + +# Pre-push: Thorough checks (< 30s) +pre-push: + commands: + clippy: + glob: "*.rs" + run: cargo clippy --all-targets -- -D warnings + test: + glob: "*.rs" + run: cargo test --lib --quiet diff --git a/mise.toml b/mise.toml index 3ed6ff4..db5f4c4 100644 --- a/mise.toml +++ b/mise.toml @@ -1,2 +1,3 @@ [tools] -rust = "1.93" +lefthook = "2.1.9" +rust = { version = "1.93.1", profile = "default" } diff --git a/src/app/mod.rs b/src/app/mod.rs index dcabdc2..3f337e6 100644 --- a/src/app/mod.rs +++ b/src/app/mod.rs @@ -1,4 +1,3 @@ -pub mod monitor; pub mod runner; use thiserror::Error; @@ -11,8 +10,8 @@ pub enum ApplicationError { ExtractionError(#[from] crate::services::ExtractionError), #[error("Upload error: {0}")] UploadError(#[from] crate::services::AreaUploadError), - #[error("Storage error: {0}")] - StorageError(#[from] crate::services::StorageError), + #[error("Kubo service error: {0}")] + KuboServiceError(#[from] crate::services::KuboServiceError), #[error("IO error: {0}")] IoError(#[from] std::io::Error), } diff --git a/src/app/monitor.rs b/src/app/monitor.rs deleted file mode 100644 index bb00792..0000000 --- a/src/app/monitor.rs +++ /dev/null @@ -1,53 +0,0 @@ -use crate::services::{StorageService, StorageStatus}; -use indicatif::{ProgressBar, ProgressStyle}; -use std::sync::Arc; -use std::time::Duration; -use tokio::time::interval; - -pub fn create_node_status_progress_bar() -> ProgressBar { - let pb = ProgressBar::new_spinner(); - pb.set_style( - ProgressStyle::with_template("{spinner:.green} {msg}") - .unwrap() - .tick_strings(&["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]), - ); - pb.enable_steady_tick(Duration::from_millis(100)); - pb -} - -pub async fn monitor_node_status( - storage_service: Arc, - progress_bar: ProgressBar, -) { - let mut tick = interval(Duration::from_secs(2)); - - loop { - tick.tick().await; - - let status = storage_service.get_status().await; - - match storage_service.get_node_info().await { - Ok(node_info) => { - let status_str = format_status(&status); - progress_bar.set_message(format!( - "Status: {} | Discovery: {} nodes", - status_str, node_info.discovery_node_count - )); - } - Err(_) => { - let status_str = format_status(&status); - progress_bar.set_message(format!("Status: {}", status_str)); - } - } - } -} - -pub fn format_status(status: &StorageStatus) -> &'static str { - match status { - StorageStatus::Disconnected => "Disconnected", - StorageStatus::Initialized => "Initialized", - StorageStatus::Connecting => "Connecting", - StorageStatus::Connected => "Connected", - StorageStatus::Error => "Error", - } -} diff --git a/src/app/runner.rs b/src/app/runner.rs index 5b92a1b..f791d68 100644 --- a/src/app/runner.rs +++ b/src/app/runner.rs @@ -1,9 +1,6 @@ -use crate::app::monitor::{create_node_status_progress_bar, monitor_node_status}; use crate::config::Config; use crate::initialization::print_final_stats; -use crate::services::{ - AreaUploadService, CountryService, ExtractionService, StorageService, -}; +use crate::services::{AreaUploadService, CountryService, ExtractionService, KuboService}; use std::sync::Arc; use tracing::{error, info, warn}; @@ -11,7 +8,7 @@ use super::ApplicationResult; pub struct NodeRunner { config: Arc, - storage_service: Arc, + kubo_service: Arc, extraction_service: ExtractionService, upload_service: AreaUploadService, country_service: CountryService, @@ -22,7 +19,7 @@ pub struct NodeRunner { impl NodeRunner { pub fn new( config: Arc, - storage_service: Arc, + kubo_service: Arc, extraction_service: ExtractionService, upload_service: AreaUploadService, country_service: CountryService, @@ -31,7 +28,7 @@ impl NodeRunner { ) -> Self { Self { config, - storage_service, + kubo_service, extraction_service, upload_service, country_service, @@ -41,9 +38,9 @@ impl NodeRunner { } pub async fn run(&self) -> ApplicationResult<()> { - info!("Starting storage node..."); - self.storage_service.start_node().await?; - info!("Storage node started successfully"); + info!("Connecting to Kubo API..."); + self.kubo_service.check_alive().await?; + info!("Connected to Kubo API successfully"); if !self.skip_extract { info!("Extracting PMTiles from planet file..."); @@ -77,63 +74,35 @@ impl NodeRunner { let stats = self.upload_service.get_stats().await; print_final_stats(&stats); - self.display_node_info().await; - Ok(()) } - async fn display_node_info(&self) { - match self.storage_service.get_node_info().await { + pub async fn display_summary(&self) { + info!("=== AnyNode Summary ==="); + + match self.kubo_service.get_node_info().await { Ok(node_info) => { - info!("Storage node is now running and serving files to the network..."); - if let Some(peer_id) = node_info.peer_id { - info!("Peer ID: {}", peer_id); + info!("Kubo Node: {}", node_info.peer_id); + info!("Kubo Version: {}", node_info.version); + if let Some(agent) = &node_info.agent_version { + info!("Agent: {}", agent); } + info!("Peers Connected: {}", node_info.peers_connected); if !node_info.addresses.is_empty() { - info!("Node Addresses:"); + info!("Listen Addresses:"); for addr in &node_info.addresses { - info!(" {}", addr); - } - } - if !node_info.announce_addresses.is_empty() { - info!("Announce Addresses:"); - for addr in &node_info.announce_addresses { - info!(" {}", addr); + info!(" - {}", addr); } } - if let Some(spr) = node_info.spr { - info!("Signed Peer Record:\n {}", spr); - } - info!("Discovery table nodes: {}", node_info.discovery_node_count); - if node_info.discovery_node_count > 0 { - info!("Successfully connected to the network via bootstrap nodes"); - } else { - warn!("No peers in discovery table - bootstrap may have failed"); - } - if let Some(version) = node_info.version { - info!("Storage version: {}", version); - } } Err(e) => { - info!("Storage node is now running and serving files to the network..."); - warn!("Failed to get node info: {}", e); + warn!("Failed to get node info for summary: {}", e); } } - } - - pub fn start_monitoring(&self) -> tokio::task::JoinHandle<()> { - let progress_bar = create_node_status_progress_bar(); - let storage_service = self.storage_service.clone(); - - tokio::spawn(async move { - monitor_node_status(storage_service, progress_bar).await; - }) - } - pub async fn shutdown(&self) -> Result<(), crate::services::StorageError> { - info!("Stopping storage node..."); - self.storage_service.stop_node().await?; - info!("Storage node stopped successfully"); - Ok(()) + let stats = self.upload_service.get_stats().await; + info!("Areas Uploaded: {}", stats.total_uploaded); + info!("Areas Failed: {}", stats.total_failed); + info!("Total Bytes Uploaded: {}", stats.total_bytes_uploaded); } } diff --git a/src/cli.rs b/src/cli.rs index ecbfa5d..9226807 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -16,19 +16,6 @@ pub struct Cli { #[arg(long, help = "Skip extracting PMTiles from planet files")] pub no_extract: bool, - #[arg( - long, - help = "Port for the Storage node (overrides STORAGE_DISCOVERY_PORT env var)" - )] - pub port: Option, - - #[arg( - long, - value_name = "DIR", - help = "Data directory for Storage node (overrides STORAGE_DATA_DIR env var)" - )] - pub data_dir: Option, - #[arg( short, long, @@ -37,33 +24,18 @@ pub struct Cli { )] pub config: Option, + #[arg(long, help = "Override Kubo RPC API URL")] + pub kubo_api_url: Option, + + #[arg(long, help = "Skip pinning content after upload")] + pub no_pin: bool, + #[arg(short, long, help = "Verbose output")] pub verbose: bool, #[arg(short, long, help = "Quiet mode (minimal output)")] pub quiet: bool, - #[arg( - long, - value_name = "SPR_URI", - help = "Bootstrap node SPR URI (can be repeated for multiple nodes)" - )] - pub bootstrap: Vec, - - #[arg( - long, - value_name = "METHOD", - help = "NAT traversal method: any, none, upnp, pmp, or extip: (overrides STORAGE_NAT env var)" - )] - pub nat: Option, - - #[arg( - long, - value_name = "ADDRS", - help = "Listen addresses (comma-separated multi-addresses, overrides STORAGE_LISTEN_ADDRS env var)" - )] - pub listen_addrs: Option, - #[arg( long, value_name = "IDS", @@ -77,14 +49,6 @@ impl Cli { Self::parse() } - pub fn get_port(&self, env_port: Option) -> Option { - self.port.or(env_port) - } - - pub fn get_data_dir(&self, env_dir: Option) -> Option { - self.data_dir.clone().or(env_dir) - } - pub fn is_non_interactive(&self) -> bool { self.non_interactive } @@ -107,30 +71,6 @@ impl Cli { } } - pub fn get_bootstrap_nodes(&self, env_nodes: Vec) -> Vec { - if !self.bootstrap.is_empty() { - self.bootstrap.clone() - } else { - env_nodes - } - } - - pub fn get_nat(&self, env_nat: String) -> String { - self.nat.clone().unwrap_or(env_nat) - } - - pub fn get_listen_addrs(&self, env_addrs: Vec) -> Vec { - if let Some(addrs) = &self.listen_addrs { - addrs - .split(',') - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) - .collect() - } else { - env_addrs - } - } - pub fn get_area_ids(&self, env_ids: Vec) -> Vec { if let Some(ids) = &self.area_ids { ids.split(',') diff --git a/src/config.rs b/src/config.rs index bf804fa..e5da188 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,6 +1,18 @@ -use dotenvy::dotenv; -use std::env; +use std::env::VarError; use std::path::PathBuf; +use std::time::Duration; + +pub trait Environment: Send + Sync { + fn var(&self, key: &str) -> Result; +} + +pub struct RealEnv; + +impl Environment for RealEnv { + fn var(&self, key: &str) -> Result { + std::env::var(key) + } +} #[derive(Debug)] pub enum ConfigError { @@ -11,7 +23,9 @@ pub enum ConfigError { impl std::fmt::Display for ConfigError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - ConfigError::MissingEnvVar(var) => write!(f, "Missing required environment variable: {}", var), + ConfigError::MissingEnvVar(var) => { + write!(f, "Missing required environment variable: {}", var) + } ConfigError::InvalidValue(msg) => write!(f, "Invalid configuration value: {}", msg), } } @@ -21,14 +35,11 @@ impl std::error::Error for ConfigError {} #[derive(Clone, Debug)] pub struct Config { - pub storage_data_dir: PathBuf, - pub storage_quota: u64, - pub discovery_port: u16, - pub max_peers: u32, - pub bootstrap_nodes: Vec, // TODO: Add a type for SPR URIs, with proper parsing - - pub nat: String, // TODO: properly type this - pub listen_addrs: Vec, // TODO: Add a type for those URIs as well, with proper parsing + pub kubo_api_url: String, + pub kubo_api_timeout: Duration, + pub pin_on_upload: bool, + pub kubo_api_username: Option, + pub kubo_api_password: Option, pub whosonfirst_db_path: PathBuf, pub cid_db_path: PathBuf, @@ -48,59 +59,65 @@ pub struct Config { impl Config { pub fn from_env() -> Result { - dotenv().ok(); + Self::from_env_with(&RealEnv) + } - let storage_data_dir = PathBuf::from( - env::var("STORAGE_DATA_DIR") - .map_err(|_| ConfigError::MissingEnvVar("STORAGE_DATA_DIR".to_string()))?, - ); + pub fn from_env_with(env: &E) -> Result { + let kubo_api_url = env + .var("KUBO_API_URL") + .unwrap_or_else(|_| "http://127.0.0.1:5001".to_string()); - let storage_quota_gb: u64 = env::var("STORAGE_QUOTA_GB") - .map_err(|_| ConfigError::MissingEnvVar("STORAGE_QUOTA_GB".to_string()))? - .parse() - .map_err(|e| ConfigError::InvalidValue(format!("STORAGE_QUOTA_GB: {}", e)))?; - let storage_quota = storage_quota_gb * 1024 * 1024 * 1024; // Convert GB to bytes + let kubo_api_timeout_secs: u64 = env + .var("KUBO_API_TIMEOUT_SECS") + .ok() + .filter(|s| !s.is_empty()) + .and_then(|s| s.parse().ok()) + .unwrap_or(300); + let kubo_api_timeout = Duration::from_secs(kubo_api_timeout_secs); - let discovery_port: u16 = env::var("STORAGE_DISCOVERY_PORT") - .map_err(|_| ConfigError::MissingEnvVar("STORAGE_DISCOVERY_PORT".to_string()))? - .parse() - .map_err(|e| ConfigError::InvalidValue(format!("STORAGE_DISCOVERY_PORT: {}", e)))?; + let pin_on_upload = env + .var("KUBO_PIN_ON_UPLOAD") + .ok() + .filter(|s| !s.is_empty()) + .map(|s| s.parse::().unwrap_or(true)) + .unwrap_or(true); - let max_peers: u32 = env::var("STORAGE_MAX_PEERS") - .map_err(|_| ConfigError::MissingEnvVar("STORAGE_MAX_PEERS".to_string()))? - .parse() - .map_err(|e| ConfigError::InvalidValue(format!("STORAGE_MAX_PEERS: {}", e)))?; + let kubo_api_username = env.var("KUBO_API_USERNAME").ok().filter(|s| !s.is_empty()); + let kubo_api_password = env.var("KUBO_API_PASSWORD").ok().filter(|s| !s.is_empty()); let whosonfirst_db_path = PathBuf::from( - env::var("WHOSONFIRST_DB_PATH") + env.var("WHOSONFIRST_DB_PATH") .map_err(|_| ConfigError::MissingEnvVar("WHOSONFIRST_DB_PATH".to_string()))?, ); let cid_db_path = PathBuf::from( - env::var("CID_DB_PATH") + env.var("CID_DB_PATH") .map_err(|_| ConfigError::MissingEnvVar("CID_DB_PATH".to_string()))?, ); let areas_dir = PathBuf::from( - env::var("AREAS_DIR") + env.var("AREAS_DIR") .map_err(|_| ConfigError::MissingEnvVar("AREAS_DIR".to_string()))?, ); - let bzip2_cmd = env::var("BZIP2_CMD") + let bzip2_cmd = env + .var("BZIP2_CMD") .map_err(|_| ConfigError::MissingEnvVar("BZIP2_CMD".to_string()))?; - let pmtiles_cmd = env::var("PMTILES_CMD") + let pmtiles_cmd = env + .var("PMTILES_CMD") .map_err(|_| ConfigError::MissingEnvVar("PMTILES_CMD".to_string()))?; - let target_countries: Vec = env::var("TARGET_COUNTRIES") + let target_countries: Vec = env + .var("TARGET_COUNTRIES") .map_err(|_| ConfigError::MissingEnvVar("TARGET_COUNTRIES".to_string()))? .split(',') .map(|s| s.trim().to_string()) .filter(|s| !s.is_empty()) .collect(); - // Optional - comma-separated area IDs to process (overrides TARGET_COUNTRIES) - let area_ids: Vec = env::var("AREA_IDS") + let area_ids: Vec = env + .var("AREA_IDS") .ok() .filter(|s| !s.is_empty()) .map(|s| { @@ -112,45 +129,27 @@ impl Config { }) .unwrap_or_default(); - let max_concurrent_extractions: usize = env::var("MAX_CONCURRENT_EXTRACTIONS") + let max_concurrent_extractions: usize = env + .var("MAX_CONCURRENT_EXTRACTIONS") .map_err(|_| ConfigError::MissingEnvVar("MAX_CONCURRENT_EXTRACTIONS".to_string()))? .parse() .map_err(|e| ConfigError::InvalidValue(format!("MAX_CONCURRENT_EXTRACTIONS: {}", e)))?; - // Optional - empty string means None - // Can be a local file path or a remote URL (http:// or https://) - let planet_pmtiles_location = env::var("PLANET_PMTILES_LOCATION") + let planet_pmtiles_location = env + .var("PLANET_PMTILES_LOCATION") .ok() .filter(|s| !s.is_empty()); - // Optional - comma-separated SPR URIs for bootstrap nodes - let bootstrap_nodes: Vec = env::var("STORAGE_BOOTSTRAP_NODES") - .ok() - .filter(|s| !s.is_empty()) - .map(|s| s.split(',').map(|s| s.trim().to_string()).collect()) - .unwrap_or_default(); - - let nat = env::var("STORAGE_NAT") - .map_err(|_| ConfigError::MissingEnvVar("STORAGE_NAT".to_string()))?; - - let listen_addrs: Vec = env::var("STORAGE_LISTEN_ADDRS") - .map_err(|_| ConfigError::MissingEnvVar("STORAGE_LISTEN_ADDRS".to_string()))? - .split(',') - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) - .collect(); - - let whosonfirst_db_url = env::var("WHOSONFIRST_DB_URL") + let whosonfirst_db_url = env + .var("WHOSONFIRST_DB_URL") .map_err(|_| ConfigError::MissingEnvVar("WHOSONFIRST_DB_URL".to_string()))?; Ok(Self { - storage_data_dir, - storage_quota, - discovery_port, - max_peers, - bootstrap_nodes, - nat, - listen_addrs, + kubo_api_url, + kubo_api_timeout, + pin_on_upload, + kubo_api_username, + kubo_api_password, whosonfirst_db_path, cid_db_path, areas_dir, @@ -167,4 +166,173 @@ impl Config { pub fn load() -> Result { Self::from_env() } + + pub fn apply_cli_overrides(&mut self, cli: &crate::cli::Cli) { + if let Some(url) = cli.kubo_api_url.as_ref() { + self.kubo_api_url = url.clone(); + } + if cli.no_pin { + self.pin_on_upload = false; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + struct MockEnv(HashMap<&'static str, String>); + + impl MockEnv { + fn new() -> Self { + Self(HashMap::new()) + } + + fn set(mut self, k: &'static str, v: impl Into) -> Self { + self.0.insert(k, v.into()); + self + } + } + + impl Environment for MockEnv { + fn var(&self, key: &str) -> Result { + self.0.get(key).cloned().ok_or(VarError::NotPresent) + } + } + + fn minimal_env() -> MockEnv { + MockEnv::new() + .set("WHOSONFIRST_DB_PATH", "/tmp/test.db") + .set("CID_DB_PATH", "/tmp/cid.db") + .set("AREAS_DIR", "/tmp/areas") + .set("BZIP2_CMD", "bzip2") + .set("PMTILES_CMD", "pmtiles") + .set("TARGET_COUNTRIES", "") + .set("MAX_CONCURRENT_EXTRACTIONS", "4") + .set("WHOSONFIRST_DB_URL", "http://example.com/db") + } + + #[test] + fn config_defaults_kubo_url() { + let env = minimal_env(); + let config = Config::from_env_with(&env).unwrap(); + assert_eq!(config.kubo_api_url, "http://127.0.0.1:5001"); + } + + #[test] + fn config_defaults_timeout() { + let env = minimal_env(); + let config = Config::from_env_with(&env).unwrap(); + assert_eq!(config.kubo_api_timeout, Duration::from_secs(300)); + } + + #[test] + fn config_defaults_pin_on_upload() { + let env = minimal_env(); + let config = Config::from_env_with(&env).unwrap(); + assert!(config.pin_on_upload); + } + + #[test] + fn config_custom_kubo_url() { + let env = minimal_env().set("KUBO_API_URL", "http://custom:9999"); + let config = Config::from_env_with(&env).unwrap(); + assert_eq!(config.kubo_api_url, "http://custom:9999"); + } + + #[test] + fn config_custom_timeout() { + let env = minimal_env().set("KUBO_API_TIMEOUT_SECS", "600"); + let config = Config::from_env_with(&env).unwrap(); + assert_eq!(config.kubo_api_timeout, Duration::from_secs(600)); + } + + #[test] + fn config_custom_pin_false() { + let env = minimal_env().set("KUBO_PIN_ON_UPLOAD", "false"); + let config = Config::from_env_with(&env).unwrap(); + assert!(!config.pin_on_upload); + } + + #[test] + fn config_target_countries_parsing() { + let env = minimal_env().set("TARGET_COUNTRIES", "US,CA,GB"); + let config = Config::from_env_with(&env).unwrap(); + assert_eq!(config.target_countries, vec!["US", "CA", "GB"]); + } + + #[test] + fn config_target_countries_empty() { + let env = minimal_env().set("TARGET_COUNTRIES", ""); + let config = Config::from_env_with(&env).unwrap(); + assert!(config.target_countries.is_empty()); + } + + #[test] + fn config_area_ids_parsing() { + let env = minimal_env().set("AREA_IDS", "123,456,789"); + let config = Config::from_env_with(&env).unwrap(); + assert_eq!(config.area_ids, vec![123, 456, 789]); + } + + #[test] + fn config_area_ids_empty() { + let env = minimal_env().set("AREA_IDS", ""); + let config = Config::from_env_with(&env).unwrap(); + assert!(config.area_ids.is_empty()); + } + + #[test] + #[allow(clippy::panic)] + fn config_missing_required_var() { + let env = MockEnv::new(); + let result = Config::from_env_with(&env); + assert!(result.is_err()); + if let Err(ConfigError::MissingEnvVar(var)) = result { + assert_eq!(var, "WHOSONFIRST_DB_PATH"); + } else { + panic!("Expected MissingEnvVar error for WHOSONFIRST_DB_PATH"); + } + } + + #[test] + fn config_apply_cli_kubo_url() { + let env = minimal_env().set("KUBO_API_URL", "http://original:5001"); + let mut config = Config::from_env_with(&env).unwrap(); + let cli = crate::cli::Cli { + non_interactive: false, + no_download: false, + no_extract: false, + config: None, + kubo_api_url: Some("http://override:8080".to_string()), + no_pin: false, + verbose: false, + quiet: false, + area_ids: None, + }; + + config.apply_cli_overrides(&cli); + assert_eq!(config.kubo_api_url, "http://override:8080"); + } + + #[test] + fn config_apply_cli_no_pin() { + let env = minimal_env().set("KUBO_PIN_ON_UPLOAD", "true"); + let mut config = Config::from_env_with(&env).unwrap(); + let cli = crate::cli::Cli { + non_interactive: false, + no_download: false, + no_extract: false, + config: None, + kubo_api_url: None, + no_pin: true, + verbose: false, + quiet: false, + area_ids: None, + }; + + config.apply_cli_overrides(&cli); + assert!(!config.pin_on_upload); + } } diff --git a/src/initialization/database_init.rs b/src/initialization/database_init.rs index 24d4ef5..a9c37e7 100644 --- a/src/initialization/database_init.rs +++ b/src/initialization/database_init.rs @@ -3,33 +3,45 @@ use crate::services::DatabaseService; use std::sync::Arc; use tracing::info; -use super::InitializationResult; - -pub async fn initialize_whosonfirst_db(config: &Config) -> InitializationResult> { - info!("Initializing WhosOnFirst database at {:?}", config.whosonfirst_db_path); - - let db = DatabaseService::new( - config.whosonfirst_db_path.to_str().unwrap(), - false, // Don't create CID tables for WhosOnFirst DB - ) - .await?; +use super::{InitializationError, InitializationResult}; + +pub async fn initialize_whosonfirst_db( + config: &Config, +) -> InitializationResult> { + info!( + "Initializing WhosOnFirst database at {:?}", + config.whosonfirst_db_path + ); + + let db_path = config.whosonfirst_db_path.to_str().ok_or_else(|| { + InitializationError::InvalidPath(format!( + "WhosOnFirst database path: {}", + config.whosonfirst_db_path.display() + )) + })?; + let db = DatabaseService::new(db_path, false).await?; info!("WhosOnFirst database initialized successfully"); Ok(Arc::new(db)) } pub async fn initialize_cid_db(config: &Config) -> InitializationResult> { - info!("Initializing CID mappings database at {:?}", config.cid_db_path); + info!( + "Initializing CID mappings database at {:?}", + config.cid_db_path + ); if let Some(parent) = config.cid_db_path.parent() { tokio::fs::create_dir_all(parent).await?; } - let db = DatabaseService::new( - config.cid_db_path.to_str().unwrap(), - true, // Create CID tables - ) - .await?; + let cid_path = config.cid_db_path.to_str().ok_or_else(|| { + InitializationError::InvalidPath(format!( + "CID database path: {}", + config.cid_db_path.display() + )) + })?; + let db = DatabaseService::new(cid_path, true).await?; info!("CID mappings database initialized successfully"); Ok(Arc::new(db)) diff --git a/src/initialization/init.rs b/src/initialization/init.rs index ebecb72..38c98fb 100644 --- a/src/initialization/init.rs +++ b/src/initialization/init.rs @@ -1,9 +1,8 @@ use crate::config::Config; use crate::services::{ - AreaUploadService, CountryService, DatabaseService, ExtractionService, StorageService, + AreaUploadService, CountryService, DatabaseService, ExtractionService, KuboService, }; use crate::types::UploadStats; -use std::path::PathBuf; use std::sync::Arc; use tracing::info; @@ -14,41 +13,16 @@ pub fn initialize_country_service() -> CountryService { country_service } -pub async fn initialize_storage_service( +pub async fn initialize_kubo_service( config: &Config, - port_override: Option, - data_dir_override: Option, - bootstrap_nodes: Vec, - nat_override: Option, - listen_addrs_override: Option>, -) -> super::InitializationResult> { - info!("Initializing storage service"); +) -> super::InitializationResult> { + info!("Initializing Kubo service"); - let port = port_override.unwrap_or(config.discovery_port); - let data_dir = data_dir_override.unwrap_or_else(|| config.storage_data_dir.clone()); - let nat = nat_override.unwrap_or_else(|| config.nat.clone()); - let listen_addrs = listen_addrs_override.unwrap_or_else(|| config.listen_addrs.clone()); + let kubo_service = KuboService::new(config)?; + kubo_service.check_alive().await?; - if !bootstrap_nodes.is_empty() { - info!("Using {} bootstrap node(s)", bootstrap_nodes.len()); - } - - info!("Using NAT configuration: {}", nat); - info!("Using listen addresses: {:?}", listen_addrs); - - let storage_service = StorageService::new( - &data_dir, - config.storage_quota, - port, - config.max_peers, - bootstrap_nodes, - nat, - listen_addrs, - ) - .await?; - - info!("Storage service initialized successfully"); - Ok(Arc::new(storage_service)) + info!("Connected to Kubo at {}", config.kubo_api_url); + Ok(Arc::new(kubo_service)) } pub fn initialize_extraction_service( @@ -66,7 +40,7 @@ pub fn initialize_extraction_service( pub fn initialize_area_upload_service( cid_db: Arc, whosonfirst_db: Arc, - storage: Arc, + kubo: Arc, config: &Config, area_ids: Vec, ) -> super::InitializationResult { @@ -75,7 +49,7 @@ pub fn initialize_area_upload_service( let upload_service = AreaUploadService::new( cid_db, whosonfirst_db, - storage, + kubo, config.areas_dir.clone(), config.target_countries.clone(), area_ids, @@ -91,9 +65,12 @@ pub fn print_startup_info(config: &Config, cli: &crate::cli::Cli) { info!("CID Mappings DB: {:?}", config.cid_db_path); info!("Areas Dir: {:?}", config.areas_dir); info!("Planet PMTiles: {:?}", config.planet_pmtiles_location); - info!("Storage Port: {}", config.discovery_port); - info!("Storage Data Dir: {:?}", config.storage_data_dir); - info!("Max Concurrent Extractions: {}", config.max_concurrent_extractions); + info!("Kubo API URL: {}", config.kubo_api_url); + info!("Pin on Upload: {}", config.pin_on_upload); + info!( + "Max Concurrent Extractions: {}", + config.max_concurrent_extractions + ); info!("Target Countries: {:?}", config.target_countries); info!("Non-Interactive: {}", cli.is_non_interactive()); info!("Skip Download: {}", cli.should_skip_download()); diff --git a/src/initialization/mod.rs b/src/initialization/mod.rs index 097c251..93dabf4 100644 --- a/src/initialization/mod.rs +++ b/src/initialization/mod.rs @@ -13,8 +13,8 @@ pub enum InitializationError { ConfigError(#[from] crate::config::ConfigError), #[error("Database error: {0}")] DatabaseError(#[from] crate::services::DatabaseError), - #[error("Storage error: {0}")] - StorageError(#[from] crate::services::StorageError), + #[error("Kubo service error: {0}")] + KuboServiceError(#[from] crate::services::KuboServiceError), #[error("Extraction error: {0}")] ExtractionError(#[from] crate::services::ExtractionError), #[error("IO error: {0}")] @@ -27,6 +27,8 @@ pub enum InitializationError { CmdError(#[from] crate::utils::CmdError), #[error("Database is missing and download is disabled")] DatabaseMissing, + #[error("Invalid UTF-8 in path: {0}")] + InvalidPath(String), } pub type InitializationResult = Result; @@ -35,8 +37,8 @@ pub use database_init::{initialize_cid_db, initialize_whosonfirst_db}; pub use directories_init::ensure_directories; pub use download_init::ensure_database_is_present; pub use init::{ - initialize_country_service, initialize_extraction_service, initialize_area_upload_service, - initialize_storage_service, print_final_stats, print_startup_info, + initialize_area_upload_service, initialize_country_service, initialize_extraction_service, + initialize_kubo_service, print_final_stats, print_startup_info, }; pub use tools_init::ensure_required_tools; pub use validation_init::validate_config; diff --git a/src/lib.rs b/src/lib.rs index 1808903..e9781c0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,17 +10,15 @@ pub use app::{ApplicationError, ApplicationResult, NodeRunner}; pub use cli::Cli; pub use config::{Config, ConfigError}; pub use initialization::{ - ensure_database_is_present, ensure_directories, ensure_required_tools, initialize_cid_db, - initialize_country_service, initialize_extraction_service, initialize_area_upload_service, - initialize_storage_service, initialize_whosonfirst_db, print_final_stats, print_startup_info, - validate_config, InitializationError, InitializationResult, + ensure_database_is_present, ensure_directories, ensure_required_tools, + initialize_area_upload_service, initialize_cid_db, initialize_country_service, + initialize_extraction_service, initialize_kubo_service, initialize_whosonfirst_db, + print_final_stats, print_startup_info, validate_config, InitializationError, + InitializationResult, }; pub use services::{ AreaUploadError, AreaUploadService, CountryService, DatabaseError, DatabaseService, - DownloadResult, ExtractionError, ExtractionService, NodeInfo, StorageError, StorageService, - StorageStatus, UploadResult, -}; -pub use types::{ - AdministrativeArea, AreaInfo, CompletedUpload, PaginatedAreasResult, PaginationInfo, - PendingUpload, UploadQueue, UploadStats, + ExtractionError, ExtractionService, KuboService, KuboServiceError, KuboServiceStatus, NodeInfo, + UploadResult, }; +pub use types::{AdministrativeArea, CompletedUpload, PendingUpload, UploadQueue, UploadStats}; diff --git a/src/main.rs b/src/main.rs index 76badff..18fb325 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,28 +2,29 @@ use anynode::app::NodeRunner; use anynode::cli::Cli; use anynode::config::Config; use anynode::initialization::{ - ensure_database_is_present, ensure_directories, ensure_required_tools, initialize_cid_db, - initialize_country_service, initialize_extraction_service, initialize_area_upload_service, - initialize_storage_service, initialize_whosonfirst_db, print_startup_info, validate_config, + ensure_database_is_present, ensure_directories, ensure_required_tools, + initialize_area_upload_service, initialize_cid_db, initialize_country_service, + initialize_extraction_service, initialize_kubo_service, initialize_whosonfirst_db, + print_startup_info, validate_config, }; use std::sync::Arc; use tokio::signal; use tracing::{error, info}; use tracing_indicatif::IndicatifLayer; -use tracing_subscriber::EnvFilter; use tracing_subscriber::fmt; use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; +use tracing_subscriber::EnvFilter; #[tokio::main] async fn main() -> Result<(), Box> { + drop(dotenvy::dotenv()); + let cli = Cli::parse_args(); let log_level = cli.get_log_level(); - let filter = EnvFilter::try_from_default_env() - .unwrap_or_else(|_| EnvFilter::new(log_level)); + let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(log_level)); - // Set up tracing with indicatif layer to keep progress bar visible let indicatif_layer = IndicatifLayer::new(); tracing_subscriber::registry() @@ -34,7 +35,8 @@ async fn main() -> Result<(), Box> { info!("AnyNode v0.1.0 starting..."); - let config = Config::load()?; + let mut config = Config::load()?; + config.apply_cli_overrides(&cli); let config = Arc::new(config); print_startup_info(&config, &cli); @@ -59,25 +61,14 @@ async fn main() -> Result<(), Box> { let whosonfirst_db = initialize_whosonfirst_db(&config).await?; let cid_db = initialize_cid_db(&config).await?; let country_service = initialize_country_service(); - let bootstrap_nodes = cli.get_bootstrap_nodes(config.bootstrap_nodes.clone()); - let nat = cli.get_nat(config.nat.clone()); - let listen_addrs = cli.get_listen_addrs(config.listen_addrs.clone()); - let storage_service = initialize_storage_service( - &config, - cli.get_port(Some(config.discovery_port)), - cli.get_data_dir(Some(config.storage_data_dir.clone())), - bootstrap_nodes, - Some(nat), - Some(listen_addrs), - ) - .await?; + let kubo_service = initialize_kubo_service(&config).await?; let area_ids = cli.get_area_ids(config.area_ids.clone()); let extraction_service = initialize_extraction_service(&config, whosonfirst_db.clone())?; let upload_service = initialize_area_upload_service( cid_db.clone(), whosonfirst_db.clone(), - storage_service.clone(), + kubo_service.clone(), &config, area_ids.clone(), )?; @@ -92,7 +83,7 @@ async fn main() -> Result<(), Box> { let runner = NodeRunner::new( config.clone(), - storage_service.clone(), + kubo_service.clone(), extraction_service, upload_service, country_service, @@ -100,34 +91,21 @@ async fn main() -> Result<(), Box> { cli.should_skip_extract(), ); - if let Err(e) = runner.run().await { + let result = tokio::select! { + result = runner.run() => result, + _ = signal::ctrl_c() => { + info!("Interrupted by user"); + return Err("Interrupted".into()); + } + }; + + if let Err(e) = result { error!("Application error: {}", e); return Err(e.into()); } - info!("Press Ctrl+C to stop the node gracefully"); - - let monitor_handle = runner.start_monitoring(); - - tokio::select! { - _ = async { - signal::ctrl_c().await.expect("Failed to listen for ctrl+c"); - } => { - info!("Received Ctrl+C, shutting down gracefully..."); - } - _ = async { - let mut sig_term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) - .expect("Failed to setup SIGTERM handler"); - sig_term.recv().await; - } => { - info!("Received termination signal, shutting down gracefully..."); - } - } - - monitor_handle.abort(); - - runner.shutdown().await?; + runner.display_summary().await; - info!("AnyNode shutdown complete"); + info!("AnyNode completed successfully"); Ok(()) } diff --git a/src/services/area_upload_service.rs b/src/services/area_upload_service.rs index 1bd5128..48fb340 100644 --- a/src/services/area_upload_service.rs +++ b/src/services/area_upload_service.rs @@ -1,4 +1,4 @@ -use crate::services::{DatabaseService, StorageService}; +use crate::services::{DatabaseService, KuboServiceTrait}; use crate::types::{CompletedUpload, PendingUpload, UploadQueue, UploadStats}; use futures::future::join_all; use std::sync::Arc; @@ -10,8 +10,8 @@ use tracing::{error, info, warn}; pub enum AreaUploadError { #[error("Database error: {0}")] DatabaseError(#[from] crate::services::DatabaseError), - #[error("Storage error: {0}")] - StorageError(#[from] crate::services::StorageError), + #[error("Kubo service error: {0}")] + KuboServiceError(#[from] crate::services::KuboServiceError), #[error("File error: {0}")] FileError(#[from] std::io::Error), #[error("Upload queue error: {0}")] @@ -21,7 +21,7 @@ pub enum AreaUploadError { pub struct AreaUploadService { cid_db: Arc, whosonfirst_db: Arc, - storage: Arc, + kubo: Arc, upload_queue: Arc>, stats: Arc>, areas_dir: std::path::PathBuf, @@ -33,7 +33,7 @@ impl AreaUploadService { pub fn new( cid_db: Arc, whosonfirst_db: Arc, - storage: Arc, + kubo: Arc, areas_dir: std::path::PathBuf, target_countries: Vec, area_ids: Vec, @@ -41,9 +41,9 @@ impl AreaUploadService { Self { cid_db, whosonfirst_db, - storage, + kubo, upload_queue: Arc::new(Mutex::new(UploadQueue::new(10, 100))), - stats: Arc::new(Mutex::new(UploadStats::new())), + stats: Arc::new(Mutex::new(UploadStats::default())), areas_dir, target_countries, area_ids, @@ -84,8 +84,13 @@ impl AreaUploadService { AreaUploadError::QueueError("Invalid country directory name".to_string()) })?; - if !self.target_countries.is_empty() && !self.target_countries.contains(&country_code.to_string()) { - info!("Skipping country directory (not in target list): {}", country_code); + if !self.target_countries.is_empty() + && !self.target_countries.contains(&country_code.to_string()) + { + info!( + "Skipping country directory (not in target list): {}", + country_code + ); continue; } @@ -167,11 +172,7 @@ impl AreaUploadService { AreaUploadError::QueueError(format!("Invalid area ID in filename: {}", filename)) })?; - match self - .whosonfirst_db - .get_area_by_id(area_id as i64) - .await - { + match self.whosonfirst_db.get_area_by_id(area_id as i64).await { Ok(Some(_area)) => { if self .process_file_for_upload(&file_path, country_code, area_id) @@ -219,7 +220,10 @@ impl AreaUploadService { match self.whosonfirst_db.get_area_by_id(area_id as i64).await { Ok(Some(_area)) => { - if self.process_file_for_upload(&file_path, country_code, area_id).await? { + if self + .process_file_for_upload(&file_path, country_code, area_id) + .await? + { return Ok(true); } } @@ -250,21 +254,18 @@ impl AreaUploadService { return Ok(false); } - let pending_upload = PendingUpload::new( - country_code.to_string(), - area_id, - file_path.to_path_buf(), - ); + let pending_upload = + PendingUpload::new(country_code.to_string(), area_id, file_path.to_path_buf()); { let mut queue = self.upload_queue.lock().await; - if let Err(e) = queue.add_upload(pending_upload) { - warn!("Failed to add upload to queue: {}", e); + if !queue.add_upload(pending_upload) { + warn!("Failed to add upload to queue: queue is full"); return Ok(false); } } - if self.upload_queue.lock().await.is_full() { + if self.upload_queue.lock().await.is_batch_ready() { self.process_upload_queue().await?; } @@ -348,7 +349,7 @@ impl AreaUploadService { pending.area_id, pending.country_code, file_size ); - let result = self.storage.upload_file(file_path).await.map_err(|e| { + let result = self.kubo.upload_file(file_path).await.map_err(|e| { error!("Upload failed for area {}: {}", pending.area_id, e); e })?; @@ -394,3 +395,317 @@ impl AreaUploadService { self.stats.lock().await.clone() } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::Config; + use crate::services::{ + KuboService, KuboServiceError, KuboServiceStatus, NodeInfo, UploadResult, + }; + use async_trait::async_trait; + use mockall::mock; + use rusqlite::Connection; + use std::path::PathBuf; + use std::time::Duration; + use tempfile::TempDir; + + mock! { + pub KuboService {} + + #[async_trait] + impl KuboServiceTrait for KuboService { + async fn check_alive(&self) -> Result<(), KuboServiceError>; + fn get_status(&self) -> KuboServiceStatus; + async fn get_node_info(&self) -> Result; + async fn upload_file(&self, file_path: &std::path::Path) -> Result; + fn is_connected(&self) -> bool; + } + } + + fn create_test_config() -> Config { + Config { + kubo_api_url: "http://127.0.0.1:9999".to_string(), + kubo_api_timeout: Duration::from_secs(5), + pin_on_upload: false, + kubo_api_username: None, + kubo_api_password: None, + whosonfirst_db_path: PathBuf::from(":memory:"), + cid_db_path: PathBuf::from(":memory:"), + areas_dir: PathBuf::from("/nonexistent"), + bzip2_cmd: "bzip2".to_string(), + pmtiles_cmd: "pmtiles".to_string(), + target_countries: vec![], + area_ids: vec![], + max_concurrent_extractions: 1, + planet_pmtiles_location: None, + whosonfirst_db_url: String::new(), + } + } + + fn create_test_db() -> DatabaseService { + let conn = Connection::open_in_memory().expect("Failed to create in-memory database"); + DatabaseService::from_connection(conn, true).expect("Failed to create DatabaseService") + } + + fn create_spr_table(conn: &Connection) { + conn.execute( + "CREATE TABLE IF NOT EXISTS spr ( + id INTEGER PRIMARY KEY, + name TEXT, + country TEXT, + placetype TEXT, + latitude REAL, + longitude REAL, + min_longitude REAL, + min_latitude REAL, + max_longitude REAL, + max_latitude REAL, + is_current INTEGER DEFAULT 1, + is_deprecated INTEGER DEFAULT 0 + )", + [], + ) + .expect("Failed to create spr table"); + } + + fn insert_test_area(conn: &Connection, id: i64, name: &str, country: &str, placetype: &str) { + conn.execute( + "INSERT INTO spr (id, name, country, placetype, latitude, longitude, min_longitude, min_latitude, max_longitude, max_latitude, is_current, is_deprecated) + VALUES (?1, ?2, ?3, ?4, 37.5, -119.5, -124.5, 32.5, -114.1, 42.0, 1, 0)", + rusqlite::params![id, name, country, placetype], + ) + .expect("Failed to insert test area"); + } + + fn create_test_db_with_areas(area_ids: &[i64]) -> DatabaseService { + let conn = Connection::open_in_memory().expect("Failed to create in-memory database"); + create_spr_table(&conn); + for id in area_ids { + insert_test_area(&conn, *id, &format!("Area {}", id), "US", "region"); + } + DatabaseService::from_connection(conn, true).expect("Failed to create DatabaseService") + } + + fn create_test_kubo() -> Arc { + let config = create_test_config(); + Arc::new(KuboService::new(&config).expect("Failed to create KuboService")) + } + + fn create_test_service( + areas_dir: PathBuf, + target_countries: Vec, + area_ids: Vec, + ) -> AreaUploadService { + let cid_db = Arc::new(create_test_db()); + let whosonfirst_db = Arc::new(create_test_db()); + let kubo = create_test_kubo(); + + AreaUploadService::new( + cid_db, + whosonfirst_db, + kubo, + areas_dir, + target_countries, + area_ids, + ) + } + + fn create_mock_service( + areas_dir: PathBuf, + target_countries: Vec, + area_ids: Vec, + mock_kubo: MockKuboService, + ) -> AreaUploadService { + let cid_db = Arc::new(create_test_db()); + let whosonfirst_db = Arc::new(create_test_db()); + let kubo = Arc::new(mock_kubo) as Arc; + + AreaUploadService::new( + cid_db, + whosonfirst_db, + kubo, + areas_dir, + target_countries, + area_ids, + ) + } + + #[test] + fn new_creates_service() { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let service = create_test_service(temp_dir.path().to_path_buf(), vec![], vec![]); + assert!(service.areas_dir.exists()); + } + + #[tokio::test] + async fn get_stats_initial() { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let service = create_test_service(temp_dir.path().to_path_buf(), vec![], vec![]); + let stats = service.get_stats().await; + assert_eq!(stats.total_uploaded, 0); + assert_eq!(stats.total_failed, 0); + assert_eq!(stats.total_bytes_uploaded, 0); + } + + #[tokio::test] + async fn process_areas_missing_dir() { + let service = create_test_service(PathBuf::from("/nonexistent/path"), vec![], vec![]); + let result = service.process_areas().await; + assert!(result.is_ok()); + } + + #[tokio::test] + async fn process_areas_by_ids_empty() { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let service = create_test_service(temp_dir.path().to_path_buf(), vec![], vec![]); + let result = service.process_areas().await; + assert!(result.is_ok()); + } + + #[tokio::test] + async fn process_areas_with_file() { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let country_dir = temp_dir.path().join("US"); + std::fs::create_dir_all(&country_dir).expect("Failed to create country dir"); + + let pmtiles_path = country_dir.join("12345.pmtiles"); + std::fs::write(&pmtiles_path, b"fake pmtiles data").expect("Failed to write test file"); + + let mut mock_kubo = MockKuboService::new(); + mock_kubo.expect_upload_file().returning(|_| { + Ok(UploadResult { + cid: "QmTestCID123".to_string(), + size: 18, + }) + }); + + let cid_db = Arc::new(create_test_db()); + let whosonfirst_db = Arc::new(create_test_db_with_areas(&[12345])); + + let service = AreaUploadService::new( + cid_db, + whosonfirst_db, + Arc::new(mock_kubo), + temp_dir.path().to_path_buf(), + vec!["US".to_string()], + vec![], + ); + + let result = service.process_areas().await; + assert!(result.is_ok()); + + let stats = service.get_stats().await; + assert_eq!(stats.total_uploaded, 1); + } + + #[tokio::test] + async fn upload_single_file() { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let file_path = temp_dir.path().join("test.pmtiles"); + std::fs::write(&file_path, b"test data").expect("Failed to write test file"); + + let mut mock_kubo = MockKuboService::new(); + mock_kubo.expect_upload_file().returning(|_| { + Ok(UploadResult { + cid: "QmTestCID456".to_string(), + size: 9, + }) + }); + + let service = create_mock_service(temp_dir.path().to_path_buf(), vec![], vec![], mock_kubo); + + let pending = PendingUpload::new("US".to_string(), 12345, file_path); + + let result = service.upload_single_file(pending).await; + assert!(result.is_ok()); + + let upload = result.unwrap(); + assert_eq!(upload.cid, "QmTestCID456"); + assert_eq!(upload.area_id, 12345); + } + + #[tokio::test] + async fn upload_single_file_not_found() { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + + let mock_kubo = MockKuboService::new(); + + let service = create_mock_service(temp_dir.path().to_path_buf(), vec![], vec![], mock_kubo); + + let pending = PendingUpload::new( + "US".to_string(), + 99999, + PathBuf::from("/nonexistent/file.pmtiles"), + ); + + let result = service.upload_single_file(pending).await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn upload_single_file_kubo_error() { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let file_path = temp_dir.path().join("test.pmtiles"); + std::fs::write(&file_path, b"test data").expect("Failed to write test file"); + + let mut mock_kubo = MockKuboService::new(); + mock_kubo.expect_upload_file().returning(|_| { + Err(KuboServiceError::UploadFailed( + "Connection refused".to_string(), + )) + }); + + let service = create_mock_service(temp_dir.path().to_path_buf(), vec![], vec![], mock_kubo); + + let pending = PendingUpload::new("US".to_string(), 12345, file_path); + + let result = service.upload_single_file(pending).await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn batch_update_cid_mappings() { + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let country_dir = temp_dir.path().join("US"); + std::fs::create_dir_all(&country_dir).expect("Failed to create country dir"); + + let file1 = country_dir.join("100.pmtiles"); + let file2 = country_dir.join("200.pmtiles"); + std::fs::write(&file1, b"data1").expect("Failed to write test file"); + std::fs::write(&file2, b"data2").expect("Failed to write test file"); + + let mut mock_kubo = MockKuboService::new(); + mock_kubo + .expect_upload_file() + .returning(|_| { + Ok(UploadResult { + cid: "QmCID1".to_string(), + size: 5, + }) + }) + .times(2); + + let cid_db = Arc::new(create_test_db()); + let whosonfirst_db = Arc::new(create_test_db_with_areas(&[100, 200])); + + let service = AreaUploadService::new( + cid_db.clone(), + whosonfirst_db, + Arc::new(mock_kubo), + temp_dir.path().to_path_buf(), + vec!["US".to_string()], + vec![], + ); + + let result = service.process_areas().await; + assert!(result.is_ok()); + + let stats = service.get_stats().await; + assert_eq!(stats.total_uploaded, 2); + assert!(stats.total_bytes_uploaded > 0); + + assert!(cid_db.has_cid_mapping("US", 100).await.unwrap()); + assert!(cid_db.has_cid_mapping("US", 200).await.unwrap()); + } +} diff --git a/src/services/country_service.rs b/src/services/country_service.rs index 61dd7ab..9e82739 100644 --- a/src/services/country_service.rs +++ b/src/services/country_service.rs @@ -1,22 +1,23 @@ use tracing::info; const ALL_COUNTRIES: &[&str] = &[ - "AD", "AE", "AF", "AG", "AI", "AL", "AM", "AN", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", - "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", - "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", "CU", - "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", - "ET", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", "GM", - "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", "ID", "IE", - "IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", - "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", - "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", - "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", - "NZ", "Nl", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", - "QA", "RE", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", - "SM", "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", - "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "UN", "US", "UY", "UZ", "VA", - "VC", "VE", "VG", "VI", "VN", "VU", "WF", "WS", "XK", "XN", "XS", "XX", "XY", "XZ", "YE", "YT", "ZA", - "ZM", "ZW", + "AD", "AE", "AF", "AG", "AI", "AL", "AM", "AN", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", + "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", + "BS", "BT", "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", + "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", + "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", + "GG", "GH", "GI", "GL", "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", + "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", + "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", + "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", + "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", + "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", "NZ", "Nl", "OM", "PA", "PE", "PF", + "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", + "RW", "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", "SN", "SO", "SR", + "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", "TL", "TM", "TN", + "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "UN", "US", "UY", "UZ", "VA", "VC", "VE", + "VG", "VI", "VN", "VU", "WF", "WS", "XK", "XN", "XS", "XX", "XY", "XZ", "YE", "YT", "ZA", "ZM", + "ZW", ]; pub struct CountryService; @@ -41,16 +42,104 @@ impl CountryService { .filter(|country| ALL_COUNTRIES.contains(&country.as_str())) .cloned() .collect(); - + if valid_countries.len() != target_countries.len() { let invalid: Vec<_> = target_countries .iter() .filter(|c| !valid_countries.contains(c)) .collect(); - info!("Some requested countries not found in country list: {:?}", invalid); + info!( + "Some requested countries not found in country list: {:?}", + invalid + ); } - + valid_countries } } } + +#[cfg(test)] +mod tests { + use super::*; + + fn get_all_countries() -> Vec { + ALL_COUNTRIES.iter().map(|s| s.to_string()).collect() + } + + #[test] + fn get_countries_empty_targets() { + let service = CountryService::new(); + let result = service.get_countries_to_process(&[]); + assert_eq!(result, get_all_countries()); + } + + #[test] + fn get_countries_all_keyword() { + let service = CountryService::new(); + let result = service.get_countries_to_process(&["ALL".to_string()]); + assert_eq!(result, get_all_countries()); + } + + #[test] + fn get_countries_specific_valid() { + let service = CountryService::new(); + let result = service.get_countries_to_process(&[ + "US".to_string(), + "CA".to_string(), + "GB".to_string(), + ]); + assert_eq!(result, vec!["US", "CA", "GB"]); + } + + #[test] + fn get_countries_mixed_valid_invalid() { + let service = CountryService::new(); + let result = service.get_countries_to_process(&[ + "US".to_string(), + "INVALID".to_string(), + "CA".to_string(), + ]); + assert_eq!(result, vec!["US", "CA"]); + } + + #[test] + fn get_countries_empty_after_filter() { + let service = CountryService::new(); + let result = service.get_countries_to_process(&[ + "INVALID1".to_string(), + "INVALID2".to_string(), + "ZZZ".to_string(), + ]); + assert!(result.is_empty()); + } + + #[test] + fn get_countries_preserves_order() { + let service = CountryService::new(); + let result = service.get_countries_to_process(&[ + "GB".to_string(), + "US".to_string(), + "CA".to_string(), + ]); + assert_eq!(result, vec!["GB", "US", "CA"]); + } + + #[test] + fn get_countries_case_sensitive() { + let service = CountryService::new(); + let result = service.get_countries_to_process(&["us".to_string(), "US".to_string()]); + assert_eq!(result, vec!["US"]); + } + + #[test] + fn get_countries_duplicates() { + let service = CountryService::new(); + let result = service.get_countries_to_process(&[ + "US".to_string(), + "CA".to_string(), + "US".to_string(), + ]); + assert_eq!(result, vec!["US", "CA", "US"]); + } +} diff --git a/src/services/database_service.rs b/src/services/database_service.rs index a8c6237..8fddddf 100644 --- a/src/services/database_service.rs +++ b/src/services/database_service.rs @@ -33,36 +33,53 @@ impl DatabaseService { Ok(service) } + fn create_cid_tables_sync(conn: &Connection) -> Result<(), DatabaseError> { + let create_cid_table = r#" + CREATE TABLE IF NOT EXISTS area_cids ( + country_code TEXT NOT NULL, + area_id INTEGER NOT NULL, + cid TEXT NOT NULL, + upload_time DATETIME DEFAULT CURRENT_TIMESTAMP, + file_size INTEGER, + PRIMARY KEY (country_code, area_id) + ) + "#; + + let create_cid_index = r#" + CREATE INDEX IF NOT EXISTS idx_area_cids_lookup + ON area_cids(country_code, area_id) + "#; + + conn.execute(create_cid_table, [])?; + conn.execute(create_cid_index, [])?; + + Ok(()) + } + async fn create_cid_tables(&self) -> Result<(), DatabaseError> { let conn = self.conn.clone(); tokio::task::spawn_blocking(move || { let conn = conn.blocking_lock(); - - let create_cid_table = r#" - CREATE TABLE IF NOT EXISTS area_cids ( - country_code TEXT NOT NULL, - area_id INTEGER NOT NULL, - cid TEXT NOT NULL, - upload_time DATETIME DEFAULT CURRENT_TIMESTAMP, - file_size INTEGER, - PRIMARY KEY (country_code, area_id) - ) - "#; - - let create_cid_index = r#" - CREATE INDEX IF NOT EXISTS idx_area_cids_lookup - ON area_cids(country_code, area_id) - "#; - - conn.execute(create_cid_table, [])?; - conn.execute(create_cid_index, [])?; - + Self::create_cid_tables_sync(&conn)?; Ok::<(), DatabaseError>(()) }) .await? } + #[cfg(test)] + pub fn from_connection( + conn: Connection, + create_cid_tables: bool, + ) -> Result { + if create_cid_tables { + Self::create_cid_tables_sync(&conn)?; + } + Ok(Self { + conn: Arc::new(Mutex::new(conn)), + }) + } + pub async fn get_country_areas( &self, country_code: &str, @@ -95,7 +112,7 @@ impl DatabaseService { ); let mut stmt = conn.prepare(&query_str)?; - let rows = stmt.query_map([&country_code], |row| AdministrativeArea::from_row(row))?; + let rows = stmt.query_map([&country_code], AdministrativeArea::from_row)?; let areas = rows.collect::, _>>()?; Ok(areas) @@ -103,32 +120,6 @@ impl DatabaseService { .await? } - pub async fn get_country_area_count( - &self, - country_code: &str, - ) -> Result { - let conn = self.conn.clone(); - let country_code = country_code.to_string(); - - tokio::task::spawn_blocking(move || { - let conn = conn.blocking_lock(); - - let conditions = [ - "placetype IN ('region', 'county')", - "is_current = 1", - "is_deprecated = 0", - "country = ?1", - ]; - - let where_clause = conditions.join(" AND "); - let query_str = format!("SELECT COUNT(*) as count FROM spr WHERE {}", where_clause); - - let count = conn.query_row(&query_str, [&country_code], |row| row.get::<_, i64>(0))?; - Ok(count as u32) - }) - .await? - } - pub async fn get_area_by_id( &self, area_id: i64, @@ -145,7 +136,7 @@ impl DatabaseService { "#; let mut stmt = conn.prepare(query)?; - let rows = stmt.query_map([&area_id], |row| AdministrativeArea::from_row(row))?; + let rows = stmt.query_map([&area_id], AdministrativeArea::from_row)?; let areas: Result, _> = rows.collect(); match areas { @@ -182,7 +173,7 @@ impl DatabaseService { let mut stmt = conn.prepare(&query_str)?; let params: Vec<&dyn rusqlite::ToSql> = area_ids.iter().map(|id| id as &dyn rusqlite::ToSql).collect(); - let rows = stmt.query_map(params.as_slice(), |row| AdministrativeArea::from_row(row))?; + let rows = stmt.query_map(params.as_slice(), AdministrativeArea::from_row)?; let areas = rows.collect::, _>>()?; Ok(areas) @@ -213,12 +204,7 @@ impl DatabaseService { let file_size_i64 = file_size as i64; tx.execute( query, - rusqlite::params![ - &country_code, - &area_id_i64, - &cid, - &file_size_i64, - ], + rusqlite::params![&country_code, &area_id_i64, &cid, &file_size_i64,], )?; } @@ -255,21 +241,304 @@ impl DatabaseService { }) .await? } +} - pub async fn get_cid_mapping_stats(&self) -> Result<(u64, u64), DatabaseError> { - let conn = self.conn.clone(); +#[cfg(test)] +mod tests { + use super::*; + + fn create_spr_table(conn: &Connection) { + conn.execute( + "CREATE TABLE IF NOT EXISTS spr ( + id INTEGER PRIMARY KEY, + name TEXT, + country TEXT, + placetype TEXT, + latitude REAL, + longitude REAL, + min_longitude REAL, + min_latitude REAL, + max_longitude REAL, + max_latitude REAL, + is_current INTEGER DEFAULT 1, + is_deprecated INTEGER DEFAULT 0 + )", + [], + ) + .expect("Failed to create spr table"); + } - tokio::task::spawn_blocking(move || { - let conn = conn.blocking_lock(); + fn insert_test_area( + conn: &Connection, + id: i64, + name: &str, + country: &str, + placetype: &str, + is_current: i32, + is_deprecated: i32, + ) { + conn.execute( + "INSERT INTO spr (id, name, country, placetype, latitude, longitude, min_longitude, min_latitude, max_longitude, max_latitude, is_current, is_deprecated) + VALUES (?1, ?2, ?3, ?4, 37.5, -119.5, -124.5, 32.5, -114.1, 42.0, ?5, ?6)", + rusqlite::params![id, name, country, placetype, is_current, is_deprecated], + ) + .expect("Failed to insert test area"); + } + + fn create_test_db_with_spr_data() -> DatabaseService { + let conn = Connection::open_in_memory().expect("Failed to create in-memory database"); + create_spr_table(&conn); - let total_query = "SELECT COUNT(*) as count FROM area_cids"; - let total_count = conn.query_row(total_query, [], |row| row.get::<_, i64>(0))?; + insert_test_area(&conn, 1, "California", "US", "region", 1, 0); + insert_test_area(&conn, 2, "Texas", "US", "region", 1, 0); + insert_test_area(&conn, 3, "Los Angeles County", "US", "county", 1, 0); + insert_test_area(&conn, 4, "Los Angeles", "US", "locality", 1, 0); + insert_test_area(&conn, 5, "Deprecated Region", "US", "region", 1, 1); + insert_test_area(&conn, 6, "Inactive Region", "US", "region", 0, 0); - let countries_query = "SELECT COUNT(DISTINCT country_code) as count FROM area_cids"; - let countries_count = conn.query_row(countries_query, [], |row| row.get::<_, i64>(0))?; + DatabaseService::from_connection(conn, true).expect("Failed to create DatabaseService") + } + + #[tokio::test] + async fn new_in_memory_no_tables() { + let conn = Connection::open_in_memory().expect("Failed to create in-memory database"); + let service = DatabaseService::from_connection(conn, false); + assert!(service.is_ok()); + } - Ok((total_count as u64, countries_count as u64)) + #[tokio::test] + async fn new_in_memory_creates_tables() { + let conn = Connection::open_in_memory().expect("Failed to create in-memory database"); + let service = + DatabaseService::from_connection(conn, true).expect("Failed to create service"); + + let exists: bool = tokio::task::spawn_blocking(move || { + service + .conn + .blocking_lock() + .query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='area_cids')", + [], + |row| row.get(0), + ) + .expect("Failed to check table existence") }) - .await? + .await + .expect("spawn_blocking failed"); + assert!(exists); + } + + #[tokio::test] + async fn has_cid_mapping_false() { + let conn = Connection::open_in_memory().expect("Failed to create in-memory database"); + let service = + DatabaseService::from_connection(conn, true).expect("Failed to create service"); + + let has_mapping = service + .has_cid_mapping("US", 12345) + .await + .expect("has_cid_mapping failed"); + assert!(!has_mapping); + } + + #[tokio::test] + async fn has_cid_mapping_true() { + let conn = Connection::open_in_memory().expect("Failed to create in-memory database"); + let service = + DatabaseService::from_connection(conn, true).expect("Failed to create service"); + + service + .batch_insert_cid_mappings(&[("US".to_string(), 12345, "cid123".to_string(), 1024)]) + .await + .expect("batch_insert failed"); + + let has_mapping = service + .has_cid_mapping("US", 12345) + .await + .expect("has_cid_mapping failed"); + assert!(has_mapping); + } + + #[tokio::test] + async fn batch_insert_cid_mappings() { + let conn = Connection::open_in_memory().expect("Failed to create in-memory database"); + let service = + DatabaseService::from_connection(conn, true).expect("Failed to create service"); + + let mappings = vec![ + ("US".to_string(), 1, "cid1".to_string(), 100), + ("US".to_string(), 2, "cid2".to_string(), 200), + ("CA".to_string(), 3, "cid3".to_string(), 300), + ]; + + service + .batch_insert_cid_mappings(&mappings) + .await + .expect("batch_insert failed"); + + assert!(service + .has_cid_mapping("US", 1) + .await + .expect("has_cid_mapping failed")); + assert!(service + .has_cid_mapping("US", 2) + .await + .expect("has_cid_mapping failed")); + assert!(service + .has_cid_mapping("CA", 3) + .await + .expect("has_cid_mapping failed")); + } + + #[tokio::test] + async fn batch_insert_cid_mappings_replaces() { + let conn = Connection::open_in_memory().expect("Failed to create in-memory database"); + let service = + DatabaseService::from_connection(conn, true).expect("Failed to create service"); + + service + .batch_insert_cid_mappings(&[("US".to_string(), 1, "cid_old".to_string(), 100)]) + .await + .expect("batch_insert failed"); + + service + .batch_insert_cid_mappings(&[("US".to_string(), 1, "cid_new".to_string(), 200)]) + .await + .expect("batch_insert failed"); + + let (count, cid) = tokio::task::spawn_blocking(move || { + let guard = service.conn.blocking_lock(); + let count: i64 = guard + .query_row( + "SELECT COUNT(*) FROM area_cids WHERE country_code = 'US' AND area_id = 1", + [], + |row| row.get(0), + ) + .expect("Failed to count rows"); + let cid: String = guard + .query_row( + "SELECT cid FROM area_cids WHERE country_code = 'US' AND area_id = 1", + [], + |row| row.get(0), + ) + .expect("Failed to get cid"); + (count, cid) + }) + .await + .expect("spawn_blocking failed"); + + assert_eq!(count, 1); + assert_eq!(cid, "cid_new"); + } + + #[tokio::test] + async fn get_country_areas_empty_db() { + let service = create_test_db_with_spr_data(); + + let areas = service + .get_country_areas("DE") + .await + .expect("get_country_areas failed"); + assert!(areas.is_empty()); + } + + #[tokio::test] + async fn get_country_areas_returns_areas() { + let service = create_test_db_with_spr_data(); + + let areas = service + .get_country_areas("US") + .await + .expect("get_country_areas failed"); + + assert_eq!(areas.len(), 3); + let names: Vec<&str> = areas.iter().map(|a| a.name.as_str()).collect(); + assert!(names.contains(&"California")); + assert!(names.contains(&"Texas")); + assert!(names.contains(&"Los Angeles County")); + } + + #[tokio::test] + async fn get_country_areas_filters_placetype() { + let service = create_test_db_with_spr_data(); + + let areas = service + .get_country_areas("US") + .await + .expect("get_country_areas failed"); + + for area in &areas { + assert!(area.placetype == "region" || area.placetype == "county"); + assert_ne!(area.placetype, "locality"); + } + } + + #[tokio::test] + async fn get_country_areas_filters_deprecated() { + let service = create_test_db_with_spr_data(); + + let areas = service + .get_country_areas("US") + .await + .expect("get_country_areas failed"); + + let names: Vec<&str> = areas.iter().map(|a| a.name.as_str()).collect(); + assert!(!names.contains(&"Deprecated Region")); + assert!(!names.contains(&"Inactive Region")); + } + + #[tokio::test] + async fn get_area_by_id_found() { + let service = create_test_db_with_spr_data(); + + let area = service + .get_area_by_id(1) + .await + .expect("get_area_by_id failed"); + assert!(area.is_some()); + + let area = area.unwrap(); + assert_eq!(area.id, 1); + assert_eq!(area.name, "California"); + assert_eq!(area.country, "US"); + assert_eq!(area.placetype, "region"); + } + + #[tokio::test] + async fn get_area_by_id_not_found() { + let service = create_test_db_with_spr_data(); + + let area = service + .get_area_by_id(99999) + .await + .expect("get_area_by_id failed"); + assert!(area.is_none()); + } + + #[tokio::test] + async fn get_areas_by_ids_empty() { + let service = create_test_db_with_spr_data(); + + let areas = service + .get_areas_by_ids(&[]) + .await + .expect("get_areas_by_ids failed"); + assert!(areas.is_empty()); + } + + #[tokio::test] + async fn get_areas_by_ids_multiple() { + let service = create_test_db_with_spr_data(); + + let areas = service + .get_areas_by_ids(&[1, 2, 3]) + .await + .expect("get_areas_by_ids failed"); + + assert_eq!(areas.len(), 3); + let ids: Vec = areas.iter().map(|a| a.id).collect(); + assert!(ids.contains(&1)); + assert!(ids.contains(&2)); + assert!(ids.contains(&3)); } } diff --git a/src/services/extraction_service.rs b/src/services/extraction_service.rs index fe5d313..16f30e3 100644 --- a/src/services/extraction_service.rs +++ b/src/services/extraction_service.rs @@ -20,6 +20,8 @@ pub enum ExtractionError { DatabaseError(String), #[error("IO error: {0}")] IoError(#[from] std::io::Error), + #[error("Internal error: {0}")] + InternalError(String), } #[derive(Clone, Debug)] @@ -29,10 +31,6 @@ pub enum PlanetSource { } impl PlanetSource { - pub fn is_remote(&self) -> bool { - matches!(self, PlanetSource::Remote(_)) - } - pub fn as_str(&self) -> &str { match self { PlanetSource::Local(path) => path.to_str().unwrap_or(""), @@ -88,10 +86,7 @@ impl ExtractionService { let bbox = format!( "{},{},{},{}", - area.min_longitude, - area.min_latitude, - area.max_longitude, - area.max_latitude + area.min_longitude, area.min_latitude, area.max_longitude, area.max_latitude ); info!( @@ -99,11 +94,17 @@ impl ExtractionService { area.placetype, area.id, area.name, bbox ); + let output_path_str = output_path.to_str().ok_or_else(|| { + ExtractionError::ExtractionFailed( + area.id, + format!("Invalid UTF-8 in output path: {}", output_path.display()), + ) + })?; let output = tokio::process::Command::new(&self.config.pmtiles_cmd) .args([ "extract", planet_source.as_str(), - output_path.to_str().unwrap(), + output_path_str, &format!("--bbox={}", bbox), ]) .output() @@ -112,7 +113,10 @@ impl ExtractionService { if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); - error!("Extraction failed for {} {}: {}", area.placetype, area.id, stderr); + error!( + "Extraction failed for {} {}: {}", + area.placetype, area.id, stderr + ); return Err(ExtractionError::ExtractionFailed( area.id, stderr.to_string(), @@ -131,10 +135,7 @@ impl ExtractionService { } } - pub async fn extract_areas( - &self, - country_codes: &[String], - ) -> Result<(), ExtractionError> { + pub async fn extract_areas(&self, country_codes: &[String]) -> Result<(), ExtractionError> { let planet_source = self.get_planet_source()?; for country_code in country_codes { @@ -156,11 +157,7 @@ impl ExtractionService { continue; } - info!( - "Found {} areas for country: {}", - areas.len(), - country_code - ); + info!("Found {} areas for country: {}", areas.len(), country_code); let mut existing_count = 0; for area in &areas { @@ -198,7 +195,9 @@ impl ExtractionService { let completed_count = completed_count.clone(); let task = tokio::spawn(async move { - let _permit = semaphore.acquire().await.unwrap(); + let _permit = semaphore.acquire().await.map_err(|e| { + ExtractionError::InternalError(format!("semaphore error: {}", e)) + })?; let result = extraction_service .extract_area(&area, &planet_source, &country_dir) .await; @@ -248,10 +247,7 @@ impl ExtractionService { Ok(()) } - pub async fn extract_areas_by_ids( - &self, - area_ids: &[u32], - ) -> Result<(), ExtractionError> { + pub async fn extract_areas_by_ids(&self, area_ids: &[u32]) -> Result<(), ExtractionError> { let planet_source = self.get_planet_source()?; let areas = self @@ -271,8 +267,7 @@ impl ExtractionService { area_ids.len() ); - let found_ids: std::collections::HashSet = - areas.iter().map(|a| a.id).collect(); + let found_ids: std::collections::HashSet = areas.iter().map(|a| a.id).collect(); for id in area_ids { if !found_ids.contains(&(*id as i64)) { warn!( @@ -286,7 +281,7 @@ impl ExtractionService { for area in areas { by_country .entry(area.country.clone()) - .or_insert_with(Vec::new) + .or_default() .push(area); } @@ -306,7 +301,15 @@ impl ExtractionService { let extraction_service = self.clone(); let task = tokio::spawn(async move { - let _permit = semaphore.acquire().await.unwrap(); + let _permit = match semaphore.acquire().await { + Ok(p) => p, + Err(e) => { + return Err(ExtractionError::InternalError(format!( + "semaphore error: {}", + e + ))) + } + }; extraction_service .extract_area(&area, &planet_source, &country_dir) .await @@ -342,46 +345,133 @@ impl ExtractionService { Ok(()) } +} - pub async fn get_pmtiles_file_count(&self, country_code: &str) -> Result { - let country_dir = self.config.areas_dir.join(country_code); +impl Clone for ExtractionService { + fn clone(&self) -> Self { + Self { + config: self.config.clone(), + db_service: self.db_service.clone(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::Config; + use crate::services::DatabaseService; + use std::path::PathBuf; + use std::time::Duration; + + fn make_config(planet_location: Option) -> Arc { + Arc::new(Config { + kubo_api_url: "http://127.0.0.1:5001".to_string(), + kubo_api_timeout: Duration::from_secs(30), + pin_on_upload: false, + kubo_api_username: None, + kubo_api_password: None, + whosonfirst_db_path: PathBuf::from("/tmp/whosonfirst.db"), + cid_db_path: PathBuf::from("/tmp/cid.db"), + areas_dir: PathBuf::from("/tmp/areas"), + bzip2_cmd: "bzip2".to_string(), + pmtiles_cmd: "pmtiles".to_string(), + target_countries: vec![], + area_ids: vec![], + max_concurrent_extractions: 1, + planet_pmtiles_location: planet_location, + whosonfirst_db_url: "http://example.com".to_string(), + }) + } - if !country_dir.exists() { - return Ok(0); + fn make_service(planet_location: Option) -> ExtractionService { + let config = make_config(planet_location); + let conn = rusqlite::Connection::open_in_memory().unwrap(); + let db_service = Arc::new(DatabaseService::from_connection(conn, false).unwrap()); + ExtractionService::new(config, db_service) + } + + #[test] + #[allow(clippy::panic)] + fn get_planet_source_none_configured() { + let service = make_service(None); + let result = service.get_planet_source(); + assert!(result.is_err()); + match result.unwrap_err() { + ExtractionError::PlanetLocationNotConfigured => {} + other => panic!("expected PlanetLocationNotConfigured, got {:?}", other), } + } - let mut count = 0; - let mut entries = tokio::fs::read_dir(&country_dir).await?; - while let Some(entry) = entries.next_entry().await? { - let path = entry.path(); - if path.extension().and_then(|s| s.to_str()) == Some("pmtiles") { - count += 1; + #[test] + #[allow(clippy::panic)] + fn get_planet_source_remote_url_https() { + let service = make_service(Some("https://example.com/planet.pmtiles".to_string())); + let result = service.get_planet_source(); + assert!(result.is_ok()); + match result.unwrap() { + PlanetSource::Remote(url) => { + assert_eq!(url, "https://example.com/planet.pmtiles"); } + PlanetSource::Local(_) => panic!("expected Remote, got Local"), } + } - Ok(count) + #[test] + #[allow(clippy::panic)] + fn get_planet_source_remote_url_http() { + let service = make_service(Some("http://example.com/planet.pmtiles".to_string())); + let result = service.get_planet_source(); + assert!(result.is_ok()); + match result.unwrap() { + PlanetSource::Remote(url) => { + assert_eq!(url, "http://example.com/planet.pmtiles"); + } + PlanetSource::Local(_) => panic!("expected Remote, got Local"), + } } - pub async fn batch_get_pmtiles_file_count( - &self, - country_codes: &[String], - ) -> Result, ExtractionError> { - let mut counts = HashMap::new(); + #[test] + #[allow(clippy::panic)] + fn get_planet_source_local_exists() { + let dir = tempfile::tempdir().unwrap(); + let file_path = dir.path().join("planet.pmtiles"); + std::fs::write(&file_path, b"fake").unwrap(); + + let service = make_service(Some(file_path.to_str().unwrap().to_string())); + let result = service.get_planet_source(); + assert!(result.is_ok()); + match result.unwrap() { + PlanetSource::Local(path) => { + assert_eq!(path, file_path); + } + PlanetSource::Remote(_) => panic!("expected Local, got Remote"), + } + } - for country_code in country_codes { - let count = self.get_pmtiles_file_count(country_code).await?; - counts.insert(country_code.clone(), count); + #[test] + #[allow(clippy::panic)] + fn get_planet_source_local_not_found() { + let service = make_service(Some("/tmp/nonexistent_planet_xyz.pmtiles".to_string())); + let result = service.get_planet_source(); + assert!(result.is_err()); + match result.unwrap_err() { + ExtractionError::PlanetFileNotFound(path) => { + assert!(path.contains("nonexistent_planet_xyz")); + } + other => panic!("expected PlanetFileNotFound, got {:?}", other), } + } - Ok(counts) + #[test] + fn planet_source_as_str_local() { + let source = PlanetSource::Local(PathBuf::from("/data/planet.pmtiles")); + assert_eq!(source.as_str(), "/data/planet.pmtiles"); } -} -impl Clone for ExtractionService { - fn clone(&self) -> Self { - Self { - config: self.config.clone(), - db_service: self.db_service.clone(), - } + #[test] + fn planet_source_as_str_remote() { + let source = PlanetSource::Remote("https://example.com/planet.pmtiles".to_string()); + assert_eq!(source.as_str(), "https://example.com/planet.pmtiles"); } } diff --git a/src/services/kubo_client.rs b/src/services/kubo_client.rs new file mode 100644 index 0000000..750b4f1 --- /dev/null +++ b/src/services/kubo_client.rs @@ -0,0 +1,609 @@ +use std::path::Path; +use std::time::Duration; + +use reqwest::multipart; +use serde::Deserialize; +use tracing::{debug, info}; + +#[derive(Debug, Clone)] +pub struct KuboVersion { + pub version: String, +} + +#[derive(Debug, Clone)] +pub struct KuboId { + pub id: String, + pub addresses: Vec, + pub agent_version: String, +} + +#[derive(Debug, Clone)] +pub struct KuboAddResponse { + pub hash: String, + pub size: u64, +} + +#[derive(Debug, Clone)] +pub struct KuboSwarmPeers { + pub peers: Vec, +} + +#[derive(Debug, Clone)] +pub struct KuboPeer { + pub addr: String, + pub peer: String, +} + +#[derive(Debug, thiserror::Error)] +pub enum KuboClientError { + #[error("Kubo API unavailable at {0}: {1}")] + ApiUnavailable(String, String), + #[error("Kubo API error: {0}")] + ApiError(String), + #[error("Upload failed: {0}")] + UploadFailed(String), + #[error("Pin failed for CID {0}: {1}")] + PinFailed(String, String), + #[error("Failed to parse Kubo response: {0}")] + ParseError(String), + #[error("IO error: {0}")] + IoError(#[from] std::io::Error), + #[error("HTTP request error: {0}")] + RequestError(#[from] reqwest::Error), +} + +#[allow(non_snake_case)] +#[derive(Deserialize)] +struct KuboVersionRaw { + Version: String, +} + +#[allow(non_snake_case)] +#[derive(Deserialize)] +struct KuboIdRaw { + ID: String, + Addresses: Vec, + AgentVersion: String, +} + +#[allow(non_snake_case)] +#[derive(Deserialize)] +struct KuboAddLineRaw { + Hash: String, + Size: String, +} + +#[allow(non_snake_case)] +#[derive(Deserialize)] +struct KuboSwarmPeersRaw { + Peers: Vec, +} + +#[allow(non_snake_case)] +#[derive(Deserialize)] +struct KuboPeerRaw { + Addr: String, + Peer: String, +} + +#[allow(non_snake_case)] +#[derive(Deserialize)] +struct KuboErrorRaw { + Message: String, +} + +pub struct KuboClient { + client: reqwest::Client, + api_url: String, + username: Option, + password: Option, +} + +impl KuboClient { + pub fn new( + api_url: &str, + timeout: Duration, + username: Option, + password: Option, + ) -> Result { + let client = reqwest::Client::builder() + .timeout(timeout) + .build() + .map_err(KuboClientError::RequestError)?; + + let api_url = api_url.trim_end_matches('/').to_string(); + + Ok(Self { + client, + api_url, + username, + password, + }) + } + + fn add_auth(&self, request: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + if let (Some(user), Some(pass)) = (&self.username, &self.password) { + request.basic_auth(user, Some(pass)) + } else { + request + } + } + + pub async fn version(&self) -> Result { + let url = format!("{}/api/v0/version", self.api_url); + debug!("Requesting Kubo version from {}", url); + + let request = self.add_auth(self.client.post(&url)); + let response = request + .send() + .await + .map_err(|e| KuboClientError::ApiUnavailable(self.api_url.clone(), e.to_string()))?; + + let status = response.status(); + let body = response.text().await?; + + if !status.is_success() { + let msg = Self::parse_error_message(&body); + return Err(KuboClientError::ApiError(msg)); + } + + let raw: KuboVersionRaw = + serde_json::from_str(&body).map_err(|e| KuboClientError::ParseError(e.to_string()))?; + + info!("Kubo version: {}", raw.Version); + + Ok(KuboVersion { + version: raw.Version, + }) + } + + pub async fn id(&self) -> Result { + let url = format!("{}/api/v0/id", self.api_url); + debug!("Requesting Kubo node ID from {}", url); + + let request = self.add_auth(self.client.post(&url)); + let response = request + .send() + .await + .map_err(|e| KuboClientError::ApiUnavailable(self.api_url.clone(), e.to_string()))?; + + let status = response.status(); + let body = response.text().await?; + + if !status.is_success() { + let msg = Self::parse_error_message(&body); + return Err(KuboClientError::ApiError(msg)); + } + + let raw: KuboIdRaw = + serde_json::from_str(&body).map_err(|e| KuboClientError::ParseError(e.to_string()))?; + + Ok(KuboId { + id: raw.ID, + addresses: raw.Addresses, + agent_version: raw.AgentVersion, + }) + } + + pub async fn add( + &self, + file_path: &Path, + pin: bool, + ) -> Result { + let url = format!("{}/api/v0/add?pin={}&progress=false", self.api_url, pin); + debug!( + "Uploading file to Kubo: {} (pin={})", + file_path.display(), + pin + ); + + let file_name = file_path + .file_name() + .ok_or_else(|| { + KuboClientError::UploadFailed(format!( + "No filename in path: {}", + file_path.display() + )) + })? + .to_string_lossy() + .into_owned(); + + let file_bytes = tokio::fs::read(file_path).await?; + + let part = multipart::Part::bytes(file_bytes) + .file_name(file_name) + .mime_str("application/octet-stream") + .map_err(|e| KuboClientError::UploadFailed(e.to_string()))?; + + let form = multipart::Form::new().part("file", part); + + let request = self.add_auth(self.client.post(&url).multipart(form)); + let response = request.send().await?; + + let status = response.status(); + let body = response.text().await?; + + if !status.is_success() { + let msg = Self::parse_error_message(&body); + return Err(KuboClientError::UploadFailed(msg)); + } + + let last_line = body + .lines() + .rfind(|line| !line.trim().is_empty()) + .ok_or_else(|| { + KuboClientError::ParseError("Empty response from add endpoint".to_string()) + })?; + + let raw: KuboAddLineRaw = serde_json::from_str(last_line) + .map_err(|e| KuboClientError::ParseError(e.to_string()))?; + + let size = raw.Size.parse::().map_err(|e| { + KuboClientError::ParseError(format!("Failed to parse size '{}': {}", raw.Size, e)) + })?; + + info!("File uploaded to Kubo. CID: {}, size: {}", raw.Hash, size); + + Ok(KuboAddResponse { + hash: raw.Hash, + size, + }) + } + + pub async fn swarm_peers(&self) -> Result { + let url = format!("{}/api/v0/swarm/peers", self.api_url); + debug!("Requesting swarm peers from Kubo"); + + let request = self.add_auth(self.client.post(&url)); + let response = request + .send() + .await + .map_err(|e| KuboClientError::ApiUnavailable(self.api_url.clone(), e.to_string()))?; + + let status = response.status(); + let body = response.text().await?; + + if !status.is_success() { + let msg = Self::parse_error_message(&body); + return Err(KuboClientError::ApiError(msg)); + } + + let raw: KuboSwarmPeersRaw = + serde_json::from_str(&body).map_err(|e| KuboClientError::ParseError(e.to_string()))?; + + let peers = raw + .Peers + .into_iter() + .map(|p| KuboPeer { + addr: p.Addr, + peer: p.Peer, + }) + .collect(); + + Ok(KuboSwarmPeers { peers }) + } + + fn parse_error_message(body: &str) -> String { + serde_json::from_str::(body) + .map(|e| e.Message) + .unwrap_or_else(|_| body.to_string()) + } +} + +impl Clone for KuboClient { + fn clone(&self) -> Self { + Self { + client: self.client.clone(), + api_url: self.api_url.clone(), + username: self.username.clone(), + password: self.password.clone(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + use tempfile::NamedTempFile; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + #[test] + fn new_strips_trailing_slash() { + let client = KuboClient::new( + "http://localhost:5001/", + Duration::from_secs(30), + None, + None, + ) + .unwrap(); + assert_eq!(client.api_url, "http://localhost:5001"); + } + + #[tokio::test] + async fn version_success() { + let server = MockServer::start().await; + + Mock::given(method("POST")) + .and(path("/api/v0/version")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "Version": "0.20.0", + "Commit": "abc123", + "Repo": "10" + }))) + .mount(&server) + .await; + + let client = KuboClient::new(&server.uri(), Duration::from_secs(30), None, None).unwrap(); + let result = client.version().await.unwrap(); + + assert_eq!(result.version, "0.20.0"); + } + + #[tokio::test] + async fn version_api_unavailable() { + let client = KuboClient::new( + "http://127.0.0.1:59999", + Duration::from_millis(100), + None, + None, + ) + .unwrap(); + let result = client.version().await; + + assert!(matches!(result, Err(KuboClientError::ApiUnavailable(_, _)))); + } + + #[tokio::test] + async fn version_api_error_500() { + let server = MockServer::start().await; + + Mock::given(method("POST")) + .and(path("/api/v0/version")) + .respond_with(ResponseTemplate::new(500).set_body_json(serde_json::json!({ + "Message": "internal server error" + }))) + .mount(&server) + .await; + + let client = KuboClient::new(&server.uri(), Duration::from_secs(30), None, None).unwrap(); + let result = client.version().await; + + assert!(matches!(result, Err(KuboClientError::ApiError(_)))); + } + + #[tokio::test] + async fn version_parse_error() { + let server = MockServer::start().await; + + Mock::given(method("POST")) + .and(path("/api/v0/version")) + .respond_with(ResponseTemplate::new(200).set_body_string("not valid json")) + .mount(&server) + .await; + + let client = KuboClient::new(&server.uri(), Duration::from_secs(30), None, None).unwrap(); + let result = client.version().await; + + assert!(matches!(result, Err(KuboClientError::ParseError(_)))); + } + + #[tokio::test] + async fn id_success() { + let server = MockServer::start().await; + + Mock::given(method("POST")) + .and(path("/api/v0/id")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "ID": "QmPeerId123", + "Addresses": ["/ip4/127.0.0.1/tcp/4001/p2p/QmPeerId123"], + "AgentVersion": "kubo/0.20.0" + }))) + .mount(&server) + .await; + + let client = KuboClient::new(&server.uri(), Duration::from_secs(30), None, None).unwrap(); + let result = client.id().await.unwrap(); + + assert_eq!(result.id, "QmPeerId123"); + assert_eq!( + result.addresses, + vec!["/ip4/127.0.0.1/tcp/4001/p2p/QmPeerId123"] + ); + assert_eq!(result.agent_version, "kubo/0.20.0"); + } + + #[tokio::test] + async fn id_api_unavailable() { + let client = KuboClient::new( + "http://127.0.0.1:59999", + Duration::from_millis(100), + None, + None, + ) + .unwrap(); + let result = client.id().await; + + assert!(matches!(result, Err(KuboClientError::ApiUnavailable(_, _)))); + } + + #[tokio::test] + async fn id_api_error() { + let server = MockServer::start().await; + + Mock::given(method("POST")) + .and(path("/api/v0/id")) + .respond_with(ResponseTemplate::new(403).set_body_json(serde_json::json!({ + "Message": "access denied" + }))) + .mount(&server) + .await; + + let client = KuboClient::new(&server.uri(), Duration::from_secs(30), None, None).unwrap(); + let result = client.id().await; + + assert!(matches!(result, Err(KuboClientError::ApiError(_)))); + } + + #[tokio::test] + async fn add_success() { + let server = MockServer::start().await; + + let mut temp = NamedTempFile::new().unwrap(); + temp.write_all(b"test data").unwrap(); + let temp_path = temp.path(); + + Mock::given(method("POST")) + .and(path("/api/v0/add")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "Name": "test.txt", + "Hash": "QmTestHash123", + "Size": "12345" + }))) + .mount(&server) + .await; + + let client = KuboClient::new(&server.uri(), Duration::from_secs(30), None, None).unwrap(); + let result = client.add(temp_path, false).await.unwrap(); + + assert_eq!(result.hash, "QmTestHash123"); + assert_eq!(result.size, 12345); + } + + #[tokio::test] + async fn add_multiline_response() { + let server = MockServer::start().await; + + let mut temp = NamedTempFile::new().unwrap(); + temp.write_all(b"test data").unwrap(); + let temp_path = temp.path(); + + let multiline_body = r#"{"Name":"test.txt","Hash":"QmFirst","Size":"100"} +{"Name":"test.txt","Hash":"QmSecond","Size":"200"} +{"Name":"test.txt","Hash":"QmLastHash","Size":"300"} +"#; + + Mock::given(method("POST")) + .and(path("/api/v0/add")) + .respond_with(ResponseTemplate::new(200).set_body_string(multiline_body)) + .mount(&server) + .await; + + let client = KuboClient::new(&server.uri(), Duration::from_secs(30), None, None).unwrap(); + let result = client.add(temp_path, false).await.unwrap(); + + assert_eq!(result.hash, "QmLastHash"); + assert_eq!(result.size, 300); + } + + #[tokio::test] + async fn add_api_error() { + let server = MockServer::start().await; + + let mut temp = NamedTempFile::new().unwrap(); + temp.write_all(b"test data").unwrap(); + let temp_path = temp.path(); + + Mock::given(method("POST")) + .and(path("/api/v0/add")) + .respond_with(ResponseTemplate::new(500).set_body_json(serde_json::json!({ + "Message": "no space left on device" + }))) + .mount(&server) + .await; + + let client = KuboClient::new(&server.uri(), Duration::from_secs(30), None, None).unwrap(); + let result = client.add(temp_path, false).await; + + assert!(matches!(result, Err(KuboClientError::UploadFailed(_)))); + } + + #[tokio::test] + async fn add_file_not_found() { + let server = MockServer::start().await; + + Mock::given(method("POST")) + .and(path("/api/v0/add")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "Hash": "QmTest", + "Size": "100" + }))) + .mount(&server) + .await; + + let client = KuboClient::new(&server.uri(), Duration::from_secs(30), None, None).unwrap(); + let result = client + .add(Path::new("/nonexistent/path/file.txt"), false) + .await; + + assert!(matches!(result, Err(KuboClientError::IoError(_)))); + } + + #[tokio::test] + async fn swarm_peers_success() { + let server = MockServer::start().await; + + Mock::given(method("POST")) + .and(path("/api/v0/swarm/peers")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "Peers": [ + { + "Addr": "/ip4/192.168.1.1/tcp/4001", + "Peer": "QmPeer1" + }, + { + "Addr": "/ip4/192.168.1.2/tcp/4001", + "Peer": "QmPeer2" + } + ] + }))) + .mount(&server) + .await; + + let client = KuboClient::new(&server.uri(), Duration::from_secs(30), None, None).unwrap(); + let result = client.swarm_peers().await.unwrap(); + + assert_eq!(result.peers.len(), 2); + assert_eq!(result.peers[0].addr, "/ip4/192.168.1.1/tcp/4001"); + assert_eq!(result.peers[0].peer, "QmPeer1"); + assert_eq!(result.peers[1].addr, "/ip4/192.168.1.2/tcp/4001"); + assert_eq!(result.peers[1].peer, "QmPeer2"); + } + + #[tokio::test] + async fn swarm_peers_empty() { + let server = MockServer::start().await; + + Mock::given(method("POST")) + .and(path("/api/v0/swarm/peers")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "Peers": [] + }))) + .mount(&server) + .await; + + let client = KuboClient::new(&server.uri(), Duration::from_secs(30), None, None).unwrap(); + let result = client.swarm_peers().await.unwrap(); + + assert!(result.peers.is_empty()); + } + + #[tokio::test] + async fn swarm_peers_api_error() { + let server = MockServer::start().await; + + Mock::given(method("POST")) + .and(path("/api/v0/swarm/peers")) + .respond_with(ResponseTemplate::new(503).set_body_json(serde_json::json!({ + "Message": "service unavailable" + }))) + .mount(&server) + .await; + + let client = KuboClient::new(&server.uri(), Duration::from_secs(30), None, None).unwrap(); + let result = client.swarm_peers().await; + + assert!(matches!(result, Err(KuboClientError::ApiError(_)))); + } +} diff --git a/src/services/kubo_service.rs b/src/services/kubo_service.rs new file mode 100644 index 0000000..851312f --- /dev/null +++ b/src/services/kubo_service.rs @@ -0,0 +1,386 @@ +use std::path::Path; +use std::sync::{Arc, RwLock}; + +use async_trait::async_trait; +use thiserror::Error; +use tracing::{error, info}; + +use crate::config::Config; +use crate::services::kubo_client::{KuboClient, KuboClientError}; +use crate::services::traits::KuboServiceTrait; + +#[derive(Error, Debug)] +pub enum KuboServiceError { + #[error("Kubo API unavailable: {0}")] + ApiUnavailable(String), + #[error("Kubo API error: {0}")] + ApiError(String), + #[error("Upload failed: {0}")] + UploadFailed(String), + #[error("Pin failed: {0}")] + PinFailed(String), + #[error("IO error: {0}")] + IoError(#[from] std::io::Error), +} + +impl From for KuboServiceError { + fn from(err: KuboClientError) -> Self { + match err { + KuboClientError::ApiUnavailable(url, detail) => { + KuboServiceError::ApiUnavailable(format!("{}: {}", url, detail)) + } + KuboClientError::ApiError(msg) => KuboServiceError::ApiError(msg), + KuboClientError::UploadFailed(msg) => KuboServiceError::UploadFailed(msg), + KuboClientError::PinFailed(cid, msg) => { + KuboServiceError::PinFailed(format!("CID {}: {}", cid, msg)) + } + KuboClientError::ParseError(msg) => KuboServiceError::ApiError(msg), + KuboClientError::IoError(e) => KuboServiceError::IoError(e), + KuboClientError::RequestError(e) => KuboServiceError::ApiUnavailable(e.to_string()), + } + } +} + +#[derive(Debug, Clone, PartialEq)] +pub enum KuboServiceStatus { + Disconnected, + Connected, + Error(String), +} + +#[derive(Debug, Clone)] +pub struct UploadResult { + pub cid: String, + pub size: u64, +} + +#[derive(Debug, Clone)] +pub struct NodeInfo { + pub peer_id: String, + pub version: String, + pub agent_version: Option, + pub addresses: Vec, + pub peers_connected: usize, +} + +pub struct KuboService { + client: KuboClient, + pin_on_upload: bool, + status: Arc>, +} + +impl KuboService { + pub fn new(config: &Config) -> Result { + let client = KuboClient::new( + &config.kubo_api_url, + config.kubo_api_timeout, + config.kubo_api_username.clone(), + config.kubo_api_password.clone(), + )?; + + Ok(Self { + client, + pin_on_upload: config.pin_on_upload, + status: Arc::new(RwLock::new(KuboServiceStatus::Disconnected)), + }) + } + + pub async fn check_alive(&self) -> Result<(), KuboServiceError> { + match self.client.version().await { + Ok(_) => { + let mut status = self.status.write().unwrap_or_else(|e| e.into_inner()); + *status = KuboServiceStatus::Connected; + info!("Connected to Kubo API"); + Ok(()) + } + Err(e) => { + let mut status = self.status.write().unwrap_or_else(|e| e.into_inner()); + *status = KuboServiceStatus::Error(e.to_string()); + error!("Kubo API unreachable: {}", e); + Err(e.into()) + } + } + } + + pub fn get_status(&self) -> KuboServiceStatus { + self.status + .read() + .unwrap_or_else(|e| e.into_inner()) + .clone() + } + + pub fn is_connected(&self) -> bool { + matches!(self.get_status(), KuboServiceStatus::Connected) + } + + pub async fn get_node_info(&self) -> Result { + let id_info = self.client.id().await?; + let version_info = self.client.version().await?; + let swarm_info = self.client.swarm_peers().await?; + + Ok(NodeInfo { + peer_id: id_info.id, + version: version_info.version, + agent_version: Some(id_info.agent_version), + addresses: id_info.addresses, + peers_connected: swarm_info.peers.len(), + }) + } + + pub async fn upload_file(&self, file_path: &Path) -> Result { + if !file_path.exists() { + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("File not found: {}", file_path.display()), + ) + .into()); + } + + let file_size = tokio::fs::metadata(file_path).await?.len(); + + info!( + "Uploading file: {} ({} bytes)", + file_path.display(), + file_size + ); + + let result = self.client.add(file_path, self.pin_on_upload).await?; + + info!("Upload complete. CID: {}", result.hash); + + Ok(UploadResult { + cid: result.hash, + size: result.size, + }) + } +} + +#[async_trait] +impl KuboServiceTrait for KuboService { + async fn check_alive(&self) -> Result<(), KuboServiceError> { + self.check_alive().await + } + + fn get_status(&self) -> KuboServiceStatus { + self.get_status() + } + + async fn get_node_info(&self) -> Result { + self.get_node_info().await + } + + async fn upload_file(&self, file_path: &Path) -> Result { + self.upload_file(file_path).await + } + + fn is_connected(&self) -> bool { + self.is_connected() + } +} + +impl Clone for KuboService { + fn clone(&self) -> Self { + Self { + client: self.client.clone(), + pin_on_upload: self.pin_on_upload, + status: Arc::clone(&self.status), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + use tempfile::NamedTempFile; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + fn create_test_config(api_url: &str) -> Config { + Config { + kubo_api_url: api_url.to_string(), + kubo_api_timeout: std::time::Duration::from_secs(30), + pin_on_upload: true, + kubo_api_username: None, + kubo_api_password: None, + whosonfirst_db_path: std::path::PathBuf::from("/tmp/test.db"), + cid_db_path: std::path::PathBuf::from("/tmp/cid.db"), + areas_dir: std::path::PathBuf::from("/tmp/areas"), + bzip2_cmd: "bzip2".to_string(), + pmtiles_cmd: "pmtiles".to_string(), + target_countries: vec![], + area_ids: vec![], + max_concurrent_extractions: 4, + planet_pmtiles_location: None, + whosonfirst_db_url: "http://example.com/db".to_string(), + } + } + + #[tokio::test] + async fn new_creates_service() { + let server = MockServer::start().await; + let config = create_test_config(&server.uri()); + let service = KuboService::new(&config).unwrap(); + assert_eq!(service.get_status(), KuboServiceStatus::Disconnected); + } + + #[tokio::test] + async fn check_alive_success() { + let server = MockServer::start().await; + + Mock::given(method("POST")) + .and(path("/api/v0/version")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "Version": "0.20.0" + }))) + .mount(&server) + .await; + + let config = create_test_config(&server.uri()); + let service = KuboService::new(&config).unwrap(); + service.check_alive().await.unwrap(); + + assert_eq!(service.get_status(), KuboServiceStatus::Connected); + } + + #[tokio::test] + async fn check_alive_failure() { + let config = create_test_config("http://127.0.0.1:59999"); + let service = KuboService::new(&config).unwrap(); + let result = service.check_alive().await; + + assert!(result.is_err()); + assert!(matches!(service.get_status(), KuboServiceStatus::Error(_))); + } + + #[tokio::test] + async fn get_status_reflects_state() { + let server = MockServer::start().await; + + Mock::given(method("POST")) + .and(path("/api/v0/version")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "Version": "0.20.0" + }))) + .mount(&server) + .await; + + let config = create_test_config(&server.uri()); + let service = KuboService::new(&config).unwrap(); + assert_eq!(service.get_status(), KuboServiceStatus::Disconnected); + + service.check_alive().await.unwrap(); + assert_eq!(service.get_status(), KuboServiceStatus::Connected); + } + + #[tokio::test] + async fn get_node_info_success() { + let server = MockServer::start().await; + + Mock::given(method("POST")) + .and(path("/api/v0/id")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "ID": "QmPeerId123", + "Addresses": ["/ip4/127.0.0.1/tcp/4001/p2p/QmPeerId123"], + "AgentVersion": "kubo/0.20.0" + }))) + .mount(&server) + .await; + + Mock::given(method("POST")) + .and(path("/api/v0/version")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "Version": "0.20.0" + }))) + .mount(&server) + .await; + + Mock::given(method("POST")) + .and(path("/api/v0/swarm/peers")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "Peers": [ + {"Addr": "/ip4/192.168.1.1/tcp/4001", "Peer": "QmPeer1"}, + {"Addr": "/ip4/192.168.1.2/tcp/4001", "Peer": "QmPeer2"} + ] + }))) + .mount(&server) + .await; + + let config = create_test_config(&server.uri()); + let service = KuboService::new(&config).unwrap(); + let info = service.get_node_info().await.unwrap(); + + assert_eq!(info.peer_id, "QmPeerId123"); + assert_eq!(info.version, "0.20.0"); + assert_eq!(info.agent_version, Some("kubo/0.20.0".to_string())); + assert_eq!( + info.addresses, + vec!["/ip4/127.0.0.1/tcp/4001/p2p/QmPeerId123"] + ); + assert_eq!(info.peers_connected, 2); + } + + #[tokio::test] + async fn upload_file_success() { + let server = MockServer::start().await; + + let mut temp = NamedTempFile::new().unwrap(); + temp.write_all(b"test data").unwrap(); + + Mock::given(method("POST")) + .and(path("/api/v0/add")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "Name": "test.txt", + "Hash": "QmTestHash123", + "Size": "12345" + }))) + .mount(&server) + .await; + + let config = create_test_config(&server.uri()); + let service = KuboService::new(&config).unwrap(); + let result = service.upload_file(temp.path()).await.unwrap(); + + assert_eq!(result.cid, "QmTestHash123"); + assert_eq!(result.size, 12345); + } + + #[tokio::test] + async fn upload_file_missing_file() { + let server = MockServer::start().await; + let config = create_test_config(&server.uri()); + let service = KuboService::new(&config).unwrap(); + let result = service + .upload_file(Path::new("/nonexistent/path/file.txt")) + .await; + + assert!(result.is_err()); + assert!(matches!(result.unwrap_err(), KuboServiceError::IoError(_))); + } + + #[tokio::test] + async fn is_connected_initially_false() { + let server = MockServer::start().await; + let config = create_test_config(&server.uri()); + let service = KuboService::new(&config).unwrap(); + assert!(!service.is_connected()); + } + + #[tokio::test] + async fn is_connected_after_check() { + let server = MockServer::start().await; + + Mock::given(method("POST")) + .and(path("/api/v0/version")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "Version": "0.20.0" + }))) + .mount(&server) + .await; + + let config = create_test_config(&server.uri()); + let service = KuboService::new(&config).unwrap(); + service.check_alive().await.unwrap(); + assert!(service.is_connected()); + } +} diff --git a/src/services/mod.rs b/src/services/mod.rs index 4b28687..75a2a88 100644 --- a/src/services/mod.rs +++ b/src/services/mod.rs @@ -2,12 +2,14 @@ pub mod area_upload_service; pub mod country_service; pub mod database_service; pub mod extraction_service; -pub mod storage_service; +pub mod kubo_client; +pub mod kubo_service; +pub mod traits; pub use area_upload_service::{AreaUploadError, AreaUploadService}; pub use country_service::CountryService; pub use database_service::{DatabaseError, DatabaseService}; pub use extraction_service::{ExtractionError, ExtractionService}; -pub use storage_service::{ - DownloadResult, NodeInfo, StorageError, StorageService, StorageStatus, UploadResult, -}; +pub use kubo_client::KuboClientError; +pub use kubo_service::{KuboService, KuboServiceError, KuboServiceStatus, NodeInfo, UploadResult}; +pub use traits::KuboServiceTrait; diff --git a/src/services/storage_service.rs b/src/services/storage_service.rs deleted file mode 100644 index 7c5ad4a..0000000 --- a/src/services/storage_service.rs +++ /dev/null @@ -1,312 +0,0 @@ -use std::sync::Arc; -use storage_bindings::node::config::RepoKind; -use storage_bindings::{debug, upload_file, StorageConfig, StorageNode, LogLevel}; -use thiserror::Error; -use tokio::sync::{Mutex, RwLock}; -use tracing::info; - -#[derive(Error, Debug)] -pub enum StorageError { - #[error("Node creation failed: {0}")] - NodeCreation(String), - #[error("Node start failed: {0}")] - NodeStart(String), - #[error("Node stop failed: {0}")] - NodeStop(String), - #[error("Node not initialized")] - NodeNotInitialized, - #[error("Node not started")] - NodeNotStarted, - #[error("Upload failed: {0}")] - UploadFailed(String), - #[error("Download failed: {0}")] - DownloadFailed(String), - #[error("Connection failed: {0}")] - ConnectionFailed(String), - #[error("IO error: {0}")] - IoError(#[from] std::io::Error), -} - -#[derive(Debug, Clone, PartialEq, Default)] -pub enum StorageStatus { - #[default] - Disconnected, - Initialized, - Connecting, - Connected, - Error, -} - -#[derive(Debug, Clone)] -pub struct UploadResult { - pub cid: String, - pub size: u64, -} - -#[derive(Debug, Clone)] -pub struct DownloadResult { - pub cid: String, - pub size: usize, -} - -#[derive(Debug, Clone)] -pub struct NodeInfo { - pub peer_id: Option, - pub version: Option, - pub repo_path: Option, - pub addresses: Vec, - pub announce_addresses: Vec, - pub spr: Option, - pub discovery_node_count: usize, -} - -pub struct StorageService { - node: Arc>>, - config: StorageConfig, - status: Arc>, -} - -impl StorageService { - pub async fn new( - data_dir: &std::path::Path, - storage_quota: u64, - discovery_port: u16, - max_peers: u32, - bootstrap_nodes: Vec, - nat: String, - listen_addrs: Vec, - ) -> Result { - let mut config = StorageConfig::new() - .log_level(LogLevel::Info) - .data_dir(data_dir) - .storage_quota(storage_quota) - .max_peers(max_peers) - .discovery_port(discovery_port) - .repo_kind(RepoKind::LevelDb) - .nat(nat); - - for addr in listen_addrs { - config = config.add_listen_addr(addr); - } - - for node in bootstrap_nodes { - config = config.add_bootstrap_node(node); - } - - let service = Self { - node: Arc::new(Mutex::new(None)), - config, - status: Arc::new(RwLock::new(StorageStatus::Disconnected)), - }; - - service.initialize_node().await?; - - Ok(service) - } - - pub async fn initialize_node(&self) -> Result<(), StorageError> { - { - let mut status = self.status.write().await; - *status = StorageStatus::Connecting; - } - - { - let node_guard = self.node.lock().await; - if node_guard.is_some() { - let mut status = self.status.write().await; - *status = StorageStatus::Initialized; - return Ok(()); - } - } - - let node = StorageNode::new(self.config.clone()) - .await - .map_err(|e| StorageError::NodeCreation(e.to_string()))?; - - { - let mut node_guard = self.node.lock().await; - *node_guard = Some(node); - } - - { - let mut status = self.status.write().await; - *status = StorageStatus::Initialized; - } - - info!("Storage node initialized"); - Ok(()) - } - - pub async fn start_node(&self) -> Result<(), StorageError> { - { - let mut status = self.status.write().await; - *status = StorageStatus::Connecting; - } - - let node = { - let mut node_guard = self.node.lock().await; - match node_guard.take() { - Some(node) => node, - None => { - drop(node_guard); - self.initialize_node().await?; - let mut node_guard = self.node.lock().await; - node_guard.take().ok_or(StorageError::NodeNotInitialized)? - } - } - }; - - node.start() - .await - .map_err(|e| StorageError::NodeStart(e.to_string()))?; - - { - let mut node_guard = self.node.lock().await; - *node_guard = Some(node); - } - - { - let mut status = self.status.write().await; - *status = StorageStatus::Connected; - } - - info!("Storage node started"); - Ok(()) - } - - pub async fn stop_node(&self) -> Result<(), StorageError> { - { - let mut status = self.status.write().await; - *status = StorageStatus::Disconnected; - } - - { - let node_option = { - let mut node_guard = self.node.lock().await; - node_guard.take() - }; - - if let Some(node) = node_option { - node.stop() - .await - .map_err(|e| StorageError::NodeStop(e.to_string()))?; - - let mut node_guard = self.node.lock().await; - *node_guard = Some(node); - } - } - - { - let mut status = self.status.write().await; - *status = StorageStatus::Initialized; - } - - info!("Storage node stopped"); - Ok(()) - } - - pub async fn get_status(&self) -> StorageStatus { - self.status.read().await.clone() - } - - pub async fn get_node_info(&self) -> Result { - let node = { - let node_guard = self.node.lock().await; - node_guard - .as_ref() - .ok_or(StorageError::NodeNotInitialized)? - .clone() - }; - - let peer_id = node.peer_id().await.ok(); - let version = node.version().await.ok(); - let repo_path = node.repo().await.ok(); - - let debug_info = debug(&node).await.ok(); - let (addresses, announce_addresses, spr, discovery_node_count) = match debug_info { - Some(info) => { - let spr = if info.spr.is_empty() { None } else { Some(info.spr.clone()) }; - let node_count = info.discovery_node_count(); - (info.addrs, info.announce_addresses, spr, node_count) - } - None => (Vec::new(), Vec::new(), None, 0), - }; - - Ok(NodeInfo { - peer_id, - version, - repo_path, - addresses, - announce_addresses, - spr, - discovery_node_count, - }) - } - - pub async fn upload_file(&self, file_path: &std::path::Path) -> Result { - let node = { - let node_guard = self.node.lock().await; - node_guard - .as_ref() - .ok_or(StorageError::NodeNotInitialized)? - .clone() - }; - - if !node.is_started() { - return Err(StorageError::NodeNotStarted); - } - - if !file_path.exists() { - return Err(std::io::Error::new( - std::io::ErrorKind::NotFound, - format!("File not found: {}", file_path.display()), - ).into()); - } - - let file_size = tokio::fs::metadata(file_path).await?.len(); - - info!( - "Uploading file: {} ({} bytes)", - file_path.display(), - file_size - ); - - let file_path_owned = file_path.to_path_buf(); - let upload_options = storage_bindings::UploadOptions::new() - .filepath(&file_path_owned) - .on_progress(move |progress| { - let percentage = (progress.percentage * 100.0) as u32; - info!("Upload progress: {}%", percentage); - }); - - let result = upload_file(&node, upload_options) - .await - .map_err(|e| StorageError::UploadFailed(e.to_string()))?; - - info!("Upload complete. CID: {}", result.cid); - - Ok(UploadResult { - cid: result.cid, - size: file_size, - }) - } - - pub async fn is_started(&self) -> bool { - let node_guard = self.node.lock().await; - if let Some(node) = node_guard.as_ref() { - node.is_started() - } else { - false - } - } -} - -impl Clone for StorageService { - fn clone(&self) -> Self { - Self { - node: Arc::clone(&self.node), - config: self.config.clone(), - status: Arc::clone(&self.status), - } - } -} diff --git a/src/services/traits.rs b/src/services/traits.rs new file mode 100644 index 0000000..ff55ffe --- /dev/null +++ b/src/services/traits.rs @@ -0,0 +1,13 @@ +use async_trait::async_trait; +use std::path::Path; + +use super::{KuboServiceError, KuboServiceStatus, NodeInfo, UploadResult}; + +#[async_trait] +pub trait KuboServiceTrait: Send + Sync { + async fn check_alive(&self) -> Result<(), KuboServiceError>; + fn get_status(&self) -> KuboServiceStatus; + async fn get_node_info(&self) -> Result; + async fn upload_file(&self, file_path: &Path) -> Result; + fn is_connected(&self) -> bool; +} diff --git a/src/types/area.rs b/src/types/area.rs index 198b134..01910d9 100644 --- a/src/types/area.rs +++ b/src/types/area.rs @@ -33,34 +33,126 @@ impl AdministrativeArea { } } -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AreaInfo { - #[serde(flatten)] - pub area: AdministrativeArea, - pub file_size: u64, - pub cid: String, -} +#[cfg(test)] +mod tests { + use super::*; + use rusqlite::Connection; -impl AreaInfo { - pub fn new(area: AdministrativeArea, file_size: u64, cid: String) -> Self { - Self { - area, - file_size, - cid, + fn create_test_area() -> AdministrativeArea { + AdministrativeArea { + id: 12345, + name: "California".to_string(), + country: "US".to_string(), + placetype: "region".to_string(), + latitude: 37.5, + longitude: -119.5, + min_longitude: -124.5, + min_latitude: 32.5, + max_longitude: -114.1, + max_latitude: 42.0, } } -} -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PaginatedAreasResult { - pub areas: Vec, - pub pagination: PaginationInfo, -} + #[test] + fn administrative_area_fields() { + let area = create_test_area(); -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PaginationInfo { - pub page: u32, - pub limit: u32, - pub total: u32, - pub total_pages: u32, + assert_eq!(area.id, 12345); + assert_eq!(area.name, "California"); + assert_eq!(area.country, "US"); + assert_eq!(area.placetype, "region"); + assert_eq!(area.latitude, 37.5); + assert_eq!(area.longitude, -119.5); + assert_eq!(area.min_longitude, -124.5); + assert_eq!(area.min_latitude, 32.5); + assert_eq!(area.max_longitude, -114.1); + assert_eq!(area.max_latitude, 42.0); + } + + #[test] + fn administrative_area_clone() { + let area = create_test_area(); + let cloned = area.clone(); + + assert_eq!(area.id, cloned.id); + assert_eq!(area.name, cloned.name); + assert_eq!(area.country, cloned.country); + assert_eq!(area.placetype, cloned.placetype); + assert_eq!(area.latitude, cloned.latitude); + assert_eq!(area.longitude, cloned.longitude); + assert_eq!(area.min_longitude, cloned.min_longitude); + assert_eq!(area.min_latitude, cloned.min_latitude); + assert_eq!(area.max_longitude, cloned.max_longitude); + assert_eq!(area.max_latitude, cloned.max_latitude); + } + + #[test] + fn administrative_area_serialization() { + let area = create_test_area(); + + let json = serde_json::to_string(&area).expect("Failed to serialize"); + let deserialized: AdministrativeArea = + serde_json::from_str(&json).expect("Failed to deserialize"); + + assert_eq!(area.id, deserialized.id); + assert_eq!(area.name, deserialized.name); + assert_eq!(area.country, deserialized.country); + assert_eq!(area.placetype, deserialized.placetype); + assert_eq!(area.latitude, deserialized.latitude); + assert_eq!(area.longitude, deserialized.longitude); + assert_eq!(area.min_longitude, deserialized.min_longitude); + assert_eq!(area.min_latitude, deserialized.min_latitude); + assert_eq!(area.max_longitude, deserialized.max_longitude); + assert_eq!(area.max_latitude, deserialized.max_latitude); + } + + #[test] + fn administrative_area_from_row() { + let conn = Connection::open_in_memory().expect("Failed to create in-memory database"); + + conn.execute( + "CREATE TABLE spr ( + id INTEGER PRIMARY KEY, + name TEXT, + country TEXT, + placetype TEXT, + latitude REAL, + longitude REAL, + min_longitude REAL, + min_latitude REAL, + max_longitude REAL, + max_latitude REAL, + is_current INTEGER, + is_deprecated INTEGER + )", + [], + ) + .expect("Failed to create table"); + + conn.execute( + "INSERT INTO spr (id, name, country, placetype, latitude, longitude, min_longitude, min_latitude, max_longitude, max_latitude, is_current, is_deprecated) + VALUES (12345, 'California', 'US', 'region', 37.5, -119.5, -124.5, 32.5, -114.1, 42.0, 1, 0)", + [], + ) + .expect("Failed to insert test row"); + + let area: AdministrativeArea = conn + .query_row( + "SELECT id, name, country, placetype, latitude, longitude, min_longitude, min_latitude, max_longitude, max_latitude FROM spr WHERE id = 12345", + [], + AdministrativeArea::from_row, + ) + .expect("Failed to query row"); + + assert_eq!(area.id, 12345); + assert_eq!(area.name, "California"); + assert_eq!(area.country, "US"); + assert_eq!(area.placetype, "region"); + assert_eq!(area.latitude, 37.5); + assert_eq!(area.longitude, -119.5); + assert_eq!(area.min_longitude, -124.5); + assert_eq!(area.min_latitude, 32.5); + assert_eq!(area.max_longitude, -114.1); + assert_eq!(area.max_latitude, 42.0); + } } diff --git a/src/types/mod.rs b/src/types/mod.rs index a6d62de..a22d79f 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -1,5 +1,5 @@ pub mod area; pub mod storage; -pub use area::{AdministrativeArea, AreaInfo, PaginatedAreasResult, PaginationInfo}; +pub use area::AdministrativeArea; pub use storage::{CompletedUpload, PendingUpload, UploadQueue, UploadStats}; diff --git a/src/types/storage.rs b/src/types/storage.rs index 3e99ae9..0f0ab20 100644 --- a/src/types/storage.rs +++ b/src/types/storage.rs @@ -1,6 +1,5 @@ use std::collections::VecDeque; use std::path::PathBuf; -use thiserror::Error; #[derive(Debug, Clone)] pub struct PendingUpload { @@ -38,12 +37,6 @@ impl CompletedUpload { } } -#[derive(Debug, Error)] -pub enum QueueError { - #[error("Upload queue is full")] - QueueFull, -} - #[derive(Debug)] pub struct UploadQueue { pending_uploads: VecDeque, @@ -60,22 +53,26 @@ impl UploadQueue { } } - pub fn add_upload(&mut self, upload: PendingUpload) -> Result<(), QueueError> { + pub fn add_upload(&mut self, upload: PendingUpload) -> bool { if self.pending_uploads.len() >= self.max_queue_size { - return Err(QueueError::QueueFull); + return false; } self.pending_uploads.push_back(upload); - Ok(()) + true } pub fn take_batch(&mut self) -> Vec { let batch_size = std::cmp::min(self.batch_size, self.pending_uploads.len()); - (0..batch_size) - .map(|_| self.pending_uploads.pop_front().unwrap()) - .collect() + let mut batch = Vec::with_capacity(batch_size); + for _ in 0..batch_size { + if let Some(upload) = self.pending_uploads.pop_front() { + batch.push(upload); + } + } + batch } - pub fn is_full(&self) -> bool { + pub fn is_batch_ready(&self) -> bool { self.pending_uploads.len() >= self.batch_size } @@ -96,10 +93,6 @@ pub struct UploadStats { } impl UploadStats { - pub fn new() -> Self { - Self::default() - } - pub fn increment_uploaded(&mut self, bytes: u64) { self.total_uploaded += 1; self.total_bytes_uploaded += bytes; @@ -109,3 +102,160 @@ impl UploadStats { self.total_failed += 1; } } + +#[cfg(test)] +mod tests { + use super::*; + + fn sample_pending_upload() -> PendingUpload { + PendingUpload::new("US".to_string(), 12345, PathBuf::from("/tmp/test.pmtiles")) + } + + #[test] + fn pending_upload_new() { + let upload = + PendingUpload::new("DE".to_string(), 54321, PathBuf::from("/data/test.pmtiles")); + assert_eq!(upload.country_code, "DE"); + assert_eq!(upload.area_id, 54321); + assert_eq!(upload.file_path, PathBuf::from("/data/test.pmtiles")); + } + + #[test] + fn completed_upload_new() { + let completed = + CompletedUpload::new("FR".to_string(), 11111, "QmTestCid123".to_string(), 98765); + assert_eq!(completed.country_code, "FR"); + assert_eq!(completed.area_id, 11111); + assert_eq!(completed.cid, "QmTestCid123"); + assert_eq!(completed.file_size, 98765); + } + + #[test] + fn upload_queue_new() { + let queue = UploadQueue::new(10, 100); + assert!(queue.is_empty()); + assert_eq!(queue.len(), 0); + } + + #[test] + fn upload_queue_add_success() { + let mut queue = UploadQueue::new(10, 100); + let result = queue.add_upload(sample_pending_upload()); + assert!(result); + } + + #[test] + fn upload_queue_add_when_full() { + let mut queue = UploadQueue::new(10, 3); + queue.add_upload(sample_pending_upload()); + queue.add_upload(sample_pending_upload()); + queue.add_upload(sample_pending_upload()); + let result = queue.add_upload(sample_pending_upload()); + assert!(!result); + } + + #[test] + fn upload_queue_add_increments_len() { + let mut queue = UploadQueue::new(10, 100); + assert_eq!(queue.len(), 0); + queue.add_upload(sample_pending_upload()); + assert_eq!(queue.len(), 1); + queue.add_upload(sample_pending_upload()); + assert_eq!(queue.len(), 2); + } + + #[test] + fn upload_queue_take_batch_empty() { + let mut queue = UploadQueue::new(10, 100); + let batch = queue.take_batch(); + assert!(batch.is_empty()); + } + + #[test] + fn upload_queue_take_batch_partial() { + let mut queue = UploadQueue::new(10, 100); + queue.add_upload(sample_pending_upload()); + queue.add_upload(sample_pending_upload()); + let batch = queue.take_batch(); + assert_eq!(batch.len(), 2); + } + + #[test] + fn upload_queue_take_batch_full() { + let mut queue = UploadQueue::new(3, 100); + for _ in 0..5 { + queue.add_upload(sample_pending_upload()); + } + let batch = queue.take_batch(); + assert_eq!(batch.len(), 3); + } + + #[test] + fn upload_queue_take_batch_removes_items() { + let mut queue = UploadQueue::new(3, 100); + for _ in 0..5 { + queue.add_upload(sample_pending_upload()); + } + let batch = queue.take_batch(); + assert_eq!(batch.len(), 3); + assert_eq!(queue.len(), 2); + } + + #[test] + fn upload_queue_is_batch_ready_false() { + let mut queue = UploadQueue::new(5, 100); + queue.add_upload(sample_pending_upload()); + queue.add_upload(sample_pending_upload()); + assert!(!queue.is_batch_ready()); + } + + #[test] + fn upload_queue_is_batch_ready_true() { + let mut queue = UploadQueue::new(3, 100); + queue.add_upload(sample_pending_upload()); + queue.add_upload(sample_pending_upload()); + queue.add_upload(sample_pending_upload()); + assert!(queue.is_batch_ready()); + } + + #[test] + fn upload_queue_is_empty() { + let mut queue = UploadQueue::new(10, 100); + assert!(queue.is_empty()); + queue.add_upload(sample_pending_upload()); + assert!(!queue.is_empty()); + queue.take_batch(); + assert!(queue.is_empty()); + } + + #[test] + fn upload_queue_len() { + let mut queue = UploadQueue::new(10, 100); + assert_eq!(queue.len(), 0); + queue.add_upload(sample_pending_upload()); + assert_eq!(queue.len(), 1); + queue.add_upload(sample_pending_upload()); + assert_eq!(queue.len(), 2); + queue.take_batch(); + assert_eq!(queue.len(), 0); + } + + #[test] + fn upload_stats_default() { + let stats = UploadStats::default(); + assert_eq!(stats.total_uploaded, 0); + assert_eq!(stats.total_failed, 0); + assert_eq!(stats.total_bytes_uploaded, 0); + } + + #[test] + fn upload_stats_record_upload() { + let mut stats = UploadStats::default(); + stats.increment_uploaded(1000); + assert_eq!(stats.total_uploaded, 1); + assert_eq!(stats.total_bytes_uploaded, 1000); + stats.increment_uploaded(500); + assert_eq!(stats.total_uploaded, 2); + assert_eq!(stats.total_bytes_uploaded, 1500); + } +} diff --git a/src/utils/file.rs b/src/utils/file.rs index ec480f4..446c931 100644 --- a/src/utils/file.rs +++ b/src/utils/file.rs @@ -44,7 +44,7 @@ pub async fn download_file_with_progress(url: &str, destination: &Path) -> Resul tokio::time::sleep(tokio::time::Duration::from_secs(RETRY_DELAY_SECS)).await; } else { // Clean up temp file on final failure - let _ = tokio::fs::remove_file(&temp_path).await; + drop(tokio::fs::remove_file(&temp_path).await); return Err(e); } } @@ -72,13 +72,11 @@ fn get_temp_path(destination: &Path) -> PathBuf { /// Create a progress bar with standard styling fn create_progress_bar(total_size: u64) -> ProgressBar { let pb = ProgressBar::new(total_size); - pb.set_style( - ProgressStyle::with_template( - "{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {bytes}/{total_bytes} ({eta})", - ) - .unwrap() - .progress_chars("#>-"), - ); + let style = ProgressStyle::with_template( + "{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {bytes}/{total_bytes} ({eta})", + ) + .unwrap_or_else(|_| ProgressStyle::default_bar()); + pb.set_style(style.progress_chars("#>-")); pb } @@ -125,7 +123,7 @@ async fn download_attempt( // Parse total size from "bytes start-end/total" let total = content_range .split('/') - .last() + .next_back() .and_then(|s| s.parse::().ok()) .unwrap_or(existing_size); @@ -194,3 +192,123 @@ async fn download_attempt( info!("Download completed: {}", temp_path.display()); Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + use wiremock::{ + matchers::{method, path}, + Mock, MockServer, ResponseTemplate, + }; + + fn setup_time_mock() { + tokio::time::pause(); + } + + #[tokio::test] + async fn download_success() { + let server = MockServer::start().await; + + Mock::given(method("GET")) + .and(path("/file.db")) + .respond_with(ResponseTemplate::new(200).set_body_string("test database content")) + .mount(&server) + .await; + + let temp_dir = TempDir::new().unwrap(); + let dest = temp_dir.path().join("file.db"); + + let result = download_file_with_progress(&format!("{}/file.db", server.uri()), &dest).await; + + assert!(result.is_ok()); + } + + #[tokio::test] + async fn download_creates_file() { + let server = MockServer::start().await; + + Mock::given(method("GET")) + .and(path("/data.bin")) + .respond_with(ResponseTemplate::new(200).set_body_string("binary data")) + .mount(&server) + .await; + + let temp_dir = TempDir::new().unwrap(); + let dest = temp_dir.path().join("data.bin"); + + download_file_with_progress(&format!("{}/data.bin", server.uri()), &dest) + .await + .unwrap(); + + assert!(dest.exists()); + } + + #[tokio::test] + async fn download_http_error_404() { + setup_time_mock(); + + let server = MockServer::start().await; + + Mock::given(method("GET")) + .and(path("/missing.db")) + .respond_with(ResponseTemplate::new(404)) + .mount(&server) + .await; + + let temp_dir = TempDir::new().unwrap(); + let dest = temp_dir.path().join("missing.db"); + + let result = + download_file_with_progress(&format!("{}/missing.db", server.uri()), &dest).await; + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(err.to_string().contains("HTTP error")); + } + + #[tokio::test] + async fn download_http_error_500() { + setup_time_mock(); + + let server = MockServer::start().await; + + Mock::given(method("GET")) + .and(path("/error.db")) + .respond_with(ResponseTemplate::new(500)) + .mount(&server) + .await; + + let temp_dir = TempDir::new().unwrap(); + let dest = temp_dir.path().join("error.db"); + + let result = + download_file_with_progress(&format!("{}/error.db", server.uri()), &dest).await; + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(err.to_string().contains("HTTP error")); + } + + #[tokio::test] + async fn download_content_correct() { + let server = MockServer::start().await; + + let expected_content = "test database content with specific data"; + Mock::given(method("GET")) + .and(path("/file.db")) + .respond_with(ResponseTemplate::new(200).set_body_string(expected_content)) + .mount(&server) + .await; + + let temp_dir = TempDir::new().unwrap(); + let dest = temp_dir.path().join("file.db"); + + download_file_with_progress(&format!("{}/file.db", server.uri()), &dest) + .await + .unwrap(); + + let content = std::fs::read_to_string(&dest).unwrap(); + assert_eq!(content, expected_content); + } +}