diff --git a/CHANGELOG.md b/CHANGELOG.md index 10c1e25..75b08dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,40 @@ # Changelog +## Unreleased — resourceType rename + queued time + ip dim + gauge fill + +### Breaking + +- Renamed the `resource` dimension to `resourceType` across the + events, gauges, and events-daily tables. All Metric column + constants, schema definitions, indexes, projections + (`p_by_resource*` → `p_by_resourceType*`), materialized-view + dimension list, and the `Metric::getResource()` getter (now + `Metric::getResourceType()`) follow the new name. Callers must + rename the `resource` tag key on writes and any queries that + filter/group by the old name. A one-time ClickHouse column rename + plus data backfill is required for existing deployments; that + migration lives in the cloud repo, not in this library. + +### Added + +- `Accumulator::collect()` accepts an optional `\DateTime $time` + argument. When provided, the row is written at that moment; when + omitted the ClickHouse adapter still stamps `now()`. Events with + the same (tenant, metric, tags) fold into the earliest supplied + time; gauges follow last-write-wins on both value and time. +- `Usage::addBatch()` payloads may carry a `time` field per row + (`DateTime|string`) — the ClickHouse adapter threads it through + `formatDateTime()`. +- New event-only `ip` dimension in `Metric::EVENT_COLUMNS` and + `Metric::getEventSchema()`. Stored as + `LowCardinality(Nullable(String))` in ClickHouse, indexed with a + `bloom_filter`, size 45 chars (IPv6 max). Gauges are unchanged. + `Metric::getIp()` is a typed accessor. The base-table primary key + is intentionally not extended with `ip`. +- Gauge time-series reads now carry the last observed value forward + across empty buckets inside the window (LOCF) instead of collapsing + to zero. Events keep zero-fill. Return shape is unchanged. + ## 0.4.0 — Per-dimension ClickHouse projections and auto-routing This release accelerates grouped reads against the events and gauges diff --git a/README.md b/README.md index 9591632..9272193 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ $accumulator->collect('project_123', 'bandwidth', 5000, Usage::TYPE_EVENT, [ 'method' => 'POST', 'status' => '201', 'service' => 'storage', - 'resource' => 'bucket', + 'resourceType' => 'bucket', 'resourceId' => 'abc123', 'resourceInternalId' => '42', 'teamId' => 'team_x', diff --git a/src/Usage/Accumulator.php b/src/Usage/Accumulator.php index 965c229..b898be9 100644 --- a/src/Usage/Accumulator.php +++ b/src/Usage/Accumulator.php @@ -16,12 +16,7 @@ class Accumulator private Usage $usage; /** - * In-memory buffer for metrics. - * Keyed by "{tenant}:{metric}:{type}:{tagsHash}" — events are summed, gauges - * use last-write-wins. Tenant is part of the key so metrics for different - * tenants never collapse into one entry. - * - * @var array}> + * @var array, time?: \DateTime}> */ private array $buffer = []; @@ -40,18 +35,12 @@ public function __construct(Usage $usage) /** * Collect a metric into the in-memory buffer for deferred flushing. * - * For event type: multiple collect() calls for the same metric are summed. - * For gauge type: last-write-wins semantics. - * No period fan-out — raw timestamps are used. + * Events fold additively; earliest non-null time wins on merge. + * Gauges use last-write-wins. * - * @param string $tenant Tenant scope (shared-tables mode) - * @param string $metric Metric name - * @param int $value Value - * @param string $type Metric type: 'event' or 'gauge' - * @param array $tags Optional tags - * @return self + * @param array $tags */ - public function collect(string $tenant, string $metric, int $value, string $type, array $tags = []): self + public function collect(string $tenant, string $metric, int $value, string $type, array $tags = [], ?\DateTime $time = null): self { // Compare against '' rather than empty(): the string "0" is a valid // tenant/metric id but empty("0") is true in PHP. @@ -77,17 +66,23 @@ public function collect(string $tenant, string $metric, int $value, string $type $key = md5(json_encode([$tenant, $metric, $type, $canonicalTags], JSON_THROW_ON_ERROR)); if ($type === Usage::TYPE_EVENT && isset($this->buffer[$key])) { - // Additive: sum values for the same tenant + metric + tags combination + // earliest time wins on merge $this->buffer[$key]['value'] += $value; + if ($time !== null && (!isset($this->buffer[$key]['time']) || $time < $this->buffer[$key]['time'])) { + $this->buffer[$key]['time'] = $time; + } } else { - // New event entry, or gauge (last-write-wins) - $this->buffer[$key] = [ + $entry = [ 'tenant' => $tenant, 'metric' => $metric, 'value' => $value, 'type' => $type, 'tags' => $tags, ]; + if ($time !== null) { + $entry['time'] = $time; + } + $this->buffer[$key] = $entry; } return $this; diff --git a/src/Usage/Adapter.php b/src/Usage/Adapter.php index ddbf659..135f5de 100644 --- a/src/Usage/Adapter.php +++ b/src/Usage/Adapter.php @@ -141,7 +141,7 @@ abstract public function sum(string $tenant, array $queries = [], string $attrib * the daily events table — gauges are never pre-aggregated. * * @param string $tenant Tenant scope (shared-tables mode) - * @param array<\Utopia\Query\Query> $queries Filters (metric, time range, resource, etc.) + * @param array<\Utopia\Query\Query> $queries Filters (metric, time range, resourceType, etc.) * @return array */ abstract public function findDaily(string $tenant, array $queries = []): array; diff --git a/src/Usage/Adapter/ClickHouse.php b/src/Usage/Adapter/ClickHouse.php index 9cf0cd5..8a88899 100644 --- a/src/Usage/Adapter/ClickHouse.php +++ b/src/Usage/Adapter/ClickHouse.php @@ -23,7 +23,7 @@ * This adapter stores usage metrics in ClickHouse using HTTP interface. * Uses two separate tables: * - Events table (MergeTree): raw request events with metadata columns - * (path, method, status, resource, resourceId) + * (path, method, status, resourceType, resourceId) * - Gauges table (MergeTree): simple resource snapshots (metric, value, time, tags) * * A SummingMergeTree materialized view pre-aggregates events by day for fast @@ -694,9 +694,9 @@ private function formatParamValue(mixed $value): string */ private const GAUGE_PROJECTIONS = [ ['name' => 'p_by_service', 'dims' => ['service']], - ['name' => 'p_by_resource', 'dims' => ['resource']], + ['name' => 'p_by_resourceType', 'dims' => ['resourceType']], ['name' => 'p_by_resourceId', 'dims' => ['resourceId']], - ['name' => 'p_by_resource_resourceId', 'dims' => ['resource', 'resourceId']], + ['name' => 'p_by_resourceType_resourceId', 'dims' => ['resourceType', 'resourceId']], ]; /** @@ -727,6 +727,8 @@ public function setup(): void $this->getEventIndexes() ); + $this->ensureEventDimColumns(); + // --- Events daily table (SummingMergeTree) --- $this->createDailyTable(); @@ -776,23 +778,41 @@ private function setLightweightMutationProjectionMode(string $baseTable): void $this->query($sql); } + private const BASE_KEY_COLUMNS = ['id', 'metric', 'value', 'time', 'tenant']; + + private function ensureGaugeDimColumns(): void + { + $this->ensureDimColumns($this->getGaugesTableName(), Metric::GAUGE_COLUMNS, 'gauge'); + } + + private function ensureEventDimColumns(): void + { + $this->ensureDimColumns($this->getEventsTableName(), Metric::EVENT_COLUMNS, 'event'); + } + /** - * Backfill the service / resource columns on an existing gauges table. - * setup() uses CREATE TABLE IF NOT EXISTS, so deployments that came up - * before these columns were added never receive them — the gauge - * projections would then fail because their SELECT references columns - * the source table lacks. + * @param array $columns */ - private function ensureGaugeDimColumns(): void + private function ensureDimColumns(string $tableName, array $columns, string $type): void { - $gaugesTable = $this->escapeIdentifier($this->database) - . '.' . $this->escapeIdentifier($this->getGaugesTableName()); + $escapedTable = $this->escapeIdentifier($this->database) + . '.' . $this->escapeIdentifier($tableName); - $sql = "ALTER TABLE {$gaugesTable} " - . 'ADD COLUMN IF NOT EXISTS service LowCardinality(Nullable(String)), ' - . 'ADD COLUMN IF NOT EXISTS resource LowCardinality(Nullable(String))'; + $adds = []; + foreach ($columns as $column) { + if (in_array($column, self::BASE_KEY_COLUMNS, true)) { + continue; + } + $adds[] = 'ADD COLUMN IF NOT EXISTS ' + . $this->escapeIdentifier($column) + . ' ' . $this->getColumnType($column, $type); + } - $this->query($sql); + if ($adds === []) { + return; + } + + $this->query("ALTER TABLE {$escapedTable} " . implode(', ', $adds)); } /** @@ -916,7 +936,7 @@ private function createDailyTable(): void 'metric String', 'value Int64', "time DateTime64(3, 'UTC')", - 'resource LowCardinality(Nullable(String))', + 'resourceType LowCardinality(Nullable(String))', 'resourceId Nullable(String)', 'resourceInternalId Nullable(String)', 'teamId Nullable(String)', @@ -930,8 +950,8 @@ private function createDailyTable(): void $columnDefs = implode(",\n ", $columns); $dailyOrderBy = $this->sharedTables - ? '(tenant, metric, time, resource, resourceId, resourceInternalId, teamId, teamInternalId)' - : '(metric, time, resource, resourceId, resourceInternalId, teamId, teamInternalId)'; + ? '(tenant, metric, time, resourceType, resourceId, resourceInternalId, teamId, teamInternalId)' + : '(metric, time, resourceType, resourceId, resourceInternalId, teamId, teamInternalId)'; $createDailyTableSql = " CREATE TABLE IF NOT EXISTS {$escapedDailyTable} ( @@ -961,7 +981,7 @@ private function createDailyMaterializedView(): void $escapedDailyTable = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($dailyTableName); $escapedDailyMv = $this->escapeIdentifier($this->database) . '.' . $this->escapeIdentifier($dailyMvName); - $dimensions = 'resource, resourceId, resourceInternalId, teamId, teamInternalId'; + $dimensions = 'resourceType, resourceId, resourceInternalId, teamId, teamInternalId'; if ($this->sharedTables) { $innerSelect = "metric, tenant, {$dimensions}, sum(value) as value, toStartOfDay(time, 'UTC') as d"; @@ -1045,7 +1065,7 @@ private function validateGroupByAttribute(string $attribute, string $type): bool */ private const DAILY_COLUMNS = [ 'metric', 'value', 'time', - 'resource', 'resourceId', 'resourceInternalId', + 'resourceType', 'resourceId', 'resourceInternalId', 'teamId', 'teamInternalId', ]; @@ -1122,12 +1142,12 @@ private function getColumnType(string $id, string $type = 'event'): string } $lowCardinality = [ - 'country', 'region', 'service', 'resource', + 'country', 'region', 'service', 'resourceType', 'osCode', 'osName', 'osVersion', 'clientType', 'clientCode', 'clientName', 'clientVersion', 'clientEngine', 'clientEngineVersion', 'deviceName', 'deviceBrand', 'deviceModel', - 'hostname', + 'hostname', 'ip', ]; if (in_array($id, $lowCardinality, true)) { @@ -1257,7 +1277,7 @@ private function validateMetricsBatch(array $metrics, string $type): void /** * Add metrics in batch (raw append to appropriate table). * - * For events: extracts path/method/status/resource/resourceId from tags into + * For events: extracts path/method/status/resourceType/resourceId from tags into * dedicated columns; remaining tags stay in the tags JSON column. * For gauges: simple metric/value/time/tags insert. * @@ -1296,11 +1316,16 @@ public function addBatch(array $metrics, string $type, int $batchSize = self::IN $columns = Metric::extractColumns($tags, $type); + $rawTime = $metricData['time'] ?? null; + $emittedAt = ($rawTime instanceof DateTime || is_string($rawTime)) + ? $rawTime + : null; + $row = array_merge([ 'id' => $this->generateId(), 'metric' => $metric, 'value' => $value, - 'time' => $this->formatDateTime(null), + 'time' => $this->formatDateTime($emittedAt), ], $columns); if ($this->sharedTables) { @@ -2497,7 +2522,7 @@ public function findDaily(string $tenant, array $queries = []): array $groupByColumns = $this->sharedTables ? ['tenant'] : []; $groupByColumns[] = 'metric'; $groupByColumns[] = 'time'; - foreach (['resource', 'resourceId', 'resourceInternalId', 'teamId', 'teamInternalId'] as $dim) { + foreach (['resourceType', 'resourceId', 'resourceInternalId', 'teamId', 'teamInternalId'] as $dim) { $groupByColumns[] = $dim; } @@ -2675,6 +2700,8 @@ public function getTimeSeries(string $tenant, array $metrics, string $interval, $typesToQuery[] = Usage::TYPE_GAUGE; } + $metricTypes = []; + foreach ($typesToQuery as $queryType) { // Skip a table when its schema can't satisfy every filter attribute // (e.g. `path` on a gauge query); avoids "Invalid attribute name" @@ -2691,6 +2718,10 @@ public function getTimeSeries(string $tenant, array $metrics, string $interval, continue; } + if (!empty($metricData['data'])) { + $metricTypes[$metricName] = $queryType; + } + $output[$metricName]['total'] += $metricData['total']; $output[$metricName]['data'] = array_merge( $output[$metricName]['data'], @@ -2699,15 +2730,24 @@ public function getTimeSeries(string $tenant, array $metrics, string $interval, } } - // Zero-fill gaps if requested if ($zeroFill) { foreach ($output as $metricName => &$metricData) { - $metricData['data'] = $this->zeroFillTimeSeries( - $metricData['data'], - $interval, - $startDate, - $endDate - ); + $fillType = $metricTypes[$metricName] ?? $type ?? Usage::TYPE_EVENT; + if ($fillType === Usage::TYPE_GAUGE) { + $metricData['data'] = $this->locfFillTimeSeries( + $metricData['data'], + $interval, + $startDate, + $endDate + ); + } else { + $metricData['data'] = $this->zeroFillTimeSeries( + $metricData['data'], + $interval, + $startDate, + $endDate + ); + } } unset($metricData); } @@ -2830,7 +2870,7 @@ private function zeroFillTimeSeries(array $data, string $interval, string $start $dt = new DateTime($point['date']); $key = $dt->format($format); // If multiple points in the same bucket, sum them - $existing[$key] = ($existing[$key] ?? 0) + $point['value']; + $existing[$key] = ($existing[$key] ?? 0.0) + $point['value']; } // Generate all time buckets in range @@ -2843,7 +2883,55 @@ private function zeroFillTimeSeries(array $data, string $interval, string $start while ($current <= $end) { $key = $current->format($format); $result[] = [ - 'value' => $existing[$key] ?? 0, + 'value' => $existing[$key] ?? 0.0, + 'date' => $key, + ]; + $current->modify($step); + } + + return $result; + } + + /** + * Fill missing gauge buckets by carrying the last observation forward. + * + * Multiple points in the same bucket collapse to the most recent point's + * value (last-write-wins), matching argMax(value, time) on the write side. + * + * @param array $data + * @return array + */ + private function locfFillTimeSeries(array $data, string $interval, string $startDate, string $endDate): array + { + $format = $interval === '1h' ? 'Y-m-d\TH:00:00+00:00' : 'Y-m-d\T00:00:00+00:00'; + $step = $interval === '1h' ? '+1 hour' : '+1 day'; + + usort($data, fn (array $a, array $b): int => strcmp($a['date'], $b['date'])); + + $existing = []; + foreach ($data as $point) { + $dt = new DateTime($point['date']); + $key = $dt->format($format); + $existing[$key] = $point['value']; + } + + $start = new DateTime($startDate); + $end = new DateTime($endDate); + + $result = []; + $current = clone $start; + $lastValue = 0.0; + $seenAny = false; + + while ($current <= $end) { + $key = $current->format($format); + if (array_key_exists($key, $existing)) { + $lastValue = $existing[$key]; + $seenAny = true; + } + $result[] = [ + // Pre-window buckets fall back to 0 rather than fabricating a value. + 'value' => $seenAny ? $lastValue : 0.0, 'date' => $key, ]; $current->modify($step); diff --git a/src/Usage/Metric.php b/src/Usage/Metric.php index c597685..c68b485 100644 --- a/src/Usage/Metric.php +++ b/src/Usage/Metric.php @@ -24,7 +24,7 @@ * 'method' => 'POST', * 'status' => '201', * 'service' => 'storage', - * 'resource' => 'bucket', + * 'resourceType' => 'bucket', * 'resourceId' => 'abc123', * 'resourceInternalId' => '42', * 'teamId' => 'team_x', @@ -51,9 +51,9 @@ class Metric extends ArrayObject */ public const EVENT_COLUMNS = [ 'path', 'method', 'status', - 'service', 'resource', 'resourceId', 'resourceInternalId', + 'service', 'resourceType', 'resourceId', 'resourceInternalId', 'teamId', 'teamInternalId', - 'country', 'region', 'hostname', + 'country', 'region', 'hostname', 'ip', 'osCode', 'osName', 'osVersion', 'clientType', 'clientCode', 'clientName', 'clientVersion', 'clientEngine', 'clientEngineVersion', @@ -63,7 +63,7 @@ class Metric extends ArrayObject /** * Gauge-specific column names that are extracted from tags into dedicated columns. */ - public const GAUGE_COLUMNS = ['service', 'resource', 'teamId', 'teamInternalId', 'resourceId', 'resourceInternalId']; + public const GAUGE_COLUMNS = ['service', 'resourceType', 'teamId', 'teamInternalId', 'resourceId', 'resourceInternalId']; /** * Construct a new metric object. @@ -79,9 +79,9 @@ class Metric extends ArrayObject * Event-only dimension columns (see EVENT_COLUMNS): * - path / method / status: HTTP shape * - service: API service segment (storage, databases, …) - * - resource / resourceId / resourceInternalId: resource identity + * - resourceType / resourceId / resourceInternalId: resource identity * - teamId / teamInternalId: owning team identity - * - country / region / hostname: geographic + caller origin + * - country / region / hostname / ip: geographic + caller origin * - osCode / osName / osVersion: parsed user-agent OS fields * - clientType / clientCode / clientName / clientVersion: parsed client * - clientEngine / clientEngineVersion: parsed client engine @@ -219,10 +219,10 @@ public function getStatus(): ?string * * @return string|null The resource type, or null if not set */ - public function getResource(): ?string + public function getResourceType(): ?string { - $resource = $this->getAttribute('resource', null); - return is_string($resource) ? $resource : null; + $resourceType = $this->getAttribute('resourceType', null); + return is_string($resourceType) ? $resourceType : null; } /** @@ -305,6 +305,15 @@ public function getHostname(): ?string return is_string($v) ? $v : null; } + /** + * Get caller IP address (event metrics). + */ + public function getIp(): ?string + { + $v = $this->getAttribute('ip', null); + return is_string($v) ? $v : null; + } + /** * Get OS short code (event metrics). */ @@ -561,7 +570,7 @@ public function toArray(): array * * Returns the attribute schema for the events table which stores * raw request events with metadata columns for path, method, status, - * resource, and resourceId. + * resourceType, and resourceId. * * @return array> */ @@ -610,7 +619,7 @@ public static function getEventSchema(): array $stringColumn('method', 16), $stringColumn('status', 16), $stringColumn('service', 256), - $stringColumn('resource', 256), + $stringColumn('resourceType', 256), $stringColumn('resourceId', 255), $stringColumn('resourceInternalId', 255), $stringColumn('teamId', 255), @@ -618,6 +627,7 @@ public static function getEventSchema(): array $stringColumn('country', 2), $stringColumn('region', 64), $stringColumn('hostname', 255), + $stringColumn('ip', 45), $stringColumn('osCode', 256), $stringColumn('osName', 256), $stringColumn('osVersion', 255), @@ -683,7 +693,7 @@ public static function getGaugeSchema(): array 'filters' => ['datetime'], ], $stringColumn('service', 256), - $stringColumn('resource', 256), + $stringColumn('resourceType', 256), $stringColumn('teamId', 255), $stringColumn('teamInternalId', 255), $stringColumn('resourceId', 255), @@ -713,9 +723,9 @@ public static function getEventIndexes(): array { $indexed = [ 'path', 'method', 'status', - 'service', 'resource', 'resourceId', 'resourceInternalId', + 'service', 'resourceType', 'resourceId', 'resourceInternalId', 'teamId', 'teamInternalId', - 'country', 'region', 'hostname', + 'country', 'region', 'hostname', 'ip', 'osName', 'clientType', 'clientName', 'deviceName', ]; @@ -745,9 +755,9 @@ static function (string $col) use ($setIndexed): array { */ public static function getGaugeIndexes(): array { - $indexed = ['service', 'resource', 'resourceId', 'resourceInternalId', 'teamId', 'teamInternalId']; + $indexed = ['service', 'resourceType', 'resourceId', 'resourceInternalId', 'teamId', 'teamInternalId']; - $setIndexed = ['service', 'resource']; + $setIndexed = ['service', 'resourceType']; return array_map( static fn (string $col): array => [ diff --git a/src/Usage/Usage.php b/src/Usage/Usage.php index 7897158..a5f65bc 100644 --- a/src/Usage/Usage.php +++ b/src/Usage/Usage.php @@ -66,7 +66,7 @@ public function setup(): void * Each metric carries its own `tenant` (shared-tables mode), so a single * batch may span multiple tenants. * - * @param array}> $metrics + * @param array, time?: \DateTime|string}> $metrics * @param string $type Metric type: 'event' or 'gauge' * @param int $batchSize Maximum number of metrics per INSERT statement * @return bool diff --git a/tests/Benchmark/EventsBench.php b/tests/Benchmark/EventsBench.php index 1e4c1e3..ad00ada 100644 --- a/tests/Benchmark/EventsBench.php +++ b/tests/Benchmark/EventsBench.php @@ -117,12 +117,12 @@ public function testBenchmarks(): void ], Usage::TYPE_EVENT); }); - $this->runBench('bench_events_topN_path_30d_filtered_resource', function (string $queryId) use ($start, $end): void { + $this->runBench('bench_events_topN_path_30d_filtered_resourceType', function (string $queryId) use ($start, $end): void { $this->adapter->setNextQueryId($queryId); $this->usage->find($this->tenant, [ UsageQuery::groupBy('path'), Query::equal('metric', [$this->metric]), - Query::equal('resource', ['function']), + Query::equal('resourceType', ['function']), Query::greaterThanEqual('time', $start), Query::lessThanEqual('time', $end), Query::limit(500), diff --git a/tests/Benchmark/GaugesBench.php b/tests/Benchmark/GaugesBench.php index f84eee2..41f4a56 100644 --- a/tests/Benchmark/GaugesBench.php +++ b/tests/Benchmark/GaugesBench.php @@ -49,10 +49,10 @@ public function testBenchmarks(): void ], Usage::TYPE_GAUGE); }); - $this->runBench('bench_gauges_topN_resource_30d', function (string $queryId) use ($start30d, $endClosed): void { + $this->runBench('bench_gauges_topN_resourceType_30d', function (string $queryId) use ($start30d, $endClosed): void { $this->adapter->setNextQueryId($queryId); $this->usage->find($this->tenant, [ - UsageQuery::groupBy('resource'), + UsageQuery::groupBy('resourceType'), Query::equal('metric', [$this->metric]), Query::greaterThanEqual('time', $start30d), Query::lessThanEqual('time', $endClosed), @@ -75,7 +75,7 @@ public function testBenchmarks(): void $expectedProjections = [ 'bench_gauges_topN_service_30d' => 'p_by_service', - 'bench_gauges_topN_resource_30d' => 'p_by_resource', + 'bench_gauges_topN_resourceType_30d' => 'p_by_resourceType', 'bench_gauges_topN_service_today_partial' => 'p_by_service', ]; foreach ($expectedProjections as $scenario => $projection) { diff --git a/tests/Benchmark/README.md b/tests/Benchmark/README.md index 025356e..1b66e36 100644 --- a/tests/Benchmark/README.md +++ b/tests/Benchmark/README.md @@ -77,7 +77,7 @@ Gauges (`GaugesBench`): |---|---|---| | `bench_gauges_latest_in_window` | gauge `getTotal` argMax | p50 < 50ms | | `bench_gauges_topN_service_30d` | closed-day gauge by_service AMT | p50 < 100ms | -| `bench_gauges_topN_resource_30d` | closed-day gauge by_resource AMT | p50 < 100ms | +| `bench_gauges_topN_resourceType_30d` | closed-day gauge by_resourceType AMT | p50 < 100ms | | `bench_gauges_topN_service_today_partial` | gauge by_service hybrid | p50 < 80ms | Budgets are guidance, not gates. diff --git a/tests/Benchmark/fixtures/seed.sql b/tests/Benchmark/fixtures/seed.sql index 5bc2698..03b1273 100644 --- a/tests/Benchmark/fixtures/seed.sql +++ b/tests/Benchmark/fixtures/seed.sql @@ -10,10 +10,10 @@ -- * 1000 distinct paths (api-style surface) — bounded but high-cardinality. -- * 6 HTTP methods, 6 status codes — bounded cross-product (~36 keys). -- * 30-day span, 1-minute resolution — matches expected production density. --- * Country / service / resource cycled for routing-fallback tests. +-- * Country / service / resourceType cycled for routing-fallback tests. INSERT INTO {TABLE} (id, metric, value, time, path, method, status, service, country, - region, hostname, resource, resourceId, tenant) + region, hostname, resourceType, resourceId, tenant) SELECT lower(hex(randomString(16))) AS id, '{METRIC}' AS metric, @@ -38,7 +38,7 @@ SELECT concat('host-', toString(number % 10), '.example.com') AS hostname, [ 'project','bucket','database','function' - ][1 + (number % 4)] AS resource, + ][1 + (number % 4)] AS resourceType, concat('resource-', toString(number % 5000)) AS resourceId, '{TENANT}' AS tenant FROM numbers({ROWS}); diff --git a/tests/Usage/AccumulatorTest.php b/tests/Usage/AccumulatorTest.php index f63808a..8495c48 100644 --- a/tests/Usage/AccumulatorTest.php +++ b/tests/Usage/AccumulatorTest.php @@ -287,4 +287,80 @@ public function testInvalidTypeThrows(): void $this->expectException(\InvalidArgumentException::class); $this->accumulator->collect('t1', 'requests', 10, 'invalid'); } + + public function testCollectWithoutTimeOmitsField(): void + { + // Callers that don't set $time must not surface a null/empty time + // in the buffered entry — the adapter interprets "no time" as + // "use now() at write". + $this->accumulator->collect('t1', 'requests', 10, Usage::TYPE_EVENT); + + $this->assertTrue($this->accumulator->flush()); + $this->assertArrayNotHasKey('time', $this->adapter->batches[0]['metrics'][0]); + } + + public function testCollectThreadsQueuedTimeToBatch(): void + { + $emittedAt = new \DateTime('2026-04-15 12:34:56'); + $this->accumulator->collect('t1', 'requests', 10, Usage::TYPE_EVENT, [], $emittedAt); + + $this->assertTrue($this->accumulator->flush()); + + $entry = $this->adapter->batches[0]['metrics'][0]; + $this->assertArrayHasKey('time', $entry); + $this->assertSame($emittedAt, $entry['time']); + } + + public function testEventCollectPreservesEarliestQueuedTime(): void + { + // Two collect() calls fold into one entry; the earliest queued + // time survives so buckets don't slide forward on late arrivals. + $earlier = new \DateTime('2026-04-15 12:00:00'); + $later = new \DateTime('2026-04-15 12:05:00'); + + $this->accumulator->collect('t1', 'requests', 10, Usage::TYPE_EVENT, [], $earlier); + $this->accumulator->collect('t1', 'requests', 5, Usage::TYPE_EVENT, [], $later); + + $this->assertEquals(1, $this->accumulator->count()); + $this->assertTrue($this->accumulator->flush()); + + $entry = $this->adapter->batches[0]['metrics'][0]; + $this->assertEquals(15, $entry['value']); + $this->assertSame($earlier, $entry['time']); + } + + public function testEventCollectKeepsEarliestTimeOnOutOfOrderMerge(): void + { + // Out-of-order redelivery: the later time arrives FIRST, the earlier + // time second. The merged bucket must carry the earlier time so + // buckets never slide forward when late-arriving events fold in. + $later = new \DateTime('2026-04-15 12:05:00'); + $earlier = new \DateTime('2026-04-15 12:00:00'); + + $this->accumulator->collect('t1', 'requests', 10, Usage::TYPE_EVENT, [], $later); + $this->accumulator->collect('t1', 'requests', 5, Usage::TYPE_EVENT, [], $earlier); + + $this->assertEquals(1, $this->accumulator->count()); + $this->assertTrue($this->accumulator->flush()); + + $entry = $this->adapter->batches[0]['metrics'][0]; + $this->assertEquals(15, $entry['value']); + $this->assertSame($earlier, $entry['time']); + } + + public function testGaugeCollectUsesLastWriteWinsForTime(): void + { + // Gauge collect() is last-write-wins on value; the queued time + // supplied on the last call wins alongside it. + $t1 = new \DateTime('2026-04-15 12:00:00'); + $t2 = new \DateTime('2026-04-15 12:05:00'); + + $this->accumulator->collect('t1', 'storage', 100, Usage::TYPE_GAUGE, [], $t1); + $this->accumulator->collect('t1', 'storage', 200, Usage::TYPE_GAUGE, [], $t2); + + $this->assertTrue($this->accumulator->flush()); + $entry = $this->adapter->batches[0]['metrics'][0]; + $this->assertEquals(200, $entry['value']); + $this->assertSame($t2, $entry['time']); + } } diff --git a/tests/Usage/Adapter/ClickHouseDimRoutingTest.php b/tests/Usage/Adapter/ClickHouseDimRoutingTest.php index b3b6912..f6129dc 100644 --- a/tests/Usage/Adapter/ClickHouseDimRoutingTest.php +++ b/tests/Usage/Adapter/ClickHouseDimRoutingTest.php @@ -144,7 +144,7 @@ public function testMultiDimNotInAnyProjectionFallsBackToTable(): void public function testFilterOnExtraColumnStillRoutesToProjectionWhenColumnPresent(): void { - // resource is a column on the events table but not in p_by_path's + // resourceType is a column on the events table but not in p_by_path's // projection; the optimizer cannot satisfy this query from the // projection and must scan the base table. $start = (new DateTime('-7 days'))->format('Y-m-d H:i:s'); @@ -155,7 +155,7 @@ public function testFilterOnExtraColumnStillRoutesToProjectionWhenColumnPresent( $this->usage->find('1', [ UsageQuery::groupBy('path'), Query::equal('metric', [$this->metric]), - Query::equal('resource', ['function']), + Query::equal('resourceType', ['function']), Query::greaterThanEqual('time', $start), Query::lessThanEqual('time', $end), ], Usage::TYPE_EVENT); diff --git a/tests/Usage/Adapter/ClickHouseGaugeDimRoutingTest.php b/tests/Usage/Adapter/ClickHouseGaugeDimRoutingTest.php index 562a81b..21f2c26 100644 --- a/tests/Usage/Adapter/ClickHouseGaugeDimRoutingTest.php +++ b/tests/Usage/Adapter/ClickHouseGaugeDimRoutingTest.php @@ -12,7 +12,7 @@ /** * Routing tests for the gauge per-dim projection slate (p_by_service, - * p_by_resource). Each grouped scenario asserts (a) latest-per-group + * p_by_resourceType). Each grouped scenario asserts (a) latest-per-group * values match the raw scan, and (b) the optimizer picked the matching * argMaxState projection per `system.query_log.projections`. */ @@ -40,13 +40,13 @@ protected function setUp(): void $this->usage->setup(); $this->usage->purge('1'); - $this->seedHistoricalRow($this->metric, 100, '-5 days', ['service' => 'storage', 'resource' => 'file', 'resourceId' => 'f1']); - $this->seedHistoricalRow($this->metric, 200, '-4 days', ['service' => 'storage', 'resource' => 'file', 'resourceId' => 'f2']); - $this->seedHistoricalRow($this->metric, 50, '-3 days', ['service' => 'databases', 'resource' => 'database', 'resourceId' => 'db1']); - $this->seedHistoricalRow($this->metric, 80, '-3 days', ['service' => 'functions', 'resource' => 'function', 'resourceId' => 'fn1']); + $this->seedHistoricalRow($this->metric, 100, '-5 days', ['service' => 'storage', 'resourceType' => 'file', 'resourceId' => 'f1']); + $this->seedHistoricalRow($this->metric, 200, '-4 days', ['service' => 'storage', 'resourceType' => 'file', 'resourceId' => 'f2']); + $this->seedHistoricalRow($this->metric, 50, '-3 days', ['service' => 'databases', 'resourceType' => 'database', 'resourceId' => 'db1']); + $this->seedHistoricalRow($this->metric, 80, '-3 days', ['service' => 'functions', 'resourceType' => 'function', 'resourceId' => 'fn1']); $this->usage->addBatch([ - ['tenant' => '1', 'metric' => $this->metric, 'value' => 999, 'tags' => ['service' => 'storage', 'resource' => 'file', 'resourceId' => 'f1']], + ['tenant' => '1', 'metric' => $this->metric, 'value' => 999, 'tags' => ['service' => 'storage', 'resourceType' => 'file', 'resourceId' => 'f1']], ], Usage::TYPE_GAUGE); } @@ -74,7 +74,7 @@ private function seedHistoricalRow(string $metric, int $value, string $modifier, "'{$time}'", "'1'", ]; - foreach (['service', 'resource'] as $tag) { + foreach (['service', 'resourceType'] as $tag) { if (isset($tags[$tag])) { $cols[] = $tag; $vals[] = "'" . addslashes($tags[$tag]) . "'"; @@ -91,10 +91,10 @@ private function seedHistoricalRow(string $metric, int $value, string $modifier, public static function topGaugesProjectionProvider(): array { return [ - 'by_service' => [['service'], 'p_by_service'], - 'by_resource' => [['resource'], 'p_by_resource'], - 'by_resourceId' => [['resourceId'], 'p_by_resourceId'], - 'by_resource_resourceId' => [['resource', 'resourceId'], 'p_by_resource_resourceId'], + 'by_service' => [['service'], 'p_by_service'], + 'by_resourceType' => [['resourceType'], 'p_by_resourceType'], + 'by_resourceId' => [['resourceId'], 'p_by_resourceId'], + 'by_resourceType_resourceId' => [['resourceType', 'resourceId'], 'p_by_resourceType_resourceId'], ]; } diff --git a/tests/Usage/Adapter/ClickHouseSchemaTest.php b/tests/Usage/Adapter/ClickHouseSchemaTest.php index 37c9e84..281ef27 100644 --- a/tests/Usage/Adapter/ClickHouseSchemaTest.php +++ b/tests/Usage/Adapter/ClickHouseSchemaTest.php @@ -4,6 +4,7 @@ use Utopia\Tests\Usage\Adapter\ClickHouseTestCase; use Utopia\Usage\Adapter\ClickHouse as ClickHouseAdapter; +use Utopia\Usage\Metric; use Utopia\Usage\Usage; /** @@ -93,7 +94,7 @@ public function testGaugesTableCarriesServiceAndResource(): void $ddl = $this->showCreate($this->resolveTableName($this->adapter, 'getGaugesTableName')); $this->assertStringContainsString('`service` LowCardinality(Nullable(String))', $ddl); - $this->assertStringContainsString('`resource` LowCardinality(Nullable(String))', $ddl); + $this->assertStringContainsString('`resourceType` LowCardinality(Nullable(String))', $ddl); $this->assertStringContainsString("`time` DateTime64(3, 'UTC') CODEC(Delta(4), LZ4)", $ddl); $this->assertStringContainsString('`resourceId` Nullable(String) CODEC(ZSTD(3))', $ddl); $this->assertStringContainsString('`teamId` Nullable(String) CODEC(ZSTD(3))', $ddl); @@ -104,7 +105,7 @@ public function testGaugesTableSwapsBloomForSetOnLowCardinality(): void $ddl = $this->showCreate($this->resolveTableName($this->adapter, 'getGaugesTableName')); $this->assertStringContainsString('`index-service` service TYPE set(0)', $ddl); - $this->assertStringContainsString('`index-resource` resource TYPE set(0)', $ddl); + $this->assertStringContainsString('`index-resourceType` resourceType TYPE set(0)', $ddl); $this->assertStringContainsString('`index-resourceId` resourceId TYPE bloom_filter', $ddl); $this->assertStringContainsString('`index-teamId` teamId TYPE bloom_filter', $ddl); } @@ -132,10 +133,6 @@ public function testSetupBackfillsServiceResourceOnLegacyGaugesTable(): void metric String, value Int64, time DateTime64(3), - resourceId Nullable(String), - resourceInternalId Nullable(String), - teamId Nullable(String), - teamInternalId Nullable(String), tenant Nullable(String) ) ENGINE = MergeTree() @@ -147,18 +144,101 @@ public function testSetupBackfillsServiceResourceOnLegacyGaugesTable(): void $usage = new Usage($legacyAdapter); $usage->setup(); - $rawString = $this->queryRaw($legacyAdapter, "SHOW CREATE TABLE {$fullName} FORMAT JSON"); + $ddl = $this->showCreateFor($legacyAdapter, $fullName); + + foreach ($this->expectedDimAssertions(Metric::GAUGE_COLUMNS, 'gauge') as $expected) { + $this->assertStringContainsString($expected, $ddl); + } + } + + public function testSetupBackfillsIpOnLegacyEventsTable(): void + { + $legacyAdapter = new ClickHouseAdapter( + getenv('CLICKHOUSE_HOST') ?: 'clickhouse', + getenv('CLICKHOUSE_USER') ?: 'default', + getenv('CLICKHOUSE_PASSWORD') ?: 'clickhouse', + (int) (getenv('CLICKHOUSE_PORT') ?: 8123), + (bool) (getenv('CLICKHOUSE_SECURE') ?: false), + namespace: 'utopia_usage_schema_legacy_event', + database: getenv('CLICKHOUSE_DATABASE') ?: 'default', + sharedTables: true, + ); + $database = $this->databaseName($legacyAdapter); + $eventsTable = $this->resolveTableName($legacyAdapter, 'getEventsTableName'); + $dailyTable = $this->resolveTableName($legacyAdapter, 'getEventsDailyTableName'); + $dailyMv = $this->resolveTableName($legacyAdapter, 'getTableName') . '_events_daily_mv'; + $fullName = "`{$database}`.`{$eventsTable}`"; + $fullDaily = "`{$database}`.`{$dailyTable}`"; + $fullMv = "`{$database}`.`{$dailyMv}`"; + + $this->queryRaw($legacyAdapter, "DROP TABLE IF EXISTS {$fullMv}"); + $this->queryRaw($legacyAdapter, "DROP TABLE IF EXISTS {$fullDaily}"); + $this->queryRaw($legacyAdapter, "DROP TABLE IF EXISTS {$fullName}"); + $this->queryRaw($legacyAdapter, " + CREATE TABLE {$fullName} ( + id String, + metric String, + value Int64, + time DateTime64(3, 'UTC'), + tenant Nullable(String) + ) + ENGINE = MergeTree() + ORDER BY (tenant, metric, time, id) + PARTITION BY toYYYYMM(time) + SETTINGS allow_nullable_key = 1 + "); + + $usage = new Usage($legacyAdapter); + $usage->setup(); + + $ddl = $this->showCreateFor($legacyAdapter, $fullName); + + foreach ($this->expectedDimAssertions(Metric::EVENT_COLUMNS, 'event') as $expected) { + $this->assertStringContainsString($expected, $ddl); + } + } + + /** + * @param array $columns + * @return array + */ + private function expectedDimAssertions(array $columns, string $type): array + { + $lowCardinality = [ + 'country', 'region', 'service', 'resourceType', + 'osCode', 'osName', 'osVersion', + 'clientType', 'clientCode', 'clientName', 'clientVersion', + 'clientEngine', 'clientEngineVersion', + 'deviceName', 'deviceBrand', 'deviceModel', + 'hostname', 'ip', + ]; + + $baseKey = ['id', 'metric', 'value', 'time', 'tenant']; + + $expected = []; + foreach ($columns as $column) { + if (in_array($column, $baseKey, true)) { + continue; + } + $suffix = in_array($column, $lowCardinality, true) + ? 'LowCardinality(Nullable(String))' + : 'Nullable(String)'; + $expected[] = "`{$column}` {$suffix}"; + } + return $expected; + } + + private function showCreateFor(ClickHouseAdapter $adapter, string $fullName): string + { + $rawString = $this->queryRaw($adapter, "SHOW CREATE TABLE {$fullName} FORMAT JSON"); $json = json_decode($rawString, true); - $ddl = ''; if (is_array($json) && isset($json['data']) && is_array($json['data']) && isset($json['data'][0]) && is_array($json['data'][0])) { $statement = $json['data'][0]['statement'] ?? ''; if (is_string($statement)) { - $ddl = $statement; + return $statement; } } - - $this->assertStringContainsString('`service` LowCardinality(Nullable(String))', $ddl); - $this->assertStringContainsString('`resource` LowCardinality(Nullable(String))', $ddl); + return ''; } private function showCreate(string $table): string diff --git a/tests/Usage/Adapter/ClickHouseTest.php b/tests/Usage/Adapter/ClickHouseTest.php index aa7e031..83aea81 100644 --- a/tests/Usage/Adapter/ClickHouseTest.php +++ b/tests/Usage/Adapter/ClickHouseTest.php @@ -235,7 +235,7 @@ public function testEventColumnsExtractedFromTags(): void 'method' => 'POST', 'status' => '201', 'service' => 'storage', - 'resource' => 'bucket', + 'resourceType' => 'bucket', 'resourceId' => 'bucket123', 'resourceInternalId' => '42', 'teamId' => 'team_x', @@ -272,7 +272,7 @@ public function testEventColumnsExtractedFromTags(): void $this->assertEquals('POST', $metric->getMethod()); $this->assertEquals('201', $metric->getStatus()); $this->assertEquals('storage', $metric->getService()); - $this->assertEquals('bucket', $metric->getResource()); + $this->assertEquals('bucket', $metric->getResourceType()); $this->assertEquals('bucket123', $metric->getResourceId()); $this->assertEquals('42', $metric->getResourceInternalId()); $this->assertEquals('team_x', $metric->getTeamId()); @@ -309,7 +309,7 @@ public function testGaugeColumnsRoundTrip(): void 'value' => 500, 'tags' => [ 'service' => 'storage', - 'resource' => 'file', + 'resourceType' => 'file', 'teamId' => 'team_x', 'teamInternalId' => '7', 'resourceId' => 'r1', @@ -325,7 +325,7 @@ public function testGaugeColumnsRoundTrip(): void $this->assertCount(1, $results); $metric = $results[0]; $this->assertEquals('storage', $metric->getService()); - $this->assertEquals('file', $metric->getResource()); + $this->assertEquals('file', $metric->getResourceType()); $this->assertEquals('team_x', $metric->getTeamId()); $this->assertEquals('7', $metric->getTeamInternalId()); $this->assertEquals('r1', $metric->getResourceId()); @@ -396,7 +396,7 @@ public function testEventsSchemaPersistsAllNewColumns(): void $tags = [ 'path' => '/v1/x', 'method' => 'GET', 'status' => '200', - 'service' => 'storage', 'resource' => 'bucket', + 'service' => 'storage', 'resourceType' => 'bucket', 'resourceId' => 'r1', 'resourceInternalId' => '42', 'teamId' => 't1', 'teamInternalId' => '7', 'country' => 'us', 'region' => 'fra', 'hostname' => 'h.example.com', @@ -432,9 +432,9 @@ public function testQueryEventsByColumns(): void $this->usage->purge('1', [], Usage::TYPE_EVENT); $this->assertTrue($this->usage->addBatch([ - ['tenant' => '1', 'metric' => 'req', 'value' => 10, 'tags' => ['path' => '/v1/storage', 'method' => 'GET', 'status' => '200', 'resource' => 'project', 'resourceId' => 'p1']], - ['tenant' => '1', 'metric' => 'req', 'value' => 20, 'tags' => ['path' => '/v1/databases', 'method' => 'POST', 'status' => '201', 'resource' => 'database', 'resourceId' => 'db1']], - ['tenant' => '1', 'metric' => 'req', 'value' => 30, 'tags' => ['path' => '/v1/storage', 'method' => 'GET', 'status' => '404', 'resource' => 'project', 'resourceId' => 'p1']], + ['tenant' => '1', 'metric' => 'req', 'value' => 10, 'tags' => ['path' => '/v1/storage', 'method' => 'GET', 'status' => '200', 'resourceType' => 'project', 'resourceId' => 'p1']], + ['tenant' => '1', 'metric' => 'req', 'value' => 20, 'tags' => ['path' => '/v1/databases', 'method' => 'POST', 'status' => '201', 'resourceType' => 'database', 'resourceId' => 'db1']], + ['tenant' => '1', 'metric' => 'req', 'value' => 30, 'tags' => ['path' => '/v1/storage', 'method' => 'GET', 'status' => '404', 'resourceType' => 'project', 'resourceId' => 'p1']], ], Usage::TYPE_EVENT)); // Filter by path @@ -457,9 +457,9 @@ public function testQueryEventsByColumns(): void $this->assertCount(1, $results); $this->assertEquals(30, $results[0]->getValue()); - // Filter by resource + // Filter by resourceType $results = $this->usage->find('1', [ - \Utopia\Query\Query::equal('resource', ['database']), + \Utopia\Query\Query::equal('resourceType', ['database']), ], Usage::TYPE_EVENT); $this->assertCount(1, $results); @@ -493,7 +493,7 @@ public function testGaugeTableSimpleSchema(): void $this->assertNull($results[0]->getPath()); $this->assertNull($results[0]->getMethod()); $this->assertNull($results[0]->getStatus()); - $this->assertNull($results[0]->getResource()); + $this->assertNull($results[0]->getResourceType()); $this->assertEquals('r1', $results[0]->getResourceId()); } @@ -1167,4 +1167,142 @@ public function testEqualRejectsEmptyValues(): void new Query(Query::TYPE_EQUAL, 'metric', []), ], Usage::TYPE_EVENT); } + + /** + * Gauge getTimeSeries() carries the last-observed value forward + * across buckets that have no snapshot, instead of collapsing to + * zero. Two hourly snapshots six hours apart should backfill the + * middle buckets with the earlier snapshot's value; buckets after + * the last snapshot should stick at that snapshot's value; buckets + * before the first snapshot fall back to 0 (no observation yet). + */ + public function testGaugeTimeSeriesFillsMissingBucketsWithLastKnownValue(): void + { + $this->usage->purge('1', [], Usage::TYPE_GAUGE); + + $windowStart = (new \DateTime('2026-06-01 00:00:00'))->format('Y-m-d H:i:s'); + $windowEnd = (new \DateTime('2026-06-01 10:00:00'))->format('Y-m-d H:i:s'); + + // Two snapshots inside the window: 02:00 -> 100, 08:00 -> 300. + // We ingest them with an explicit `time` so bucket alignment is + // deterministic. The addBatch payload accepts a `time` field + // when threaded through (see commit that adds queued-time on + // Accumulator::collect); this test exercises the same path. + $this->assertTrue($this->usage->addBatch([ + [ + 'tenant' => '1', + 'metric' => 'gauge-locf-fill', + 'value' => 100, + 'time' => new \DateTime('2026-06-01 02:00:00'), + 'tags' => [], + ], + [ + 'tenant' => '1', + 'metric' => 'gauge-locf-fill', + 'value' => 300, + 'time' => new \DateTime('2026-06-01 08:00:00'), + 'tags' => [], + ], + ], Usage::TYPE_GAUGE)); + + $results = $this->usage->getTimeSeries( + '1', + ['gauge-locf-fill'], + '1h', + $windowStart, + $windowEnd, + [], + true, + Usage::TYPE_GAUGE, + ); + + $this->assertArrayHasKey('gauge-locf-fill', $results); + $data = $results['gauge-locf-fill']['data']; + + $indexed = []; + foreach ($data as $point) { + $key = (new \DateTime($point['date']))->format('H'); + $indexed[$key] = $point['value']; + } + + $expected = [ + // Pre-first-snapshot buckets fall back to 0. + '00' => 0.0, + '01' => 0.0, + // First snapshot lands at 02:00. + '02' => 100.0, + // 03..07 carry the 100 forward until the next snapshot. + '03' => 100.0, + '04' => 100.0, + '05' => 100.0, + '06' => 100.0, + '07' => 100.0, + // Second snapshot at 08:00 updates the value. + '08' => 300.0, + // Post-snapshot buckets stick at 300. + '09' => 300.0, + '10' => 300.0, + ]; + foreach ($expected as $hour => $value) { + $actual = $indexed[$hour] ?? null; + $this->assertNotNull($actual, "missing bucket {$hour}:00"); + $this->assertSame($value, $actual, "hour {$hour}:00 filled with wrong value"); + } + } + + /** + * Event getTimeSeries() must keep the classic zero-fill semantics: + * buckets with no additive activity remain 0, and are NOT carried + * forward from earlier buckets. + */ + public function testEventTimeSeriesStillZeroFillsMissingBuckets(): void + { + $this->usage->purge('1', [], Usage::TYPE_EVENT); + + $windowStart = (new \DateTime('2026-06-02 00:00:00'))->format('Y-m-d H:i:s'); + $windowEnd = (new \DateTime('2026-06-02 04:00:00'))->format('Y-m-d H:i:s'); + + // Single event at 01:00 with value 50. Later buckets must be 0, + // not 50. + $this->assertTrue($this->usage->addBatch([ + [ + 'tenant' => '1', + 'metric' => 'event-zero-fill', + 'value' => 50, + 'time' => new \DateTime('2026-06-02 01:00:00'), + 'tags' => [], + ], + ], Usage::TYPE_EVENT)); + + $results = $this->usage->getTimeSeries( + '1', + ['event-zero-fill'], + '1h', + $windowStart, + $windowEnd, + [], + true, + Usage::TYPE_EVENT, + ); + + $data = $results['event-zero-fill']['data']; + $indexed = []; + foreach ($data as $point) { + $key = (new \DateTime($point['date']))->format('H'); + $indexed[$key] = $point['value']; + } + + $expected = [ + '00' => 0.0, + '01' => 50.0, + '02' => 0.0, + '03' => 0.0, + '04' => 0.0, + ]; + foreach ($expected as $hour => $value) { + $actual = $indexed[$hour] ?? null; + $this->assertNotNull($actual, "missing bucket {$hour}:00"); + $this->assertSame($value, $actual, "hour {$hour}:00 filled with wrong value"); + } + } } diff --git a/tests/Usage/Adapter/DatabaseTest.php b/tests/Usage/Adapter/DatabaseTest.php index 5c0ee1e..3352b3a 100644 --- a/tests/Usage/Adapter/DatabaseTest.php +++ b/tests/Usage/Adapter/DatabaseTest.php @@ -76,7 +76,7 @@ public function testEventColumnsExtractedFromTags(): void 'method' => 'POST', 'status' => '201', 'service' => 'storage', - 'resource' => 'bucket', + 'resourceType' => 'bucket', 'resourceId' => 'bucket123', 'resourceInternalId' => '42', 'teamId' => 'team_x', diff --git a/tests/Usage/MetricTest.php b/tests/Usage/MetricTest.php index eba1029..90eba04 100644 --- a/tests/Usage/MetricTest.php +++ b/tests/Usage/MetricTest.php @@ -99,9 +99,9 @@ public function testEventIndexesCoverNewFilterableColumns(): void $indexed = array_merge($indexed, $attrs); } foreach ([ - 'path', 'method', 'status', 'service', 'resource', + 'path', 'method', 'status', 'service', 'resourceType', 'resourceId', 'resourceInternalId', 'teamId', 'teamInternalId', - 'country', 'region', 'hostname', + 'country', 'region', 'hostname', 'ip', 'osName', 'clientType', 'clientName', 'deviceName', ] as $col) { $this->assertContains($col, $indexed, "Event indexes missing {$col}"); @@ -143,7 +143,7 @@ public function testValidateAcceptsValidEventData(): void 'path' => '/v1/storage/files', 'method' => 'POST', 'status' => '201', - 'resource' => 'bucket', + 'resourceType' => 'bucket', 'resourceId' => 'abc123', 'region' => 'us', ]; @@ -310,7 +310,7 @@ public function testConstructorInitializesWithData(): void 'path' => '/v1/storage/files', 'method' => 'POST', 'status' => '201', - 'resource' => 'bucket', + 'resourceType' => 'bucket', 'resourceId' => 'abc123', ]; @@ -323,7 +323,7 @@ public function testConstructorInitializesWithData(): void $this->assertEquals('/v1/storage/files', $metric->getPath()); $this->assertEquals('POST', $metric->getMethod()); $this->assertEquals('201', $metric->getStatus()); - $this->assertEquals('bucket', $metric->getResource()); + $this->assertEquals('bucket', $metric->getResourceType()); $this->assertEquals('abc123', $metric->getResourceId()); } @@ -399,7 +399,7 @@ public function testEventGettersReturnNullWhenNotSet(): void $this->assertNull($metric->getPath()); $this->assertNull($metric->getMethod()); $this->assertNull($metric->getStatus()); - $this->assertNull($metric->getResource()); + $this->assertNull($metric->getResourceType()); $this->assertNull($metric->getResourceId()); } @@ -412,13 +412,13 @@ public function testEventGettersReturnCorrectValues(): void 'path' => '/v1/databases', 'method' => 'GET', 'status' => '200', - 'resource' => 'database', + 'resourceType' => 'database', 'resourceId' => 'db123', ]); $this->assertEquals('/v1/databases', $metric->getPath()); $this->assertEquals('GET', $metric->getMethod()); $this->assertEquals('200', $metric->getStatus()); - $this->assertEquals('database', $metric->getResource()); + $this->assertEquals('database', $metric->getResourceType()); $this->assertEquals('db123', $metric->getResourceId()); } @@ -606,9 +606,9 @@ public function testEventColumnsConstant(): void { $expected = [ 'path', 'method', 'status', - 'service', 'resource', 'resourceId', 'resourceInternalId', + 'service', 'resourceType', 'resourceId', 'resourceInternalId', 'teamId', 'teamInternalId', - 'country', 'region', 'hostname', + 'country', 'region', 'hostname', 'ip', 'osCode', 'osName', 'osVersion', 'clientType', 'clientCode', 'clientName', 'clientVersion', 'clientEngine', 'clientEngineVersion', @@ -622,7 +622,7 @@ public function testEventColumnsConstant(): void */ public function testGaugeColumnsConstant(): void { - $expected = ['service', 'resource', 'teamId', 'teamInternalId', 'resourceId', 'resourceInternalId']; + $expected = ['service', 'resourceType', 'teamId', 'teamInternalId', 'resourceId', 'resourceInternalId']; $this->assertSame($expected, Metric::GAUGE_COLUMNS); } @@ -696,4 +696,35 @@ public function testDroppedGettersDoNotExist(): void $this->assertFalse(method_exists(Metric::class, 'getUserAgent')); $this->assertFalse(method_exists(Metric::class, 'getTags')); } + + /** + * The ip dimension is event-only and round-trips through the typed + * accessor. + */ + public function testGetIpReturnsAddress(): void + { + $m = new Metric(['ip' => '203.0.113.42']); + $this->assertSame('203.0.113.42', $m->getIp()); + + $m6 = new Metric(['ip' => '2001:db8::1']); + $this->assertSame('2001:db8::1', $m6->getIp()); + + $missing = new Metric([]); + $this->assertNull($missing->getIp()); + } + + /** + * ip is event-only; the gauge column set must not include it. + */ + public function testIpIsEventOnly(): void + { + $this->assertContains('ip', Metric::EVENT_COLUMNS); + $this->assertNotContains('ip', Metric::GAUGE_COLUMNS); + + $eventIds = array_column(Metric::getEventSchema(), '$id'); + $this->assertContains('ip', $eventIds); + + $gaugeIds = array_column(Metric::getGaugeSchema(), '$id'); + $this->assertNotContains('ip', $gaugeIds); + } } diff --git a/tests/Usage/UsageBase.php b/tests/Usage/UsageBase.php index a104b7e..ea126c4 100644 --- a/tests/Usage/UsageBase.php +++ b/tests/Usage/UsageBase.php @@ -27,9 +27,9 @@ public function createUsageMetrics(): void { // Events: additive metrics $this->assertTrue($this->usage->addBatch([ - ['tenant' => '1', 'metric' => 'requests', 'value' => 100, 'tags' => ['region' => 'us-east', 'path' => '/v1/storage', 'method' => 'GET', 'status' => '200', 'resource' => 'project', 'resourceId' => 'p1']], - ['tenant' => '1', 'metric' => 'requests', 'value' => 150, 'tags' => ['region' => 'us-west', 'path' => '/v1/databases', 'method' => 'POST', 'status' => '201', 'resource' => 'database', 'resourceId' => 'db1']], - ['tenant' => '1', 'metric' => 'bandwidth', 'value' => 5000, 'tags' => ['region' => 'us-east', 'path' => '/v1/storage/files', 'method' => 'POST', 'status' => '201', 'resource' => 'bucket', 'resourceId' => 'b1']], + ['tenant' => '1', 'metric' => 'requests', 'value' => 100, 'tags' => ['region' => 'us-east', 'path' => '/v1/storage', 'method' => 'GET', 'status' => '200', 'resourceType' => 'project', 'resourceId' => 'p1']], + ['tenant' => '1', 'metric' => 'requests', 'value' => 150, 'tags' => ['region' => 'us-west', 'path' => '/v1/databases', 'method' => 'POST', 'status' => '201', 'resourceType' => 'database', 'resourceId' => 'db1']], + ['tenant' => '1', 'metric' => 'bandwidth', 'value' => 5000, 'tags' => ['region' => 'us-east', 'path' => '/v1/storage/files', 'method' => 'POST', 'status' => '201', 'resourceType' => 'bucket', 'resourceId' => 'b1']], ], Usage::TYPE_EVENT)); // Gauges: point-in-time snapshots