Skip to content

Operators limits #911

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
abnegate merged 21 commits into from
Jul 6, 2026
Merged

Operators limits #911

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
0ffcaa6
run ci
fogelito Jul 1, 2026
dc15617
Merge branch 'main' of github.com:utopia-php/database into operators-…
fogelito Jul 1, 2026
cc44213
Update main
fogelito Jul 1, 2026
4d60463
fix edge cases
fogelito Jul 1, 2026
3fd8f84
MAX_ARRAY_OPERATOR_SIZE
fogelito Jul 1, 2026
cb070cb
Remove check
fogelito Jul 1, 2026
5523af2
Revert pint changes
fogelito Jul 1, 2026
846a924
Merge branch 'main' of github.com:utopia-php/database into operators-…
fogelito Jul 1, 2026
4262ff8
On source of trues
fogelito Jul 1, 2026
3b22166
Remove throw
fogelito Jul 1, 2026
e84f450
Catch throws
fogelito Jul 1, 2026
c9db3ae
Adress comments
fogelito Jul 1, 2026
a172cfa
Run tests
fogelito Jul 1, 2026
7874ad1
lint
fogelito Jul 1, 2026
5dec9cb
update
fogelito Jul 1, 2026
edaea0d
update
fogelito Jul 1, 2026
adb2bbb
edge cases
fogelito Jul 1, 2026
db28249
Address greptile review
fogelito Jul 1, 2026
991564b
zero power
fogelito Jul 5, 2026
5575da1
Merge branch 'main' of github.com:utopia-php/database into operators-…
fogelito Jul 5, 2026
dd16c25
Check is operator
fogelito Jul 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 54 additions & 46 deletions src/Database/Adapter/MariaDB.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1038,21 +1038,13 @@ public function updateDocument(Document $collection, string $id, Document $docum
*/
$keyIndex = 0;
$operatorBinds = [];
$operators = [];

// Separate regular attributes from operators
foreach ($attributes as $attribute => $value) {
if (Operator::isOperator($value)) {
$operators[$attribute] = $value;
}
}

foreach ($attributes as $attribute => $value) {
$column = $this->filter($attribute);

// Check if this is an operator or regular attribute
if (isset($operators[$attribute])) {
$operatorSQL = $this->getOperatorSQL($column, $operators[$attribute], $operatorBinds);
if (Operator::isOperator($value)) {
$operatorSQL = $this->getOperatorSQL($column, $value, $operatorBinds);
$columns .= $operatorSQL . ',';
} else {
$bindKey = 'key_' . $keyIndex;
Expand Down Expand Up @@ -1086,7 +1078,7 @@ public function updateDocument(Document $collection, string $id, Document $docum
$keyIndex = 0;
foreach ($attributes as $attribute => $value) {
// Handle operators separately
if (isset($operators[$attribute])) {
if (Operator::isOperator($value)) {
continue;
}

Expand Down Expand Up @@ -1966,9 +1958,11 @@ protected function getOperatorSQL(string $column, Operator $operator, array &$bi
$bindKey = $this->registerOperatorBind($binds, $values[0] ?? 1);
if (isset($values[1])) {
$maxKey = $this->registerOperatorBind($binds, $values[1]);
// Compare with the operand moved across (`col > max - val`) instead of
// `col + val > max`, so the guard never overflows BIGINT when col is near the
// integer range limit. Inclusive: a result landing exactly on max still applies.
return "{$quotedColumn} = CASE
WHEN COALESCE({$quotedColumn}, 0) >= :$maxKey THEN :$maxKey
WHEN COALESCE({$quotedColumn}, 0) > :$maxKey - :$bindKey THEN :$maxKey
WHEN COALESCE({$quotedColumn}, 0) > :$maxKey - :$bindKey THEN COALESCE({$quotedColumn}, 0)
ELSE COALESCE({$quotedColumn}, 0) + :$bindKey
END";
}
Expand All @@ -1978,9 +1972,10 @@ protected function getOperatorSQL(string $column, Operator $operator, array &$bi
$bindKey = $this->registerOperatorBind($binds, $values[0] ?? 1);
if (isset($values[1])) {
$minKey = $this->registerOperatorBind($binds, $values[1]);
// `col < min + val` rather than `col - val < min`: overflow-safe near the
// integer range limit. Inclusive: a result landing exactly on min still applies.
return "{$quotedColumn} = CASE
WHEN COALESCE({$quotedColumn}, 0) <= :$minKey THEN :$minKey
WHEN COALESCE({$quotedColumn}, 0) < :$minKey + :$bindKey THEN :$minKey
WHEN COALESCE({$quotedColumn}, 0) < :$minKey + :$bindKey THEN COALESCE({$quotedColumn}, 0)
ELSE COALESCE({$quotedColumn}, 0) - :$bindKey
END";
}
Expand All @@ -1990,10 +1985,12 @@ protected function getOperatorSQL(string $column, Operator $operator, array &$bi
$bindKey = $this->registerOperatorBind($binds, $values[0] ?? 1);
if (isset($values[1])) {
$maxKey = $this->registerOperatorBind($binds, $values[1]);
// Compare via division (`col > max/val`, sign-aware) instead of computing
// `col * val`, which would overflow BIGINT for large operands. The factor's
// sign flips the inequality. Inclusive: a result exactly on max still applies.
return "{$quotedColumn} = CASE
WHEN COALESCE({$quotedColumn}, 0) >= :$maxKey THEN :$maxKey
WHEN :$bindKey > 0 AND COALESCE({$quotedColumn}, 0) > :$maxKey / :$bindKey THEN :$maxKey
WHEN :$bindKey < 0 AND COALESCE({$quotedColumn}, 0) < :$maxKey / :$bindKey THEN :$maxKey
WHEN :$bindKey > 0 AND COALESCE({$quotedColumn}, 0) > :$maxKey / :$bindKey THEN COALESCE({$quotedColumn}, 0)
WHEN :$bindKey < 0 AND COALESCE({$quotedColumn}, 0) < :$maxKey / :$bindKey THEN COALESCE({$quotedColumn}, 0)
ELSE COALESCE({$quotedColumn}, 0) * :$bindKey
END";
}
Expand All @@ -2004,7 +2001,7 @@ protected function getOperatorSQL(string $column, Operator $operator, array &$bi
if (isset($values[1])) {
$minKey = $this->registerOperatorBind($binds, $values[1]);
return "{$quotedColumn} = CASE
WHEN :$bindKey != 0 AND COALESCE({$quotedColumn}, 0) / :$bindKey <= :$minKey THEN :$minKey
WHEN :$bindKey != 0 AND COALESCE({$quotedColumn}, 0) / :$bindKey < :$minKey THEN COALESCE({$quotedColumn}, 0)
ELSE COALESCE({$quotedColumn}, 0) / :$bindKey
END";
}
Expand All @@ -2015,15 +2012,46 @@ protected function getOperatorSQL(string $column, Operator $operator, array &$bi
return "{$quotedColumn} = MOD(COALESCE({$quotedColumn}, 0), :$bindKey)";

case Operator::TYPE_POWER:
$bindKey = $this->registerOperatorBind($binds, $values[0] ?? 1);
$exponent = $values[0] ?? 1;
$bindKey = $this->registerOperatorBind($binds, $exponent);
if (isset($values[1])) {
$maxKey = $this->registerOperatorBind($binds, $values[1]);
return "{$quotedColumn} = CASE
WHEN COALESCE({$quotedColumn}, 0) >= :$maxKey THEN :$maxKey
WHEN COALESCE({$quotedColumn}, 0) <= 1 THEN COALESCE({$quotedColumn}, 0)
WHEN :$bindKey * LOG(COALESCE({$quotedColumn}, 1)) > LOG(:$maxKey) THEN :$maxKey
ELSE POWER(COALESCE({$quotedColumn}, 0), :$bindKey)
END";
$col = "COALESCE({$quotedColumn}, 0)";

// Leave the value unchanged only for undefined inputs, then apply the power if
// the result stays within the max. The exponent is constant, so only the
// undefined guard its value can actually trigger is emitted.
$oddInteger = \floor($exponent) == $exponent && ((int) $exponent) % 2 !== 0;

$whens = [];
if ($exponent < 0) {
// 0 to a negative power is undefined (POWER would error / return NULL).
$whens[] = "WHEN {$col} = 0 THEN {$col}";
}
if (\floor($exponent) != $exponent) {
// A negative base to a fractional exponent is not a real number.
$whens[] = "WHEN {$col} < 0 THEN {$col}";
}
// Cap by magnitude via logarithms so POWER() never runs on a value that would
// overflow (base^exp > max <=> exp * LOG(base) > LOG(max)).
if ($exponent == 0) {
// Every base to the zeroth power is 1 (including 0^0), which the magnitude
// check below can't see for a base of 0. The result 1 exceeds the max when
// max < 1, i.e. LOG(max) < 0 (LOG also coerces the bound value numerically).
$whens[] = "WHEN LOG(:$maxKey) < 0 THEN {$col}";
} elseif ($oddInteger) {
// An odd exponent keeps a negative base negative, and a negative result is
// always within a positive max, so only cap positive bases; negative bases
// fall through to POWER() and their (negative) result is applied.
$whens[] = "WHEN {$col} > 0 AND :$bindKey * LOG({$col}) > LOG(:$maxKey) THEN {$col}";
} else {
// Otherwise the result is non-negative, so its magnitude equals its value —
// cap either sign. ABS() keeps LOG() defined for a negative even-power base.
$whens[] = "WHEN {$col} <> 0 AND :$bindKey * LOG(ABS({$col})) > LOG(:$maxKey) THEN {$col}";
}

$whenSql = \implode(' ', $whens);
return "{$quotedColumn} = CASE {$whenSql} ELSE POWER({$col}, :$bindKey) END";
Comment thread
coderabbitai[bot] marked this conversation as resolved.
}
return "{$quotedColumn} = POWER(COALESCE({$quotedColumn}, 0), :$bindKey)";

Expand All @@ -2043,16 +2071,10 @@ protected function getOperatorSQL(string $column, Operator $operator, array &$bi

// Array operators
case Operator::TYPE_ARRAY_APPEND:
if (\count($values) > self::MAX_ARRAY_OPERATOR_SIZE) {
throw new DatabaseException("Array size " . \count($values) . " exceeds maximum allowed size of " . self::MAX_ARRAY_OPERATOR_SIZE . " for array operations");
}
$bindKey = $this->registerOperatorBind($binds, json_encode($values));
return "{$quotedColumn} = JSON_MERGE_PRESERVE(IFNULL({$quotedColumn}, JSON_ARRAY()), :$bindKey)";

case Operator::TYPE_ARRAY_PREPEND:
if (\count($values) > self::MAX_ARRAY_OPERATOR_SIZE) {
throw new DatabaseException("Array size " . \count($values) . " exceeds maximum allowed size of " . self::MAX_ARRAY_OPERATOR_SIZE . " for array operations");
}
$bindKey = $this->registerOperatorBind($binds, json_encode($values));
return "{$quotedColumn} = JSON_MERGE_PRESERVE(:$bindKey, IFNULL({$quotedColumn}, JSON_ARRAY()))";

Expand Down Expand Up @@ -2086,9 +2108,6 @@ protected function getOperatorSQL(string $column, Operator $operator, array &$bi
), JSON_ARRAY())";

case Operator::TYPE_ARRAY_INTERSECT:
if (\count($values) > self::MAX_ARRAY_OPERATOR_SIZE) {
throw new DatabaseException("Array size " . \count($values) . " exceeds maximum allowed size of " . self::MAX_ARRAY_OPERATOR_SIZE . " for array operations");
}
$bindKey = $this->registerOperatorBind($binds, json_encode($values));
return "{$quotedColumn} = IFNULL((
SELECT JSON_ARRAYAGG(jt1.value)
Expand All @@ -2100,9 +2119,6 @@ protected function getOperatorSQL(string $column, Operator $operator, array &$bi
), JSON_ARRAY())";

case Operator::TYPE_ARRAY_DIFF:
if (\count($values) > self::MAX_ARRAY_OPERATOR_SIZE) {
throw new DatabaseException("Array size " . \count($values) . " exceeds maximum allowed size of " . self::MAX_ARRAY_OPERATOR_SIZE . " for array operations");
}
$bindKey = $this->registerOperatorBind($binds, json_encode($values));
return "{$quotedColumn} = IFNULL((
SELECT JSON_ARRAYAGG(jt1.value)
Expand All @@ -2115,14 +2131,6 @@ protected function getOperatorSQL(string $column, Operator $operator, array &$bi

case Operator::TYPE_ARRAY_FILTER:
$condition = $values[0] ?? 'equal';
$validConditions = [
'equal', 'notEqual', // Comparison
'greaterThan', 'greaterThanEqual', 'lessThan', 'lessThanEqual', // Numeric
'isNull', 'isNotNull' // Null checks
];
if (!in_array($condition, $validConditions, true)) {
throw new DatabaseException("Invalid filter condition: {$condition}. Must be one of: " . implode(', ', $validConditions));
}
$filterValue = $values[1] ?? null;
$conditionKey = $this->registerOperatorBind($binds, $condition);
$valueKey = $this->registerOperatorBind($binds, $filterValue === null ? null : json_encode($filterValue));
Expand Down
55 changes: 44 additions & 11 deletions src/Database/Adapter/Memory.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
use Utopia\Database\Document;
use Utopia\Database\Exception as DatabaseException;
use Utopia\Database\Exception\Duplicate as DuplicateException;
use Utopia\Database\Exception\Limit as LimitException;
use Utopia\Database\Exception\NotFound as NotFoundException;
use Utopia\Database\Exception\Operator as OperatorException;
use Utopia\Database\Operator;
Expand Down Expand Up @@ -3467,12 +3468,13 @@ protected function applyOperator(mixed $current, Operator $operator): mixed
$max = $values[1] ?? null;
$base = \is_numeric($current) ? $current + 0 : 0;
if ($max !== null) {
// Compare *remaining headroom* against $by so we never
// overflow PHP's int range (which would silently demote
// the result to float and corrupt downstream Range
// validators).
if ($base >= $max || ($max - $base) <= $by) {
return $this->preserveNumericType($base, $max);
// Compare *remaining headroom* against $by so we never overflow PHP's int
// range. Guard: if the RESULT would exceed the max, leave it unchanged.
// Note: we must NOT short-circuit on `$base >= $max` — a negative $by moves
// the value down, so an already-over-max base can still land within bound
// (e.g. 52 + (-5) = 47 <= 50 must apply).
if (($max - $base) < $by) {
return $this->preserveNumericType($base, $base);
}
}

Expand All @@ -3483,8 +3485,10 @@ protected function applyOperator(mixed $current, Operator $operator): mixed
$min = $values[1] ?? null;
$base = \is_numeric($current) ? $current + 0 : 0;
if ($min !== null) {
if ($base <= $min || ($base - $min) <= $by) {
return $this->preserveNumericType($base, $min);
// Guard: leave unchanged only if the RESULT would go below min. Don't
// short-circuit on `$base <= $min` — a negative $by moves the value up.
if (($base - $min) < $by) {
return $this->preserveNumericType($base, $base);
}
}

Expand All @@ -3494,8 +3498,12 @@ protected function applyOperator(mixed $current, Operator $operator): mixed
$by = $values[0] ?? 1;
$max = $values[1] ?? null;
$base = \is_numeric($current) ? $current + 0 : 0;
$result = $base * $by;
if ($max !== null && $result > $max) {
return $this->preserveNumericType($base, $base);
}

return $this->applyNumericLimit($base * $by, $max, true);
return $this->preserveNumericType($base, $result);

case Operator::TYPE_DIVIDE:
$by = $values[0] ?? 1;
Expand All @@ -3504,8 +3512,12 @@ protected function applyOperator(mixed $current, Operator $operator): mixed
return $current;
}
$base = \is_numeric($current) ? $current + 0 : 0;
$result = $base / $by;
if ($min !== null && $result < $min) {
return $this->preserveNumericType($base, $base);
}

return $this->applyNumericLimit($base / $by, $min, false);
return $this->preserveNumericType($base, $result);

case Operator::TYPE_MODULO:
$by = $values[0] ?? 1;
Expand All @@ -3520,8 +3532,29 @@ protected function applyOperator(mixed $current, Operator $operator): mixed
$by = $values[0] ?? 1;
$max = $values[1] ?? null;
$base = \is_numeric($current) ? $current + 0 : 0;
if ($max !== null) {
// Leave the value unchanged for undefined inputs (0 to a negative power, or a
// negative base to a fractional exponent) — they produce INF/NaN, not a number.
if (($base == 0 && $by < 0) || ($base < 0 && \floor($by) != $by)) {
return $this->preserveNumericType($base, $base);
}
$result = $base ** $by;
// A result that overflows (INF) or exceeds the max also leaves the value as-is.
if (!\is_finite($result) || $result > $max) {
return $this->preserveNumericType($base, $base);
}

return $this->preserveNumericType($base, $result);
}

// 0 to a negative power, or a negative base to a fractional exponent, is not a real
// number. Fail loudly with a clear exception rather than storing INF/NaN.
$result = $base ** $by;
if (!\is_finite($result)) {
throw new LimitException('Value out of range');
}

return $this->applyNumericLimit($base ** $by, $max, true);
return $this->preserveNumericType($base, $result);

case Operator::TYPE_STRING_CONCAT:
return ((string) ($current ?? '')).(string) ($values[0] ?? '');
Expand Down
Loading
Loading