From 0ef3443b94f8e3fa148f15f0346b55189568dd45 Mon Sep 17 00:00:00 2001 From: Daniel Frankcom Date: Wed, 15 Jul 2026 12:28:57 -0700 Subject: [PATCH] Fix autoCompact intermittent busy error Signed-off-by: Daniel Frankcom --- .../test_autoCompact_fstmb_bounds.py | 9 +++-- .../test_autoCompact_operational.py | 29 +++++++++++----- .../autoCompact/test_autoCompact_success.py | 8 +++-- .../autoCompact/test_smoke_autoCompact.py | 8 +++-- .../autoCompact/utils/autoCompact_common.py | 33 +++++++------------ documentdb_tests/framework/error_codes.py | 3 +- 6 files changed, 52 insertions(+), 38 deletions(-) diff --git a/documentdb_tests/compatibility/tests/system/administration/commands/autoCompact/test_autoCompact_fstmb_bounds.py b/documentdb_tests/compatibility/tests/system/administration/commands/autoCompact/test_autoCompact_fstmb_bounds.py index 52372da22..e17d9115f 100644 --- a/documentdb_tests/compatibility/tests/system/administration/commands/autoCompact/test_autoCompact_fstmb_bounds.py +++ b/documentdb_tests/compatibility/tests/system/administration/commands/autoCompact/test_autoCompact_fstmb_bounds.py @@ -13,8 +13,8 @@ ensure_autocompact_idle, ) from documentdb_tests.framework.assertions import assertResult -from documentdb_tests.framework.error_codes import BAD_VALUE_ERROR -from documentdb_tests.framework.executor import execute_admin_command +from documentdb_tests.framework.error_codes import BAD_VALUE_ERROR, OBJECT_IS_BUSY_ERROR +from documentdb_tests.framework.executor import execute_admin_with_retry_command from documentdb_tests.framework.parametrize import pytest_params from documentdb_tests.framework.property_checks import Eq from documentdb_tests.framework.test_constants import ( @@ -285,7 +285,10 @@ def test_autoCompact_fstmb_bounds(database_client, collection, test): # Ensure autoCompact is idle first: a leftover config from a prior test # would otherwise conflict. ensure_autocompact_idle(collection) - result = execute_admin_command(collection, test.build_command(ctx)) + # Tolerate the transient busy state while a prior compaction winds down. + result = execute_admin_with_retry_command( + collection, test.build_command(ctx), retry_code=OBJECT_IS_BUSY_ERROR + ) assertResult( result, expected=test.build_expected(ctx), diff --git a/documentdb_tests/compatibility/tests/system/administration/commands/autoCompact/test_autoCompact_operational.py b/documentdb_tests/compatibility/tests/system/administration/commands/autoCompact/test_autoCompact_operational.py index bd7a8bde0..df716d06b 100644 --- a/documentdb_tests/compatibility/tests/system/administration/commands/autoCompact/test_autoCompact_operational.py +++ b/documentdb_tests/compatibility/tests/system/administration/commands/autoCompact/test_autoCompact_operational.py @@ -13,11 +13,12 @@ ) from documentdb_tests.framework.assertions import assertResult from documentdb_tests.framework.error_codes import ( - CONFLICTING_OPERATION_IN_PROGRESS_ERROR, + ALREADY_INITIALIZED_ERROR, + OBJECT_IS_BUSY_ERROR, UNAUTHORIZED_ERROR, ) from documentdb_tests.framework.executor import ( - execute_admin_command, + execute_admin_with_retry_command, execute_command, ) from documentdb_tests.framework.parametrize import pytest_params @@ -68,14 +69,19 @@ def test_autoCompact_reconfigure_conflict(collection): ensure_autocompact_idle(collection) # Establish a running config. - execute_admin_command(collection, {"autoCompact": True, "freeSpaceTargetMB": 30}) + execute_admin_with_retry_command( + collection, {"autoCompact": True, "freeSpaceTargetMB": 30}, retry_code=OBJECT_IS_BUSY_ERROR + ) - # Second should be rejected as the value differs. - result = execute_admin_command(collection, {"autoCompact": True, "freeSpaceTargetMB": 50}) + # A differing reconfigure is rejected while running; tolerate the transient + # busy state while a prior compaction winds down. + result = execute_admin_with_retry_command( + collection, {"autoCompact": True, "freeSpaceTargetMB": 50}, retry_code=OBJECT_IS_BUSY_ERROR + ) assertResult( result, - error_code=CONFLICTING_OPERATION_IN_PROGRESS_ERROR, + error_code=ALREADY_INITIALIZED_ERROR, msg="autoCompact should reject reconfiguring an enabled compaction with a different config", raw_res=True, ) @@ -91,10 +97,15 @@ def test_autoCompact_reconfigure_idempotent(collection): ensure_autocompact_idle(collection) # Establish a running config. - execute_admin_command(collection, {"autoCompact": True, "freeSpaceTargetMB": 30}) + execute_admin_with_retry_command( + collection, {"autoCompact": True, "freeSpaceTargetMB": 30}, retry_code=OBJECT_IS_BUSY_ERROR + ) - # Second should be accepted as a no-op since the value is the same. - result = execute_admin_command(collection, {"autoCompact": True, "freeSpaceTargetMB": 30}) + # Re-enabling an identical config is a no-op; tolerate the transient busy + # state while a prior compaction winds down. + result = execute_admin_with_retry_command( + collection, {"autoCompact": True, "freeSpaceTargetMB": 30}, retry_code=OBJECT_IS_BUSY_ERROR + ) assertResult( result, diff --git a/documentdb_tests/compatibility/tests/system/administration/commands/autoCompact/test_autoCompact_success.py b/documentdb_tests/compatibility/tests/system/administration/commands/autoCompact/test_autoCompact_success.py index b24e6c915..8b60d285a 100644 --- a/documentdb_tests/compatibility/tests/system/administration/commands/autoCompact/test_autoCompact_success.py +++ b/documentdb_tests/compatibility/tests/system/administration/commands/autoCompact/test_autoCompact_success.py @@ -12,7 +12,8 @@ ensure_autocompact_idle, ) from documentdb_tests.framework.assertions import assertResult -from documentdb_tests.framework.executor import execute_admin_command +from documentdb_tests.framework.error_codes import OBJECT_IS_BUSY_ERROR +from documentdb_tests.framework.executor import execute_admin_with_retry_command from documentdb_tests.framework.parametrize import pytest_params from documentdb_tests.framework.property_checks import Eq, IsType, NotExists @@ -116,7 +117,10 @@ def test_autoCompact_success(database_client, collection, test): # Ensure autoCompact is idle first: a leftover config from a prior test # would otherwise conflict. ensure_autocompact_idle(collection) - result = execute_admin_command(collection, test.build_command(ctx)) + # Tolerate the transient busy state while a prior compaction winds down. + result = execute_admin_with_retry_command( + collection, test.build_command(ctx), retry_code=OBJECT_IS_BUSY_ERROR + ) assertResult( result, expected=test.build_expected(ctx), diff --git a/documentdb_tests/compatibility/tests/system/administration/commands/autoCompact/test_smoke_autoCompact.py b/documentdb_tests/compatibility/tests/system/administration/commands/autoCompact/test_smoke_autoCompact.py index 5bb8b7954..26aba9f12 100644 --- a/documentdb_tests/compatibility/tests/system/administration/commands/autoCompact/test_smoke_autoCompact.py +++ b/documentdb_tests/compatibility/tests/system/administration/commands/autoCompact/test_smoke_autoCompact.py @@ -10,7 +10,8 @@ ensure_autocompact_idle, ) from documentdb_tests.framework.assertions import assertSuccessPartial -from documentdb_tests.framework.executor import execute_admin_command +from documentdb_tests.framework.error_codes import OBJECT_IS_BUSY_ERROR +from documentdb_tests.framework.executor import execute_admin_with_retry_command pytestmark = [pytest.mark.smoke, pytest.mark.no_parallel] @@ -20,7 +21,10 @@ def test_smoke_autoCompact(collection): # Ensure autoCompact is idle first: a leftover non-default config would make # this plain enable conflict instead of returning ok. ensure_autocompact_idle(collection) - result = execute_admin_command(collection, {"autoCompact": True}) + # Tolerate the transient busy state while a prior compaction winds down. + result = execute_admin_with_retry_command( + collection, {"autoCompact": True}, retry_code=OBJECT_IS_BUSY_ERROR + ) expected = {"ok": 1.0} assertSuccessPartial(result, expected, msg="Should support autoCompact command") diff --git a/documentdb_tests/compatibility/tests/system/administration/commands/autoCompact/utils/autoCompact_common.py b/documentdb_tests/compatibility/tests/system/administration/commands/autoCompact/utils/autoCompact_common.py index 4b325ed7a..ee9b6a833 100644 --- a/documentdb_tests/compatibility/tests/system/administration/commands/autoCompact/utils/autoCompact_common.py +++ b/documentdb_tests/compatibility/tests/system/administration/commands/autoCompact/utils/autoCompact_common.py @@ -2,32 +2,23 @@ from __future__ import annotations -import time - -from documentdb_tests.framework.executor import execute_admin_command +from documentdb_tests.framework.error_codes import OBJECT_IS_BUSY_ERROR +from documentdb_tests.framework.executor import execute_admin_with_retry_command def ensure_autocompact_idle(collection): - """Disable autoCompact, retrying until it reaches a deterministic idle state. + """Reset autoCompact to a disabled baseline before a test runs. - autoCompact is a server-wide setting, so a test inherits prior state, and a - single disable returns before the background wind-down finishes. This sends - disable repeatedly with a short pause between calls, returns only after - several consecutive disables succeed (so the async wind-down has time to - finish), and raises if it never settles within a bounded number of attempts. + autoCompact is a server-wide setting, so a test inherits prior state. The + background compaction winds down asynchronously, so we must try to disable + until the server stops complaining. Callers must be marked no_parallel: this only resets state left by a prior - test, not a concurrent worker mutating the shared setting between settling + test, not a concurrent worker mutating the shared setting between the reset and the command under test. """ - consecutive = 0 - for _ in range(200): - result = execute_admin_command(collection, {"autoCompact": False}) - if isinstance(result, dict) and result.get("ok") == 1.0: - consecutive += 1 - if consecutive >= 3: - return - else: - consecutive = 0 - time.sleep(0.05) - raise RuntimeError("autoCompact did not reach an idle state") + result = execute_admin_with_retry_command( + collection, {"autoCompact": False}, retry_code=OBJECT_IS_BUSY_ERROR + ) + if not (isinstance(result, dict) and result.get("ok") == 1.0): + raise RuntimeError(f"autoCompact did not reach an idle state: {result!r}") diff --git a/documentdb_tests/framework/error_codes.py b/documentdb_tests/framework/error_codes.py index ef220d300..8a71478fa 100644 --- a/documentdb_tests/framework/error_codes.py +++ b/documentdb_tests/framework/error_codes.py @@ -14,7 +14,7 @@ OVERFLOW_ERROR = 15 INVALID_LENGTH_ERROR = 16 ILLEGAL_OPERATION_ERROR = 20 -CONFLICTING_OPERATION_IN_PROGRESS_ERROR = 23 +ALREADY_INITIALIZED_ERROR = 23 NAMESPACE_NOT_FOUND_ERROR = 26 INDEX_NOT_FOUND_ERROR = 27 PATH_NOT_VIABLE_ERROR = 28 @@ -66,6 +66,7 @@ NO_QUERY_EXECUTION_PLANS_ERROR = 291 QUERY_EXCEEDED_MEMORY_NO_DISK_USE_ERROR = 292 FEATURE_NOT_SUPPORTED_ERROR = 303 +OBJECT_IS_BUSY_ERROR = 314 API_VERSION_ERROR = 322 API_STRICT_ERROR = 323 CANNOT_CONVERT_INDEX_TO_UNIQUE_ERROR = 359