diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh index d4616143a..6dadc2687 100755 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh +++ b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh @@ -35,6 +35,10 @@ if [ "${EVAL_ONLY}" = "true" ]; then MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" fi +if [ "$MAX_MODEL_LEN" -lt 9472 ]; then + MAX_MODEL_LEN=9472 +fi + # If the machine runs a MEC FW older than 177, RCCL # cannot reclaim some memory. # Disable that features to avoid crashes. @@ -47,6 +51,12 @@ fi export VLLM_ROCM_USE_AITER=1 export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 +export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 +export VLLM_ROCM_USE_SKINNY_GEMM=0 +export AITER_MXFP4_INTERMEDIATE=1 +export AITER_BYPASS_TUNE_CONFIG=0 +export AITER_MOE_SORT_BACKEND=auto +export OMP_NUM_THREADS=1 # Disable AITER RMSNorm for TP < 8 due to accuracy issues if [ "${TP}" -lt 8 ]; then @@ -59,9 +69,6 @@ else EP=" " fi -# following AMD andy luo's recipe -# https://x.com/linluo77/status/2017024513595301985 - # Start GPU monitoring (power, temperature, clocks every second) start_gpu_monitor @@ -71,8 +78,11 @@ vllm serve $MODEL --port $PORT \ $EP \ --gpu-memory-utilization 0.90 \ --max-model-len $MAX_MODEL_LEN \ ---block-size=1 \ ---no-enable-prefix-caching \ +--kv-cache-dtype fp8 \ +--block-size 16 \ +--max-num-batched-tokens 16384 \ +--max-num-seqs 512 \ +--async-scheduling \ --trust-remote-code \ --no-enable-prefix-caching \ --mm-encoder-tp-mode data > $SERVER_LOG 2>&1 & diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 4b1ea4aed..2327426cc 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -770,7 +770,8 @@ kimik2.5-int4-mi300x-vllm: - { tp: 8, conc-start: 4, conc-end: 64 } kimik2.5-fp4-mi355x-vllm: - image: vllm/vllm-openai-rocm:v0.24.0 + # TODO: replace with the official vLLM ROCm image containing AITER #3832. + image: TBD model: amd/Kimi-K2.5-MXFP4 model-prefix: kimik2.5 runner: mi355x @@ -782,13 +783,13 @@ kimik2.5-fp4-mi355x-vllm: - isl: 1024 osl: 1024 search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - { tp: 4, conc-start: 4, conc-end: 64 } + - { tp: 8, conc-start: 4, conc-end: 128 } + - { tp: 4, conc-start: 4, conc-end: 128 } - isl: 8192 osl: 1024 search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - { tp: 4, conc-start: 4, conc-end: 64 } + - { tp: 8, conc-start: 4, conc-end: 128 } + - { tp: 4, conc-start: 4, conc-end: 128 } kimik2.5-fp4-mi355x-vllm-agentic: image: vllm/vllm-openai-rocm:v0.22.0 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d77c35f1f..c2151d778 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4781,3 +4781,10 @@ - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260708" - "Clean the export envs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2198 + +- config-keys: + - kimik2.5-fp4-mi355x-vllm + description: + - "Apply the accuracy-gated Kimi-K2.5 MXFP4 TP4 settings: tuned AITER MXFP4 MoE, fused shared experts, FP8 KV cache, block size 16, 16384 batched tokens, 512 sequences, async scheduling, and the AITER BF16 GEMM path" + - "Extend the existing TP4 and TP8 concurrency sweeps from 64 to 128 without changing other search dimensions; the image remains TBD until an official vLLM ROCm image contains AITER #3832" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213