MODEL := "Qwen/Qwen3-30B-A3B-FP8"

tp PORT:
  vllm serve {{MODEL}} \
    --port {{PORT}} \
    --tensor-parallel-size 2 \
    --enforce-eager \
    --disable-log-requests

dp_a_internal_lb PORT:
  vllm serve {{MODEL}} \
    --port {{PORT}} \
    --data-parallel-size 4 \
    --data-parallel-size-local 2 \
    --data-parallel-rpc-port 5555 \
    --enable-expert-parallel \
    --enforce-eager \
    --disable-log-requests

dp_b_internal_lb:
  vllm serve {{MODEL}} \
    --headless \
    --data-parallel-size 4 \
    --data-parallel-size-local 2 \
    --data-parallel-start-rank 2 \
    --data-parallel-rpc-port 5555 \
    --enable-expert-parallel \
    --enforce-eager \
    --disable-log-requests

dp_a_external_lb PORT:
   vllm serve {{MODEL}} \
    --port 8100 \
    --data-parallel-size 2 \
    --data-parallel-rank 0 \
    --data-parallel-rpc-port 5555 \
    --enable-expert-parallel \
    --enforce-eager \
    --disable-log-requests

dp_b_external_lb PORT:
  vllm serve {{MODEL}} \
    --port {{PORT}} \
    --data-parallel-size 2 \
    --data-parallel-rank 1 \
    --data-parallel-rpc-port 5555 \
    --enable-expert-parallel \
    --enforce-eager \
    --disable-log-requests


eval PORT CONCURRENT LIMIT:
  lm_eval --model local-completions --tasks gsm8k \
    --model_args model={{MODEL}},base_url=http://127.0.0.1:{{PORT}}/v1/completions,num_concurrent={{CONCURRENT}},num_retries=0,tokenized_requests=False \
    --limit {{LIMIT}}

metrics PORT:
   curl http://localhost:{{PORT}}/metrics

# TYPE vllm:num_requests_running gauge
vllm:num_requests_running{engine="0",model_name="Qwen/Qwen3-30B-A3B-FP8"} 4.0
# HELP vllm:num_requests_waiting Number of requests waiting to be processed.
# TYPE vllm:num_requests_waiting gauge
vllm:num_requests_waiting{engine="0",model_name="Qwen/Qwen3-30B-A3B-FP8"} 0.0

INFO 07-20 18:05:20 [loggers.py:122] Engine 000: Avg prompt throughput: 0.0 tokens/s, Avg generation throughput: 0.0 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0%

just dp_a_internal_lb 8100
just dp_b_internal_lb 8100
just eval 8100
just metrics 8100

vllm:kv_cache_usage_perc{engine="0",model_name="Qwen/Qwen3-30B-A3B-FP8"} 2.6821875921956284e-05
vllm:kv_cache_usage_perc{engine="1",model_name="Qwen/Qwen3-30B-A3B-FP8"} 2.686799752815716e-05
vllm:kv_cache_usage_perc{engine="2",model_name="Qwen/Qwen3-30B-A3B-FP8"} 2.686799752815716e-05
vllm:kv_cache_usage_perc{engine="3",model_name="Qwen/Qwen3-30B-A3B-FP8"} 2.663896214605277e-05

INFO 07-20 18:10:58 [loggers.py:122] Engine 000: Avg prompt throughput: 3064.0 tokens/s, Avg generation throughput: 359.0 tokens/s, Running: 26 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.3%, Prefix cache hit rate: 0.0%
INFO 07-20 18:10:58 [loggers.py:122] Engine 001: Avg prompt throughput: 2508.4 tokens/s, Avg generation throughput: 353.7 tokens/s, Running: 25 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.4%, Prefix cache hit rate: 0.0%
INFO 07-20 18:10:58 [loggers.py:122] Engine 002: Avg prompt throughput: 1962.9 tokens/s, Avg generation throughput: 353.5 tokens/s, Running: 24 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.3%, Prefix cache hit rate: 0.0%
INFO 07-20 18:10:58 [loggers.py:122] Engine 003: Avg prompt throughput: 2619.2 tokens/s, Avg generation throughput: 354.6 tokens/s, Running: 25 reqs, Waiting: 0 reqs, GPU KV cache usage: 5.2%, Prefix cache hit rate: 0.6%

just dp_a_external_lb 8100
just dp_b_external_lb 8100
just eval 8100
just eval 8200
just metrics 8100
just metrics 8200

vllm:request_success_total{engine="0",finished_reason="stop",model_name="Qwen/Qwen3-30B-A3B-FP8"} 88.0
vllm:request_success_total{engine="0",finished_reason="length",model_name="Qwen/Qwen3-30B-A3B-FP8"} 12.0
vllm:request_success_total{engine="0",finished_reason="abort",model_name="Qwen/Qwen3-30B-A3B-FP8"} 0.0

INFO 07-20 18:15:37 [loggers.py:122] Engine 000: Avg prompt throughput: 10130.6 tokens/s, Avg generation throughput: 506.1 tokens/s, Running: 99 reqs, Waiting: 0 reqs, GPU KV cache usage: 20.0%, Prefix cache hit rate: 0.0%

vllm:request_success_total{engine="1",finished_reason="stop",model_name="Qwen/Qwen3-30B-A3B-FP8"} 88.0
vllm:request_success_total{engine="1",finished_reason="length",model_name="Qwen/Qwen3-30B-A3B-FP8"} 12.0
vllm:request_success_total{engine="1",finished_reason="abort",model_name="Qwen/Qwen3-30B-A3B-FP8"} 0.0

INFO 07-20 18:15:47 [loggers.py:122] Engine 001: Avg prompt throughput: 10129.2 tokens/s, Avg generation throughput: 894.6 tokens/s, Running: 69 reqs, Waiting: 0 reqs, GPU KV cache usage: 14.4%, Prefix cache hit rate: 0.0%