Skip to content

vllm.utils.cpu_resource_utils

Functions:

_synthesize_cpu_list() cached

Synthesize a flat CPU list: each logical CPU is its own core on NUMA node 0. Used when lscpu output is unavailable or unparsable (e.g. macOS, RISC-V).

Source code in vllm/utils/cpu_resource_utils.py 
281
282
283
284
285
286
287
288
@cache
def _synthesize_cpu_list() -> list[LogicalCPUInfo]:
    """Synthesize a flat CPU list: each logical CPU is its own core on
    NUMA node 0.  Used when lscpu output is unavailable or unparsable
    (e.g. macOS, RISC-V)."""
    cpu_count = os.cpu_count()
    assert cpu_count
    return [LogicalCPUInfo(i, i, 0) for i in range(cpu_count)]

check_cgroup_memory_available(required_bytes, allocation_name)

Log cgroup memory headroom for an upcoming allocation.

Parameters:

  • required_bytes

    (int) –

    Bytes required by the allocation.

  • allocation_name

    (str) –

    Human-readable name used in log messages.

Low headroom logs a warning, but does not reject the allocation because cgroup usage can include reclaimable memory. If the cgroup limit or usage cannot be read, the check is skipped.

Source code in vllm/utils/cpu_resource_utils.py 
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
def check_cgroup_memory_available(
    required_bytes: int,
    allocation_name: str,
) -> None:
    """Log cgroup memory headroom for an upcoming allocation.

    Args:
        required_bytes: Bytes required by the allocation.
        allocation_name: Human-readable name used in log messages.

    Low headroom logs a warning, but does not reject the allocation because
    cgroup usage can include reclaimable memory. If the cgroup limit or usage
    cannot be read, the check is skipped.

    """
    cgroup_limit = get_cgroup_memory_limit()
    cgroup_usage = get_cgroup_memory_usage()
    if cgroup_limit is None or cgroup_usage is None:
        return

    cgroup_available = max(0, cgroup_limit - cgroup_usage)
    mib = 1 << 20
    remaining_bytes = cgroup_available - required_bytes
    log_fn = logger.debug if remaining_bytes >= 0 else logger.warning
    status = (
        "current headroom meets the requested allocation"
        if remaining_bytes >= 0
        else "current headroom is below the requested allocation; allocation "
        "will still be attempted because cgroup usage may be reclaimable"
    )
    log_fn(
        "Cgroup memory preflight for %s: %.0f MiB required, %.0f MiB current "
        "usage, %.0f MiB available under %.0f MiB limit, %.0f MiB remaining "
        "after allocation based on current usage; %s.",
        allocation_name,
        required_bytes / mib,
        cgroup_usage / mib,
        cgroup_available / mib,
        cgroup_limit / mib,
        remaining_bytes / mib,
        status,
    )

get_cgroup_memory_limit() cached

Return the cgroup memory limit in bytes, or None.

Supports both cgroup v2 (unified) and v1. Returns None when not running under a constrained cgroup (e.g. bare metal, or limit reported as max/an unrealistically large value).

Source code in vllm/utils/cpu_resource_utils.py 
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
@cache
def get_cgroup_memory_limit() -> int | None:
    """Return the cgroup memory limit in bytes, or None.

    Supports both cgroup v2 (unified) and v1. Returns None when
    not running under a constrained cgroup (e.g. bare metal, or limit
    reported as `max`/an unrealistically large value).
    """
    if sys.platform != "linux":
        return None

    # cgroup v2 unified hierarchy
    v2_limit = _read_int_file("/sys/fs/cgroup/memory.max")
    if v2_limit is not None:
        return v2_limit

    # cgroup v1
    v1_limit = _read_int_file("/sys/fs/cgroup/memory/memory.limit_in_bytes")
    if v1_limit is not None:
        # cgroup v1 reports a huge sentinel (close to PAGE_COUNTER_MAX)
        # when unlimited. Treat absurdly large values as "no limit".
        if v1_limit >= (1 << 62):
            return None
        return v1_limit

    return None

get_cgroup_memory_usage()

Return the current cgroup memory usage in bytes, or None.

The usage value is intentionally read on every call because cgroup memory usage changes while the process is running.

Source code in vllm/utils/cpu_resource_utils.py 
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
def get_cgroup_memory_usage() -> int | None:
    """Return the current cgroup memory usage in bytes, or None.

    The usage value is intentionally read on every call because cgroup
    memory usage changes while the process is running.
    """
    if sys.platform != "linux":
        return None

    # cgroup v2 unified hierarchy
    if _read_int_file("/sys/fs/cgroup/memory.max") is not None:
        return _read_int_file("/sys/fs/cgroup/memory.current")

    # cgroup v1
    v1_limit = _read_int_file("/sys/fs/cgroup/memory/memory.limit_in_bytes")
    if v1_limit is not None and v1_limit < (1 << 62):
        return _read_int_file("/sys/fs/cgroup/memory/memory.usage_in_bytes")

    return None

get_visible_memory_node(assigned_node_ids=None)

Memory nodes visible to this process.

assigned_node_ids is the per-DP-rank list of NUMA node ids from parallel_config.assigned_physical_gpu_ids. It takes priority over the CPU_VISIBLE_MEMORY_NODES env var, which still applies when it is None.

Source code in vllm/utils/cpu_resource_utils.py 
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
def get_visible_memory_node(assigned_node_ids: list[int] | None = None) -> list[int]:
    """Memory nodes visible to this process.

    assigned_node_ids is the per-DP-rank list of NUMA node ids from
    parallel_config.assigned_physical_gpu_ids. It takes priority over the
    CPU_VISIBLE_MEMORY_NODES env var, which still applies when it is None.
    """
    if sys.platform == "darwin":
        return [0]

    allowed_memory_node_list = get_memory_affinity()

    if "VLLM_CPU_SIM_MULTI_NUMA" in os.environ:
        return allowed_memory_node_list

    if assigned_node_ids is not None:
        return [node for node in assigned_node_ids if node in allowed_memory_node_list]

    env_key = DEVICE_CONTROL_ENV_VAR
    if env_key in os.environ and os.environ[env_key] != "":
        visible_nodes = [int(s) for s in os.environ[env_key].split(",")]
        visible_nodes = [
            node for node in visible_nodes if node in allowed_memory_node_list
        ]
        return visible_nodes

    return allowed_memory_node_list

parse_id_list(raw_str)

Parses strings like '0-2,4,7-8' into [0, 1, 2, 4, 7, 8].

Source code in vllm/utils/cpu_resource_utils.py 
174
175
176
177
178
179
180
181
182
183
184
185
186
def parse_id_list(raw_str: str) -> list[int]:
    """Parses strings like '0-2,4,7-8' into [0, 1, 2, 4, 7, 8]."""
    result: list[int] = []
    if not raw_str:
        return result

    for part in raw_str.split(","):
        if "-" in part:
            start, end = map(int, part.split("-"))
            result.extend(range(start, end + 1))
        else:
            result.append(int(part))
    return sorted(list(set(result)))