monarch_rdma/config.rs
1/*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD-style license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9//! RDMA configuration attributes.
10
11use std::time::Duration;
12
13use hyperactor_config::CONFIG;
14use hyperactor_config::ConfigAttr;
15use hyperactor_config::NonZeroUsize;
16use hyperactor_config::attrs::declare_attrs;
17
18declare_attrs! {
19 /// Maximum chunk size in MiB for TCP-based RDMA transfers.
20 @meta(CONFIG = ConfigAttr::new(
21 Some("MONARCH_RDMA_MAX_CHUNK_SIZE_MB".to_string()),
22 Some("rdma_max_chunk_size_mb".to_string()),
23 ))
24 pub attr RDMA_MAX_CHUNK_SIZE_MB: usize = 64;
25
26 /// Allow TCP fallback when ibverbs hardware is unavailable.
27 ///
28 /// When true (the default), RDMA operations fall back to chunked
29 /// hyperactor messaging over the default channel transport. When
30 /// false, operations fail if no ibverbs backend is available.
31 @meta(CONFIG = ConfigAttr::new(
32 Some("MONARCH_RDMA_ALLOW_TCP_FALLBACK".to_string()),
33 Some("rdma_allow_tcp_fallback".to_string()),
34 ))
35 pub attr RDMA_ALLOW_TCP_FALLBACK: bool = true;
36
37 /// Disable ibverbs even when hardware is present.
38 ///
39 /// When true, `RdmaManagerActor` skips ibverbs initialization and
40 /// relies on the TCP fallback (if enabled). Useful for testing the
41 /// TCP transport on machines that have RDMA hardware.
42 @meta(CONFIG = ConfigAttr::new(
43 Some("MONARCH_RDMA_DISABLE_IBVERBS".to_string()),
44 Some("rdma_disable_ibverbs".to_string()),
45 ))
46 pub attr RDMA_DISABLE_IBVERBS: bool = false;
47
48 /// Use one implicit-ODP registration per mlx5 device for newly registered
49 /// host buffers. Destination ranges receive asynchronous write advice;
50 /// per-allocation host-memory registration remains the default.
51 @meta(CONFIG = ConfigAttr::new(
52 Some("MONARCH_RDMA_HOST_ODP".to_string()),
53 Some("rdma_host_odp".to_string()),
54 ))
55 pub attr RDMA_HOST_ODP: bool = false;
56
57 /// Number of parallel channels for TCP fallback transfers.
58 ///
59 /// When greater than 1, each [`TcpManagerActor`] serves this many
60 /// direct `hyperactor::channel` connections for bulk data transfer,
61 /// bypassing the single-socket actor mailbox. Default is 1
62 /// (sequential, existing behavior).
63 @meta(CONFIG = ConfigAttr::new(
64 Some("MONARCH_RDMA_TCP_FALLBACK_PARALLELISM".to_string()),
65 Some("rdma_tcp_fallback_parallelism".to_string()),
66 ))
67 pub attr RDMA_TCP_FALLBACK_PARALLELISM: usize = 1;
68
69 /// Per-side budget for the `QueuePairInitializer` handshake. The
70 /// timer arms once when we send `EnsureQueuePair` and is rearmed
71 /// after we hit RTS while still waiting for the peer's
72 /// `NotifyRts`. If it fires the entry is tombstoned with a
73 /// `qp_initializer_failed` so further `RequestQueuePair` calls
74 /// for the same key surface the same error rather than hanging.
75 @meta(CONFIG = ConfigAttr::new(
76 Some("MONARCH_RDMA_QP_INIT_TIMEOUT".to_string()),
77 Some("rdma_qp_init_timeout".to_string()),
78 ))
79 pub attr RDMA_QP_INIT_TIMEOUT: Duration = Duration::from_secs(30);
80
81 /// Default ibverbs device target for managers without an explicit target.
82 ///
83 /// Accepted forms are `cpu:<numa>`, `gpu:<ordinal>`, and `nic:<name>`.
84 /// The empty default preserves automatic device selection. Non-empty value
85 /// syntax is validated when the RDMA manager starts.
86 @meta(CONFIG = ConfigAttr::new(
87 Some("MONARCH_RDMA_IBVERBS_TARGET".to_string()),
88 Some("rdma_ibverbs_target".to_string()),
89 ))
90 pub attr RDMA_IBVERBS_TARGET: String = String::new();
91
92 /// Which peer NICs each local NIC may pair with for a transfer.
93 ///
94 /// Accepted forms are `any`, `match_name`, and `groups:` followed by any
95 /// number of `|`-separated groups, each naming any number of devices,
96 /// comma-separated — `groups:mlx5_0,mlx5_1|mlx5_2,mlx5_3|mlx5_4,mlx5_5` and
97 /// `groups:mlx5_0` are both valid. Groups must be disjoint. Parsed into a
98 /// [`PeerDeviceAffinityPolicy`](crate::backend::ibverbs::device_selection::PeerDeviceAffinityPolicy).
99 /// The empty default means `any`, since a `String` attribute cannot default
100 /// to anything else. Value syntax is validated when the RDMA manager starts.
101 @meta(CONFIG = ConfigAttr::new(
102 Some("MONARCH_RDMA_PEER_DEVICE_AFFINITY".to_string()),
103 Some("rdma_peer_device_affinity".to_string()),
104 ))
105 pub attr RDMA_PEER_DEVICE_AFFINITY: String = String::new();
106
107 /// How many NICs a buffer is registered on, at most. `None`, the default,
108 /// sets no limit: every equally good NIC serves the buffer. An empty
109 /// environment value parses to `None`.
110 ///
111 /// Depending on where in memory a buffer lives, there may be many
112 /// NICs that would be equally good for serving it. Registering the
113 /// buffer on each NIC improves flexibility and provides performance
114 /// optimization opportunities, but comes at the cost of an expensive
115 /// memory registration per NIC. This config attribute lets the user
116 /// tune this tradeoff.
117 ///
118 /// When multiple optimal NICs are available for a buffer, selecting
119 /// among them depends on [`RDMA_PEER_DEVICE_AFFINITY`]:
120 /// - `any`: first chosen at random, then the rest are taken
121 /// lexicographically starting from the first.
122 /// - `match_name`: taken lexicographically.
123 /// - `groups`: round robin across groups, taken lexicographically
124 /// within each group (so `groups:nic0,nic1|nic2,nic3`
125 /// with `RDMA_MAX_NICS_PER_BUFFER = 2` would choose
126 /// `nic0` and `nic2`).
127 ///
128 /// A manager pinned to one device by [`RDMA_IBVERBS_TARGET`] registers
129 /// there and ignores this.
130 @meta(CONFIG = ConfigAttr::new(
131 Some("MONARCH_RDMA_MAX_NICS_PER_BUFFER".to_string()),
132 Some("rdma_max_nics_per_buffer".to_string()),
133 ))
134 pub attr RDMA_MAX_NICS_PER_BUFFER: Option<NonZeroUsize> = None;
135
136 /// Minimum RDMA stripe size, in KiB.
137 ///
138 /// No stripe of a transfer is smaller than this, which limits how many
139 /// stripes a transfer is split into. Transfers smaller than the minimum
140 /// use one stripe.
141 @meta(CONFIG = ConfigAttr::new(
142 Some("MONARCH_RDMA_MIN_STRIPE_SIZE_KB".to_string()),
143 Some("rdma_min_stripe_size_kb".to_string()),
144 ))
145 pub attr RDMA_MIN_STRIPE_SIZE_KB: NonZeroUsize =
146 NonZeroUsize::new(512).expect("512 is non-zero");
147
148 /// How many queue pairs share one completion queue.
149 ///
150 /// Sharing is what lets one poller reap for several queue pairs. Each
151 /// completion queue is sized to hold every sharer's work requests at once
152 /// (`rdma_qps_per_cq * max_send_wr` entries), so raising this trades
153 /// completion-queue memory for fewer completion queues to poll. Opening a
154 /// device fails outright if it cannot hold a completion queue that large.
155 /// The default is 64.
156 @meta(CONFIG = ConfigAttr::new(
157 Some("MONARCH_RDMA_QPS_PER_CQ".to_string()),
158 Some("rdma_qps_per_cq".to_string()),
159 ))
160 pub attr RDMA_QPS_PER_CQ: NonZeroUsize =
161 NonZeroUsize::new(64).expect("64 is non-zero");
162
163 /// How many queue pairs to create for each local-NIC/peer/remote-NIC route.
164 ///
165 /// A transfer is split into at least this many stripes, rounded up to a
166 /// multiple of its NIC pair count, so a transfer over one NIC pair still
167 /// uses every queue pair. Stripes take each route's queue pairs in round
168 /// robin order. [`RDMA_MIN_STRIPE_SIZE_KB`] lowers the stripe count for
169 /// small transfers.
170 @meta(CONFIG = ConfigAttr::new(
171 Some("MONARCH_RDMA_QPS_PER_PEER".to_string()),
172 Some("rdma_qps_per_peer".to_string()),
173 ))
174 pub attr RDMA_QPS_PER_PEER: NonZeroUsize =
175 NonZeroUsize::new(4).expect("4 is non-zero");
176
177 /// Whether each device gets its own completion-queue poller.
178 ///
179 /// When true (the default), the manager spawns one poller per RDMA device.
180 /// When false, one poller serves every device in the process.
181 @meta(CONFIG = ConfigAttr::new(
182 Some("MONARCH_RDMA_CQ_POLLER_PER_DEVICE".to_string()),
183 Some("rdma_cq_poller_per_device".to_string()),
184 ))
185 pub attr RDMA_CQ_POLLER_PER_DEVICE: bool = true;
186
187 /// Worker-thread count for the shared rdma data-plane runtime, which runs
188 /// RDMA actors and queue-pair worker tasks. The default is 4.
189 ///
190 /// The runtime is built once, lazily, so this value is latched at the
191 /// first RDMA use in a process and later changes have no effect.
192 @meta(CONFIG = ConfigAttr::new(
193 Some("MONARCH_RDMA_RUNTIME_WORKER_THREADS".to_string()),
194 Some("rdma_runtime_worker_threads".to_string()),
195 ))
196 pub attr RDMA_RUNTIME_WORKER_THREADS: usize = 4;
197}