monarch_rdma/backend/
ibverbs.rs

1/*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD-style license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9//! ibverbs backend implementation for RDMA operations.
10
11use hyperactor::ActorRef;
12use hyperactor::actor::Referable;
13use serde::Deserialize;
14use serde::Serialize;
15use typeuri::Named;
16
17pub mod device;
18pub mod device_selection;
19pub(crate) mod domain;
20pub mod efa_device;
21pub mod efa_domain;
22pub mod manager_actor;
23pub mod memory_region;
24pub mod mlx_device;
25pub mod mlx_domain;
26pub mod mlx_queue_pair;
27pub mod primitives;
28pub mod queue_pair;
29
30use manager_actor::IbvManagerActor;
31use mlx_device::MlxDevice;
32pub use queue_pair::IbvQueuePair;
33pub use queue_pair::PollTarget;
34
35#[cfg(test)]
36mod doorbell_test_utils;
37#[cfg(test)]
38mod doorbell_tests;
39#[cfg(test)]
40mod mlx5dv_tests;
41
42use crate::RdmaOpType;
43use crate::local_memory::KeepaliveLocalMemory;
44
45/// Lazily-initialized ibverbs transport details for a registered memory
46/// region. Retrieved on demand from the [`IbvManagerActor`] via
47/// [`IbvManagerMessage::RequestBuffer`].
48#[derive(Debug, Clone, Serialize, Deserialize, Named)]
49pub struct IbvBuffer {
50    pub lkey: u32,
51    pub rkey: u32,
52    /// RDMA address (may differ from virtual address for CUDA memory).
53    pub addr: usize,
54    pub size: usize,
55    /// Name of the RDMA device this buffer is associated with (e.g., "mlx5_0").
56    pub device_name: String,
57}
58
59impl From<&memory_region::IbvMemoryRegionView> for IbvBuffer {
60    /// The wire transport details are fully derived from the registered MR
61    /// view: the keys, the RDMA address, the size, and the device name.
62    fn from(view: &memory_region::IbvMemoryRegionView) -> Self {
63        Self {
64            lkey: view.lkey,
65            rkey: view.rkey,
66            addr: view.rdma_addr,
67            size: view.size,
68            device_name: view.device_name.clone(),
69        }
70    }
71}
72
73/// A single RDMA op for the [`IbvBackend`](manager_actor::IbvBackend).
74///
75/// Generic over the manager actor type so unit tests can swap in a
76/// mock; production code uses the default `IbvOp<IbvManagerActor>`.
77#[derive(Debug, Named)]
78pub struct IbvOp<M: Referable = IbvManagerActor<MlxDevice>> {
79    pub op_type: RdmaOpType,
80    pub local_memory: KeepaliveLocalMemory,
81    pub remote_buffer: IbvBuffer,
82    pub remote_manager: ActorRef<M>,
83}
84
85// Hand-rolled `Clone` to avoid the `M: Clone` bound the derive macro
86// would add (`ActorRef<M>` is `Clone` for all `M: Referable`).
87impl<M: Referable> Clone for IbvOp<M> {
88    fn clone(&self) -> Self {
89        Self {
90            op_type: self.op_type,
91            local_memory: self.local_memory.clone(),
92            remote_buffer: self.remote_buffer.clone(),
93            remote_manager: self.remote_manager.clone(),
94        }
95    }
96}
monarch_rdma/backend/ibverbs.rs

monarch_rdma/backend/
ibverbs.rs
