monarch_rdma/
lib.rs

1/*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD-style license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9// RDMA requires frequent unsafe code blocks
10#![allow(clippy::undocumented_unsafe_blocks)]
11
12use local_memory::KeepaliveLocalMemory;
13use serde::Deserialize;
14use serde::Serialize;
15
16#[macro_use]
17mod macros;
18
19mod action;
20pub mod backend;
21pub mod config;
22pub mod device_selection;
23pub mod efa;
24pub mod local_memory;
25mod rdma_components;
26mod rdma_manager_actor;
27mod rdma_runtime;
28
29pub use backend::ibverbs::primitives::*;
30
31/// Whether any RDMA backend is available on this system.
32///
33/// Returns true if ibverbs hardware is present, or if TCP fallback
34/// is enabled via [`config::RDMA_ALLOW_TCP_FALLBACK`].
35pub fn rdma_supported() -> bool {
36    ibverbs_supported() || hyperactor_config::global::get(config::RDMA_ALLOW_TCP_FALLBACK)
37}
38pub use action::RdmaAction;
39// Re-export the CUDA segment scanner API for the extension/test crates to
40// install a process-wide scanner (see `backend::ibverbs::mlx_domain`).
41pub use backend::ibverbs::mlx_domain::CudaSegmentScanner;
42pub use backend::ibverbs::mlx_domain::ScannedSegment;
43pub use backend::ibverbs::mlx_domain::register_cuda_segment_scanner;
44pub use rdma_components::RdmaRemoteBuffer;
45pub use rdma_components::*;
46pub use rdma_manager_actor::*;
47// Re-export rdmaxcel_sys for extension crate to access types
48pub use rdmaxcel_sys;
49pub use test_utils::is_cuda_available;
50
51/// Type of RDMA operation.
52#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
53pub enum RdmaOpType {
54    ReadIntoLocal,
55    WriteFromLocal,
56}
57
58/// A single RDMA operation to be submitted to a backend.
59#[derive(Debug)]
60pub struct RdmaOp {
61    pub op_type: RdmaOpType,
62    pub local: KeepaliveLocalMemory,
63    pub remote: RdmaRemoteBuffer,
64}
65
66/// Transport level, ordered slowest to fastest.
67///
68/// The `Ord` implementation reflects this ordering, enabling transport
69/// selection via comparison (e.g., "at least NIC speed").
70#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
71pub enum RdmaTransportLevel {
72    /// TCP/IP sockets (fallback transport).
73    Tcp,
74    /// RDMA NIC (RoCE, InfiniBand, EFA).
75    Nic,
76    /// Direct memory access (NVLink, shared memory).
77    Memory,
78}
79
80/// Print comprehensive RDMA device information for debugging.
81/// Controlled by MONARCH_DEBUG_RDMA environment variable.
82pub fn print_device_info_if_debug_enabled(context: *mut rdmaxcel_sys::ibv_context) {
83    if std::env::var("MONARCH_DEBUG_RDMA").is_ok() {
84        unsafe {
85            rdmaxcel_sys::rdmaxcel_print_device_info(context);
86        }
87    }
88}
89
90/// Print comprehensive RDMA device information for debugging (always prints).
91pub fn print_device_info(context: *mut rdmaxcel_sys::ibv_context) {
92    unsafe {
93        rdmaxcel_sys::rdmaxcel_print_device_info(context);
94    }
95}
96
97mod test_utils;
monarch_rdma/lib.rs

monarch_rdma/
lib.rs
