monarch_rdma/
lib.rs

1/*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD-style license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9// RDMA requires frequent unsafe code blocks
10#![allow(clippy::undocumented_unsafe_blocks)]
11
12use local_memory::KeepaliveLocalMemory;
13use serde::Deserialize;
14use serde::Serialize;
15
16#[macro_use]
17mod macros;
18
19mod action;
20pub mod backend;
21pub mod config;
22pub mod device_selection;
23pub mod efa;
24mod errors;
25pub mod local_memory;
26mod rdma_components;
27mod rdma_manager_actor;
28mod rdma_manager_owner;
29mod rdma_runtime;
30
31pub use backend::ibverbs::primitives::*;
32
33/// Whether any RDMA backend is available on this system.
34///
35/// Returns true if ibverbs hardware is present, or if TCP fallback
36/// is enabled via [`config::RDMA_ALLOW_TCP_FALLBACK`].
37pub fn rdma_supported() -> bool {
38    ibverbs_supported() || hyperactor_config::global::get(config::RDMA_ALLOW_TCP_FALLBACK)
39}
40pub use action::RdmaAction;
41// Re-export the CUDA segment scanner API for the extension/test crates to
42// install a process-wide scanner (see `backend::ibverbs::mlx_domain`).
43pub use backend::ibverbs::mlx_domain::CudaSegmentScanner;
44pub use backend::ibverbs::mlx_domain::ScannedSegment;
45pub use backend::ibverbs::mlx_domain::register_cuda_segment_scanner;
46pub use errors::RdmaInitError;
47pub use rdma_components::RdmaRemoteBuffer;
48pub use rdma_components::*;
49pub use rdma_manager_actor::*;
50pub use rdma_manager_owner::*;
51// Re-export rdmaxcel_sys for extension crate to access types
52pub use rdmaxcel_sys;
53pub use test_utils::is_cuda_available;
54
55/// Type of RDMA operation.
56#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
57pub enum RdmaOpType {
58    ReadIntoLocal,
59    WriteFromLocal,
60}
61
62/// A single RDMA operation to be submitted to a backend.
63#[derive(Debug)]
64pub struct RdmaOp {
65    pub op_type: RdmaOpType,
66    pub local: KeepaliveLocalMemory,
67    pub remote: RdmaRemoteBuffer,
68}
69
70/// Transport level, ordered slowest to fastest.
71///
72/// The `Ord` implementation reflects this ordering, enabling transport
73/// selection via comparison (e.g., "at least NIC speed").
74#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
75pub enum RdmaTransportLevel {
76    /// TCP/IP sockets (fallback transport).
77    Tcp,
78    /// RDMA NIC (RoCE, InfiniBand, EFA).
79    Nic,
80    /// Direct memory access (NVLink, shared memory).
81    Memory,
82}
83
84/// Print comprehensive RDMA device information for debugging.
85/// Controlled by MONARCH_DEBUG_RDMA environment variable.
86pub fn print_device_info_if_debug_enabled(context: *mut rdmaxcel_sys::ibv_context) {
87    if std::env::var("MONARCH_DEBUG_RDMA").is_ok() {
88        unsafe {
89            rdmaxcel_sys::rdmaxcel_print_device_info(context);
90        }
91    }
92}
93
94/// Print comprehensive RDMA device information for debugging (always prints).
95pub fn print_device_info(context: *mut rdmaxcel_sys::ibv_context) {
96    unsafe {
97        rdmaxcel_sys::rdmaxcel_print_device_info(context);
98    }
99}
100
101mod test_utils;
monarch_rdma/lib.rs

monarch_rdma/
lib.rs
