monarch_rdma/
efa.rs

1/*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD-style license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9//! EFA (Elastic Fabric Adapter) specific RDMA operations.
10//!
11//! This module contains EFA-specific helpers for device detection and configuration.
12//! Connect and post operations are handled by C functions in rdmaxcel.c.
13
14use std::sync::OnceLock;
15
16/// Cached result of EFA device check.
17static EFA_DEVICE_CACHE: OnceLock<bool> = OnceLock::new();
18
19/// Checks if any EFA device is available in the system.
20///
21/// Uses `efadv_query_device()` to detect EFA hardware.
22/// The result is cached after the first call.
23pub fn is_efa_device() -> bool {
24    *EFA_DEVICE_CACHE.get_or_init(is_efa_device_impl)
25}
26
27fn is_efa_device_impl() -> bool {
28    // SAFETY: We are calling C functions from libibverbs and libefa.
29    unsafe {
30        let mut num_devices = 0;
31        let device_list = rdmaxcel_sys::ibv_get_device_list(&mut num_devices);
32        if device_list.is_null() || num_devices == 0 {
33            return false;
34        }
35        let mut found = false;
36        for i in 0..num_devices {
37            let device = *device_list.add(i as usize);
38            if device.is_null() {
39                continue;
40            }
41            let context = rdmaxcel_sys::ibv_open_device(device);
42            if context.is_null() {
43                continue;
44            }
45            if rdmaxcel_sys::rdmaxcel_is_efa_dev(context) != 0 {
46                found = true;
47                rdmaxcel_sys::ibv_close_device(context);
48                break;
49            }
50            rdmaxcel_sys::ibv_close_device(context);
51        }
52        rdmaxcel_sys::ibv_free_device_list(device_list);
53        found
54    }
55}
56
57/// Returns the MR access flags appropriate for EFA devices.
58///
59/// EFA does not support `IBV_ACCESS_REMOTE_ATOMIC`, so this returns only
60/// local write, remote write, and remote read flags.
61pub fn mr_access_flags() -> rdmaxcel_sys::ibv_access_flags {
62    rdmaxcel_sys::ibv_access_flags::IBV_ACCESS_LOCAL_WRITE
63        | rdmaxcel_sys::ibv_access_flags::IBV_ACCESS_REMOTE_WRITE
64        | rdmaxcel_sys::ibv_access_flags::IBV_ACCESS_REMOTE_READ
65}
monarch_rdma/efa.rs

monarch_rdma/
efa.rs
