hyperactor_mesh/
lib.rs

1/*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD-style license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9//! This crate provides hyperactor's mesh abstractions.
10
11#![feature(associated_type_defaults)]
12#![feature(impl_trait_in_bindings)]
13#![feature(get_disjoint_mut_helpers)]
14#![feature(exact_size_is_empty)]
15#![feature(async_fn_track_caller)]
16// EnumAsInner generates code that triggers a false positive
17// unused_assignments lint on struct variant fields. #[allow] on the
18// enum itself doesn't propagate into derive-macro-generated code, so
19// the suppression must be at module scope.
20#![allow(unused_assignments)]
21
22pub mod actor_mesh;
23mod assign;
24pub mod bootstrap;
25pub mod casting;
26pub mod comm;
27pub mod config;
28pub mod config_dump;
29pub mod connect;
30pub mod global_context;
31pub mod host;
32pub mod host_mesh;
33pub mod introspect;
34pub mod logging;
35pub mod mesh;
36pub mod mesh_admin;
37pub mod mesh_admin_client;
38pub mod mesh_controller;
39pub mod mesh_id;
40pub mod mesh_selection;
41mod metrics;
42pub mod proc_agent;
43pub mod proc_launcher;
44pub mod proc_mesh;
45pub mod pyspy;
46pub mod reference;
47pub mod resource;
48pub mod shared_cell;
49pub mod shortuuid;
50pub mod supervision;
51#[cfg(target_os = "linux")]
52mod systemd;
53pub mod test_utils;
54pub mod testactor;
55pub mod testing;
56mod testresource;
57pub mod transport;
58pub mod value_mesh {
59    pub use hyperactor::value_mesh::*;
60}
61
62use std::io;
63
64pub use actor_mesh::ActorMesh;
65pub use actor_mesh::ActorMeshRef;
66pub use bootstrap::Bootstrap;
67pub use bootstrap::bootstrap;
68pub use bootstrap::bootstrap_or_die;
69pub use casting::CastError;
70pub use comm::CommActor;
71pub use dashmap;
72use enum_as_inner::EnumAsInner;
73pub use global_context::GlobalClientActor;
74pub use global_context::GlobalContext;
75pub use global_context::context;
76pub use global_context::this_host;
77pub use global_context::this_proc;
78pub use host_mesh::HostMeshRef;
79use hyperactor::ActorAddr;
80use hyperactor::ActorRef;
81use hyperactor::ProcAddr;
82use hyperactor::mailbox::MailboxSenderError;
83pub use hyperactor_mesh_macros::sel;
84pub use mesh::Mesh;
85// Re-exported for internal test binaries that don't have ndslice as a direct dependency
86pub use ndslice::extent;
87use ndslice::view;
88pub use proc_mesh::ProcMesh;
89pub use proc_mesh::ProcMeshRef;
90pub use value_mesh::ValueMesh;
91
92use crate::host::HostError;
93use crate::host_mesh::HostAgent;
94use crate::host_mesh::HostMeshRefParseError;
95use crate::host_mesh::host_agent::ProcState;
96use crate::resource::RankedValues;
97use crate::resource::Status;
98use crate::supervision::MeshFailure;
99
100/// A mesh of per-rank lifecycle statuses.
101///
102/// `StatusMesh` is `ValueMesh<Status>` and supports dense or
103/// compressed encodings. Updates are applied via sparse overlays with
104/// **last-writer-wins** semantics (see
105/// [`ValueMesh::merge_from_overlay`]). The mesh's `Region` defines
106/// the rank space; all updates must match that region.
107pub type StatusMesh = ValueMesh<Status>;
108
109/// A sparse set of `(Range<usize>, Status)` updates for a
110/// [`StatusMesh`].
111///
112/// `StatusOverlay` carries **normalized** runs (sorted,
113/// non-overlapping, and coalesced). Applying an overlay to a
114/// `StatusMesh` uses **right-wins** semantics on overlap and
115/// preserves first-appearance order in the compressed table.
116/// Construct via `ValueOverlay::try_from_runs` after normalizing.
117pub type StatusOverlay = value_mesh::ValueOverlay<Status>;
118
119inventory::submit! {
120    hyperactor::accum::ReducerFactory {
121        typehash_f: <hyperactor::value_mesh::ValueOverlayReducer<crate::resource::Status> as typeuri::Named>::typehash,
122        builder_f: |_| Ok(Box::new(hyperactor::value_mesh::ValueOverlayReducer::<crate::resource::Status>::new())),
123    }
124}
125
126// Reducer for per-proc `State<ProcState>` overlays, so `ProcMeshRef::states`
127// can reduce partial per-host meshes up the cast tree (see `GetHostProcStates`).
128inventory::submit! {
129    hyperactor::accum::ReducerFactory {
130        typehash_f: <hyperactor::value_mesh::ValueOverlayReducer<crate::resource::State<crate::host_mesh::host_agent::ProcState>> as typeuri::Named>::typehash,
131        builder_f: |_| Ok(Box::new(hyperactor::value_mesh::ValueOverlayReducer::<crate::resource::State<crate::host_mesh::host_agent::ProcState>>::new())),
132    }
133}
134
135/// Errors that occur during mesh operations.
136#[derive(Debug, EnumAsInner, thiserror::Error)]
137pub enum Error {
138    #[error("invalid mesh ref: expected {expected} ranks, but contains {actual} ranks")]
139    InvalidRankCardinality { expected: usize, actual: usize },
140
141    #[error(transparent)]
142    ResourceIdParseError(#[from] mesh_id::ResourceIdParseError),
143
144    #[error(transparent)]
145    HostMeshRefParseError(#[from] HostMeshRefParseError),
146
147    #[error(transparent)]
148    ChannelError(#[from] Box<hyperactor::channel::ChannelError>),
149
150    #[error(transparent)]
151    MailboxError(#[from] Box<hyperactor::mailbox::MailboxError>),
152
153    #[error(transparent)]
154    CodecError(#[from] CodecError),
155
156    #[error("error during mesh configuration: {0}")]
157    ConfigurationError(anyhow::Error),
158
159    // This is a temporary error to ensure we don't create unroutable
160    // meshes.
161    #[error("configuration error: mesh is unroutable")]
162    UnroutableMesh(),
163
164    #[error("error while calling actor {0}: {1}")]
165    CallError(ActorAddr, anyhow::Error),
166
167    #[error("actor not registered for type {0}")]
168    ActorTypeNotRegistered(String),
169
170    // TODO: this should be a valuemesh of statuses
171    #[error("error while spawning actor {0}: {1}")]
172    GspawnError(mesh_id::ActorMeshId, String),
173
174    #[error("error while sending message to actor {0}: {1}")]
175    SendingError(ActorAddr, Box<MailboxSenderError>),
176
177    #[error("error while casting message to {0}: {1}")]
178    CastingError(mesh_id::ActorMeshId, anyhow::Error),
179
180    #[error("error configuring host mesh agent {0}: {1}")]
181    HostMeshAgentConfigurationError(ActorAddr, String),
182
183    /// HM-2 / HM-3 / HM-4: structured per-host failure from
184    /// `HostMeshRef::push_config()`. See the HM-* invariant block in
185    /// `host_mesh.rs` for the contract this surfaces.
186    #[error(transparent)]
187    ConfigPushFailed(#[from] crate::host_mesh::ConfigPushError),
188
189    #[error(
190        "error creating proc (host rank {host_rank}) on host mesh agent {mesh_agent}, state: {state}"
191    )]
192    ProcCreationError {
193        state: Box<resource::State<ProcState>>,
194        host_rank: usize,
195        mesh_agent: ActorRef<HostAgent>,
196    },
197
198    #[error(
199        "error spawning proc mesh: statuses: {}",
200        RankedValues::invert(statuses)
201    )]
202    ProcSpawnError { statuses: RankedValues<Status> },
203
204    #[error(
205        "error spawning actor mesh: statuses: {}",
206        RankedValues::invert(statuses)
207    )]
208    ActorSpawnError { statuses: RankedValues<Status> },
209
210    #[error(
211        "error stopping actor mesh: statuses: {}",
212        RankedValues::invert(statuses)
213    )]
214    ActorStopError { statuses: RankedValues<Status> },
215
216    #[error(
217        "error stopping proc mesh: statuses: {}",
218        RankedValues::invert(statuses)
219    )]
220    ProcMeshStopError { statuses: RankedValues<Status> },
221
222    #[error("error spawning actor: {0}")]
223    SingletonActorSpawnError(anyhow::Error),
224
225    #[error("error spawning controller actor for mesh {0}: {1}")]
226    ControllerActorSpawnError(mesh_id::ResourceId, anyhow::Error),
227
228    #[error("proc {0} must be direct-addressable")]
229    RankedProc(ProcAddr),
230
231    #[error("{0}")]
232    Supervision(Box<MeshFailure>),
233
234    #[error("error: {0} does not exist")]
235    NotExist(mesh_id::ResourceId),
236
237    #[error(transparent)]
238    Io(#[from] io::Error),
239
240    #[error(transparent)]
241    Host(#[from] HostError),
242
243    #[error(transparent)]
244    Other(#[from] anyhow::Error),
245}
246
247/// Errors that occur during serialization and deserialization.
248#[derive(Debug, thiserror::Error)]
249pub enum CodecError {
250    #[error(transparent)]
251    BincodeEncodeError(#[from] Box<bincode::error::EncodeError>),
252    #[error(transparent)]
253    BincodeDecodeError(#[from] Box<bincode::error::DecodeError>),
254    #[error(transparent)]
255    JsonError(#[from] Box<serde_json::Error>),
256    #[error(transparent)]
257    Base64Error(#[from] Box<base64::DecodeError>),
258    #[error(transparent)]
259    Utf8Error(#[from] Box<std::str::Utf8Error>),
260}
261
262impl From<bincode::error::EncodeError> for Error {
263    fn from(e: bincode::error::EncodeError) -> Self {
264        Error::CodecError(Box::new(e).into())
265    }
266}
267
268impl From<bincode::error::DecodeError> for Error {
269    fn from(e: bincode::error::DecodeError) -> Self {
270        Error::CodecError(Box::new(e).into())
271    }
272}
273
274impl From<serde_json::Error> for Error {
275    fn from(e: serde_json::Error) -> Self {
276        Error::CodecError(Box::new(e).into())
277    }
278}
279
280impl From<base64::DecodeError> for Error {
281    fn from(e: base64::DecodeError) -> Self {
282        Error::CodecError(Box::new(e).into())
283    }
284}
285
286impl From<std::str::Utf8Error> for Error {
287    fn from(e: std::str::Utf8Error) -> Self {
288        Error::CodecError(Box::new(e).into())
289    }
290}
291
292impl From<hyperactor::channel::ChannelError> for Error {
293    fn from(e: hyperactor::channel::ChannelError) -> Self {
294        Error::ChannelError(Box::new(e))
295    }
296}
297
298impl From<hyperactor::mailbox::MailboxError> for Error {
299    fn from(e: hyperactor::mailbox::MailboxError) -> Self {
300        Error::MailboxError(Box::new(e))
301    }
302}
303
304impl From<view::InvalidCardinality> for Error {
305    fn from(e: view::InvalidCardinality) -> Self {
306        Error::InvalidRankCardinality {
307            expected: e.expected,
308            actual: e.actual,
309        }
310    }
311}
312
313impl From<hyperactor::value_mesh::ValueMeshError> for Error {
314    fn from(e: hyperactor::value_mesh::ValueMeshError) -> Self {
315        match e {
316            hyperactor::value_mesh::ValueMeshError::InvalidRankCardinality { expected, actual } => {
317                Error::InvalidRankCardinality { expected, actual }
318            }
319        }
320    }
321}
322
323/// The type of result used in `hyperactor_mesh`.
324pub type Result<T> = std::result::Result<T, Error>;
325
326/// Construct a per-actor display name from a mesh-level base name and a
327/// rank's coordinates. Inserts `point.format_as_dict()` before the last
328/// `>` in `base`, or appends it if no `>` is found. Returns `base`
329/// unchanged for scalar (empty) points.
330pub(crate) fn actor_display_name(base: &str, point: &view::Point) -> String {
331    if point.is_empty() {
332        return base.to_string();
333    }
334    let coords = point.format_as_dict();
335    if let Some(pos) = base.rfind('>') {
336        format!("{}{}{}", &base[..pos], coords, &base[pos..])
337    } else {
338        format!("{}{}", base, coords)
339    }
340}
341
342#[cfg(test)]
343mod tests {
344
345    #[test]
346    fn basic() {
347        use ndslice::selection::dsl;
348        use ndslice::selection::structurally_equal;
349
350        let actual = sel!(*, 0:4, *);
351        let expected = dsl::all(dsl::range(
352            ndslice::shape::Range(0, Some(4), 1),
353            dsl::all(dsl::true_()),
354        ));
355        assert!(structurally_equal(&actual, &expected));
356    }
357
358    #[cfg(false)]
359    #[test]
360    fn shouldnt_compile() {
361        let _ = sel!(foobar);
362    }
363    // error: sel! parse failed: unexpected token: Ident { sym: foobar, span: #0 bytes(605..611) }
364    //   --> fbcode/monarch/hyperactor_mesh_macros/tests/basic.rs:19:13
365    //    |
366    // 19 |     let _ = sel!(foobar);
367    //    |             ^^^^^^^^^^^^ in this macro invocation
368    //   --> fbcode/monarch/hyperactor_mesh_macros/src/lib.rs:12:1
369    //    |
370    //    = note: in this expansion of `sel!`
371
372    use hyperactor_mesh_macros::sel;
373    use ndslice::assert_round_trip;
374    use ndslice::assert_structurally_eq;
375    use ndslice::selection::Selection;
376
377    macro_rules! assert_round_trip_match {
378        ($left:expr, $right:expr) => {{
379            assert_structurally_eq!($left, $right);
380            assert_round_trip!($left);
381            assert_round_trip!($right);
382        }};
383    }
384
385    #[test]
386    fn token_parser() {
387        use ndslice::selection::dsl::*;
388        use ndslice::shape;
389
390        assert_round_trip_match!(all(true_()), sel!(*));
391        assert_round_trip_match!(range(3, true_()), sel!(3));
392        assert_round_trip_match!(range(1..4, true_()), sel!(1:4));
393        assert_round_trip_match!(all(range(1..4, true_())), sel!(*, 1:4));
394        assert_round_trip_match!(range(shape::Range(0, None, 1), true_()), sel!(:));
395        assert_round_trip_match!(any(true_()), sel!(?));
396        assert_round_trip_match!(any(range(1..4, all(true_()))), sel!(?, 1:4, *));
397        assert_round_trip_match!(union(range(0, true_()), range(1, true_())), sel!(0 | 1));
398        assert_round_trip_match!(
399            intersection(range(0..4, true_()), range(2..6, true_())),
400            sel!(0:4 & 2:6)
401        );
402        assert_round_trip_match!(range(shape::Range(0, None, 1), true_()), sel!(:));
403        assert_round_trip_match!(all(true_()), sel!(*));
404        assert_round_trip_match!(any(true_()), sel!(?));
405        assert_round_trip_match!(all(all(all(true_()))), sel!(*, *, *));
406        assert_round_trip_match!(intersection(all(true_()), all(true_())), sel!(* & *));
407        assert_round_trip_match!(
408            all(all(union(
409                range(0..2, true_()),
410                range(shape::Range(6, None, 1), true_())
411            ))),
412            sel!(*, *, (:2|6:))
413        );
414        assert_round_trip_match!(
415            all(all(range(shape::Range(1, None, 2), true_()))),
416            sel!(*, *, 1::2)
417        );
418        assert_round_trip_match!(
419            range(
420                shape::Range(0, Some(1), 1),
421                any(range(shape::Range(0, Some(4), 1), true_()))
422            ),
423            sel!(0, ?, :4)
424        );
425        assert_round_trip_match!(range(shape::Range(1, Some(4), 2), true_()), sel!(1:4:2));
426        assert_round_trip_match!(range(shape::Range(0, None, 2), true_()), sel!(::2));
427        assert_round_trip_match!(
428            union(range(0..4, true_()), range(4..8, true_())),
429            sel!(0:4 | 4:8)
430        );
431        assert_round_trip_match!(
432            intersection(range(0..4, true_()), range(2..6, true_())),
433            sel!(0:4 & 2:6)
434        );
435        assert_round_trip_match!(
436            all(union(range(1..4, all(true_())), range(5..6, all(true_())))),
437            sel!(*, (1:4 | 5:6), *)
438        );
439        assert_round_trip_match!(
440            range(
441                0,
442                intersection(
443                    range(1..4, range(7, true_())),
444                    range(2..5, range(7, true_()))
445                )
446            ),
447            sel!(0, (1:4 & 2:5), 7)
448        );
449        assert_round_trip_match!(
450            all(all(union(
451                union(range(0..2, true_()), range(4..6, true_())),
452                range(shape::Range(6, None, 1), true_())
453            ))),
454            sel!(*, *, (:2 | 4:6 | 6:))
455        );
456        assert_round_trip_match!(intersection(all(true_()), all(true_())), sel!(* & *));
457        assert_round_trip_match!(union(all(true_()), all(true_())), sel!(* | *));
458        assert_round_trip_match!(
459            intersection(
460                range(0..2, true_()),
461                union(range(1, true_()), range(2, true_()))
462            ),
463            sel!(0:2 & (1 | 2))
464        );
465        assert_round_trip_match!(
466            all(all(intersection(
467                range(1..2, true_()),
468                range(2..3, true_())
469            ))),
470            sel!(*,*,(1:2&2:3))
471        );
472        assert_round_trip_match!(
473            intersection(all(all(all(true_()))), all(all(all(true_())))),
474            sel!((*,*,*) & (*,*,*))
475        );
476        assert_round_trip_match!(
477            intersection(
478                range(0, all(all(true_()))),
479                range(0, union(range(1, all(true_())), range(3, all(true_()))))
480            ),
481            sel!((0, *, *) & (0, (1 | 3), *))
482        );
483        assert_round_trip_match!(
484            intersection(
485                range(0, all(all(true_()))),
486                range(
487                    0,
488                    union(
489                        range(1, range(2..5, true_())),
490                        range(3, range(2..5, true_()))
491                    )
492                )
493            ),
494            sel!((0, *, *) & (0, (1 | 3), 2:5))
495        );
496        assert_round_trip_match!(all(true_()), sel!((*)));
497        assert_round_trip_match!(range(1..4, range(2, true_())), sel!(((1:4), 2)));
498        assert_round_trip_match!(sel!(1:4 & 5:6 | 7:8), sel!((1:4 & 5:6) | 7:8));
499        assert_round_trip_match!(
500            union(
501                intersection(all(all(true_())), all(all(true_()))),
502                all(all(true_()))
503            ),
504            sel!((*,*) & (*,*) | (*,*))
505        );
506        assert_round_trip_match!(all(true_()), sel!(*));
507        assert_round_trip_match!(sel!(((1:4))), sel!(1:4));
508        assert_round_trip_match!(sel!(*, (*)), sel!(*, *));
509        assert_round_trip_match!(
510            intersection(
511                range(0, range(1..4, true_())),
512                range(0, union(range(2, all(true_())), range(3, all(true_()))))
513            ),
514            sel!((0,1:4)&(0,(2|3),*))
515        );
516
517        //assert_round_trip_match!(true_(), sel!(foo)); // sel! macro: parse error: Parsing Error: Error { input: "foo", code: Tag }
518
519        assert_round_trip_match!(
520            sel!(0 & (0, (1|3), *)),
521            intersection(
522                range(0, true_()),
523                range(0, union(range(1, all(true_())), range(3, all(true_()))))
524            )
525        );
526        assert_round_trip_match!(
527            sel!(0 & (0, (3|1), *)),
528            intersection(
529                range(0, true_()),
530                range(0, union(range(3, all(true_())), range(1, all(true_()))))
531            )
532        );
533        assert_round_trip_match!(
534            sel!((*, *, *) & (*, *, (2 | 4))),
535            intersection(
536                all(all(all(true_()))),
537                all(all(union(range(2, true_()), range(4, true_()))))
538            )
539        );
540        assert_round_trip_match!(
541            sel!((*, *, *) & (*, *, (4 | 2))),
542            intersection(
543                all(all(all(true_()))),
544                all(all(union(range(4, true_()), range(2, true_()))))
545            )
546        );
547        assert_round_trip_match!(
548            sel!((*, (1|2)) & (*, (2|1))),
549            intersection(
550                all(union(range(1, true_()), range(2, true_()))),
551                all(union(range(2, true_()), range(1, true_())))
552            )
553        );
554        assert_round_trip_match!(
555            sel!((*, *, *) & *),
556            intersection(all(all(all(true_()))), all(true_()))
557        );
558        assert_round_trip_match!(
559            sel!(* & (*, *, *)),
560            intersection(all(true_()), all(all(all(true_()))))
561        );
562
563        assert_round_trip_match!(
564            sel!( (*, *, *) & ((*, *, *) & (*, *, *)) ),
565            intersection(
566                all(all(all(true_()))),
567                intersection(all(all(all(true_()))), all(all(all(true_()))))
568            )
569        );
570        assert_round_trip_match!(
571            sel!((1, *, *) | (0 & (0, 3, *))),
572            union(
573                range(1, all(all(true_()))),
574                intersection(range(0, true_()), range(0, range(3, all(true_()))))
575            )
576        );
577        assert_round_trip_match!(
578            sel!(((0, *)| (1, *)) & ((1, *) | (0, *))),
579            intersection(
580                union(range(0, all(true_())), range(1, all(true_()))),
581                union(range(1, all(true_())), range(0, all(true_())))
582            )
583        );
584        assert_round_trip_match!(sel!(*, 8:8), all(range(8..8, true_())));
585        assert_round_trip_match!(
586            sel!((*, 1) & (*, 8 : 8)),
587            intersection(all(range(1..2, true_())), all(range(8..8, true_())))
588        );
589        assert_round_trip_match!(
590            sel!((*, 8 : 8) | (*, 1)),
591            union(all(range(8..8, true_())), all(range(1..2, true_())))
592        );
593        assert_round_trip_match!(
594            sel!((*, 1) | (*, 2:8)),
595            union(all(range(1..2, true_())), all(range(2..8, true_())))
596        );
597        assert_round_trip_match!(
598            sel!((*, *, *) & (*, *, 2:8)),
599            intersection(all(all(all(true_()))), all(all(range(2..8, true_()))))
600        );
601    }
602}
hyperactor_mesh/lib.rs

hyperactor_mesh/
lib.rs
