hyperactor_mesh/
lib.rs

1/*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD-style license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9//! This crate provides hyperactor's mesh abstractions.
10
11#![feature(associated_type_defaults)]
12#![feature(impl_trait_in_bindings)]
13#![feature(get_disjoint_mut_helpers)]
14#![feature(exact_size_is_empty)]
15#![feature(async_fn_track_caller)]
16// EnumAsInner generates code that triggers a false positive
17// unused_assignments lint on struct variant fields. #[allow] on the
18// enum itself doesn't propagate into derive-macro-generated code, so
19// the suppression must be at module scope.
20#![allow(unused_assignments)]
21// `Error` is a public enum with established structured variants. Changed-file
22// lint reports its size through functions returning `crate::Result`.
23#![allow(clippy::result_large_err)]
24
25pub mod actor_mesh;
26mod assign;
27pub mod bootstrap;
28pub mod casting;
29pub mod client_root;
30pub mod config;
31pub mod config_dump;
32pub mod connect;
33pub mod global_context;
34pub mod host;
35pub mod host_mesh;
36pub mod introspect;
37pub mod logging;
38pub mod mesh;
39pub mod mesh_admin;
40pub mod mesh_admin_client;
41pub mod mesh_controller;
42pub mod mesh_id;
43pub mod mesh_selection;
44mod metrics;
45pub mod monitor;
46pub mod proc_agent;
47pub mod proc_launcher;
48pub mod proc_mesh;
49pub mod pyspy;
50pub mod reference;
51pub mod resource;
52pub mod shared_cell;
53pub mod shortuuid;
54pub mod supervision;
55#[cfg(target_os = "linux")]
56mod systemd;
57pub mod test_utils;
58pub mod testactor;
59pub mod testing;
60mod testresource;
61pub mod transport;
62pub mod value_mesh {
63    pub use hyperactor::value_mesh::*;
64}
65
66use std::io;
67
68pub use actor_mesh::ActorMesh;
69pub use actor_mesh::ActorMeshRef;
70pub use actor_mesh::DataActorMesh;
71pub use bootstrap::Bootstrap;
72pub use bootstrap::HostBootstrapReady;
73pub use bootstrap::bootstrap;
74pub use bootstrap::bootstrap_or_die;
75pub use casting::CastError;
76pub use dashmap;
77use enum_as_inner::EnumAsInner;
78pub use global_context::GlobalClientActor;
79pub use global_context::GlobalContext;
80pub use global_context::context;
81pub use global_context::this_host;
82pub use global_context::this_proc;
83pub use host_mesh::HostMeshRef;
84use hyperactor::ActorAddr;
85use hyperactor::ActorRef;
86use hyperactor::ProcAddr;
87use hyperactor::mailbox::MailboxSenderError;
88pub use hyperactor_mesh_macros::sel;
89pub use mesh::Mesh;
90pub use monitor::MeshMonitor;
91// Re-exported for internal test binaries that don't have ndslice as a direct dependency
92pub use ndslice::extent;
93use ndslice::view;
94pub use proc_mesh::ProcMesh;
95pub use proc_mesh::ProcMeshRef;
96pub use value_mesh::ValueMesh;
97
98use crate::host::HostError;
99use crate::host_mesh::HostAgent;
100use crate::host_mesh::HostMeshRefParseError;
101use crate::host_mesh::host_agent::ProcState;
102use crate::resource::RankedValues;
103use crate::resource::Status;
104
105/// A mesh of per-rank lifecycle statuses.
106///
107/// `StatusMesh` is `ValueMesh<Status>` and supports dense or
108/// compressed encodings. Updates are applied via sparse overlays with
109/// **last-writer-wins** semantics (see
110/// [`ValueMesh::merge_from_overlay`]). The mesh's `Region` defines
111/// the rank space; all updates must match that region.
112pub type StatusMesh = ValueMesh<Status>;
113
114/// A sparse set of `(Range<usize>, Status)` updates for a
115/// [`StatusMesh`].
116///
117/// `StatusOverlay` carries **normalized** runs (sorted,
118/// non-overlapping, and coalesced). Applying an overlay to a
119/// `StatusMesh` uses **right-wins** semantics on overlap and
120/// preserves first-appearance order in the compressed table.
121/// Construct via `ValueOverlay::try_from_runs` after normalizing.
122pub type StatusOverlay = value_mesh::ValueOverlay<Status>;
123
124inventory::submit! {
125    hyperactor::accum::ReducerFactory {
126        typehash_f: <hyperactor::value_mesh::ValueOverlayReducer<crate::resource::Status> as typeuri::Named>::typehash,
127        builder_f: |_| Ok(Box::new(hyperactor::value_mesh::ValueOverlayReducer::<crate::resource::Status>::new())),
128    }
129}
130
131// Reducer for per-proc `State<ProcState>` overlays, so `ProcMeshRef::states`
132// can reduce partial per-host meshes up the cast tree (see `GetHostProcStates`).
133inventory::submit! {
134    hyperactor::accum::ReducerFactory {
135        typehash_f: <hyperactor::value_mesh::ValueOverlayReducer<crate::resource::State<crate::host_mesh::host_agent::ProcState>> as typeuri::Named>::typehash,
136        builder_f: |_| Ok(Box::new(hyperactor::value_mesh::ValueOverlayReducer::<crate::resource::State<crate::host_mesh::host_agent::ProcState>>::new())),
137    }
138}
139
140/// Errors that occur during mesh operations.
141#[derive(Debug, EnumAsInner, thiserror::Error)]
142pub enum Error {
143    #[error("invalid mesh ref: expected {expected} ranks, but contains {actual} ranks")]
144    InvalidRankCardinality { expected: usize, actual: usize },
145
146    #[error(transparent)]
147    ResourceIdParseError(#[from] mesh_id::ResourceIdParseError),
148
149    #[error(transparent)]
150    HostMeshRefParseError(#[from] HostMeshRefParseError),
151
152    #[error(transparent)]
153    ChannelError(#[from] Box<hyperactor::channel::ChannelError>),
154
155    #[error(transparent)]
156    MailboxError(#[from] Box<hyperactor::mailbox::MailboxError>),
157
158    #[error(transparent)]
159    CodecError(#[from] CodecError),
160
161    #[error("error during mesh configuration: {0}")]
162    ConfigurationError(anyhow::Error),
163
164    // This is a temporary error to ensure we don't create unroutable
165    // meshes.
166    #[error("configuration error: mesh is unroutable")]
167    UnroutableMesh(),
168
169    #[error("error while calling actor {0}: {1}")]
170    CallError(ActorAddr, anyhow::Error),
171
172    #[error("actor not registered for type {0}")]
173    ActorTypeNotRegistered(String),
174
175    #[error(transparent)]
176    ClientRootError(#[from] crate::client_root::ClientRootError),
177
178    #[error("client-root capability is absent from this actor's environment")]
179    MissingClientRoot,
180
181    // TODO: this should be a valuemesh of statuses
182    #[error("error while spawning actor {0}: {1}")]
183    GspawnError(mesh_id::ActorMeshId, String),
184
185    #[error("error while sending message to actor {0}: {1}")]
186    SendingError(ActorAddr, Box<MailboxSenderError>),
187
188    #[error("error while casting message to {0}: {1}")]
189    CastingError(mesh_id::ActorMeshId, anyhow::Error),
190
191    #[error("error configuring host mesh agent {0}: {1}")]
192    HostMeshAgentConfigurationError(ActorAddr, String),
193
194    /// HM-2 / HM-3 / HM-4: structured per-host failure from
195    /// `HostMeshRef::push_config()`. See the HM-* invariant block in
196    /// `host_mesh.rs` for the contract this surfaces.
197    #[error(transparent)]
198    ConfigPushFailed(#[from] crate::host_mesh::ConfigPushError),
199
200    #[error(
201        "error creating proc (host rank {host_rank}) on host mesh agent {mesh_agent}, state: {state}"
202    )]
203    ProcCreationError {
204        state: Box<resource::State<ProcState>>,
205        host_rank: usize,
206        mesh_agent: ActorRef<HostAgent>,
207    },
208
209    #[error(
210        "error spawning proc mesh: statuses: {}",
211        RankedValues::invert(statuses)
212    )]
213    ProcSpawnError { statuses: RankedValues<Status> },
214
215    #[error(
216        "error spawning actor mesh: statuses: {}",
217        RankedValues::invert(statuses)
218    )]
219    ActorSpawnError { statuses: RankedValues<Status> },
220
221    #[error(
222        "error stopping actor mesh: statuses: {}",
223        RankedValues::invert(statuses)
224    )]
225    ActorStopError { statuses: RankedValues<Status> },
226
227    #[error(
228        "error stopping proc mesh: statuses: {}",
229        RankedValues::invert(statuses)
230    )]
231    ProcMeshStopError { statuses: RankedValues<Status> },
232
233    #[error("error spawning actor: {0}")]
234    SingletonActorSpawnError(anyhow::Error),
235
236    #[error("error spawning controller actor for mesh {0}: {1}")]
237    ControllerActorSpawnError(mesh_id::ResourceId, anyhow::Error),
238
239    #[error("proc {0} must be direct-addressable")]
240    RankedProc(ProcAddr),
241
242    #[error("error: {0} does not exist")]
243    NotExist(mesh_id::ResourceId),
244
245    #[error(transparent)]
246    Io(#[from] io::Error),
247
248    #[error(transparent)]
249    Host(#[from] HostError),
250
251    #[error(transparent)]
252    Other(#[from] anyhow::Error),
253}
254
255/// Errors that occur during serialization and deserialization.
256#[derive(Debug, thiserror::Error)]
257pub enum CodecError {
258    #[error(transparent)]
259    BincodeEncodeError(#[from] Box<bincode::error::EncodeError>),
260    #[error(transparent)]
261    BincodeDecodeError(#[from] Box<bincode::error::DecodeError>),
262    #[error(transparent)]
263    JsonError(#[from] Box<serde_json::Error>),
264    #[error(transparent)]
265    Base64Error(#[from] Box<base64::DecodeError>),
266    #[error(transparent)]
267    Utf8Error(#[from] Box<std::str::Utf8Error>),
268}
269
270impl From<bincode::error::EncodeError> for Error {
271    fn from(e: bincode::error::EncodeError) -> Self {
272        Error::CodecError(Box::new(e).into())
273    }
274}
275
276impl From<bincode::error::DecodeError> for Error {
277    fn from(e: bincode::error::DecodeError) -> Self {
278        Error::CodecError(Box::new(e).into())
279    }
280}
281
282impl From<serde_json::Error> for Error {
283    fn from(e: serde_json::Error) -> Self {
284        Error::CodecError(Box::new(e).into())
285    }
286}
287
288impl From<base64::DecodeError> for Error {
289    fn from(e: base64::DecodeError) -> Self {
290        Error::CodecError(Box::new(e).into())
291    }
292}
293
294impl From<std::str::Utf8Error> for Error {
295    fn from(e: std::str::Utf8Error) -> Self {
296        Error::CodecError(Box::new(e).into())
297    }
298}
299
300impl From<hyperactor::channel::ChannelError> for Error {
301    fn from(e: hyperactor::channel::ChannelError) -> Self {
302        Error::ChannelError(Box::new(e))
303    }
304}
305
306impl From<hyperactor::mailbox::MailboxError> for Error {
307    fn from(e: hyperactor::mailbox::MailboxError) -> Self {
308        Error::MailboxError(Box::new(e))
309    }
310}
311
312impl From<view::InvalidCardinality> for Error {
313    fn from(e: view::InvalidCardinality) -> Self {
314        Error::InvalidRankCardinality {
315            expected: e.expected,
316            actual: e.actual,
317        }
318    }
319}
320
321impl From<hyperactor::value_mesh::ValueMeshError> for Error {
322    fn from(e: hyperactor::value_mesh::ValueMeshError) -> Self {
323        match e {
324            hyperactor::value_mesh::ValueMeshError::InvalidRankCardinality { expected, actual } => {
325                Error::InvalidRankCardinality { expected, actual }
326            }
327        }
328    }
329}
330
331/// The type of result used in `hyperactor_mesh`.
332pub type Result<T> = std::result::Result<T, Error>;
333
334/// Construct a per-actor display name from a mesh-level base name and a
335/// rank's coordinates. Inserts `point.format_as_dict()` before the last
336/// `>` in `base`, or appends it if no `>` is found. Returns `base`
337/// unchanged for scalar (empty) points.
338pub(crate) fn actor_display_name(base: &str, point: &view::Point) -> String {
339    if point.is_empty() {
340        return base.to_string();
341    }
342    let coords = point.format_as_dict();
343    if let Some(pos) = base.rfind('>') {
344        format!("{}{}{}", &base[..pos], coords, &base[pos..])
345    } else {
346        format!("{}{}", base, coords)
347    }
348}
349
350#[cfg(test)]
351mod tests {
352
353    #[test]
354    fn basic() {
355        use ndslice::selection::dsl;
356        use ndslice::selection::structurally_equal;
357
358        let actual = sel!(*, 0:4, *);
359        let expected = dsl::all(dsl::range(
360            ndslice::shape::Range(0, Some(4), 1),
361            dsl::all(dsl::true_()),
362        ));
363        assert!(structurally_equal(&actual, &expected));
364    }
365
366    #[cfg(false)]
367    #[test]
368    fn shouldnt_compile() {
369        let _ = sel!(foobar);
370    }
371    // error: sel! parse failed: unexpected token: Ident { sym: foobar, span: #0 bytes(605..611) }
372    //   --> fbcode/monarch/hyperactor_mesh_macros/tests/basic.rs:19:13
373    //    |
374    // 19 |     let _ = sel!(foobar);
375    //    |             ^^^^^^^^^^^^ in this macro invocation
376    //   --> fbcode/monarch/hyperactor_mesh_macros/src/lib.rs:12:1
377    //    |
378    //    = note: in this expansion of `sel!`
379
380    use hyperactor_mesh_macros::sel;
381    use ndslice::assert_round_trip;
382    use ndslice::assert_structurally_eq;
383    use ndslice::selection::Selection;
384
385    macro_rules! assert_round_trip_match {
386        ($left:expr, $right:expr) => {{
387            assert_structurally_eq!($left, $right);
388            assert_round_trip!($left);
389            assert_round_trip!($right);
390        }};
391    }
392
393    #[test]
394    fn token_parser() {
395        use ndslice::selection::dsl::*;
396        use ndslice::shape;
397
398        assert_round_trip_match!(all(true_()), sel!(*));
399        assert_round_trip_match!(range(3, true_()), sel!(3));
400        assert_round_trip_match!(range(1..4, true_()), sel!(1:4));
401        assert_round_trip_match!(all(range(1..4, true_())), sel!(*, 1:4));
402        assert_round_trip_match!(range(shape::Range(0, None, 1), true_()), sel!(:));
403        assert_round_trip_match!(any(true_()), sel!(?));
404        assert_round_trip_match!(any(range(1..4, all(true_()))), sel!(?, 1:4, *));
405        assert_round_trip_match!(union(range(0, true_()), range(1, true_())), sel!(0 | 1));
406        assert_round_trip_match!(
407            intersection(range(0..4, true_()), range(2..6, true_())),
408            sel!(0:4 & 2:6)
409        );
410        assert_round_trip_match!(range(shape::Range(0, None, 1), true_()), sel!(:));
411        assert_round_trip_match!(all(true_()), sel!(*));
412        assert_round_trip_match!(any(true_()), sel!(?));
413        assert_round_trip_match!(all(all(all(true_()))), sel!(*, *, *));
414        assert_round_trip_match!(intersection(all(true_()), all(true_())), sel!(* & *));
415        assert_round_trip_match!(
416            all(all(union(
417                range(0..2, true_()),
418                range(shape::Range(6, None, 1), true_())
419            ))),
420            sel!(*, *, (:2|6:))
421        );
422        assert_round_trip_match!(
423            all(all(range(shape::Range(1, None, 2), true_()))),
424            sel!(*, *, 1::2)
425        );
426        assert_round_trip_match!(
427            range(
428                shape::Range(0, Some(1), 1),
429                any(range(shape::Range(0, Some(4), 1), true_()))
430            ),
431            sel!(0, ?, :4)
432        );
433        assert_round_trip_match!(range(shape::Range(1, Some(4), 2), true_()), sel!(1:4:2));
434        assert_round_trip_match!(range(shape::Range(0, None, 2), true_()), sel!(::2));
435        assert_round_trip_match!(
436            union(range(0..4, true_()), range(4..8, true_())),
437            sel!(0:4 | 4:8)
438        );
439        assert_round_trip_match!(
440            intersection(range(0..4, true_()), range(2..6, true_())),
441            sel!(0:4 & 2:6)
442        );
443        assert_round_trip_match!(
444            all(union(range(1..4, all(true_())), range(5..6, all(true_())))),
445            sel!(*, (1:4 | 5:6), *)
446        );
447        assert_round_trip_match!(
448            range(
449                0,
450                intersection(
451                    range(1..4, range(7, true_())),
452                    range(2..5, range(7, true_()))
453                )
454            ),
455            sel!(0, (1:4 & 2:5), 7)
456        );
457        assert_round_trip_match!(
458            all(all(union(
459                union(range(0..2, true_()), range(4..6, true_())),
460                range(shape::Range(6, None, 1), true_())
461            ))),
462            sel!(*, *, (:2 | 4:6 | 6:))
463        );
464        assert_round_trip_match!(intersection(all(true_()), all(true_())), sel!(* & *));
465        assert_round_trip_match!(union(all(true_()), all(true_())), sel!(* | *));
466        assert_round_trip_match!(
467            intersection(
468                range(0..2, true_()),
469                union(range(1, true_()), range(2, true_()))
470            ),
471            sel!(0:2 & (1 | 2))
472        );
473        assert_round_trip_match!(
474            all(all(intersection(
475                range(1..2, true_()),
476                range(2..3, true_())
477            ))),
478            sel!(*,*,(1:2&2:3))
479        );
480        assert_round_trip_match!(
481            intersection(all(all(all(true_()))), all(all(all(true_())))),
482            sel!((*,*,*) & (*,*,*))
483        );
484        assert_round_trip_match!(
485            intersection(
486                range(0, all(all(true_()))),
487                range(0, union(range(1, all(true_())), range(3, all(true_()))))
488            ),
489            sel!((0, *, *) & (0, (1 | 3), *))
490        );
491        assert_round_trip_match!(
492            intersection(
493                range(0, all(all(true_()))),
494                range(
495                    0,
496                    union(
497                        range(1, range(2..5, true_())),
498                        range(3, range(2..5, true_()))
499                    )
500                )
501            ),
502            sel!((0, *, *) & (0, (1 | 3), 2:5))
503        );
504        assert_round_trip_match!(all(true_()), sel!((*)));
505        assert_round_trip_match!(range(1..4, range(2, true_())), sel!(((1:4), 2)));
506        assert_round_trip_match!(sel!(1:4 & 5:6 | 7:8), sel!((1:4 & 5:6) | 7:8));
507        assert_round_trip_match!(
508            union(
509                intersection(all(all(true_())), all(all(true_()))),
510                all(all(true_()))
511            ),
512            sel!((*,*) & (*,*) | (*,*))
513        );
514        assert_round_trip_match!(all(true_()), sel!(*));
515        assert_round_trip_match!(sel!(((1:4))), sel!(1:4));
516        assert_round_trip_match!(sel!(*, (*)), sel!(*, *));
517        assert_round_trip_match!(
518            intersection(
519                range(0, range(1..4, true_())),
520                range(0, union(range(2, all(true_())), range(3, all(true_()))))
521            ),
522            sel!((0,1:4)&(0,(2|3),*))
523        );
524
525        //assert_round_trip_match!(true_(), sel!(foo)); // sel! macro: parse error: Parsing Error: Error { input: "foo", code: Tag }
526
527        assert_round_trip_match!(
528            sel!(0 & (0, (1|3), *)),
529            intersection(
530                range(0, true_()),
531                range(0, union(range(1, all(true_())), range(3, all(true_()))))
532            )
533        );
534        assert_round_trip_match!(
535            sel!(0 & (0, (3|1), *)),
536            intersection(
537                range(0, true_()),
538                range(0, union(range(3, all(true_())), range(1, all(true_()))))
539            )
540        );
541        assert_round_trip_match!(
542            sel!((*, *, *) & (*, *, (2 | 4))),
543            intersection(
544                all(all(all(true_()))),
545                all(all(union(range(2, true_()), range(4, true_()))))
546            )
547        );
548        assert_round_trip_match!(
549            sel!((*, *, *) & (*, *, (4 | 2))),
550            intersection(
551                all(all(all(true_()))),
552                all(all(union(range(4, true_()), range(2, true_()))))
553            )
554        );
555        assert_round_trip_match!(
556            sel!((*, (1|2)) & (*, (2|1))),
557            intersection(
558                all(union(range(1, true_()), range(2, true_()))),
559                all(union(range(2, true_()), range(1, true_())))
560            )
561        );
562        assert_round_trip_match!(
563            sel!((*, *, *) & *),
564            intersection(all(all(all(true_()))), all(true_()))
565        );
566        assert_round_trip_match!(
567            sel!(* & (*, *, *)),
568            intersection(all(true_()), all(all(all(true_()))))
569        );
570
571        assert_round_trip_match!(
572            sel!( (*, *, *) & ((*, *, *) & (*, *, *)) ),
573            intersection(
574                all(all(all(true_()))),
575                intersection(all(all(all(true_()))), all(all(all(true_()))))
576            )
577        );
578        assert_round_trip_match!(
579            sel!((1, *, *) | (0 & (0, 3, *))),
580            union(
581                range(1, all(all(true_()))),
582                intersection(range(0, true_()), range(0, range(3, all(true_()))))
583            )
584        );
585        assert_round_trip_match!(
586            sel!(((0, *)| (1, *)) & ((1, *) | (0, *))),
587            intersection(
588                union(range(0, all(true_())), range(1, all(true_()))),
589                union(range(1, all(true_())), range(0, all(true_())))
590            )
591        );
592        assert_round_trip_match!(sel!(*, 8:8), all(range(8..8, true_())));
593        assert_round_trip_match!(
594            sel!((*, 1) & (*, 8 : 8)),
595            intersection(all(range(1..2, true_())), all(range(8..8, true_())))
596        );
597        assert_round_trip_match!(
598            sel!((*, 8 : 8) | (*, 1)),
599            union(all(range(8..8, true_())), all(range(1..2, true_())))
600        );
601        assert_round_trip_match!(
602            sel!((*, 1) | (*, 2:8)),
603            union(all(range(1..2, true_())), all(range(2..8, true_())))
604        );
605        assert_round_trip_match!(
606            sel!((*, *, *) & (*, *, 2:8)),
607            intersection(all(all(all(true_()))), all(all(range(2..8, true_()))))
608        );
609    }
610}
hyperactor_mesh/lib.rs

hyperactor_mesh/
lib.rs
