1#![feature(assert_matches)]
12#![feature(exit_status_error)]
13#![feature(impl_trait_in_bindings)]
14#![feature(get_disjoint_mut_helpers)]
15#![feature(exact_size_is_empty)]
16#![feature(async_fn_track_caller)]
17#![allow(unused_assignments)]
22
23pub mod actor_mesh;
24pub mod alloc;
25mod assign;
26pub mod bootstrap;
27pub mod casting;
28pub mod comm;
29pub mod config;
30pub mod config_dump;
31pub mod connect;
32pub mod global_context;
33pub mod host_mesh;
34pub mod introspect;
35pub mod logging;
36pub mod mesh;
37pub mod mesh_admin;
38pub mod mesh_admin_client;
39pub mod mesh_controller;
40pub mod mesh_selection;
41mod metrics;
42pub mod namespace;
43pub mod proc_agent;
44pub mod proc_launcher;
45pub mod proc_mesh;
46pub mod pyspy;
47pub mod reference;
48pub mod resource;
49pub mod shared_cell;
50pub mod shortuuid;
51pub mod supervision;
52#[cfg(target_os = "linux")]
53mod systemd;
54pub mod test_utils;
55pub mod testactor;
56pub mod testing;
57mod testresource;
58pub mod transport;
59pub mod value_mesh;
60
61use std::io;
62use std::str::FromStr;
63
64pub use actor_mesh::ActorMesh;
65pub use actor_mesh::ActorMeshRef;
66pub use bootstrap::Bootstrap;
67pub use bootstrap::bootstrap;
68pub use bootstrap::bootstrap_or_die;
69pub use casting::CastError;
70pub use comm::CommActor;
71pub use dashmap;
72use enum_as_inner::EnumAsInner;
73pub use global_context::GlobalClientActor;
74pub use global_context::GlobalContext;
75pub use global_context::context;
76pub use global_context::this_host;
77pub use global_context::this_proc;
78pub use host_mesh::HostMeshRef;
79use hyperactor::host::HostError;
80use hyperactor::mailbox::MailboxSenderError;
81use hyperactor::reference as hyperactor_reference;
82pub use hyperactor_mesh_macros::sel;
83pub use mesh::Mesh;
84pub use ndslice::extent;
86use ndslice::view;
87pub use proc_mesh::ProcMesh;
88pub use proc_mesh::ProcMeshRef;
89use serde::Deserialize;
90use serde::Serialize;
91use typeuri::Named;
92pub use value_mesh::ValueMesh;
93
94use crate::host_mesh::HostAgent;
95use crate::host_mesh::HostMeshRefParseError;
96use crate::host_mesh::host_agent::ProcState;
97use crate::resource::RankedValues;
98use crate::resource::Status;
99use crate::shortuuid::ShortUuid;
100use crate::supervision::MeshFailure;
101
102pub type StatusMesh = ValueMesh<Status>;
110
111pub type StatusOverlay = value_mesh::ValueOverlay<Status>;
120
121#[derive(Debug, EnumAsInner, thiserror::Error)]
123pub enum Error {
124 #[error("invalid mesh ref: expected {expected} ranks, but contains {actual} ranks")]
125 InvalidRankCardinality { expected: usize, actual: usize },
126
127 #[error(transparent)]
128 NameParseError(#[from] NameParseError),
129
130 #[error(transparent)]
131 HostMeshRefParseError(#[from] HostMeshRefParseError),
132
133 #[error(transparent)]
134 AllocatorError(#[from] Box<crate::alloc::AllocatorError>),
135
136 #[error(transparent)]
137 ChannelError(#[from] Box<hyperactor::channel::ChannelError>),
138
139 #[error(transparent)]
140 MailboxError(#[from] Box<hyperactor::mailbox::MailboxError>),
141
142 #[error(transparent)]
143 CodecError(#[from] CodecError),
144
145 #[error("error during mesh configuration: {0}")]
146 ConfigurationError(anyhow::Error),
147
148 #[error("configuration error: mesh is unroutable")]
151 UnroutableMesh(),
152
153 #[error("error while calling actor {0}: {1}")]
154 CallError(hyperactor_reference::ActorId, anyhow::Error),
155
156 #[error("actor not registered for type {0}")]
157 ActorTypeNotRegistered(String),
158
159 #[error("error while spawning actor {0}: {1}")]
161 GspawnError(Name, String),
162
163 #[error("error while sending message to actor {0}: {1}")]
164 SendingError(hyperactor_reference::ActorId, Box<MailboxSenderError>),
165
166 #[error("error while casting message to {0}: {1}")]
167 CastingError(Name, anyhow::Error),
168
169 #[error("error configuring host mesh agent {0}: {1}")]
170 HostMeshAgentConfigurationError(hyperactor_reference::ActorId, String),
171
172 #[error(
173 "error creating proc (host rank {host_rank}) on host mesh agent {mesh_agent}, state: {state}"
174 )]
175 ProcCreationError {
176 state: Box<resource::State<ProcState>>,
177 host_rank: usize,
178 mesh_agent: hyperactor_reference::ActorRef<HostAgent>,
179 },
180
181 #[error(
182 "error spawning proc mesh: statuses: {}",
183 RankedValues::invert(statuses)
184 )]
185 ProcSpawnError { statuses: RankedValues<Status> },
186
187 #[error(
188 "error spawning actor mesh: statuses: {}",
189 RankedValues::invert(statuses)
190 )]
191 ActorSpawnError { statuses: RankedValues<Status> },
192
193 #[error(
194 "error stopping actor mesh: statuses: {}",
195 RankedValues::invert(statuses)
196 )]
197 ActorStopError { statuses: RankedValues<Status> },
198
199 #[error("error spawning actor: {0}")]
200 SingletonActorSpawnError(anyhow::Error),
201
202 #[error("error spawning controller actor for mesh {0}: {1}")]
203 ControllerActorSpawnError(Name, anyhow::Error),
204
205 #[error("proc {0} must be direct-addressable")]
206 RankedProc(hyperactor_reference::ProcId),
207
208 #[error("{0}")]
209 Supervision(Box<MeshFailure>),
210
211 #[error("error: {0} does not exist")]
212 NotExist(Name),
213
214 #[error(transparent)]
215 Io(#[from] io::Error),
216
217 #[error(transparent)]
218 Host(#[from] HostError),
219
220 #[error(transparent)]
221 Other(#[from] anyhow::Error),
222}
223
224#[derive(Debug, thiserror::Error)]
226pub enum CodecError {
227 #[error(transparent)]
228 BincodeError(#[from] Box<bincode::Error>),
229 #[error(transparent)]
230 JsonError(#[from] Box<serde_json::Error>),
231 #[error(transparent)]
232 Base64Error(#[from] Box<base64::DecodeError>),
233 #[error(transparent)]
234 Utf8Error(#[from] Box<std::str::Utf8Error>),
235}
236
237impl From<bincode::Error> for Error {
238 fn from(e: bincode::Error) -> Self {
239 Error::CodecError(Box::new(e).into())
240 }
241}
242
243impl From<serde_json::Error> for Error {
244 fn from(e: serde_json::Error) -> Self {
245 Error::CodecError(Box::new(e).into())
246 }
247}
248
249impl From<base64::DecodeError> for Error {
250 fn from(e: base64::DecodeError) -> Self {
251 Error::CodecError(Box::new(e).into())
252 }
253}
254
255impl From<std::str::Utf8Error> for Error {
256 fn from(e: std::str::Utf8Error) -> Self {
257 Error::CodecError(Box::new(e).into())
258 }
259}
260
261impl From<crate::alloc::AllocatorError> for Error {
262 fn from(e: crate::alloc::AllocatorError) -> Self {
263 Error::AllocatorError(Box::new(e))
264 }
265}
266
267impl From<hyperactor::channel::ChannelError> for Error {
268 fn from(e: hyperactor::channel::ChannelError) -> Self {
269 Error::ChannelError(Box::new(e))
270 }
271}
272
273impl From<hyperactor::mailbox::MailboxError> for Error {
274 fn from(e: hyperactor::mailbox::MailboxError) -> Self {
275 Error::MailboxError(Box::new(e))
276 }
277}
278
279impl From<view::InvalidCardinality> for Error {
280 fn from(e: view::InvalidCardinality) -> Self {
281 Error::InvalidRankCardinality {
282 expected: e.expected,
283 actual: e.actual,
284 }
285 }
286}
287
288pub type Result<T> = std::result::Result<T, Error>;
290
291pub(crate) fn actor_display_name(base: &str, point: &view::Point) -> String {
296 if point.is_empty() {
297 return base.to_string();
298 }
299 let coords = point.format_as_dict();
300 if let Some(pos) = base.rfind('>') {
301 format!("{}{}{}", &base[..pos], coords, &base[pos..])
302 } else {
303 format!("{}{}", base, coords)
304 }
305}
306
307#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Named, EnumAsInner)]
312pub enum Name {
313 Suffixed(String, ShortUuid),
315 Reserved(String),
317}
318wirevalue::register_type!(Name);
319
320static NAME_SUFFIX_DELIMITER: &str = "-";
324
325impl Name {
326 pub fn new(name: impl Into<String>) -> Result<Self> {
328 Ok(Self::new_with_uuid(name, Some(ShortUuid::generate()))?)
329 }
330
331 pub fn new_reserved(name: impl Into<String>) -> Result<Self> {
333 Ok(Self::new_with_uuid(name, None)?)
334 }
335
336 fn new_with_uuid(
337 name: impl Into<String>,
338 uuid: Option<ShortUuid>,
339 ) -> std::result::Result<Self, NameParseError> {
340 let mut name = name.into();
341 if name.is_empty() {
342 name = "unnamed".to_string();
343 }
344 if !hyperactor_reference::is_valid_ident(&name) {
345 return Err(NameParseError::InvalidName(name));
346 }
347 if let Some(uuid) = uuid {
348 Ok(Self::Suffixed(name, uuid))
349 } else {
350 Ok(Self::Reserved(name))
351 }
352 }
353
354 pub fn name(&self) -> &str {
356 match self {
357 Self::Suffixed(n, _) => n,
358 Self::Reserved(n) => n,
359 }
360 }
361
362 pub fn uuid(&self) -> Option<&ShortUuid> {
365 match self {
366 Self::Suffixed(_, uuid) => Some(uuid),
367 Self::Reserved(_) => None,
368 }
369 }
370}
371
372impl Serialize for Name {
373 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
374 where
375 S: serde::Serializer,
376 {
377 serializer.serialize_str(&self.to_string())
379 }
380}
381
382impl<'de> Deserialize<'de> for Name {
383 fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
384 where
385 D: serde::Deserializer<'de>,
386 {
387 let s = String::deserialize(deserializer)?;
388 Name::from_str(&s).map_err(serde::de::Error::custom)
389 }
390}
391
392#[derive(thiserror::Error, Debug)]
394pub enum NameParseError {
395 #[error("invalid name: missing name")]
396 MissingName,
397
398 #[error("invalid name: missing uuid")]
399 MissingUuid,
400
401 #[error(
405 "invalid name '{0}': names must contain only alphanumeric characters \
406 and underscores, and must start with a letter or underscore"
407 )]
408 InvalidName(String),
409
410 #[error(transparent)]
411 InvalidUuid(#[from] <ShortUuid as FromStr>::Err),
412
413 #[error("invalid name: missing separator")]
414 MissingSeparator,
415}
416
417impl FromStr for Name {
418 type Err = NameParseError;
419
420 fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
421 if let Some((name, uuid)) = s.split_once(NAME_SUFFIX_DELIMITER) {
424 if name.is_empty() {
425 return Err(NameParseError::MissingName);
426 }
427 if uuid.is_empty() {
428 return Err(NameParseError::MissingName);
429 }
430
431 Name::new_with_uuid(name.to_string(), Some(uuid.parse()?))
432 } else {
433 if s.is_empty() {
434 return Err(NameParseError::MissingName);
435 }
436 Name::new_with_uuid(s, None)
437 }
438 }
439}
440
441impl std::fmt::Display for Name {
442 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
443 match self {
444 Self::Suffixed(n, uuid) => {
445 write!(f, "{}{}", n, NAME_SUFFIX_DELIMITER)?;
446 uuid.format(f, true )
447 }
448 Self::Reserved(n) => write!(f, "{}", n),
449 }
450 }
451}
452
453#[cfg(test)]
454mod tests {
455
456 #[test]
457 fn basic() {
458 use ndslice::selection::dsl;
459 use ndslice::selection::structurally_equal;
460
461 let actual = sel!(*, 0:4, *);
462 let expected = dsl::all(dsl::range(
463 ndslice::shape::Range(0, Some(4), 1),
464 dsl::all(dsl::true_()),
465 ));
466 assert!(structurally_equal(&actual, &expected));
467 }
468
469 #[cfg(FALSE)]
470 #[test]
471 fn shouldnt_compile() {
472 let _ = sel!(foobar);
473 }
474 use std::str::FromStr;
484
485 use hyperactor_mesh_macros::sel;
486 use ndslice::assert_round_trip;
487 use ndslice::assert_structurally_eq;
488 use ndslice::selection::Selection;
489
490 macro_rules! assert_round_trip_match {
491 ($left:expr, $right:expr) => {{
492 assert_structurally_eq!($left, $right);
493 assert_round_trip!($left);
494 assert_round_trip!($right);
495 }};
496 }
497
498 #[test]
499 fn token_parser() {
500 use ndslice::selection::dsl::*;
501 use ndslice::shape;
502
503 assert_round_trip_match!(all(true_()), sel!(*));
504 assert_round_trip_match!(range(3, true_()), sel!(3));
505 assert_round_trip_match!(range(1..4, true_()), sel!(1:4));
506 assert_round_trip_match!(all(range(1..4, true_())), sel!(*, 1:4));
507 assert_round_trip_match!(range(shape::Range(0, None, 1), true_()), sel!(:));
508 assert_round_trip_match!(any(true_()), sel!(?));
509 assert_round_trip_match!(any(range(1..4, all(true_()))), sel!(?, 1:4, *));
510 assert_round_trip_match!(union(range(0, true_()), range(1, true_())), sel!(0 | 1));
511 assert_round_trip_match!(
512 intersection(range(0..4, true_()), range(2..6, true_())),
513 sel!(0:4 & 2:6)
514 );
515 assert_round_trip_match!(range(shape::Range(0, None, 1), true_()), sel!(:));
516 assert_round_trip_match!(all(true_()), sel!(*));
517 assert_round_trip_match!(any(true_()), sel!(?));
518 assert_round_trip_match!(all(all(all(true_()))), sel!(*, *, *));
519 assert_round_trip_match!(intersection(all(true_()), all(true_())), sel!(* & *));
520 assert_round_trip_match!(
521 all(all(union(
522 range(0..2, true_()),
523 range(shape::Range(6, None, 1), true_())
524 ))),
525 sel!(*, *, (:2|6:))
526 );
527 assert_round_trip_match!(
528 all(all(range(shape::Range(1, None, 2), true_()))),
529 sel!(*, *, 1::2)
530 );
531 assert_round_trip_match!(
532 range(
533 shape::Range(0, Some(1), 1),
534 any(range(shape::Range(0, Some(4), 1), true_()))
535 ),
536 sel!(0, ?, :4)
537 );
538 assert_round_trip_match!(range(shape::Range(1, Some(4), 2), true_()), sel!(1:4:2));
539 assert_round_trip_match!(range(shape::Range(0, None, 2), true_()), sel!(::2));
540 assert_round_trip_match!(
541 union(range(0..4, true_()), range(4..8, true_())),
542 sel!(0:4 | 4:8)
543 );
544 assert_round_trip_match!(
545 intersection(range(0..4, true_()), range(2..6, true_())),
546 sel!(0:4 & 2:6)
547 );
548 assert_round_trip_match!(
549 all(union(range(1..4, all(true_())), range(5..6, all(true_())))),
550 sel!(*, (1:4 | 5:6), *)
551 );
552 assert_round_trip_match!(
553 range(
554 0,
555 intersection(
556 range(1..4, range(7, true_())),
557 range(2..5, range(7, true_()))
558 )
559 ),
560 sel!(0, (1:4 & 2:5), 7)
561 );
562 assert_round_trip_match!(
563 all(all(union(
564 union(range(0..2, true_()), range(4..6, true_())),
565 range(shape::Range(6, None, 1), true_())
566 ))),
567 sel!(*, *, (:2 | 4:6 | 6:))
568 );
569 assert_round_trip_match!(intersection(all(true_()), all(true_())), sel!(* & *));
570 assert_round_trip_match!(union(all(true_()), all(true_())), sel!(* | *));
571 assert_round_trip_match!(
572 intersection(
573 range(0..2, true_()),
574 union(range(1, true_()), range(2, true_()))
575 ),
576 sel!(0:2 & (1 | 2))
577 );
578 assert_round_trip_match!(
579 all(all(intersection(
580 range(1..2, true_()),
581 range(2..3, true_())
582 ))),
583 sel!(*,*,(1:2&2:3))
584 );
585 assert_round_trip_match!(
586 intersection(all(all(all(true_()))), all(all(all(true_())))),
587 sel!((*,*,*) & (*,*,*))
588 );
589 assert_round_trip_match!(
590 intersection(
591 range(0, all(all(true_()))),
592 range(0, union(range(1, all(true_())), range(3, all(true_()))))
593 ),
594 sel!((0, *, *) & (0, (1 | 3), *))
595 );
596 assert_round_trip_match!(
597 intersection(
598 range(0, all(all(true_()))),
599 range(
600 0,
601 union(
602 range(1, range(2..5, true_())),
603 range(3, range(2..5, true_()))
604 )
605 )
606 ),
607 sel!((0, *, *) & (0, (1 | 3), 2:5))
608 );
609 assert_round_trip_match!(all(true_()), sel!((*)));
610 assert_round_trip_match!(range(1..4, range(2, true_())), sel!(((1:4), 2)));
611 assert_round_trip_match!(sel!(1:4 & 5:6 | 7:8), sel!((1:4 & 5:6) | 7:8));
612 assert_round_trip_match!(
613 union(
614 intersection(all(all(true_())), all(all(true_()))),
615 all(all(true_()))
616 ),
617 sel!((*,*) & (*,*) | (*,*))
618 );
619 assert_round_trip_match!(all(true_()), sel!(*));
620 assert_round_trip_match!(sel!(((1:4))), sel!(1:4));
621 assert_round_trip_match!(sel!(*, (*)), sel!(*, *));
622 assert_round_trip_match!(
623 intersection(
624 range(0, range(1..4, true_())),
625 range(0, union(range(2, all(true_())), range(3, all(true_()))))
626 ),
627 sel!((0,1:4)&(0,(2|3),*))
628 );
629
630 assert_round_trip_match!(
633 sel!(0 & (0, (1|3), *)),
634 intersection(
635 range(0, true_()),
636 range(0, union(range(1, all(true_())), range(3, all(true_()))))
637 )
638 );
639 assert_round_trip_match!(
640 sel!(0 & (0, (3|1), *)),
641 intersection(
642 range(0, true_()),
643 range(0, union(range(3, all(true_())), range(1, all(true_()))))
644 )
645 );
646 assert_round_trip_match!(
647 sel!((*, *, *) & (*, *, (2 | 4))),
648 intersection(
649 all(all(all(true_()))),
650 all(all(union(range(2, true_()), range(4, true_()))))
651 )
652 );
653 assert_round_trip_match!(
654 sel!((*, *, *) & (*, *, (4 | 2))),
655 intersection(
656 all(all(all(true_()))),
657 all(all(union(range(4, true_()), range(2, true_()))))
658 )
659 );
660 assert_round_trip_match!(
661 sel!((*, (1|2)) & (*, (2|1))),
662 intersection(
663 all(union(range(1, true_()), range(2, true_()))),
664 all(union(range(2, true_()), range(1, true_())))
665 )
666 );
667 assert_round_trip_match!(
668 sel!((*, *, *) & *),
669 intersection(all(all(all(true_()))), all(true_()))
670 );
671 assert_round_trip_match!(
672 sel!(* & (*, *, *)),
673 intersection(all(true_()), all(all(all(true_()))))
674 );
675
676 assert_round_trip_match!(
677 sel!( (*, *, *) & ((*, *, *) & (*, *, *)) ),
678 intersection(
679 all(all(all(true_()))),
680 intersection(all(all(all(true_()))), all(all(all(true_()))))
681 )
682 );
683 assert_round_trip_match!(
684 sel!((1, *, *) | (0 & (0, 3, *))),
685 union(
686 range(1, all(all(true_()))),
687 intersection(range(0, true_()), range(0, range(3, all(true_()))))
688 )
689 );
690 assert_round_trip_match!(
691 sel!(((0, *)| (1, *)) & ((1, *) | (0, *))),
692 intersection(
693 union(range(0, all(true_())), range(1, all(true_()))),
694 union(range(1, all(true_())), range(0, all(true_())))
695 )
696 );
697 assert_round_trip_match!(sel!(*, 8:8), all(range(8..8, true_())));
698 assert_round_trip_match!(
699 sel!((*, 1) & (*, 8 : 8)),
700 intersection(all(range(1..2, true_())), all(range(8..8, true_())))
701 );
702 assert_round_trip_match!(
703 sel!((*, 8 : 8) | (*, 1)),
704 union(all(range(8..8, true_())), all(range(1..2, true_())))
705 );
706 assert_round_trip_match!(
707 sel!((*, 1) | (*, 2:8)),
708 union(all(range(1..2, true_())), all(range(2..8, true_())))
709 );
710 assert_round_trip_match!(
711 sel!((*, *, *) & (*, *, 2:8)),
712 intersection(all(all(all(true_()))), all(all(range(2..8, true_()))))
713 );
714 }
715
716 #[test]
717 fn test_name_unique() {
718 use super::Name;
719 assert_ne!(Name::new("foo").unwrap(), Name::new("foo").unwrap());
720 let name = Name::new("foo").unwrap();
721 assert_eq!(name, name);
722 }
723
724 #[test]
725 fn test_name_roundtrip() {
726 use super::Name;
727 use super::ShortUuid;
728 let uuid = "111111111111".parse::<ShortUuid>().unwrap();
729 let name = Name::new_with_uuid("foo", Some(uuid)).unwrap();
730 let str = name.to_string();
731 assert_eq!(str, "foo-111111111111");
732 assert_eq!(name, Name::from_str(&str).unwrap());
733 }
734
735 #[test]
736 fn test_name_roundtrip_with_underscore() {
737 use super::Name;
738 use super::ShortUuid;
739 let uuid = "_1a2b3c4d5e6f".parse::<ShortUuid>().unwrap();
742 let name = Name::new_with_uuid("foo", Some(uuid)).unwrap();
743 let str = name.to_string();
744 assert_eq!(str, "foo-1a2b3c4d5e6f");
746 assert_eq!(name, Name::from_str(&str).unwrap());
747 }
748
749 #[test]
750 fn test_name_roundtrip_random() {
751 use super::Name;
752 let name = Name::new("foo").unwrap();
753 assert_eq!(name, Name::from_str(&name.to_string()).unwrap());
754 }
755
756 #[test]
757 fn test_name_roundtrip_reserved() {
758 use super::Name;
759 let name = Name::new_reserved("foo").unwrap();
760 let str = name.to_string();
761 assert_eq!(str, "foo");
762 assert_eq!(name, Name::from_str(&str).unwrap());
763 }
764
765 #[test]
766 fn test_name_parse() {
767 use super::Name;
768 let name = Name::from_str("foo_bar_1a2b3c4d5e6f").unwrap();
771 assert_eq!(format!("{}", name), "foo_bar_1a2b3c4d5e6f");
772 }
773
774 #[test]
775 fn test_invalid() {
776 use super::Name;
777 assert!(Name::new("").is_ok());
779 assert!(Name::new("foo-").is_err());
781 assert!(Name::new("foo-bar").is_err());
782 }
783}