1use std::env;
22use std::fs::File;
23use std::io::Read;
24use std::path::Path;
25
26use derive_more::Display;
27use derive_more::From;
28use derive_more::Into;
29use serde::Deserialize;
30use serde::Serialize;
31use shell_quote::QuoteRefExt;
32use thiserror::Error;
33use typeuri::Named;
34
35pub mod attrs;
36pub mod flattrs;
37pub mod global;
38
39pub use attrs::AttrKeyInfo;
41pub use attrs::AttrValue;
42pub use attrs::Attrs;
43pub use attrs::Key;
44pub use attrs::SerializableValue;
45#[doc(hidden)]
47pub use bincode;
48pub use flattrs::Flattrs;
49pub use hyperactor_config_macros::AttrValue;
51pub use inventory::submit;
53pub use paste::paste;
54#[doc(hidden)]
56pub use typeuri;
57
58#[derive(Clone, Debug, Serialize, Deserialize)]
76pub struct ConfigAttr {
77 pub env_name: Option<String>,
79
80 pub py_name: Option<String>,
83
84 pub propagate: bool,
87}
88
89impl ConfigAttr {
90 pub fn new(env_name: Option<String>, py_name: Option<String>) -> Self {
96 Self {
97 env_name,
98 py_name,
99 propagate: true,
100 }
101 }
102
103 pub fn process_local(mut self) -> Self {
108 self.propagate = false;
109 self
110 }
111}
112
113impl Named for ConfigAttr {
114 fn typename() -> &'static str {
115 "hyperactor_config::ConfigAttr"
116 }
117}
118
119impl AttrValue for ConfigAttr {
120 fn display(&self) -> String {
121 serde_json::to_string(self).unwrap_or_else(|_| "<invalid ConfigAttr>".into())
122 }
123 fn parse(s: &str) -> Result<Self, anyhow::Error> {
124 Ok(serde_json::from_str(s)?)
125 }
126}
127
128#[derive(
129 Clone,
130 Copy,
131 Debug,
132 Display,
133 From,
134 Into,
135 PartialEq,
136 Eq,
137 Serialize,
138 Deserialize
139)]
140#[display("{}", _0)]
141pub struct NonZeroUsize(std::num::NonZeroUsize);
142
143#[derive(Clone, Copy, Debug, Error, PartialEq, Eq)]
144#[error("expected non-zero usize")]
145pub struct NonZeroUsizeError;
146
147impl NonZeroUsize {
148 pub const MIN: Self = Self(std::num::NonZeroUsize::MIN);
149
150 pub const fn new(value: usize) -> Option<Self> {
151 match std::num::NonZeroUsize::new(value) {
152 Some(value) => Some(Self(value)),
153 None => None,
154 }
155 }
156
157 pub const fn get(self) -> usize {
158 self.0.get()
159 }
160
161 pub const fn into_std(self) -> std::num::NonZeroUsize {
162 self.0
163 }
164}
165
166impl Named for NonZeroUsize {
167 fn typename() -> &'static str {
168 "hyperactor_config::NonZeroUsize"
169 }
170}
171
172impl TryFrom<usize> for NonZeroUsize {
173 type Error = NonZeroUsizeError;
174
175 fn try_from(value: usize) -> Result<Self, Self::Error> {
176 Self::new(value).ok_or(NonZeroUsizeError)
177 }
178}
179
180impl From<NonZeroUsize> for usize {
181 fn from(value: NonZeroUsize) -> Self {
182 value.get()
183 }
184}
185
186impl AttrValue for NonZeroUsize {
187 fn display(&self) -> String {
188 self.to_string()
189 }
190
191 fn parse(s: &str) -> Result<Self, anyhow::Error> {
192 Ok(s.parse::<usize>()?.try_into()?)
193 }
194}
195
196#[derive(Clone, Debug, Serialize, Deserialize)]
207pub struct IntrospectAttr {
208 pub name: String,
210
211 pub desc: String,
213}
214
215impl Named for IntrospectAttr {
216 fn typename() -> &'static str {
217 "hyperactor_config::IntrospectAttr"
218 }
219}
220
221impl AttrValue for IntrospectAttr {
222 fn display(&self) -> String {
223 serde_json::to_string(self).unwrap_or_else(|_| "<invalid IntrospectAttr>".into())
224 }
225 fn parse(s: &str) -> Result<Self, anyhow::Error> {
226 Ok(serde_json::from_str(s)?)
227 }
228}
229
230declare_attrs! {
231 pub attr CONFIG: ConfigAttr;
243
244 pub attr INTROSPECT: IntrospectAttr;
256}
257
258pub fn from_env() -> Attrs {
260 let mut config = Attrs::new();
261 let mut output = String::new();
262
263 fn export(env_var: &str, value: Option<&dyn SerializableValue>) -> String {
264 let env_var: String = env_var.quoted(shell_quote::Bash);
265 let value: String = value
266 .map_or("".to_string(), SerializableValue::display)
267 .quoted(shell_quote::Bash);
268 format!("export {}={}\n", env_var, value)
269 }
270
271 for key in inventory::iter::<AttrKeyInfo>() {
272 let Some(cfg_meta) = key.meta.get(CONFIG) else {
277 continue;
278 };
279 let Some(env_var) = cfg_meta.env_name.as_deref() else {
280 continue;
281 };
282
283 let Ok(val) = env::var(env_var) else {
284 output.push_str("# ");
286 output.push_str(&export(env_var, key.default));
287 continue;
288 };
289
290 match (key.parse)(&val) {
291 Err(e) => {
292 tracing::error!(
293 "failed to override config key {} from value \"{}\" in ${}: {})",
294 key.name,
295 val,
296 env_var,
297 e
298 );
299 output.push_str("# ");
300 output.push_str(&export(env_var, key.default));
301 }
302 Ok(parsed) => {
303 output.push_str("# ");
304 output.push_str(&export(env_var, key.default));
305 output.push_str(&export(env_var, Some(parsed.as_ref())));
306 config.insert_value_by_name_unchecked(key.name, parsed);
307 }
308 }
309 }
310
311 tracing::info!(
312 "loaded configuration from environment:\n{}",
313 output.trim_end()
314 );
315
316 config
317}
318
319pub fn from_yaml<P: AsRef<Path>>(path: P) -> Result<Attrs, anyhow::Error> {
321 let mut file = File::open(path)?;
322 let mut contents = String::new();
323 file.read_to_string(&mut contents)?;
324 Ok(serde_yaml::from_str(&contents)?)
325}
326
327pub fn to_yaml<P: AsRef<Path>>(attrs: &Attrs, path: P) -> Result<(), anyhow::Error> {
329 let yaml = serde_yaml::to_string(attrs)?;
330 std::fs::write(path, yaml)?;
331 Ok(())
332}
333
334#[cfg(test)]
335mod tests {
336 use std::collections::HashSet;
337 use std::net::Ipv4Addr;
338
339 use indoc::indoc;
340
341 use crate::AttrValue;
342 use crate::CONFIG;
343 use crate::ConfigAttr;
344 use crate::NonZeroUsize;
345 use crate::attrs::declare_attrs;
346 use crate::from_env;
347 use crate::from_yaml;
348 use crate::to_yaml;
349
350 fn logs_assert_unscoped(f: impl Fn(&[&str]) -> Result<(), String>) {
354 let buf = tracing_test::internal::global_buf().lock().unwrap();
355 let logs_str = std::str::from_utf8(&buf).expect("Logs contain invalid UTF8");
356 let lines: Vec<&str> = logs_str.lines().collect();
357 match f(&lines) {
358 Ok(()) => {}
359 Err(msg) => panic!("{}", msg),
360 }
361 }
362
363 #[derive(
364 Debug,
365 Clone,
366 Copy,
367 PartialEq,
368 Eq,
369 serde::Serialize,
370 serde::Deserialize
371 )]
372 pub(crate) enum TestMode {
373 Development,
374 Staging,
375 Production,
376 }
377
378 impl std::fmt::Display for TestMode {
379 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
380 match self {
381 TestMode::Development => write!(f, "dev"),
382 TestMode::Staging => write!(f, "staging"),
383 TestMode::Production => write!(f, "prod"),
384 }
385 }
386 }
387
388 impl std::str::FromStr for TestMode {
389 type Err = anyhow::Error;
390
391 fn from_str(s: &str) -> Result<Self, Self::Err> {
392 match s {
393 "dev" => Ok(TestMode::Development),
394 "staging" => Ok(TestMode::Staging),
395 "prod" => Ok(TestMode::Production),
396 _ => Err(anyhow::anyhow!("unknown mode: {}", s)),
397 }
398 }
399 }
400
401 impl typeuri::Named for TestMode {
402 fn typename() -> &'static str {
403 "hyperactor_config::tests::TestMode"
404 }
405 }
406
407 impl crate::attrs::AttrValue for TestMode {
408 fn display(&self) -> String {
409 self.to_string()
410 }
411
412 fn parse(s: &str) -> Result<Self, anyhow::Error> {
413 s.parse()
414 }
415 }
416
417 declare_attrs! {
418 @meta(CONFIG = ConfigAttr::new(
419 Some("TEST_USIZE_KEY".to_string()),
420 None,
421 ))
422 pub attr USIZE_KEY: usize = 10;
423
424 @meta(CONFIG = ConfigAttr::new(
425 Some("TEST_STRING_KEY".to_string()),
426 None,
427 ))
428 pub attr STRING_KEY: String = String::new();
429
430 @meta(CONFIG = ConfigAttr::new(
431 Some("TEST_BOOL_KEY".to_string()),
432 None,
433 ))
434 pub attr BOOL_KEY: bool = false;
435
436 @meta(CONFIG = ConfigAttr::new(
437 Some("TEST_I64_KEY".to_string()),
438 None,
439 ))
440 pub attr I64_KEY: i64 = -42;
441
442 @meta(CONFIG = ConfigAttr::new(
443 Some("TEST_F64_KEY".to_string()),
444 None,
445 ))
446 pub attr F64_KEY: f64 = 3.14;
447
448 @meta(CONFIG = ConfigAttr::new(
449 Some("TEST_U32_KEY".to_string()),
450 Some("test_u32_key".to_string()),
451 ))
452 pub attr U32_KEY: u32 = 100;
453
454 @meta(CONFIG = ConfigAttr::new(
455 Some("TEST_DURATION_KEY".to_string()),
456 None,
457 ))
458 pub attr DURATION_KEY: std::time::Duration = std::time::Duration::from_mins(1);
459
460 @meta(CONFIG = ConfigAttr::new(
461 Some("TEST_MODE_KEY".to_string()),
462 None,
463 ))
464 pub attr MODE_KEY: TestMode = TestMode::Development;
465
466 @meta(CONFIG = ConfigAttr::new(
467 Some("TEST_IP_KEY".to_string()),
468 None,
469 ))
470 pub attr IP_KEY: Ipv4Addr = Ipv4Addr::new(127, 0, 0, 1);
471
472 @meta(CONFIG = ConfigAttr::new(
473 Some("TEST_SYSTEMTIME_KEY".to_string()),
474 None,
475 ))
476 pub attr SYSTEMTIME_KEY: std::time::SystemTime = std::time::UNIX_EPOCH;
477
478 @meta(CONFIG = ConfigAttr::new(
479 None,
480 Some("test_no_env_key".to_string()),
481 ))
482 pub attr NO_ENV_KEY: usize = 999;
483 }
484
485 #[tracing_test::traced_test]
486 #[test]
487 #[cfg_attr(not(fbcode_build), ignore)]
489 fn test_from_env() {
490 unsafe { std::env::set_var("TEST_USIZE_KEY", "1024") };
493 unsafe { std::env::set_var("TEST_STRING_KEY", "world") };
495 unsafe { std::env::set_var("TEST_BOOL_KEY", "true") };
497 unsafe { std::env::set_var("TEST_I64_KEY", "-999") };
499 unsafe { std::env::set_var("TEST_F64_KEY", "2.718") };
501 unsafe { std::env::set_var("TEST_U32_KEY", "500") };
503 unsafe { std::env::set_var("TEST_DURATION_KEY", "5s") };
505 unsafe { std::env::set_var("TEST_MODE_KEY", "prod") };
507 unsafe { std::env::set_var("TEST_IP_KEY", "192.168.1.1") };
509 unsafe { std::env::set_var("TEST_SYSTEMTIME_KEY", "2024-01-15T10:30:00Z") };
511
512 let config = from_env();
513
514 assert_eq!(config[USIZE_KEY], 1024);
516 assert_eq!(config[STRING_KEY], "world");
517 assert!(config[BOOL_KEY]);
518 assert_eq!(config[I64_KEY], -999);
519 assert_eq!(config[F64_KEY], 2.718);
520 assert_eq!(config[U32_KEY], 500);
521 assert_eq!(config[DURATION_KEY], std::time::Duration::from_secs(5));
522 assert_eq!(config[MODE_KEY], TestMode::Production);
523 assert_eq!(config[IP_KEY], Ipv4Addr::new(192, 168, 1, 1));
524 assert_eq!(
525 config[SYSTEMTIME_KEY],
526 std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1705314600) );
528
529 assert_eq!(config[NO_ENV_KEY], 999);
531
532 let expected_lines: HashSet<&str> = indoc! {"
533 # export TEST_USIZE_KEY=10
534 export TEST_USIZE_KEY=1024
535 # export TEST_STRING_KEY=''
536 export TEST_STRING_KEY=world
537 # export TEST_BOOL_KEY=0
538 export TEST_BOOL_KEY=1
539 # export TEST_I64_KEY=-42
540 export TEST_I64_KEY=-999
541 # export TEST_F64_KEY=3.14
542 export TEST_F64_KEY=2.718
543 # export TEST_U32_KEY=100
544 export TEST_U32_KEY=500
545 # export TEST_DURATION_KEY=1m
546 export TEST_DURATION_KEY=5s
547 # export TEST_MODE_KEY=dev
548 export TEST_MODE_KEY=prod
549 # export TEST_IP_KEY=127.0.0.1
550 export TEST_IP_KEY=192.168.1.1
551 "}
552 .trim_end()
553 .lines()
554 .collect();
555
556 logs_assert_unscoped(|logged_lines: &[&str]| {
560 let mut expected_lines = expected_lines.clone(); for logged in logged_lines {
562 expected_lines.remove(logged);
563 }
564
565 if expected_lines.is_empty() {
566 Ok(())
567 } else {
568 Err(format!("missing log lines: {:?}", expected_lines))
569 }
570 });
571
572 unsafe { std::env::remove_var("TEST_USIZE_KEY") };
575 unsafe { std::env::remove_var("TEST_STRING_KEY") };
577 unsafe { std::env::remove_var("TEST_BOOL_KEY") };
579 unsafe { std::env::remove_var("TEST_I64_KEY") };
581 unsafe { std::env::remove_var("TEST_F64_KEY") };
583 unsafe { std::env::remove_var("TEST_U32_KEY") };
585 unsafe { std::env::remove_var("TEST_DURATION_KEY") };
587 unsafe { std::env::remove_var("TEST_MODE_KEY") };
589 unsafe { std::env::remove_var("TEST_IP_KEY") };
591 unsafe { std::env::remove_var("TEST_SYSTEMTIME_KEY") };
593 }
594
595 #[test]
596 fn test_yaml_round_trip() {
597 let temp_path = std::env::temp_dir().join("test_config.yaml");
598
599 let mut config = crate::Attrs::new();
600 config.set(USIZE_KEY, 2048);
601 config.set(STRING_KEY, "hello_yaml".to_string());
602 config.set(BOOL_KEY, true);
603 config.set(I64_KEY, -123);
604 config.set(F64_KEY, 1.414);
605 config.set(U32_KEY, 777);
606 config.set(DURATION_KEY, std::time::Duration::from_mins(2));
607 config.set(MODE_KEY, TestMode::Staging);
608 config.set(IP_KEY, Ipv4Addr::new(10, 0, 0, 1));
609 config.set(
610 SYSTEMTIME_KEY,
611 std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1609459200),
612 );
613
614 to_yaml(&config, &temp_path).unwrap();
615
616 let yaml_content = std::fs::read_to_string(&temp_path).unwrap();
617
618 eprintln!("YAML content:\n{}", yaml_content);
619
620 assert!(yaml_content.contains("2048"));
621 assert!(yaml_content.contains("hello_yaml"));
622 assert!(yaml_content.contains("Staging"));
623
624 let loaded_config = from_yaml(&temp_path).unwrap();
625
626 assert_eq!(loaded_config[USIZE_KEY], 2048);
627 assert_eq!(loaded_config[STRING_KEY], "hello_yaml");
628 assert!(loaded_config[BOOL_KEY]);
629 assert_eq!(loaded_config[I64_KEY], -123);
630 assert_eq!(loaded_config[F64_KEY], 1.414);
631 assert_eq!(loaded_config[U32_KEY], 777);
632 assert_eq!(
633 loaded_config[DURATION_KEY],
634 std::time::Duration::from_mins(2)
635 );
636 assert_eq!(loaded_config[MODE_KEY], TestMode::Staging);
637 assert_eq!(loaded_config[IP_KEY], Ipv4Addr::new(10, 0, 0, 1));
638 assert_eq!(
639 loaded_config[SYSTEMTIME_KEY],
640 std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1609459200)
641 );
642
643 let _ = std::fs::remove_file(&temp_path);
644 }
645
646 #[test]
647 fn test_nonzero_usize_attr_value() {
648 let value = NonZeroUsize::parse("16").unwrap();
649 assert_eq!(value.get(), 16);
650 assert_eq!(value.display(), "16");
651 assert_eq!(usize::from(value), 16);
652 assert_eq!(
653 std::num::NonZeroUsize::from(value),
654 std::num::NonZeroUsize::new(16).unwrap()
655 );
656 assert_eq!(Option::<NonZeroUsize>::parse("").unwrap(), None);
657 assert_eq!(
658 Option::<NonZeroUsize>::parse("16")
659 .unwrap()
660 .map(NonZeroUsize::get),
661 Some(16)
662 );
663 assert!(NonZeroUsize::try_from(0).is_err());
664 assert!(NonZeroUsize::parse("0").is_err());
665 assert!(Option::<NonZeroUsize>::parse("0").is_err());
666 }
667
668 #[test]
678 fn test_introspect_meta_key() {
679 use crate::INTROSPECT;
680 use crate::IntrospectAttr;
681
682 declare_attrs! {
683 @meta(INTROSPECT = IntrospectAttr {
684 name: "test_key".into(),
685 desc: "A test introspection key".into(),
686 })
687 attr TEST_INTROSPECT_KEY: String;
688 }
689 let meta = TEST_INTROSPECT_KEY.attrs();
690 let introspect = meta
691 .get(INTROSPECT)
692 .expect("INTROSPECT meta-attr should be set");
693 assert_eq!(introspect.name, "test_key");
694 assert_eq!(introspect.desc, "A test introspection key");
695 }
696}