1use std::env;
22use std::fs::File;
23use std::io::Read;
24use std::path::Path;
25
26use derive_more::Display;
27use derive_more::From;
28use derive_more::Into;
29use serde::Deserialize;
30use serde::Serialize;
31use shell_quote::QuoteRefExt;
32use thiserror::Error;
33use typeuri::Named;
34
35pub mod attrs;
36pub mod flattrs;
37pub mod global;
38
39pub use attrs::AttrKeyInfo;
41pub use attrs::AttrValue;
42pub use attrs::Attrs;
43pub use attrs::Key;
44pub use attrs::SerializableValue;
45#[doc(hidden)]
47pub use bincode;
48pub use flattrs::Flattrs;
49pub use flattrs::FlattrsValidationError;
50pub use hyperactor_config_macros::AttrValue;
52pub use inventory::submit;
54pub use paste::paste;
55#[doc(hidden)]
57pub use typeuri;
58
59#[derive(Clone, Debug, Serialize, Deserialize)]
77pub struct ConfigAttr {
78 pub env_name: Option<String>,
80
81 pub py_name: Option<String>,
84
85 pub propagate: bool,
88}
89
90impl ConfigAttr {
91 pub fn new(env_name: Option<String>, py_name: Option<String>) -> Self {
97 Self {
98 env_name,
99 py_name,
100 propagate: true,
101 }
102 }
103
104 pub fn process_local(mut self) -> Self {
109 self.propagate = false;
110 self
111 }
112}
113
114impl Named for ConfigAttr {
115 fn typename() -> &'static str {
116 "hyperactor_config::ConfigAttr"
117 }
118}
119
120impl AttrValue for ConfigAttr {
121 fn display(&self) -> String {
122 serde_json::to_string(self).unwrap_or_else(|_| "<invalid ConfigAttr>".into())
123 }
124 fn parse(s: &str) -> Result<Self, anyhow::Error> {
125 Ok(serde_json::from_str(s)?)
126 }
127}
128
129#[derive(
130 Clone,
131 Copy,
132 Debug,
133 Display,
134 From,
135 Into,
136 PartialEq,
137 Eq,
138 Serialize,
139 Deserialize
140)]
141#[display("{}", _0)]
142pub struct NonZeroUsize(std::num::NonZeroUsize);
143
144#[derive(Clone, Copy, Debug, Error, PartialEq, Eq)]
145#[error("expected non-zero usize")]
146pub struct NonZeroUsizeError;
147
148impl NonZeroUsize {
149 pub const MIN: Self = Self(std::num::NonZeroUsize::MIN);
150
151 pub const fn new(value: usize) -> Option<Self> {
152 match std::num::NonZeroUsize::new(value) {
153 Some(value) => Some(Self(value)),
154 None => None,
155 }
156 }
157
158 pub const fn get(self) -> usize {
159 self.0.get()
160 }
161
162 pub const fn into_std(self) -> std::num::NonZeroUsize {
163 self.0
164 }
165}
166
167impl Named for NonZeroUsize {
168 fn typename() -> &'static str {
169 "hyperactor_config::NonZeroUsize"
170 }
171}
172
173impl TryFrom<usize> for NonZeroUsize {
174 type Error = NonZeroUsizeError;
175
176 fn try_from(value: usize) -> Result<Self, Self::Error> {
177 Self::new(value).ok_or(NonZeroUsizeError)
178 }
179}
180
181impl From<NonZeroUsize> for usize {
182 fn from(value: NonZeroUsize) -> Self {
183 value.get()
184 }
185}
186
187impl AttrValue for NonZeroUsize {
188 fn display(&self) -> String {
189 self.to_string()
190 }
191
192 fn parse(s: &str) -> Result<Self, anyhow::Error> {
193 Ok(s.parse::<usize>()?.try_into()?)
194 }
195}
196
197#[derive(Clone, Debug, Serialize, Deserialize)]
208pub struct IntrospectAttr {
209 pub name: String,
211
212 pub desc: String,
214}
215
216impl Named for IntrospectAttr {
217 fn typename() -> &'static str {
218 "hyperactor_config::IntrospectAttr"
219 }
220}
221
222impl AttrValue for IntrospectAttr {
223 fn display(&self) -> String {
224 serde_json::to_string(self).unwrap_or_else(|_| "<invalid IntrospectAttr>".into())
225 }
226 fn parse(s: &str) -> Result<Self, anyhow::Error> {
227 Ok(serde_json::from_str(s)?)
228 }
229}
230
231declare_attrs! {
232 pub attr CONFIG: ConfigAttr;
244
245 pub attr INTROSPECT: IntrospectAttr;
257}
258
259pub fn from_env() -> Attrs {
261 let mut config = Attrs::new();
262 let mut output = String::new();
263
264 fn export(env_var: &str, value: Option<&dyn SerializableValue>) -> String {
265 let env_var: String = env_var.quoted(shell_quote::Bash);
266 let value: String = value
267 .map_or("".to_string(), SerializableValue::display)
268 .quoted(shell_quote::Bash);
269 format!("export {}={}\n", env_var, value)
270 }
271
272 for key in inventory::iter::<AttrKeyInfo>() {
273 let Some(cfg_meta) = key.meta.get(CONFIG) else {
278 continue;
279 };
280 let Some(env_var) = cfg_meta.env_name.as_deref() else {
281 continue;
282 };
283
284 let Ok(val) = env::var(env_var) else {
285 output.push_str("# ");
287 output.push_str(&export(env_var, key.default));
288 continue;
289 };
290
291 match (key.parse)(&val) {
292 Err(e) => {
293 tracing::error!(
294 "failed to override config key {} from value \"{}\" in ${}: {})",
295 key.name,
296 val,
297 env_var,
298 e
299 );
300 output.push_str("# ");
301 output.push_str(&export(env_var, key.default));
302 }
303 Ok(parsed) => {
304 output.push_str("# ");
305 output.push_str(&export(env_var, key.default));
306 output.push_str(&export(env_var, Some(parsed.as_ref())));
307 config.insert_value_by_name_unchecked(key.name, parsed);
308 }
309 }
310 }
311
312 tracing::info!(
313 "loaded configuration from environment:\n{}",
314 output.trim_end()
315 );
316
317 config
318}
319
320pub fn from_yaml<P: AsRef<Path>>(path: P) -> Result<Attrs, anyhow::Error> {
322 let mut file = File::open(path)?;
323 let mut contents = String::new();
324 file.read_to_string(&mut contents)?;
325 Ok(serde_yaml::from_str(&contents)?)
326}
327
328pub fn to_yaml<P: AsRef<Path>>(attrs: &Attrs, path: P) -> Result<(), anyhow::Error> {
330 let yaml = serde_yaml::to_string(attrs)?;
331 std::fs::write(path, yaml)?;
332 Ok(())
333}
334
335#[cfg(test)]
336mod tests {
337 use std::collections::HashSet;
338 use std::net::Ipv4Addr;
339
340 use indoc::indoc;
341
342 use crate::AttrValue;
343 use crate::CONFIG;
344 use crate::ConfigAttr;
345 use crate::NonZeroUsize;
346 use crate::attrs::declare_attrs;
347 use crate::from_env;
348 use crate::from_yaml;
349 use crate::to_yaml;
350
351 fn logs_assert_unscoped(f: impl Fn(&[&str]) -> Result<(), String>) {
355 let buf = tracing_test::internal::global_buf().lock().unwrap();
356 let logs_str = std::str::from_utf8(&buf).expect("Logs contain invalid UTF8");
357 let lines: Vec<&str> = logs_str.lines().collect();
358 match f(&lines) {
359 Ok(()) => {}
360 Err(msg) => panic!("{}", msg),
361 }
362 }
363
364 #[derive(
365 Debug,
366 Clone,
367 Copy,
368 PartialEq,
369 Eq,
370 serde::Serialize,
371 serde::Deserialize
372 )]
373 pub(crate) enum TestMode {
374 Development,
375 Staging,
376 Production,
377 }
378
379 impl std::fmt::Display for TestMode {
380 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
381 match self {
382 TestMode::Development => write!(f, "dev"),
383 TestMode::Staging => write!(f, "staging"),
384 TestMode::Production => write!(f, "prod"),
385 }
386 }
387 }
388
389 impl std::str::FromStr for TestMode {
390 type Err = anyhow::Error;
391
392 fn from_str(s: &str) -> Result<Self, Self::Err> {
393 match s {
394 "dev" => Ok(TestMode::Development),
395 "staging" => Ok(TestMode::Staging),
396 "prod" => Ok(TestMode::Production),
397 _ => Err(anyhow::anyhow!("unknown mode: {}", s)),
398 }
399 }
400 }
401
402 impl typeuri::Named for TestMode {
403 fn typename() -> &'static str {
404 "hyperactor_config::tests::TestMode"
405 }
406 }
407
408 impl crate::attrs::AttrValue for TestMode {
409 fn display(&self) -> String {
410 self.to_string()
411 }
412
413 fn parse(s: &str) -> Result<Self, anyhow::Error> {
414 s.parse()
415 }
416 }
417
418 declare_attrs! {
419 @meta(CONFIG = ConfigAttr::new(
420 Some("TEST_USIZE_KEY".to_string()),
421 None,
422 ))
423 pub attr USIZE_KEY: usize = 10;
424
425 @meta(CONFIG = ConfigAttr::new(
426 Some("TEST_STRING_KEY".to_string()),
427 None,
428 ))
429 pub attr STRING_KEY: String = String::new();
430
431 @meta(CONFIG = ConfigAttr::new(
432 Some("TEST_BOOL_KEY".to_string()),
433 None,
434 ))
435 pub attr BOOL_KEY: bool = false;
436
437 @meta(CONFIG = ConfigAttr::new(
438 Some("TEST_I64_KEY".to_string()),
439 None,
440 ))
441 pub attr I64_KEY: i64 = -42;
442
443 @meta(CONFIG = ConfigAttr::new(
444 Some("TEST_F64_KEY".to_string()),
445 None,
446 ))
447 pub attr F64_KEY: f64 = 3.14;
448
449 @meta(CONFIG = ConfigAttr::new(
450 Some("TEST_U32_KEY".to_string()),
451 Some("test_u32_key".to_string()),
452 ))
453 pub attr U32_KEY: u32 = 100;
454
455 @meta(CONFIG = ConfigAttr::new(
456 Some("TEST_DURATION_KEY".to_string()),
457 None,
458 ))
459 pub attr DURATION_KEY: std::time::Duration = std::time::Duration::from_mins(1);
460
461 @meta(CONFIG = ConfigAttr::new(
462 Some("TEST_MODE_KEY".to_string()),
463 None,
464 ))
465 pub attr MODE_KEY: TestMode = TestMode::Development;
466
467 @meta(CONFIG = ConfigAttr::new(
468 Some("TEST_IP_KEY".to_string()),
469 None,
470 ))
471 pub attr IP_KEY: Ipv4Addr = Ipv4Addr::new(127, 0, 0, 1);
472
473 @meta(CONFIG = ConfigAttr::new(
474 Some("TEST_SYSTEMTIME_KEY".to_string()),
475 None,
476 ))
477 pub attr SYSTEMTIME_KEY: std::time::SystemTime = std::time::UNIX_EPOCH;
478
479 @meta(CONFIG = ConfigAttr::new(
480 None,
481 Some("test_no_env_key".to_string()),
482 ))
483 pub attr NO_ENV_KEY: usize = 999;
484 }
485
486 #[tracing_test::traced_test]
487 #[test]
488 #[cfg_attr(not(fbcode_build), ignore)]
490 fn test_from_env() {
491 unsafe { std::env::set_var("TEST_USIZE_KEY", "1024") };
494 unsafe { std::env::set_var("TEST_STRING_KEY", "world") };
496 unsafe { std::env::set_var("TEST_BOOL_KEY", "true") };
498 unsafe { std::env::set_var("TEST_I64_KEY", "-999") };
500 unsafe { std::env::set_var("TEST_F64_KEY", "2.718") };
502 unsafe { std::env::set_var("TEST_U32_KEY", "500") };
504 unsafe { std::env::set_var("TEST_DURATION_KEY", "5s") };
506 unsafe { std::env::set_var("TEST_MODE_KEY", "prod") };
508 unsafe { std::env::set_var("TEST_IP_KEY", "192.168.1.1") };
510 unsafe { std::env::set_var("TEST_SYSTEMTIME_KEY", "2024-01-15T10:30:00Z") };
512
513 let config = from_env();
514
515 assert_eq!(config[USIZE_KEY], 1024);
517 assert_eq!(config[STRING_KEY], "world");
518 assert!(config[BOOL_KEY]);
519 assert_eq!(config[I64_KEY], -999);
520 assert_eq!(config[F64_KEY], 2.718);
521 assert_eq!(config[U32_KEY], 500);
522 assert_eq!(config[DURATION_KEY], std::time::Duration::from_secs(5));
523 assert_eq!(config[MODE_KEY], TestMode::Production);
524 assert_eq!(config[IP_KEY], Ipv4Addr::new(192, 168, 1, 1));
525 assert_eq!(
526 config[SYSTEMTIME_KEY],
527 std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1705314600) );
529
530 assert_eq!(config[NO_ENV_KEY], 999);
532
533 let expected_lines: HashSet<&str> = indoc! {"
534 # export TEST_USIZE_KEY=10
535 export TEST_USIZE_KEY=1024
536 # export TEST_STRING_KEY=''
537 export TEST_STRING_KEY=world
538 # export TEST_BOOL_KEY=0
539 export TEST_BOOL_KEY=1
540 # export TEST_I64_KEY=-42
541 export TEST_I64_KEY=-999
542 # export TEST_F64_KEY=3.14
543 export TEST_F64_KEY=2.718
544 # export TEST_U32_KEY=100
545 export TEST_U32_KEY=500
546 # export TEST_DURATION_KEY=1m
547 export TEST_DURATION_KEY=5s
548 # export TEST_MODE_KEY=dev
549 export TEST_MODE_KEY=prod
550 # export TEST_IP_KEY=127.0.0.1
551 export TEST_IP_KEY=192.168.1.1
552 "}
553 .trim_end()
554 .lines()
555 .collect();
556
557 logs_assert_unscoped(|logged_lines: &[&str]| {
561 let mut expected_lines = expected_lines.clone(); for logged in logged_lines {
563 expected_lines.remove(logged);
564 }
565
566 if expected_lines.is_empty() {
567 Ok(())
568 } else {
569 Err(format!("missing log lines: {:?}", expected_lines))
570 }
571 });
572
573 unsafe { std::env::remove_var("TEST_USIZE_KEY") };
576 unsafe { std::env::remove_var("TEST_STRING_KEY") };
578 unsafe { std::env::remove_var("TEST_BOOL_KEY") };
580 unsafe { std::env::remove_var("TEST_I64_KEY") };
582 unsafe { std::env::remove_var("TEST_F64_KEY") };
584 unsafe { std::env::remove_var("TEST_U32_KEY") };
586 unsafe { std::env::remove_var("TEST_DURATION_KEY") };
588 unsafe { std::env::remove_var("TEST_MODE_KEY") };
590 unsafe { std::env::remove_var("TEST_IP_KEY") };
592 unsafe { std::env::remove_var("TEST_SYSTEMTIME_KEY") };
594 }
595
596 #[test]
597 fn test_yaml_round_trip() {
598 let temp_path = std::env::temp_dir().join("test_config.yaml");
599
600 let mut config = crate::Attrs::new();
601 config.set(USIZE_KEY, 2048);
602 config.set(STRING_KEY, "hello_yaml".to_string());
603 config.set(BOOL_KEY, true);
604 config.set(I64_KEY, -123);
605 config.set(F64_KEY, 1.414);
606 config.set(U32_KEY, 777);
607 config.set(DURATION_KEY, std::time::Duration::from_mins(2));
608 config.set(MODE_KEY, TestMode::Staging);
609 config.set(IP_KEY, Ipv4Addr::new(10, 0, 0, 1));
610 config.set(
611 SYSTEMTIME_KEY,
612 std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1609459200),
613 );
614
615 to_yaml(&config, &temp_path).unwrap();
616
617 let yaml_content = std::fs::read_to_string(&temp_path).unwrap();
618
619 eprintln!("YAML content:\n{}", yaml_content);
620
621 assert!(yaml_content.contains("2048"));
622 assert!(yaml_content.contains("hello_yaml"));
623 assert!(yaml_content.contains("Staging"));
624
625 let loaded_config = from_yaml(&temp_path).unwrap();
626
627 assert_eq!(loaded_config[USIZE_KEY], 2048);
628 assert_eq!(loaded_config[STRING_KEY], "hello_yaml");
629 assert!(loaded_config[BOOL_KEY]);
630 assert_eq!(loaded_config[I64_KEY], -123);
631 assert_eq!(loaded_config[F64_KEY], 1.414);
632 assert_eq!(loaded_config[U32_KEY], 777);
633 assert_eq!(
634 loaded_config[DURATION_KEY],
635 std::time::Duration::from_mins(2)
636 );
637 assert_eq!(loaded_config[MODE_KEY], TestMode::Staging);
638 assert_eq!(loaded_config[IP_KEY], Ipv4Addr::new(10, 0, 0, 1));
639 assert_eq!(
640 loaded_config[SYSTEMTIME_KEY],
641 std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1609459200)
642 );
643
644 let _ = std::fs::remove_file(&temp_path);
645 }
646
647 #[test]
648 fn test_nonzero_usize_attr_value() {
649 let value = NonZeroUsize::parse("16").unwrap();
650 assert_eq!(value.get(), 16);
651 assert_eq!(value.display(), "16");
652 assert_eq!(usize::from(value), 16);
653 assert_eq!(
654 std::num::NonZeroUsize::from(value),
655 std::num::NonZeroUsize::new(16).unwrap()
656 );
657 assert_eq!(Option::<NonZeroUsize>::parse("").unwrap(), None);
658 assert_eq!(
659 Option::<NonZeroUsize>::parse("16")
660 .unwrap()
661 .map(NonZeroUsize::get),
662 Some(16)
663 );
664 assert!(NonZeroUsize::try_from(0).is_err());
665 assert!(NonZeroUsize::parse("0").is_err());
666 assert!(Option::<NonZeroUsize>::parse("0").is_err());
667 }
668
669 #[test]
679 fn test_introspect_meta_key() {
680 use crate::INTROSPECT;
681 use crate::IntrospectAttr;
682
683 declare_attrs! {
684 @meta(INTROSPECT = IntrospectAttr {
685 name: "test_key".into(),
686 desc: "A test introspection key".into(),
687 })
688 attr TEST_INTROSPECT_KEY: String;
689 }
690 let meta = TEST_INTROSPECT_KEY.attrs();
691 let introspect = meta
692 .get(INTROSPECT)
693 .expect("INTROSPECT meta-attr should be set");
694 assert_eq!(introspect.name, "test_key");
695 assert_eq!(introspect.desc, "A test introspection key");
696 }
697}