{ "meta": { "reportDate": "2026-10-16", "hardware": "NVIDIA GB300", "cuda": "13.0", "T": 2048, "D": 7168, "E": 384, "K": 6, "F": 3072, "sharedExperts": 1, "sharedPrecision": "BF16", "windows": 5, "iterationsPerWindow": 30, "warmups": 10, "acceptedExecutions": 110, "rankTraces": 3960, "matchedWorkloads": 44, "distPipelinePairs": 66, "balancedRows": 12288, "profiles": { "A": [ 7.5, 2.5, 4.5, 0.6, 1.8, 0.9, 2.2, 0.7, 1.6, 0.8 ], "B": [ 0.7, 5.8, 1.2, 0.6, 3.6, 1, 7.2, 0.8, 2.3, 0.7 ] } }, "cases": [ { "id": "mok-inference-ep16-bf16-native-single_layer-i1-F3072", "backend": "mok", "mode": "inference", "ep": 16, "precision": "bf16", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 1.6715509176254273, "forwardMS": 1.669378133614858, "backwardMS": null, "usefulTFPerGPU": 1133.127658609822, "meanTXGBps": 194.51260901398047, "meanRXGBps": 194.94566377160004, "modeledHBMGBps": 2239.516267513941, "peakAllocatedGiB": 13.062159061431885, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-inference-ep16-bf16-native-single_layer-i4-F3072", "backend": "mok", "mode": "inference", "ep": 16, "precision": "bf16", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 4.0, "profile": null, "layers": 1, "iterationMS": 4.9203754425048825, "forwardMS": 4.9182570298512776, "backwardMS": null, "usefulTFPerGPU": 384.94635209620395, "meanTXGBps": 67.86348803914717, "meanRXGBps": 68.00574402253062, "modeledHBMGBps": 760.8089089425793, "peakAllocatedGiB": 13.062166690826416, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-inference-ep16-bf16-native-single_layer-i8-F3072", "backend": "mok", "mode": "inference", "ep": 16, "precision": "bf16", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 8.0, "profile": null, "layers": 1, "iterationMS": 9.400529098510741, "forwardMS": 9.398408476511637, "backwardMS": null, "usefulTFPerGPU": 201.48659268934827, "meanTXGBps": 35.763429713986966, "meanRXGBps": 35.8365001187476, "modeledHBMGBps": 398.21859309951503, "peakAllocatedGiB": 13.062174320220947, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-inference-ep16-bf16-native-width-i1-F1536", "backend": "mok", "mode": "inference", "ep": 16, "precision": "bf16", "pipeline": "native", "category": "width", "F": 1536, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 1.0702122608820597, "forwardMS": 1.0680853247642517, "backwardMS": null, "usefulTFPerGPU": 884.9088385396161, "meanTXGBps": 305.8452380790594, "meanRXGBps": 306.52906895452656, "modeledHBMGBps": 1872.41163388319, "peakAllocatedGiB": 9.238916873931885, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-inference-ep16-bf16-native-width-i1-F3072", "backend": "mok", "mode": "inference", "ep": 16, "precision": "bf16", "pipeline": "native", "category": "width", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 1.6637120048205059, "forwardMS": 1.6615050673484801, "backwardMS": null, "usefulTFPerGPU": 1138.4666168471558, "meanTXGBps": 192.68035127589158, "meanRXGBps": 193.1100662650084, "modeledHBMGBps": 2250.0681975928123, "peakAllocatedGiB": 13.062181949615479, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-inference-ep16-bf16-native-width-i1-F6144", "backend": "mok", "mode": "inference", "ep": 16, "precision": "bf16", "pipeline": "native", "category": "width", "F": 6144, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 2.9354037046432495, "forwardMS": 2.9331850449244183, "backwardMS": null, "usefulTFPerGPU": 1290.5077243991518, "meanTXGBps": 111.78113039028469, "meanRXGBps": 112.03027543462287, "modeledHBMGBps": 2460.52719378093, "peakAllocatedGiB": 20.708643436431885, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep16-bf16-standard-single_layer-i1-F3072", "backend": "dist_moe", "mode": "inference", "ep": 16, "precision": "bf16", "pipeline": "standard", "category": "single_layer", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 1.5077610651652018, "forwardMS": 1.505510401725769, "backwardMS": null, "usefulTFPerGPU": 1256.2206448330526, "meanTXGBps": 213.69504387269245, "meanRXGBps": 213.69480145762518, "modeledHBMGBps": 2482.797545637536, "peakAllocatedGiB": 6.459131240844727, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep16-bf16-standard-single_layer-i4-F3072", "backend": "dist_moe", "mode": "inference", "ep": 16, "precision": "bf16", "pipeline": "standard", "category": "single_layer", "F": 3072, "imbalance": 4.0, "profile": null, "layers": 1, "iterationMS": 4.443144528071086, "forwardMS": 4.440957864125569, "backwardMS": null, "usefulTFPerGPU": 426.2928125721542, "meanTXGBps": 74.70760905673686, "meanRXGBps": 74.69382040844467, "modeledHBMGBps": 842.52615424715, "peakAllocatedGiB": 6.459134101867676, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep16-bf16-standard-single_layer-i8-F3072", "backend": "dist_moe", "mode": "inference", "ep": 16, "precision": "bf16", "pipeline": "standard", "category": "single_layer", "F": 3072, "imbalance": 8.0, "profile": null, "layers": 1, "iterationMS": 8.481332270304362, "forwardMS": 8.479137134552001, "backwardMS": null, "usefulTFPerGPU": 223.32347291329845, "meanTXGBps": 39.68064133004407, "meanRXGBps": 39.71918529810548, "modeledHBMGBps": 441.37705642154515, "peakAllocatedGiB": 6.459136962890625, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep16-bf16-standard-width-i1-F1536", "backend": "dist_moe", "mode": "inference", "ep": 16, "precision": "bf16", "pipeline": "standard", "category": "width", "F": 1536, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 0.9688565333684286, "forwardMS": 0.9666570663452149, "backwardMS": null, "usefulTFPerGPU": 977.4824818236195, "meanTXGBps": 337.58947435168176, "meanRXGBps": 337.5892919540424, "modeledHBMGBps": 2068.2916603071326, "peakAllocatedGiB": 4.077295303344727, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep16-bf16-standard-width-i1-F3072", "backend": "dist_moe", "mode": "inference", "ep": 16, "precision": "bf16", "pipeline": "standard", "category": "width", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 1.494101333618164, "forwardMS": 1.4917898615201315, "backwardMS": null, "usefulTFPerGPU": 1267.7055664954353, "meanTXGBps": 215.9565357159041, "meanRXGBps": 215.95650462812517, "modeledHBMGBps": 2505.4963728160947, "peakAllocatedGiB": 6.459139823913574, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep16-bf16-standard-width-i1-F6144", "backend": "dist_moe", "mode": "inference", "ep": 16, "precision": "bf16", "pipeline": "standard", "category": "width", "F": 6144, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 2.7351711988449097, "forwardMS": 2.7329045295715333, "backwardMS": null, "usefulTFPerGPU": 1384.9813703331545, "meanTXGBps": 118.65799327304867, "meanRXGBps": 118.65834228201328, "modeledHBMGBps": 2640.653953599027, "peakAllocatedGiB": 11.222803115844727, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep16-mxfp8-mega-single_layer-i1-F3072", "backend": "dist_moe", "mode": "inference", "ep": 16, "precision": "mxfp8", "pipeline": "mega", "category": "single_layer", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 1.0116458674271902, "forwardMS": 1.0094784080982209, "backwardMS": null, "usefulTFPerGPU": 1872.2762960056475, "meanTXGBps": 238.61710452411782, "meanRXGBps": 238.6169457027265, "modeledHBMGBps": 2182.1507378014194, "peakAllocatedGiB": 7.228292465209961, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep16-mxfp8-mega-single_layer-i4-F3072", "backend": "dist_moe", "mode": "inference", "ep": 16, "precision": "mxfp8", "pipeline": "mega", "category": "single_layer", "F": 3072, "imbalance": 4.0, "profile": null, "layers": 1, "iterationMS": 2.703762110074361, "forwardMS": 2.7015359958012897, "backwardMS": null, "usefulTFPerGPU": 700.5352173841609, "meanTXGBps": 92.13982626541791, "meanRXGBps": 92.10172108190467, "modeledHBMGBps": 816.4785532626929, "peakAllocatedGiB": 7.22829532623291, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep16-mxfp8-mega-single_layer-i8-F3072", "backend": "dist_moe", "mode": "inference", "ep": 16, "precision": "mxfp8", "pipeline": "mega", "category": "single_layer", "F": 3072, "imbalance": 8.0, "profile": null, "layers": 1, "iterationMS": 4.9685546875, "forwardMS": 4.966378657023112, "backwardMS": null, "usefulTFPerGPU": 381.2135916106891, "meanTXGBps": 50.9920206114828, "meanRXGBps": 50.97785753009126, "modeledHBMGBps": 444.30702988010535, "peakAllocatedGiB": 7.228298187255859, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep16-mxfp8-mega-width-i1-F1536", "backend": "dist_moe", "mode": "inference", "ep": 16, "precision": "mxfp8", "pipeline": "mega", "category": "width", "F": 1536, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 0.8253205299377442, "forwardMS": 0.8231765290101369, "backwardMS": null, "usefulTFPerGPU": 1147.4818018151536, "meanTXGBps": 299.9492899143489, "meanRXGBps": 299.9497054795852, "modeledHBMGBps": 1497.5115669220402, "peakAllocatedGiB": 4.186378479003906, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep16-mxfp8-mega-width-i1-F3072", "backend": "dist_moe", "mode": "inference", "ep": 16, "precision": "mxfp8", "pipeline": "mega", "category": "width", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 1.0179690678914388, "forwardMS": 1.015747199455897, "backwardMS": null, "usefulTFPerGPU": 1860.6464943569326, "meanTXGBps": 237.31117760937136, "meanRXGBps": 237.3099774642648, "modeledHBMGBps": 2168.5961250007504, "peakAllocatedGiB": 7.228301048278809, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep16-mxfp8-mega-width-i1-F6144", "backend": "dist_moe", "mode": "inference", "ep": 16, "precision": "mxfp8", "pipeline": "mega", "category": "width", "F": 6144, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 1.7821408033370971, "forwardMS": 1.7793557365735373, "backwardMS": null, "usefulTFPerGPU": 2125.623939465718, "meanTXGBps": 138.61159206764157, "meanRXGBps": 138.61215518068215, "modeledHBMGBps": 2329.1297972794673, "peakAllocatedGiB": 13.31212043762207, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "mok-inference-ep16-mxfp8-native-single_layer-i1-F3072", "backend": "mok", "mode": "inference", "ep": 16, "precision": "mxfp8", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 1.243453868230184, "forwardMS": 1.241322668393453, "backwardMS": null, "usefulTFPerGPU": 1523.2415338671608, "meanTXGBps": 262.6464184490944, "meanRXGBps": 263.23209065872334, "modeledHBMGBps": 1775.3483522006652, "peakAllocatedGiB": 14.994311332702637, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-inference-ep16-mxfp8-native-single_layer-i4-F3072", "backend": "mok", "mode": "inference", "ep": 16, "precision": "mxfp8", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 4.0, "profile": null, "layers": 1, "iterationMS": 3.463299202919006, "forwardMS": 3.4611861228942873, "backwardMS": null, "usefulTFPerGPU": 546.9006477810505, "meanTXGBps": 96.64107333783943, "meanRXGBps": 96.84406922812249, "modeledHBMGBps": 637.4164190432572, "peakAllocatedGiB": 14.994318962097168, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-inference-ep16-mxfp8-native-single_layer-i8-F3072", "backend": "mok", "mode": "inference", "ep": 16, "precision": "mxfp8", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 8.0, "profile": null, "layers": 1, "iterationMS": 6.5984660943349205, "forwardMS": 6.596349875132243, "backwardMS": null, "usefulTFPerGPU": 287.0486186421649, "meanTXGBps": 50.89465061168268, "meanRXGBps": 51.00583780136146, "modeledHBMGBps": 334.55711440198087, "peakAllocatedGiB": 14.9943265914917, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-inference-ep16-mxfp8-native-width-i1-F1536", "backend": "mok", "mode": "inference", "ep": 16, "precision": "mxfp8", "pipeline": "native", "category": "width", "F": 1536, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 1.004699742794037, "forwardMS": 1.002565332253774, "backwardMS": null, "usefulTFPerGPU": 942.6102629769887, "meanTXGBps": 326.8552212496704, "meanRXGBps": 327.58463566839674, "modeledHBMGBps": 1230.1456717436072, "peakAllocatedGiB": 9.794839859008789, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-inference-ep16-mxfp8-native-width-i1-F3072", "backend": "mok", "mode": "inference", "ep": 16, "precision": "mxfp8", "pipeline": "native", "category": "width", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 1.2424010634422302, "forwardMS": 1.2402837475140889, "backwardMS": null, "usefulTFPerGPU": 1524.5323215421345, "meanTXGBps": 263.2844701038325, "meanRXGBps": 263.87180094645555, "modeledHBMGBps": 1776.852774001709, "peakAllocatedGiB": 14.99433422088623, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-inference-ep16-mxfp8-native-width-i1-F6144", "backend": "mok", "mode": "inference", "ep": 16, "precision": "mxfp8", "pipeline": "native", "category": "width", "F": 6144, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 2.12193386554718, "forwardMS": 2.1197343905766806, "backwardMS": null, "usefulTFPerGPU": 1785.2399721681015, "meanTXGBps": 153.48095430716478, "meanRXGBps": 153.82376093317868, "modeledHBMGBps": 1956.1576896411095, "peakAllocatedGiB": 25.39325475692749, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep16-mxfp8-staged-single_layer-i1-F3072", "backend": "dist_moe", "mode": "inference", "ep": 16, "precision": "mxfp8", "pipeline": "staged", "category": "single_layer", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 1.0236405332883198, "forwardMS": 1.0214410622914631, "backwardMS": null, "usefulTFPerGPU": 1850.337609679736, "meanTXGBps": 236.53668725464166, "meanRXGBps": 236.53701697896497, "modeledHBMGBps": 2156.581049900859, "peakAllocatedGiB": 6.900919437408447, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep16-mxfp8-staged-single_layer-i4-F3072", "backend": "dist_moe", "mode": "inference", "ep": 16, "precision": "mxfp8", "pipeline": "staged", "category": "single_layer", "F": 3072, "imbalance": 4.0, "profile": null, "layers": 1, "iterationMS": 2.7799829324086507, "forwardMS": 2.77718292872111, "backwardMS": null, "usefulTFPerGPU": 681.3281317144341, "meanTXGBps": 89.25877710821285, "meanRXGBps": 89.20646957721101, "modeledHBMGBps": 794.0925644774762, "peakAllocatedGiB": 6.9009222984313965, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep16-mxfp8-staged-single_layer-i8-F3072", "backend": "dist_moe", "mode": "inference", "ep": 16, "precision": "mxfp8", "pipeline": "staged", "category": "single_layer", "F": 3072, "imbalance": 8.0, "profile": null, "layers": 1, "iterationMS": 5.163138103485108, "forwardMS": 5.160971736907959, "backwardMS": null, "usefulTFPerGPU": 366.8467779813016, "meanTXGBps": 48.94789442394608, "meanRXGBps": 48.90930546456842, "modeledHBMGBps": 427.5624110286531, "peakAllocatedGiB": 6.900925159454346, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep16-mxfp8-staged-width-i1-F1536", "backend": "dist_moe", "mode": "inference", "ep": 16, "precision": "mxfp8", "pipeline": "staged", "category": "width", "F": 1536, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 0.7512128015359243, "forwardMS": 0.7490602652231853, "backwardMS": null, "usefulTFPerGPU": 1260.6817759650637, "meanTXGBps": 330.8557079761134, "meanRXGBps": 330.8556372240554, "modeledHBMGBps": 1645.2422502292727, "peakAllocatedGiB": 4.022135257720947, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep16-mxfp8-staged-width-i1-F3072", "backend": "dist_moe", "mode": "inference", "ep": 16, "precision": "mxfp8", "pipeline": "staged", "category": "width", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 1.0270101388295492, "forwardMS": 1.0247349341710408, "backwardMS": null, "usefulTFPerGPU": 1844.2666785107142, "meanTXGBps": 236.7695055152275, "meanRXGBps": 236.76949888264346, "modeledHBMGBps": 2149.5053383950917, "peakAllocatedGiB": 6.900928020477295, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep16-mxfp8-staged-width-i1-F6144", "backend": "dist_moe", "mode": "inference", "ep": 16, "precision": "mxfp8", "pipeline": "staged", "category": "width", "F": 6144, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 1.8397141377131143, "forwardMS": 1.8369461297988892, "backwardMS": null, "usefulTFPerGPU": 2059.1031385892015, "meanTXGBps": 133.23586103729812, "meanRXGBps": 133.23441924558767, "modeledHBMGBps": 2256.2403380558694, "peakAllocatedGiB": 12.658487796783447, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "mok-inference-ep8-bf16-native-single_layer-i1-F3072", "backend": "mok", "mode": "inference", "ep": 8, "precision": "bf16", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 1.787875195344289, "forwardMS": 1.7856789231300354, "backwardMS": null, "usefulTFPerGPU": 1059.4031297420954, "meanTXGBps": 169.52262635223173, "meanRXGBps": 169.71243371672944, "modeledHBMGBps": 3867.3612755550926, "peakAllocatedGiB": 16.014551639556885, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-inference-ep8-bf16-native-single_layer-i4-F3072", "backend": "mok", "mode": "inference", "ep": 8, "precision": "bf16", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 4.0, "profile": null, "layers": 1, "iterationMS": 5.0953877131144205, "forwardMS": 5.0932085355122885, "backwardMS": null, "usefulTFPerGPU": 371.7245250368384, "meanTXGBps": 60.66189378036346, "meanRXGBps": 60.74762511727981, "modeledHBMGBps": 1356.983940241474, "peakAllocatedGiB": 16.014559268951416, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-inference-ep8-bf16-native-single_layer-i8-F3072", "backend": "mok", "mode": "inference", "ep": 8, "precision": "bf16", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 8.0, "profile": null, "layers": 1, "iterationMS": 9.61128740310669, "forwardMS": 9.609075228373209, "backwardMS": null, "usefulTFPerGPU": 197.06835287474286, "meanTXGBps": 32.6805128932596, "meanRXGBps": 32.70121549663304, "modeledHBMGBps": 142.0512202723946, "peakAllocatedGiB": 16.015299320220947, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep8-bf16-standard-single_layer-i1-F3072", "backend": "dist_moe", "mode": "inference", "ep": 8, "precision": "bf16", "pipeline": "standard", "category": "single_layer", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 1.664148251215617, "forwardMS": 1.6618794679641724, "backwardMS": null, "usefulTFPerGPU": 1138.1681747118525, "meanTXGBps": 182.72546402007197, "meanRXGBps": 182.72519194189806, "modeledHBMGBps": 4154.893826886661, "peakAllocatedGiB": 9.412256240844727, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep8-bf16-standard-single_layer-i4-F3072", "backend": "dist_moe", "mode": "inference", "ep": 8, "precision": "bf16", "pipeline": "standard", "category": "single_layer", "F": 3072, "imbalance": 4.0, "profile": null, "layers": 1, "iterationMS": 4.7187829653422035, "forwardMS": 4.716485341389974, "backwardMS": null, "usefulTFPerGPU": 401.3917553418654, "meanTXGBps": 65.08478593965845, "meanRXGBps": 65.08315096134613, "modeledHBMGBps": 1465.2844487198352, "peakAllocatedGiB": 9.412259101867676, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep8-bf16-standard-single_layer-i8-F3072", "backend": "dist_moe", "mode": "inference", "ep": 8, "precision": "bf16", "pipeline": "standard", "category": "single_layer", "F": 3072, "imbalance": 8.0, "profile": null, "layers": 1, "iterationMS": 8.988277339935303, "forwardMS": 8.985945701599121, "backwardMS": null, "usefulTFPerGPU": 210.7278743080744, "meanTXGBps": 34.836528968126586, "meanRXGBps": 34.86877732927121, "modeledHBMGBps": 151.89730494117433, "peakAllocatedGiB": 9.412261962890625, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep8-mxfp8-mega-single_layer-i1-F3072", "backend": "dist_moe", "mode": "inference", "ep": 8, "precision": "mxfp8", "pipeline": "mega", "category": "single_layer", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 1.010262397925059, "forwardMS": 1.0080469310283662, "backwardMS": null, "usefulTFPerGPU": 1874.8402211407483, "meanTXGBps": 222.097816157575, "meanRXGBps": 222.0968838760028, "modeledHBMGBps": 3803.522641139654, "peakAllocatedGiB": 11.772241592407227, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep8-mxfp8-mega-single_layer-i4-F3072", "backend": "dist_moe", "mode": "inference", "ep": 8, "precision": "mxfp8", "pipeline": "mega", "category": "single_layer", "F": 3072, "imbalance": 4.0, "profile": null, "layers": 1, "iterationMS": 2.7500863949457806, "forwardMS": 2.74781973361969, "backwardMS": null, "usefulTFPerGPU": 688.7349361158317, "meanTXGBps": 85.7175328849763, "meanRXGBps": 85.2095117542776, "modeledHBMGBps": 1397.2491595398617, "peakAllocatedGiB": 11.772244453430176, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep8-mxfp8-mega-single_layer-i8-F3072", "backend": "dist_moe", "mode": "inference", "ep": 8, "precision": "mxfp8", "pipeline": "mega", "category": "single_layer", "F": 3072, "imbalance": 8.0, "profile": null, "layers": 1, "iterationMS": 4.988869349161784, "forwardMS": 4.98656636873881, "backwardMS": null, "usefulTFPerGPU": 379.66129096049355, "meanTXGBps": 47.53677564691786, "meanRXGBps": 47.56611582147786, "modeledHBMGBps": 196.70181985521, "peakAllocatedGiB": 11.772247314453125, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "mok-inference-ep8-mxfp8-native-single_layer-i1-F3072", "backend": "mok", "mode": "inference", "ep": 8, "precision": "mxfp8", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 1.3321663896242777, "forwardMS": 1.330029877026876, "backwardMS": null, "usefulTFPerGPU": 1421.8048077839615, "meanTXGBps": 229.69714227408338, "meanRXGBps": 229.95394065433854, "modeledHBMGBps": 2884.44141357128, "peakAllocatedGiB": 20.992103099822998, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-inference-ep8-mxfp8-native-single_layer-i4-F3072", "backend": "mok", "mode": "inference", "ep": 8, "precision": "mxfp8", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 4.0, "profile": null, "layers": 1, "iterationMS": 3.540881069501241, "forwardMS": 3.5387210766474406, "backwardMS": null, "usefulTFPerGPU": 534.9178750594962, "meanTXGBps": 87.62804548332966, "meanRXGBps": 87.69303099552808, "modeledHBMGBps": 1085.1976749790279, "peakAllocatedGiB": 20.99211072921753, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-inference-ep8-mxfp8-native-single_layer-i8-F3072", "backend": "mok", "mode": "inference", "ep": 8, "precision": "mxfp8", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 8.0, "profile": null, "layers": 1, "iterationMS": 6.610096025466919, "forwardMS": 6.607921075820923, "backwardMS": null, "usefulTFPerGPU": 286.5435797360005, "meanTXGBps": 47.465586625847514, "meanRXGBps": 47.504334052390156, "modeledHBMGBps": 148.45770412702626, "peakAllocatedGiB": 20.9928617477417, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep8-mxfp8-staged-single_layer-i1-F3072", "backend": "dist_moe", "mode": "inference", "ep": 8, "precision": "mxfp8", "pipeline": "staged", "category": "single_layer", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 1.0744874715805053, "forwardMS": 1.0722901344299316, "backwardMS": null, "usefulTFPerGPU": 1762.7758607086628, "meanTXGBps": 210.38791397755767, "meanRXGBps": 210.38696454299998, "modeledHBMGBps": 3576.175623851468, "peakAllocatedGiB": 11.434977054595947, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep8-mxfp8-staged-single_layer-i4-F3072", "backend": "dist_moe", "mode": "inference", "ep": 8, "precision": "mxfp8", "pipeline": "staged", "category": "single_layer", "F": 3072, "imbalance": 4.0, "profile": null, "layers": 1, "iterationMS": 2.830664539337158, "forwardMS": 2.8283498764038084, "backwardMS": null, "usefulTFPerGPU": 669.1292985143082, "meanTXGBps": 82.10512612167761, "meanRXGBps": 82.10745524511582, "modeledHBMGBps": 1357.4748440165895, "peakAllocatedGiB": 11.434979915618896, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-inference-ep8-mxfp8-staged-single_layer-i8-F3072", "backend": "dist_moe", "mode": "inference", "ep": 8, "precision": "mxfp8", "pipeline": "staged", "category": "single_layer", "F": 3072, "imbalance": 8.0, "profile": null, "layers": 1, "iterationMS": 5.111320543289184, "forwardMS": 5.109170166651408, "backwardMS": null, "usefulTFPerGPU": 370.56579830877536, "meanTXGBps": 45.98521161511446, "meanRXGBps": 45.97940000643108, "modeledHBMGBps": 191.98946176216748, "peakAllocatedGiB": 11.434982776641846, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "mok-training-ep32-bf16-native-packing-A-F3072", "backend": "mok", "mode": "training", "ep": 32, "precision": "bf16", "pipeline": "native", "category": "packing", "F": 3072, "imbalance": 1.0, "profile": "A", "layers": 10, "iterationMS": 102.78860524495443, "forwardMS": 30.803587086995442, "backwardMS": 71.98617172241211, "usefulTFPerGPU": 552.8085257180707, "meanTXGBps": 68.50862584922736, "meanRXGBps": 68.65453702711864, "modeledHBMGBps": 629.8417674383019, "peakAllocatedGiB": 71.86364412307739, "peakReservedBytes": 98329165824, "fastMath": false, "sharedSchedule": "concurrent", "capacity": { "requestedRowsPerLayer": 30720, "retainedRowsPerLayer": 32768, "minibatchRows": 4096, "effectiveFactor": 2.6666666666666665, "rank0ScheduleRows": [ 92160, 30720, 55296, 9216, 24576, 12288, 27648, 9216, 21504, 12288 ], "rank0ReplayRows": [ 59392, 0, 22528, 0, 0, 0, 0, 0, 0, 0 ], "totalReplayRows": 81920, "rankLayerReplayCount": 2, "sourcePolicyDerived": true } }, { "id": "mok-training-ep32-bf16-native-packing-B-F3072", "backend": "mok", "mode": "training", "ep": 32, "precision": "bf16", "pipeline": "native", "category": "packing", "F": 3072, "imbalance": 1.0, "profile": "B", "layers": 10, "iterationMS": 110.35971298217774, "forwardMS": 31.99002653757731, "backwardMS": 78.34827982584635, "usefulTFPerGPU": 514.8836997723653, "meanTXGBps": 63.791216604191945, "meanRXGBps": 63.92944735789125, "modeledHBMGBps": 586.6321599663377, "peakAllocatedGiB": 71.86365175247192, "peakReservedBytes": 98329165824, "fastMath": false, "sharedSchedule": "concurrent", "capacity": { "requestedRowsPerLayer": 30720, "retainedRowsPerLayer": 32768, "minibatchRows": 4096, "effectiveFactor": 2.6666666666666665, "rank0ScheduleRows": [ 9216, 73728, 15360, 9216, 46080, 12288, 89088, 12288, 30720, 9216 ], "rank0ReplayRows": [ 0, 40960, 0, 0, 13312, 0, 56320, 0, 0, 0 ], "totalReplayRows": 110592, "rankLayerReplayCount": 3, "sourcePolicyDerived": true } }, { "id": "mok-training-ep32-bf16-native-single_layer-i1-F3072", "backend": "mok", "mode": "training", "ep": 32, "precision": "bf16", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 4.483590412139892, "forwardMS": 1.5771477341651916, "backwardMS": 2.90733118057251, "usefulTFPerGPU": 1267.3418422036514, "meanTXGBps": 152.92921558789257, "meanRXGBps": 153.27038865018963, "modeledHBMGBps": 1443.9444920014703, "peakAllocatedGiB": 18.84887170791626, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-training-ep32-bf16-native-single_layer-i4-F3072", "backend": "mok", "mode": "training", "ep": 32, "precision": "bf16", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 4.0, "profile": null, "layers": 1, "iterationMS": 14.412835184733073, "forwardMS": 4.966465044021606, "backwardMS": 9.447564792633056, "usefulTFPerGPU": 394.2487137178233, "meanTXGBps": 48.71711764349094, "meanRXGBps": 48.81433902809871, "modeledHBMGBps": 449.1868252859578, "peakAllocatedGiB": 18.84887933731079, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-training-ep32-bf16-native-single_layer-i8-F3072", "backend": "mok", "mode": "training", "ep": 32, "precision": "bf16", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 8.0, "profile": null, "layers": 1, "iterationMS": 27.804005177815757, "forwardMS": 9.533583068847657, "backwardMS": 18.271653175354004, "usefulTFPerGPU": 204.36774113183318, "meanTXGBps": 25.08527313179457, "meanRXGBps": 25.137203366963497, "modeledHBMGBps": 232.8461543074922, "peakAllocatedGiB": 18.848886966705322, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep32-bf16-standard-packing-A-F3072", "backend": "dist_moe", "mode": "training", "ep": 32, "precision": "bf16", "pipeline": "standard", "category": "packing", "F": 3072, "imbalance": 1.0, "profile": "A", "layers": 10, "iterationMS": 84.58301137288412, "forwardMS": 29.94110825856527, "backwardMS": 54.64611625671387, "usefulTFPerGPU": 671.7946831613554, "meanTXGBps": 82.77924404604461, "meanRXGBps": 82.78052999226226, "modeledHBMGBps": 765.4085111085881, "peakAllocatedGiB": 69.13515949249268, "peakReservedBytes": 97693728768, "fastMath": false, "sharedSchedule": "concurrent", "capacity": { "slotBytes": 12142510080, "slotFactor": 2.5, "scratchBytes": 5838471168, "scratchFactor": 8.0, "hostScratchBytes": 0, "slots": 1, "savedRank0Bytes": 9318909952, "maxSavedRankBytes": 9318909952, "capacityRecomputeFlags": 0, "inspectedRankLayers": 320 } }, { "id": "dist_moe-training-ep32-bf16-standard-packing-B-F3072", "backend": "dist_moe", "mode": "training", "ep": 32, "precision": "bf16", "pipeline": "standard", "category": "packing", "F": 3072, "imbalance": 1.0, "profile": "B", "layers": 10, "iterationMS": 88.25528106689453, "forwardMS": 31.24911263783773, "backwardMS": 57.014715830485024, "usefulTFPerGPU": 643.8415541729512, "meanTXGBps": 79.25045487672756, "meanRXGBps": 79.25046257053836, "modeledHBMGBps": 733.5601452668747, "peakAllocatedGiB": 69.13516235351562, "peakReservedBytes": 97630814208, "fastMath": false, "sharedSchedule": "concurrent", "capacity": { "slotBytes": 12142510080, "slotFactor": 2.5, "scratchBytes": 5838471168, "scratchFactor": 8.0, "hostScratchBytes": 0, "slots": 1, "savedRank0Bytes": 9580650496, "maxSavedRankBytes": 9580650496, "capacityRecomputeFlags": 0, "inspectedRankLayers": 320 } }, { "id": "dist_moe-training-ep32-bf16-standard-single_layer-i1-F3072", "backend": "dist_moe", "mode": "training", "ep": 32, "precision": "bf16", "pipeline": "standard", "category": "single_layer", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 4.198279476165771, "forwardMS": 1.5382431983947753, "backwardMS": 2.716723171869914, "usefulTFPerGPU": 1353.4691448882556, "meanTXGBps": 164.71379328065413, "meanRXGBps": 164.7141555090394, "modeledHBMGBps": 1542.0735367319242, "peakAllocatedGiB": 14.431884765625, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep32-bf16-standard-single_layer-i4-F3072", "backend": "dist_moe", "mode": "training", "ep": 32, "precision": "bf16", "pipeline": "standard", "category": "single_layer", "F": 3072, "imbalance": 4.0, "profile": null, "layers": 1, "iterationMS": 13.03269001642863, "forwardMS": 4.519406938552857, "backwardMS": 8.565122111638386, "usefulTFPerGPU": 435.9991471787583, "meanTXGBps": 53.49219607113179, "meanRXGBps": 53.48909967756661, "modeledHBMGBps": 496.7551343459404, "peakAllocatedGiB": 14.43188762664795, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep32-bf16-standard-single_layer-i8-F3072", "backend": "dist_moe", "mode": "training", "ep": 32, "precision": "bf16", "pipeline": "standard", "category": "single_layer", "F": 3072, "imbalance": 8.0, "profile": null, "layers": 1, "iterationMS": 25.07634245554606, "forwardMS": 8.511569054921468, "backwardMS": 16.570217641194663, "usefulTFPerGPU": 226.59770828545513, "meanTXGBps": 27.820978210319844, "meanRXGBps": 27.807646000556677, "modeledHBMGBps": 258.1738421971563, "peakAllocatedGiB": 14.431890487670898, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep32-mxfp8-mega-packing-A-F3072", "backend": "dist_moe", "mode": "training", "ep": 32, "precision": "mxfp8", "pipeline": "mega", "category": "packing", "F": 3072, "imbalance": 1.0, "profile": "A", "layers": 10, "iterationMS": 55.833187103271484, "forwardMS": 21.050084495544432, "backwardMS": 34.96414705912272, "usefulTFPerGPU": 1017.7176026326205, "meanTXGBps": 124.93995440383753, "meanRXGBps": 124.94444553220093, "modeledHBMGBps": 746.9040103848292, "peakAllocatedGiB": 76.07246160507202, "peakReservedBytes": 105746792448, "fastMath": true, "sharedSchedule": "concurrent", "capacity": { "slotBytes": 11786772480, "slotFactor": 2.5, "scratchBytes": 5408378880, "scratchFactor": 8.0, "hostScratchBytes": 0, "slots": 1, "savedRank0Bytes": 8359378944, "maxSavedRankBytes": 8359378944, "capacityRecomputeFlags": 0, "inspectedRankLayers": 320 } }, { "id": "dist_moe-training-ep32-mxfp8-mega-packing-B-F3072", "backend": "dist_moe", "mode": "training", "ep": 32, "precision": "mxfp8", "pipeline": "mega", "category": "packing", "F": 3072, "imbalance": 1.0, "profile": "B", "layers": 10, "iterationMS": 57.623893356323244, "forwardMS": 21.83856970469157, "backwardMS": 36.00784861246745, "usefulTFPerGPU": 986.0912551450275, "meanTXGBps": 120.92069438729355, "meanRXGBps": 120.92377977519321, "modeledHBMGBps": 723.6934009670472, "peakAllocatedGiB": 76.07246446609497, "peakReservedBytes": 105434316800, "fastMath": true, "sharedSchedule": "concurrent", "capacity": { "slotBytes": 11786772480, "slotFactor": 2.5, "scratchBytes": 5408378880, "scratchFactor": 8.0, "hostScratchBytes": 0, "slots": 1, "savedRank0Bytes": 8605040640, "maxSavedRankBytes": 8605040640, "capacityRecomputeFlags": 0, "inspectedRankLayers": 320 } }, { "id": "dist_moe-training-ep32-mxfp8-mega-single_layer-i1-F3072", "backend": "dist_moe", "mode": "training", "ep": 32, "precision": "mxfp8", "pipeline": "mega", "category": "single_layer", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 3.123201060295105, "forwardMS": 1.2514901280403137, "backwardMS": 1.9649301369984944, "usefulTFPerGPU": 1819.3646911963767, "meanTXGBps": 215.51539574704833, "meanRXGBps": 215.5150223973928, "modeledHBMGBps": 1335.2336450622126, "peakAllocatedGiB": 14.6922287940979, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep32-mxfp8-mega-single_layer-i4-F3072", "backend": "dist_moe", "mode": "training", "ep": 32, "precision": "mxfp8", "pipeline": "mega", "category": "single_layer", "F": 3072, "imbalance": 4.0, "profile": null, "layers": 1, "iterationMS": 8.395078404744465, "forwardMS": 3.355942400296529, "backwardMS": 5.29029229482015, "usefulTFPerGPU": 676.8539206729374, "meanTXGBps": 82.61397204512566, "meanRXGBps": 82.6124618842168, "modeledHBMGBps": 496.74379856216905, "peakAllocatedGiB": 14.69223165512085, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep32-mxfp8-mega-single_layer-i8-F3072", "backend": "dist_moe", "mode": "training", "ep": 32, "precision": "mxfp8", "pipeline": "mega", "category": "single_layer", "F": 3072, "imbalance": 8.0, "profile": null, "layers": 1, "iterationMS": 15.711488024393718, "forwardMS": 6.021947765350342, "backwardMS": 9.920983441670735, "usefulTFPerGPU": 361.6615895188113, "meanTXGBps": 44.22167556215456, "meanRXGBps": 44.22602631025934, "modeledHBMGBps": 265.423817879333, "peakAllocatedGiB": 14.692234516143799, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "mok-training-ep32-mxfp8-native-packing-A-F3072", "backend": "mok", "mode": "training", "ep": 32, "precision": "mxfp8", "pipeline": "native", "category": "packing", "F": 3072, "imbalance": 1.0, "profile": "A", "layers": 10, "iterationMS": 74.3393564860026, "forwardMS": 23.48637440999349, "backwardMS": 50.85444564819336, "usefulTFPerGPU": 764.3652031986464, "meanTXGBps": 94.53150407335303, "meanRXGBps": 94.75207151535636, "modeledHBMGBps": 560.9684201107135, "peakAllocatedGiB": 78.10159111022949, "peakReservedBytes": 105795026944, "fastMath": false, "sharedSchedule": "concurrent", "capacity": { "requestedRowsPerLayer": 30720, "retainedRowsPerLayer": 32768, "minibatchRows": 4096, "effectiveFactor": 2.6666666666666665, "rank0ScheduleRows": [ 92160, 30720, 55296, 9216, 24576, 12288, 27648, 9216, 21504, 12288 ], "rank0ReplayRows": [ 59392, 0, 22528, 0, 0, 0, 0, 0, 0, 0 ], "totalReplayRows": 81920, "rankLayerReplayCount": 2, "sourcePolicyDerived": true } }, { "id": "mok-training-ep32-mxfp8-native-packing-B-F3072", "backend": "mok", "mode": "training", "ep": 32, "precision": "mxfp8", "pipeline": "native", "category": "packing", "F": 3072, "imbalance": 1.0, "profile": "B", "layers": 10, "iterationMS": 80.5498779296875, "forwardMS": 24.295484606424967, "backwardMS": 56.25278803507487, "usefulTFPerGPU": 705.4314517482031, "meanTXGBps": 87.45529067125614, "meanRXGBps": 87.64559547068068, "modeledHBMGBps": 517.7168784340303, "peakAllocatedGiB": 78.10159873962402, "peakReservedBytes": 105795026944, "fastMath": false, "sharedSchedule": "concurrent", "capacity": { "requestedRowsPerLayer": 30720, "retainedRowsPerLayer": 32768, "minibatchRows": 4096, "effectiveFactor": 2.6666666666666665, "rank0ScheduleRows": [ 9216, 73728, 15360, 9216, 46080, 12288, 89088, 12288, 30720, 9216 ], "rank0ReplayRows": [ 0, 40960, 0, 0, 13312, 0, 56320, 0, 0, 0 ], "totalReplayRows": 110592, "rankLayerReplayCount": 3, "sourcePolicyDerived": true } }, { "id": "mok-training-ep32-mxfp8-native-single_layer-i1-F3072", "backend": "mok", "mode": "training", "ep": 32, "precision": "mxfp8", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 3.369385592142741, "forwardMS": 1.2231263955434164, "backwardMS": 2.147494427363078, "usefulTFPerGPU": 1686.432608324419, "meanTXGBps": 204.2899064781516, "meanRXGBps": 204.74549304971777, "modeledHBMGBps": 1237.6746507507867, "peakAllocatedGiB": 19.37209701538086, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-training-ep32-mxfp8-native-single_layer-i4-F3072", "backend": "mok", "mode": "training", "ep": 32, "precision": "mxfp8", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 4.0, "profile": null, "layers": 1, "iterationMS": 9.534516334533691, "forwardMS": 3.5655797322591147, "backwardMS": 5.969721635182698, "usefulTFPerGPU": 595.9653886193593, "meanTXGBps": 72.75124402604712, "meanRXGBps": 72.91470419389026, "modeledHBMGBps": 437.37962049481916, "peakAllocatedGiB": 19.37210464477539, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-training-ep32-mxfp8-native-single_layer-i8-F3072", "backend": "mok", "mode": "training", "ep": 32, "precision": "mxfp8", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 8.0, "profile": null, "layers": 1, "iterationMS": 18.184922472635904, "forwardMS": 6.94224747021993, "backwardMS": 11.249847571055094, "usefulTFPerGPU": 312.4699454264079, "meanTXGBps": 38.32214830696627, "meanRXGBps": 38.40452661252416, "modeledHBMGBps": 229.32201895692378, "peakAllocatedGiB": 19.372112274169922, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep32-mxfp8-staged-packing-A-F3072", "backend": "dist_moe", "mode": "training", "ep": 32, "precision": "mxfp8", "pipeline": "staged", "category": "packing", "F": 3072, "imbalance": 1.0, "profile": "A", "layers": 10, "iterationMS": 56.21011390686035, "forwardMS": 20.246474520365396, "backwardMS": 36.12820154825847, "usefulTFPerGPU": 1010.89311827822, "meanTXGBps": 123.98236163247414, "meanRXGBps": 123.97704585883399, "modeledHBMGBps": 741.8955141969627, "peakAllocatedGiB": 75.75152349472046, "peakReservedBytes": 104907931648, "fastMath": true, "sharedSchedule": "concurrent", "capacity": { "slotBytes": 11786772480, "slotFactor": 2.5, "scratchBytes": 5063774208, "scratchFactor": 8.0, "hostScratchBytes": 0, "slots": 1, "savedRank0Bytes": 8359378944, "maxSavedRankBytes": 8359378944, "capacityRecomputeFlags": 0, "inspectedRankLayers": 320 } }, { "id": "dist_moe-training-ep32-mxfp8-staged-packing-B-F3072", "backend": "dist_moe", "mode": "training", "ep": 32, "precision": "mxfp8", "pipeline": "staged", "category": "packing", "F": 3072, "imbalance": 1.0, "profile": "B", "layers": 10, "iterationMS": 58.00989011128743, "forwardMS": 20.820731735229494, "backwardMS": 37.20214195251465, "usefulTFPerGPU": 979.5298218471132, "meanTXGBps": 119.04345062328181, "meanRXGBps": 119.06084968161134, "modeledHBMGBps": 718.8779582239841, "peakAllocatedGiB": 75.75152635574341, "peakReservedBytes": 104847114240, "fastMath": true, "sharedSchedule": "concurrent", "capacity": { "slotBytes": 11786772480, "slotFactor": 2.5, "scratchBytes": 5063774208, "scratchFactor": 8.0, "hostScratchBytes": 0, "slots": 1, "savedRank0Bytes": 8605040640, "maxSavedRankBytes": 8605040640, "capacityRecomputeFlags": 0, "inspectedRankLayers": 320 } }, { "id": "dist_moe-training-ep32-mxfp8-staged-single_layer-i1-F3072", "backend": "dist_moe", "mode": "training", "ep": 32, "precision": "mxfp8", "pipeline": "staged", "category": "single_layer", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 3.0559871832529706, "forwardMS": 1.1713087956110637, "backwardMS": 1.9698624014854431, "usefulTFPerGPU": 1859.380092870511, "meanTXGBps": 224.6352066728914, "meanRXGBps": 224.63592571667277, "modeledHBMGBps": 1364.6009901000282, "peakAllocatedGiB": 14.371290683746338, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep32-mxfp8-staged-single_layer-i4-F3072", "backend": "dist_moe", "mode": "training", "ep": 32, "precision": "mxfp8", "pipeline": "staged", "category": "single_layer", "F": 3072, "imbalance": 4.0, "profile": null, "layers": 1, "iterationMS": 8.623060321807861, "forwardMS": 3.2583199977874755, "backwardMS": 5.536523739496867, "usefulTFPerGPU": 658.9588290641452, "meanTXGBps": 81.00387750955038, "meanRXGBps": 81.00195433041483, "modeledHBMGBps": 483.6105721600356, "peakAllocatedGiB": 14.371293544769287, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep32-mxfp8-staged-single_layer-i8-F3072", "backend": "dist_moe", "mode": "training", "ep": 32, "precision": "mxfp8", "pipeline": "staged", "category": "single_layer", "F": 3072, "imbalance": 8.0, "profile": null, "layers": 1, "iterationMS": 16.010382843017577, "forwardMS": 5.835179726282756, "backwardMS": 10.32259407043457, "usefulTFPerGPU": 354.9097974934516, "meanTXGBps": 43.46486734926951, "meanRXGBps": 43.45276631800239, "modeledHBMGBps": 260.4686706675913, "peakAllocatedGiB": 14.371296405792236, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "mok-training-ep64-bf16-native-packing-A-F3072", "backend": "mok", "mode": "training", "ep": 64, "precision": "bf16", "pipeline": "native", "category": "packing", "F": 3072, "imbalance": 1.0, "profile": "A", "layers": 10, "iterationMS": 97.99274978637695, "forwardMS": 29.80009822845459, "backwardMS": 68.22826080322265, "usefulTFPerGPU": 579.8634842878907, "meanTXGBps": 72.58797395203167, "meanRXGBps": 72.91259116847009, "modeledHBMGBps": 417.97840359914204, "peakAllocatedGiB": 49.74450349807739, "peakReservedBytes": 64703430656, "fastMath": false, "sharedSchedule": "concurrent", "capacity": { "requestedRowsPerLayer": 30720, "retainedRowsPerLayer": 32768, "minibatchRows": 4096, "effectiveFactor": 2.6666666666666665, "rank0ScheduleRows": [ 92160, 30720, 55296, 7680, 23040, 12288, 27648, 9216, 19968, 10752 ], "rank0ReplayRows": [ 59392, 0, 22528, 0, 0, 0, 0, 0, 0, 0 ], "totalReplayRows": 81920, "rankLayerReplayCount": 2, "sourcePolicyDerived": true } }, { "id": "mok-training-ep64-bf16-native-packing-B-F3072", "backend": "mok", "mode": "training", "ep": 64, "precision": "bf16", "pipeline": "native", "category": "packing", "F": 3072, "imbalance": 1.0, "profile": "B", "layers": 10, "iterationMS": 104.94571406046549, "forwardMS": 30.836193211873372, "backwardMS": 74.11048482259115, "usefulTFPerGPU": 541.4458116254391, "meanTXGBps": 67.909178098486, "meanRXGBps": 68.21228962826574, "modeledHBMGBps": 390.28609683289363, "peakAllocatedGiB": 49.744511127471924, "peakReservedBytes": 64703430656, "fastMath": false, "sharedSchedule": "concurrent", "capacity": { "requestedRowsPerLayer": 30720, "retainedRowsPerLayer": 32768, "minibatchRows": 4096, "effectiveFactor": 2.6666666666666665, "rank0ScheduleRows": [ 9216, 72192, 15360, 7680, 44544, 12288, 89088, 10752, 29184, 9216 ], "rank0ReplayRows": [ 0, 39424, 0, 0, 11776, 0, 56320, 0, 0, 0 ], "totalReplayRows": 107520, "rankLayerReplayCount": 3, "sourcePolicyDerived": true } }, { "id": "mok-training-ep64-bf16-native-single_layer-i1-F3072", "backend": "mok", "mode": "training", "ep": 64, "precision": "bf16", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 4.320784044265747, "forwardMS": 1.5578442573547364, "backwardMS": 2.7614282608032226, "usefulTFPerGPU": 1315.0950555256486, "meanTXGBps": 161.1770729421647, "meanRXGBps": 161.8963525416307, "modeledHBMGBps": 947.9495549970339, "peakAllocatedGiB": 16.63695764541626, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-training-ep64-bf16-native-single_layer-i4-F3072", "backend": "mok", "mode": "training", "ep": 64, "precision": "bf16", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 4.0, "profile": null, "layers": 1, "iterationMS": 14.069906107584636, "forwardMS": 4.87127997080485, "backwardMS": 9.204820410410564, "usefulTFPerGPU": 403.85782884115235, "meanTXGBps": 50.1064888561508, "meanRXGBps": 50.33332166885287, "modeledHBMGBps": 291.1096407240443, "peakAllocatedGiB": 16.63696527481079, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-training-ep64-bf16-native-single_layer-i8-F3072", "backend": "mok", "mode": "training", "ep": 64, "precision": "bf16", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 8.0, "profile": null, "layers": 1, "iterationMS": 27.78475507100423, "forwardMS": 9.500474611918131, "backwardMS": 18.28512535095215, "usefulTFPerGPU": 204.50933319681863, "meanTXGBps": 25.494282002558915, "meanRXGBps": 25.610186347124408, "modeledHBMGBps": 147.4148431948715, "peakAllocatedGiB": 16.636972904205322, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-training-ep64-bf16-native-width-i1-F1536", "backend": "mok", "mode": "training", "ep": 64, "precision": "bf16", "pipeline": "native", "category": "width", "F": 1536, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 2.870951437950134, "forwardMS": 1.0658506592114767, "backwardMS": 1.80632746219635, "usefulTFPerGPU": 989.6095171615186, "meanTXGBps": 244.22990998716313, "meanRXGBps": 245.31959276630934, "modeledHBMGBps": 851.4174359372973, "peakAllocatedGiB": 11.91723108291626, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-training-ep64-bf16-native-width-i1-F3072", "backend": "mok", "mode": "training", "ep": 64, "precision": "bf16", "pipeline": "native", "category": "width", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 4.333810138702392, "forwardMS": 1.568504520257314, "backwardMS": 2.766218654314677, "usefulTFPerGPU": 1311.14228605994, "meanTXGBps": 161.24675535571583, "meanRXGBps": 161.96637649567543, "modeledHBMGBps": 945.1003114839658, "peakAllocatedGiB": 16.636980533599854, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-training-ep64-bf16-native-width-i1-F6144", "backend": "mok", "mode": "training", "ep": 64, "precision": "bf16", "pipeline": "native", "category": "width", "F": 6144, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 8.224929110209148, "forwardMS": 2.795353587468465, "backwardMS": 5.430443795522054, "usefulTFPerGPU": 1381.7120260781212, "meanTXGBps": 85.68211320244845, "meanRXGBps": 86.06383618676546, "modeledHBMGBps": 899.5700282469494, "peakAllocatedGiB": 26.07641077041626, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep64-bf16-standard-packing-A-F3072", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "bf16", "pipeline": "standard", "category": "packing", "F": 3072, "imbalance": 1.0, "profile": "A", "layers": 10, "iterationMS": 80.4921137491862, "forwardMS": 28.62822208404541, "backwardMS": 51.91948356628418, "usefulTFPerGPU": 705.9376959976341, "meanTXGBps": 88.0414573943102, "meanRXGBps": 88.04009155489122, "modeledHBMGBps": 508.85547927872756, "peakAllocatedGiB": 46.986721992492676, "peakReservedBytes": 63615008768, "fastMath": false, "sharedSchedule": "concurrent", "capacity": { "slotBytes": 12142510080, "slotFactor": 2.5, "scratchBytes": 5838471168, "scratchFactor": 8.0, "hostScratchBytes": 0, "slots": 1, "savedRank0Bytes": 9318909952, "maxSavedRankBytes": 9318909952, "capacityRecomputeFlags": 0, "inspectedRankLayers": 640 } }, { "id": "dist_moe-training-ep64-bf16-standard-packing-B-F3072", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "bf16", "pipeline": "standard", "category": "packing", "F": 3072, "imbalance": 1.0, "profile": "B", "layers": 10, "iterationMS": 83.66207122802734, "forwardMS": 29.627559407552084, "backwardMS": 54.06590957641602, "usefulTFPerGPU": 679.1897031954442, "meanTXGBps": 84.75375884071295, "meanRXGBps": 84.75376743668501, "modeledHBMGBps": 489.5749354371532, "peakAllocatedGiB": 46.986724853515625, "peakReservedBytes": 63573065728, "fastMath": false, "sharedSchedule": "concurrent", "capacity": { "slotBytes": 12142510080, "slotFactor": 2.5, "scratchBytes": 5838471168, "scratchFactor": 8.0, "hostScratchBytes": 0, "slots": 1, "savedRank0Bytes": 9580650496, "maxSavedRankBytes": 9580650496, "capacityRecomputeFlags": 0, "inspectedRankLayers": 640 } }, { "id": "dist_moe-training-ep64-bf16-standard-single_layer-i1-F3072", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "bf16", "pipeline": "standard", "category": "single_layer", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 4.12414186000824, "forwardMS": 1.5676991860071818, "backwardMS": 2.5875850836435954, "usefulTFPerGPU": 1377.7997764113397, "meanTXGBps": 169.37024769556024, "meanRXGBps": 169.3706810867693, "modeledHBMGBps": 993.148502411558, "peakAllocatedGiB": 12.217041015625, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep64-bf16-standard-single_layer-i4-F3072", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "bf16", "pipeline": "standard", "category": "single_layer", "F": 3072, "imbalance": 4.0, "profile": null, "layers": 1, "iterationMS": 12.713062381744384, "forwardMS": 4.396715720494588, "backwardMS": 8.327931785583496, "usefulTFPerGPU": 446.9608943921762, "meanTXGBps": 55.58293772999353, "meanRXGBps": 55.583051529145436, "modeledHBMGBps": 322.17928214381936, "peakAllocatedGiB": 12.21704387664795, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep64-bf16-standard-single_layer-i8-F3072", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "bf16", "pipeline": "standard", "category": "single_layer", "F": 3072, "imbalance": 8.0, "profile": null, "layers": 1, "iterationMS": 24.46835091908773, "forwardMS": 8.346006472905477, "backwardMS": 16.127088006337484, "usefulTFPerGPU": 232.22822622571147, "meanTXGBps": 28.90565447606519, "meanRXGBps": 28.924486686219453, "modeledHBMGBps": 167.39523335856708, "peakAllocatedGiB": 12.217046737670898, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep64-bf16-standard-width-i1-F1536", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "bf16", "pipeline": "standard", "category": "width", "F": 1536, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 2.5324000120162964, "forwardMS": 0.9834250628948211, "backwardMS": 1.5503840049107869, "usefulTFPerGPU": 1121.9084081593808, "meanTXGBps": 274.3917437623349, "meanRXGBps": 274.3922070836073, "modeledHBMGBps": 965.2417076296673, "peakAllocatedGiB": 8.874267578125, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep64-bf16-standard-width-i1-F3072", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "bf16", "pipeline": "standard", "category": "width", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 4.0959391593933105, "forwardMS": 1.5564042687416078, "backwardMS": 2.5801589330037435, "usefulTFPerGPU": 1387.2866542894774, "meanTXGBps": 168.67515540329381, "meanRXGBps": 168.67653492636668, "modeledHBMGBps": 999.9868534684684, "peakAllocatedGiB": 12.217049598693848, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep64-bf16-standard-width-i1-F6144", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "bf16", "pipeline": "standard", "category": "width", "F": 6144, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 7.696775499979655, "forwardMS": 2.790287979443868, "backwardMS": 5.031280008951823, "usefulTFPerGPU": 1476.5252624616685, "meanTXGBps": 91.84089365081071, "meanRXGBps": 91.84059598772811, "modeledHBMGBps": 961.2986259011398, "peakAllocatedGiB": 18.902587890625, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep64-mxfp8-mega-packing-A-F3072", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "mega", "category": "packing", "F": 3072, "imbalance": 1.0, "profile": "A", "layers": 10, "iterationMS": 52.927848434448244, "forwardMS": 20.08404604593913, "backwardMS": 33.061967849731445, "usefulTFPerGPU": 1073.5826036166052, "meanTXGBps": 134.19533934463664, "meanRXGBps": 134.19789263421455, "modeledHBMGBps": 556.2211816802127, "peakAllocatedGiB": 49.72999334335327, "peakReservedBytes": 67054338048, "fastMath": true, "sharedSchedule": "concurrent", "capacity": { "slotBytes": 11348090880, "slotFactor": 2.5, "scratchBytes": 5366722560, "scratchFactor": 8.0, "hostScratchBytes": 0, "slots": 1, "savedRank0Bytes": 8306737152, "maxSavedRankBytes": 8306737152, "capacityRecomputeFlags": 0, "inspectedRankLayers": 640 } }, { "id": "dist_moe-training-ep64-mxfp8-mega-packing-B-F3072", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "mega", "category": "packing", "F": 3072, "imbalance": 1.0, "profile": "B", "layers": 10, "iterationMS": 54.63423233032226, "forwardMS": 20.762462933858234, "backwardMS": 33.98707516988119, "usefulTFPerGPU": 1040.0515373315366, "meanTXGBps": 129.9477586702332, "meanRXGBps": 129.9487166422721, "modeledHBMGBps": 538.8487976184281, "peakAllocatedGiB": 49.72999620437622, "peakReservedBytes": 66930606080, "fastMath": true, "sharedSchedule": "concurrent", "capacity": { "slotBytes": 11348090880, "slotFactor": 2.5, "scratchBytes": 5366722560, "scratchFactor": 8.0, "hostScratchBytes": 0, "slots": 1, "savedRank0Bytes": 8552398848, "maxSavedRankBytes": 8552398848, "capacityRecomputeFlags": 0, "inspectedRankLayers": 640 } }, { "id": "dist_moe-training-ep64-mxfp8-mega-single_layer-i1-F3072", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "mega", "category": "single_layer", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 2.931769585609436, "forwardMS": 1.262119472026825, "backwardMS": 1.795084802309672, "usefulTFPerGPU": 1938.1610889543404, "meanTXGBps": 238.15087054315995, "meanRXGBps": 238.1516586155365, "modeledHBMGBps": 1004.157712273978, "peakAllocatedGiB": 11.933184146881104, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep64-mxfp8-mega-single_layer-i4-F3072", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "mega", "category": "single_layer", "F": 3072, "imbalance": 4.0, "profile": null, "layers": 1, "iterationMS": 8.114518388112385, "forwardMS": 3.268566377957662, "backwardMS": 5.098596286773682, "usefulTFPerGPU": 700.2561903035892, "meanTXGBps": 86.61120239312814, "meanRXGBps": 86.61156331528308, "modeledHBMGBps": 362.8014503378098, "peakAllocatedGiB": 11.933187007904053, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep64-mxfp8-mega-single_layer-i8-F3072", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "mega", "category": "single_layer", "F": 3072, "imbalance": 8.0, "profile": null, "layers": 1, "iterationMS": 15.505108292897543, "forwardMS": 5.972500212987264, "backwardMS": 9.786971728006998, "usefulTFPerGPU": 366.47546249069904, "meanTXGBps": 45.570834456827036, "meanRXGBps": 45.572345424959316, "modeledHBMGBps": 189.8702662624127, "peakAllocatedGiB": 11.933189868927002, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep64-mxfp8-mega-width-i1-F1536", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "mega", "category": "width", "F": 1536, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 2.0520320097605387, "forwardMS": 0.9178005377451579, "backwardMS": 1.1560799956321717, "usefulTFPerGPU": 1384.5402278279003, "meanTXGBps": 334.5960004689717, "meanRXGBps": 334.59730466620323, "modeledHBMGBps": 910.5194105710048, "peakAllocatedGiB": 8.17231798171997, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep64-mxfp8-mega-width-i1-F3072", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "mega", "category": "width", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 2.9315274556477866, "forwardMS": 1.2663861354192099, "backwardMS": 1.7707253376642862, "usefulTFPerGPU": 1938.3211716679562, "meanTXGBps": 238.30144691543762, "meanRXGBps": 238.30208318367866, "modeledHBMGBps": 1004.2406508348619, "peakAllocatedGiB": 11.933192729949951, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep64-mxfp8-mega-width-i1-F6144", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "mega", "category": "width", "F": 6144, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 5.225540272394816, "forwardMS": 2.108567484219869, "backwardMS": 3.397762147585551, "usefulTFPerGPU": 2174.795881920888, "meanTXGBps": 133.44954581161335, "meanRXGBps": 133.45053248155307, "modeledHBMGBps": 975.0278253362285, "peakAllocatedGiB": 19.45491647720337, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "mok-training-ep64-mxfp8-native-packing-A-F3072", "backend": "mok", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "native", "category": "packing", "F": 3072, "imbalance": 1.0, "profile": "A", "layers": 10, "iterationMS": 70.73482360839844, "forwardMS": 23.157299359639484, "backwardMS": 47.57805658976237, "usefulTFPerGPU": 803.3160249421106, "meanTXGBps": 100.60137973251769, "meanRXGBps": 101.04993908018349, "modeledHBMGBps": 416.1965620071836, "peakAllocatedGiB": 48.36899948120117, "peakReservedBytes": 64032342016, "fastMath": false, "sharedSchedule": "concurrent", "capacity": { "requestedRowsPerLayer": 30720, "retainedRowsPerLayer": 32768, "minibatchRows": 4096, "effectiveFactor": 2.6666666666666665, "rank0ScheduleRows": [ 92160, 30720, 55296, 7680, 23040, 12288, 27648, 9216, 19968, 10752 ], "rank0ReplayRows": [ 59392, 0, 22528, 0, 0, 0, 0, 0, 0, 0 ], "totalReplayRows": 81920, "rankLayerReplayCount": 2, "sourcePolicyDerived": true } }, { "id": "mok-training-ep64-mxfp8-native-packing-B-F3072", "backend": "mok", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "native", "category": "packing", "F": 3072, "imbalance": 1.0, "profile": "B", "layers": 10, "iterationMS": 76.54772109985352, "forwardMS": 24.008308283487956, "backwardMS": 52.54009157816569, "usefulTFPerGPU": 742.3136379456336, "meanTXGBps": 93.02035301315787, "meanRXGBps": 93.43417035455624, "modeledHBMGBps": 384.5913369726213, "peakAllocatedGiB": 48.3690071105957, "peakReservedBytes": 64032342016, "fastMath": false, "sharedSchedule": "concurrent", "capacity": { "requestedRowsPerLayer": 30720, "retainedRowsPerLayer": 32768, "minibatchRows": 4096, "effectiveFactor": 2.6666666666666665, "rank0ScheduleRows": [ 9216, 72192, 15360, 7680, 44544, 12288, 89088, 10752, 29184, 9216 ], "rank0ReplayRows": [ 0, 39424, 0, 0, 11776, 0, 56320, 0, 0, 0 ], "totalReplayRows": 107520, "rankLayerReplayCount": 3, "sourcePolicyDerived": true } }, { "id": "mok-training-ep64-mxfp8-native-single_layer-i1-F3072", "backend": "mok", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 3.1929973284403483, "forwardMS": 1.224243195851644, "backwardMS": 1.9691392024358114, "usefulTFPerGPU": 1779.5948909808667, "meanTXGBps": 216.84045887785928, "meanRXGBps": 217.8085750336365, "modeledHBMGBps": 922.0048553682964, "peakAllocatedGiB": 16.39890480041504, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-training-ep64-mxfp8-native-single_layer-i4-F3072", "backend": "mok", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 4.0, "profile": null, "layers": 1, "iterationMS": 9.400117365519206, "forwardMS": 3.605842145284017, "backwardMS": 5.785875225067139, "usefulTFPerGPU": 604.4862539111657, "meanTXGBps": 75.41413238880197, "meanRXGBps": 75.75519811694899, "modeledHBMGBps": 313.18322160516914, "peakAllocatedGiB": 16.39891242980957, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-training-ep64-mxfp8-native-single_layer-i8-F3072", "backend": "mok", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "native", "category": "single_layer", "F": 3072, "imbalance": 8.0, "profile": null, "layers": 1, "iterationMS": 18.023618125915526, "forwardMS": 6.995222409566243, "backwardMS": 11.03064324061076, "usefulTFPerGPU": 315.2664294655525, "meanTXGBps": 39.267511741043684, "meanRXGBps": 39.440984694821914, "modeledHBMGBps": 163.33895999310954, "peakAllocatedGiB": 16.3989200592041, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-training-ep64-mxfp8-native-width-i1-F1536", "backend": "mok", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "native", "category": "width", "F": 1536, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 2.4470272064208984, "forwardMS": 1.0420234600702922, "backwardMS": 1.4057226777076721, "usefulTFPerGPU": 1161.0499706946518, "meanTXGBps": 286.33296549809097, "meanRXGBps": 287.61069695271516, "modeledHBMGBps": 763.5448314989536, "peakAllocatedGiB": 11.415620803833008, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-training-ep64-mxfp8-native-width-i1-F3072", "backend": "mok", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "native", "category": "width", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 3.1921759923299153, "forwardMS": 1.229091211160024, "backwardMS": 1.9637493332227072, "usefulTFPerGPU": 1780.0527747408526, "meanTXGBps": 217.97955968788017, "meanRXGBps": 218.95195805834473, "modeledHBMGBps": 922.2420841061629, "peakAllocatedGiB": 16.398927688598633, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "mok-training-ep64-mxfp8-native-width-i1-F6144", "backend": "mok", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "native", "category": "width", "F": 6144, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 5.781945594151815, "forwardMS": 2.10239253838857, "backwardMS": 3.6825034856796264, "usefulTFPerGPU": 1965.5120028646893, "meanTXGBps": 121.49764824691981, "meanRXGBps": 122.04021068112925, "modeledHBMGBps": 881.199431062343, "peakAllocatedGiB": 26.36547327041626, "peakReservedBytes": null, "fastMath": false, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep64-mxfp8-staged-packing-A-F3072", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "staged", "category": "packing", "F": 3072, "imbalance": 1.0, "profile": "A", "layers": 10, "iterationMS": 54.57124354044596, "forwardMS": 20.034262275695802, "backwardMS": 34.66030082702637, "usefulTFPerGPU": 1041.2520155228926, "meanTXGBps": 130.00310004112256, "meanRXGBps": 129.99416943097606, "modeledHBMGBps": 539.4707631718267, "peakAllocatedGiB": 49.411527156829834, "peakReservedBytes": 66299363328, "fastMath": true, "sharedSchedule": "concurrent", "capacity": { "slotBytes": 11348090880, "slotFactor": 2.5, "scratchBytes": 5024772096, "scratchFactor": 8.0, "hostScratchBytes": 0, "slots": 1, "savedRank0Bytes": 8306737152, "maxSavedRankBytes": 8306737152, "capacityRecomputeFlags": 0, "inspectedRankLayers": 640 } }, { "id": "dist_moe-training-ep64-mxfp8-staged-packing-B-F3072", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "staged", "category": "packing", "F": 3072, "imbalance": 1.0, "profile": "B", "layers": 10, "iterationMS": 56.49126930236817, "forwardMS": 20.6727081934611, "backwardMS": 35.82345403035482, "usefulTFPerGPU": 1005.86193278716, "meanTXGBps": 125.94141941424158, "meanRXGBps": 125.94191223310654, "modeledHBMGBps": 521.1352260899874, "peakAllocatedGiB": 49.41153001785278, "peakReservedBytes": 66238545920, "fastMath": true, "sharedSchedule": "concurrent", "capacity": { "slotBytes": 11348090880, "slotFactor": 2.5, "scratchBytes": 5024772096, "scratchFactor": 8.0, "hostScratchBytes": 0, "slots": 1, "savedRank0Bytes": 8552398848, "maxSavedRankBytes": 8552398848, "capacityRecomputeFlags": 0, "inspectedRankLayers": 640 } }, { "id": "dist_moe-training-ep64-mxfp8-staged-single_layer-i1-F3072", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "staged", "category": "single_layer", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 2.9545536120732625, "forwardMS": 1.1696277340253194, "backwardMS": 1.8665450771649679, "usefulTFPerGPU": 1923.2149687142316, "meanTXGBps": 235.2076547803986, "meanRXGBps": 235.20844276071318, "modeledHBMGBps": 996.4141547372944, "peakAllocatedGiB": 11.614717960357666, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep64-mxfp8-staged-single_layer-i4-F3072", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "staged", "category": "single_layer", "F": 3072, "imbalance": 4.0, "profile": null, "layers": 1, "iterationMS": 8.408859697977702, "forwardMS": 3.221996784210205, "backwardMS": 5.3461824258168535, "usefulTFPerGPU": 675.7446237299639, "meanTXGBps": 84.21351120225322, "meanRXGBps": 84.21824698571265, "modeledHBMGBps": 350.1020525658206, "peakAllocatedGiB": 11.614720821380615, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep64-mxfp8-staged-single_layer-i8-F3072", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "staged", "category": "single_layer", "F": 3072, "imbalance": 8.0, "profile": null, "layers": 1, "iterationMS": 15.845878410339356, "forwardMS": 5.784082126617432, "backwardMS": 10.19619197845459, "usefulTFPerGPU": 358.5943035445966, "meanTXGBps": 44.61052784330817, "meanRXGBps": 44.60166669478133, "modeledHBMGBps": 185.7870522393433, "peakAllocatedGiB": 11.614723682403564, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep64-mxfp8-staged-width-i1-F1536", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "staged", "category": "width", "F": 1536, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 2.2116469462712605, "forwardMS": 0.942217602332433, "backwardMS": 1.27036372423172, "usefulTFPerGPU": 1284.617723951829, "meanTXGBps": 315.85027171449934, "meanRXGBps": 315.85124415857604, "modeledHBMGBps": 844.8070697495663, "peakAllocatedGiB": 8.013084888458252, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep64-mxfp8-staged-width-i1-F3072", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "staged", "category": "width", "F": 3072, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 2.9556373357772827, "forwardMS": 1.1810080011685689, "backwardMS": 1.8550357302029927, "usefulTFPerGPU": 1922.5097963900455, "meanTXGBps": 236.36102225586237, "meanRXGBps": 236.36179010113497, "modeledHBMGBps": 996.0488062469913, "peakAllocatedGiB": 11.614726543426514, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" }, { "id": "dist_moe-training-ep64-mxfp8-staged-width-i1-F6144", "backend": "dist_moe", "mode": "training", "ep": 64, "precision": "mxfp8", "pipeline": "staged", "category": "width", "F": 6144, "imbalance": 1.0, "profile": null, "layers": 1, "iterationMS": 5.174753077824911, "forwardMS": 2.0504672129948935, "backwardMS": 3.344034131368001, "usefulTFPerGPU": 2196.1402397953257, "meanTXGBps": 135.51754889793904, "meanRXGBps": 135.51813725617092, "modeledHBMGBps": 984.597156883394, "peakAllocatedGiB": 19.348761081695557, "peakReservedBytes": null, "fastMath": true, "sharedSchedule": "concurrent" } ], "pairs": [ { "dist": "dist_moe-inference-ep16-bf16-standard-single_layer-i1-F3072", "mok": "mok-inference-ep16-bf16-native-single_layer-i1-F3072", "allDistPipelines": [ "dist_moe-inference-ep16-bf16-standard-single_layer-i1-F3072" ] }, { "dist": "dist_moe-inference-ep16-bf16-standard-single_layer-i4-F3072", "mok": "mok-inference-ep16-bf16-native-single_layer-i4-F3072", "allDistPipelines": [ "dist_moe-inference-ep16-bf16-standard-single_layer-i4-F3072" ] }, { "dist": "dist_moe-inference-ep16-bf16-standard-single_layer-i8-F3072", "mok": "mok-inference-ep16-bf16-native-single_layer-i8-F3072", "allDistPipelines": [ "dist_moe-inference-ep16-bf16-standard-single_layer-i8-F3072" ] }, { "dist": "dist_moe-inference-ep16-bf16-standard-width-i1-F1536", "mok": "mok-inference-ep16-bf16-native-width-i1-F1536", "allDistPipelines": [ "dist_moe-inference-ep16-bf16-standard-width-i1-F1536" ] }, { "dist": "dist_moe-inference-ep16-bf16-standard-width-i1-F3072", "mok": "mok-inference-ep16-bf16-native-width-i1-F3072", "allDistPipelines": [ "dist_moe-inference-ep16-bf16-standard-width-i1-F3072" ] }, { "dist": "dist_moe-inference-ep16-bf16-standard-width-i1-F6144", "mok": "mok-inference-ep16-bf16-native-width-i1-F6144", "allDistPipelines": [ "dist_moe-inference-ep16-bf16-standard-width-i1-F6144" ] }, { "dist": "dist_moe-inference-ep16-mxfp8-mega-single_layer-i1-F3072", "mok": "mok-inference-ep16-mxfp8-native-single_layer-i1-F3072", "allDistPipelines": [ "dist_moe-inference-ep16-mxfp8-mega-single_layer-i1-F3072", "dist_moe-inference-ep16-mxfp8-staged-single_layer-i1-F3072" ] }, { "dist": "dist_moe-inference-ep16-mxfp8-mega-single_layer-i4-F3072", "mok": "mok-inference-ep16-mxfp8-native-single_layer-i4-F3072", "allDistPipelines": [ "dist_moe-inference-ep16-mxfp8-mega-single_layer-i4-F3072", "dist_moe-inference-ep16-mxfp8-staged-single_layer-i4-F3072" ] }, { "dist": "dist_moe-inference-ep16-mxfp8-mega-single_layer-i8-F3072", "mok": "mok-inference-ep16-mxfp8-native-single_layer-i8-F3072", "allDistPipelines": [ "dist_moe-inference-ep16-mxfp8-mega-single_layer-i8-F3072", "dist_moe-inference-ep16-mxfp8-staged-single_layer-i8-F3072" ] }, { "dist": "dist_moe-inference-ep16-mxfp8-staged-width-i1-F1536", "mok": "mok-inference-ep16-mxfp8-native-width-i1-F1536", "allDistPipelines": [ "dist_moe-inference-ep16-mxfp8-mega-width-i1-F1536", "dist_moe-inference-ep16-mxfp8-staged-width-i1-F1536" ] }, { "dist": "dist_moe-inference-ep16-mxfp8-mega-width-i1-F3072", "mok": "mok-inference-ep16-mxfp8-native-width-i1-F3072", "allDistPipelines": [ "dist_moe-inference-ep16-mxfp8-mega-width-i1-F3072", "dist_moe-inference-ep16-mxfp8-staged-width-i1-F3072" ] }, { "dist": "dist_moe-inference-ep16-mxfp8-mega-width-i1-F6144", "mok": "mok-inference-ep16-mxfp8-native-width-i1-F6144", "allDistPipelines": [ "dist_moe-inference-ep16-mxfp8-mega-width-i1-F6144", "dist_moe-inference-ep16-mxfp8-staged-width-i1-F6144" ] }, { "dist": "dist_moe-inference-ep8-bf16-standard-single_layer-i1-F3072", "mok": "mok-inference-ep8-bf16-native-single_layer-i1-F3072", "allDistPipelines": [ "dist_moe-inference-ep8-bf16-standard-single_layer-i1-F3072" ] }, { "dist": "dist_moe-inference-ep8-bf16-standard-single_layer-i4-F3072", "mok": "mok-inference-ep8-bf16-native-single_layer-i4-F3072", "allDistPipelines": [ "dist_moe-inference-ep8-bf16-standard-single_layer-i4-F3072" ] }, { "dist": "dist_moe-inference-ep8-bf16-standard-single_layer-i8-F3072", "mok": "mok-inference-ep8-bf16-native-single_layer-i8-F3072", "allDistPipelines": [ "dist_moe-inference-ep8-bf16-standard-single_layer-i8-F3072" ] }, { "dist": "dist_moe-inference-ep8-mxfp8-mega-single_layer-i1-F3072", "mok": "mok-inference-ep8-mxfp8-native-single_layer-i1-F3072", "allDistPipelines": [ "dist_moe-inference-ep8-mxfp8-mega-single_layer-i1-F3072", "dist_moe-inference-ep8-mxfp8-staged-single_layer-i1-F3072" ] }, { "dist": "dist_moe-inference-ep8-mxfp8-mega-single_layer-i4-F3072", "mok": "mok-inference-ep8-mxfp8-native-single_layer-i4-F3072", "allDistPipelines": [ "dist_moe-inference-ep8-mxfp8-mega-single_layer-i4-F3072", "dist_moe-inference-ep8-mxfp8-staged-single_layer-i4-F3072" ] }, { "dist": "dist_moe-inference-ep8-mxfp8-mega-single_layer-i8-F3072", "mok": "mok-inference-ep8-mxfp8-native-single_layer-i8-F3072", "allDistPipelines": [ "dist_moe-inference-ep8-mxfp8-mega-single_layer-i8-F3072", "dist_moe-inference-ep8-mxfp8-staged-single_layer-i8-F3072" ] }, { "dist": "dist_moe-training-ep32-bf16-standard-packing-A-F3072", "mok": "mok-training-ep32-bf16-native-packing-A-F3072", "allDistPipelines": [ "dist_moe-training-ep32-bf16-standard-packing-A-F3072" ] }, { "dist": "dist_moe-training-ep32-bf16-standard-packing-B-F3072", "mok": "mok-training-ep32-bf16-native-packing-B-F3072", "allDistPipelines": [ "dist_moe-training-ep32-bf16-standard-packing-B-F3072" ] }, { "dist": "dist_moe-training-ep32-bf16-standard-single_layer-i1-F3072", "mok": "mok-training-ep32-bf16-native-single_layer-i1-F3072", "allDistPipelines": [ "dist_moe-training-ep32-bf16-standard-single_layer-i1-F3072" ] }, { "dist": "dist_moe-training-ep32-bf16-standard-single_layer-i4-F3072", "mok": "mok-training-ep32-bf16-native-single_layer-i4-F3072", "allDistPipelines": [ "dist_moe-training-ep32-bf16-standard-single_layer-i4-F3072" ] }, { "dist": "dist_moe-training-ep32-bf16-standard-single_layer-i8-F3072", "mok": "mok-training-ep32-bf16-native-single_layer-i8-F3072", "allDistPipelines": [ "dist_moe-training-ep32-bf16-standard-single_layer-i8-F3072" ] }, { "dist": "dist_moe-training-ep32-mxfp8-mega-packing-A-F3072", "mok": "mok-training-ep32-mxfp8-native-packing-A-F3072", "allDistPipelines": [ "dist_moe-training-ep32-mxfp8-mega-packing-A-F3072", "dist_moe-training-ep32-mxfp8-staged-packing-A-F3072" ] }, { "dist": "dist_moe-training-ep32-mxfp8-mega-packing-B-F3072", "mok": "mok-training-ep32-mxfp8-native-packing-B-F3072", "allDistPipelines": [ "dist_moe-training-ep32-mxfp8-mega-packing-B-F3072", "dist_moe-training-ep32-mxfp8-staged-packing-B-F3072" ] }, { "dist": "dist_moe-training-ep32-mxfp8-staged-single_layer-i1-F3072", "mok": "mok-training-ep32-mxfp8-native-single_layer-i1-F3072", "allDistPipelines": [ "dist_moe-training-ep32-mxfp8-mega-single_layer-i1-F3072", "dist_moe-training-ep32-mxfp8-staged-single_layer-i1-F3072" ] }, { "dist": "dist_moe-training-ep32-mxfp8-mega-single_layer-i4-F3072", "mok": "mok-training-ep32-mxfp8-native-single_layer-i4-F3072", "allDistPipelines": [ "dist_moe-training-ep32-mxfp8-mega-single_layer-i4-F3072", "dist_moe-training-ep32-mxfp8-staged-single_layer-i4-F3072" ] }, { "dist": "dist_moe-training-ep32-mxfp8-mega-single_layer-i8-F3072", "mok": "mok-training-ep32-mxfp8-native-single_layer-i8-F3072", "allDistPipelines": [ "dist_moe-training-ep32-mxfp8-mega-single_layer-i8-F3072", "dist_moe-training-ep32-mxfp8-staged-single_layer-i8-F3072" ] }, { "dist": "dist_moe-training-ep64-bf16-standard-packing-A-F3072", "mok": "mok-training-ep64-bf16-native-packing-A-F3072", "allDistPipelines": [ "dist_moe-training-ep64-bf16-standard-packing-A-F3072" ] }, { "dist": "dist_moe-training-ep64-bf16-standard-packing-B-F3072", "mok": "mok-training-ep64-bf16-native-packing-B-F3072", "allDistPipelines": [ "dist_moe-training-ep64-bf16-standard-packing-B-F3072" ] }, { "dist": "dist_moe-training-ep64-bf16-standard-single_layer-i1-F3072", "mok": "mok-training-ep64-bf16-native-single_layer-i1-F3072", "allDistPipelines": [ "dist_moe-training-ep64-bf16-standard-single_layer-i1-F3072" ] }, { "dist": "dist_moe-training-ep64-bf16-standard-single_layer-i4-F3072", "mok": "mok-training-ep64-bf16-native-single_layer-i4-F3072", "allDistPipelines": [ "dist_moe-training-ep64-bf16-standard-single_layer-i4-F3072" ] }, { "dist": "dist_moe-training-ep64-bf16-standard-single_layer-i8-F3072", "mok": "mok-training-ep64-bf16-native-single_layer-i8-F3072", "allDistPipelines": [ "dist_moe-training-ep64-bf16-standard-single_layer-i8-F3072" ] }, { "dist": "dist_moe-training-ep64-bf16-standard-width-i1-F1536", "mok": "mok-training-ep64-bf16-native-width-i1-F1536", "allDistPipelines": [ "dist_moe-training-ep64-bf16-standard-width-i1-F1536" ] }, { "dist": "dist_moe-training-ep64-bf16-standard-width-i1-F3072", "mok": "mok-training-ep64-bf16-native-width-i1-F3072", "allDistPipelines": [ "dist_moe-training-ep64-bf16-standard-width-i1-F3072" ] }, { "dist": "dist_moe-training-ep64-bf16-standard-width-i1-F6144", "mok": "mok-training-ep64-bf16-native-width-i1-F6144", "allDistPipelines": [ "dist_moe-training-ep64-bf16-standard-width-i1-F6144" ] }, { "dist": "dist_moe-training-ep64-mxfp8-mega-packing-A-F3072", "mok": "mok-training-ep64-mxfp8-native-packing-A-F3072", "allDistPipelines": [ "dist_moe-training-ep64-mxfp8-mega-packing-A-F3072", "dist_moe-training-ep64-mxfp8-staged-packing-A-F3072" ] }, { "dist": "dist_moe-training-ep64-mxfp8-mega-packing-B-F3072", "mok": "mok-training-ep64-mxfp8-native-packing-B-F3072", "allDistPipelines": [ "dist_moe-training-ep64-mxfp8-mega-packing-B-F3072", "dist_moe-training-ep64-mxfp8-staged-packing-B-F3072" ] }, { "dist": "dist_moe-training-ep64-mxfp8-mega-single_layer-i1-F3072", "mok": "mok-training-ep64-mxfp8-native-single_layer-i1-F3072", "allDistPipelines": [ "dist_moe-training-ep64-mxfp8-mega-single_layer-i1-F3072", "dist_moe-training-ep64-mxfp8-staged-single_layer-i1-F3072" ] }, { "dist": "dist_moe-training-ep64-mxfp8-mega-single_layer-i4-F3072", "mok": "mok-training-ep64-mxfp8-native-single_layer-i4-F3072", "allDistPipelines": [ "dist_moe-training-ep64-mxfp8-mega-single_layer-i4-F3072", "dist_moe-training-ep64-mxfp8-staged-single_layer-i4-F3072" ] }, { "dist": "dist_moe-training-ep64-mxfp8-mega-single_layer-i8-F3072", "mok": "mok-training-ep64-mxfp8-native-single_layer-i8-F3072", "allDistPipelines": [ "dist_moe-training-ep64-mxfp8-mega-single_layer-i8-F3072", "dist_moe-training-ep64-mxfp8-staged-single_layer-i8-F3072" ] }, { "dist": "dist_moe-training-ep64-mxfp8-mega-width-i1-F1536", "mok": "mok-training-ep64-mxfp8-native-width-i1-F1536", "allDistPipelines": [ "dist_moe-training-ep64-mxfp8-mega-width-i1-F1536", "dist_moe-training-ep64-mxfp8-staged-width-i1-F1536" ] }, { "dist": "dist_moe-training-ep64-mxfp8-mega-width-i1-F3072", "mok": "mok-training-ep64-mxfp8-native-width-i1-F3072", "allDistPipelines": [ "dist_moe-training-ep64-mxfp8-mega-width-i1-F3072", "dist_moe-training-ep64-mxfp8-staged-width-i1-F3072" ] }, { "dist": "dist_moe-training-ep64-mxfp8-staged-width-i1-F6144", "mok": "mok-training-ep64-mxfp8-native-width-i1-F6144", "allDistPipelines": [ "dist_moe-training-ep64-mxfp8-mega-width-i1-F6144", "dist_moe-training-ep64-mxfp8-staged-width-i1-F6144" ] } ], "allPipelinePairs": [ { "dist": "dist_moe-inference-ep16-bf16-standard-single_layer-i1-F3072", "mok": "mok-inference-ep16-bf16-native-single_layer-i1-F3072" }, { "dist": "dist_moe-inference-ep16-bf16-standard-single_layer-i4-F3072", "mok": "mok-inference-ep16-bf16-native-single_layer-i4-F3072" }, { "dist": "dist_moe-inference-ep16-bf16-standard-single_layer-i8-F3072", "mok": "mok-inference-ep16-bf16-native-single_layer-i8-F3072" }, { "dist": "dist_moe-inference-ep16-bf16-standard-width-i1-F1536", "mok": "mok-inference-ep16-bf16-native-width-i1-F1536" }, { "dist": "dist_moe-inference-ep16-bf16-standard-width-i1-F3072", "mok": "mok-inference-ep16-bf16-native-width-i1-F3072" }, { "dist": "dist_moe-inference-ep16-bf16-standard-width-i1-F6144", "mok": "mok-inference-ep16-bf16-native-width-i1-F6144" }, { "dist": "dist_moe-inference-ep16-mxfp8-mega-single_layer-i1-F3072", "mok": "mok-inference-ep16-mxfp8-native-single_layer-i1-F3072" }, { "dist": "dist_moe-inference-ep16-mxfp8-staged-single_layer-i1-F3072", "mok": "mok-inference-ep16-mxfp8-native-single_layer-i1-F3072" }, { "dist": "dist_moe-inference-ep16-mxfp8-mega-single_layer-i4-F3072", "mok": "mok-inference-ep16-mxfp8-native-single_layer-i4-F3072" }, { "dist": "dist_moe-inference-ep16-mxfp8-staged-single_layer-i4-F3072", "mok": "mok-inference-ep16-mxfp8-native-single_layer-i4-F3072" }, { "dist": "dist_moe-inference-ep16-mxfp8-mega-single_layer-i8-F3072", "mok": "mok-inference-ep16-mxfp8-native-single_layer-i8-F3072" }, { "dist": "dist_moe-inference-ep16-mxfp8-staged-single_layer-i8-F3072", "mok": "mok-inference-ep16-mxfp8-native-single_layer-i8-F3072" }, { "dist": "dist_moe-inference-ep16-mxfp8-mega-width-i1-F1536", "mok": "mok-inference-ep16-mxfp8-native-width-i1-F1536" }, { "dist": "dist_moe-inference-ep16-mxfp8-staged-width-i1-F1536", "mok": "mok-inference-ep16-mxfp8-native-width-i1-F1536" }, { "dist": "dist_moe-inference-ep16-mxfp8-mega-width-i1-F3072", "mok": "mok-inference-ep16-mxfp8-native-width-i1-F3072" }, { "dist": "dist_moe-inference-ep16-mxfp8-staged-width-i1-F3072", "mok": "mok-inference-ep16-mxfp8-native-width-i1-F3072" }, { "dist": "dist_moe-inference-ep16-mxfp8-mega-width-i1-F6144", "mok": "mok-inference-ep16-mxfp8-native-width-i1-F6144" }, { "dist": "dist_moe-inference-ep16-mxfp8-staged-width-i1-F6144", "mok": "mok-inference-ep16-mxfp8-native-width-i1-F6144" }, { "dist": "dist_moe-inference-ep8-bf16-standard-single_layer-i1-F3072", "mok": "mok-inference-ep8-bf16-native-single_layer-i1-F3072" }, { "dist": "dist_moe-inference-ep8-bf16-standard-single_layer-i4-F3072", "mok": "mok-inference-ep8-bf16-native-single_layer-i4-F3072" }, { "dist": "dist_moe-inference-ep8-bf16-standard-single_layer-i8-F3072", "mok": "mok-inference-ep8-bf16-native-single_layer-i8-F3072" }, { "dist": "dist_moe-inference-ep8-mxfp8-mega-single_layer-i1-F3072", "mok": "mok-inference-ep8-mxfp8-native-single_layer-i1-F3072" }, { "dist": "dist_moe-inference-ep8-mxfp8-staged-single_layer-i1-F3072", "mok": "mok-inference-ep8-mxfp8-native-single_layer-i1-F3072" }, { "dist": "dist_moe-inference-ep8-mxfp8-mega-single_layer-i4-F3072", "mok": "mok-inference-ep8-mxfp8-native-single_layer-i4-F3072" }, { "dist": "dist_moe-inference-ep8-mxfp8-staged-single_layer-i4-F3072", "mok": "mok-inference-ep8-mxfp8-native-single_layer-i4-F3072" }, { "dist": "dist_moe-inference-ep8-mxfp8-mega-single_layer-i8-F3072", "mok": "mok-inference-ep8-mxfp8-native-single_layer-i8-F3072" }, { "dist": "dist_moe-inference-ep8-mxfp8-staged-single_layer-i8-F3072", "mok": "mok-inference-ep8-mxfp8-native-single_layer-i8-F3072" }, { "dist": "dist_moe-training-ep32-bf16-standard-packing-A-F3072", "mok": "mok-training-ep32-bf16-native-packing-A-F3072" }, { "dist": "dist_moe-training-ep32-bf16-standard-packing-B-F3072", "mok": "mok-training-ep32-bf16-native-packing-B-F3072" }, { "dist": "dist_moe-training-ep32-bf16-standard-single_layer-i1-F3072", "mok": "mok-training-ep32-bf16-native-single_layer-i1-F3072" }, { "dist": "dist_moe-training-ep32-bf16-standard-single_layer-i4-F3072", "mok": "mok-training-ep32-bf16-native-single_layer-i4-F3072" }, { "dist": "dist_moe-training-ep32-bf16-standard-single_layer-i8-F3072", "mok": "mok-training-ep32-bf16-native-single_layer-i8-F3072" }, { "dist": "dist_moe-training-ep32-mxfp8-mega-packing-A-F3072", "mok": "mok-training-ep32-mxfp8-native-packing-A-F3072" }, { "dist": "dist_moe-training-ep32-mxfp8-staged-packing-A-F3072", "mok": "mok-training-ep32-mxfp8-native-packing-A-F3072" }, { "dist": "dist_moe-training-ep32-mxfp8-mega-packing-B-F3072", "mok": "mok-training-ep32-mxfp8-native-packing-B-F3072" }, { "dist": "dist_moe-training-ep32-mxfp8-staged-packing-B-F3072", "mok": "mok-training-ep32-mxfp8-native-packing-B-F3072" }, { "dist": "dist_moe-training-ep32-mxfp8-mega-single_layer-i1-F3072", "mok": "mok-training-ep32-mxfp8-native-single_layer-i1-F3072" }, { "dist": "dist_moe-training-ep32-mxfp8-staged-single_layer-i1-F3072", "mok": "mok-training-ep32-mxfp8-native-single_layer-i1-F3072" }, { "dist": "dist_moe-training-ep32-mxfp8-mega-single_layer-i4-F3072", "mok": "mok-training-ep32-mxfp8-native-single_layer-i4-F3072" }, { "dist": "dist_moe-training-ep32-mxfp8-staged-single_layer-i4-F3072", "mok": "mok-training-ep32-mxfp8-native-single_layer-i4-F3072" }, { "dist": "dist_moe-training-ep32-mxfp8-mega-single_layer-i8-F3072", "mok": "mok-training-ep32-mxfp8-native-single_layer-i8-F3072" }, { "dist": "dist_moe-training-ep32-mxfp8-staged-single_layer-i8-F3072", "mok": "mok-training-ep32-mxfp8-native-single_layer-i8-F3072" }, { "dist": "dist_moe-training-ep64-bf16-standard-packing-A-F3072", "mok": "mok-training-ep64-bf16-native-packing-A-F3072" }, { "dist": "dist_moe-training-ep64-bf16-standard-packing-B-F3072", "mok": "mok-training-ep64-bf16-native-packing-B-F3072" }, { "dist": "dist_moe-training-ep64-bf16-standard-single_layer-i1-F3072", "mok": "mok-training-ep64-bf16-native-single_layer-i1-F3072" }, { "dist": "dist_moe-training-ep64-bf16-standard-single_layer-i4-F3072", "mok": "mok-training-ep64-bf16-native-single_layer-i4-F3072" }, { "dist": "dist_moe-training-ep64-bf16-standard-single_layer-i8-F3072", "mok": "mok-training-ep64-bf16-native-single_layer-i8-F3072" }, { "dist": "dist_moe-training-ep64-bf16-standard-width-i1-F1536", "mok": "mok-training-ep64-bf16-native-width-i1-F1536" }, { "dist": "dist_moe-training-ep64-bf16-standard-width-i1-F3072", "mok": "mok-training-ep64-bf16-native-width-i1-F3072" }, { "dist": "dist_moe-training-ep64-bf16-standard-width-i1-F6144", "mok": "mok-training-ep64-bf16-native-width-i1-F6144" }, { "dist": "dist_moe-training-ep64-mxfp8-mega-packing-A-F3072", "mok": "mok-training-ep64-mxfp8-native-packing-A-F3072" }, { "dist": "dist_moe-training-ep64-mxfp8-staged-packing-A-F3072", "mok": "mok-training-ep64-mxfp8-native-packing-A-F3072" }, { "dist": "dist_moe-training-ep64-mxfp8-mega-packing-B-F3072", "mok": "mok-training-ep64-mxfp8-native-packing-B-F3072" }, { "dist": "dist_moe-training-ep64-mxfp8-staged-packing-B-F3072", "mok": "mok-training-ep64-mxfp8-native-packing-B-F3072" }, { "dist": "dist_moe-training-ep64-mxfp8-mega-single_layer-i1-F3072", "mok": "mok-training-ep64-mxfp8-native-single_layer-i1-F3072" }, { "dist": "dist_moe-training-ep64-mxfp8-staged-single_layer-i1-F3072", "mok": "mok-training-ep64-mxfp8-native-single_layer-i1-F3072" }, { "dist": "dist_moe-training-ep64-mxfp8-mega-single_layer-i4-F3072", "mok": "mok-training-ep64-mxfp8-native-single_layer-i4-F3072" }, { "dist": "dist_moe-training-ep64-mxfp8-staged-single_layer-i4-F3072", "mok": "mok-training-ep64-mxfp8-native-single_layer-i4-F3072" }, { "dist": "dist_moe-training-ep64-mxfp8-mega-single_layer-i8-F3072", "mok": "mok-training-ep64-mxfp8-native-single_layer-i8-F3072" }, { "dist": "dist_moe-training-ep64-mxfp8-staged-single_layer-i8-F3072", "mok": "mok-training-ep64-mxfp8-native-single_layer-i8-F3072" }, { "dist": "dist_moe-training-ep64-mxfp8-mega-width-i1-F1536", "mok": "mok-training-ep64-mxfp8-native-width-i1-F1536" }, { "dist": "dist_moe-training-ep64-mxfp8-staged-width-i1-F1536", "mok": "mok-training-ep64-mxfp8-native-width-i1-F1536" }, { "dist": "dist_moe-training-ep64-mxfp8-mega-width-i1-F3072", "mok": "mok-training-ep64-mxfp8-native-width-i1-F3072" }, { "dist": "dist_moe-training-ep64-mxfp8-staged-width-i1-F3072", "mok": "mok-training-ep64-mxfp8-native-width-i1-F3072" }, { "dist": "dist_moe-training-ep64-mxfp8-mega-width-i1-F6144", "mok": "mok-training-ep64-mxfp8-native-width-i1-F6144" }, { "dist": "dist_moe-training-ep64-mxfp8-staged-width-i1-F6144", "mok": "mok-training-ep64-mxfp8-native-width-i1-F6144" } ] }