{ "estimate": { "items": [ { "offloadLayers": 0, "fullOffloaded": false, "ram": { "handleLayers": 32, "handleLastLayer": 31, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 1170336728, "nonuma": 1327623128 }, "vrams": [ { "handleLayers": 0, "handleLastLayer": -1, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 0, "nonuma": 955768832 }, { "handleLayers": 0, "handleLastLayer": -1, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 0, "nonuma": 857735168 } ] }, { "offloadLayers": 1, "fullOffloaded": false, "ram": { "handleLayers": 31, "handleLastLayer": 30, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 1212279768, "nonuma": 1369566168 }, "vrams": [ { "handleLayers": 1, "handleLastLayer": 31, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 156270592, "nonuma": 1137209344 }, { "handleLayers": 0, "handleLastLayer": -1, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 0, "nonuma": 882905088 } ] }, { "offloadLayers": 2, "fullOffloaded": false, "ram": { "handleLayers": 30, "handleLastLayer": 29, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 1178725336, "nonuma": 1336011736 }, "vrams": [ { "handleLayers": 1, "handleLastLayer": 30, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 156270592, "nonuma": 1137209344 }, { "handleLayers": 1, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 33554432, "nonuma": 1039175680 } ] }, { "offloadLayers": 3, "fullOffloaded": false, "ram": { "handleLayers": 29, "handleLastLayer": 28, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 1145170904, "nonuma": 1302457304 }, "vrams": [ { "handleLayers": 2, "handleLastLayer": 30, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 312541184, "nonuma": 1293479936 }, { "handleLayers": 1, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 33554432, "nonuma": 1039175680 } ] }, { "offloadLayers": 4, "fullOffloaded": false, "ram": { "handleLayers": 28, "handleLastLayer": 27, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 1111616472, "nonuma": 1268902872 }, "vrams": [ { "handleLayers": 2, "handleLastLayer": 29, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 312541184, "nonuma": 1293479936 }, { "handleLayers": 2, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 67108864, "nonuma": 1195446272 } ] }, { "offloadLayers": 5, "fullOffloaded": false, "ram": { "handleLayers": 27, "handleLastLayer": 26, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 1078062040, "nonuma": 1235348440 }, "vrams": [ { "handleLayers": 3, "handleLastLayer": 29, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 468811776, "nonuma": 1449750528 }, { "handleLayers": 2, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 67108864, "nonuma": 1195446272 } ] }, { "offloadLayers": 6, "fullOffloaded": false, "ram": { "handleLayers": 26, "handleLastLayer": 25, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 1044507608, "nonuma": 1201794008 }, "vrams": [ { "handleLayers": 3, "handleLastLayer": 28, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 468811776, "nonuma": 1449750528 }, { "handleLayers": 3, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 100663296, "nonuma": 1351716864 } ] }, { "offloadLayers": 7, "fullOffloaded": false, "ram": { "handleLayers": 25, "handleLastLayer": 24, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 1010953176, "nonuma": 1168239576 }, "vrams": [ { "handleLayers": 4, "handleLastLayer": 28, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 625082368, "nonuma": 1606021120 }, { "handleLayers": 3, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 100663296, "nonuma": 1351716864 } ] }, { "offloadLayers": 8, "fullOffloaded": false, "ram": { "handleLayers": 24, "handleLastLayer": 23, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 977398744, "nonuma": 1134685144 }, "vrams": [ { "handleLayers": 4, "handleLastLayer": 27, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 625082368, "nonuma": 1606021120 }, { "handleLayers": 4, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 134217728, "nonuma": 1507987456 } ] }, { "offloadLayers": 9, "fullOffloaded": false, "ram": { "handleLayers": 23, "handleLastLayer": 22, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 943844312, "nonuma": 1101130712 }, "vrams": [ { "handleLayers": 5, "handleLastLayer": 27, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 781352960, "nonuma": 1762291712 }, { "handleLayers": 4, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 134217728, "nonuma": 1507987456 } ] }, { "offloadLayers": 10, "fullOffloaded": false, "ram": { "handleLayers": 22, "handleLastLayer": 21, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 910289880, "nonuma": 1067576280 }, "vrams": [ { "handleLayers": 5, "handleLastLayer": 26, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 781352960, "nonuma": 1762291712 }, { "handleLayers": 5, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 167772160, "nonuma": 1664258048 } ] }, { "offloadLayers": 11, "fullOffloaded": false, "ram": { "handleLayers": 21, "handleLastLayer": 20, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 876735448, "nonuma": 1034021848 }, "vrams": [ { "handleLayers": 6, "handleLastLayer": 26, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 937623552, "nonuma": 1918562304 }, { "handleLayers": 5, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 167772160, "nonuma": 1664258048 } ] }, { "offloadLayers": 12, "fullOffloaded": false, "ram": { "handleLayers": 20, "handleLastLayer": 19, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 843181016, "nonuma": 1000467416 }, "vrams": [ { "handleLayers": 6, "handleLastLayer": 25, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 937623552, "nonuma": 1918562304 }, { "handleLayers": 6, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 201326592, "nonuma": 1820528640 } ] }, { "offloadLayers": 13, "fullOffloaded": false, "ram": { "handleLayers": 19, "handleLastLayer": 18, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 809626584, "nonuma": 966912984 }, "vrams": [ { "handleLayers": 7, "handleLastLayer": 25, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 1093894144, "nonuma": 2074832896 }, { "handleLayers": 6, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 201326592, "nonuma": 1820528640 } ] }, { "offloadLayers": 14, "fullOffloaded": false, "ram": { "handleLayers": 18, "handleLastLayer": 17, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 776072152, "nonuma": 933358552 }, "vrams": [ { "handleLayers": 7, "handleLastLayer": 24, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 1093894144, "nonuma": 2074832896 }, { "handleLayers": 7, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 234881024, "nonuma": 1976799232 } ] }, { "offloadLayers": 15, "fullOffloaded": false, "ram": { "handleLayers": 17, "handleLastLayer": 16, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 742517720, "nonuma": 899804120 }, "vrams": [ { "handleLayers": 8, "handleLastLayer": 24, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 1250164736, "nonuma": 2231103488 }, { "handleLayers": 7, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 234881024, "nonuma": 1976799232 } ] }, { "offloadLayers": 16, "fullOffloaded": false, "ram": { "handleLayers": 16, "handleLastLayer": 15, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 708963288, "nonuma": 866249688 }, "vrams": [ { "handleLayers": 8, "handleLastLayer": 23, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 1250164736, "nonuma": 2231103488 }, { "handleLayers": 8, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 268435456, "nonuma": 2133069824 } ] }, { "offloadLayers": 17, "fullOffloaded": false, "ram": { "handleLayers": 15, "handleLastLayer": 14, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 675408856, "nonuma": 832695256 }, "vrams": [ { "handleLayers": 9, "handleLastLayer": 23, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 1406435328, "nonuma": 2387374080 }, { "handleLayers": 8, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 268435456, "nonuma": 2133069824 } ] }, { "offloadLayers": 18, "fullOffloaded": false, "ram": { "handleLayers": 14, "handleLastLayer": 13, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 641854424, "nonuma": 799140824 }, "vrams": [ { "handleLayers": 9, "handleLastLayer": 22, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 1406435328, "nonuma": 2387374080 }, { "handleLayers": 9, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 301989888, "nonuma": 2289340416 } ] }, { "offloadLayers": 19, "fullOffloaded": false, "ram": { "handleLayers": 13, "handleLastLayer": 12, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 608299992, "nonuma": 765586392 }, "vrams": [ { "handleLayers": 10, "handleLastLayer": 22, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 1562705920, "nonuma": 2543644672 }, { "handleLayers": 9, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 301989888, "nonuma": 2289340416 } ] }, { "offloadLayers": 20, "fullOffloaded": false, "ram": { "handleLayers": 12, "handleLastLayer": 11, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 574745560, "nonuma": 732031960 }, "vrams": [ { "handleLayers": 10, "handleLastLayer": 21, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 1562705920, "nonuma": 2543644672 }, { "handleLayers": 10, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 335544320, "nonuma": 2445611008 } ] }, { "offloadLayers": 21, "fullOffloaded": false, "ram": { "handleLayers": 11, "handleLastLayer": 10, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 541191128, "nonuma": 698477528 }, "vrams": [ { "handleLayers": 11, "handleLastLayer": 21, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 1718976512, "nonuma": 2699915264 }, { "handleLayers": 10, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 335544320, "nonuma": 2445611008 } ] }, { "offloadLayers": 22, "fullOffloaded": false, "ram": { "handleLayers": 10, "handleLastLayer": 9, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 507636696, "nonuma": 664923096 }, "vrams": [ { "handleLayers": 11, "handleLastLayer": 20, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 1718976512, "nonuma": 2699915264 }, { "handleLayers": 11, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 369098752, "nonuma": 2601881600 } ] }, { "offloadLayers": 23, "fullOffloaded": false, "ram": { "handleLayers": 9, "handleLastLayer": 8, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 474082264, "nonuma": 631368664 }, "vrams": [ { "handleLayers": 12, "handleLastLayer": 20, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 1875247104, "nonuma": 2856185856 }, { "handleLayers": 11, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 369098752, "nonuma": 2601881600 } ] }, { "offloadLayers": 24, "fullOffloaded": false, "ram": { "handleLayers": 8, "handleLastLayer": 7, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 440527832, "nonuma": 597814232 }, "vrams": [ { "handleLayers": 12, "handleLastLayer": 19, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 1875247104, "nonuma": 2856185856 }, { "handleLayers": 12, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 402653184, "nonuma": 2758152192 } ] }, { "offloadLayers": 25, "fullOffloaded": false, "ram": { "handleLayers": 7, "handleLastLayer": 6, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 406973400, "nonuma": 564259800 }, "vrams": [ { "handleLayers": 13, "handleLastLayer": 19, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 2031517696, "nonuma": 3012456448 }, { "handleLayers": 12, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 402653184, "nonuma": 2758152192 } ] }, { "offloadLayers": 26, "fullOffloaded": false, "ram": { "handleLayers": 6, "handleLastLayer": 5, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 373418968, "nonuma": 530705368 }, "vrams": [ { "handleLayers": 13, "handleLastLayer": 18, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 2031517696, "nonuma": 3012456448 }, { "handleLayers": 13, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 436207616, "nonuma": 2914422784 } ] }, { "offloadLayers": 27, "fullOffloaded": false, "ram": { "handleLayers": 5, "handleLastLayer": 4, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 339864536, "nonuma": 497150936 }, "vrams": [ { "handleLayers": 14, "handleLastLayer": 18, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 2187788288, "nonuma": 3168727040 }, { "handleLayers": 13, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 436207616, "nonuma": 2914422784 } ] }, { "offloadLayers": 28, "fullOffloaded": false, "ram": { "handleLayers": 4, "handleLastLayer": 3, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 306310104, "nonuma": 463596504 }, "vrams": [ { "handleLayers": 14, "handleLastLayer": 17, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 2187788288, "nonuma": 3168727040 }, { "handleLayers": 14, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 469762048, "nonuma": 3070693376 } ] }, { "offloadLayers": 29, "fullOffloaded": false, "ram": { "handleLayers": 3, "handleLastLayer": 2, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 272755672, "nonuma": 430042072 }, "vrams": [ { "handleLayers": 15, "handleLastLayer": 17, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 2344058880, "nonuma": 3324997632 }, { "handleLayers": 14, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 469762048, "nonuma": 3070693376 } ] }, { "offloadLayers": 30, "fullOffloaded": false, "ram": { "handleLayers": 2, "handleLastLayer": 1, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 239201240, "nonuma": 396487640 }, "vrams": [ { "handleLayers": 15, "handleLastLayer": 16, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 2344058880, "nonuma": 3324997632 }, { "handleLayers": 15, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 503316480, "nonuma": 3226963968 } ] }, { "offloadLayers": 31, "fullOffloaded": false, "ram": { "handleLayers": 1, "handleLastLayer": 0, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 205646808, "nonuma": 362933208 }, "vrams": [ { "handleLayers": 16, "handleLastLayer": 16, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 2500329472, "nonuma": 3481268224 }, { "handleLayers": 15, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 503316480, "nonuma": 3226963968 } ] }, { "offloadLayers": 32, "fullOffloaded": false, "ram": { "handleLayers": 0, "handleLastLayer": -1, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 172092376, "nonuma": 329378776 }, "vrams": [ { "handleLayers": 16, "handleLastLayer": 15, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 2500329472, "nonuma": 3481268224 }, { "handleLayers": 16, "handleLastLayer": 31, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 536870912, "nonuma": 3383234560 } ] }, { "offloadLayers": 33, "fullOffloaded": true, "ram": { "handleLayers": 0, "handleLastLayer": -1, "handleOutputLayer": false, "remote": false, "position": 0, "uma": 171562968, "nonuma": 328849368 }, "vrams": [ { "handleLayers": 17, "handleLastLayer": 16, "handleOutputLayer": false, "remote": true, "position": 0, "uma": 2656600064, "nonuma": 3539505152 }, { "handleLayers": 15, "handleLastLayer": 31, "handleOutputLayer": true, "remote": false, "position": 0, "uma": 503845888, "nonuma": 3658449920 } ] } ], "type": "model", "architecture": "llama", "contextSize": 8192, "flashAttention": false, "noMMap": false, "embeddingOnly": false, "reranking": false, "distributable": true, "logicalBatchSize": 2048, "physicalBatchSize": 512 } }