File size: 3,399 Bytes
e9009a3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 | <svg viewBox="0 0 1100 660" xmlns="http://www.w3.org/2000/svg" font-family="-apple-system, Helvetica, Arial, sans-serif">
<rect width="1100" height="660" fill="#ffffff"/>
<text x="50" y="58" font-size="34" font-weight="700" fill="#0f172a">Sensitivity-graded 3.6-bit recipe</text>
<text x="50" y="92" font-size="20" fill="#475569">3.62 bits/weight · 465 GB · routed experts ≈ 99% of params set the size</text>
<!-- bit scale legend -->
<g font-size="16" fill="#334155">
<rect x="700" y="40" width="22" height="22" rx="4" fill="#f59e0b"/><text x="728" y="57">3-bit</text>
<rect x="790" y="40" width="22" height="22" rx="4" fill="#f97316"/><text x="818" y="57">4-bit</text>
<rect x="880" y="40" width="22" height="22" rx="4" fill="#3b82f6"/><text x="908" y="57">6-bit</text>
<rect x="970" y="40" width="22" height="22" rx="4" fill="#10b981"/><text x="998" y="57">bf16</text>
</g>
<!-- rows: x bar origin 360, scale 70px per bit -->
<g font-size="20">
<!-- Experts gate/up : 3-bit -->
<text x="50" y="168" font-weight="600" fill="#0f172a">Experts gate / up</text>
<text x="50" y="192" font-size="15" fill="#64748b">≈ 66% of all params</text>
<rect x="360" y="148" width="210" height="40" rx="6" fill="#f59e0b"/>
<text x="585" y="175" font-size="18" fill="#334155">3-bit · g64</text>
<!-- Experts down_proj : 3-4-bit -->
<text x="50" y="246" font-weight="600" fill="#0f172a">Experts down_proj</text>
<text x="50" y="270" font-size="15" fill="#64748b">≈ 33% · the residual-writing matrix</text>
<rect x="360" y="226" width="210" height="40" rx="6" fill="#f59e0b"/>
<rect x="360" y="226" width="280" height="40" rx="6" fill="#f97316" opacity="0.0"/>
<rect x="570" y="226" width="70" height="40" rx="0" fill="#f97316"/>
<text x="655" y="253" font-size="18" fill="#334155">3-bit, upgraded to 4-bit on 16 / 60 layers</text>
<!-- Attention MLA : 6-bit -->
<text x="50" y="324" font-weight="600" fill="#0f172a">Attention (MLA)</text>
<text x="50" y="348" font-size="15" fill="#64748b">q_a / q_b / kv_a / kv_b / o</text>
<rect x="360" y="304" width="420" height="40" rx="6" fill="#3b82f6"/>
<text x="795" y="331" font-size="18" fill="#334155">6-bit</text>
<!-- Shared + dense : 6-bit -->
<text x="50" y="402" font-weight="600" fill="#0f172a">Shared expert · dense MLP</text>
<rect x="360" y="382" width="420" height="40" rx="6" fill="#3b82f6"/>
<text x="795" y="409" font-size="18" fill="#334155">6-bit</text>
<!-- Embed/head : 6-bit -->
<text x="50" y="480" font-weight="600" fill="#0f172a">Token embedding · LM head</text>
<rect x="360" y="460" width="420" height="40" rx="6" fill="#3b82f6"/>
<text x="795" y="487" font-size="18" fill="#334155">6-bit</text>
<!-- Router : bf16 -->
<text x="50" y="558" font-weight="600" fill="#0f172a">MoE router gate</text>
<text x="50" y="582" font-size="15" fill="#64748b">drives discrete top-8 routing</text>
<rect x="360" y="538" width="430" height="40" rx="6" fill="#10b981"/>
<text x="805" y="565" font-size="18" fill="#334155">bf16 — full precision</text>
</g>
<line x1="50" y1="612" x2="1050" y2="612" stroke="#e2e8f0" stroke-width="1.5"/>
<text x="50" y="640" font-size="16" fill="#64748b">Cheap bits on the robust expert bulk; full precision on the discrete-control path.</text>
</svg>
|