/* ffn_timing_test: measure each FFN phase in isolation AND in the real * interleaved pipeline, with the real buffer footprint, to find why in-engine * GEMMs run ~55-62% slower than the gemm_tune microbenchmark. * * The engine's FFN working set per layer (M=256, C=chunk): * bvnni_d device C*FF_INTER = 4.5 MB (B operand, reused gate/up/down) * suma_d device (FF_INTER/64)*C = 278 KB (B operand, reused) * act_scale shared C = 1 KB * x shared C*D_MODEL = 5.2 MB (residual, RMW by down-proj) * ffn_g shared C*FF_INTER = 17.8 MB (gate output) * ffn_u shared C*FF_INTER = 17.8 MB (up output) * ffn_t shared C*FF_INTER = 17.8 MB (swiglu output, down-proj B src) * Total live: ~75 MB >> 16 MB L2. * * Builds with engine_sources_wyut (the GEMM is shared; DeltaNet impl irrelevant). */ #include #include #include #include #include #include #include #include #include #include "qxmx.h" #include "qxmx_gpu.h" #include "qxmx_gemm.h" using namespace qx; static std::mt19937_64 rng(0xF1FFULL); static uint16_t f32_to_f16_bits(float f) { return sycl::bit_cast(sycl::half(f)); } static float f16_to_f32_local(uint16_t b) { return float(sycl::bit_cast(b)); } static std::vector synth_q2(int M, int K) { int ngrp = K / QX_QK2; std::vector W((size_t)M * ngrp); std::normal_distribution g(0, 1); for (int m = 0; m < M; m++) for (int gg = 0; gg < ngrp; gg++) { W[(size_t)m * ngrp + gg].d = f32_to_f16_bits(0.05f * std::fabs(g(rng)) + 0.001f); for (int b = 0; b < QX_QK2 / 4; b++) { uint8_t p = 0; for (int k = 0; k < 4; k++) p |= (uint8_t)((rng() % 3) << (k * 2)); W[(size_t)m * ngrp + gg].qs[b] = p; } } return W; } /* Repack to A_codes [M/8][ngrp][2][64B] + A_sc_t [ngrp][M] with M padded to 256 * (WGM8=32 default loads a 256-row tile unguarded). */ static dev_q2 up(sycl::queue& q, const std::vector& W, int M, int K) { int ngrp = K / QX_QK2; int Mp = (M + 255) & ~255; size_t csz = (size_t)Mp * ngrp * 16; size_t ssz = (size_t)ngrp * M + 256; uint8_t* ch = (uint8_t*)calloc(1, csz); float* sh = (float*)calloc(1, ssz * 4); for (int m8 = 0; m8 < M / 8; m8++) for (int gg = 0; gg < ngrp; gg++) for (int hf = 0; hf < 2; hf++) for (int r = 0; r < 8; r++) memcpy(ch + (((size_t)(m8 * ngrp + gg) * 2 + hf) * 64 + r * 8), W[(m8 * 8 + r) * ngrp + gg].qs + hf * 8, 8); for (int m = 0; m < M; m++) for (int gg = 0; gg < ngrp; gg++) sh[gg * M + m] = f16_to_f32_local(W[m * ngrp + gg].d); dev_q2 d; d.codes = sycl::malloc_shared(csz, q); d.scales = sycl::malloc_shared(ssz, q); q.memcpy(d.codes, ch, csz).wait(); q.memcpy(d.scales, sh, ssz * 4).wait(); free(ch); free(sh); d.M = M; d.K = K; d.src = nullptr; return d; } static float* upf(sycl::queue& q, const std::vector& v) { float* d = sycl::malloc_shared(v.size(), q); q.memcpy(d, v.data(), v.size() * 4).wait(); return d; } int main() { sycl::queue q(sycl::gpu_selector_v, sycl::property::queue::in_order{}); const int C = 256; /* chunk size */ const int Md = QX_D_MODEL; /* 5120 */ const int Mf = QX_FF_INTER; /* 17408 */ printf("ffn_timing_test: M=%d (chunk). FFN working set ~75 MB >> 16 MB L2.\n", C); dev_ffn_blk f; f.gate = up(q, synth_q2(Mf, Md), Mf, Md); f.up = up(q, synth_q2(Mf, Md), Mf, Md); f.down = up(q, synth_q2(Md, Mf), Md, Mf); { std::normal_distribution g(0,1); std::vector w(Md); for(int i=0;i((size_t)Mf * C, q); /* 4.5 MB */ int32_t* sum = sycl::malloc_device((size_t)(Mf / QX_QK2) * C, q); /* 278 KB */ float* asc = sycl::malloc_shared(C, q); float* g_b = sycl::malloc_shared((size_t)C * Mf, q); /* 17.8 MB */ float* u_b = sycl::malloc_shared((size_t)C * Mf, q); /* 17.8 MB */ float* t_b = sycl::malloc_shared((size_t)C * Mf, q); /* 17.8 MB */ float* x = sycl::malloc_shared((size_t)C * Md, q); /* 5.2 MB */ /* init x with varied row scales (matches the engine's residual stream) */ { std::normal_distribution gn(0,1); for (int t = 0; t < C; t++) { float sc = (t%17==3)?0.0f:powf(10,-2+3*((t%13)/12.0f)); for (int i = 0; i < Md; i++) x[(size_t)t*Md+i] = gn(rng)*sc; } } auto time = [&](const char* name, auto fn, int reps) { q.wait(); auto t0 = std::chrono::steady_clock::now(); for (int r = 0; r < reps; r++) fn(); q.wait(); auto t1 = std::chrono::steady_clock::now(); printf(" %-20s %.3f ms/call\n", name, std::chrono::duration(t1-t0).count()/reps); }; /* ---- 1. Each phase in isolation (cold L2 each rep via interleaved * dummy traffic between reps would be ideal, but the engine itself is * the real interleaver; the isolated numbers here match gemm_tune). */ printf("\n[1] Phase in isolation (same buffers, no interleaving):\n"); time("rmsnorm_quant", [&]{ gpu_rmsnorm_quant(q,x,f.post_norm,Md,bvn,sum,asc,C); }, 30); time("gate+up dual", [&]{ gemm_gpu_v4_dual(q,f.gate.codes,f.gate.scales, f.up.codes,f.up.scales, bvn,sum,asc,g_b,u_b,Mf,Md,C, nullptr,nullptr,false,true); }, 30); time("swiglu_quant", [&]{ gpu_swiglu_quant(q,g_b,u_b,t_b,Mf,bvn,sum,asc,C); }, 30); time("down (accum=x)",[&]{ gemm_gpu_v4(q,f.down.codes,f.down.scales,bvn,sum,asc, x,Md,Mf,C,/*accum=*/x,false,true); }, 30); /* ---- 2. The real interleaved FFN pipeline (one full ffn_forward_b call). */ printf("\n[2] Full ffn_forward_b (real interleaving):\n"); time("ffn_forward_b", [&]{ ffn_forward_b(q,&f,x,C,bvn,sum,asc,g_b,u_b,t_b); }, 30); /* ---- 3. GEMM-only with a cold-L2 producer in front (rmsnorm_quant then * immediately the GEMM, no other traffic) — isolates whether the * rmsnorm_quant immediately preceding evicts what the GEMM wants. */ printf("\n[3] rmsnorm_quant + gate+up dual back-to-back (B produced by rmsnorm):\n"); time("rmsnorm+gateup", [&]{ gpu_rmsnorm_quant(q,x,f.post_norm,Md,bvn,sum,asc,C); gemm_gpu_v4_dual(q,f.gate.codes,f.gate.scales,f.up.codes,f.up.scales, bvn,sum,asc,g_b,u_b,Mf,Md,C,nullptr,nullptr,false,true); }, 30); /* ---- 4. Does the down-proj's accum=x RMW hurt? Compare with/without. */ printf("\n[4] down GEMM: accum=x (RMW residual) vs no accum (fresh write):\n"); time("down accum=x", [&]{ gemm_gpu_v4(q,f.down.codes,f.down.scales,bvn,sum,asc,x,Md,Mf,C,/*accum=*/x,false,true); }, 30); time("down no accum", [&]{ gemm_gpu_v4(q,f.down.codes,f.down.scales,bvn,sum,asc,t_b,Md,Mf,C,nullptr,false,true); }, 30); /* ---- 5. Repeated GEMM (same shape, B already in L2 from prior rep). */ printf("\n[5] gate+up dual repeated (B warm in L2 from prior rep):\n"); time("gate+up x60 warm", [&]{ gemm_gpu_v4_dual(q,f.gate.codes,f.gate.scales,f.up.codes,f.up.scales,bvn,sum,asc,g_b,u_b,Mf,Md,C,nullptr,nullptr,false,true); }, 60); /* ---- 6. USM kind A/B/C: shared (engine) vs device. The engine uses * malloc_shared for ffn_g/u/t/x (the GEMM C outputs + accum); the * microbenchmark used malloc_device. Shared USM on Xe2 may run * through a host-coherent path that's slower for device-only traffic. */ printf("\n[6] USM kind: C output shared (engine) vs device:\n"); float* g_dev = sycl::malloc_device((size_t)C * Mf, q); float* u_dev = sycl::malloc_device((size_t)C * Mf, q); time("gate+up C=shared", [&]{ gemm_gpu_v4_dual(q,f.gate.codes,f.gate.scales,f.up.codes,f.up.scales,bvn,sum,asc,g_b,u_b,Mf,Md,C,nullptr,nullptr,false,true); }, 60); time("gate+up C=device", [&]{ gemm_gpu_v4_dual(q,f.gate.codes,f.gate.scales,f.up.codes,f.up.scales,bvn,sum,asc,g_dev,u_dev,Mf,Md,C,nullptr,nullptr,false,true); }, 60); printf("\n[7] USM kind: B operand shared vs device (B is bvnni from rmsnorm):\n"); int8_t* bvn_sh = sycl::malloc_shared((size_t)Mf * C, q); q.memcpy(bvn_sh, bvn, (size_t)Mf * C).wait(); time("gate+up B=device", [&]{ gemm_gpu_v4_dual(q,f.gate.codes,f.gate.scales,f.up.codes,f.up.scales,bvn,sum,asc,g_dev,u_dev,Mf,Md,C,nullptr,nullptr,false,true); }, 60); time("gate+up B=shared", [&]{ gemm_gpu_v4_dual(q,f.gate.codes,f.gate.scales,f.up.codes,f.up.scales,bvn_sh,sum,asc,g_dev,u_dev,Mf,Md,C,nullptr,nullptr,false,true); }, 60); printf("\n[8] USM kind: A_codes/A_sc shared (engine upload_q2) vs device:\n"); /* f.gate/up.codes/scales are already malloc_shared (engine layout). Make * device-only copies to compare. */ size_t gc_sz = (size_t)((f.gate.M + 255 & ~255)) * (f.gate.K / QX_QK2) * 16; size_t gs_sz = (size_t)(f.gate.K / QX_QK2) * f.gate.M + 256; uint8_t* g_codes_dev = sycl::malloc_device(gc_sz, q); float* g_sc_dev = sycl::malloc_device(gs_sz, q); uint8_t* u_codes_dev = sycl::malloc_device(gc_sz, q); float* u_sc_dev = sycl::malloc_device(gs_sz, q); q.memcpy(g_codes_dev, f.gate.codes, gc_sz).wait(); q.memcpy(g_sc_dev, f.gate.scales, gs_sz*4).wait(); q.memcpy(u_codes_dev, f.up.codes, gc_sz).wait(); q.memcpy(u_sc_dev, f.up.scales, gs_sz*4).wait(); time("gate+up A=shared (engine)", [&]{ gemm_gpu_v4_dual(q,f.gate.codes,f.gate.scales,f.up.codes,f.up.scales,bvn,sum,asc,g_dev,u_dev,Mf,Md,C,nullptr,nullptr,false,true); }, 60); time("gate+up A=device", [&]{ gemm_gpu_v4_dual(q,g_codes_dev,g_sc_dev,u_codes_dev,u_sc_dev,bvn,sum,asc,g_dev,u_dev,Mf,Md,C,nullptr,nullptr,false,true); }, 60); /* ---- 9. The gemm_tune artifact: trivial zero A (0x55) + zero B + 1.0 * scales. Does the degenerate workload run faster than real values? */ printf("\n[9] Trivial zero A (0x55) + zero B + 1.0 scales (gemm_tune's load):\n"); size_t codes_sz = (size_t)((Mf + 255 & ~255)) * (Md / QX_QK2) * 16; size_t sc_sz = (size_t)(Md / QX_QK2) * Mf + 256; uint8_t* codes_z = sycl::malloc_shared(codes_sz, q); float* sc_z = sycl::malloc_shared(sc_sz, q); int8_t* bvn_z = sycl::malloc_shared((size_t)Md * C, q); /* K-sized */ int32_t* sum_z = sycl::malloc_shared((size_t)(Md / QX_QK2) * C, q); float* asc_z = sycl::malloc_shared(C, q); q.memset(codes_z, 0x55, codes_sz).wait(); { std::vector ones(sc_sz, 1.0f); q.memcpy(sc_z, ones.data(), sc_sz*4).wait(); } q.memset(bvn_z, 0, (size_t)Md * C).wait(); q.memset(sum_z, 0, (size_t)(Md / QX_QK2) * C * 4).wait(); { std::vector ones(C, 1.0f); q.memcpy(asc_z, ones.data(), C*4).wait(); } time("gate+up trivial-zero", [&]{ gemm_gpu_v4_dual(q,codes_z,sc_z,codes_z,sc_z,bvn_z,sum_z,asc_z,g_dev,u_dev,Mf,Md,C,nullptr,nullptr,false,true); }, 60); time("gate+up real-B", [&]{ gemm_gpu_v4_dual(q,f.gate.codes,f.gate.scales,f.up.codes,f.up.scales,bvn,sum,asc,g_dev,u_dev,Mf,Md,C,nullptr,nullptr,false,true); }, 60); sycl::free(codes_z,q); sycl::free(sc_z,q); sycl::free(bvn_z,q); sycl::free(sum_z,q); sycl::free(asc_z,q); sycl::free(g_dev,q); sycl::free(u_dev,q); sycl::free(bvn_sh,q); sycl::free(g_codes_dev,q); sycl::free(g_sc_dev,q); sycl::free(u_codes_dev,q); sycl::free(u_sc_dev,q); sycl::free(bvn,q); sycl::free(sum,q); sycl::free(asc,q); sycl::free(g_b,q); sycl::free(u_b,q); sycl::free(t_b,q); sycl::free(x,q); return 0; }