#include #include "../la/amx.hpp" #define FMT_HEADER_ONLY #include #include #include #include #include void debug_simple_multiplication() { std::cout << "=== Debug Simple K-Group Multiplication ===" << std::endl; // Very small test case for debugging const int m = 32; // 1 M_STEP const int n = 32; // 1 N_STEP const int k = 512; // Must be at least K_BLOCK (512) const int k_group_size = 128; std::cout << fmt::format("Parameters: m={}, n={}, k={}, k_group_size={}\n", m, n, k, k_group_size); using Kernel = amx::GemmKernel224Int4KGroup; using BufferA = Kernel::BufferA; using BufferB = Kernel::BufferB; using BufferC = Kernel::BufferC; // Allocate buffers void* buffer_a = std::aligned_alloc(64, BufferA::required_size(m, k, k_group_size)); void* buffer_b = std::aligned_alloc(64, BufferB::required_size(n, k, k_group_size)); void* buffer_c = std::aligned_alloc(64, BufferC::required_size(m, n)); auto ba = std::make_shared(m, k, k_group_size, buffer_a); auto bb = std::make_shared(n, k, k_group_size, buffer_b); auto bc = std::make_shared(m, n, buffer_c); // Create identity-like matrices for easy verification std::vector input_a(m * k); std::vector input_b(k * n); // Initialize A as mostly zeros with a few ones for (int i = 0; i < m * k; i++) { input_a[i] = ggml_compute_fp32_to_bf16(0.0f); } // Set A[0,0] = 1 input_a[0] = ggml_compute_fp32_to_bf16(1.0f); // Initialize B as mostly zeros with a few ones for (int i = 0; i < k * n; i++) { input_b[i] = ggml_compute_fp32_to_bf16(0.0f); } // Set B[0,0] = 1 input_b[0] = ggml_compute_fp32_to_bf16(1.0f); // Expected result: C[0,0] = 1*1 = 1, rest = 0 std::cout << "\nExpected result: C[0,0] = 1.0, rest = 0.0\n" << std::endl; // Quantize inputs ba->from_mat(m, input_a.data(), 0, 1); bb->from_mat(input_b.data(), 0, 1); // Print scales for debugging std::cout << "BufferA scales for row 0:" << std::endl; for (int kg = 0; kg < k / k_group_size; kg++) { float scale = *ba->get_scale(m, 0, k, kg * k_group_size); std::cout << fmt::format(" k_group[{}]: scale = {:.6f}\n", kg, scale); } std::cout << "\nBufferB scales for col 0:" << std::endl; for (int kg = 0; kg < k / k_group_size; kg++) { float scale = *bb->get_scale(n, 0, k, kg * k_group_size); std::cout << fmt::format(" k_group[{}]: scale = {:.6f}\n", kg, scale); } // Configure AMX Kernel::config(); // Run matrix multiplication amx::mat_mul_kgroup(m, n, k, k_group_size, ba, bb, bc, 0, 1); // Get output std::vector output(m * n); bc->to_mat(m, output.data(), 0, 1); // Print results std::cout << "\nActual result (first 5x5):" << std::endl; for (int i = 0; i < std::min(5, m); i++) { for (int j = 0; j < std::min(5, n); j++) { float val = ggml_compute_bf16_to_fp32(output[i * n + j]); std::cout << fmt::format("{:8.4f} ", val); } std::cout << std::endl; } free(buffer_a); free(buffer_b); free(buffer_c); } void debug_pattern_multiplication() { std::cout << "\n=== Debug Pattern Multiplication ===" << std::endl; const int m = 32; const int n = 32; const int k = 512; // Must be at least K_BLOCK (512) const int k_group_size = 128; using Kernel = amx::GemmKernel224Int4KGroup; using BufferA = Kernel::BufferA; using BufferB = Kernel::BufferB; using BufferC = Kernel::BufferC; void* buffer_a = std::aligned_alloc(64, BufferA::required_size(m, k, k_group_size)); void* buffer_b = std::aligned_alloc(64, BufferB::required_size(n, k, k_group_size)); void* buffer_c = std::aligned_alloc(64, BufferC::required_size(m, n)); auto ba = std::make_shared(m, k, k_group_size, buffer_a); auto bb = std::make_shared(n, k, k_group_size, buffer_b); auto bc = std::make_shared(m, n, buffer_c); // Create constant matrices std::vector input_a(m * k); std::vector input_b(k * n); // Fill A with 0.1 and B with 0.1 for (int i = 0; i < m * k; i++) { input_a[i] = ggml_compute_fp32_to_bf16(0.1f); } for (int i = 0; i < k * n; i++) { input_b[i] = ggml_compute_fp32_to_bf16(0.1f); } // Expected: Each element should be 0.1 * 0.1 * k = 0.01 * 512 = 5.12 float expected = 0.1f * 0.1f * k; std::cout << fmt::format("\nExpected result: all elements = {:.4f}\n", expected); // Quantize ba->from_mat(m, input_a.data(), 0, 1); bb->from_mat(input_b.data(), 0, 1); // Run Kernel::config(); amx::mat_mul_kgroup(m, n, k, k_group_size, ba, bb, bc, 0, 1); // Get output std::vector output(m * n); bc->to_mat(m, output.data(), 0, 1); // Check results float max_error = 0.0f; float avg_error = 0.0f; for (int i = 0; i < m * n; i++) { float actual = ggml_compute_bf16_to_fp32(output[i]); float error = std::abs(actual - expected); max_error = std::max(max_error, error); avg_error += error; } avg_error /= (m * n); std::cout << fmt::format("Max error: {:.6f}\n", max_error); std::cout << fmt::format("Avg error: {:.6f}\n", avg_error); std::cout << fmt::format("Relative error: {:.2f}%\n", (max_error / expected) * 100); // Print sample values std::cout << "\nSample values (first 5x5):" << std::endl; for (int i = 0; i < std::min(5, m); i++) { for (int j = 0; j < std::min(5, n); j++) { float val = ggml_compute_bf16_to_fp32(output[i * n + j]); std::cout << fmt::format("{:8.4f} ", val); } std::cout << std::endl; } free(buffer_a); free(buffer_b); free(buffer_c); } void compare_with_regular_int4() { std::cout << "\n=== Compare K-Group vs Regular INT4 ===" << std::endl; const int m = 32; const int n = 32; const int k = 512; const int k_group_size = 128; // Create test data std::vector input_a(m * k); std::vector input_b(k * n); std::mt19937 gen(42); std::uniform_real_distribution dist(-0.1f, 0.1f); for (int i = 0; i < m * k; i++) { input_a[i] = ggml_compute_fp32_to_bf16(dist(gen)); } for (int i = 0; i < k * n; i++) { input_b[i] = ggml_compute_fp32_to_bf16(dist(gen)); } // Test with regular INT4 { using Kernel = amx::GemmKernel224Int4; using BufferA = Kernel::BufferA; using BufferB = Kernel::BufferB; using BufferC = Kernel::BufferC; void* buffer_a = std::aligned_alloc(64, BufferA::required_size(m, k)); void* buffer_b = std::aligned_alloc(64, BufferB::required_size(n, k)); // Fixed: n, k not k, n void* buffer_c = std::aligned_alloc(64, BufferC::required_size(m, n)); auto ba = std::make_shared(m, k, buffer_a); auto bb = std::make_shared(n, k, buffer_b); // Fixed: n, k not k, n auto bc = std::make_shared(m, n, buffer_c); ba->from_mat(m, input_a.data(), 0, 1); bb->from_mat(input_b.data(), 0, 1); Kernel::config(); amx::mat_mul(m, n, k, ba, bb, bc, 0, 1); std::vector output_regular(m * n); bc->to_mat(m, output_regular.data(), 0, 1); std::cout << "Regular INT4 results (first 3x3):" << std::endl; for (int i = 0; i < 3; i++) { for (int j = 0; j < 3; j++) { float val = ggml_compute_bf16_to_fp32(output_regular[i * n + j]); std::cout << fmt::format("{:8.4f} ", val); } std::cout << std::endl; } free(buffer_a); free(buffer_b); free(buffer_c); } // Test with K-Group INT4 { using Kernel = amx::GemmKernel224Int4KGroup; using BufferA = Kernel::BufferA; using BufferB = Kernel::BufferB; using BufferC = Kernel::BufferC; void* buffer_a = std::aligned_alloc(64, BufferA::required_size(m, k, k_group_size)); void* buffer_b = std::aligned_alloc(64, BufferB::required_size(n, k, k_group_size)); void* buffer_c = std::aligned_alloc(64, BufferC::required_size(m, n)); auto ba = std::make_shared(m, k, k_group_size, buffer_a); auto bb = std::make_shared(n, k, k_group_size, buffer_b); auto bc = std::make_shared(m, n, buffer_c); ba->from_mat(m, input_a.data(), 0, 1); bb->from_mat(input_b.data(), 0, 1); Kernel::config(); amx::mat_mul_kgroup(m, n, k, k_group_size, ba, bb, bc, 0, 1); std::vector output_kgroup(m * n); bc->to_mat(m, output_kgroup.data(), 0, 1); std::cout << "\nK-Group INT4 results (first 3x3):" << std::endl; for (int i = 0; i < 3; i++) { for (int j = 0; j < 3; j++) { float val = ggml_compute_bf16_to_fp32(output_kgroup[i * n + j]); std::cout << fmt::format("{:8.4f} ", val); } std::cout << std::endl; } free(buffer_a); free(buffer_b); free(buffer_c); } } int main() { std::cout << "Starting K-Group Debugging\n" << std::endl; debug_simple_multiplication(); debug_pattern_multiplication(); compare_with_regular_int4(); return 0; }