#include #include #include #include #include "../la/amx.hpp" void test_kgroup_128() { std::cout << "=== Testing K-Group with k_group_size = 128 ===\n" << std::endl; const int m = 32; // Simple case const int n = 32; const int k = 512; // Multiple of 128 const int k_group_size = 128; std::cout << "Matrix dimensions: " << m << " x " << n << " x " << k << std::endl; std::cout << "K-group size: " << k_group_size << std::endl; std::cout << "Number of k-groups: " << k / k_group_size << std::endl; using Kernel = amx::GemmKernel224Int4KGroup; using BufferA = Kernel::BufferA; using BufferB = Kernel::BufferB; using BufferC = Kernel::BufferC; void* buffer_a = std::aligned_alloc(64, BufferA::required_size(m, k, k_group_size)); void* buffer_b = std::aligned_alloc(64, BufferB::required_size(n, k, k_group_size)); void* buffer_c = std::aligned_alloc(64, BufferC::required_size(m, n)); auto ba = std::make_shared(m, k, k_group_size, buffer_a); auto bb = std::make_shared(n, k, k_group_size, buffer_b); auto bc = std::make_shared(m, n, buffer_c); Kernel::config(); // Test 1: All ones std::cout << "\n--- Test 1: All ones (expected = " << k << ") ---" << std::endl; { std::vector input_a(m * k); std::vector input_b(k * n); for (int i = 0; i < m * k; i++) { input_a[i] = ggml_compute_fp32_to_bf16(1.0f); } for (int i = 0; i < k * n; i++) { input_b[i] = ggml_compute_fp32_to_bf16(1.0f); } ba->from_mat(m, input_a.data(), 0, 1); bb->from_mat(input_b.data(), 0, 1); amx::mat_mul_kgroup(m, n, k, k_group_size, ba, bb, bc, 0, 1); std::vector output(m * n); bc->to_mat(m, output.data(), 0, 1); float actual = ggml_compute_bf16_to_fp32(output[0]); float error = std::abs(actual - k) / k * 100; std::cout << "Result[0,0]: " << actual << " (error: " << error << "%)" << std::endl; } // Test 2: Values in quantization sweet spot (0.5) std::cout << "\n--- Test 2: All 0.5 (expected = " << 0.5f * 0.5f * k << ") ---" << std::endl; { std::vector input_a(m * k); std::vector input_b(k * n); for (int i = 0; i < m * k; i++) { input_a[i] = ggml_compute_fp32_to_bf16(0.5f); } for (int i = 0; i < k * n; i++) { input_b[i] = ggml_compute_fp32_to_bf16(0.5f); } ba->from_mat(m, input_a.data(), 0, 1); bb->from_mat(input_b.data(), 0, 1); amx::mat_mul_kgroup(m, n, k, k_group_size, ba, bb, bc, 0, 1); std::vector output(m * n); bc->to_mat(m, output.data(), 0, 1); float expected = 0.5f * 0.5f * k; float actual = ggml_compute_bf16_to_fp32(output[0]); float error = std::abs(actual - expected) / expected * 100; std::cout << "Result[0,0]: " << actual << " (expected: " << expected << ", error: " << error << "%)" << std::endl; } // Test 3: Different values per k-group std::cout << "\n--- Test 3: Different values per k-group ---" << std::endl; { std::vector input_a(m * k); std::vector input_b(k * n); // Each k-group has different value for (int i = 0; i < m; i++) { for (int j = 0; j < k; j++) { int kg = j / k_group_size; float val = (kg + 1) * 0.25f; // 0.25, 0.5, 0.75, 1.0 input_a[i * k + j] = ggml_compute_fp32_to_bf16(val); } } for (int i = 0; i < k * n; i++) { input_b[i] = ggml_compute_fp32_to_bf16(0.5f); } ba->from_mat(m, input_a.data(), 0, 1); bb->from_mat(input_b.data(), 0, 1); amx::mat_mul_kgroup(m, n, k, k_group_size, ba, bb, bc, 0, 1); std::vector output(m * n); bc->to_mat(m, output.data(), 0, 1); // Expected: sum of (kg+1)*0.25 * 0.5 * k_group_size for all k-groups float expected = 0.0f; for (int kg = 0; kg < k / k_group_size; kg++) { expected += (kg + 1) * 0.25f * 0.5f * k_group_size; } float actual = ggml_compute_bf16_to_fp32(output[0]); float error = std::abs(actual - expected) / expected * 100; std::cout << "Expected: " << expected << ", Actual: " << actual << std::endl; std::cout << "Error: " << error << "%" << std::endl; } // Test 4: Pattern test std::cout << "\n--- Test 4: Pattern with alternating values ---" << std::endl; { std::vector input_a(m * k); std::vector input_b(k * n); // Alternating pattern in A for (int i = 0; i < m * k; i++) { float val = (i % 2 == 0) ? 0.25f : 0.75f; input_a[i] = ggml_compute_fp32_to_bf16(val); } // Constant in B for (int i = 0; i < k * n; i++) { input_b[i] = ggml_compute_fp32_to_bf16(0.4f); } ba->from_mat(m, input_a.data(), 0, 1); bb->from_mat(input_b.data(), 0, 1); amx::mat_mul_kgroup(m, n, k, k_group_size, ba, bb, bc, 0, 1); std::vector output(m * n); bc->to_mat(m, output.data(), 0, 1); // Expected: average of 0.25 and 0.75 is 0.5, so 0.5 * 0.4 * k float expected = 0.5f * 0.4f * k; float actual = ggml_compute_bf16_to_fp32(output[0]); float error = std::abs(actual - expected) / expected * 100; std::cout << "Expected: " << expected << ", Actual: " << actual << std::endl; std::cout << "Error: " << error << "%" << std::endl; } // Test 5: Check all output elements std::cout << "\n--- Test 5: Verify all output elements (0.1 × 0.1) ---" << std::endl; { std::vector input_a(m * k); std::vector input_b(k * n); for (int i = 0; i < m * k; i++) { input_a[i] = ggml_compute_fp32_to_bf16(0.1f); } for (int i = 0; i < k * n; i++) { input_b[i] = ggml_compute_fp32_to_bf16(0.1f); } ba->from_mat(m, input_a.data(), 0, 1); bb->from_mat(input_b.data(), 0, 1); amx::mat_mul_kgroup(m, n, k, k_group_size, ba, bb, bc, 0, 1); std::vector output(m * n); bc->to_mat(m, output.data(), 0, 1); float expected = 0.1f * 0.1f * k; float max_error = 0.0f; float avg_error = 0.0f; int error_count = 0; for (int i = 0; i < m * n; i++) { float actual = ggml_compute_bf16_to_fp32(output[i]); float error = std::abs(actual - expected) / expected * 100; max_error = std::max(max_error, error); avg_error += error; if (error > 5.0f) error_count++; } avg_error /= (m * n); std::cout << "Expected value: " << expected << std::endl; std::cout << "Max error: " << max_error << "%" << std::endl; std::cout << "Average error: " << avg_error << "%" << std::endl; std::cout << "Elements with >5% error: " << error_count << "/" << m * n << std::endl; } // Test 6: Random normal distribution (like real model weights) std::cout << "\n--- Test 6: Random normal distribution ---" << std::endl; { std::vector input_a(m * k); std::vector input_b(k * n); std::mt19937 gen(42); std::normal_distribution dist(0.0f, 0.1f); for (int i = 0; i < m * k; i++) { input_a[i] = ggml_compute_fp32_to_bf16(dist(gen)); } for (int i = 0; i < k * n; i++) { input_b[i] = ggml_compute_fp32_to_bf16(dist(gen)); } // Compute reference with float32 std::vector ref_result(m * n, 0.0f); for (int i = 0; i < m; i++) { for (int j = 0; j < n; j++) { float sum = 0.0f; for (int l = 0; l < k; l++) { float a_val = ggml_compute_bf16_to_fp32(input_a[i * k + l]); float b_val = ggml_compute_bf16_to_fp32(input_b[l * n + j]); sum += a_val * b_val; } ref_result[i * n + j] = sum; } } ba->from_mat(m, input_a.data(), 0, 1); bb->from_mat(input_b.data(), 0, 1); amx::mat_mul_kgroup(m, n, k, k_group_size, ba, bb, bc, 0, 1); std::vector output(m * n); bc->to_mat(m, output.data(), 0, 1); // Compute errors float max_abs_error = 0.0f; float max_rel_error = 0.0f; float avg_rel_error = 0.0f; int large_error_count = 0; for (int i = 0; i < m * n; i++) { float actual = ggml_compute_bf16_to_fp32(output[i]); float ref = ref_result[i]; float abs_error = std::abs(actual - ref); float rel_error = std::abs(ref) > 1e-6 ? abs_error / std::abs(ref) : 0.0f; max_abs_error = std::max(max_abs_error, abs_error); max_rel_error = std::max(max_rel_error, rel_error); avg_rel_error += rel_error; if (rel_error > 0.2f) { // 20% error large_error_count++; if (large_error_count <= 5) { std::cout << " [" << i / n << "," << i % n << "]: actual=" << actual << ", ref=" << ref << ", rel_error=" << (rel_error * 100) << "%" << std::endl; } } } avg_rel_error /= (m * n); std::cout << "Max absolute error: " << max_abs_error << std::endl; std::cout << "Max relative error: " << (max_rel_error * 100) << "%" << std::endl; std::cout << "Average relative error: " << (avg_rel_error * 100) << "%" << std::endl; std::cout << "Elements with >20% error: " << large_error_count << "/" << m * n << std::endl; } free(buffer_a); free(buffer_b); free(buffer_c); } int main() { test_kgroup_128(); return 0; }