chore: import upstream snapshot with attribution

2026-07-13 13:30:03 +08:00
commit ec436095dd
1232 changed files with 404407 additions and 0 deletions
@@ -0,0 +1,88 @@
+#include <arm_sve.h>
+
+#include <cmath>
+#include <cstdlib>
+#include <fstream>
+#include <stdexcept>
+
+#include "../../reduce.hpp"
+#include "../../rms-norm.hpp"
+#include "../../rope.hpp"
+#include "../../softmax.hpp"
+#include "../la/arm_kml.hpp"
+#include "llama.cpp/ggml-common.h"
+#include "llama.cpp/ggml.h"
+
+void bf16_to_fp16(const ggml_bf16_t* src, ggml_fp16_t* dst, size_t n) {
+  for (size_t i = 0; i < n; ++i) {
+    float x = ggml_bf16_to_fp32(src[i]);
+    dst[i] = ggml_fp32_to_fp16(x);
+  }
+}
+
+void debug_rope() {
+  float16_t* fp16 = new float16_t[1024 * 64];
+
+  for (size_t i = 0; i < 1024 * 64; i++) {
+    fp16[i] = static_cast<double>(std::rand()) / RAND_MAX;
+  }
+  std::ofstream("before_rope", std::ios::binary).write((char*)fp16, 1024 * 64 * sizeof(float16_t));
+
+  DeepseekV3YarnRotaryEmbedding rope(64, 163840, 10000, 40, 4096, 32, 1, 1, 1);
+
+  rope.init(1024);
+
+  Rope<DeepseekV3YarnRotaryEmbedding, float16_t> rope_applier;
+  rope_applier.apply_multiple(rope, fp16, 64, 64, 0, 1024);
+
+  std::ofstream("cos", std::ios::binary).write((char*)rope.cos(0), 1024 * 32 * sizeof(float));
+  std::ofstream("sin", std::ios::binary).write((char*)rope.sin(0), 1024 * 32 * sizeof(float));
+
+  std::ofstream("after_rope", std::ios::binary).write((char*)fp16, 1024 * 64 * sizeof(float16_t));
+}
+
+void debug_softmax() {
+  float16_t* fp16 = new float16_t[64 * 1024];
+
+  for (size_t i = 0; i < 1024 * 64; i++) {
+    fp16[i] = static_cast<double>(std::rand()) / RAND_MAX * 10;
+    if (i % 12 == 0) {
+      fp16[i] -= std::numeric_limits<float16_t>::infinity();
+    }
+  }
+  std::ofstream("before_softmax", std::ios::binary).write((char*)fp16, 1024 * 64 * sizeof(float16_t));
+
+  Softmax<float16_t>::apply_multiple(64, fp16, 1024, 1024);
+  std::ofstream("after_softmax", std::ios::binary).write((char*)fp16, 1024 * 64 * sizeof(float16_t));
+}
+
+void debug_inf() {
+  float16_t x, y;
+  // x = std::numeric_limits<float16_t>::infinity(); // 0.00
+  // y = -std::numeric_limits<float16_t>::infinity(); // -0.00
+  // x = 1e10;
+  x = std::numeric_limits<float>::infinity();   // inf
+  y = -std::numeric_limits<float>::infinity();  // -inf
+  printf("x = %f, y = %f\n", x, y);
+}
+
+void debug_reduce() {
+  std::vector<float16_t*> fp16s(128);
+  for (size_t i = 0; i < 128; i++) {
+    fp16s[i] = new float16_t[1024];
+    for (size_t j = 0; j < 1024; j++) {
+      fp16s[i][j] = i;
+    }
+  }
+
+  reduce_sum(fp16s.data(), 128, 0, 10);
+  for (int i = 0; i < 10; i++) {
+    printf("%f ", fp16s[0][i]);
+  }
+}
+
+int main() {
+  debug_reduce();
+
+  return 0;
+}
@@ -0,0 +1,59 @@
+#ifndef KML_DEBUG_HPP
+#define KML_DEBUG_HPP
+
+#include <arm_sve.h>
+
+#include <cstdint>
+#include <cstdlib>
+#include <fstream>
+#include <string>
+
+inline std::string get_env_or_default(const char* var_name, const std::string& default_value) {
+  const char* value = std::getenv(var_name);
+  return (value != nullptr) ? std::string(value) : default_value;
+}
+
+inline void dump_bin(std::string file_name, float16_t* data, size_t count) {
+  file_name = get_env_or_default("KML_DEBUG_PATH", "debug") + "/" + file_name + ".f16";
+  std::ofstream f(file_name, std::ios::binary);
+  f.write(reinterpret_cast<const char*>(data), count * sizeof(*data));
+  f.close();
+}
+inline void dump_bin(std::string file_name, float* data, size_t count) {
+  file_name = get_env_or_default("KML_DEBUG_PATH", "debug") + "/" + file_name + ".f32";
+  std::ofstream f(file_name, std::ios::binary);
+  f.write(reinterpret_cast<const char*>(data), count * sizeof(*data));
+  f.close();
+}
+inline void dump_bin(std::string file_name, int64_t* data, size_t count) {
+  file_name = get_env_or_default("KML_DEBUG_PATH", "debug") + "/" + file_name + ".int64";
+  std::ofstream f(file_name, std::ios::binary);
+  f.write(reinterpret_cast<const char*>(data), count * sizeof(*data));
+  f.close();
+}
+
+inline void dump_bin(std::string file_name, int8_t* data, size_t count) {
+  file_name = get_env_or_default("KML_DEBUG_PATH", "debug") + "/" + file_name + ".int8";
+  std::ofstream f(file_name, std::ios::binary);
+  f.write(reinterpret_cast<const char*>(data), count * sizeof(*data));
+  f.close();
+}
+
+inline void dump_bin(std::string file_name, int32_t* data, size_t count) {
+  file_name = get_env_or_default("KML_DEBUG_PATH", "debug") + "/" + file_name + ".int32";
+  std::ofstream f(file_name, std::ios::binary);
+  f.write(reinterpret_cast<const char*>(data), count * sizeof(*data));
+  f.close();
+}
+
+inline void load_bin(std::string file_name, float* data, size_t count) {
+  file_name = get_env_or_default("KML_DEBUG_PATH", "debug") + "/" + file_name + ".f32";
+  std::ifstream f(file_name, std::ios::binary);
+  if (!f.is_open()) {
+    throw std::runtime_error("Failed to open file: " + file_name);
+  }
+  f.read(reinterpret_cast<char*>(data), count * sizeof(*data));
+  f.close();
+}
+
+#endif
@@ -0,0 +1,110 @@
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+
+#include "../la/arm_kml.hpp"
+#include "debug.hpp"
+#include "kblas.h"
+const int M = 1, K = 7168, N = 8;
+
+int main() {
+  // 随机生成a, b, c矩阵
+  arm_kml::GemmKernelInt4::BufferA buffer_a(M, K);
+  arm_kml::GemmKernelInt4::BufferB buffer_b(N, K, true);
+  arm_kml::GemmKernelInt4::BufferC buffer_c(M, N);
+
+  arm_kml::GemmKernelInt8::BufferA buffer_a_check(M, K);
+  arm_kml::GemmKernelInt8::BufferB buffer_b_check(N, K, true);
+  arm_kml::GemmKernelInt8::BufferC buffer_c_check(M, N);
+
+  float* a = (float*)aligned_alloc(64, sizeof(float) * M * K);
+  float* b = (float*)aligned_alloc(64, sizeof(float) * K * N);
+  float* c = (float*)aligned_alloc(64, sizeof(float) * M * N);
+  float* c_check = (float*)aligned_alloc(64, sizeof(float) * M * N);
+  int8_t* buffer_a_data = (int8_t*)aligned_alloc(64, buffer_a.required_size());
+  int4_2_t* buffer_b_data = (int4_2_t*)aligned_alloc(64, buffer_b.required_size());
+  int32_t* c_data = (int32_t*)aligned_alloc(64, buffer_c.required_size());
+  int8_t* buffer_a_data_check = (int8_t*)aligned_alloc(64, buffer_a_check.required_size());
+  int8_t* buffer_b_data_check = (int8_t*)aligned_alloc(64, buffer_b_check.required_size());
+  int32_t* c_data_check = (int32_t*)aligned_alloc(64, buffer_c_check.required_size());
+  // 初始化元素内容
+  load_bin("input.bin", a, M * K);
+  load_bin("local_q_a_proj_quant.bin", b, N * K);
+
+  // for (int i = 0; i < M * K; i++) {
+  //   // 随机浮点数
+  //   // a[i] = (static_cast<float>(rand()) / (float)RAND_MAX) / 25 - 0.02;
+  //   a[i] = -(static_cast<float>(rand()) / (float)RAND_MAX) / 25;
+  //   // a[i] = i % 10;
+  //   // a[i] = 1;
+  // }
+  // for (int i = 0; i < K * N; i++) {
+  //   // 随机浮点数
+  //   // b[i] = (static_cast<float>(rand()) / (float)RAND_MAX) / 25 - 0.02;
+  //   b[i] = -(static_cast<float>(rand()) / (float)RAND_MAX) / 25;
+  //   // b[i] = i % 10;
+  //   // b[i] = 1;
+  // }
+  // // // // 设置离群值
+  // for (int i = 0; i < N; i++) {
+  //   b[i * K] = 0.06f; // 设置第一列为离群值
+  // }
+  // // 打印一下输入矩阵和权重矩阵
+  // printf("Input matrix a:\n");
+  // for (int i = 0; i < M; i++) {
+  //   for (int j = 0; j < K; j++) {
+  //     printf("%f ", a[i * K + j]);
+  //   }
+  //   printf("\n");
+  // }
+  // printf("Weight matrix b:\n");
+  // for (int i = 0; i < N; i++) {
+  //   for (int j = 0; j < K; j++) {
+  //     printf("%f ", b[i * K + j]);
+  //   }
+  //   printf("\n");
+  // }
+  buffer_a.set_data(buffer_a_data);
+  buffer_b.set_data(buffer_b_data);
+  buffer_c.set_data(c_data);
+  buffer_a_check.set_data(buffer_a_data_check);
+  buffer_b_check.set_data(buffer_b_data_check);
+  buffer_c_check.set_data(c_data_check);
+  //   调用 from mat 进行量化
+  buffer_a.from_mat(M, a, 0, M);
+  for (int i = 0; i <= arm_kml::GemmKernelInt4::recommended_nth(N); i++) {
+    buffer_b.from_mat(b, i, arm_kml::GemmKernelInt4::recommended_nth(N));
+  }
+  buffer_a_check.from_mat(M, a, 0, M);
+  for (int i = 0; i <= arm_kml::GemmKernelInt8::recommended_nth(N); i++) {
+    buffer_b_check.from_mat(b, i, arm_kml::GemmKernelInt8::recommended_nth(N));
+  }
+  // 进行乘法
+  arm_kml::MatRef<int8_t> a_ref(buffer_a.a, M, K, K, CblasRowMajor);
+  arm_kml::MatRef<int4_2_t> b_ref(buffer_b.b, K, N, K, CblasColMajor, CblasNoTrans, buffer_b.if_pack);
+  arm_kml::MatRef<int32_t> c_ref(buffer_c.c, M, N, N, CblasRowMajor);
+  b_ref = b_ref.offset_col(0, N);
+
+  arm_kml::MatRef<int8_t> a_ref_check(buffer_a_check.a, M, K, K, CblasRowMajor);
+  arm_kml::MatRef<int8_t> b_ref_check(buffer_b_check.b, K, N, K, CblasColMajor, CblasNoTrans, buffer_b_check.if_pack);
+  arm_kml::MatRef<int32_t> c_ref_check(buffer_c_check.c, M, N, N, CblasRowMajor);
+
+  arm_kml::decode_mul_mat_clearc(a_ref, b_ref, c_ref);
+  arm_kml::decode_mul_mat_clearc(a_ref_check, b_ref_check, c_ref_check);
+  //   反量化，apply scale
+  arm_kml::GemmKernelInt4::apply_scale(c, N, &buffer_a, &buffer_b, &buffer_c, 0, M, 0, N, true);
+  arm_kml::GemmKernelInt8::apply_scale(c_check, N, &buffer_a_check, &buffer_b_check, &buffer_c_check, 0, M, 0, N, true);
+  // 打印结果,比较 c 和 c_check
+  const float threashold = 0.05;
+  for (int i = 0; i < M * N; i++) {
+    float diff_relative = (c[i] - c_check[i]) / (c_check[i] + 1e-6);
+
+    if (diff_relative > threashold || diff_relative < -threashold) {
+      printf("diff_relative: %f\n", diff_relative);
+      printf("Mismatch at index %d: c = %f, c_check = %f\n", i, c[i], c_check[i]);
+    } else {
+      printf("Match at index %d: c = %f, c_check = %f\n", i, c[i], c_check[i]);
+    }
+  }
+  return 0;
+}
@@ -0,0 +1,22 @@
+#include "arm_kml.hpp"
+
+int main() {
+  const size_t M = 128, N = 64;
+  float16_t* a = new float16_t[M * N];
+  float16_t* b = new float16_t[M * N];
+  float16_t* c = new float16_t[M * M];
+  float16_t* c_check = new float16_t[M * M];
+  for (size_t i = 0; i < M * N; i++) {
+    a[i] = static_cast<double>(std::rand()) / RAND_MAX / 10.0;
+    b[i] = static_cast<double>(std::rand()) / RAND_MAX / 10.0;
+  }
+
+  arm_kml::MatRef<float16_t> aref(a, M, N, M, CblasColMajor);
+  arm_kml::MatRef<float16_t> bref(b, N, M, M, CblasColMajor);
+  arm_kml::MatRef<float16_t> cref(c, M, M, M, CblasColMajor);
+  {
+    memset(c, 0, M * M * sizeof(float16_t));
+    memset(c_check, 0, M * M * sizeof(float16_t));
+    arm_kml::mul_mat(aref, bref, cref);
+  }
+}
@@ -0,0 +1,63 @@
+// #pragma once
+#ifdef TEST_UTIL
+#include <arm_neon.h>
+#include <arm_sve.h>
+#include <stdio.h>
+
+static inline void sve_32xbf16_to_32xfp32(const bfloat16_t* src, float* dst0, float* dst1) {
+#ifdef __ARM_FEATURE_SVE
+  // 全真谓词，对应每个 16‑bit 元素
+#else
+// fallback: scalar or NEON
+#endif
+}
+
+static inline void neon_32xbf16_to_32xfp32(const uint16_t* src, float* dst0, float* dst1) {
+  // src 指向 32 个连续的 BF16（uint16_t）
+  // dst0、dst1 各指向 16 个 float 的缓冲
+
+  for (int block = 0; block < 4; ++block) {
+    // 每次处理 8 个 BF16 → 8 个 FP32（拆为两次 4→4 存储）
+    uint16x8_t v_bf16 = vld1q_u16(src + block * 8);  // load 8×BF16 :contentReference[oaicite:6]{index=6}
+
+    // 拆低半、高半各 4 个到 u32
+    uint32x4_t lo_u32 = vmovl_u16(vget_low_u16(v_bf16));   // lower 4 → u32 :contentReference[oaicite:7]{index=7}
+    uint32x4_t hi_u32 = vmovl_u16(vget_high_u16(v_bf16));  // upper 4 → u32 :contentReference[oaicite:8]{index=8}
+
+    // 左移 16 位，相当于将 BF16 的 16 位 mantissa+exp 放到 FP32 高位
+    lo_u32 = vshlq_n_u32(lo_u32, 16);  // shift left 16 :contentReference[oaicite:9]{index=9}
+    hi_u32 = vshlq_n_u32(hi_u32, 16);  // shift left 16 :contentReference[oaicite:10]{index=10}
+
+    // 重新解释为 float32x4_t
+    float32x4_t lo_f32 = vreinterpretq_f32_u32(lo_u32);  // bits → FP32 :contentReference[oaicite:11]{index=11}
+    float32x4_t hi_f32 = vreinterpretq_f32_u32(hi_u32);  // bits → FP32 :contentReference[oaicite:12]{index=12}
+
+    // 存储到 dst0 或 dst1，每次存 8 个
+    if (block < 2) {
+      vst1q_f32(dst0 + block * 4, lo_f32);      // store 4 floats :contentReference[oaicite:13]{index=13}
+      vst1q_f32(dst0 + block * 4 + 4, hi_f32);  // store next 4 floats :contentReference[oaicite:14]{index=14}
+    } else {
+      int b = block - 2;
+      vst1q_f32(dst1 + b * 4, lo_f32);      // store 4 floats :contentReference[oaicite:15]{index=15}
+      vst1q_f32(dst1 + b * 4 + 4, hi_f32);  // store next 4 floats :contentReference[oaicite:16]{index=16}
+    }
+  }
+}
+
+int main() {
+  // 测试代码
+  uint16_t bf16_data[32] = {0};  // 假设这里填充了一些 BF16 数据
+  float f32_data0[16] = {0};
+  float f32_data1[16] = {0};
+
+  neon_32xbf16_to_32xfp32(bf16_data, f32_data0, f32_data1);
+
+  // 打印结果
+  for (int i = 0; i < 16; ++i) {
+    printf("f32_data0[%d]: %f\n", i, f32_data0[i]);
+    printf("f32_data1[%d]: %f\n", i, f32_data1[i]);
+  }
+
+  return 0;
+}
+#endif