chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,115 @@
|
||||
# Simple autogenerated Python bindings for ggml
|
||||
|
||||
This folder contains:
|
||||
|
||||
- Scripts to generate full Python bindings from ggml headers (+ stubs for autocompletion in IDEs)
|
||||
- Some barebones utils (see [ggml/utils.py](./ggml/utils.py)):
|
||||
- `ggml.utils.init` builds a context that's freed automatically when the pointer gets GC'd
|
||||
- `ggml.utils.copy` **copies between same-shaped tensors (numpy or ggml), w/ automatic (de/re)quantization**
|
||||
- `ggml.utils.numpy` returns a numpy view over a ggml tensor; if it's quantized, it returns a copy (requires `allow_copy=True`)
|
||||
- Very basic examples (anyone wants to port [llama2.c](https://github.com/karpathy/llama2.c)?)
|
||||
|
||||
Provided you set `GGML_LIBRARY=.../path/to/libggml_shared.so` (see instructions below), it's trivial to do some operations on quantized tensors:
|
||||
|
||||
```python
|
||||
# Make sure libllama.so is in your [DY]LD_LIBRARY_PATH, or set GGML_LIBRARY=.../libggml_shared.so
|
||||
|
||||
from ggml import lib, ffi
|
||||
from ggml.utils import init, copy, numpy
|
||||
import numpy as np
|
||||
|
||||
ctx = init(mem_size=12*1024*1024)
|
||||
n = 256
|
||||
n_threads = 4
|
||||
|
||||
a = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n)
|
||||
b = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n) # Can't both be quantized
|
||||
sum = lib.ggml_add(ctx, a, b) # all zeroes for now. Will be quantized too!
|
||||
|
||||
gf = ffi.new('struct ggml_cgraph*')
|
||||
lib.ggml_build_forward_expand(gf, sum)
|
||||
|
||||
copy(np.array([i for i in range(n)], np.float32), a)
|
||||
copy(np.array([i*100 for i in range(n)], np.float32), b)
|
||||
|
||||
lib.ggml_graph_compute_with_ctx(ctx, gf, n_threads)
|
||||
|
||||
print(numpy(a, allow_copy=True))
|
||||
# 0. 1.0439453 2.0878906 3.131836 4.1757812 5.2197266. ...
|
||||
print(numpy(b))
|
||||
# 0. 100. 200. 300. 400. 500. ...
|
||||
print(numpy(sum, allow_copy=True))
|
||||
# 0. 105.4375 210.875 316.3125 421.75 527.1875 ...
|
||||
```
|
||||
|
||||
### Prerequisites
|
||||
|
||||
You'll need a shared library of ggml to use the bindings.
|
||||
|
||||
#### Build libggml_shared.so or libllama.so
|
||||
|
||||
As of this writing the best is to use [ggerganov/llama.cpp](https://github.com/ggerganov/llama.cpp)'s generated `libggml_shared.so` or `libllama.so`, which you can build as follows:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/ggerganov/llama.cpp
|
||||
# On a CUDA-enabled system add -DLLAMA_CUDA=1
|
||||
# On a Mac add -DLLAMA_METAL=1
|
||||
cmake llama.cpp \
|
||||
-B llama_build \
|
||||
-DCMAKE_C_FLAGS=-Ofast \
|
||||
-DLLAMA_NATIVE=1 \
|
||||
-DLLAMA_LTO=1 \
|
||||
-DBUILD_SHARED_LIBS=1 \
|
||||
-DLLAMA_MPI=1 \
|
||||
-DLLAMA_BUILD_TESTS=0 \
|
||||
-DLLAMA_BUILD_EXAMPLES=0
|
||||
( cd llama_build && make -j )
|
||||
|
||||
# On Mac, this will be libggml_shared.dylib instead
|
||||
export GGML_LIBRARY=$PWD/llama_build/libggml_shared.so
|
||||
# Alternatively, you can just copy it to your system's lib dir, e.g /usr/local/lib
|
||||
```
|
||||
|
||||
#### (Optional) Regenerate the bindings and stubs
|
||||
|
||||
If you added or changed any signatures of the C API, you'll want to regenerate the bindings ([ggml/cffi.py](./ggml/cffi.py)) and stubs ([ggml/__init__.pyi](./ggml/__init__.pyi)).
|
||||
|
||||
Luckily it's a one-liner using [regenerate.py](./regenerate.py):
|
||||
|
||||
```bash
|
||||
pip install -q cffi
|
||||
|
||||
python regenerate.py
|
||||
```
|
||||
|
||||
By default it assumes `llama.cpp` was cloned in ../../../llama.cpp (alongside the ggml folder). You can override this with:
|
||||
|
||||
```bash
|
||||
C_INCLUDE_DIR=$LLAMA_CPP_DIR python regenerate.py
|
||||
```
|
||||
|
||||
You can also edit [api.h](./api.h) to control which files should be included in the generated bindings (defaults to `llama.cpp/ggml*.h`)
|
||||
|
||||
In fact, if you wanted to only generate bindings for the current version of the `ggml` repo itself (instead of `llama.cpp`; you'd loose support for k-quants), you could run:
|
||||
|
||||
```bash
|
||||
API=../../include/ggml.h python regenerate.py
|
||||
```
|
||||
|
||||
## Develop
|
||||
|
||||
Run tests:
|
||||
|
||||
```bash
|
||||
pytest
|
||||
```
|
||||
|
||||
### Alternatives
|
||||
|
||||
This example's goal is to showcase [cffi](https://cffi.readthedocs.io/)-generated bindings that are trivial to use and update, but there are already alternatives in the wild:
|
||||
|
||||
- https://github.com/abetlen/ggml-python: these bindings seem to be hand-written and use [ctypes](https://docs.python.org/3/library/ctypes.html). It has [high-quality API reference docs](https://ggml-python.readthedocs.io/en/latest/api-reference/#ggml.ggml) that can be used with these bindings too, but it doesn't expose Metal, CUDA, MPI or OpenCL calls, doesn't support transparent (de/re)quantization like this example does (see [ggml.utils](./ggml/utils.py) module), and won't pick up your local changes.
|
||||
|
||||
- https://github.com/abetlen/llama-cpp-python: these expose the C++ `llama.cpp` interface, which this example cannot easily be extended to support (`cffi` only generates bindings of C libraries)
|
||||
|
||||
- [pybind11](https://github.com/pybind/pybind11) and [nanobind](https://github.com/wjakob/nanobind) are two alternatives to cffi that support binding C++ libraries, but it doesn't seem either of them have an automatic generator (writing bindings is rather time-consuming).
|
||||
@@ -0,0 +1,14 @@
|
||||
/*
|
||||
List here all the headers you want to expose in the Python bindings,
|
||||
then run `python regenerate.py` (see details in README.md)
|
||||
*/
|
||||
|
||||
#include "ggml.h"
|
||||
#include "ggml-metal.h"
|
||||
#include "ggml-opencl.h"
|
||||
|
||||
// Headers below are currently only present in the llama.cpp repository, comment them out if you don't have them.
|
||||
#include "k_quants.h"
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-cuda.h"
|
||||
#include "ggml-mpi.h"
|
||||
@@ -0,0 +1,25 @@
|
||||
from ggml import lib, ffi
|
||||
from ggml.utils import init, copy, numpy
|
||||
import numpy as np
|
||||
|
||||
ctx = init(mem_size=12*1024*1024) # automatically freed when pointer is GC'd
|
||||
n = 256
|
||||
n_threads = 4
|
||||
|
||||
a = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n)
|
||||
b = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n) # can't both be quantized
|
||||
sum = lib.ggml_add(ctx, a, b) # all zeroes for now. Will be quantized too!
|
||||
|
||||
# See cffi's doc on how to allocate native memory: it's very simple!
|
||||
# https://cffi.readthedocs.io/en/latest/ref.html#ffi-interface
|
||||
gf = ffi.new('struct ggml_cgraph*')
|
||||
lib.ggml_build_forward_expand(gf, sum)
|
||||
|
||||
copy(np.array([i for i in range(n)], np.float32), a)
|
||||
copy(np.array([i*100 for i in range(n)], np.float32), b)
|
||||
|
||||
lib.ggml_graph_compute_with_ctx(ctx, gf, n_threads)
|
||||
|
||||
print(numpy(a, allow_copy=True))
|
||||
print(numpy(b))
|
||||
print(numpy(sum, allow_copy=True))
|
||||
@@ -0,0 +1,68 @@
|
||||
from ggml import ffi, lib
|
||||
from ggml.utils import init, numpy, copy
|
||||
import numpy as np
|
||||
from math import pi, cos, sin, ceil
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
ctx = init(mem_size=100*1024*1024) # Will be auto-GC'd
|
||||
n = 256
|
||||
|
||||
orig = np.array([
|
||||
[
|
||||
cos(j * 2 * pi / n) * (sin(i * 2 * pi / n))
|
||||
for j in range(n)
|
||||
]
|
||||
for i in range(n)
|
||||
], np.float32)
|
||||
orig_tensor = lib.ggml_new_tensor_2d(ctx, lib.GGML_TYPE_F32, n, n)
|
||||
copy(orig, orig_tensor)
|
||||
|
||||
quants = [
|
||||
type for type in range(lib.GGML_TYPE_COUNT)
|
||||
if lib.ggml_is_quantized(type) and
|
||||
type not in [lib.GGML_TYPE_Q8_1, lib.GGML_TYPE_Q8_K] # Apparently not supported
|
||||
]
|
||||
# quants = [lib.GGML_TYPE_Q2_K] # Test a single one
|
||||
|
||||
def get_name(type):
|
||||
name = lib.ggml_type_name(type)
|
||||
return ffi.string(name).decode('utf-8') if name else '?'
|
||||
|
||||
quants.sort(key=get_name)
|
||||
quants.insert(0, None)
|
||||
print(quants)
|
||||
|
||||
ncols=4
|
||||
nrows = ceil(len(quants) / ncols)
|
||||
|
||||
plt.figure(figsize=(ncols * 5, nrows * 5), layout='tight')
|
||||
|
||||
for i, type in enumerate(quants):
|
||||
plt.subplot(nrows, ncols, i + 1)
|
||||
try:
|
||||
if type == None:
|
||||
plt.title('Original')
|
||||
plt.imshow(orig)
|
||||
else:
|
||||
quantized_tensor = lib.ggml_new_tensor_2d(ctx, type, n, n)
|
||||
copy(orig_tensor, quantized_tensor)
|
||||
quantized = numpy(quantized_tensor, allow_copy=True)
|
||||
d = quantized - orig
|
||||
results = {
|
||||
"l2": np.linalg.norm(d, 2),
|
||||
"linf": np.linalg.norm(d, np.inf),
|
||||
"compression":
|
||||
round(lib.ggml_nbytes(orig_tensor) /
|
||||
lib.ggml_nbytes(quantized_tensor), 1)
|
||||
}
|
||||
name = get_name(type)
|
||||
print(f'{name}: {results}')
|
||||
|
||||
plt.title(f'{name} ({results["compression"]}x smaller)')
|
||||
plt.imshow(quantized, interpolation='nearest')
|
||||
|
||||
except Exception as e:
|
||||
print(f'Error: {e}')
|
||||
|
||||
plt.show()
|
||||
@@ -0,0 +1,58 @@
|
||||
"""
|
||||
Python bindings for the ggml library.
|
||||
|
||||
Usage example:
|
||||
|
||||
from ggml import lib, ffi
|
||||
from ggml.utils import init, copy, numpy
|
||||
import numpy as np
|
||||
|
||||
ctx = init(mem_size=10*1024*1024)
|
||||
n = 1024
|
||||
n_threads = 4
|
||||
|
||||
a = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n)
|
||||
b = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n)
|
||||
sum = lib.ggml_add(ctx, a, b)
|
||||
|
||||
gf = ffi.new('struct ggml_cgraph*')
|
||||
lib.ggml_build_forward_expand(gf, sum)
|
||||
|
||||
copy(np.array([i for i in range(n)], np.float32), a)
|
||||
copy(np.array([i*100 for i in range(n)], np.float32), b)
|
||||
lib.ggml_graph_compute_with_ctx(ctx, gf, n_threads)
|
||||
|
||||
print(numpy(sum, allow_copy=True))
|
||||
|
||||
See https://cffi.readthedocs.io/en/latest/cdef.html for more on cffi.
|
||||
"""
|
||||
|
||||
try:
|
||||
from ggml.cffi import ffi as ffi
|
||||
except ImportError as e:
|
||||
raise ImportError(f"Couldn't find ggml bindings ({e}). Run `python regenerate.py` or check your PYTHONPATH.")
|
||||
|
||||
import os, platform
|
||||
|
||||
__exact_library = os.environ.get("GGML_LIBRARY")
|
||||
if __exact_library:
|
||||
__candidates = [__exact_library]
|
||||
elif platform.system() == "Windows":
|
||||
__candidates = ["ggml_shared.dll", "llama.dll"]
|
||||
else:
|
||||
__candidates = ["libggml_shared.so", "libllama.so"]
|
||||
if platform.system() == "Darwin":
|
||||
__candidates += ["libggml_shared.dylib", "libllama.dylib"]
|
||||
|
||||
for i, name in enumerate(__candidates):
|
||||
try:
|
||||
# This is where all the functions, enums and constants are defined
|
||||
lib = ffi.dlopen(name)
|
||||
except OSError:
|
||||
if i < len(__candidates) - 1:
|
||||
continue
|
||||
raise OSError(f"Couldn't find ggml's shared library (tried names: {__candidates}). Add its directory to DYLD_LIBRARY_PATH (on Mac) or LD_LIBRARY_PATH, or define GGML_LIBRARY.")
|
||||
|
||||
# This contains the cffi helpers such as new, cast, string, etc.
|
||||
# https://cffi.readthedocs.io/en/latest/ref.html#ffi-interface
|
||||
ffi = ffi
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
@@ -0,0 +1,7 @@
|
||||
# Phony stubs.
|
||||
|
||||
class CData:
|
||||
pass
|
||||
|
||||
class CType:
|
||||
pass
|
||||
@@ -0,0 +1,182 @@
|
||||
"""
|
||||
Common helpers for working with ggml + numpy
|
||||
"""
|
||||
from ggml import ffi, lib
|
||||
from typing import Union, Optional
|
||||
import numpy as np
|
||||
|
||||
def init(mem_size: int, mem_buffer: ffi.CData = ffi.NULL, no_alloc: bool = False) -> ffi.CData:
|
||||
"""
|
||||
Initialize a ggml context, which will be freed automatically when the pointer is garbage collected.
|
||||
"""
|
||||
params = ffi.new('struct ggml_init_params*')
|
||||
params.mem_size = mem_size
|
||||
params.mem_buffer = mem_buffer
|
||||
params.no_alloc = no_alloc
|
||||
return ffi.gc(lib.ggml_init(params[0]), lib.ggml_free)
|
||||
|
||||
TensorLike = Union[ffi.CData, np.ndarray]
|
||||
|
||||
def copy(from_tensor: TensorLike, to_tensor: TensorLike, allow_requantize: bool = True):
|
||||
"""
|
||||
Copy the contents of one tensor to another, doing any necessary (de/re)quantization transparently.
|
||||
Works across numpy & ggml tensors, but they must have the same shape (and be contiguous).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
from_tensor : TensorLike
|
||||
The tensor to copy from (a numpy array or possibly-quantized ggml tensor)
|
||||
to_tensor : TensorLike
|
||||
The tensor to copy to (a numpy array or possibly-quantized ggml tensor)
|
||||
allow_requantize : bool
|
||||
If False, will throw an error if requantization is required (i.e. both from_tensor
|
||||
and to_tensor are quantized with different quantization types)
|
||||
"""
|
||||
if id(from_tensor) == id(to_tensor):
|
||||
return
|
||||
|
||||
__expect_same_layout("source", from_tensor, "destination", to_tensor)
|
||||
__check_shape_consistent_with_type(from_tensor)
|
||||
__check_shape_consistent_with_type(to_tensor)
|
||||
|
||||
from_type = __get_type(from_tensor)
|
||||
to_type = __get_type(to_tensor)
|
||||
|
||||
if from_type == to_type:
|
||||
ffi.memmove(__get_data(to_tensor), __get_data(from_tensor), __get_nbytes(from_tensor))
|
||||
else:
|
||||
assert allow_requantize or not lib.ggml_is_quantized(from_type) or not lib.ggml_is_quantized(to_type), \
|
||||
f"Requantizing from {__type_name(from_type)} to {__type_name(to_type)} is disabled. Force with allow_requantize=True"
|
||||
|
||||
__set_floats(to_tensor, __get_floats(from_tensor))
|
||||
|
||||
def numpy(tensor: ffi.CData, allow_copy: Union[bool, np.ndarray] = False, allow_requantize=False) -> np.ndarray:
|
||||
"""
|
||||
Convert a ggml tensor to a numpy array.
|
||||
If the tensor isn't quantized, the returned numpy array will be a view over its data.
|
||||
|
||||
If it is quantized (and allow_copy is True), the copy will involve dequantization and the returned array will
|
||||
be a copy of the original tensor (any changes to the numpy array won't then be reflected back to the tensor).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
tensor : ffi.CData
|
||||
The tensor to convert to a numpy array
|
||||
allow_copy : bool or np.ndarray
|
||||
If False, will throw an error if the tensor is quantized (since dequantization requires extra memory).
|
||||
If True, will dequantize the tensor and return a copy of the data in a new float32 numpy array.
|
||||
If an np.ndarray, will copy the data into the given array (which must be the same shape as the tensor) when dequantization is needed
|
||||
allow_requantize : bool
|
||||
If allow_copy is a tensor with a different quantization type than the source tensor, will throw an error unless allow_requantize is True.
|
||||
"""
|
||||
shape = __get_shape(tensor)
|
||||
|
||||
if lib.ggml_is_quantized(tensor.type):
|
||||
if allow_copy == False:
|
||||
raise ValueError(f"{__describe(tensor)} is quantized, conversion to numpy requires a copy (pass allow_copy=True; changes to the numpy array won't affect the original).")
|
||||
elif isinstance(allow_copy, np.ndarray):
|
||||
__expect_same_layout("source tensor", tensor, "dequantization output tensor", allow_copy)
|
||||
destination = allow_copy
|
||||
else:
|
||||
destination = np.empty(shape, dtype=np.float32)
|
||||
|
||||
copy(tensor, destination, allow_requantize=allow_requantize)
|
||||
return destination
|
||||
else:
|
||||
dtype = __type_to_dtype(tensor.type)
|
||||
if not dtype:
|
||||
raise NotImplementedError(f'Cannot convert {__describe(tensor)} to numpy')
|
||||
|
||||
assert __is_contiguous(tensor), f"Cannot convert {__describe(tensor)} to numpy (support contiguous tensors only)"
|
||||
nbytes = lib.ggml_nelements(tensor) * lib.ggml_type_size(tensor.type)
|
||||
array = np.frombuffer(ffi.buffer(lib.ggml_get_data(tensor), nbytes), dtype=dtype)
|
||||
array.shape = shape
|
||||
return array
|
||||
|
||||
def __type_name(type: int) -> str:
|
||||
name = lib.ggml_type_name(type)
|
||||
return ffi.string(name).decode('utf-8') if name else None
|
||||
|
||||
__k_quant_types = set([
|
||||
lib.GGML_TYPE_Q2_K,
|
||||
lib.GGML_TYPE_Q3_K,
|
||||
lib.GGML_TYPE_Q4_K,
|
||||
lib.GGML_TYPE_Q5_K,
|
||||
lib.GGML_TYPE_Q6_K,
|
||||
lib.GGML_TYPE_Q8_K,
|
||||
])
|
||||
|
||||
__type_to_dtype_dict = {
|
||||
lib.GGML_TYPE_I8: np.int8,
|
||||
lib.GGML_TYPE_I16: np.int16,
|
||||
lib.GGML_TYPE_I32: np.int32,
|
||||
lib.GGML_TYPE_F16: np.float16,
|
||||
lib.GGML_TYPE_F32: np.float32,
|
||||
}
|
||||
|
||||
def __type_to_dtype(type: int) -> Optional[np.dtype]: return __type_to_dtype_dict.get(type)
|
||||
def __dtype_to_type(dtype: np.dtype):
|
||||
if dtype == np.float32: return lib.GGML_TYPE_F32
|
||||
elif dtype == np.float16: return lib.GGML_TYPE_F16
|
||||
elif dtype == np.int32: return lib.GGML_TYPE_I32
|
||||
elif dtype == np.int16: return lib.GGML_TYPE_I16
|
||||
elif dtype == np.int8: return lib.GGML_TYPE_I8
|
||||
else: raise ValueError(f"Unsupported dtype: {dtype}")
|
||||
|
||||
def __describe(tensor: ffi.CType): return f'Tensor[{__type_name(__get_type(tensor))}, {__get_shape(tensor)}]'
|
||||
def __get_type(tensor: TensorLike): return __dtype_to_type(tensor.dtype) if isinstance(tensor, np.ndarray) else tensor.type
|
||||
def __get_shape(x: TensorLike): return x.shape if isinstance(x, np.ndarray) else tuple([x.ne[i] for i in range(x.n_dims)])
|
||||
def __get_strides(x: TensorLike): return x.strides if isinstance(x, np.ndarray) else tuple([x.nb[i] for i in range(x.n_dims)])
|
||||
def __get_data(x: TensorLike) -> ffi.CData: return ffi.from_buffer(x) if isinstance(x, np.ndarray) else lib.ggml_get_data(x)
|
||||
def __get_nbytes(tensor: TensorLike): return tensor.nbytes if isinstance(tensor, np.ndarray) else lib.ggml_nbytes(tensor)
|
||||
def __get_nelements(tensor: TensorLike): return tensor.size if isinstance(tensor, np.ndarray) else lib.ggml_nelements(tensor)
|
||||
def __is_contiguous(tensor: TensorLike): return tensor.flags['C_CONTIGUOUS'] if isinstance(tensor, np.ndarray) else lib.ggml_is_contiguous(tensor)
|
||||
|
||||
def __get_floats(tensor: TensorLike) -> ffi.CData:
|
||||
data, type = __get_data(tensor), __get_type(tensor)
|
||||
if type == lib.GGML_TYPE_F32:
|
||||
return ffi.cast('float*', data)
|
||||
else:
|
||||
nelements = __get_nelements(tensor)
|
||||
floats = ffi.new('float[]', nelements)
|
||||
if type == lib.GGML_TYPE_F16:
|
||||
lib.ggml_fp16_to_fp32_row(ffi.cast('uint16_t*', data), floats, nelements)
|
||||
elif lib.ggml_is_quantized(type):
|
||||
qtype = lib.ggml_internal_get_type_traits(type)
|
||||
assert qtype.to_float, f"Type {__type_name(type)} is not supported by ggml"
|
||||
qtype.to_float(data, floats, nelements)
|
||||
else:
|
||||
raise NotImplementedError(f'Cannot read floats from {__describe(tensor)}')
|
||||
return floats
|
||||
|
||||
def __set_floats(tensor: TensorLike, f32_data: ffi.CData) -> None:
|
||||
data, type, nbytes = __get_data(tensor), __get_type(tensor), __get_nbytes(tensor)
|
||||
if type == lib.GGML_TYPE_F32:
|
||||
ffi.memmove(data, f32_data, nbytes)
|
||||
else:
|
||||
nelements = __get_nelements(tensor)
|
||||
if type == lib.GGML_TYPE_F16:
|
||||
lib.ggml_fp32_to_fp16_row(f32_data, ffi.cast('uint16_t*', data), nelements)
|
||||
elif lib.ggml_is_quantized(type):
|
||||
qtype = lib.ggml_internal_get_type_traits(type)
|
||||
assert qtype.from_float, f"Type {__type_name(type)} is not supported by ggml"
|
||||
qtype.from_float(f32_data, data, nelements)
|
||||
else:
|
||||
raise NotImplementedError(f'Cannot write floats to {__describe(tensor)}')
|
||||
|
||||
def __expect_same_layout(name1: str, tensor1: TensorLike, name2: str, tensor2: TensorLike):
|
||||
shape1, shape2 = __get_shape(tensor1), __get_shape(tensor2)
|
||||
assert shape1 == shape2, f"Shape mismatch: {name1} has {shape1} but {name2} has {shape2}"
|
||||
assert __is_contiguous(tensor1) and __is_contiguous(tensor2), f"Only contiguous tensors are supported (got {name1} with strides {__get_strides(tensor1)} and {name2} with strides {__get_strides(tensor2)})"
|
||||
|
||||
def __check_shape_consistent_with_type(tensor: TensorLike):
|
||||
type = __get_type(tensor)
|
||||
if not lib.ggml_is_quantized(type):
|
||||
return
|
||||
shape = __get_shape(tensor)
|
||||
|
||||
block_size = lib.ggml_blck_size(type)
|
||||
assert not (block_size == 0 and type in __k_quant_types), f"Can't quantize, native library was not compiled with USE_K_QUANTS!"
|
||||
assert block_size > 0, f"Invalid block size {block_size} for type {__type_name(type)}"
|
||||
for i, d in enumerate(shape):
|
||||
assert d % block_size == 0, f"Dimension {i} of {__describe(tensor)} is not divisible by {block_size}, required for quantization."
|
||||
@@ -0,0 +1,42 @@
|
||||
# Generates bindings for the ggml library.
|
||||
#
|
||||
# cffi requires prior C preprocessing of the headers, and it uses pycparser which chokes on a couple of things
|
||||
# so we help it a bit (e.g. replace sizeof expressions with their value, remove exotic syntax found in Darwin headers).
|
||||
import os, sys, re, subprocess
|
||||
import cffi
|
||||
from stubs import generate_stubs
|
||||
|
||||
API = os.environ.get('API', 'api.h')
|
||||
CC = os.environ.get('CC') or 'gcc'
|
||||
C_INCLUDE_DIR = os.environ.get('C_INCLUDE_DIR', '../../../llama.cpp')
|
||||
CPPFLAGS = [
|
||||
"-I", C_INCLUDE_DIR,
|
||||
'-D__fp16=uint16_t', # pycparser doesn't support __fp16
|
||||
'-D__attribute__(x)=',
|
||||
'-D_Static_assert(x, m)=',
|
||||
] + [x for x in os.environ.get('CPPFLAGS', '').split(' ') if x != '']
|
||||
|
||||
try: header = subprocess.run([CC, "-E", *CPPFLAGS, API], capture_output=True, text=True, check=True).stdout
|
||||
except subprocess.CalledProcessError as e: print(f'{e.stderr}\n{e}', file=sys.stderr); raise
|
||||
|
||||
header = '\n'.join([l for l in header.split('\n') if '__darwin_va_list' not in l]) # pycparser hates this
|
||||
|
||||
# Replace constant size expressions w/ their value (compile & run a mini exe for each, because why not).
|
||||
# First, extract anyting *inside* square brackets and anything that looks like a sizeof call.
|
||||
for expr in set(re.findall(f'(?<=\\[)[^\\]]+(?=])|sizeof\\s*\\([^()]+\\)', header)):
|
||||
if re.match(r'^(\d+|\s*)$', expr): continue # skip constants and empty bracket contents
|
||||
subprocess.run([CC, "-o", "eval_size_expr", *CPPFLAGS, "-x", "c", "-"], text=True, check=True,
|
||||
input=f'''#include <stdio.h>
|
||||
#include "{API}"
|
||||
int main() {{ printf("%lu", (size_t)({expr})); }}''')
|
||||
size = subprocess.run(["./eval_size_expr"], capture_output=True, text=True, check=True).stdout
|
||||
print(f'Computed constexpr {expr} = {size}')
|
||||
header = header.replace(expr, size)
|
||||
|
||||
ffibuilder = cffi.FFI()
|
||||
ffibuilder.cdef(header)
|
||||
ffibuilder.set_source(f'ggml.cffi', None) # we're not compiling a native extension, as this quickly gets hairy
|
||||
ffibuilder.compile(verbose=True)
|
||||
|
||||
with open("ggml/__init__.pyi", "wt") as f:
|
||||
f.write(generate_stubs(header))
|
||||
@@ -0,0 +1,128 @@
|
||||
"""
|
||||
This generates .pyi stubs for the cffi Python bindings generated by regenerate.py
|
||||
"""
|
||||
import sys, re, itertools
|
||||
sys.path.extend(['.', '..']) # for pycparser
|
||||
|
||||
from pycparser import c_ast, parse_file, CParser
|
||||
import pycparser.plyparser
|
||||
from pycparser.c_ast import PtrDecl, TypeDecl, FuncDecl, EllipsisParam, IdentifierType, Struct, Enum, Typedef
|
||||
from typing import Tuple
|
||||
|
||||
__c_type_to_python_type = {
|
||||
'void': 'None', '_Bool': 'bool',
|
||||
'char': 'int', 'short': 'int', 'int': 'int', 'long': 'int',
|
||||
'ptrdiff_t': 'int', 'size_t': 'int',
|
||||
'int8_t': 'int', 'uint8_t': 'int',
|
||||
'int16_t': 'int', 'uint16_t': 'int',
|
||||
'int32_t': 'int', 'uint32_t': 'int',
|
||||
'int64_t': 'int', 'uint64_t': 'int',
|
||||
'float': 'float', 'double': 'float',
|
||||
'ggml_fp16_t': 'np.float16',
|
||||
}
|
||||
|
||||
def format_type(t: TypeDecl):
|
||||
if isinstance(t, PtrDecl) or isinstance(t, Struct):
|
||||
return 'ffi.CData'
|
||||
if isinstance(t, Enum):
|
||||
return 'int'
|
||||
if isinstance(t, TypeDecl):
|
||||
return format_type(t.type)
|
||||
if isinstance(t, IdentifierType):
|
||||
assert len(t.names) == 1, f'Expected a single name, got {t.names}'
|
||||
return __c_type_to_python_type.get(t.names[0]) or 'ffi.CData'
|
||||
return t.name
|
||||
|
||||
class PythonStubFuncDeclVisitor(c_ast.NodeVisitor):
|
||||
def __init__(self):
|
||||
self.sigs = {}
|
||||
self.sources = {}
|
||||
|
||||
def get_source_snippet_lines(self, coord: pycparser.plyparser.Coord) -> Tuple[list[str], list[str]]:
|
||||
if coord.file not in self.sources:
|
||||
with open(coord.file, 'rt') as f:
|
||||
self.sources[coord.file] = f.readlines()
|
||||
source_lines = self.sources[coord.file]
|
||||
ncomment_lines = len(list(itertools.takewhile(lambda i: re.search(r'^\s*(//|/\*)', source_lines[i]), range(coord.line - 2, -1, -1))))
|
||||
comment_lines = [l.strip() for l in source_lines[coord.line - 1 - ncomment_lines:coord.line - 1]]
|
||||
decl_lines = []
|
||||
for line in source_lines[coord.line - 1:]:
|
||||
decl_lines.append(line.rstrip())
|
||||
if (';' in line) or ('{' in line): break
|
||||
return (comment_lines, decl_lines)
|
||||
|
||||
def visit_Enum(self, node: Enum):
|
||||
if node.values is not None:
|
||||
for e in node.values.enumerators:
|
||||
self.sigs[e.name] = f' @property\n def {e.name}(self) -> int: ...'
|
||||
|
||||
def visit_Typedef(self, node: Typedef):
|
||||
pass
|
||||
|
||||
def visit_FuncDecl(self, node: FuncDecl):
|
||||
ret_type = node.type
|
||||
is_ptr = False
|
||||
while isinstance(ret_type, PtrDecl):
|
||||
ret_type = ret_type.type
|
||||
is_ptr = True
|
||||
|
||||
fun_name = ret_type.declname
|
||||
if fun_name.startswith('__'):
|
||||
return
|
||||
|
||||
args = []
|
||||
argnames = []
|
||||
def gen_name(stem):
|
||||
i = 1
|
||||
while True:
|
||||
new_name = stem if i == 1 else f'{stem}{i}'
|
||||
if new_name not in argnames: return new_name
|
||||
i += 1
|
||||
|
||||
for a in node.args.params:
|
||||
if isinstance(a, EllipsisParam):
|
||||
arg_name = gen_name('args')
|
||||
argnames.append(arg_name)
|
||||
args.append('*' + gen_name('args'))
|
||||
elif format_type(a.type) == 'None':
|
||||
continue
|
||||
else:
|
||||
arg_name = a.name or gen_name('arg')
|
||||
argnames.append(arg_name)
|
||||
args.append(f'{arg_name}: {format_type(a.type)}')
|
||||
|
||||
ret = format_type(ret_type if not is_ptr else node.type)
|
||||
|
||||
comment_lines, decl_lines = self.get_source_snippet_lines(node.coord)
|
||||
|
||||
lines = [f' def {fun_name}({", ".join(args)}) -> {ret}:']
|
||||
if len(comment_lines) == 0 and len(decl_lines) == 1:
|
||||
lines += [f' """{decl_lines[0]}"""']
|
||||
else:
|
||||
lines += [' """']
|
||||
lines += [f' {c.lstrip("/* ")}' for c in comment_lines]
|
||||
if len(comment_lines) > 0:
|
||||
lines += ['']
|
||||
lines += [f' {d}' for d in decl_lines]
|
||||
lines += [' """']
|
||||
lines += [' ...']
|
||||
self.sigs[fun_name] = '\n'.join(lines)
|
||||
|
||||
def generate_stubs(header: str):
|
||||
"""
|
||||
Generates a .pyi Python stub file for the GGML API using C header files.
|
||||
"""
|
||||
|
||||
v = PythonStubFuncDeclVisitor()
|
||||
v.visit(CParser().parse(header, "<input>"))
|
||||
|
||||
keys = list(v.sigs.keys())
|
||||
keys.sort()
|
||||
|
||||
return '\n'.join([
|
||||
'# auto-generated file',
|
||||
'import ggml.ffi as ffi',
|
||||
'import numpy as np',
|
||||
'class lib:',
|
||||
*[v.sigs[k] for k in keys]
|
||||
])
|
||||
@@ -0,0 +1,258 @@
|
||||
import pytest
|
||||
from pytest import raises
|
||||
|
||||
from ggml import lib, ffi
|
||||
from ggml.utils import init, copy, numpy
|
||||
import numpy as np
|
||||
import numpy.testing as npt
|
||||
|
||||
@pytest.fixture()
|
||||
def ctx():
|
||||
print("setup")
|
||||
yield init(mem_size=10*1024*1024)
|
||||
print("teardown")
|
||||
|
||||
class TestNumPy:
|
||||
|
||||
# Single element
|
||||
|
||||
def test_set_get_single_i32(self, ctx):
|
||||
i = lib.ggml_new_i32(ctx, 42)
|
||||
assert lib.ggml_get_i32_1d(i, 0) == 42
|
||||
assert numpy(i) == np.array([42], dtype=np.int32)
|
||||
|
||||
def test_set_get_single_f32(self, ctx):
|
||||
i = lib.ggml_new_f32(ctx, 4.2)
|
||||
|
||||
epsilon = 0.000001 # Not sure why so large a difference??
|
||||
pytest.approx(lib.ggml_get_f32_1d(i, 0), 4.2, epsilon)
|
||||
pytest.approx(numpy(i), np.array([4.2], dtype=np.float32), epsilon)
|
||||
|
||||
def _test_copy_np_to_ggml(self, a: np.ndarray, t: ffi.CData):
|
||||
a2 = a.copy() # Clone original
|
||||
copy(a, t)
|
||||
npt.assert_array_equal(numpy(t), a2)
|
||||
|
||||
# I32
|
||||
|
||||
def test_copy_np_to_ggml_1d_i32(self, ctx):
|
||||
t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_I32, 10)
|
||||
a = np.arange(10, dtype=np.int32)
|
||||
self._test_copy_np_to_ggml(a, t)
|
||||
|
||||
def test_copy_np_to_ggml_2d_i32(self, ctx):
|
||||
t = lib.ggml_new_tensor_2d(ctx, lib.GGML_TYPE_I32, 2, 3)
|
||||
a = np.arange(2 * 3, dtype=np.int32).reshape((2, 3))
|
||||
self._test_copy_np_to_ggml(a, t)
|
||||
|
||||
def test_copy_np_to_ggml_3d_i32(self, ctx):
|
||||
t = lib.ggml_new_tensor_3d(ctx, lib.GGML_TYPE_I32, 2, 3, 4)
|
||||
a = np.arange(2 * 3 * 4, dtype=np.int32).reshape((2, 3, 4))
|
||||
self._test_copy_np_to_ggml(a, t)
|
||||
|
||||
def test_copy_np_to_ggml_4d_i32(self, ctx):
|
||||
t = lib.ggml_new_tensor_4d(ctx, lib.GGML_TYPE_I32, 2, 3, 4, 5)
|
||||
a = np.arange(2 * 3 * 4 * 5, dtype=np.int32).reshape((2, 3, 4, 5))
|
||||
self._test_copy_np_to_ggml(a, t)
|
||||
|
||||
def test_copy_np_to_ggml_4d_n_i32(self, ctx):
|
||||
dims = [2, 3, 4, 5] # GGML_MAX_DIMS is 4, going beyond would crash
|
||||
pdims = ffi.new('int64_t[]', len(dims))
|
||||
for i, d in enumerate(dims): pdims[i] = d
|
||||
t = lib.ggml_new_tensor(ctx, lib.GGML_TYPE_I32, len(dims), pdims)
|
||||
a = np.arange(np.prod(dims), dtype=np.int32).reshape(tuple(pdims))
|
||||
self._test_copy_np_to_ggml(a, t)
|
||||
|
||||
# F32
|
||||
|
||||
def test_copy_np_to_ggml_1d_f32(self, ctx):
|
||||
t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 10)
|
||||
a = np.arange(10, dtype=np.float32)
|
||||
self._test_copy_np_to_ggml(a, t)
|
||||
|
||||
def test_copy_np_to_ggml_2d_f32(self, ctx):
|
||||
t = lib.ggml_new_tensor_2d(ctx, lib.GGML_TYPE_F32, 2, 3)
|
||||
a = np.arange(2 * 3, dtype=np.float32).reshape((2, 3))
|
||||
self._test_copy_np_to_ggml(a, t)
|
||||
|
||||
def test_copy_np_to_ggml_3d_f32(self, ctx):
|
||||
t = lib.ggml_new_tensor_3d(ctx, lib.GGML_TYPE_F32, 2, 3, 4)
|
||||
a = np.arange(2 * 3 * 4, dtype=np.float32).reshape((2, 3, 4))
|
||||
self._test_copy_np_to_ggml(a, t)
|
||||
|
||||
def test_copy_np_to_ggml_4d_f32(self, ctx):
|
||||
t = lib.ggml_new_tensor_4d(ctx, lib.GGML_TYPE_F32, 2, 3, 4, 5)
|
||||
a = np.arange(2 * 3 * 4 * 5, dtype=np.float32).reshape((2, 3, 4, 5))
|
||||
self._test_copy_np_to_ggml(a, t)
|
||||
|
||||
def test_copy_np_to_ggml_4d_n_f32(self, ctx):
|
||||
dims = [2, 3, 4, 5] # GGML_MAX_DIMS is 4, going beyond would crash
|
||||
pdims = ffi.new('int64_t[]', len(dims))
|
||||
for i, d in enumerate(dims): pdims[i] = d
|
||||
t = lib.ggml_new_tensor(ctx, lib.GGML_TYPE_F32, len(dims), pdims)
|
||||
a = np.arange(np.prod(dims), dtype=np.float32).reshape(tuple(pdims))
|
||||
self._test_copy_np_to_ggml(a, t)
|
||||
|
||||
# F16
|
||||
|
||||
def test_copy_np_to_ggml_1d_f16(self, ctx):
|
||||
t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F16, 10)
|
||||
a = np.arange(10, dtype=np.float16)
|
||||
self._test_copy_np_to_ggml(a, t)
|
||||
|
||||
def test_copy_np_to_ggml_2d_f16(self, ctx):
|
||||
t = lib.ggml_new_tensor_2d(ctx, lib.GGML_TYPE_F16, 2, 3)
|
||||
a = np.arange(2 * 3, dtype=np.float16).reshape((2, 3))
|
||||
self._test_copy_np_to_ggml(a, t)
|
||||
|
||||
def test_copy_np_to_ggml_3d_f16(self, ctx):
|
||||
t = lib.ggml_new_tensor_3d(ctx, lib.GGML_TYPE_F16, 2, 3, 4)
|
||||
a = np.arange(2 * 3 * 4, dtype=np.float16).reshape((2, 3, 4))
|
||||
self._test_copy_np_to_ggml(a, t)
|
||||
|
||||
def test_copy_np_to_ggml_4d_f16(self, ctx):
|
||||
t = lib.ggml_new_tensor_4d(ctx, lib.GGML_TYPE_F16, 2, 3, 4, 5)
|
||||
a = np.arange(2 * 3 * 4 * 5, dtype=np.float16).reshape((2, 3, 4, 5))
|
||||
self._test_copy_np_to_ggml(a, t)
|
||||
|
||||
def test_copy_np_to_ggml_4d_n_f16(self, ctx):
|
||||
dims = [2, 3, 4, 5] # GGML_MAX_DIMS is 4, going beyond would crash
|
||||
pdims = ffi.new('int64_t[]', len(dims))
|
||||
for i, d in enumerate(dims): pdims[i] = d
|
||||
t = lib.ggml_new_tensor(ctx, lib.GGML_TYPE_F16, len(dims), pdims)
|
||||
a = np.arange(np.prod(dims), dtype=np.float16).reshape(tuple(pdims))
|
||||
self._test_copy_np_to_ggml(a, t)
|
||||
|
||||
# Mismatching shapes
|
||||
|
||||
def test_copy_mismatching_shapes_1d(self, ctx):
|
||||
t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 10)
|
||||
a = np.arange(10, dtype=np.float32)
|
||||
copy(a, t) # OK
|
||||
|
||||
a = a.reshape((5, 2))
|
||||
with raises(AssertionError): copy(a, t)
|
||||
with raises(AssertionError): copy(t, a)
|
||||
|
||||
def test_copy_mismatching_shapes_2d(self, ctx):
|
||||
t = lib.ggml_new_tensor_2d(ctx, lib.GGML_TYPE_F32, 2, 3)
|
||||
a = np.arange(6, dtype=np.float32)
|
||||
copy(a.reshape((2, 3)), t) # OK
|
||||
|
||||
a = a.reshape((3, 2))
|
||||
with raises(AssertionError): copy(a, t)
|
||||
with raises(AssertionError): copy(t, a)
|
||||
|
||||
def test_copy_mismatching_shapes_3d(self, ctx):
|
||||
t = lib.ggml_new_tensor_3d(ctx, lib.GGML_TYPE_F32, 2, 3, 4)
|
||||
a = np.arange(24, dtype=np.float32)
|
||||
copy(a.reshape((2, 3, 4)), t) # OK
|
||||
|
||||
a = a.reshape((2, 4, 3))
|
||||
with raises(AssertionError): copy(a, t)
|
||||
with raises(AssertionError): copy(t, a)
|
||||
|
||||
def test_copy_mismatching_shapes_4d(self, ctx):
|
||||
t = lib.ggml_new_tensor_4d(ctx, lib.GGML_TYPE_F32, 2, 3, 4, 5)
|
||||
a = np.arange(24*5, dtype=np.float32)
|
||||
copy(a.reshape((2, 3, 4, 5)), t) # OK
|
||||
|
||||
a = a.reshape((2, 3, 5, 4))
|
||||
with raises(AssertionError): copy(a, t)
|
||||
with raises(AssertionError): copy(t, a)
|
||||
|
||||
def test_copy_f16_to_f32(self, ctx):
|
||||
t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 1)
|
||||
a = np.array([123.45], dtype=np.float16)
|
||||
copy(a, t)
|
||||
np.testing.assert_allclose(lib.ggml_get_f32_1d(t, 0), 123.45, rtol=1e-3)
|
||||
|
||||
def test_copy_f32_to_f16(self, ctx):
|
||||
t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F16, 1)
|
||||
a = np.array([123.45], dtype=np.float32)
|
||||
copy(a, t)
|
||||
np.testing.assert_allclose(lib.ggml_get_f32_1d(t, 0), 123.45, rtol=1e-3)
|
||||
|
||||
def test_copy_f16_to_Q5_K(self, ctx):
|
||||
n = 256
|
||||
t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n)
|
||||
a = np.arange(n, dtype=np.float16)
|
||||
copy(a, t)
|
||||
np.testing.assert_allclose(a, numpy(t, allow_copy=True), rtol=0.05)
|
||||
|
||||
def test_copy_Q5_K_to_f16(self, ctx):
|
||||
n = 256
|
||||
t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n)
|
||||
copy(np.arange(n, dtype=np.float32), t)
|
||||
a = np.arange(n, dtype=np.float16)
|
||||
copy(t, a)
|
||||
np.testing.assert_allclose(a, numpy(t, allow_copy=True), rtol=0.05)
|
||||
|
||||
def test_copy_i16_f32_mismatching_types(self, ctx):
|
||||
t = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 1)
|
||||
a = np.arange(1, dtype=np.int16)
|
||||
with raises(NotImplementedError): copy(a, t)
|
||||
with raises(NotImplementedError): copy(t, a)
|
||||
|
||||
class TestTensorCopy:
|
||||
|
||||
def test_copy_self(self, ctx):
|
||||
t = lib.ggml_new_i32(ctx, 42)
|
||||
copy(t, t)
|
||||
assert lib.ggml_get_i32_1d(t, 0) == 42
|
||||
|
||||
def test_copy_1d(self, ctx):
|
||||
t1 = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 10)
|
||||
t2 = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, 10)
|
||||
a = np.arange(10, dtype=np.float32)
|
||||
copy(a, t1)
|
||||
copy(t1, t2)
|
||||
assert np.allclose(a, numpy(t2))
|
||||
assert np.allclose(numpy(t1), numpy(t2))
|
||||
|
||||
class TestGraph:
|
||||
|
||||
def test_add(self, ctx):
|
||||
n = 256
|
||||
ta = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n)
|
||||
tb = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n)
|
||||
tsum = lib.ggml_add(ctx, ta, tb)
|
||||
assert tsum.type == lib.GGML_TYPE_F32
|
||||
|
||||
gf = ffi.new('struct ggml_cgraph*')
|
||||
lib.ggml_build_forward_expand(gf, tsum)
|
||||
|
||||
a = np.arange(0, n, dtype=np.float32)
|
||||
b = np.arange(n, 0, -1, dtype=np.float32)
|
||||
copy(a, ta)
|
||||
copy(b, tb)
|
||||
|
||||
lib.ggml_graph_compute_with_ctx(ctx, gf, 1)
|
||||
|
||||
assert np.allclose(numpy(tsum, allow_copy=True), a + b)
|
||||
|
||||
class TestQuantization:
|
||||
|
||||
def test_quantized_add(self, ctx):
|
||||
n = 256
|
||||
ta = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_Q5_K, n)
|
||||
tb = lib.ggml_new_tensor_1d(ctx, lib.GGML_TYPE_F32, n)
|
||||
tsum = lib.ggml_add(ctx, ta, tb)
|
||||
assert tsum.type == lib.GGML_TYPE_Q5_K
|
||||
|
||||
gf = ffi.new('struct ggml_cgraph*')
|
||||
lib.ggml_build_forward_expand(gf, tsum)
|
||||
|
||||
a = np.arange(0, n, dtype=np.float32)
|
||||
b = np.arange(n, 0, -1, dtype=np.float32)
|
||||
copy(a, ta)
|
||||
copy(b, tb)
|
||||
|
||||
lib.ggml_graph_compute_with_ctx(ctx, gf, 1)
|
||||
|
||||
unquantized_sum = a + b
|
||||
sum = numpy(tsum, allow_copy=True)
|
||||
|
||||
diff = np.linalg.norm(unquantized_sum - sum, np.inf)
|
||||
assert diff > 4
|
||||
assert diff < 5
|
||||
Reference in New Issue
Block a user