wehub-resource-sync
505 lines
20 KiB
Python
505 lines
20 KiB
Python
import numpy as np
|
|
|
|
|
|
def float_to_fp32_bits(value):
|
|
bits = np.float32(value).view(np.uint32)
|
|
sign = (bits >> 31) & 1
|
|
exponent = (bits >> 23) & 0xFF
|
|
mantissa = bits & 0x7FFFFF
|
|
return {"sign": int(sign), "exponent": int(exponent), "mantissa": int(mantissa),
|
|
"exponent_bits": format(int(exponent), '08b'),
|
|
"mantissa_bits": format(int(mantissa), '023b'),
|
|
"value": float(value),
|
|
"actual_exponent": int(exponent) - 127}
|
|
|
|
|
|
def float_to_fp16_bits(value):
|
|
fp16 = np.float16(value)
|
|
bits = fp16.view(np.uint16)
|
|
sign = (bits >> 15) & 1
|
|
exponent = (bits >> 10) & 0x1F
|
|
mantissa = bits & 0x3FF
|
|
return {"sign": int(sign), "exponent": int(exponent), "mantissa": int(mantissa),
|
|
"exponent_bits": format(int(exponent), '05b'),
|
|
"mantissa_bits": format(int(mantissa), '010b'),
|
|
"value": float(fp16),
|
|
"actual_exponent": int(exponent) - 15}
|
|
|
|
|
|
def float_to_bf16_bits(value):
|
|
fp32_bits = np.float32(value).view(np.uint32)
|
|
bf16_bits = (fp32_bits >> 16).astype(np.uint16)
|
|
sign = (bf16_bits >> 15) & 1
|
|
exponent = (bf16_bits >> 7) & 0xFF
|
|
mantissa = bf16_bits & 0x7F
|
|
reconstructed = np.uint32(bf16_bits.astype(np.uint32) << 16).view(np.float32)
|
|
return {"sign": int(sign), "exponent": int(exponent), "mantissa": int(mantissa),
|
|
"exponent_bits": format(int(exponent), '08b'),
|
|
"mantissa_bits": format(int(mantissa), '07b'),
|
|
"value": float(reconstructed),
|
|
"actual_exponent": int(exponent) - 127}
|
|
|
|
|
|
def simulate_fp8_e4m3(value):
|
|
sign = 1 if value < 0 else 0
|
|
abs_val = abs(value)
|
|
max_val = 448.0
|
|
abs_val = min(abs_val, max_val)
|
|
if abs_val == 0:
|
|
return {"sign": sign, "exponent": 0, "mantissa": 0, "value": 0.0,
|
|
"exponent_bits": "0000", "mantissa_bits": "000"}
|
|
exp = int(np.floor(np.log2(abs_val)))
|
|
exp = max(-6, min(8, exp))
|
|
mantissa_val = abs_val / (2.0 ** exp) - 1.0
|
|
mantissa_quant = round(mantissa_val * 8) / 8
|
|
mantissa_quant = max(0, min(0.875, mantissa_quant))
|
|
reconstructed = (1.0 + mantissa_quant) * (2.0 ** exp)
|
|
if sign:
|
|
reconstructed = -reconstructed
|
|
mantissa_int = int(round(mantissa_quant * 8))
|
|
return {"sign": sign, "exponent": exp + 7, "mantissa": mantissa_int,
|
|
"exponent_bits": format(exp + 7, '04b'),
|
|
"mantissa_bits": format(mantissa_int, '03b'),
|
|
"value": float(reconstructed),
|
|
"actual_exponent": exp}
|
|
|
|
|
|
def display_format_comparison(value):
|
|
fp32 = float_to_fp32_bits(value)
|
|
fp16 = float_to_fp16_bits(value)
|
|
bf16 = float_to_bf16_bits(value)
|
|
fp8 = simulate_fp8_e4m3(value)
|
|
|
|
print(f"\n Value: {value}")
|
|
print(f" {'Format':<8} {'Stored Value':>14} {'Error':>12} {'Sign':>5} {'Exp Bits':>10} {'Man Bits':>25}")
|
|
print(f" {'-'*76}")
|
|
print(f" {'FP32':<8} {fp32['value']:>14.6f} {abs(fp32['value'] - value):>12.8f} {fp32['sign']:>5} {fp32['exponent_bits']:>10} {fp32['mantissa_bits']:>25}")
|
|
print(f" {'FP16':<8} {fp16['value']:>14.6f} {abs(fp16['value'] - value):>12.8f} {fp16['sign']:>5} {fp16['exponent_bits']:>10} {fp16['mantissa_bits']:>25}")
|
|
print(f" {'BF16':<8} {bf16['value']:>14.6f} {abs(bf16['value'] - value):>12.8f} {bf16['sign']:>5} {bf16['exponent_bits']:>10} {bf16['mantissa_bits']:>25}")
|
|
print(f" {'FP8e4m3':<8} {fp8['value']:>14.6f} {abs(fp8['value'] - value):>12.8f} {fp8['sign']:>5} {fp8['exponent_bits']:>10} {fp8['mantissa_bits']:>25}")
|
|
|
|
|
|
def quantize_symmetric(tensor, num_bits=8):
|
|
qmin = -(2 ** (num_bits - 1))
|
|
qmax = 2 ** (num_bits - 1) - 1
|
|
abs_max = np.max(np.abs(tensor))
|
|
if abs_max == 0:
|
|
return np.zeros_like(tensor, dtype=np.int32), 1.0
|
|
scale = abs_max / qmax
|
|
quantized = np.clip(np.round(tensor / scale), qmin, qmax).astype(np.int32)
|
|
return quantized, float(scale)
|
|
|
|
|
|
def dequantize_symmetric(quantized, scale):
|
|
return quantized.astype(np.float64) * scale
|
|
|
|
|
|
def quantize_per_channel(tensor, num_bits=8, axis=0):
|
|
qmin = -(2 ** (num_bits - 1))
|
|
qmax = 2 ** (num_bits - 1) - 1
|
|
|
|
if axis == 0:
|
|
abs_max = np.max(np.abs(tensor), axis=1, keepdims=True)
|
|
else:
|
|
abs_max = np.max(np.abs(tensor), axis=0, keepdims=True)
|
|
|
|
abs_max = np.where(abs_max == 0, 1.0, abs_max)
|
|
scales = abs_max / qmax
|
|
quantized = np.clip(np.round(tensor / scales), qmin, qmax).astype(np.int32)
|
|
return quantized, scales.squeeze()
|
|
|
|
|
|
def dequantize_per_channel(quantized, scales, axis=0):
|
|
if axis == 0:
|
|
return quantized.astype(np.float64) * scales.reshape(-1, 1)
|
|
else:
|
|
return quantized.astype(np.float64) * scales.reshape(1, -1)
|
|
|
|
|
|
def quantize_asymmetric(tensor, num_bits=8):
|
|
qmin = 0
|
|
qmax = 2 ** num_bits - 1
|
|
t_min = np.min(tensor)
|
|
t_max = np.max(tensor)
|
|
if t_max == t_min:
|
|
return np.zeros_like(tensor, dtype=np.int32), 1.0, 0
|
|
scale = (t_max - t_min) / (qmax - qmin)
|
|
zero_point = int(np.round(qmin - t_min / scale))
|
|
zero_point = max(qmin, min(qmax, zero_point))
|
|
quantized = np.clip(np.round(tensor / scale + zero_point), qmin, qmax).astype(np.int32)
|
|
return quantized, float(scale), int(zero_point)
|
|
|
|
|
|
def dequantize_asymmetric(quantized, scale, zero_point):
|
|
return (quantized.astype(np.float64) - zero_point) * scale
|
|
|
|
|
|
def quantization_error(original, reconstructed):
|
|
diff = original - reconstructed
|
|
mse = float(np.mean(diff ** 2))
|
|
rmse = float(np.sqrt(mse))
|
|
max_error = float(np.max(np.abs(diff)))
|
|
signal_power = float(np.mean(original ** 2))
|
|
snr_db = 10 * np.log10(signal_power / max(mse, 1e-20))
|
|
|
|
orig_flat = original.flatten()
|
|
recon_flat = reconstructed.flatten()
|
|
norm_orig = np.linalg.norm(orig_flat)
|
|
norm_recon = np.linalg.norm(recon_flat)
|
|
if norm_orig == 0 or norm_recon == 0:
|
|
cosine_sim = 0.0
|
|
else:
|
|
cosine_sim = float(np.dot(orig_flat, recon_flat) / (norm_orig * norm_recon))
|
|
|
|
return {"mse": mse, "rmse": rmse, "max_error": max_error,
|
|
"snr_db": float(snr_db), "cosine_similarity": cosine_sim}
|
|
|
|
|
|
def compare_quantization_methods(tensor, num_bits=8):
|
|
q_pt, s_pt = quantize_symmetric(tensor, num_bits)
|
|
recon_pt = dequantize_symmetric(q_pt, s_pt)
|
|
err_pt = quantization_error(tensor, recon_pt)
|
|
|
|
q_pc, s_pc = quantize_per_channel(tensor, num_bits, axis=0)
|
|
recon_pc = dequantize_per_channel(q_pc, s_pc, axis=0)
|
|
err_pc = quantization_error(tensor, recon_pc)
|
|
|
|
q_asym, s_asym, zp = quantize_asymmetric(tensor, num_bits)
|
|
recon_asym = dequantize_asymmetric(q_asym, s_asym, zp)
|
|
err_asym = quantization_error(tensor, recon_asym)
|
|
|
|
print(f"\n Quantization Comparison ({num_bits}-bit, tensor shape {tensor.shape}):")
|
|
print(f" {'Method':<20} {'MSE':>12} {'SNR (dB)':>10} {'Cosine Sim':>12} {'Max Error':>12}")
|
|
print(f" {'-'*68}")
|
|
print(f" {'Per-tensor sym':<20} {err_pt['mse']:>12.8f} {err_pt['snr_db']:>10.2f} {err_pt['cosine_similarity']:>12.8f} {err_pt['max_error']:>12.8f}")
|
|
print(f" {'Per-channel sym':<20} {err_pc['mse']:>12.8f} {err_pc['snr_db']:>10.2f} {err_pc['cosine_similarity']:>12.8f} {err_pc['max_error']:>12.8f}")
|
|
print(f" {'Asymmetric':<20} {err_asym['mse']:>12.8f} {err_asym['snr_db']:>10.2f} {err_asym['cosine_similarity']:>12.8f} {err_asym['max_error']:>12.8f}")
|
|
|
|
return {"per_tensor": err_pt, "per_channel": err_pc, "asymmetric": err_asym}
|
|
|
|
|
|
def bit_width_sweep(tensor):
|
|
print(f"\n Bit-Width Sweep (tensor shape {tensor.shape}):")
|
|
print(f" {'Bits':>6} {'Levels':>8} {'MSE':>14} {'SNR (dB)':>10} {'Cosine Sim':>12} {'Compression':>12}")
|
|
print(f" {'-'*64}")
|
|
|
|
results = []
|
|
for bits in [2, 3, 4, 8, 16]:
|
|
q, s = quantize_per_channel(tensor, bits, axis=0)
|
|
recon = dequantize_per_channel(q, s, axis=0)
|
|
err = quantization_error(tensor, recon)
|
|
levels = 2 ** bits
|
|
compression = 32.0 / bits
|
|
|
|
print(f" {bits:>6} {levels:>8} {err['mse']:>14.8f} {err['snr_db']:>10.2f} {err['cosine_similarity']:>12.8f} {compression:>11.1f}x")
|
|
results.append({"bits": bits, "levels": levels, "error": err, "compression": compression})
|
|
|
|
return results
|
|
|
|
|
|
def simulate_transformer_layer(input_data, weights, kv_scale=1.0):
|
|
hidden = input_data @ weights["qkv"]
|
|
seq_len = hidden.shape[1]
|
|
d_model = weights["qkv"].shape[1] // 3
|
|
q, k, v = hidden[:, :, :d_model], hidden[:, :, d_model:2*d_model], hidden[:, :, 2*d_model:]
|
|
|
|
attn_scores = (q @ k.transpose(0, 2, 1)) / np.sqrt(d_model) * kv_scale
|
|
attn_max = np.max(attn_scores, axis=-1, keepdims=True)
|
|
attn_exp = np.exp(attn_scores - attn_max)
|
|
attn_weights = attn_exp / np.sum(attn_exp, axis=-1, keepdims=True)
|
|
|
|
attn_output = attn_weights @ v
|
|
output = attn_output @ weights["out"]
|
|
return output, {"q": q, "k": k, "v": v, "attn_scores": attn_scores,
|
|
"attn_weights": attn_weights, "attn_output": attn_output}
|
|
|
|
|
|
def sensitivity_experiment(batch_size=2, seq_len=16, d_model=64, num_bits=8):
|
|
np.random.seed(42)
|
|
input_data = np.random.randn(batch_size, seq_len, d_model) * 0.1
|
|
|
|
weights = {
|
|
"qkv": np.random.randn(d_model, 3 * d_model) * (2.0 / d_model) ** 0.5,
|
|
"out": np.random.randn(d_model, d_model) * (2.0 / d_model) ** 0.5,
|
|
}
|
|
|
|
baseline_output, baseline_internals = simulate_transformer_layer(input_data, weights)
|
|
|
|
experiments = {}
|
|
|
|
q_qkv, s_qkv = quantize_per_channel(weights["qkv"], num_bits, axis=0)
|
|
q_out, s_out = quantize_per_channel(weights["out"], num_bits, axis=0)
|
|
quantized_weights = {
|
|
"qkv": dequantize_per_channel(q_qkv, s_qkv, axis=0),
|
|
"out": dequantize_per_channel(q_out, s_out, axis=0),
|
|
}
|
|
weight_quant_output, _ = simulate_transformer_layer(input_data, quantized_weights)
|
|
experiments["Weights only"] = quantization_error(baseline_output, weight_quant_output)
|
|
|
|
_, fresh_internals = simulate_transformer_layer(input_data, weights)
|
|
q_act, s_act = quantize_per_channel(
|
|
fresh_internals["attn_output"].reshape(-1, d_model), num_bits, axis=0
|
|
)
|
|
quant_attn_out = dequantize_per_channel(q_act, s_act, axis=0).reshape(batch_size, seq_len, d_model)
|
|
act_quant_output = quant_attn_out @ weights["out"]
|
|
experiments["Activations only"] = quantization_error(baseline_output, act_quant_output)
|
|
|
|
q_k, s_k = quantize_per_channel(fresh_internals["k"].reshape(-1, d_model), num_bits, axis=0)
|
|
q_v, s_v = quantize_per_channel(fresh_internals["v"].reshape(-1, d_model), num_bits, axis=0)
|
|
quant_k = dequantize_per_channel(q_k, s_k, axis=0).reshape(batch_size, seq_len, d_model)
|
|
quant_v = dequantize_per_channel(q_v, s_v, axis=0).reshape(batch_size, seq_len, d_model)
|
|
attn_scores_kv = (fresh_internals["q"] @ quant_k.transpose(0, 2, 1)) / np.sqrt(d_model)
|
|
attn_max_kv = np.max(attn_scores_kv, axis=-1, keepdims=True)
|
|
attn_exp_kv = np.exp(attn_scores_kv - attn_max_kv)
|
|
attn_weights_kv = attn_exp_kv / np.sum(attn_exp_kv, axis=-1, keepdims=True)
|
|
kv_quant_output = (attn_weights_kv @ quant_v) @ weights["out"]
|
|
experiments["KV cache only"] = quantization_error(baseline_output, kv_quant_output)
|
|
|
|
noise_scale = np.std(fresh_internals["attn_scores"]) * 0.05
|
|
noisy_scores = fresh_internals["attn_scores"] + np.random.randn(*fresh_internals["attn_scores"].shape) * noise_scale
|
|
noisy_max = np.max(noisy_scores, axis=-1, keepdims=True)
|
|
noisy_exp = np.exp(noisy_scores - noisy_max)
|
|
noisy_weights = noisy_exp / np.sum(noisy_exp, axis=-1, keepdims=True)
|
|
attn_quant_output = (noisy_weights @ fresh_internals["v"]) @ weights["out"]
|
|
experiments["Attention logits (5% noise)"] = quantization_error(baseline_output, attn_quant_output)
|
|
|
|
print(f"\n Sensitivity Experiment ({num_bits}-bit quantization):")
|
|
print(f" {'Component':<30} {'MSE':>14} {'SNR (dB)':>10} {'Cosine Sim':>12}")
|
|
print(f" {'-'*68}")
|
|
for name, err in sorted(experiments.items(), key=lambda x: x[1]["mse"]):
|
|
print(f" {name:<30} {err['mse']:>14.8f} {err['snr_db']:>10.2f} {err['cosine_similarity']:>12.8f}")
|
|
|
|
return experiments
|
|
|
|
|
|
def simulated_gptq(weight_matrix, calibration_inputs, num_bits=4):
|
|
n_in, n_out = weight_matrix.shape
|
|
qmin = -(2 ** (num_bits - 1))
|
|
qmax = 2 ** (num_bits - 1) - 1
|
|
|
|
H = np.zeros((n_in, n_in))
|
|
for x in calibration_inputs:
|
|
x = x.reshape(-1, 1) if x.ndim == 1 else x
|
|
for row in range(x.shape[0]):
|
|
xi = x[row].reshape(-1, 1)
|
|
H += xi @ xi.T
|
|
H /= len(calibration_inputs)
|
|
H += np.eye(n_in) * 1e-4
|
|
|
|
weight_importance = np.diag(H)
|
|
|
|
quantized = np.zeros_like(weight_matrix, dtype=np.int32)
|
|
scales = np.zeros(n_out)
|
|
errors = np.zeros(n_out)
|
|
|
|
W = weight_matrix.copy()
|
|
|
|
for col in range(n_out):
|
|
w_col = W[:, col]
|
|
abs_max = np.max(np.abs(w_col))
|
|
if abs_max == 0:
|
|
scales[col] = 1.0
|
|
continue
|
|
scale = abs_max / qmax
|
|
scales[col] = scale
|
|
|
|
q_col = np.clip(np.round(w_col / scale), qmin, qmax).astype(np.int32)
|
|
quantized[:, col] = q_col
|
|
|
|
quant_error = w_col - q_col * scale
|
|
errors[col] = np.sqrt(np.mean(quant_error ** 2))
|
|
|
|
if col < n_out - 1:
|
|
importance_weights = weight_importance / (np.max(weight_importance) + 1e-10)
|
|
for next_col in range(col + 1, min(col + 4, n_out)):
|
|
compensation = quant_error * importance_weights * 0.1
|
|
W[:, next_col] += compensation
|
|
|
|
return quantized, scales, {"column_errors": errors,
|
|
"mean_error": float(np.mean(errors)),
|
|
"max_error": float(np.max(errors))}
|
|
|
|
|
|
def dequantize_gptq(quantized, scales):
|
|
result = np.zeros_like(quantized, dtype=np.float64)
|
|
for col in range(quantized.shape[1]):
|
|
result[:, col] = quantized[:, col] * scales[col]
|
|
return result
|
|
|
|
|
|
def simulated_awq(weight_matrix, calibration_inputs, num_bits=4, salient_fraction=0.01):
|
|
n_in, n_out = weight_matrix.shape
|
|
qmin = -(2 ** (num_bits - 1))
|
|
qmax = 2 ** (num_bits - 1) - 1
|
|
|
|
activation_magnitudes = np.zeros(n_in)
|
|
for x in calibration_inputs:
|
|
if x.ndim == 1:
|
|
activation_magnitudes += np.abs(x)
|
|
else:
|
|
activation_magnitudes += np.mean(np.abs(x), axis=0)
|
|
activation_magnitudes /= len(calibration_inputs)
|
|
|
|
n_salient = max(1, int(n_in * salient_fraction))
|
|
salient_indices = np.argsort(activation_magnitudes)[-n_salient:]
|
|
|
|
scale_factors = np.ones(n_in)
|
|
for idx in salient_indices:
|
|
col_max = np.max(np.abs(weight_matrix[idx, :]))
|
|
if col_max > 0:
|
|
scale_factors[idx] = min(4.0, 1.0 / (col_max + 1e-8) * np.mean(np.abs(weight_matrix)))
|
|
|
|
scaled_weights = weight_matrix * scale_factors.reshape(-1, 1)
|
|
|
|
quantized, scales = quantize_per_channel(scaled_weights, num_bits, axis=0)
|
|
dequantized = dequantize_per_channel(quantized, scales, axis=0)
|
|
|
|
result = dequantized / scale_factors.reshape(-1, 1)
|
|
|
|
err = quantization_error(weight_matrix, result)
|
|
|
|
return result, {"salient_indices": salient_indices,
|
|
"scale_factors": scale_factors[salient_indices],
|
|
"error": err,
|
|
"n_salient": n_salient}
|
|
|
|
|
|
def full_quantization_comparison(d_in=256, d_out=512, num_bits=4, n_calibration=32):
|
|
np.random.seed(42)
|
|
|
|
weight = np.random.randn(d_in, d_out) * 0.02
|
|
outlier_rows = np.random.choice(d_in, size=5, replace=False)
|
|
weight[outlier_rows] *= 10
|
|
|
|
calibration = [np.random.randn(8, d_in) * 0.1 for _ in range(n_calibration)]
|
|
|
|
q_naive, s_naive = quantize_symmetric(weight, num_bits)
|
|
recon_naive = dequantize_symmetric(q_naive, s_naive)
|
|
err_naive = quantization_error(weight, recon_naive)
|
|
|
|
q_pc, s_pc = quantize_per_channel(weight, num_bits, axis=0)
|
|
recon_pc = dequantize_per_channel(q_pc, s_pc, axis=0)
|
|
err_pc = quantization_error(weight, recon_pc)
|
|
|
|
q_gptq, s_gptq, gptq_info = simulated_gptq(weight, calibration, num_bits)
|
|
recon_gptq = dequantize_gptq(q_gptq, s_gptq)
|
|
err_gptq = quantization_error(weight, recon_gptq)
|
|
|
|
recon_awq, awq_info = simulated_awq(weight, calibration, num_bits)
|
|
err_awq = awq_info["error"]
|
|
|
|
print(f"\n Full Quantization Comparison ({num_bits}-bit, {d_in}x{d_out} matrix)")
|
|
print(f" Matrix has {len(outlier_rows)} outlier rows (10x scale)")
|
|
print()
|
|
print(f" {'Method':<20} {'MSE':>14} {'SNR (dB)':>10} {'Cosine Sim':>12}")
|
|
print(f" {'-'*58}")
|
|
print(f" {'Naive per-tensor':<20} {err_naive['mse']:>14.8f} {err_naive['snr_db']:>10.2f} {err_naive['cosine_similarity']:>12.8f}")
|
|
print(f" {'Per-channel':<20} {err_pc['mse']:>14.8f} {err_pc['snr_db']:>10.2f} {err_pc['cosine_similarity']:>12.8f}")
|
|
print(f" {'Simulated GPTQ':<20} {err_gptq['mse']:>14.8f} {err_gptq['snr_db']:>10.2f} {err_gptq['cosine_similarity']:>12.8f}")
|
|
print(f" {'Simulated AWQ':<20} {err_awq['mse']:>14.8f} {err_awq['snr_db']:>10.2f} {err_awq['cosine_similarity']:>12.8f}")
|
|
|
|
test_input = np.random.randn(4, d_in) * 0.1
|
|
baseline = test_input @ weight
|
|
output_naive = test_input @ recon_naive
|
|
output_pc = test_input @ recon_pc
|
|
output_gptq = test_input @ recon_gptq
|
|
output_awq = test_input @ recon_awq
|
|
|
|
print(f"\n End-to-End Output Error (matmul with test input):")
|
|
print(f" {'Method':<20} {'Output MSE':>14} {'Output Cosine':>14}")
|
|
print(f" {'-'*50}")
|
|
for name, output in [("Naive", output_naive), ("Per-channel", output_pc),
|
|
("GPTQ", output_gptq), ("AWQ", output_awq)]:
|
|
out_err = quantization_error(baseline, output)
|
|
print(f" {name:<20} {out_err['mse']:>14.8f} {out_err['cosine_similarity']:>14.8f}")
|
|
|
|
return {"naive": err_naive, "per_channel": err_pc, "gptq": err_gptq, "awq": err_awq}
|
|
|
|
|
|
def memory_calculator(num_params_billions, bits_per_param):
|
|
bytes_per_param = bits_per_param / 8
|
|
total_bytes = num_params_billions * 1e9 * bytes_per_param
|
|
total_gb = total_bytes / (1024 ** 3)
|
|
return total_gb
|
|
|
|
|
|
def print_memory_table():
|
|
print("\n Memory Requirements by Model and Precision:")
|
|
print(f" {'Model':<15} {'FP32':>8} {'FP16':>8} {'FP8':>8} {'INT8':>8} {'INT4':>8} {'INT2':>8}")
|
|
print(f" {'-'*64}")
|
|
for name, params in [("7B", 7), ("13B", 13), ("34B", 34), ("70B", 70), ("405B", 405)]:
|
|
fp32 = memory_calculator(params, 32)
|
|
fp16 = memory_calculator(params, 16)
|
|
fp8 = memory_calculator(params, 8)
|
|
int8 = memory_calculator(params, 8)
|
|
int4 = memory_calculator(params, 4)
|
|
int2 = memory_calculator(params, 2)
|
|
print(f" {name:<15} {fp32:>7.1f}G {fp16:>7.1f}G {fp8:>7.1f}G {int8:>7.1f}G {int4:>7.1f}G {int2:>7.1f}G")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
np.random.seed(42)
|
|
|
|
print("=" * 70)
|
|
print("QUANTIZATION: MAKING MODELS FIT")
|
|
print("=" * 70)
|
|
|
|
print("\nSTEP 1: Number Format Comparison")
|
|
print("-" * 50)
|
|
for val in [0.1, 3.14159, -0.00073, 42.5, 0.0000012]:
|
|
display_format_comparison(val)
|
|
|
|
print("\n\nSTEP 2: Memory Requirements")
|
|
print("-" * 50)
|
|
print_memory_table()
|
|
|
|
print("\n\nSTEP 3: Quantization Methods Comparison")
|
|
print("-" * 50)
|
|
weight_matrix = np.random.randn(128, 256) * 0.02
|
|
weight_matrix[0] *= 15
|
|
weight_matrix[42] *= 8
|
|
compare_quantization_methods(weight_matrix, num_bits=8)
|
|
compare_quantization_methods(weight_matrix, num_bits=4)
|
|
|
|
print("\n\nSTEP 4: Bit-Width Sweep")
|
|
print("-" * 50)
|
|
sweep_tensor = np.random.randn(64, 128) * 0.05
|
|
bit_width_sweep(sweep_tensor)
|
|
|
|
print("\n\nSTEP 5: Sensitivity Experiment")
|
|
print("-" * 50)
|
|
print("\n INT8:")
|
|
sensitivity_experiment(num_bits=8)
|
|
print("\n INT4:")
|
|
sensitivity_experiment(num_bits=4)
|
|
|
|
print("\n\nSTEP 6: GPTQ vs AWQ vs Naive (INT4)")
|
|
print("-" * 50)
|
|
full_quantization_comparison(d_in=256, d_out=512, num_bits=4)
|
|
|
|
print("\n\nSTEP 7: Distribution Analysis")
|
|
print("-" * 50)
|
|
np.random.seed(0)
|
|
simulated_weights = np.random.randn(1000) * 0.02
|
|
abs_vals = np.abs(simulated_weights)
|
|
pct_in_range = np.mean(abs_vals < 0.1) * 100
|
|
print(f"\n Simulated weight distribution (1000 params, std=0.02):")
|
|
print(f" Weights in [-0.1, 0.1]: {pct_in_range:.1f}%")
|
|
print(f" Weights in [-0.05, 0.05]: {np.mean(abs_vals < 0.05) * 100:.1f}%")
|
|
print(f" Weights in [-0.01, 0.01]: {np.mean(abs_vals < 0.01) * 100:.1f}%")
|
|
print(f" Max absolute value: {np.max(abs_vals):.6f}")
|
|
print(f" Mean absolute value: {np.mean(abs_vals):.6f}")
|
|
|
|
histogram = np.histogram(simulated_weights, bins=20)
|
|
print(f"\n Weight histogram:")
|
|
max_count = max(histogram[0])
|
|
for i in range(len(histogram[0])):
|
|
bar_len = int(histogram[0][i] / max_count * 40)
|
|
lo = histogram[1][i]
|
|
hi = histogram[1][i + 1]
|
|
print(f" [{lo:>7.4f}, {hi:>7.4f}] {'#' * bar_len} ({histogram[0][i]})")
|
|
|
|
print("\n\n" + "=" * 70)
|
|
print("DONE")
|
|
print("=" * 70)
|