wehub-resource-sync
a06f331eb8
CI / benchmark (push) Has been skipped
install-script / posix-syntax (push) Successful in 6m1s
CI / build-onnx (push) Failing after 6m43s
init-smoke / dry-run (push) Failing after 15m57s
security / govulncheck (push) Has been cancelled
security / trivy-fs (push) Has been cancelled
CI / test (1.26, ubuntu-latest) (push) Has been cancelled
Scorecard supply-chain security / Scorecard analysis (push) Has been cancelled
CI / test (1.26, macos-latest) (push) Has been cancelled
CI / build-windows (push) Has been cancelled
CI / lint (push) Has been cancelled
install-script / powershell-syntax (push) Has been cancelled
install-script / install (macos-14) (push) Has been cancelled
install-script / install (ubuntu-latest) (push) Has been cancelled
95 lines
2.8 KiB
Go
95 lines
2.8 KiB
Go
package tokens
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestDiskCache_RoundTrip(t *testing.T) {
|
|
dir := t.TempDir()
|
|
content := strings.Repeat("func foo() int { return bar() }\n", 300)
|
|
want := Count(content)
|
|
|
|
c := NewDiskCache(dir)
|
|
if got := c.Count(content); got != want { // miss: compute + store
|
|
t.Fatalf("first Count = %d, want %d", got, want)
|
|
}
|
|
|
|
// A fresh cache over the same directory must read the persisted
|
|
// entry rather than recompute — the cache survives a restart.
|
|
c2 := NewDiskCache(dir)
|
|
if got := c2.Count(content); got != want {
|
|
t.Errorf("persisted Count = %d, want %d", got, want)
|
|
}
|
|
}
|
|
|
|
func TestDiskCache_HitReturnsStored(t *testing.T) {
|
|
// A hit returns the stored value verbatim: pre-seed a deliberately
|
|
// wrong count and confirm Count trusts the entry over recomputing.
|
|
dir := t.TempDir()
|
|
c := NewDiskCache(dir)
|
|
content := "content the cache will answer for"
|
|
c.write(c.key(content), 999999)
|
|
if got := c.Count(content); got != 999999 {
|
|
t.Errorf("Count = %d, want the stored 999999 (cache not consulted)", got)
|
|
}
|
|
}
|
|
|
|
func TestDiskCache_RevisionIsolation(t *testing.T) {
|
|
dir := t.TempDir()
|
|
content := "content shared across tokenizer revisions"
|
|
|
|
old := &DiskCache{dir: dir, revision: "0/old"}
|
|
old.write(old.key(content), 4242)
|
|
|
|
// A cache on a different revision must not see the old entry —
|
|
// counts from a different tokenizer are never trusted.
|
|
cur := &DiskCache{dir: dir, revision: "1/cl100k_base"}
|
|
if _, ok := cur.read(cur.key(content)); ok {
|
|
t.Error("revision change must make old entries unreachable")
|
|
}
|
|
// The same revision still hits.
|
|
if n, ok := old.read(old.key(content)); !ok || n != 4242 {
|
|
t.Errorf("same-revision read = (%d,%v), want (4242,true)", n, ok)
|
|
}
|
|
}
|
|
|
|
func TestCachedCount(t *testing.T) {
|
|
xdg := t.TempDir()
|
|
t.Setenv("XDG_CACHE_HOME", xdg)
|
|
|
|
small := strings.Repeat("x", minCacheBytes-1)
|
|
large := strings.Repeat("token stream ", 600)
|
|
|
|
// CachedCount is value-identical to Count regardless of size.
|
|
if got, want := CachedCount(small), Count(small); got != want {
|
|
t.Errorf("CachedCount(small) = %d, want %d", got, want)
|
|
}
|
|
if got, want := CachedCount(large), Count(large); got != want {
|
|
t.Errorf("CachedCount(large) = %d, want %d", got, want)
|
|
}
|
|
|
|
// Only the large input crossed the threshold and was persisted.
|
|
cacheRoot := filepath.Join(xdg, "gortex", "token-counts")
|
|
entries := countCacheEntries(t, cacheRoot)
|
|
if entries == 0 {
|
|
t.Error("a large input should have produced a cache entry")
|
|
}
|
|
}
|
|
|
|
// countCacheEntries counts regular files under root (zero when root is
|
|
// absent), used to assert the small-input fast path skips the disk.
|
|
func countCacheEntries(t *testing.T, root string) int {
|
|
t.Helper()
|
|
n := 0
|
|
_ = filepath.Walk(root, func(_ string, info os.FileInfo, err error) error {
|
|
if err == nil && info != nil && info.Mode().IsRegular() {
|
|
n++
|
|
}
|
|
return nil
|
|
})
|
|
return n
|
|
}
|