wehub-resource-sync
f99010fae1
Desktop Artifacts / Desktop Build (Linux) (push) Waiting to run
Desktop Artifacts / Desktop Build (Windows) (push) Waiting to run
Desktop Artifacts / Desktop Build (Linux (arm64)) (push) Waiting to run
Desktop Artifacts (macOS) / Desktop Build (macOS (aarch64)) (push) Waiting to run
Desktop Artifacts (macOS) / Desktop Build (macOS (x86_64)) (push) Waiting to run
CI / lint (push) Failing after 1s
CI / frontend (push) Failing after 1s
CI / scripts (push) Failing after 1s
CI / Go Test (ubuntu-latest) (push) Failing after 0s
CI / frontend-node-25 (push) Failing after 1s
CI / docs (push) Failing after 0s
CI / coverage (push) Failing after 0s
CI / e2e (push) Failing after 0s
Docker / build-and-push (push) Failing after 1s
CI / integration (push) Failing after 4m43s
CI / Go Test (windows-latest) (push) Has been cancelled
CI / Desktop Unit Tests (Windows) (push) Has been cancelled
459 lines
13 KiB
Go
459 lines
13 KiB
Go
package parser
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestJSONLSourceSetDiscoverRecursiveStableSources(t *testing.T) {
|
|
root := t.TempDir()
|
|
writeSourceFile(t, filepath.Join(root, "b.jsonl"), "{}\n")
|
|
writeSourceFile(t, filepath.Join(root, "a.jsonl"), "{}\n")
|
|
writeSourceFile(t, filepath.Join(root, "nested", "c.jsonl"), "{}\n")
|
|
writeSourceFile(t, filepath.Join(root, "nested", "ignored.txt"), "{}\n")
|
|
writeSourceFile(t, filepath.Join(root, "nested", "upper.JSONL"), "{}\n")
|
|
|
|
roots := []string{root}
|
|
sources := NewJSONLSourceSet(AgentCodex, roots,
|
|
WithRecursive(),
|
|
WithKey(func(root, path string) string {
|
|
return mustRelSlash(t, root, path)
|
|
}),
|
|
WithProjectHint(func(root, path string) string {
|
|
rel := mustRelSlash(t, root, filepath.Dir(path))
|
|
if rel == "." {
|
|
return ""
|
|
}
|
|
return rel
|
|
}),
|
|
)
|
|
roots[0] = filepath.Join(root, "mutated")
|
|
|
|
discovered, err := sources.Discover(context.Background())
|
|
require.NoError(t, err)
|
|
require.Len(t, discovered, 3)
|
|
|
|
assert.Equal(t, []string{
|
|
"a.jsonl",
|
|
"b.jsonl",
|
|
"nested/c.jsonl",
|
|
}, sourceKeys(discovered))
|
|
assert.Equal(t, []string{"", "", "nested"}, sourceProjects(discovered))
|
|
for _, source := range discovered {
|
|
assert.Equal(t, AgentCodex, source.Provider)
|
|
assert.Equal(t, source.DisplayPath, source.FingerprintKey)
|
|
assert.NotEmpty(t, source.DisplayPath)
|
|
assert.IsType(t, JSONLSource{}, source.Opaque)
|
|
}
|
|
}
|
|
|
|
func TestJSONLSourceSetShallowDiscoveryAndFilters(t *testing.T) {
|
|
root := t.TempDir()
|
|
writeSourceFile(t, filepath.Join(root, "keep.jsonl"), "{}\n")
|
|
writeSourceFile(t, filepath.Join(root, "keep.ndjson"), "{}\n")
|
|
writeSourceFile(t, filepath.Join(root, "drop.jsonl"), "{}\n")
|
|
writeSourceFile(t, filepath.Join(root, "nested", "skip.jsonl"), "{}\n")
|
|
|
|
sources := NewJSONLSourceSet(AgentGptme, []string{root},
|
|
WithExtensions(".jsonl", ".ndjson"),
|
|
WithInclude(func(path string, _ os.FileInfo) bool {
|
|
return filepath.Base(path) != "drop.jsonl"
|
|
}),
|
|
)
|
|
|
|
discovered, err := sources.Discover(context.Background())
|
|
require.NoError(t, err)
|
|
|
|
assert.Equal(t, []string{
|
|
filepath.Join(root, "keep.jsonl"),
|
|
filepath.Join(root, "keep.ndjson"),
|
|
}, sourceDisplayPaths(discovered))
|
|
}
|
|
|
|
func TestJSONLSourceSetWatchChangedPathFindAndFingerprint(t *testing.T) {
|
|
root := t.TempDir()
|
|
path := filepath.Join(root, "nested", "session-1.jsonl")
|
|
content := "{\"role\":\"user\"}\n"
|
|
writeSourceFile(t, path, content)
|
|
writeSourceFile(t, filepath.Join(root, "nested", "notes.txt"), "{}\n")
|
|
|
|
sources := NewJSONLSourceSet(AgentCodex, []string{root},
|
|
WithRecursive(),
|
|
WithContentHashing(),
|
|
)
|
|
|
|
plan, err := sources.WatchPlan(context.Background())
|
|
require.NoError(t, err)
|
|
require.Len(t, plan.Roots, 1)
|
|
assert.Equal(t, root, plan.Roots[0].Path)
|
|
assert.True(t, plan.Roots[0].Recursive)
|
|
assert.Equal(t, []string{"*.jsonl"}, plan.Roots[0].IncludeGlobs)
|
|
assert.NotEmpty(t, plan.Roots[0].DebounceKey)
|
|
|
|
changed, err := sources.SourcesForChangedPath(
|
|
context.Background(),
|
|
ChangedPathRequest{Path: path, EventKind: "write", WatchRoot: root},
|
|
)
|
|
require.NoError(t, err)
|
|
require.Len(t, changed, 1)
|
|
assert.Equal(t, path, changed[0].Key)
|
|
assert.Equal(t, path, changed[0].DisplayPath)
|
|
assert.Equal(t, path, changed[0].FingerprintKey)
|
|
|
|
ignored, err := sources.SourcesForChangedPath(
|
|
context.Background(),
|
|
ChangedPathRequest{
|
|
Path: filepath.Join(root, "nested", "notes.txt"),
|
|
EventKind: "write",
|
|
WatchRoot: root,
|
|
},
|
|
)
|
|
require.NoError(t, err)
|
|
assert.Empty(t, ignored)
|
|
|
|
outside, err := sources.SourcesForChangedPath(
|
|
context.Background(),
|
|
ChangedPathRequest{
|
|
Path: filepath.Join(t.TempDir(), "session-1.jsonl"),
|
|
EventKind: "write",
|
|
WatchRoot: root,
|
|
},
|
|
)
|
|
require.NoError(t, err)
|
|
assert.Empty(t, outside)
|
|
|
|
found, ok, err := sources.FindSource(context.Background(), FindSourceRequest{
|
|
StoredFilePath: path,
|
|
})
|
|
require.NoError(t, err)
|
|
require.True(t, ok)
|
|
assert.Equal(t, path, found.DisplayPath)
|
|
|
|
foundByID, ok, err := sources.FindSource(context.Background(), FindSourceRequest{
|
|
RawSessionID: "session-1",
|
|
})
|
|
require.NoError(t, err)
|
|
require.True(t, ok)
|
|
assert.Equal(t, found.DisplayPath, foundByID.DisplayPath)
|
|
|
|
withoutOpaque := found
|
|
withoutOpaque.Opaque = nil
|
|
fingerprint, err := sources.Fingerprint(context.Background(), withoutOpaque)
|
|
require.NoError(t, err)
|
|
|
|
info, err := os.Stat(path)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, path, fingerprint.Key)
|
|
assert.Equal(t, info.Size(), fingerprint.Size)
|
|
assert.Equal(t, info.ModTime().UnixNano(), fingerprint.MTimeNS)
|
|
assert.Equal(t, fmt.Sprintf("%x", sha256.Sum256([]byte(content))), fingerprint.Hash)
|
|
}
|
|
|
|
func TestJSONLSourceSetChangedPathClassifiesDeletedFiles(t *testing.T) {
|
|
root := t.TempDir()
|
|
path := filepath.Join(root, "nested", "deleted.jsonl")
|
|
sources := NewJSONLSourceSet(AgentCodex, []string{root},
|
|
WithRecursive(),
|
|
)
|
|
|
|
changed, err := sources.SourcesForChangedPath(
|
|
context.Background(),
|
|
ChangedPathRequest{Path: path, EventKind: "remove", WatchRoot: root},
|
|
)
|
|
require.NoError(t, err)
|
|
require.Len(t, changed, 1)
|
|
assert.Equal(t, path, changed[0].Key)
|
|
assert.Equal(t, path, changed[0].DisplayPath)
|
|
assert.Equal(t, path, changed[0].FingerprintKey)
|
|
assert.Equal(t, "nested/deleted.jsonl", changed[0].Opaque.(JSONLSource).RelPath)
|
|
|
|
shallowPath := filepath.Join(root, "nested", "ignored.jsonl")
|
|
shallowSources := NewJSONLSourceSet(AgentCodex, []string{root})
|
|
changed, err = shallowSources.SourcesForChangedPath(
|
|
context.Background(),
|
|
ChangedPathRequest{Path: shallowPath, EventKind: "remove", WatchRoot: root},
|
|
)
|
|
require.NoError(t, err)
|
|
assert.Empty(t, changed)
|
|
}
|
|
|
|
func TestJSONLSourceSetChangedPathRejectsExistingNonRegularPath(t *testing.T) {
|
|
root := t.TempDir()
|
|
path := filepath.Join(root, "nested", "not-a-source.jsonl")
|
|
require.NoError(t, os.MkdirAll(path, 0o755))
|
|
|
|
sources := NewJSONLSourceSet(AgentCodex, []string{root},
|
|
WithRecursive(),
|
|
)
|
|
|
|
changed, err := sources.SourcesForChangedPath(
|
|
context.Background(),
|
|
ChangedPathRequest{Path: path, EventKind: "write", WatchRoot: root},
|
|
)
|
|
require.NoError(t, err)
|
|
assert.Empty(t, changed)
|
|
}
|
|
|
|
func TestJSONLSourceSetChangedPathUsesPathOnlyFilterForDeletedFiles(t *testing.T) {
|
|
root := t.TempDir()
|
|
sources := NewJSONLSourceSet(AgentCodex, []string{root},
|
|
WithRecursive(),
|
|
WithIncludePath(func(root, path string) bool {
|
|
return filepath.Base(path) == "events.jsonl"
|
|
}),
|
|
)
|
|
|
|
ignored, err := sources.SourcesForChangedPath(
|
|
context.Background(),
|
|
ChangedPathRequest{
|
|
Path: filepath.Join(root, "session", "notes.jsonl"),
|
|
EventKind: "remove",
|
|
WatchRoot: root,
|
|
},
|
|
)
|
|
require.NoError(t, err)
|
|
assert.Empty(t, ignored)
|
|
|
|
changed, err := sources.SourcesForChangedPath(
|
|
context.Background(),
|
|
ChangedPathRequest{
|
|
Path: filepath.Join(root, "session", "events.jsonl"),
|
|
EventKind: "remove",
|
|
WatchRoot: root,
|
|
},
|
|
)
|
|
require.NoError(t, err)
|
|
require.Len(t, changed, 1)
|
|
assert.Equal(t, filepath.Join(root, "session", "events.jsonl"), changed[0].DisplayPath)
|
|
}
|
|
|
|
func TestJSONLSourceSetDescendPathPrunesSources(t *testing.T) {
|
|
root := t.TempDir()
|
|
keepPath := filepath.Join(root, "keep", "session.jsonl")
|
|
skipPath := filepath.Join(root, "skip", "session.jsonl")
|
|
writeSourceFile(t, keepPath, "{}\n")
|
|
writeSourceFile(t, skipPath, "{}\n")
|
|
|
|
sources := NewJSONLSourceSet(AgentCodex, []string{root},
|
|
WithRecursive(),
|
|
WithDescendPath(func(root, path string) bool {
|
|
return filepath.Base(path) != "skip"
|
|
}),
|
|
)
|
|
|
|
discovered, err := sources.Discover(context.Background())
|
|
require.NoError(t, err)
|
|
require.Len(t, discovered, 1)
|
|
assert.Equal(t, keepPath, discovered[0].DisplayPath)
|
|
|
|
changed, err := sources.SourcesForChangedPath(
|
|
context.Background(),
|
|
ChangedPathRequest{Path: skipPath, EventKind: "write", WatchRoot: root},
|
|
)
|
|
require.NoError(t, err)
|
|
assert.Empty(t, changed)
|
|
|
|
removed, err := sources.SourcesForChangedPath(
|
|
context.Background(),
|
|
ChangedPathRequest{
|
|
Path: filepath.Join(root, "skip", "removed.jsonl"),
|
|
EventKind: "remove",
|
|
WatchRoot: root,
|
|
},
|
|
)
|
|
require.NoError(t, err)
|
|
assert.Empty(t, removed)
|
|
}
|
|
|
|
func TestJSONLSourceSetDuplicateKeysKeepFirstConfiguredRoot(t *testing.T) {
|
|
firstRoot := t.TempDir()
|
|
secondRoot := t.TempDir()
|
|
firstPath := filepath.Join(firstRoot, "session.jsonl")
|
|
secondPath := filepath.Join(secondRoot, "session.jsonl")
|
|
writeSourceFile(t, firstPath, "{}\n")
|
|
writeSourceFile(t, secondPath, "{}\n")
|
|
|
|
sources := NewJSONLSourceSet(AgentCodex, []string{firstRoot, secondRoot},
|
|
WithKey(func(_, path string) string {
|
|
return filepath.Base(path)
|
|
}),
|
|
)
|
|
|
|
discovered, err := sources.Discover(context.Background())
|
|
require.NoError(t, err)
|
|
require.Len(t, discovered, 1)
|
|
assert.Equal(t, firstPath, discovered[0].DisplayPath)
|
|
|
|
found, ok, err := sources.FindSource(
|
|
context.Background(),
|
|
FindSourceRequest{StoredFilePath: secondPath},
|
|
)
|
|
require.NoError(t, err)
|
|
require.True(t, ok)
|
|
assert.Equal(t, firstPath, found.DisplayPath)
|
|
|
|
changed, err := sources.SourcesForChangedPath(
|
|
context.Background(),
|
|
ChangedPathRequest{
|
|
Path: secondPath,
|
|
EventKind: "write",
|
|
WatchRoot: secondRoot,
|
|
},
|
|
)
|
|
require.NoError(t, err)
|
|
assert.Empty(t, changed)
|
|
}
|
|
|
|
func TestJSONLSourceSetFindSourceNormalizesRawSessionID(t *testing.T) {
|
|
root := t.TempDir()
|
|
path := filepath.Join(root, "session-1.jsonl")
|
|
writeSourceFile(t, path, "{}\n")
|
|
|
|
// LookupIDValid rejects the raw, un-normalized form, so a lookup only
|
|
// succeeds when RawSessionIDForLookup runs before the validity gate and
|
|
// before the SessionIDFromPath comparison in the discovery loop. The
|
|
// on-disk session ID is "session-1" (base name without extension), which
|
|
// the raw "raw:session-1" only matches once normalized.
|
|
rejectsRaw := func(rawID string) bool {
|
|
return rawID != "" && !strings.HasPrefix(rawID, "raw:")
|
|
}
|
|
|
|
normalizing := NewJSONLSourceSet(AgentCodex, []string{root},
|
|
WithRawSessionIDForLookup(func(rawID string) string {
|
|
return strings.TrimPrefix(rawID, "raw:")
|
|
}),
|
|
WithLookupIDValid(rejectsRaw),
|
|
)
|
|
|
|
found, ok, err := normalizing.FindSource(
|
|
context.Background(),
|
|
FindSourceRequest{RawSessionID: "raw:session-1"},
|
|
)
|
|
require.NoError(t, err)
|
|
require.True(t, ok, "normalized raw session ID must resolve its source")
|
|
assert.Equal(t, path, found.DisplayPath)
|
|
|
|
// Without the normalizer the identical request is gated out: the raw form
|
|
// fails LookupIDValid and never matches the on-disk session ID. This locks
|
|
// in that the normalization step is what enables both checks.
|
|
unnormalized := NewJSONLSourceSet(AgentCodex, []string{root},
|
|
WithLookupIDValid(rejectsRaw),
|
|
)
|
|
|
|
_, ok, err = unnormalized.FindSource(
|
|
context.Background(),
|
|
FindSourceRequest{RawSessionID: "raw:session-1"},
|
|
)
|
|
require.NoError(t, err)
|
|
assert.False(t, ok, "un-normalized raw session ID must not resolve")
|
|
}
|
|
|
|
func TestJSONLSourceSetMissingRootAndInvalidLookupAreNoops(t *testing.T) {
|
|
root := t.TempDir()
|
|
sources := NewJSONLSourceSet(AgentCodex, []string{
|
|
filepath.Join(root, "missing"),
|
|
})
|
|
|
|
discovered, err := sources.Discover(context.Background())
|
|
require.NoError(t, err)
|
|
assert.Empty(t, discovered)
|
|
|
|
found, ok, err := sources.FindSource(context.Background(), FindSourceRequest{
|
|
RawSessionID: "../session",
|
|
})
|
|
require.NoError(t, err)
|
|
assert.False(t, ok)
|
|
assert.Empty(t, found)
|
|
}
|
|
|
|
func writeSourceFile(t *testing.T, path, content string) {
|
|
t.Helper()
|
|
|
|
require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755))
|
|
require.NoError(t, os.WriteFile(path, []byte(content), 0o644))
|
|
}
|
|
|
|
func mustRelSlash(t *testing.T, root, path string) string {
|
|
t.Helper()
|
|
|
|
rel, err := filepath.Rel(root, path)
|
|
require.NoError(t, err)
|
|
return filepath.ToSlash(rel)
|
|
}
|
|
|
|
func sourceKeys(sources []SourceRef) []string {
|
|
keys := make([]string, 0, len(sources))
|
|
for _, source := range sources {
|
|
keys = append(keys, source.Key)
|
|
}
|
|
return keys
|
|
}
|
|
|
|
func sourceProjects(sources []SourceRef) []string {
|
|
projects := make([]string, 0, len(sources))
|
|
for _, source := range sources {
|
|
projects = append(projects, source.ProjectHint)
|
|
}
|
|
return projects
|
|
}
|
|
|
|
func sourceDisplayPaths(sources []SourceRef) []string {
|
|
paths := make([]string, 0, len(sources))
|
|
for _, source := range sources {
|
|
paths = append(paths, source.DisplayPath)
|
|
}
|
|
return paths
|
|
}
|
|
|
|
func TestCleanJSONLRootsPreservesS3Scheme(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
roots []string
|
|
want []string
|
|
}{
|
|
{
|
|
name: "s3 root kept verbatim",
|
|
roots: []string{"s3://bucket/laptop/raw/claude"},
|
|
want: []string{"s3://bucket/laptop/raw/claude"},
|
|
},
|
|
{
|
|
name: "s3 root with trailing slash kept verbatim",
|
|
roots: []string{"s3://bucket/laptop/raw/claude/"},
|
|
want: []string{"s3://bucket/laptop/raw/claude/"},
|
|
},
|
|
{
|
|
name: "local roots still cleaned",
|
|
roots: []string{"/tmp/foo/../bar", "/tmp/baz/"},
|
|
// filepath.Clean yields OS-native separators (backslashes on
|
|
// Windows), so build the expectation with FromSlash rather than
|
|
// hard-coding forward slashes.
|
|
want: []string{filepath.FromSlash("/tmp/bar"), filepath.FromSlash("/tmp/baz")},
|
|
},
|
|
{
|
|
name: "empty roots dropped",
|
|
roots: []string{"", "s3://bucket/x", ""},
|
|
want: []string{"s3://bucket/x"},
|
|
},
|
|
{
|
|
name: "mixed local and s3",
|
|
roots: []string{"/tmp/a/./b", "s3://bucket/y/"},
|
|
want: []string{filepath.FromSlash("/tmp/a/b"), "s3://bucket/y/"},
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
// filepath.Clean would collapse "s3://" to "s3:/", which defeats the
|
|
// HasPrefix("s3://") checks that route discovery to the object store.
|
|
assert.Equal(t, tt.want, cleanJSONLRoots(tt.roots))
|
|
})
|
|
}
|
|
}
|