kenn-io--agentsview/internal/secrets/rules_test.go

package secrets

import (
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestDefiniteRules(t *testing.T) {
	cases := []struct {
		name string
		rule string
		text string
		want bool
	}{
		{"github classic", "github-pat",
			"tok ghp_8Hk3Wn7Dz4Rp2Vx9Mb6Tj0Qc5Lm1Yp8Bv4Hg x", true},
		{"github fine-grained", "github-pat",
			"github_pat_8Hk3Wn7Dz4Rp2Vx9Mb6Tj0Qc5Lm1Yp8Bv4HgN_X2cWp9", true},
		{"slack bot", "slack-token",
			"xoxb-549271836401-fHk7Bm3Pz9Wt5Vx2Yq8Nc", true},
		{"stripe live", "stripe-secret",
			"sk_live_7Qh3Wn8Dk4Rp9Vx2Mb6Tj0Qc5Lm", true},
		{"google api", "google-api-key",
			"AIza7Qh3Wn8Dk4Rp9Vx2Mb6Tj0Qc5Lm1Yp8Bv4H", true},
		{"google api ending dash", "google-api-key",
			"key AIza7Qh3Wn8Dk4Rp9Vx2Mb6Tj0Qc5Lm1Yp8Bv4- end", true},
		{"pem block", "private-key-block",
			"-----BEGIN RSA PRIVATE KEY-----\n" +
				rep("MIIBSECRETKEYMATERIAL0123456789ABCDEF\n", 5) +
				"-----END RSA PRIVATE KEY-----", true},
		{"plain prose", "", "the quick brown fox jumps over", false},
		{"short ghp", "", "ghp_tooShort", false},
		{"openai project", "openai-key",
			"tok sk-proj-Xa9Kd03Lm5Qp7Rt2Vw8Zb4Nc6Hf1Yp end", true},
		{"openai svcacct", "openai-key",
			"tok sk-svcacct-Xa9Kd03Lm5Qp7Rt2Vw8Zb4Nc6Hf1Yp end", true},
		{"openai admin", "openai-key",
			"tok sk-admin-Xa9Kd03Lm5Qp7Rt2Vw8Zb4Nc6Hf1Yp end", true},
		{"openai ending dash", "openai-key",
			"tok sk-proj-Xa9Kd03Lm5Qp7Rt2Vw8Zb4Nc6Hf1Y- end", true},
		{"openai ending underscore", "openai-key",
			"tok sk-proj-Xa9Kd03Lm5Qp7Rt2Vw8Zb4Nc6Hf1Y_ end", true},
		{"openai at eot", "openai-key",
			"tok sk-proj-Xa9Kd03Lm5Qp7Rt2Vw8Zb4Nc6Hf1Yp", true},
		{"openai trailing-run placeholder", "",
			"tok sk-proj-AAAAAAAAAAAAAAAAAAAAA end", false},
		{"openai low-entropy body", "",
			"tok sk-proj-abababababababababab end", false},
		{"openai short body", "",
			"tok sk-proj-short end", false},
		{"openai bare sk- not matched", "",
			"tok sk-Xa9Kd03Lm5Qp7Rt2Vw8Zb4Nc6Hf1YpA end", false},
		{"gitlab pat", "gitlab-pat",
			"tok glpat-Xa9Kd03Lm5Qp7Rt2Vw8Zb end", true},
		{"gitlab pat ending dash", "gitlab-pat",
			"tok glpat-Xa9Kd03Lm5Qp7Rt2Vw8Z- end", true},
		{"gitlab pat at eot", "gitlab-pat",
			"tok glpat-Xa9Kd03Lm5Qp7Rt2Vw8Zb", true},
		{"gitlab placeholder repeating", "",
			"tok glpat-AAAAAAAAAAAAAAAAAAAA end", false},
		{"gitlab placeholder monotone", "",
			"tok glpat-abcdefghijklmnopqrst end", false},
		{"gitlab short body", "",
			"tok glpat-short end", false},
		{"gitlab trailing-run placeholder", "",
			"tok glpat-Xa9Kd03Lm5Qp7Rt2AAAA end", false},
		{"npm token", "npm-token",
			"tok npm_Xa9Kd03Lm5Qp7Rt2Vw8Zb4Nc6Hf1YpAbCdEf end", true},
		{"npm placeholder repeating", "",
			"tok npm_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA end", false},
		{"npm legacy bare hex not matched", "",
			"tok 0123456789abcdef0123456789abcdef0123 end", false},
		{"npm short body", "",
			"tok npm_short end", false},
		{"npm trailing-run placeholder", "",
			"tok npm_Xa9Kd03Lm5Qp7Rt2Vw8Zb4Nc6Hf1YpAbAAAA end", false},
		{"pypi token 69 tail matches", "pypi-token",
			"tok pypi-AgEIcHlwaS5vcmcCXa9Kd03Lm5Qp7Rt2Vw8Zb4Nc6Hf1Yp8Bv4HgNX2cWpQz7MjRv3Ts6Ku0Fy9DnEwPaRbTZ end", true},
		{"pypi token 68 tail rejected", "",
			"tok pypi-AgEIcHlwaS5vcmcCXa9Kd03Lm5Qp7Rt2Vw8Zb4Nc6Hf1Yp8Bv4HgNX2cWpQz7MjRv3Ts6Ku0Fy9DnEwPaRbT end", false},
		{"pypi token ending dash 69 tail", "pypi-token",
			"tok pypi-AgEIcHlwaS5vcmcCXa9Kd03Lm5Qp7Rt2Vw8Zb4Nc6Hf1Yp8Bv4HgNX2cWpQz7MjRv3Ts6Ku0Fy9DnEwPaRbT- end", true},
		{"pypi placeholder repeating", "",
			"tok pypi-AgEIcHlwaS5vcmcC" + rep("AAAAAAAAA", 8) + " end", false},
		{"pypi trailing-run placeholder", "",
			"tok pypi-AgEIcHlwaS5vcmcCXa9Kd03Lm5Qp7Rt2Vw8Zb4Nc6Hf1Yp8Bv4HgNX2cWpQz7MjRv3Ts6Ku0Fy9DnEwPAAAAA end", false},
		{"hf token", "huggingface-token",
			"tok hf_Xa9Kd03Lm5Qp7Rt2Vw8Zb4Nc6Hf1YpAbCd end", true},
		{"hf token longer", "huggingface-token",
			"tok hf_Xa9Kd03Lm5Qp7Rt2Vw8Zb4Nc6Hf1YpAbCdEfGhIj end", true},
		{"hf token at eot", "huggingface-token",
			"tok hf_Xa9Kd03Lm5Qp7Rt2Vw8Zb4Nc6Hf1YpAbCd", true},
		{"hf placeholder repeating", "",
			"tok hf_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA end", false},
		{"hf placeholder monotone", "",
			"tok hf_abcdefghijklmnopqrstuvwxyzABCD end", false},
		{"hf short body", "",
			"tok hf_short end", false},
		{"hf trailing-run placeholder", "",
			"tok hf_Xa9Kd03Lm5Qp7Rt2Vw8Zb4Nc6HAAAA end", false},
		// sendgrid-key: format SG.<22 base64url>.<43 base64url>
		// ident22 = "Xa9Kd03Lm5Qp7Rt2Vw8Zb4" (22 chars)
		// secret43 = "yifDLkDmWJ6UuVTAIjvFu7WICPhDeOZIiBOB-Y6sHrF" (43 chars)
		{"sendgrid key", "sendgrid-key",
			"tok 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 end", true},
		{"sendgrid key ending dash", "sendgrid-key",
			"tok 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 end", true},
		{"sendgrid key at eot", "sendgrid-key",
			"tok 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", true},
		{"sendgrid key trailing dot stops at dot", "sendgrid-key",
			"tok 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. end", true},
		// negative cases
		{"sendgrid missing separator", "",
			"tok SG.Xa9Kd03Lm5Qp7Rt2Vw8Zb4yifDLkDmWJ6UuVTAIjvFu7WICPhDeOZIiBOB-Y6sHrF end", false},
		// wrong ident = 20 chars ("Xa9Kd03Lm5Qp7Rt2Vw8Z", drop last 2 from ident22)
		{"sendgrid wrong identifier length", "",
			"tok SG.Xa9Kd03Lm5Qp7Rt2Vw8Z.yifDLkDmWJ6UuVTAIjvFu7WICPhDeOZIiBOB-Y6sHrF end", false},
		// wrong secret = 44 chars (append "S" to secret43)
		{"sendgrid wrong secret length", "",
			"tok SG.Xa9Kd03Lm5Qp7Rt2Vw8Zb4.yifDLkDmWJ6UuVTAIjvFu7WICPhDeOZIiBOB-Y6sHrFS end", false},
		{"sendgrid placeholder repeating", "",
			"tok SG.AAAAAAAAAAAAAAAAAAAAAA.BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB end", false},
		// trailing-run secret = first 39 of secret43 + "AAAA" = 43 chars
		// secret43[:39]="yifDLkDmWJ6UuVTAIjvFu7WICPhDeOZIiBOB-Y6" + "AAAA"
		{"sendgrid trailing-run placeholder", "",
			"tok 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 end", false},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			got := Scan(c.text)
			found := ""
			for _, m := range got {
				if m.Rule == c.rule {
					found = m.Rule
				}
			}
			if c.want {
				assert.NotEmpty(t, found,
					"expected rule %q to match %q; got %+v", c.rule, c.text, got)
			} else {
				assert.Empty(t, got,
					"expected no match for %q; got %+v", c.text, got)
			}
		})
	}
}

func TestCandidateRules(t *testing.T) {
	jwt := "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.dumm_Sig-Value12345"
	cases := []struct {
		name string
		rule string
		text string
		want bool
	}{
		{"jwt", "jwt", "auth: " + jwt, true},
		{"high entropy assignment", "high-entropy-assignment",
			"SECRET=Xa9Kd03Lm5Qp7Rt2Vw8Zb4Nc6", true},
		{"low entropy assignment", "high-entropy-assignment",
			"NAME=aaaaaaaaaaaaaaaaaaaa", false},
		{"short assignment", "high-entropy-assignment", "X=ab12", false},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			got := Scan(c.text)
			found := false
			for _, m := range got {
				if m.Rule == c.rule {
					found = true
					assert.Equal(t, ConfidenceCandidate, m.Confidence,
						"%s confidence", c.rule)
				}
			}
			assert.Equal(t, c.want, found,
				"rule %q match=%v want=%v for %q (got %+v)",
				c.rule, found, c.want, c.text, got)
		})
	}
}

// TestScanDefiniteReturnsOnlyDefinite confirms the inline-sync scan path
// reports definite vendor formats and skips the FP-prone candidate heuristics
// (high-entropy assignments, JWTs, basic-auth URLs) entirely.
func TestScanDefiniteReturnsOnlyDefinite(t *testing.T) {
	// One definite AWS key and one candidate high-entropy assignment.
	text := "aws AKIA7QHWN2DKR4FYPLJM and SECRET=Xa9Kd03Lm5Qp7Rt2Vw8Zb4Nc6"
	full := Scan(text)
	require.Len(t, full, 2,
		"precondition: Scan should report 2 matches (1 definite, 1 candidate)")
	got := ScanDefinite(text)
	require.Len(t, got, 1)
	assert.Equal(t, "aws-access-key", got[0].Rule)
	for _, m := range got {
		assert.Equal(t, ConfidenceDefinite, m.Confidence,
			"ScanDefinite returned non-definite match: %+v", m)
	}
}

// TestScanDefiniteMatchesScanDefiniteSubset confirms ScanDefinite yields the
// same spans (rule, offsets, redaction) that Scan reports for definite rules,
// so findings stored by the inline path and the full scan stay consistent.
func TestScanDefiniteMatchesScanDefiniteSubset(t *testing.T) {
	text := "key AKIA7QHWN2DKR4FYPLJM tok ghp_8Hk3Wn7Dz4Rp2Vx9Mb6Tj0Qc5Lm1Yp8Bv4Hg" +
		" SECRET=Xa9Kd03Lm5Qp7Rt2Vw8Zb4Nc6"
	var wantDef []Match
	for _, m := range Scan(text) {
		if m.Confidence == ConfidenceDefinite {
			wantDef = append(wantDef, m)
		}
	}
	got := ScanDefinite(text)
	require.Len(t, got, len(wantDef),
		"ScanDefinite count vs Scan definite count (%+v vs %+v)", got, wantDef)
	for i := range got {
		assert.Equal(t, wantDef[i].Rule, got[i].Rule, "match %d rule differs", i)
		assert.Equal(t, wantDef[i].Start, got[i].Start, "match %d start differs", i)
		assert.Equal(t, wantDef[i].End, got[i].End, "match %d end differs", i)
		assert.Equal(t, wantDef[i].Redacted, got[i].Redacted, "match %d redacted differs", i)
	}
}

// TestDefiniteRulesVersionDistinctFromFull pins the split-versioning contract:
// the inline definite-only scan stamps a version that differs from the full
// ruleset version, so secrets scan --backfill (which treats RulesVersion as
// current) re-scans inline-only sessions to pick up candidate findings.
func TestDefiniteRulesVersionDistinctFromFull(t *testing.T) {
	def := DefiniteRulesVersion()
	full := RulesVersion()
	require.NotEqual(t, full, def,
		"DefiniteRulesVersion must differ from RulesVersion (both %q)", def)
	require.NotEmpty(t, def, "versions must be non-empty")
	require.NotEmpty(t, full, "versions must be non-empty")
	assert.Equal(t, def, DefiniteRulesVersion(), "DefiniteRulesVersion not stable across calls")
	assert.Len(t, def, 64, "DefiniteRulesVersion length: %q", def)
	for _, c := range def {
		require.True(t, isLowerHex(c),
			"DefiniteRulesVersion has non-hex char %q in %q", c, def)
	}
}

func TestRulesVersionStableAndHex(t *testing.T) {
	v1 := RulesVersion()
	v2 := RulesVersion()
	require.Equal(t, v1, v2, "RulesVersion not stable")
	require.Len(t, v1, 64, "RulesVersion length: %q", v1) // sha256 hex
	for _, c := range v1 {
		require.True(t, isLowerHex(c),
			"RulesVersion has non-hex char %q in %q", c, v1)
	}
}

func TestVerify(t *testing.T) {
	// Non-grouped rule: the stored span is the full regex match.
	awsSrc := "export KEY=AKIA7QHWN2DKR4FYPLJM done"
	s := strings.Index(awsSrc, "AKIA")
	e := s + len("AKIA7QHWN2DKR4FYPLJM")
	assert.True(t, Verify("aws-access-key", awsSrc, s, e),
		"Verify should accept a valid AWS key at its coordinates")
	assert.False(t, Verify("aws-access-key", awsSrc, 0, 6),
		"Verify should reject coordinates that are not the key")
	assert.False(t, Verify("nonexistent-rule", awsSrc, s, e),
		"Verify should reject an unknown rule")
	assert.False(t, Verify("aws-access-key", awsSrc, s, len(awsSrc)+10),
		"Verify should reject out-of-bounds coordinates")
	// Grouped rule: the stored span is the captured group (the password),
	// not the full URL match. Verify must still accept it.
	urlSrc := "db=postgres://user:s3cretP4ss@host:5432/db"
	ps := strings.Index(urlSrc, "s3cretP4ss")
	pe := ps + len("s3cretP4ss")
	assert.True(t, Verify("basic-auth-url", urlSrc, ps, pe),
		"Verify should accept a grouped finding at its group coordinates")
}

// TestVerifyDetectsChangedSource locks in the core --reveal guarantee: a scan
// produces coordinates, Verify accepts them on the unchanged source, and
// rejects them once the bytes at those coordinates are no longer the secret.
func TestVerifyDetectsChangedSource(t *testing.T) {
	source := "export AWS=AKIA7QHWN2DKR4FYPLJM"
	// Seed from canonical Scan (what produces findings and what Verify uses).
	matches := Scan(source)
	require.NotEmpty(t, matches, "expected at least one match in source")
	m := matches[0]
	assert.True(t, Verify(m.Rule, source, m.Start, m.End),
		"Verify should accept unchanged source at [%d,%d)", m.Start, m.End)
	// Same length, but the secret at [Start,End) is replaced by a zero-entropy
	// run that matches no rule, so Verify must reject the stale coordinates.
	changed := source[:m.Start] + strings.Repeat("X", m.End-m.Start)
	assert.False(t, Verify(m.Rule, changed, m.Start, m.End),
		"Verify should reject when the source changed at those coords")
}

// TestVerifyRejectsSuppressedCandidate ensures Verify mirrors canonical Scan,
// not raw scanning: a candidate that overlaps a definite is suppressed by Scan,
// so Verify must reject its coordinates even though scanRaw reports it.
func TestVerifyRejectsSuppressedCandidate(t *testing.T) {
	src := "https://user:sk-ant-api03-Xa9Kd03Lm5Qp7Rt2Vw8Zb4@example.com"
	var cand Match
	for _, m := range scanRaw(src) {
		if m.Rule == "basic-auth-url" {
			cand = m
			break
		}
	}
	require.NotEmpty(t, cand.Rule,
		"precondition: scanRaw should report a basic-auth-url candidate")
	assert.False(t, Verify("basic-auth-url", src, cand.Start, cand.End),
		"Verify must reject a candidate that canonical Scan suppresses")
}

// isLowerHex reports whether c is a lowercase hexadecimal digit, the alphabet
// a SHA-256 hex digest is built from.
func isLowerHex(c rune) bool {
	return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')
}

// rep returns s repeated n times (test helper for building token bodies).
func rep(s string, n int) string {
	var out strings.Builder
	for range n {
		out.WriteString(s)
	}
	return out.String()
}

// TestHasRepeatingBlock pins the seed-pattern detector that catches
// placeholders built by repeating a short string. Block size 1 is the
// "ghp_aaaa…" shape; sizes 2..6 cover "A1b2A1b2…", "aB3_xaB3_x…", etc.
func TestHasRepeatingBlock(t *testing.T) {
	cases := []struct {
		name string
		s    string
		want bool
	}{
		{"single byte dominating", strings.Repeat("a", 36), true},
		{"block size 4 A1b2", strings.Repeat("A1b2", 20), true},
		{"block size 4 a1B2", strings.Repeat("a1B2", 8), true},
		{"block size 5 aB3_x", strings.Repeat("aB3_x", 7), true},
		{"block size 2", strings.Repeat("xy", 10), true},
		{"random body", "7Qh3Wn8Dk4Rp9Vx2Mb6Tj0Qc5Lm", false},
		{"random aws body", "7QHWN2DKR4FYPLJM", false},
		{"random pat body", "8Hk3Wn7Dz4Rp2Vx9Mb6Tj0Qc5Lm1Yp8Bv4Hg", false},
		{"too short", "abcd", false},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			assert.Equal(t, c.want, hasRepeatingBlock(c.s))
		})
	}
}

// TestHighEntropyPaddingCapture verifies that the high-entropy-assignment
// rule includes base64 '=' padding in the captured span.
func TestHighEntropyPaddingCapture(t *testing.T) {
	cases := []struct {
		name   string
		text   string
		suffix string // expected last chars of the captured span
	}{
		{"double padding",
			`KEY="Xa9Kd03Lm5Qp7Rt2Vw8Zb4N=="`,
			`==`},
		{"single padding",
			`KEY="Xa9Kd03Lm5Qp7Rt2Vw8Zb4=`,
			`=`},
		{"no padding",
			`KEY="Xa9Kd03Lm5Qp7Rt2Vw8Zb4N"`,
			`N`},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			got := Scan(c.text)
			var m *Match
			for i := range got {
				if got[i].Rule == "high-entropy-assignment" {
					m = &got[i]
					break
				}
			}
			if m == nil {
				t.Fatalf("no high-entropy match in %q; got %+v", c.text, got)
			}
			span := c.text[m.Start:m.End]
			if !strings.HasSuffix(span, c.suffix) {
				t.Errorf("captured span %q does not end with %q",
					span, c.suffix)
			}
		})
	}
}

// TestHighEntropyValueTrimsPaddingBeforeLength verifies that
// highEntropyValue trims trailing '=' before applying the 20-char
// length floor. A 19-char body padded to 21 chars must fail the new
// gate (body length 19 after trim) even though the raw input is 21
// chars — proving the trim runs before the length check.
func TestHighEntropyValueTrimsPaddingBeforeLength(t *testing.T) {
	const body = "A1b2C3d4E5f6G7h8I9j" // 19 chars
	if len(body) != 19 {
		t.Fatalf("test setup: body length = %d, want 19", len(body))
	}
	padded := body + "=="
	if highEntropyValue(padded) {
		t.Errorf("highEntropyValue(%q) = true, want false "+
			"(trimmed body is 19 chars, below the 20-char floor)",
			padded)
	}
}

// TestHighEntropyValueAcceptsPaddedRealBody verifies that the validator
// accepts a real high-entropy body whether or not '=' padding is
// appended.
func TestHighEntropyValueAcceptsPaddedRealBody(t *testing.T) {
	const body = "Xa9Kd03Lm5Qp7Rt2Vw8Zb4Nc6" // 25 chars, real high entropy
	if !highEntropyValue(body) {
		t.Fatalf("precondition: bare body should pass")
	}
	for _, pad := range []string{"=", "=="} {
		if !highEntropyValue(body + pad) {
			t.Errorf("highEntropyValue(%q) = false, want true",
				body+pad)
		}
	}
}

// TestHasMonotoneRun pins the alphabet/digit-run detector that catches
// placeholders built from sequential ASCII ("abcdef", "1234567890",
// "ZYXWVU"). The 6-char minimum is small enough to catch the dominant
// noise shapes without flagging random secrets that happen to include a
// short run by chance.
func TestHasMonotoneRun(t *testing.T) {
	cases := []struct {
		name string
		s    string
		want bool
	}{
		{"abcdef", "abcdef", true},
		{"1234567890", "1234567890", true},
		{"ZYXWVU", "ZYXWVU", true},
		{"fedcba", "fedcba", true},
		{"abcde (only 5)", "abcde", false},
		{"random", "7Qh3Wn8Dk4Rp9Vx2Mb6Tj0Qc5Lm", false},
		{"isolated +1 transitions", "549271836401", false},
		{"embedded run", "Xabcdef9", true},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			assert.Equal(t, c.want, hasMonotoneRun(c.s, 6))
		})
	}
}