Files
wehub-resource-sync 48b3ccf279
gitleaks / gitleaks (push) Has been skipped
Test / test (ubuntu-latest) (push) Failing after 0s
Test / test (windows-latest) (push) Has been cancelled
chore: import upstream snapshot with attribution
2026-07-13 12:12:29 +08:00

673 lines
19 KiB
Go

package config
import (
"errors"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/zricethezav/gitleaks/v8/regexp"
)
const configPath = "../testdata/config/"
var regexComparer = func(x, y *regexp.Regexp) bool {
if x == nil || y == nil {
return x == y
}
return x.String() == y.String()
}
type translateCase struct {
// Configuration file basename to load, from `../testdata/config/`.
cfgName string
// Expected result.
cfg Config
// Rules to compare.
rules []string
// Error to expect.
wantError error
}
func TestTranslate(t *testing.T) {
tests := []translateCase{
// Valid
{
cfgName: "generic",
cfg: Config{
Title: "gitleaks config",
Rules: map[string]Rule{"generic-api-key": {
RuleID: "generic-api-key",
Description: "Generic API Key",
Regex: regexp.MustCompile(`(?i)(?:key|api|token|secret|client|passwd|password|auth|access)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|\"|\s|=|\x60){0,5}([0-9a-z\-_.=]{10,150})(?:['|\"|\n|\r|\s|\x60|;]|$)`),
Entropy: 3.5,
Keywords: []string{"key", "api", "token", "secret", "client", "passwd", "password", "auth", "access"},
Tags: []string{},
}},
},
},
{
cfgName: "valid/rule_path_only",
cfg: Config{
Rules: map[string]Rule{"python-files-only": {
RuleID: "python-files-only",
Description: "Python Files",
Path: regexp.MustCompile(`.py`),
Keywords: []string{},
Tags: []string{},
}},
},
},
{
cfgName: "valid/rule_regex_escaped_character_group",
cfg: Config{
Rules: map[string]Rule{"pypi-upload-token": {
RuleID: "pypi-upload-token",
Description: "PyPI upload token",
Regex: regexp.MustCompile(`pypi-AgEIcHlwaS5vcmc[A-Za-z0-9\-_]{50,1000}`),
Keywords: []string{},
Tags: []string{"key", "pypi"},
}},
},
},
{
cfgName: "valid/rule_entropy_group",
cfg: Config{
Rules: map[string]Rule{"discord-api-key": {
RuleID: "discord-api-key",
Description: "Discord API key",
Regex: regexp.MustCompile(`(?i)(discord[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{64})['\"]`),
Entropy: 3.5,
SecretGroup: 3,
Keywords: []string{},
Tags: []string{},
}},
},
},
// Invalid
{
cfgName: "invalid/rule_missing_id",
cfg: Config{},
wantError: errors.New("rule |id| is missing or empty, description: Discord API key, regex: (?i)(discord[a-z0-9_ .\\-,]{0,25})(=|>|:=|\\|\\|:|<=|=>|:).{0,5}['\\\"]([a-h0-9]{64})['\\\"]"),
},
{
cfgName: "invalid/rule_no_regex_or_path",
cfg: Config{},
wantError: errors.New("discord-api-key: both |regex| and |path| are empty, this rule will have no effect"),
},
{
cfgName: "invalid/rule_bad_entropy_group",
cfg: Config{},
wantError: errors.New("discord-api-key: invalid regex secret group 5, max regex secret group 3"),
},
}
for _, tt := range tests {
t.Run(tt.cfgName, func(t *testing.T) {
testTranslate(t, tt)
})
}
}
func TestTranslateAllowlists(t *testing.T) {
tests := []translateCase{
// Global
{
cfgName: "valid/allowlist_global_old_compat",
cfg: Config{
Rules: map[string]Rule{},
Allowlists: []*Allowlist{
{
StopWords: []string{"0989c462-69c9-49fa-b7d2-30dc5c576a97"},
},
},
},
},
{
cfgName: "valid/allowlist_global_multiple",
cfg: Config{
Rules: map[string]Rule{
"test": {
RuleID: "test",
Regex: regexp.MustCompile(`token = "(.+)"`),
Keywords: []string{},
Tags: []string{},
},
},
Allowlists: []*Allowlist{
{
Regexes: []*regexp.Regexp{regexp.MustCompile("^changeit$")},
},
{
MatchCondition: AllowlistMatchAnd,
Paths: []*regexp.Regexp{regexp.MustCompile("^node_modules/.*")},
StopWords: []string{"mock"},
},
},
},
},
{
cfgName: "valid/allowlist_global_target_rules",
cfg: Config{
Rules: map[string]Rule{
"github-app-token": {
RuleID: "github-app-token",
Regex: regexp.MustCompile(`(?:ghu|ghs)_[0-9a-zA-Z]{36}`),
Tags: []string{},
Keywords: []string{},
Allowlists: []*Allowlist{
{
Paths: []*regexp.Regexp{regexp.MustCompile(`(?:^|/)@octokit/auth-token/README\.md$`)},
},
},
},
"github-oauth": {
RuleID: "github-oauth",
Regex: regexp.MustCompile(`gho_[0-9a-zA-Z]{36}`),
Tags: []string{},
Keywords: []string{},
Allowlists: nil,
},
"github-pat": {
RuleID: "github-pat",
Regex: regexp.MustCompile(`ghp_[0-9a-zA-Z]{36}`),
Tags: []string{},
Keywords: []string{},
Allowlists: []*Allowlist{
{
Paths: []*regexp.Regexp{regexp.MustCompile(`(?:^|/)@octokit/auth-token/README\.md$`)},
},
},
},
},
Allowlists: []*Allowlist{
{
Regexes: []*regexp.Regexp{regexp.MustCompile(".*fake.*")},
},
},
},
},
{
cfgName: "valid/allowlist_global_regex",
cfg: Config{
Rules: map[string]Rule{},
Allowlists: []*Allowlist{
{
MatchCondition: AllowlistMatchOr,
Regexes: []*regexp.Regexp{regexp.MustCompile("AKIALALEM.L33243OLIA")},
},
},
},
},
{
cfgName: "invalid/allowlist_global_empty",
cfg: Config{},
wantError: errors.New("[[allowlists]] must contain at least one check for: commits, paths, regexes, or stopwords"),
},
{
cfgName: "invalid/allowlist_global_old_and_new",
cfg: Config{},
wantError: errors.New("[allowlist] is deprecated, it cannot be used alongside [[allowlists]]"),
},
{
cfgName: "invalid/allowlist_global_target_rule_id",
cfg: Config{},
wantError: errors.New("[[allowlists]] target rule ID 'github-pat' does not exist"),
},
{
cfgName: "invalid/allowlist_global_regextarget",
cfg: Config{},
wantError: errors.New("[[allowlists]] unknown allowlist |regexTarget| 'mtach' (expected 'match', 'line')"),
},
// Rule
{
cfgName: "valid/allowlist_rule_old_compat",
cfg: Config{
Rules: map[string]Rule{"example": {
RuleID: "example",
Regex: regexp.MustCompile(`example\d+`),
Tags: []string{},
Keywords: []string{},
Allowlists: []*Allowlist{
{
MatchCondition: AllowlistMatchOr,
Regexes: []*regexp.Regexp{regexp.MustCompile("123")},
},
},
}},
},
},
{
cfgName: "valid/allowlist_rule_regex",
cfg: Config{
Title: "simple config with allowlist for aws",
Rules: map[string]Rule{"aws-access-key": {
RuleID: "aws-access-key",
Description: "AWS Access Key",
Regex: regexp.MustCompile("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
Keywords: []string{},
Tags: []string{"key", "AWS"},
Allowlists: []*Allowlist{
{
MatchCondition: AllowlistMatchOr,
Regexes: []*regexp.Regexp{regexp.MustCompile("AKIALALEMEL33243OLIA")},
},
},
}},
},
},
{
cfgName: "valid/allowlist_rule_commit",
cfg: Config{
Title: "simple config with allowlist for a specific commit",
Rules: map[string]Rule{"aws-access-key": {
RuleID: "aws-access-key",
Description: "AWS Access Key",
Regex: regexp.MustCompile("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
Keywords: []string{},
Tags: []string{"key", "AWS"},
Allowlists: []*Allowlist{
{
MatchCondition: AllowlistMatchOr,
Commits: []string{"allowthiscommit"},
},
},
}},
},
},
{
cfgName: "valid/allowlist_rule_path",
cfg: Config{
Title: "simple config with allowlist for .go files",
Rules: map[string]Rule{"aws-access-key": {
RuleID: "aws-access-key",
Description: "AWS Access Key",
Regex: regexp.MustCompile("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
Keywords: []string{},
Tags: []string{"key", "AWS"},
Allowlists: []*Allowlist{
{
MatchCondition: AllowlistMatchOr,
Paths: []*regexp.Regexp{regexp.MustCompile(".go")},
},
},
}},
},
},
{
cfgName: "invalid/allowlist_rule_empty",
cfg: Config{},
wantError: errors.New("example: [[rules.allowlists]] must contain at least one check for: commits, paths, regexes, or stopwords"),
},
{
cfgName: "invalid/allowlist_rule_old_and_new",
cfg: Config{},
wantError: errors.New("example: [rules.allowlist] is deprecated, it cannot be used alongside [[rules.allowlist]]"),
},
{
cfgName: "invalid/allowlist_rule_regextarget",
cfg: Config{},
wantError: errors.New("example: [[rules.allowlists]] unknown allowlist |regexTarget| 'mtach' (expected 'match', 'line')"),
},
}
for _, tt := range tests {
t.Run(tt.cfgName, func(t *testing.T) {
testTranslate(t, tt)
})
}
}
func TestTranslateExtend(t *testing.T) {
tests := []translateCase{
// Valid
{
cfgName: "valid/extend",
cfg: Config{
Rules: map[string]Rule{
"aws-access-key": {
RuleID: "aws-access-key",
Description: "AWS Access Key",
Regex: regexp.MustCompile("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
Keywords: []string{},
Tags: []string{"key", "AWS"},
},
"aws-secret-key": {
RuleID: "aws-secret-key",
Description: "AWS Secret Key",
Regex: regexp.MustCompile(`(?i)aws_(.{0,20})?=?.[\'\"0-9a-zA-Z\/+]{40}`),
Keywords: []string{},
Tags: []string{"key", "AWS"},
},
"aws-secret-key-again": {
RuleID: "aws-secret-key-again",
Description: "AWS Secret Key",
Regex: regexp.MustCompile(`(?i)aws_(.{0,20})?=?.[\'\"0-9a-zA-Z\/+]{40}`),
Keywords: []string{},
Tags: []string{"key", "AWS"},
},
},
},
},
{
cfgName: "valid/extend_disabled",
cfg: Config{
Title: "gitleaks extend disable",
Rules: map[string]Rule{
"aws-secret-key": {
RuleID: "aws-secret-key",
Regex: regexp.MustCompile(`(?i)aws_(.{0,20})?=?.[\'\"0-9a-zA-Z\/+]{40}`),
Tags: []string{"key", "AWS"},
Keywords: []string{},
},
"pypi-upload-token": {
RuleID: "pypi-upload-token",
Regex: regexp.MustCompile(`pypi-AgEIcHlwaS5vcmc[A-Za-z0-9\-_]{50,1000}`),
Tags: []string{},
Keywords: []string{},
},
},
},
},
{
cfgName: "valid/extend_rule_no_regexpath",
cfg: Config{
Rules: map[string]Rule{
"aws-secret-key-again-again": {
RuleID: "aws-secret-key-again-again",
Description: "AWS Secret Key",
Regex: regexp.MustCompile(`(?i)aws_(.{0,20})?=?.[\'\"0-9a-zA-Z\/+]{40}`),
Keywords: []string{},
Tags: []string{"key", "AWS"},
Allowlists: []*Allowlist{
{
Description: "False positive. Keys used for colors match the rule, and should be excluded.",
MatchCondition: AllowlistMatchOr,
Paths: []*regexp.Regexp{regexp.MustCompile(`something.py`)},
},
},
},
},
},
},
{
cfgName: "valid/extend_rule_override_description",
rules: []string{"aws-access-key"},
cfg: Config{
Title: "override a built-in rule's description",
Rules: map[string]Rule{"aws-access-key": {
RuleID: "aws-access-key",
Description: "Puppy Doggy",
Regex: regexp.MustCompile("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
Keywords: []string{},
Tags: []string{"key", "AWS"},
},
},
},
},
{
cfgName: "valid/extend_rule_override_path",
rules: []string{"aws-access-key"},
cfg: Config{
Title: "override a built-in rule's path",
Rules: map[string]Rule{"aws-access-key": {
RuleID: "aws-access-key",
Description: "AWS Access Key",
Regex: regexp.MustCompile("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
Path: regexp.MustCompile("(?:puppy)"),
Keywords: []string{},
Tags: []string{"key", "AWS"},
},
},
},
},
{
cfgName: "valid/extend_rule_override_regex",
rules: []string{"aws-access-key"},
cfg: Config{
Title: "override a built-in rule's regex",
Rules: map[string]Rule{"aws-access-key": {
RuleID: "aws-access-key",
Description: "AWS Access Key",
Regex: regexp.MustCompile("(?:a)"),
Keywords: []string{},
Tags: []string{"key", "AWS"},
},
},
},
},
{
cfgName: "valid/extend_rule_override_secret_group",
rules: []string{"aws-access-key"},
cfg: Config{
Title: "override a built-in rule's secretGroup",
Rules: map[string]Rule{"aws-access-key": {
RuleID: "aws-access-key",
Description: "AWS Access Key",
Regex: regexp.MustCompile("(a)(a)"),
SecretGroup: 2,
Keywords: []string{},
Tags: []string{"key", "AWS"},
},
},
},
},
{
cfgName: "valid/extend_rule_override_entropy",
rules: []string{"aws-access-key"},
cfg: Config{
Title: "override a built-in rule's entropy",
Rules: map[string]Rule{"aws-access-key": {
RuleID: "aws-access-key",
Description: "AWS Access Key",
Regex: regexp.MustCompile("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
Entropy: 999.0,
Keywords: []string{},
Tags: []string{"key", "AWS"},
},
},
},
},
{
cfgName: "valid/extend_rule_override_keywords",
rules: []string{"aws-access-key"},
cfg: Config{
Title: "override a built-in rule's keywords",
Rules: map[string]Rule{"aws-access-key": {
RuleID: "aws-access-key",
Description: "AWS Access Key",
Regex: regexp.MustCompile("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
Keywords: []string{"puppy"},
Tags: []string{"key", "AWS"},
},
},
},
},
{
cfgName: "valid/extend_rule_override_tags",
rules: []string{"aws-access-key"},
cfg: Config{
Title: "override a built-in rule's tags",
Rules: map[string]Rule{"aws-access-key": {
RuleID: "aws-access-key",
Description: "AWS Access Key",
Regex: regexp.MustCompile("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}"),
Keywords: []string{},
Tags: []string{"key", "AWS", "puppy"},
},
},
},
},
{
cfgName: "valid/extend_rule_allowlist_or",
cfg: Config{
Title: "gitleaks extended 3",
Rules: map[string]Rule{
"aws-secret-key-again-again": {
RuleID: "aws-secret-key-again-again",
Description: "AWS Secret Key",
Regex: regexp.MustCompile(`(?i)aws_(.{0,20})?=?.[\'\"0-9a-zA-Z\/+]{40}`),
Keywords: []string{},
Tags: []string{"key", "AWS"},
Allowlists: []*Allowlist{
{
MatchCondition: AllowlistMatchOr,
StopWords: []string{"fake"},
},
{
MatchCondition: AllowlistMatchOr,
Commits: []string{"abcdefg1"},
Paths: []*regexp.Regexp{regexp.MustCompile(`ignore\.xaml`)},
Regexes: []*regexp.Regexp{regexp.MustCompile(`foo.+bar`)},
RegexTarget: "line",
StopWords: []string{"example"},
},
},
},
},
},
},
{
cfgName: "valid/extend_rule_allowlist_and",
cfg: Config{
Title: "gitleaks extended 3",
Rules: map[string]Rule{
"aws-secret-key-again-again": {
RuleID: "aws-secret-key-again-again",
Description: "AWS Secret Key",
Regex: regexp.MustCompile(`(?i)aws_(.{0,20})?=?.[\'\"0-9a-zA-Z\/+]{40}`),
Keywords: []string{},
Tags: []string{"key", "AWS"},
Allowlists: []*Allowlist{
{
MatchCondition: AllowlistMatchOr,
StopWords: []string{"fake"},
},
{
MatchCondition: AllowlistMatchAnd,
Commits: []string{"abcdefg1"},
Paths: []*regexp.Regexp{regexp.MustCompile(`ignore\.xaml`)},
Regexes: []*regexp.Regexp{regexp.MustCompile(`foo.+bar`)},
RegexTarget: "line",
StopWords: []string{"example"},
},
},
},
},
},
},
// Invalid
{
cfgName: "invalid/extend_invalid_ruleid",
wantError: errors.New("rule |id| is missing or empty"),
},
}
for _, tt := range tests {
t.Run(tt.cfgName, func(t *testing.T) {
testTranslate(t, tt)
})
}
}
func testTranslate(t *testing.T, test translateCase) {
t.Helper()
t.Cleanup(func() {
extendDepth = 0
viper.Reset()
})
viper.AddConfigPath(configPath)
viper.SetConfigName(test.cfgName)
viper.SetConfigType("toml")
err := viper.ReadInConfig()
require.NoError(t, err)
var vc ViperConfig
err = viper.Unmarshal(&vc)
require.NoError(t, err)
cfg, err := vc.Translate()
if err != nil {
if test.wantError != nil {
assert.EqualError(t, err, test.wantError.Error())
} else {
require.NoError(t, err)
}
} else {
if test.wantError != nil {
t.Fatalf("expected error but got none: %v", test.wantError)
return
}
}
if len(test.rules) > 0 {
rules := make(map[string]Rule)
for _, name := range test.rules {
rules[name] = cfg.Rules[name]
}
cfg.Rules = rules
}
opts := cmp.Options{
cmp.Comparer(regexComparer),
cmpopts.IgnoreUnexported(Rule{}, Allowlist{}),
}
if diff := cmp.Diff(test.cfg.Title, cfg.Title); diff != "" {
t.Errorf("%s diff: (-want +got)\n%s", test.cfgName, diff)
}
if diff := cmp.Diff(test.cfg.Rules, cfg.Rules, opts); diff != "" {
t.Errorf("%s diff: (-want +got)\n%s", test.cfgName, diff)
}
if diff := cmp.Diff(test.cfg.Allowlists, cfg.Allowlists, opts); diff != "" {
t.Errorf("%s diff: (-want +got)\n%s", test.cfgName, diff)
}
}
func TestExtendedRuleKeywordsAreDowncase(t *testing.T) {
tests := []struct {
name string
cfgName string
expectedKeywords string
}{
{
name: "Extend base rule that includes AWS keyword with new attribute",
cfgName: "valid/extend_base_rule_including_keywords_with_attribute",
expectedKeywords: "aws",
},
{
name: "Extend base with a new rule with CMS keyword",
cfgName: "valid/extend_rule_new",
expectedKeywords: "cms",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Cleanup(func() {
viper.Reset()
})
viper.AddConfigPath(configPath)
viper.SetConfigName(tt.cfgName)
viper.SetConfigType("toml")
err := viper.ReadInConfig()
require.NoError(t, err)
var vc ViperConfig
err = viper.Unmarshal(&vc)
require.NoError(t, err)
cfg, err := vc.Translate()
require.NoError(t, err)
_, exists := cfg.Keywords[tt.expectedKeywords]
require.Truef(t, exists, "The expected keyword %s did not exist as a key of cfg.Keywords", tt.expectedKeywords)
})
}
}