chore: import upstream snapshot with attribution

detect/codec/ascii.go

@@ -0,0 +1,34 @@
+package codec
+
+var printableASCII [256]bool
+
+func init() {
+	for b := 0; b < len(printableASCII); b++ {
+		if '\x08' < b && b < '\x7f' {
+			printableASCII[b] = true
+		}
+	}
+}
+
+// isPrintableASCII returns true if all bytes are printable ASCII
+func isPrintableASCII(b []byte) bool {
+	for _, c := range b {
+		if !printableASCII[c] {
+			return false
+		}
+	}
+
+	return true
+}
+
+// hasByte can be used to check if a string has at least one of the provided
+// bytes. Note: make sure byteset is long enough to handle the largest byte in
+// the string.
+func hasByte(data string, byteset []bool) bool {
+	for i := 0; i < len(data); i++ {
+		if byteset[data[i]] {
+			return true
+		}
+	}
+	return false
+}

detect/codec/base64.go

@@ -0,0 +1,39 @@
+package codec
+
+import (
+	"encoding/base64"
+)
+
+// likelyBase64Chars is a set of characters that you would expect to find at
+// least one of in base64 encoded data. This risks missing about 1% of
+// base64 encoded data that doesn't contain these characters, but gives you
+// the performance gain of not trying to decode a lot of long symbols in code.
+var likelyBase64Chars = make([]bool, 256)
+
+func init() {
+	for _, c := range `0123456789+/-_` {
+		likelyBase64Chars[c] = true
+	}
+}
+
+// decodeBase64 decodes base64 encoded printable ASCII characters
+func decodeBase64(encodedValue string) string {
+	// Exit early if it doesn't seem like base64
+	if !hasByte(encodedValue, likelyBase64Chars) {
+		return ""
+	}
+
+	// Try standard base64 decoding
+	decodedValue, err := base64.StdEncoding.DecodeString(encodedValue)
+	if err == nil && isPrintableASCII(decodedValue) {
+		return string(decodedValue)
+	}
+
+	// Try base64url decoding
+	decodedValue, err = base64.RawURLEncoding.DecodeString(encodedValue)
+	if err == nil && isPrintableASCII(decodedValue) {
+		return string(decodedValue)
+	}
+
+	return ""
+}

detect/codec/decoder.go

@@ -0,0 +1,105 @@
+package codec
+
+import (
+	"bytes"
+
+	"github.com/zricethezav/gitleaks/v8/logging"
+)
+
+// Decoder decodes various types of data in place
+type Decoder struct {
+	decodedMap map[string]string
+}
+
+// NewDecoder creates a default decoder struct
+func NewDecoder() *Decoder {
+	return &Decoder{
+		decodedMap: make(map[string]string),
+	}
+}
+
+// Decode returns the data with the values decoded in place along with the
+// encoded segment meta data for the next pass of decoding
+func (d *Decoder) Decode(data string, predecessors []*EncodedSegment) (string, []*EncodedSegment) {
+	segments := d.findEncodedSegments(data, predecessors)
+
+	if len(segments) > 0 {
+		result := bytes.NewBuffer(make([]byte, 0, len(data)))
+		encodedStart := 0
+		for _, segment := range segments {
+			result.WriteString(data[encodedStart:segment.encoded.start])
+			result.WriteString(segment.decodedValue)
+			encodedStart = segment.encoded.end
+		}
+
+		result.WriteString(data[encodedStart:])
+		return result.String(), segments
+	}
+
+	return data, segments
+}
+
+// findEncodedSegments finds the encoded segments in the data
+func (d *Decoder) findEncodedSegments(data string, predecessors []*EncodedSegment) []*EncodedSegment {
+	if len(data) == 0 {
+		return []*EncodedSegment{}
+	}
+
+	decodedShift := 0
+	encodingMatches := findEncodingMatches(data)
+	segments := make([]*EncodedSegment, 0, len(encodingMatches))
+	for _, m := range encodingMatches {
+		encodedValue := data[m.start:m.end]
+		decodedValue, alreadyDecoded := d.decodedMap[encodedValue]
+
+		if !alreadyDecoded {
+			decodedValue = m.encoding.decode(encodedValue)
+			d.decodedMap[encodedValue] = decodedValue
+		}
+
+		if len(decodedValue) == 0 {
+			continue
+		}
+
+		segment := &EncodedSegment{
+			predecessors: predecessors,
+			original:     toOriginal(predecessors, m.startEnd),
+			encoded:      m.startEnd,
+			decoded: startEnd{
+				m.start + decodedShift,
+				m.start + decodedShift + len(decodedValue),
+			},
+			decodedValue: decodedValue,
+			encodings:    m.encoding.kind,
+			depth:        1,
+		}
+
+		// Shift decoded start and ends based on size changes
+		decodedShift += len(decodedValue) - len(encodedValue)
+
+		// Adjust depth and encoding if applicable
+		if len(segment.predecessors) != 0 {
+			// Set the depth based on the predecessors' depth in the previous pass
+			segment.depth = 1 + segment.predecessors[0].depth
+			// Adjust encodings
+			for _, p := range segment.predecessors {
+				if segment.encoded.overlaps(p.decoded) {
+					segment.encodings |= p.encodings
+				}
+			}
+		}
+
+		segments = append(segments, segment)
+		logging.Debug().
+			Str("decoder", m.encoding.kind.String()).
+			Msgf(
+				"segment found: original=%s pos=%s: %q -> %q",
+				segment.original,
+				segment.encoded,
+				encodedValue,
+				segment.decodedValue,
+			)
+	}
+
+	return segments
+}

detect/codec/decoder_test.go

@@ -0,0 +1,144 @@
+package codec
+
+import (
+	"encoding/hex"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDecode(t *testing.T) {
+	tests := []struct {
+		chunk    string
+		expected string
+		name     string
+	}{
+		{
+			name:     "only b64 chunk",
+			chunk:    `bG9uZ2VyLWVuY29kZWQtc2VjcmV0LXRlc3Q=`,
+			expected: `longer-encoded-secret-test`,
+		},
+		{
+			name:     "mixed content",
+			chunk:    `token: bG9uZ2VyLWVuY29kZWQtc2VjcmV0LXRlc3Q=`,
+			expected: `token: longer-encoded-secret-test`,
+		},
+		{
+			name:     "no chunk",
+			chunk:    ``,
+			expected: ``,
+		},
+		{
+			name:     "env var (looks like all b64 decodable but has `=` in the middle)",
+			chunk:    `some-encoded-secret=dGVzdC1zZWNyZXQtdmFsdWU=`,
+			expected: `some-encoded-secret=test-secret-value`,
+		},
+		{
+			name:     "has longer b64 inside",
+			chunk:    `some-encoded-secret="bG9uZ2VyLWVuY29kZWQtc2VjcmV0LXRlc3Q="`,
+			expected: `some-encoded-secret="longer-encoded-secret-test"`,
+		},
+		{
+			name: "many possible i := 0substrings",
+			chunk: `Many substrings in this slack message could be base64 decoded
+				but only dGhpcyBlbmNhcHN1bGF0ZWQgc2VjcmV0 should be decoded.`,
+			expected: `Many substrings in this slack message could be base64 decoded
+				but only this encapsulated secret should be decoded.`,
+		},
+		{
+			name:     "b64-url-safe: only b64 chunk",
+			chunk:    `bG9uZ2VyLWVuY29kZWQtc2VjcmV0LXRlc3Q`,
+			expected: `longer-encoded-secret-test`,
+		},
+		{
+			name:     "b64-url-safe: mixed content",
+			chunk:    `token: bG9uZ2VyLWVuY29kZWQtc2VjcmV0LXRlc3Q`,
+			expected: `token: longer-encoded-secret-test`,
+		},
+		{
+			name:     "b64-url-safe: env var (looks like all b64 decodable but has `=` in the middle)",
+			chunk:    `some-encoded-secret=dGVzdC1zZWNyZXQtdmFsdWU=`,
+			expected: `some-encoded-secret=test-secret-value`,
+		},
+		{
+			name:     "b64-url-safe: has longer b64 inside",
+			chunk:    `some-encoded-secret="bG9uZ2VyLWVuY29kZWQtc2VjcmV0LXRlc3Q"`,
+			expected: `some-encoded-secret="longer-encoded-secret-test"`,
+		},
+		{
+			name:     "b64-url-safe: hyphen url b64",
+			chunk:    `Z2l0bGVha3M-PmZpbmRzLXNlY3JldHM`,
+			expected: `gitleaks>>finds-secrets`,
+		},
+		{
+			name:     "b64-url-safe: underscore url b64",
+			chunk:    `YjY0dXJsc2FmZS10ZXN0LXNlY3JldC11bmRlcnNjb3Jlcz8_`,
+			expected: `b64urlsafe-test-secret-underscores??`,
+		},
+		{
+			name:     "invalid base64 string",
+			chunk:    `a3d3fa7c2bb99e469ba55e5834ce79ee4853a8a3`,
+			expected: `a3d3fa7c2bb99e469ba55e5834ce79ee4853a8a3`,
+		},
+		{
+			name:     "url encoded value",
+			chunk:    `secret%3D%22q%24%21%40%23%24%25%5E%26%2A%28%20asdf%22`,
+			expected: `secret="q$!@#$%^&*( asdf"`,
+		},
+		{
+			name:     "hex encoded value",
+			chunk:    `secret="466973684D617048756E6B79212121363334"`,
+			expected: `secret="FishMapHunky!!!634"`,
+		},
+		{
+			name:     "unicode encoded value",
+			chunk:    `secret=U+0061 U+0062 U+0063 U+0064 U+0065 U+0066`,
+			expected: "secret=abcdef",
+		},
+		{
+			name:     "unicode encoded value backslashed",
+			chunk:    `secret=\\u0068\\u0065\\u006c\\u006c\\u006f\\u0020\\u0077\\u006f\\u0072\\u006c\\u0064\\u0020\\u0064\\u0075\\u0064\\u0065`,
+			expected: "secret=hello world dude",
+		},
+		{
+			name:     "unicode encoded value backslashed mixed w/ hex",
+			chunk:    `secret=\u0068\u0065\u006c\u006c\u006f\u0020\u0077\u006f\u0072\u006c\u0064 6C6F76656C792070656F706C65206F66206561727468`,
+			expected: "secret=hello world lovely people of earth",
+		},
+	}
+
+	decoder := NewDecoder()
+	fullDecode := func(data string) string {
+		segments := []*EncodedSegment{}
+		for {
+			data, segments = decoder.Decode(data, segments)
+			if len(segments) == 0 {
+				return data
+			}
+		}
+	}
+
+	// Test value decoding
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.expected, fullDecode(tt.chunk))
+		})
+	}
+
+	// Percent encode the values to test percent decoding
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			encodedChunk := url.PathEscape(tt.chunk)
+			assert.Equal(t, tt.expected, fullDecode(encodedChunk))
+		})
+	}
+
+	// Hex encode the values to test hex decoding
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			encodedChunk := hex.EncodeToString([]byte(tt.chunk))
+			assert.Equal(t, tt.expected, fullDecode(encodedChunk))
+		})
+	}
+}

detect/codec/encodings.go

@@ -0,0 +1,161 @@
+package codec
+
+import (
+	"fmt"
+	"math"
+	"strings"
+
+	"github.com/zricethezav/gitleaks/v8/regexp"
+)
+
+var (
+	// encodingsRe is a regex built by combining all the encoding patterns
+	// into named capture groups so that a single pass can detect multiple
+	// encodings
+	encodingsRe *regexp.Regexp
+	// encodings contains all the encoding configurations for the detector.
+	// The precedence is important. You want more specific encodings to
+	// have a higher precedence or encodings that partially encode the
+	// values (e.g. percent) unlike encodings that fully encode the string
+	// (e.g. base64). If two encoding matches overlap the decoder will use
+	// this order to determine which encoding should wait till the next pass.
+	encodings = []*encoding{
+		{
+			kind:    percentKind,
+			pattern: `%[0-9A-Fa-f]{2}(?:.*%[0-9A-Fa-f]{2})?`,
+			decode:  decodePercent,
+		},
+		{
+			kind:    unicodeKind,
+			pattern: `(?:(?:U\+[a-fA-F0-9]{4}(?:\s|$))+|(?i)(?:\\{1,2}u[a-fA-F0-9]{4})+)`,
+			decode:  decodeUnicode,
+		},
+		{
+			kind:    hexKind,
+			pattern: `[0-9A-Fa-f]{32,}`,
+			decode:  decodeHex,
+		},
+		{
+			kind:    base64Kind,
+			pattern: `[\w\/+-]{16,}={0,2}`,
+			decode:  decodeBase64,
+		},
+	}
+)
+
+// encodingNames is used to map the encodingKinds to their name
+var encodingNames = []string{
+	"percent",
+	"unicode",
+	"hex",
+	"base64",
+}
+
+// encodingKind can be or'd together to capture all of the unique encodings
+// that were present in a segment
+type encodingKind int
+
+var (
+	// make sure these go up by powers of 2
+	percentKind = encodingKind(1)
+	unicodeKind = encodingKind(2)
+	hexKind     = encodingKind(4)
+	base64Kind  = encodingKind(8)
+)
+
+func (e encodingKind) String() string {
+	i := int(math.Log2(float64(e)))
+	if i >= len(encodingNames) {
+		return ""
+	}
+	return encodingNames[i]
+}
+
+// kinds returns a list of encodingKinds combined in this one
+func (e encodingKind) kinds() []encodingKind {
+	kinds := []encodingKind{}
+
+	for i := 0; i < len(encodingNames); i++ {
+		if kind := int(e) & int(math.Pow(2, float64(i))); kind != 0 {
+			kinds = append(kinds, encodingKind(kind))
+		}
+	}
+
+	return kinds
+}
+
+// encodingMatch represents a match of an encoding in the text
+type encodingMatch struct {
+	encoding *encoding
+	startEnd
+}
+
+// encoding represent a type of coding supported by the decoder.
+type encoding struct {
+	// the kind of decoding (e.g. base64, etc)
+	kind encodingKind
+	// the regex pattern that matches the encoding format
+	pattern string
+	// take the match and return the decoded value
+	decode func(string) string
+	// determine which encoding should win out when two overlap
+	precedence int
+}
+
+func init() {
+	count := len(encodings)
+	namedPatterns := make([]string, count)
+	for i, encoding := range encodings {
+		encoding.precedence = count - i
+		namedPatterns[i] = fmt.Sprintf(
+			"(?P<%s>%s)",
+			encoding.kind,
+			encoding.pattern,
+		)
+	}
+	encodingsRe = regexp.MustCompile(strings.Join(namedPatterns, "|"))
+}
+
+// findEncodingMatches finds as many encodings as it can for this pass
+func findEncodingMatches(data string) []encodingMatch {
+	var all []encodingMatch
+	for _, matchIndex := range encodingsRe.FindAllStringSubmatchIndex(data, -1) {
+		// Add the encodingMatch with its proper encoding
+		for i, j := 2, 0; i < len(matchIndex); i, j = i+2, j+1 {
+			if matchIndex[i] > -1 {
+				all = append(all, encodingMatch{
+					encoding: encodings[j],
+					startEnd: startEnd{
+						start: matchIndex[i],
+						end:   matchIndex[i+1],
+					},
+				})
+			}
+		}
+	}
+
+	totalMatches := len(all)
+	if totalMatches == 1 {
+		return all
+	}
+
+	// filter out lower precedence ones that overlap their neigbors
+	filtered := make([]encodingMatch, 0, len(all))
+	for i, m := range all {
+		if i > 0 {
+			prev := all[i-1]
+			if m.overlaps(prev.startEnd) && prev.encoding.precedence > m.encoding.precedence {
+				continue // skip this one
+			}
+		}
+		if i+1 < totalMatches {
+			next := all[i+1]
+			if m.overlaps(next.startEnd) && next.encoding.precedence > m.encoding.precedence {
+				continue // skip this one
+			}
+		}
+		filtered = append(filtered, m)
+	}
+
+	return filtered
+}

detect/codec/hex.go

@@ -0,0 +1,60 @@
+package codec
+
+// hexMap is a precalculated map of hex nibbles
+const hexMap = "" +
+	"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+	"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+	"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+	"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\xff\xff\xff\xff\xff\xff" +
+	"\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+	"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+	"\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+	"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+	"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+	"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+	"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+	"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+	"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+	"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+	"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +
+	"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"
+
+// likelyHexChars is a set of characters that you would expect to find at
+// least one of in hex encoded data. This risks missing some hex data that
+// doesn't contain these characters, but gives you the performance gain of not
+// trying to decode a lot of long symbols in code.
+var likelyHexChars = make([]bool, 256)
+
+func init() {
+	for _, c := range `0123456789` {
+		likelyHexChars[c] = true
+	}
+}
+
+// decodeHex decodes hex data
+func decodeHex(encodedValue string) string {
+	size := len(encodedValue)
+	// hex should have two characters per byte
+	if size%2 != 0 {
+		return ""
+	}
+	if !hasByte(encodedValue, likelyHexChars) {
+		return ""
+	}
+
+	decodedValue := make([]byte, size/2)
+	for i := 0; i < size; i += 2 {
+		n1 := hexMap[encodedValue[i]]
+		n2 := hexMap[encodedValue[i+1]]
+		if n1|n2 == '\xff' {
+			return ""
+		}
+		b := n1<<4 | n2
+		if !printableASCII[b] {
+			return ""
+		}
+		decodedValue[i/2] = b
+	}
+
+	return string(decodedValue)
+}

detect/codec/percent.go

@@ -0,0 +1,34 @@
+package codec
+
+// decodePercent decodes percent encoded strings
+func decodePercent(encodedValue string) string {
+	encLen := len(encodedValue)
+	decodedValue := make([]byte, encLen)
+	decIndex := 0
+	encIndex := 0
+
+	for encIndex < encLen {
+		if encodedValue[encIndex] == '%' && encIndex+2 < encLen {
+			n1 := hexMap[encodedValue[encIndex+1]]
+			n2 := hexMap[encodedValue[encIndex+2]]
+			// Make sure they're hex characters
+			if n1|n2 != '\xff' {
+				b := n1<<4 | n2
+				if !printableASCII[b] {
+					return ""
+				}
+
+				decodedValue[decIndex] = b
+				encIndex += 3
+				decIndex += 1
+				continue
+			}
+		}
+
+		decodedValue[decIndex] = encodedValue[encIndex]
+		encIndex += 1
+		decIndex += 1
+	}
+
+	return string(decodedValue[:decIndex])
+}

detect/codec/segment.go

@@ -0,0 +1,173 @@
+package codec
+
+import (
+	"fmt"
+)
+
+// EncodedSegment represents a portion of text that is encoded in some way.
+type EncodedSegment struct {
+	// predecessors are all of the segments from the previous decoding pass
+	predecessors []*EncodedSegment
+
+	// original start/end indices before decoding
+	original startEnd
+
+	// encoded start/end indices relative to the previous decoding pass.
+	// If it's a top level segment, original and encoded will be the
+	// same.
+	encoded startEnd
+
+	// decoded start/end indices in this pass after decoding
+	decoded startEnd
+
+	// decodedValue contains the decoded string for this segment
+	decodedValue string
+
+	// encodings is the encodings that make up this segment. encodingKind
+	// can be or'd together to hold multiple encodings
+	encodings encodingKind
+
+	// depth is how many decoding passes it took to decode this segment
+	depth int
+}
+
+// Tags returns additional meta data tags related to the types of segments
+func Tags(segments []*EncodedSegment) []string {
+	// Return an empty list if we don't have any segments
+	if len(segments) == 0 {
+		return []string{}
+	}
+
+	// Since decoding is done in passes, the depth of all the segments
+	// should be the same
+	depth := segments[0].depth
+
+	// Collect the encodings from the segments
+	encodings := segments[0].encodings
+	for i := 1; i < len(segments); i++ {
+		encodings |= segments[i].encodings
+	}
+
+	kinds := encodings.kinds()
+	tags := make([]string, len(kinds)+1)
+
+	tags[len(tags)-1] = fmt.Sprintf("decode-depth:%d", depth)
+	for i, kind := range kinds {
+		tags[i] = fmt.Sprintf("decoded:%s", kind)
+	}
+
+	return tags
+}
+
+// CurrentLine returns from the start of the line containing the segments
+// to the end of the line where the segment ends.
+func CurrentLine(segments []*EncodedSegment, currentRaw string) string {
+	// Return the whole thing if no segments are provided
+	if len(segments) == 0 {
+		return currentRaw
+	}
+
+	start := 0
+	end := len(currentRaw)
+
+	// Merge the ranges together into a single decoded value
+	decoded := segments[0].decoded
+	for i := 1; i < len(segments); i++ {
+		decoded = decoded.merge(segments[i].decoded)
+	}
+
+	// Find the start of the range
+	for i := decoded.start; i > -1; i-- {
+		c := currentRaw[i]
+		if c == '\n' {
+			start = i
+			break
+		}
+	}
+
+	// Find the end of the range
+	for i := decoded.end; i < end; i++ {
+		c := currentRaw[i]
+		if c == '\n' {
+			end = i
+			break
+		}
+	}
+
+	return currentRaw[start:end]
+}
+
+// AdjustMatchIndex maps a match index from the current decode pass back to
+// its location in the original text
+func AdjustMatchIndex(segments []*EncodedSegment, matchIndex []int) []int {
+	// Don't adjust if we're not provided any segments
+	if len(segments) == 0 {
+		return matchIndex
+	}
+
+	// Map the match to the location in the original text
+	match := startEnd{matchIndex[0], matchIndex[1]}
+
+	// Map the match to its orignal location
+	adjusted := toOriginal(segments, match)
+
+	// Return the adjusted match index
+	return []int{
+		adjusted.start,
+		adjusted.end,
+	}
+}
+
+// SegmentsWithDecodedOverlap the segments where the start and end overlap its
+// decoded range
+func SegmentsWithDecodedOverlap(segments []*EncodedSegment, start, end int) []*EncodedSegment {
+	se := startEnd{start, end}
+	overlaps := []*EncodedSegment{}
+
+	for _, segment := range segments {
+		if segment.decoded.overlaps(se) {
+			overlaps = append(overlaps, segment)
+		}
+	}
+
+	return overlaps
+}
+
+// toOriginal maps a start/end to its start/end in the original text
+// the provided start/end should be relative to the segment's decoded value
+func toOriginal(predecessors []*EncodedSegment, decoded startEnd) startEnd {
+	if len(predecessors) == 0 {
+		return decoded
+	}
+
+	// Map the decoded value one level up where it was encoded
+	encoded := startEnd{}
+
+	for _, p := range predecessors {
+		if !p.decoded.overlaps(decoded) {
+			continue // Not in scope
+		}
+
+		// If fully contained, return the segments original start/end
+		if p.decoded.contains(decoded) {
+			return p.original
+		}
+
+		// Map the value to be relative to the predecessors's decoded values
+		if encoded.end == 0 {
+			encoded = p.encoded.add(p.decoded.overflow(decoded))
+		} else {
+			encoded = encoded.merge(p.encoded.add(p.decoded.overflow(decoded)))
+		}
+	}
+
+	// Should only get here if the thing passed in wasn't in a decoded
+	// value. This shouldn't be the case
+	if encoded.end == 0 {
+		return decoded
+	}
+
+	// Climb up another level
+	// (NOTE: each segment references all the predecessors)
+	return toOriginal(predecessors[0].predecessors, encoded)
+}

detect/codec/start_end.go

@@ -0,0 +1,57 @@
+package codec
+
+import (
+	"fmt"
+)
+
+// startEnd represents the start and end of some data. It mainly exists as a
+// helper when referencing the values
+type startEnd struct {
+	start int
+	end   int
+}
+
+// sub subtracts the values of two startEnds
+func (s startEnd) sub(o startEnd) startEnd {
+	return startEnd{
+		s.start - o.start,
+		s.end - o.end,
+	}
+}
+
+// add adds the values of two startEnds
+func (s startEnd) add(o startEnd) startEnd {
+	return startEnd{
+		s.start + o.start,
+		s.end + o.end,
+	}
+}
+
+// overlaps returns true if two startEnds overlap
+func (s startEnd) overlaps(o startEnd) bool {
+	return o.start <= s.end && o.end >= s.start
+}
+
+// contains returns true if the other is fully contained within this one
+func (s startEnd) contains(o startEnd) bool {
+	return s.start <= o.start && o.end <= s.end
+}
+
+// overflow returns a startEnd that tells how much the other goes outside the
+// bounds of this one
+func (s startEnd) overflow(o startEnd) startEnd {
+	return s.merge(o).sub(s)
+}
+
+// merge takes two start/ends and returns a single one that encompasses both
+func (s startEnd) merge(o startEnd) startEnd {
+	return startEnd{
+		min(s.start, o.start),
+		max(s.end, o.end),
+	}
+}
+
+// String returns a string representation for clearer debugging
+func (s startEnd) String() string {
+	return fmt.Sprintf("[%d,%d]", s.start, s.end)
+}

detect/codec/unicode.go

@@ -0,0 +1,261 @@
+package codec
+
+import (
+	"bytes"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+
+	"github.com/zricethezav/gitleaks/v8/regexp"
+)
+
+var (
+	// Standard Unicode notation (e.g., U+1234)
+	unicodeCodePointPat = regexp.MustCompile(`U\+([a-fA-F0-9]{4}).?`)
+
+	// Multiple code points pattern - used for continuous sequences like "U+0074 U+006F U+006B..."
+	unicodeMultiCodePointPat = regexp.MustCompile(`(?:U\+[a-fA-F0-9]{4}(?:\s|$))+`)
+
+	// Common escape sequence used in programming languages (e.g., \u1234)
+	unicodeEscapePat = regexp.MustCompile(`(?i)\\{1,2}u([a-fA-F0-9]{4})`)
+
+	// Multiple escape sequences pattern - used for continuous sequences like "\u0074\u006F\u006B..."
+	unicodeMultiEscapePat = regexp.MustCompile(`(?i)(?:\\{1,2}u[a-fA-F0-9]{4})+`)
+)
+
+// Unicode characters are encoded as 1 to 4 bytes per rune.
+const maxBytesPerRune = 4
+
+// decodeUnicode decodes Unicode escape sequences in the given string
+func decodeUnicode(encodedValue string) string {
+	// First, check if we have a continuous sequence of Unicode code points
+	if matches := unicodeMultiCodePointPat.FindAllString(encodedValue, -1); len(matches) > 0 {
+		// For each detected sequence of code points
+		for _, match := range matches {
+			// Decode the entire sequence at once
+			decodedSequence := decodeMultiCodePoint(match)
+
+			// If we successfully decoded something, replace it in the original string
+			if decodedSequence != "" && decodedSequence != match {
+				encodedValue = strings.Replace(encodedValue, match, decodedSequence, 1)
+			}
+		}
+		return encodedValue
+	}
+
+	// Next, check if we have a continuous sequence of escape sequences
+	if matches := unicodeMultiEscapePat.FindAllString(encodedValue, -1); len(matches) > 0 {
+		// For each detected sequence of escape sequences
+		for _, match := range matches {
+			// Decode the entire sequence at once
+			decodedSequence := decodeMultiEscape(match)
+
+			// If we successfully decoded something, replace it in the original string
+			if decodedSequence != "" && decodedSequence != match {
+				encodedValue = strings.Replace(encodedValue, match, decodedSequence, 1)
+			}
+		}
+		return encodedValue
+	}
+
+	// If no multi-patterns were matched, fall back to the original implementation
+	// for individual code points and escape sequences
+
+	// Create a copy of the input to work with
+	data := []byte(encodedValue)
+
+	// Store the result
+	var result []byte
+
+	// Check and decode Unicode code points (U+1234 format)
+	if unicodeCodePointPat.Match(data) {
+		result = decodeIndividualCodePoints(data)
+	}
+
+	// If no code points were found or we have a mix of formats,
+	// also check for Unicode escape sequences (\u1234 format)
+	if len(result) == 0 || unicodeEscapePat.Match(data) {
+		// If we already have some result from code point decoding,
+		// continue decoding escape sequences on that result
+		if len(result) > 0 {
+			result = decodeIndividualEscapes(result)
+		} else {
+			result = decodeIndividualEscapes(data)
+		}
+	}
+
+	// If nothing was decoded, return original string
+	if len(result) == 0 || bytes.Equal(result, data) {
+		return encodedValue
+	}
+
+	return string(result)
+}
+
+// decodeMultiCodePoint decodes a continuous sequence of Unicode code points (U+XXXX format)
+func decodeMultiCodePoint(sequence string) string {
+	// If the sequence is empty, return empty string
+	if sequence == "" {
+		return ""
+	}
+
+	// Split the sequence by whitespace to get individual code points
+	codePoints := strings.Fields(sequence)
+	if len(codePoints) == 0 {
+		return sequence
+	}
+
+	// Decode each code point and build the result
+	var decodedRunes []rune
+	for _, cp := range codePoints {
+		// Check if it follows the U+XXXX pattern
+		if !strings.HasPrefix(cp, "U+") || len(cp) < 6 {
+			continue
+		}
+
+		// Extract the hexadecimal value
+		hexValue := cp[2:]
+
+		// Parse the hexadecimal value to an integer
+		unicodeInt, err := strconv.ParseInt(hexValue, 16, 32)
+		if err != nil {
+			continue
+		}
+
+		// Convert to rune and add to result
+		decodedRunes = append(decodedRunes, rune(unicodeInt))
+	}
+
+	// If we didn't decode anything, return the original sequence
+	if len(decodedRunes) == 0 {
+		return sequence
+	}
+
+	// Return the decoded string
+	return string(decodedRunes)
+}
+
+// decodeMultiEscape decodes a continuous sequence of Unicode escape sequences (\uXXXX format)
+func decodeMultiEscape(sequence string) string {
+	// If the sequence is empty, return empty string
+	if sequence == "" {
+		return ""
+	}
+
+	// Find all escape sequences
+	escapes := unicodeEscapePat.FindAllStringSubmatch(sequence, -1)
+	if len(escapes) == 0 {
+		return sequence
+	}
+
+	// Decode each escape sequence and build the result
+	var decodedRunes []rune
+	for _, esc := range escapes {
+		// Extract the hexadecimal value
+		hexValue := esc[1]
+
+		// Parse the hexadecimal value to an integer
+		unicodeInt, err := strconv.ParseInt(hexValue, 16, 32)
+		if err != nil {
+			continue
+		}
+
+		// Convert to rune and add to result
+		decodedRunes = append(decodedRunes, rune(unicodeInt))
+	}
+
+	// If we didn't decode anything, return the original sequence
+	if len(decodedRunes) == 0 {
+		return sequence
+	}
+
+	// Return the decoded string
+	return string(decodedRunes)
+}
+
+// decodeIndividualCodePoints decodes individual Unicode code points (U+1234 format)
+// This is a fallback for when we don't have a continuous sequence of code points
+func decodeIndividualCodePoints(input []byte) []byte {
+	// Find all Unicode code point sequences in the input byte slice
+	indices := unicodeCodePointPat.FindAllSubmatchIndex(input, -1)
+
+	// If none found, return original input
+	if len(indices) == 0 {
+		return input
+	}
+
+	// Iterate over found indices in reverse order to avoid modifying the slice length
+	utf8Bytes := make([]byte, maxBytesPerRune)
+	for i := len(indices) - 1; i >= 0; i-- {
+		matches := indices[i]
+
+		startIndex := matches[0]
+		endIndex := matches[1]
+		hexStartIndex := matches[2]
+		hexEndIndex := matches[3]
+
+		// If the input is like `U+1234 U+5678` we should replace `U+1234 `.
+		// Otherwise, we should only replace `U+1234`.
+		if endIndex != hexEndIndex && endIndex < len(input) && input[endIndex-1] == ' ' {
+			endIndex = endIndex - 1
+		}
+
+		// Extract the hexadecimal value from the escape sequence
+		hexValue := string(input[hexStartIndex:hexEndIndex])
+
+		// Parse the hexadecimal value to an integer
+		unicodeInt, err := strconv.ParseInt(hexValue, 16, 32)
+		if err != nil {
+			// If there's an error, continue to the next escape sequence
+			continue
+		}
+
+		// Convert the Unicode code point to a UTF-8 representation
+		utf8Len := utf8.EncodeRune(utf8Bytes, rune(unicodeInt))
+
+		// Replace the escape sequence with the UTF-8 representation
+		input = append(input[:startIndex], append(utf8Bytes[:utf8Len], input[endIndex:]...)...)
+	}
+
+	return input
+}
+
+// decodeIndividualEscapes decodes individual Unicode escape sequences (\u1234 format)
+// This is a fallback for when we don't have a continuous sequence of escape sequences
+func decodeIndividualEscapes(input []byte) []byte {
+	// Find all Unicode escape sequences in the input byte slice
+	indices := unicodeEscapePat.FindAllSubmatchIndex(input, -1)
+
+	// If none found, return original input
+	if len(indices) == 0 {
+		return input
+	}
+
+	// Iterate over found indices in reverse order to avoid modifying the slice length
+	utf8Bytes := make([]byte, maxBytesPerRune)
+	for i := len(indices) - 1; i >= 0; i-- {
+		matches := indices[i]
+
+		startIndex := matches[0]
+		hexStartIndex := matches[2]
+		endIndex := matches[3]
+
+		// Extract the hexadecimal value from the escape sequence
+		hexValue := string(input[hexStartIndex:endIndex])
+
+		// Parse the hexadecimal value to an integer
+		unicodeInt, err := strconv.ParseInt(hexValue, 16, 32)
+		if err != nil {
+			// If there's an error, continue to the next escape sequence
+			continue
+		}
+
+		// Convert the Unicode code point to a UTF-8 representation
+		utf8Len := utf8.EncodeRune(utf8Bytes, rune(unicodeInt))
+
+		// Replace the escape sequence with the UTF-8 representation
+		input = append(input[:startIndex], append(utf8Bytes[:utf8Len], input[endIndex:]...)...)
+	}
+
+	return input
+}