chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,133 @@
|
||||
package buffer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// maxLineSize is the maximum size for a single log line (10MB).
|
||||
// GitHub Actions logs can contain extremely long lines (base64 content, minified JS, etc.)
|
||||
const maxLineSize = 10 * 1024 * 1024
|
||||
|
||||
// ProcessResponseAsRingBufferToEnd reads the body of an HTTP response line by line,
|
||||
// storing only the last maxJobLogLines lines using a ring buffer (sliding window).
|
||||
// This efficiently retains the most recent lines, overwriting older ones as needed.
|
||||
//
|
||||
// Parameters:
|
||||
//
|
||||
// httpResp: The HTTP response whose body will be read.
|
||||
// maxJobLogLines: The maximum number of log lines to retain.
|
||||
//
|
||||
// Returns:
|
||||
//
|
||||
// string: The concatenated log lines (up to maxJobLogLines), separated by newlines.
|
||||
// int: The total number of lines read from the response.
|
||||
// *http.Response: The original HTTP response.
|
||||
// error: Any error encountered during reading.
|
||||
//
|
||||
// The function uses a ring buffer to efficiently store only the last maxJobLogLines lines.
|
||||
// If the response contains more lines than maxJobLogLines, only the most recent lines are kept.
|
||||
// Lines exceeding maxLineSize are truncated with a marker.
|
||||
func ProcessResponseAsRingBufferToEnd(httpResp *http.Response, maxJobLogLines int) (string, int, *http.Response, error) {
|
||||
if maxJobLogLines <= 0 {
|
||||
maxJobLogLines = 500
|
||||
}
|
||||
if maxJobLogLines > 100000 {
|
||||
maxJobLogLines = 100000
|
||||
}
|
||||
|
||||
lines := make([]string, maxJobLogLines)
|
||||
validLines := make([]bool, maxJobLogLines)
|
||||
totalLines := 0
|
||||
writeIndex := 0
|
||||
|
||||
const readBufferSize = 64 * 1024 // 64KB read buffer
|
||||
const maxDisplayLength = 1000 // Keep first 1000 chars of truncated lines
|
||||
|
||||
readBuf := make([]byte, readBufferSize)
|
||||
var currentLine strings.Builder
|
||||
lineTruncated := false
|
||||
|
||||
// storeLine saves the current line to the ring buffer and resets state
|
||||
storeLine := func() {
|
||||
line := currentLine.String()
|
||||
if lineTruncated && len(line) > maxDisplayLength {
|
||||
line = line[:maxDisplayLength]
|
||||
}
|
||||
if lineTruncated {
|
||||
line += "... [TRUNCATED]"
|
||||
}
|
||||
lines[writeIndex] = line
|
||||
validLines[writeIndex] = true
|
||||
totalLines++
|
||||
writeIndex = (writeIndex + 1) % maxJobLogLines
|
||||
currentLine.Reset()
|
||||
lineTruncated = false
|
||||
}
|
||||
|
||||
// accumulate adds bytes to currentLine up to maxLineSize, sets lineTruncated if exceeded
|
||||
accumulate := func(data []byte) {
|
||||
if lineTruncated {
|
||||
return
|
||||
}
|
||||
remaining := maxLineSize - currentLine.Len()
|
||||
if remaining <= 0 {
|
||||
lineTruncated = true
|
||||
return
|
||||
}
|
||||
if remaining > len(data) {
|
||||
remaining = len(data)
|
||||
}
|
||||
currentLine.Write(data[:remaining])
|
||||
if currentLine.Len() >= maxLineSize {
|
||||
lineTruncated = true
|
||||
}
|
||||
}
|
||||
|
||||
for {
|
||||
n, err := httpResp.Body.Read(readBuf)
|
||||
if n > 0 {
|
||||
chunk := readBuf[:n]
|
||||
for len(chunk) > 0 {
|
||||
newlineIdx := bytes.IndexByte(chunk, '\n')
|
||||
if newlineIdx < 0 {
|
||||
accumulate(chunk)
|
||||
break
|
||||
}
|
||||
accumulate(chunk[:newlineIdx])
|
||||
storeLine()
|
||||
chunk = chunk[newlineIdx+1:]
|
||||
}
|
||||
}
|
||||
|
||||
if err == io.EOF {
|
||||
if currentLine.Len() > 0 {
|
||||
storeLine()
|
||||
}
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return "", 0, httpResp, fmt.Errorf("failed to read log content: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
var result []string
|
||||
linesInBuffer := min(totalLines, maxJobLogLines)
|
||||
|
||||
startIndex := 0
|
||||
if totalLines > maxJobLogLines {
|
||||
startIndex = writeIndex
|
||||
}
|
||||
|
||||
for i := range linesInBuffer {
|
||||
idx := (startIndex + i) % maxJobLogLines
|
||||
if validLines[idx] {
|
||||
result = append(result, lines[idx])
|
||||
}
|
||||
}
|
||||
|
||||
return strings.Join(result, "\n"), totalLines, httpResp, nil
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package buffer
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestProcessResponseAsRingBufferToEnd(t *testing.T) {
|
||||
t.Run("normal lines", func(t *testing.T) {
|
||||
body := "line1\nline2\nline3\n"
|
||||
resp := &http.Response{
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
}
|
||||
|
||||
result, totalLines, respOut, err := ProcessResponseAsRingBufferToEnd(resp, 10)
|
||||
if respOut != nil && respOut.Body != nil {
|
||||
defer respOut.Body.Close()
|
||||
}
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 3, totalLines)
|
||||
assert.Equal(t, "line1\nline2\nline3", result)
|
||||
})
|
||||
|
||||
t.Run("ring buffer keeps last N lines", func(t *testing.T) {
|
||||
body := "line1\nline2\nline3\nline4\nline5\n"
|
||||
resp := &http.Response{
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
}
|
||||
|
||||
result, totalLines, respOut, err := ProcessResponseAsRingBufferToEnd(resp, 3)
|
||||
if respOut != nil && respOut.Body != nil {
|
||||
defer respOut.Body.Close()
|
||||
}
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 5, totalLines)
|
||||
assert.Equal(t, "line3\nline4\nline5", result)
|
||||
})
|
||||
|
||||
t.Run("handles very long line exceeding 10MB", func(t *testing.T) {
|
||||
// Create a line that exceeds maxLineSize (10MB)
|
||||
longLine := strings.Repeat("x", 11*1024*1024) // 11MB
|
||||
body := "line1\n" + longLine + "\nline3\n"
|
||||
resp := &http.Response{
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
}
|
||||
|
||||
result, totalLines, respOut, err := ProcessResponseAsRingBufferToEnd(resp, 100)
|
||||
if respOut != nil && respOut.Body != nil {
|
||||
defer respOut.Body.Close()
|
||||
}
|
||||
require.NoError(t, err)
|
||||
// Should have processed lines with truncation marker
|
||||
assert.Greater(t, totalLines, 0)
|
||||
assert.Contains(t, result, "TRUNCATED")
|
||||
})
|
||||
|
||||
t.Run("handles line at exactly max size", func(t *testing.T) {
|
||||
// Create a line just under maxLineSize
|
||||
longLine := strings.Repeat("a", 1024*1024) // 1MB - should work fine
|
||||
body := "start\n" + longLine + "\nend\n"
|
||||
resp := &http.Response{
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
}
|
||||
|
||||
result, totalLines, respOut, err := ProcessResponseAsRingBufferToEnd(resp, 100)
|
||||
if respOut != nil && respOut.Body != nil {
|
||||
defer respOut.Body.Close()
|
||||
}
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 3, totalLines)
|
||||
assert.Contains(t, result, "start")
|
||||
assert.Contains(t, result, "end")
|
||||
})
|
||||
|
||||
t.Run("ring buffer with long line in middle of many lines", func(t *testing.T) {
|
||||
// Create many lines with a long line in the middle
|
||||
// Ring buffer size is 5, so we should only keep the last 5 lines
|
||||
var sb strings.Builder
|
||||
for i := 1; i <= 10; i++ {
|
||||
sb.WriteString(fmt.Sprintf("line%d\n", i))
|
||||
}
|
||||
// Insert an 11MB line (exceeds maxLineSize of 10MB)
|
||||
longLine := strings.Repeat("x", 11*1024*1024)
|
||||
sb.WriteString(longLine)
|
||||
sb.WriteString("\n")
|
||||
for i := 11; i <= 20; i++ {
|
||||
sb.WriteString(fmt.Sprintf("line%d\n", i))
|
||||
}
|
||||
|
||||
resp := &http.Response{
|
||||
Body: io.NopCloser(strings.NewReader(sb.String())),
|
||||
}
|
||||
|
||||
result, totalLines, respOut, err := ProcessResponseAsRingBufferToEnd(resp, 5)
|
||||
if respOut != nil && respOut.Body != nil {
|
||||
defer respOut.Body.Close()
|
||||
}
|
||||
require.NoError(t, err)
|
||||
// 10 lines before + 1 long line + 10 lines after = 21 total
|
||||
assert.Equal(t, 21, totalLines)
|
||||
// Should only have the last 5 lines (line16 through line20)
|
||||
assert.Contains(t, result, "line16")
|
||||
assert.Contains(t, result, "line17")
|
||||
assert.Contains(t, result, "line18")
|
||||
assert.Contains(t, result, "line19")
|
||||
assert.Contains(t, result, "line20")
|
||||
// Should NOT contain earlier lines
|
||||
assert.NotContains(t, result, "line1\n")
|
||||
assert.NotContains(t, result, "line10\n")
|
||||
// The truncated line should not be in the last 5
|
||||
assert.NotContains(t, result, "TRUNCATED")
|
||||
})
|
||||
|
||||
t.Run("empty response body", func(t *testing.T) {
|
||||
resp := &http.Response{
|
||||
Body: io.NopCloser(strings.NewReader("")),
|
||||
}
|
||||
|
||||
result, totalLines, respOut, err := ProcessResponseAsRingBufferToEnd(resp, 10)
|
||||
if respOut != nil && respOut.Body != nil {
|
||||
defer respOut.Body.Close()
|
||||
}
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, totalLines)
|
||||
assert.Equal(t, "", result)
|
||||
})
|
||||
|
||||
t.Run("line at exactly maxLineSize boundary", func(t *testing.T) {
|
||||
// Create a line at exactly maxLineSize (10MB) - should be truncated
|
||||
exactLine := strings.Repeat("z", 10*1024*1024)
|
||||
body := "before\n" + exactLine + "\nafter\n"
|
||||
resp := &http.Response{
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
}
|
||||
|
||||
result, totalLines, respOut, err := ProcessResponseAsRingBufferToEnd(resp, 10)
|
||||
if respOut != nil && respOut.Body != nil {
|
||||
defer respOut.Body.Close()
|
||||
}
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 3, totalLines)
|
||||
assert.Contains(t, result, "before")
|
||||
assert.Contains(t, result, "TRUNCATED")
|
||||
assert.Contains(t, result, "after")
|
||||
})
|
||||
|
||||
t.Run("ring buffer keeps truncated line when in last N", func(t *testing.T) {
|
||||
// Long line followed by only 2 more lines, with ring buffer size 5
|
||||
longLine := strings.Repeat("y", 11*1024*1024)
|
||||
body := "line1\nline2\nline3\n" + longLine + "\nlineA\nlineB\n"
|
||||
resp := &http.Response{
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
}
|
||||
|
||||
result, totalLines, respOut, err := ProcessResponseAsRingBufferToEnd(resp, 5)
|
||||
if respOut != nil && respOut.Body != nil {
|
||||
defer respOut.Body.Close()
|
||||
}
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 6, totalLines)
|
||||
// Last 5: line2, line3, truncated, lineA, lineB
|
||||
assert.Contains(t, result, "line2")
|
||||
assert.Contains(t, result, "line3")
|
||||
assert.Contains(t, result, "TRUNCATED")
|
||||
assert.Contains(t, result, "lineA")
|
||||
assert.Contains(t, result, "lineB")
|
||||
// line1 should be rotated out
|
||||
assert.NotContains(t, result, "line1")
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user