wehub-resource-sync
134 lines
3.5 KiB
Go
134 lines
3.5 KiB
Go
package buffer
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
)
|
|
|
|
// maxLineSize is the maximum size for a single log line (10MB).
|
|
// GitHub Actions logs can contain extremely long lines (base64 content, minified JS, etc.)
|
|
const maxLineSize = 10 * 1024 * 1024
|
|
|
|
// ProcessResponseAsRingBufferToEnd reads the body of an HTTP response line by line,
|
|
// storing only the last maxJobLogLines lines using a ring buffer (sliding window).
|
|
// This efficiently retains the most recent lines, overwriting older ones as needed.
|
|
//
|
|
// Parameters:
|
|
//
|
|
// httpResp: The HTTP response whose body will be read.
|
|
// maxJobLogLines: The maximum number of log lines to retain.
|
|
//
|
|
// Returns:
|
|
//
|
|
// string: The concatenated log lines (up to maxJobLogLines), separated by newlines.
|
|
// int: The total number of lines read from the response.
|
|
// *http.Response: The original HTTP response.
|
|
// error: Any error encountered during reading.
|
|
//
|
|
// The function uses a ring buffer to efficiently store only the last maxJobLogLines lines.
|
|
// If the response contains more lines than maxJobLogLines, only the most recent lines are kept.
|
|
// Lines exceeding maxLineSize are truncated with a marker.
|
|
func ProcessResponseAsRingBufferToEnd(httpResp *http.Response, maxJobLogLines int) (string, int, *http.Response, error) {
|
|
if maxJobLogLines <= 0 {
|
|
maxJobLogLines = 500
|
|
}
|
|
if maxJobLogLines > 100000 {
|
|
maxJobLogLines = 100000
|
|
}
|
|
|
|
lines := make([]string, maxJobLogLines)
|
|
validLines := make([]bool, maxJobLogLines)
|
|
totalLines := 0
|
|
writeIndex := 0
|
|
|
|
const readBufferSize = 64 * 1024 // 64KB read buffer
|
|
const maxDisplayLength = 1000 // Keep first 1000 chars of truncated lines
|
|
|
|
readBuf := make([]byte, readBufferSize)
|
|
var currentLine strings.Builder
|
|
lineTruncated := false
|
|
|
|
// storeLine saves the current line to the ring buffer and resets state
|
|
storeLine := func() {
|
|
line := currentLine.String()
|
|
if lineTruncated && len(line) > maxDisplayLength {
|
|
line = line[:maxDisplayLength]
|
|
}
|
|
if lineTruncated {
|
|
line += "... [TRUNCATED]"
|
|
}
|
|
lines[writeIndex] = line
|
|
validLines[writeIndex] = true
|
|
totalLines++
|
|
writeIndex = (writeIndex + 1) % maxJobLogLines
|
|
currentLine.Reset()
|
|
lineTruncated = false
|
|
}
|
|
|
|
// accumulate adds bytes to currentLine up to maxLineSize, sets lineTruncated if exceeded
|
|
accumulate := func(data []byte) {
|
|
if lineTruncated {
|
|
return
|
|
}
|
|
remaining := maxLineSize - currentLine.Len()
|
|
if remaining <= 0 {
|
|
lineTruncated = true
|
|
return
|
|
}
|
|
if remaining > len(data) {
|
|
remaining = len(data)
|
|
}
|
|
currentLine.Write(data[:remaining])
|
|
if currentLine.Len() >= maxLineSize {
|
|
lineTruncated = true
|
|
}
|
|
}
|
|
|
|
for {
|
|
n, err := httpResp.Body.Read(readBuf)
|
|
if n > 0 {
|
|
chunk := readBuf[:n]
|
|
for len(chunk) > 0 {
|
|
newlineIdx := bytes.IndexByte(chunk, '\n')
|
|
if newlineIdx < 0 {
|
|
accumulate(chunk)
|
|
break
|
|
}
|
|
accumulate(chunk[:newlineIdx])
|
|
storeLine()
|
|
chunk = chunk[newlineIdx+1:]
|
|
}
|
|
}
|
|
|
|
if err == io.EOF {
|
|
if currentLine.Len() > 0 {
|
|
storeLine()
|
|
}
|
|
break
|
|
}
|
|
if err != nil {
|
|
return "", 0, httpResp, fmt.Errorf("failed to read log content: %w", err)
|
|
}
|
|
}
|
|
|
|
var result []string
|
|
linesInBuffer := min(totalLines, maxJobLogLines)
|
|
|
|
startIndex := 0
|
|
if totalLines > maxJobLogLines {
|
|
startIndex = writeIndex
|
|
}
|
|
|
|
for i := range linesInBuffer {
|
|
idx := (startIndex + i) % maxJobLogLines
|
|
if validLines[idx] {
|
|
result = append(result, lines[idx])
|
|
}
|
|
}
|
|
|
|
return strings.Join(result, "\n"), totalLines, httpResp, nil
|
|
}
|