diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28289b7..1905f04 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,9 +48,15 @@ jobs: run: go vet ./... - name: build run: go build ./... + # The GitHub Ubuntu runner disables unprivileged user namespaces (AppArmor), + # so Chrome's sandbox cannot initialize there and the secure default would + # make it refuse to start. IN_DOCKER is kage's documented escape hatch for + # exactly that case, and setting it here also exercises the container path. + # macOS does not need it, so it stays empty on that leg. - name: test env: KAGE_CHROME: ${{ steps.chrome.outputs.chrome-path }} + IN_DOCKER: ${{ matrix.os == 'ubuntu-latest' && '1' || '' }} run: go test -race -count=1 -coverprofile=coverage.out ./... - name: coverage summary if: matrix.os == 'ubuntu-latest' diff --git a/CHANGELOG.md b/CHANGELOG.md index 2669796..6a15f4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,23 @@ All notable changes to kage are recorded here. The format follows ## [Unreleased] +### Security + +- Chrome now keeps its sandbox on by default. It was previously launched with `--no-sandbox` unconditionally, which removed Chrome's main line of defense when rendering pages from the open web (reported in #10). The sandbox is now dropped only where it genuinely cannot run: inside a container, or when running as root, and the choice is logged so it is never silent. + +### Added + +- Container-aware Chrome flags. kage detects a container from the `IN_DOCKER` environment variable or a `/.dockerenv` marker and, only there, drops the sandbox and adds `--disable-dev-shm-usage` (the default 64 MB `/dev/shm` is too small for Chrome on large pages). Outside a container the faster shared memory is left in place. +- Asset downloads retry on a transient failure (a 403/429, a 5xx, or a network blip) with a short backoff, recovering files that bot-protection rejects on the first request of a burst. Permanent failures (404, 401, ...) are not retried. + +### Changed + +- Clearer crawl error reporting. Each failure is logged with a classified reason (`HTTP 403 Forbidden`, `timed out`, ...), the URL, and the page that referenced it, and the end-of-run summary lists what went wrong instead of printing only a count. + +### Fixed + +- The container image now runs. Chrome aborted on launch with `chrome_crashpad_handler: --database is required`, so kage disables Chrome's crash reporter inside a container, and the `kage` user now has a writable home (the mounted `/out` volume) so the default output, resume state, and Chrome's profile no longer fail with a permission error (issue #7). + ## [0.1.1] - 2026-06-14 ### Added diff --git a/Dockerfile b/Dockerfile index 7b20f98..c9347b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,14 @@ WORKDIR /out # Point kage at the bundled Chromium and write mirrors under /out by default: # # docker run -v "$PWD/out:/out" ghcr.io/tamnd/kage clone example.com -ENV KAGE_CHROME=/usr/bin/chromium-browser +# +# The kage user has no home directory of its own, so HOME points at the mounted +# /out volume. That keeps two things writable: kage's default output and resume +# state (it lands under $HOME/data/kage), and Chrome's profile and crash +# database. Without this both fail with a permission error in the container +# (issue #7), and the mounted volume captures nothing. +ENV KAGE_CHROME=/usr/bin/chromium-browser \ + HOME=/out VOLUME ["/out"] diff --git a/asset/download.go b/asset/download.go index cfce9b1..6d6958b 100644 --- a/asset/download.go +++ b/asset/download.go @@ -2,6 +2,7 @@ package asset import ( "context" + "errors" "fmt" "io" "net/http" @@ -18,6 +19,7 @@ type Downloader struct { Client *http.Client UserAgent string MaxBytes int64 // per-asset cap; 0 = unlimited + Retries int // extra attempts for a transient failure (0 = try once) } // NewDownloader builds a Downloader with a sane client and the given timeout. @@ -26,6 +28,10 @@ func NewDownloader(userAgent string, timeout time.Duration, maxBytes int64) *Dow Client: &http.Client{Timeout: timeout}, UserAgent: userAgent, MaxBytes: maxBytes, + // A few sites (and the bot-protection in front of them) reject the first + // request of a burst with a 403 or 429 but serve a retry fine, so give + // transient failures a couple of extra tries before giving up. + Retries: 3, } } @@ -36,9 +42,55 @@ type Result struct { IsCSS bool } +// StatusError reports a non-2xx HTTP response. It carries the code so callers +// can render a clear message ("HTTP 403 Forbidden") and decide whether a retry +// is worthwhile, without the URL baked in (the caller already has it). +type StatusError struct { + Code int +} + +func (e *StatusError) Error() string { + if t := http.StatusText(e.Code); t != "" { + return fmt.Sprintf("HTTP %d %s", e.Code, t) + } + return fmt.Sprintf("HTTP %d", e.Code) +} + // Get fetches u, sending referer as the Referer header. It reads at most // MaxBytes and reports whether the body is CSS (so the caller can rewrite it). +// A transient failure (a 403/429/5xx or a network blip) is retried with a short +// backoff up to Retries times. func (d *Downloader) Get(ctx context.Context, u *url.URL, referer string) (*Result, error) { + attempts := d.Retries + 1 + if attempts < 1 { + attempts = 1 + } + var lastErr error + for i := 0; i < attempts; i++ { + if err := ctx.Err(); err != nil { + return nil, err + } + if i > 0 { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(backoff(i)): + } + } + res, err := d.try(ctx, u, referer) + if err == nil { + return res, nil + } + lastErr = err + if !transient(err) { + break + } + } + return nil, lastErr +} + +// try performs a single fetch attempt. +func (d *Downloader) try(ctx context.Context, u *url.URL, referer string) (*Result, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { return nil, err @@ -55,7 +107,7 @@ func (d *Downloader) Get(ctx context.Context, u *url.URL, referer string) (*Resu } defer func() { _ = resp.Body.Close() }() if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return nil, fmt.Errorf("status %d for %s", resp.StatusCode, u) + return nil, &StatusError{Code: resp.StatusCode} } var r io.Reader = resp.Body if d.MaxBytes > 0 { @@ -73,6 +125,34 @@ func (d *Downloader) Get(ctx context.Context, u *url.URL, referer string) (*Resu }, nil } +// backoff returns the pause before retry attempt i (1-based): 500ms, 1s, 2s, … +func backoff(i int) time.Duration { + d := 500 * time.Millisecond << (i - 1) + if max := 5 * time.Second; d > max { + d = max + } + return d +} + +// transient reports whether an error is worth retrying. Bot-protection statuses +// (403/429), request-timeout and too-early (408/425), and 5xx server errors are +// transient; other 4xx (404, 401, 410, …) are permanent. A network error is +// retried, but a cancelled or expired context is not. +func transient(err error) bool { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + var se *StatusError + if errors.As(err, &se) { + switch se.Code { + case http.StatusForbidden, http.StatusRequestTimeout, http.StatusTooEarly, http.StatusTooManyRequests: + return true + } + return se.Code >= 500 + } + return true +} + // isCSS reports whether a response is a stylesheet, by content-type or by a // .css path when the server sends no useful type. func isCSS(contentType string, u *url.URL) bool { diff --git a/asset/download_test.go b/asset/download_test.go new file mode 100644 index 0000000..d74ffb9 --- /dev/null +++ b/asset/download_test.go @@ -0,0 +1,112 @@ +package asset + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "sync/atomic" + "testing" + "time" +) + +func TestStatusErrorMessage(t *testing.T) { + cases := map[int]string{ + 403: "HTTP 403 Forbidden", + 404: "HTTP 404 Not Found", + 999: "HTTP 999", + } + for code, want := range cases { + if got := (&StatusError{Code: code}).Error(); got != want { + t.Errorf("StatusError{%d} = %q; want %q", code, got, want) + } + } +} + +func TestGetRetriesTransientThenSucceeds(t *testing.T) { + var hits int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // 403 on the first try (like bot-protection), then serve the file. + if atomic.AddInt32(&hits, 1) == 1 { + w.WriteHeader(http.StatusForbidden) + return + } + w.Header().Set("Content-Type", "text/css") + _, _ = w.Write([]byte("body{}")) + })) + defer srv.Close() + + d := NewDownloader("kage-test", 5*time.Second, 0) + u, _ := url.Parse(srv.URL + "/style.css") + res, err := d.Get(context.Background(), u, "") + if err != nil { + t.Fatalf("Get after retry: %v", err) + } + if !res.IsCSS || string(res.Body) != "body{}" { + t.Errorf("unexpected result: css=%v body=%q", res.IsCSS, res.Body) + } + if hits < 2 { + t.Errorf("expected a retry; server saw %d hits", hits) + } +} + +func TestGetDoesNotRetryPermanent(t *testing.T) { + var hits int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&hits, 1) + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + d := NewDownloader("kage-test", 5*time.Second, 0) + u, _ := url.Parse(srv.URL + "/missing.png") + _, err := d.Get(context.Background(), u, "") + + var se *StatusError + if !errors.As(err, &se) || se.Code != 404 { + t.Fatalf("got %v; want StatusError 404", err) + } + if hits != 1 { + t.Errorf("404 should not be retried; server saw %d hits", hits) + } +} + +func TestGetGivesUpAfterRetries(t *testing.T) { + var hits int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&hits, 1) + w.WriteHeader(http.StatusTooManyRequests) + })) + defer srv.Close() + + d := NewDownloader("kage-test", 5*time.Second, 0) + d.Retries = 2 + u, _ := url.Parse(srv.URL + "/rate.css") + _, err := d.Get(context.Background(), u, "") + + var se *StatusError + if !errors.As(err, &se) || se.Code != 429 { + t.Fatalf("got %v; want StatusError 429", err) + } + if hits != 3 { // 1 try + 2 retries + t.Errorf("expected 3 attempts, server saw %d", hits) + } +} + +func TestTransientClassification(t *testing.T) { + transientCodes := []int{403, 408, 425, 429, 500, 502, 503} + for _, c := range transientCodes { + if !transient(&StatusError{Code: c}) { + t.Errorf("status %d should be transient", c) + } + } + for _, c := range []int{400, 401, 404, 410} { + if transient(&StatusError{Code: c}) { + t.Errorf("status %d should be permanent", c) + } + } + if transient(context.Canceled) { + t.Error("context.Canceled should not be transient") + } +} diff --git a/browser/pool.go b/browser/pool.go index aea6dc3..e457ec0 100644 --- a/browser/pool.go +++ b/browser/pool.go @@ -10,6 +10,7 @@ import ( "fmt" "os" "runtime" + "strings" "sync" "time" @@ -130,9 +131,30 @@ func (p *Pool) getBrowser() (*rod.Browser, error) { l := launcher.New(). Headless(p.opts.Headless). Set("disable-blink-features", "AutomationControlled"). - Set("disable-dev-shm-usage", ""). - Set("no-sandbox", ""). Set("disable-gpu", "") + + // Chrome's sandbox is the main line of defense when rendering pages from + // the open web, so kage keeps it on by default (issue #10). It is dropped + // only where it genuinely cannot initialize: inside a container, or when + // running as root, where Chrome otherwise refuses to start. The decision + // is logged so it is never silent. + if off, reason := disableSandbox(); off { + l = l.Set("no-sandbox", "") + warnSandboxDisabled(reason) + } + + // In a container, the default /dev/shm is only 64 MB, too small for + // Chrome's renderer on large pages, so steer it to a temp file instead. + // Outside a container /dev/shm is roomy and faster, so leave it alone. + // Chrome's crashpad handler also aborts with "--database is required" in a + // minimal container, which fails the whole launch (issue #7), so turn the + // crash reporter off there. kage never uploads Chrome crash dumps anyway. + if inContainer() { + l = l.Set("disable-dev-shm-usage", ""). + Set("disable-crash-reporter", ""). + Set("disable-breakpad", "") + } + if bin := p.chromeBin(); bin != "" { l = l.Bin(bin) } @@ -225,6 +247,80 @@ func systemChromeCandidates() []string { } } +// disableSandbox decides whether Chrome should launch without its sandbox, with +// a short reason for the log. The secure default is to keep the sandbox on; it +// is dropped only where it cannot run: inside a container, or when running as +// root (Chrome refuses to start a sandbox as root). +func disableSandbox() (off bool, reason string) { + if inContainer() { + return true, "container" + } + if isRoot() { + return true, "root" + } + return false, "" +} + +// warnSandboxDisabled prints why the sandbox was turned off, so dropping a +// security boundary is always visible rather than silent. +func warnSandboxDisabled(reason string) { + switch reason { + case "container": + fmt.Fprintln(os.Stderr, "kage: container detected, Chrome sandbox disabled") + case "root": + fmt.Fprintln(os.Stderr, "kage: running as root, Chrome sandbox disabled (run as a non-root user to keep it on)") + } +} + +// inContainer reports whether kage is running inside a container, where Chrome +// needs container-specific flags. It honors IN_DOCKER (set it in your image) +// and the /.dockerenv marker that Docker writes into every container. +// +// Keeping the sandbox on by default and dropping it only here was prompted by +// Dimitrios Prasakis (issue #10); the IN_DOCKER opt-in was suggested on Hacker +// News (https://news.ycombinator.com/item?id=48534865). Thanks to both. +func inContainer() bool { + if envTrue("IN_DOCKER") { + return true + } + if _, err := os.Stat("/.dockerenv"); err == nil { + return true + } + return false +} + +// isRoot reports whether the process runs as the superuser. On Windows +// os.Geteuid returns -1, so this is false there. +func isRoot() bool { + return os.Geteuid() == 0 +} + +// envTrue reports whether the named environment variable is set to a truthy +// value. +func envTrue(name string) bool { + v, ok := envBool(name) + return ok && v +} + +// envBool parses a boolean-ish environment variable. It returns ok=false when +// the variable is unset or empty. "1", "true", "yes", "on" are true and "0", +// "false", "no", "off" are false (case-insensitive); any other non-empty value +// counts as true, so IN_DOCKER=docker reads as set. +func envBool(name string) (val, ok bool) { + s := strings.TrimSpace(os.Getenv(name)) + if s == "" { + return false, false + } + switch strings.ToLower(s) { + case "1", "true", "yes", "on": + return true, true + case "0", "false", "no", "off": + return false, true + default: + return true, true + } +} + // settle waits for the network to go quiet for d, recovering from any rod // panic and capping the wait so a chatty page can never hang the worker. func settle(page *rod.Page, d time.Duration) { diff --git a/browser/pool_test.go b/browser/pool_test.go index 9c10856..5287cba 100644 --- a/browser/pool_test.go +++ b/browser/pool_test.go @@ -4,6 +4,7 @@ import ( "context" "net/http" "net/http/httptest" + "os" "strings" "testing" "time" @@ -17,6 +18,70 @@ func TestLookChromeReadsEnv(t *testing.T) { } } +func TestEnvBool(t *testing.T) { + cases := []struct { + in string + set bool + wantVal bool + wantOk bool + }{ + {"", false, false, false}, + {"1", true, true, true}, + {"true", true, true, true}, + {"TRUE", true, true, true}, + {"yes", true, true, true}, + {"on", true, true, true}, + {"0", true, false, true}, + {"false", true, false, true}, + {"off", true, false, true}, + {"no", true, false, true}, + {"docker", true, true, true}, // any other non-empty value is true + {" true ", true, true, true}, // trimmed + } + for _, c := range cases { + if c.set { + t.Setenv("KAGE_TEST_BOOL", c.in) + } else { + _ = os.Unsetenv("KAGE_TEST_BOOL") + } + val, ok := envBool("KAGE_TEST_BOOL") + if val != c.wantVal || ok != c.wantOk { + t.Errorf("envBool(%q) = (%v, %v); want (%v, %v)", c.in, val, ok, c.wantVal, c.wantOk) + } + } +} + +func TestDisableSandboxDefaultKeepsItOn(t *testing.T) { + // Not in a container and not root, the sandbox stays on. (When the test + // itself runs as root, e.g. some CI containers, "root" is the honest + // reason; accept that rather than asserting a false negative.) + t.Setenv("IN_DOCKER", "") + off, reason := disableSandbox() + if isRoot() || inContainer() { + if !off { + t.Errorf("disableSandbox() = false as root/container; want true") + } + return + } + if off { + t.Errorf("disableSandbox() = true (%q) on a normal host; want sandbox kept on", reason) + } +} + +func TestInContainerHonorsEnv(t *testing.T) { + t.Setenv("IN_DOCKER", "1") + if !inContainer() { + t.Errorf("inContainer() = false with IN_DOCKER=1; want true") + } +} + +func TestDisableSandboxContainer(t *testing.T) { + t.Setenv("IN_DOCKER", "true") + if off, reason := disableSandbox(); !off || reason != "container" { + t.Errorf("in container: got (%v, %q); want (true, container)", off, reason) + } +} + func TestRenderCapturesFinalDOM(t *testing.T) { if testing.Short() { t.Skip("render test drives Chrome; skipped under -short") diff --git a/cli/clone.go b/cli/clone.go index 7d221dc..1133223 100644 --- a/cli/clone.go +++ b/cli/clone.go @@ -186,9 +186,27 @@ func printSummary(res clone.Result) { styleAccent.Render("assets"), res.Assets) if res.PageErrors+res.AssetErrors > 0 { fmt.Fprintf(os.Stderr, " %s %d\n", styleErr.Render("errors"), res.PageErrors+res.AssetErrors) + printFailures(res) } if res.Skipped > 0 { fmt.Fprintf(os.Stderr, " %s %d\n", styleWarn.Render("skipped"), res.Skipped) } fmt.Fprintf(os.Stderr, " open %s\n", styleAccent.Render("kage serve "+res.OutDir)) } + +// printFailures lists what went wrong, grouped reason and URL, so the error +// count is actionable instead of opaque. The list is capped during the crawl; +// when it overflows, say how many more there were. +func printFailures(res clone.Result) { + total := res.PageErrors + res.AssetErrors + for _, f := range res.Failures { + line := fmt.Sprintf(" %s %s", styleErr.Render(f.Reason), f.URL) + fmt.Fprintln(os.Stderr, line) + if f.Referer != "" { + fmt.Fprintln(os.Stderr, styleDim.Render(" referenced by "+f.Referer)) + } + } + if more := total - int64(len(res.Failures)); more > 0 { + fmt.Fprintln(os.Stderr, styleDim.Render(fmt.Sprintf(" ... and %d more", more))) + } +} diff --git a/clone/cloner.go b/clone/cloner.go index ce4da92..48486db 100644 --- a/clone/cloner.go +++ b/clone/cloner.go @@ -2,6 +2,7 @@ package clone import ( "context" + "errors" "fmt" "io" "net/http" @@ -165,7 +166,7 @@ func (c *Cloner) Run(ctx context.Context) (Result, error) { } } - res := Result{Progress: c.stats.snapshot(), OutDir: c.outRoot} + res := Result{Progress: c.stats.snapshot(), OutDir: c.outRoot, Failures: c.stats.recordedFailures()} if ctx.Err() != nil { return res, ctx.Err() } @@ -236,15 +237,13 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) { res, err := c.pool.Render(ctx, j.u.String()) if err != nil { - c.stats.pageErrors.Add(1) - c.logf("page error %s: %v", j.u, err) + c.failPage(j.u.String(), fmt.Errorf("render: %w", err)) return } root, err := html.Parse(strings.NewReader(res.HTML)) if err != nil { - c.stats.pageErrors.Add(1) - c.logf("parse error %s: %v", j.u, err) + c.failPage(j.u.String(), fmt.Errorf("parse: %w", err)) return } @@ -275,12 +274,11 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) { var buf strings.Builder if err := html.Render(&buf, root); err != nil { - c.stats.pageErrors.Add(1) + c.failPage(j.u.String(), fmt.Errorf("render html: %w", err)) return } if err := c.writeFile(localFile, []byte(buf.String())); err != nil { - c.stats.pageErrors.Add(1) - c.logf("write error %s: %v", localFile, err) + c.failPage(j.u.String(), fmt.Errorf("write %s: %w", localFile, err)) return } c.front.markVisited(key) @@ -295,8 +293,7 @@ func (c *Cloner) processAsset(ctx context.Context, j assetItem) { } res, err := c.dl.Get(ctx, j.u, j.referer) if err != nil { - c.stats.assetErrors.Add(1) - c.logf("asset error %s: %v", j.u, err) + c.failAsset(j.u.String(), j.referer, err) return } @@ -312,13 +309,54 @@ func (c *Cloner) processAsset(ctx context.Context, j assetItem) { body = asset.RewriteCSS(body, j.u, cssSink) } if err := c.writeFile(localFile, body); err != nil { - c.stats.assetErrors.Add(1) - c.logf("write error %s: %v", localFile, err) + c.failAsset(j.u.String(), j.referer, fmt.Errorf("write %s: %w", localFile, err)) return } c.stats.assets.Add(1) } +// failAsset records and logs a failed asset, naming the page that referenced it +// so a 403 or 404 is traceable back to where it came from. The reason is +// classified (HTTP status, timeout, or other) for a readable line. +func (c *Cloner) failAsset(u, referer string, err error) { + c.stats.assetErrors.Add(1) + reason := classifyError(err) + c.stats.recordFailure(Failure{Kind: "asset", URL: u, Referer: referer, Reason: reason}) + if referer != "" { + c.logf("asset error: %s\n %s\n referenced by %s", reason, u, referer) + } else { + c.logf("asset error: %s\n %s", reason, u) + } +} + +// failPage records and logs a failed page. +func (c *Cloner) failPage(u string, err error) { + c.stats.pageErrors.Add(1) + reason := classifyError(err) + c.stats.recordFailure(Failure{Kind: "page", URL: u, Reason: reason}) + c.logf("page error: %s\n %s", reason, u) +} + +// classifyError turns an error into a short, human-readable reason for the log +// and the final report: an HTTP status with its name, a timeout, a cancellation, +// or the underlying message otherwise. +func classifyError(err error) string { + if err == nil { + return "" + } + var se *asset.StatusError + if errors.As(err, &se) { + return se.Error() + } + switch { + case errors.Is(err, context.DeadlineExceeded): + return "timed out" + case errors.Is(err, context.Canceled): + return "cancelled" + } + return err.Error() +} + // enqueuePage offers a page URL to the frontier, honouring the visited set, the // depth cap, and the page budget. It reports whether the page was newly queued. func (c *Cloner) enqueuePage(ctx context.Context, u *url.URL, depth int) bool { diff --git a/clone/stats.go b/clone/stats.go index 9d461fc..c153640 100644 --- a/clone/stats.go +++ b/clone/stats.go @@ -1,6 +1,14 @@ package clone -import "sync/atomic" +import ( + "sync" + "sync/atomic" +) + +// maxRecordedFailures caps how many individual failures Run keeps for the final +// report, so a huge broken site cannot grow the slice without bound. The error +// counters still count every failure. +const maxRecordedFailures = 100 // stats are the live counters of a run, read by the CLI's progress ticker. type stats struct { @@ -9,6 +17,34 @@ type stats struct { pageErrors atomic.Int64 assetErrors atomic.Int64 skipped atomic.Int64 // robots-disallowed or out of budget + + muFail sync.Mutex + failures []Failure +} + +// Failure is one thing that went wrong, kept for the end-of-run report so the +// errors are visible as a list rather than only as a count. +type Failure struct { + Kind string // "page" or "asset" + URL string + Referer string // the page that referenced it, when known + Reason string // e.g. "HTTP 403 Forbidden" +} + +func (s *stats) recordFailure(f Failure) { + s.muFail.Lock() + if len(s.failures) < maxRecordedFailures { + s.failures = append(s.failures, f) + } + s.muFail.Unlock() +} + +func (s *stats) recordedFailures() []Failure { + s.muFail.Lock() + defer s.muFail.Unlock() + out := make([]Failure, len(s.failures)) + copy(out, s.failures) + return out } // Progress is a snapshot of a run for display. @@ -34,4 +70,6 @@ func (s *stats) snapshot() Progress { type Result struct { Progress OutDir string + // Failures is a capped sample of what went wrong, for the final report. + Failures []Failure }