Compare commits
11 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 0cffea568a | |||
| b5fb185b86 | |||
| d19433e412 | |||
| ebe66ab535 | |||
| d59b7e1dff | |||
| d59c85afc8 | |||
| dab6c11ea8 | |||
| 01e75b87ec | |||
| 5d8057473b | |||
| d81b90b38c | |||
| c4895d5b92 |
@@ -38,7 +38,7 @@ jobs:
|
|||||||
id: chrome
|
id: chrome
|
||||||
- name: gofmt
|
- name: gofmt
|
||||||
run: |
|
run: |
|
||||||
unformatted=$(gofmt -s -l asset browser cli clone cmd robots sanitize urlx)
|
unformatted=$(gofmt -s -l .)
|
||||||
if [ -n "$unformatted" ]; then
|
if [ -n "$unformatted" ]; then
|
||||||
echo "These files need gofmt -s -w:"
|
echo "These files need gofmt -s -w:"
|
||||||
echo "$unformatted"
|
echo "$unformatted"
|
||||||
@@ -48,9 +48,15 @@ jobs:
|
|||||||
run: go vet ./...
|
run: go vet ./...
|
||||||
- name: build
|
- name: build
|
||||||
run: go build ./...
|
run: go build ./...
|
||||||
|
# The GitHub Ubuntu runner disables unprivileged user namespaces (AppArmor),
|
||||||
|
# so Chrome's sandbox cannot initialize there and the secure default would
|
||||||
|
# make it refuse to start. IN_DOCKER is kage's documented escape hatch for
|
||||||
|
# exactly that case, and setting it here also exercises the container path.
|
||||||
|
# macOS does not need it, so it stays empty on that leg.
|
||||||
- name: test
|
- name: test
|
||||||
env:
|
env:
|
||||||
KAGE_CHROME: ${{ steps.chrome.outputs.chrome-path }}
|
KAGE_CHROME: ${{ steps.chrome.outputs.chrome-path }}
|
||||||
|
IN_DOCKER: ${{ matrix.os == 'ubuntu-latest' && '1' || '' }}
|
||||||
run: go test -race -count=1 -coverprofile=coverage.out ./...
|
run: go test -race -count=1 -coverprofile=coverage.out ./...
|
||||||
- name: coverage summary
|
- name: coverage summary
|
||||||
if: matrix.os == 'ubuntu-latest'
|
if: matrix.os == 'ubuntu-latest'
|
||||||
@@ -101,3 +107,21 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
go mod tidy
|
go mod tidy
|
||||||
git diff --exit-code -- go.mod go.sum
|
git diff --exit-code -- go.mod go.sum
|
||||||
|
|
||||||
|
# Compile the optional native-window viewer (-tags webview, cgo) so that path
|
||||||
|
# keeps building. The default CI build is pure Go and never touches it. The
|
||||||
|
# viewer code is the same Go on every OS, only the system WebView library
|
||||||
|
# differs, so a macOS compile (WebKit ships in the SDK) catches our
|
||||||
|
# regressions without the WebKitGTK version juggling Linux runners need. It is
|
||||||
|
# build-only: actually opening a window needs a display.
|
||||||
|
webview:
|
||||||
|
runs-on: macos-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v5
|
||||||
|
- uses: actions/setup-go@v6
|
||||||
|
with:
|
||||||
|
go-version-file: go.mod
|
||||||
|
check-latest: true
|
||||||
|
cache: true
|
||||||
|
- name: build webview viewer
|
||||||
|
run: CGO_ENABLED=1 go build -tags webview ./cmd/kage
|
||||||
|
|||||||
+21
-1
@@ -6,6 +6,25 @@ All notable changes to kage are recorded here. The format follows
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
## [0.1.2] - 2026-06-15
|
||||||
|
|
||||||
|
### Security
|
||||||
|
|
||||||
|
- Chrome now keeps its sandbox on by default. It was previously launched with `--no-sandbox` unconditionally, which removed Chrome's main line of defense when rendering pages from the open web (reported in #10). The sandbox is now dropped only where it genuinely cannot run: inside a container, or when running as root, and the choice is logged so it is never silent.
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- Container-aware Chrome flags. kage detects a container from the `IN_DOCKER` environment variable or a `/.dockerenv` marker and, only there, drops the sandbox and adds `--disable-dev-shm-usage` (the default 64 MB `/dev/shm` is too small for Chrome on large pages). Outside a container the faster shared memory is left in place.
|
||||||
|
- Asset downloads retry on a transient failure (a 403/429, a 5xx, or a network blip) with a short backoff, recovering files that bot-protection rejects on the first request of a burst. Permanent failures (404, 401, ...) are not retried.
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- Clearer crawl error reporting. Each failure is logged with a classified reason (`HTTP 403 Forbidden`, `timed out`, ...), the URL, and the page that referenced it, and the end-of-run summary lists what went wrong instead of printing only a count.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- The container image now runs. Chrome aborted on launch with `chrome_crashpad_handler: --database is required`, so kage disables Chrome's crash reporter inside a container, and the `kage` user now has a writable home (the mounted `/out` volume) so the default output, resume state, and Chrome's profile no longer fail with a permission error (issue #7).
|
||||||
|
|
||||||
## [0.1.1] - 2026-06-14
|
## [0.1.1] - 2026-06-14
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
@@ -65,6 +84,7 @@ can browse offline, with every script stripped out.
|
|||||||
a multi-arch container image on GHCR (Chromium bundled), checksums, SBOMs, and
|
a multi-arch container image on GHCR (Chromium bundled), checksums, SBOMs, and
|
||||||
a cosign signature, all cut from one version tag by GoReleaser.
|
a cosign signature, all cut from one version tag by GoReleaser.
|
||||||
|
|
||||||
[Unreleased]: https://github.com/tamnd/kage/compare/v0.1.1...HEAD
|
[Unreleased]: https://github.com/tamnd/kage/compare/v0.1.2...HEAD
|
||||||
|
[0.1.2]: https://github.com/tamnd/kage/compare/v0.1.1...v0.1.2
|
||||||
[0.1.1]: https://github.com/tamnd/kage/compare/v0.1.0...v0.1.1
|
[0.1.1]: https://github.com/tamnd/kage/compare/v0.1.0...v0.1.1
|
||||||
[0.1.0]: https://github.com/tamnd/kage/releases/tag/v0.1.0
|
[0.1.0]: https://github.com/tamnd/kage/releases/tag/v0.1.0
|
||||||
|
|||||||
+8
-1
@@ -29,7 +29,14 @@ WORKDIR /out
|
|||||||
# Point kage at the bundled Chromium and write mirrors under /out by default:
|
# Point kage at the bundled Chromium and write mirrors under /out by default:
|
||||||
#
|
#
|
||||||
# docker run -v "$PWD/out:/out" ghcr.io/tamnd/kage clone example.com
|
# docker run -v "$PWD/out:/out" ghcr.io/tamnd/kage clone example.com
|
||||||
ENV KAGE_CHROME=/usr/bin/chromium-browser
|
#
|
||||||
|
# The kage user has no home directory of its own, so HOME points at the mounted
|
||||||
|
# /out volume. That keeps two things writable: kage's default output and resume
|
||||||
|
# state (it lands under $HOME/data/kage), and Chrome's profile and crash
|
||||||
|
# database. Without this both fail with a permission error in the container
|
||||||
|
# (issue #7), and the mounted volume captures nothing.
|
||||||
|
ENV KAGE_CHROME=/usr/bin/chromium-browser \
|
||||||
|
HOME=/out
|
||||||
|
|
||||||
VOLUME ["/out"]
|
VOLUME ["/out"]
|
||||||
|
|
||||||
|
|||||||
@@ -149,7 +149,7 @@ kage pack paulgraham.com --format binary -o paulgraham
|
|||||||
./paulgraham
|
./paulgraham
|
||||||
```
|
```
|
||||||
|
|
||||||
The appended archive is platform-independent; only the base executable carries the architecture. By default kage appends to itself, so you get a viewer for the machine you ran it on. Point `--base` at a kage built for another OS to produce a viewer for that platform from your own machine:
|
The appended archive is platform-independent; only the base executable carries the architecture. By default kage appends to itself, so you get a viewer for the machine you ran it on. Point `--base` at a kage built for another OS (grab one from a [release](https://github.com/tamnd/kage/releases); every platform ships one) to produce a viewer for that platform from your own machine. kage reads the base's executable header to figure out the target, so a Windows viewer automatically gets a `.exe` name:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Sitting on a Mac, build a Windows viewer
|
# Sitting on a Mac, build a Windows viewer
|
||||||
|
|||||||
+81
-1
@@ -2,6 +2,7 @@ package asset
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -18,6 +19,7 @@ type Downloader struct {
|
|||||||
Client *http.Client
|
Client *http.Client
|
||||||
UserAgent string
|
UserAgent string
|
||||||
MaxBytes int64 // per-asset cap; 0 = unlimited
|
MaxBytes int64 // per-asset cap; 0 = unlimited
|
||||||
|
Retries int // extra attempts for a transient failure (0 = try once)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewDownloader builds a Downloader with a sane client and the given timeout.
|
// NewDownloader builds a Downloader with a sane client and the given timeout.
|
||||||
@@ -26,6 +28,10 @@ func NewDownloader(userAgent string, timeout time.Duration, maxBytes int64) *Dow
|
|||||||
Client: &http.Client{Timeout: timeout},
|
Client: &http.Client{Timeout: timeout},
|
||||||
UserAgent: userAgent,
|
UserAgent: userAgent,
|
||||||
MaxBytes: maxBytes,
|
MaxBytes: maxBytes,
|
||||||
|
// A few sites (and the bot-protection in front of them) reject the first
|
||||||
|
// request of a burst with a 403 or 429 but serve a retry fine, so give
|
||||||
|
// transient failures a couple of extra tries before giving up.
|
||||||
|
Retries: 3,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -36,9 +42,55 @@ type Result struct {
|
|||||||
IsCSS bool
|
IsCSS bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StatusError reports a non-2xx HTTP response. It carries the code so callers
|
||||||
|
// can render a clear message ("HTTP 403 Forbidden") and decide whether a retry
|
||||||
|
// is worthwhile, without the URL baked in (the caller already has it).
|
||||||
|
type StatusError struct {
|
||||||
|
Code int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *StatusError) Error() string {
|
||||||
|
if t := http.StatusText(e.Code); t != "" {
|
||||||
|
return fmt.Sprintf("HTTP %d %s", e.Code, t)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("HTTP %d", e.Code)
|
||||||
|
}
|
||||||
|
|
||||||
// Get fetches u, sending referer as the Referer header. It reads at most
|
// Get fetches u, sending referer as the Referer header. It reads at most
|
||||||
// MaxBytes and reports whether the body is CSS (so the caller can rewrite it).
|
// MaxBytes and reports whether the body is CSS (so the caller can rewrite it).
|
||||||
|
// A transient failure (a 403/429/5xx or a network blip) is retried with a short
|
||||||
|
// backoff up to Retries times.
|
||||||
func (d *Downloader) Get(ctx context.Context, u *url.URL, referer string) (*Result, error) {
|
func (d *Downloader) Get(ctx context.Context, u *url.URL, referer string) (*Result, error) {
|
||||||
|
attempts := d.Retries + 1
|
||||||
|
if attempts < 1 {
|
||||||
|
attempts = 1
|
||||||
|
}
|
||||||
|
var lastErr error
|
||||||
|
for i := 0; i < attempts; i++ {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if i > 0 {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
case <-time.After(backoff(i)):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
res, err := d.try(ctx, u, referer)
|
||||||
|
if err == nil {
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
|
lastErr = err
|
||||||
|
if !transient(err) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, lastErr
|
||||||
|
}
|
||||||
|
|
||||||
|
// try performs a single fetch attempt.
|
||||||
|
func (d *Downloader) try(ctx context.Context, u *url.URL, referer string) (*Result, error) {
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -55,7 +107,7 @@ func (d *Downloader) Get(ctx context.Context, u *url.URL, referer string) (*Resu
|
|||||||
}
|
}
|
||||||
defer func() { _ = resp.Body.Close() }()
|
defer func() { _ = resp.Body.Close() }()
|
||||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
return nil, fmt.Errorf("status %d for %s", resp.StatusCode, u)
|
return nil, &StatusError{Code: resp.StatusCode}
|
||||||
}
|
}
|
||||||
var r io.Reader = resp.Body
|
var r io.Reader = resp.Body
|
||||||
if d.MaxBytes > 0 {
|
if d.MaxBytes > 0 {
|
||||||
@@ -73,6 +125,34 @@ func (d *Downloader) Get(ctx context.Context, u *url.URL, referer string) (*Resu
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// backoff returns the pause before retry attempt i (1-based): 500ms, 1s, 2s, …
|
||||||
|
func backoff(i int) time.Duration {
|
||||||
|
d := 500 * time.Millisecond << (i - 1)
|
||||||
|
if max := 5 * time.Second; d > max {
|
||||||
|
d = max
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
|
// transient reports whether an error is worth retrying. Bot-protection statuses
|
||||||
|
// (403/429), request-timeout and too-early (408/425), and 5xx server errors are
|
||||||
|
// transient; other 4xx (404, 401, 410, …) are permanent. A network error is
|
||||||
|
// retried, but a cancelled or expired context is not.
|
||||||
|
func transient(err error) bool {
|
||||||
|
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var se *StatusError
|
||||||
|
if errors.As(err, &se) {
|
||||||
|
switch se.Code {
|
||||||
|
case http.StatusForbidden, http.StatusRequestTimeout, http.StatusTooEarly, http.StatusTooManyRequests:
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return se.Code >= 500
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
// isCSS reports whether a response is a stylesheet, by content-type or by a
|
// isCSS reports whether a response is a stylesheet, by content-type or by a
|
||||||
// .css path when the server sends no useful type.
|
// .css path when the server sends no useful type.
|
||||||
func isCSS(contentType string, u *url.URL) bool {
|
func isCSS(contentType string, u *url.URL) bool {
|
||||||
|
|||||||
@@ -0,0 +1,112 @@
|
|||||||
|
package asset
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/url"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestStatusErrorMessage(t *testing.T) {
|
||||||
|
cases := map[int]string{
|
||||||
|
403: "HTTP 403 Forbidden",
|
||||||
|
404: "HTTP 404 Not Found",
|
||||||
|
999: "HTTP 999",
|
||||||
|
}
|
||||||
|
for code, want := range cases {
|
||||||
|
if got := (&StatusError{Code: code}).Error(); got != want {
|
||||||
|
t.Errorf("StatusError{%d} = %q; want %q", code, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetRetriesTransientThenSucceeds(t *testing.T) {
|
||||||
|
var hits int32
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
// 403 on the first try (like bot-protection), then serve the file.
|
||||||
|
if atomic.AddInt32(&hits, 1) == 1 {
|
||||||
|
w.WriteHeader(http.StatusForbidden)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "text/css")
|
||||||
|
_, _ = w.Write([]byte("body{}"))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
d := NewDownloader("kage-test", 5*time.Second, 0)
|
||||||
|
u, _ := url.Parse(srv.URL + "/style.css")
|
||||||
|
res, err := d.Get(context.Background(), u, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Get after retry: %v", err)
|
||||||
|
}
|
||||||
|
if !res.IsCSS || string(res.Body) != "body{}" {
|
||||||
|
t.Errorf("unexpected result: css=%v body=%q", res.IsCSS, res.Body)
|
||||||
|
}
|
||||||
|
if hits < 2 {
|
||||||
|
t.Errorf("expected a retry; server saw %d hits", hits)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetDoesNotRetryPermanent(t *testing.T) {
|
||||||
|
var hits int32
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
atomic.AddInt32(&hits, 1)
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
d := NewDownloader("kage-test", 5*time.Second, 0)
|
||||||
|
u, _ := url.Parse(srv.URL + "/missing.png")
|
||||||
|
_, err := d.Get(context.Background(), u, "")
|
||||||
|
|
||||||
|
var se *StatusError
|
||||||
|
if !errors.As(err, &se) || se.Code != 404 {
|
||||||
|
t.Fatalf("got %v; want StatusError 404", err)
|
||||||
|
}
|
||||||
|
if hits != 1 {
|
||||||
|
t.Errorf("404 should not be retried; server saw %d hits", hits)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetGivesUpAfterRetries(t *testing.T) {
|
||||||
|
var hits int32
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
atomic.AddInt32(&hits, 1)
|
||||||
|
w.WriteHeader(http.StatusTooManyRequests)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
d := NewDownloader("kage-test", 5*time.Second, 0)
|
||||||
|
d.Retries = 2
|
||||||
|
u, _ := url.Parse(srv.URL + "/rate.css")
|
||||||
|
_, err := d.Get(context.Background(), u, "")
|
||||||
|
|
||||||
|
var se *StatusError
|
||||||
|
if !errors.As(err, &se) || se.Code != 429 {
|
||||||
|
t.Fatalf("got %v; want StatusError 429", err)
|
||||||
|
}
|
||||||
|
if hits != 3 { // 1 try + 2 retries
|
||||||
|
t.Errorf("expected 3 attempts, server saw %d", hits)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTransientClassification(t *testing.T) {
|
||||||
|
transientCodes := []int{403, 408, 425, 429, 500, 502, 503}
|
||||||
|
for _, c := range transientCodes {
|
||||||
|
if !transient(&StatusError{Code: c}) {
|
||||||
|
t.Errorf("status %d should be transient", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, c := range []int{400, 401, 404, 410} {
|
||||||
|
if transient(&StatusError{Code: c}) {
|
||||||
|
t.Errorf("status %d should be permanent", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if transient(context.Canceled) {
|
||||||
|
t.Error("context.Canceled should not be transient")
|
||||||
|
}
|
||||||
|
}
|
||||||
+98
-2
@@ -10,6 +10,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"runtime"
|
"runtime"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -130,9 +131,30 @@ func (p *Pool) getBrowser() (*rod.Browser, error) {
|
|||||||
l := launcher.New().
|
l := launcher.New().
|
||||||
Headless(p.opts.Headless).
|
Headless(p.opts.Headless).
|
||||||
Set("disable-blink-features", "AutomationControlled").
|
Set("disable-blink-features", "AutomationControlled").
|
||||||
Set("disable-dev-shm-usage", "").
|
|
||||||
Set("no-sandbox", "").
|
|
||||||
Set("disable-gpu", "")
|
Set("disable-gpu", "")
|
||||||
|
|
||||||
|
// Chrome's sandbox is the main line of defense when rendering pages from
|
||||||
|
// the open web, so kage keeps it on by default (issue #10). It is dropped
|
||||||
|
// only where it genuinely cannot initialize: inside a container, or when
|
||||||
|
// running as root, where Chrome otherwise refuses to start. The decision
|
||||||
|
// is logged so it is never silent.
|
||||||
|
if off, reason := disableSandbox(); off {
|
||||||
|
l = l.Set("no-sandbox", "")
|
||||||
|
warnSandboxDisabled(reason)
|
||||||
|
}
|
||||||
|
|
||||||
|
// In a container, the default /dev/shm is only 64 MB, too small for
|
||||||
|
// Chrome's renderer on large pages, so steer it to a temp file instead.
|
||||||
|
// Outside a container /dev/shm is roomy and faster, so leave it alone.
|
||||||
|
// Chrome's crashpad handler also aborts with "--database is required" in a
|
||||||
|
// minimal container, which fails the whole launch (issue #7), so turn the
|
||||||
|
// crash reporter off there. kage never uploads Chrome crash dumps anyway.
|
||||||
|
if inContainer() {
|
||||||
|
l = l.Set("disable-dev-shm-usage", "").
|
||||||
|
Set("disable-crash-reporter", "").
|
||||||
|
Set("disable-breakpad", "")
|
||||||
|
}
|
||||||
|
|
||||||
if bin := p.chromeBin(); bin != "" {
|
if bin := p.chromeBin(); bin != "" {
|
||||||
l = l.Bin(bin)
|
l = l.Bin(bin)
|
||||||
}
|
}
|
||||||
@@ -225,6 +247,80 @@ func systemChromeCandidates() []string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// disableSandbox decides whether Chrome should launch without its sandbox, with
|
||||||
|
// a short reason for the log. The secure default is to keep the sandbox on; it
|
||||||
|
// is dropped only where it cannot run: inside a container, or when running as
|
||||||
|
// root (Chrome refuses to start a sandbox as root).
|
||||||
|
func disableSandbox() (off bool, reason string) {
|
||||||
|
if inContainer() {
|
||||||
|
return true, "container"
|
||||||
|
}
|
||||||
|
if isRoot() {
|
||||||
|
return true, "root"
|
||||||
|
}
|
||||||
|
return false, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// warnSandboxDisabled prints why the sandbox was turned off, so dropping a
|
||||||
|
// security boundary is always visible rather than silent.
|
||||||
|
func warnSandboxDisabled(reason string) {
|
||||||
|
switch reason {
|
||||||
|
case "container":
|
||||||
|
fmt.Fprintln(os.Stderr, "kage: container detected, Chrome sandbox disabled")
|
||||||
|
case "root":
|
||||||
|
fmt.Fprintln(os.Stderr, "kage: running as root, Chrome sandbox disabled (run as a non-root user to keep it on)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// inContainer reports whether kage is running inside a container, where Chrome
|
||||||
|
// needs container-specific flags. It honors IN_DOCKER (set it in your image)
|
||||||
|
// and the /.dockerenv marker that Docker writes into every container.
|
||||||
|
//
|
||||||
|
// Keeping the sandbox on by default and dropping it only here was prompted by
|
||||||
|
// Dimitrios Prasakis (issue #10); the IN_DOCKER opt-in was suggested on Hacker
|
||||||
|
// News (https://news.ycombinator.com/item?id=48534865). Thanks to both.
|
||||||
|
func inContainer() bool {
|
||||||
|
if envTrue("IN_DOCKER") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if _, err := os.Stat("/.dockerenv"); err == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// isRoot reports whether the process runs as the superuser. On Windows
|
||||||
|
// os.Geteuid returns -1, so this is false there.
|
||||||
|
func isRoot() bool {
|
||||||
|
return os.Geteuid() == 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// envTrue reports whether the named environment variable is set to a truthy
|
||||||
|
// value.
|
||||||
|
func envTrue(name string) bool {
|
||||||
|
v, ok := envBool(name)
|
||||||
|
return ok && v
|
||||||
|
}
|
||||||
|
|
||||||
|
// envBool parses a boolean-ish environment variable. It returns ok=false when
|
||||||
|
// the variable is unset or empty. "1", "true", "yes", "on" are true and "0",
|
||||||
|
// "false", "no", "off" are false (case-insensitive); any other non-empty value
|
||||||
|
// counts as true, so IN_DOCKER=docker reads as set.
|
||||||
|
func envBool(name string) (val, ok bool) {
|
||||||
|
s := strings.TrimSpace(os.Getenv(name))
|
||||||
|
if s == "" {
|
||||||
|
return false, false
|
||||||
|
}
|
||||||
|
switch strings.ToLower(s) {
|
||||||
|
case "1", "true", "yes", "on":
|
||||||
|
return true, true
|
||||||
|
case "0", "false", "no", "off":
|
||||||
|
return false, true
|
||||||
|
default:
|
||||||
|
return true, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// settle waits for the network to go quiet for d, recovering from any rod
|
// settle waits for the network to go quiet for d, recovering from any rod
|
||||||
// panic and capping the wait so a chatty page can never hang the worker.
|
// panic and capping the wait so a chatty page can never hang the worker.
|
||||||
func settle(page *rod.Page, d time.Duration) {
|
func settle(page *rod.Page, d time.Duration) {
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -17,6 +18,70 @@ func TestLookChromeReadsEnv(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestEnvBool(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
in string
|
||||||
|
set bool
|
||||||
|
wantVal bool
|
||||||
|
wantOk bool
|
||||||
|
}{
|
||||||
|
{"", false, false, false},
|
||||||
|
{"1", true, true, true},
|
||||||
|
{"true", true, true, true},
|
||||||
|
{"TRUE", true, true, true},
|
||||||
|
{"yes", true, true, true},
|
||||||
|
{"on", true, true, true},
|
||||||
|
{"0", true, false, true},
|
||||||
|
{"false", true, false, true},
|
||||||
|
{"off", true, false, true},
|
||||||
|
{"no", true, false, true},
|
||||||
|
{"docker", true, true, true}, // any other non-empty value is true
|
||||||
|
{" true ", true, true, true}, // trimmed
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
if c.set {
|
||||||
|
t.Setenv("KAGE_TEST_BOOL", c.in)
|
||||||
|
} else {
|
||||||
|
_ = os.Unsetenv("KAGE_TEST_BOOL")
|
||||||
|
}
|
||||||
|
val, ok := envBool("KAGE_TEST_BOOL")
|
||||||
|
if val != c.wantVal || ok != c.wantOk {
|
||||||
|
t.Errorf("envBool(%q) = (%v, %v); want (%v, %v)", c.in, val, ok, c.wantVal, c.wantOk)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDisableSandboxDefaultKeepsItOn(t *testing.T) {
|
||||||
|
// Not in a container and not root, the sandbox stays on. (When the test
|
||||||
|
// itself runs as root, e.g. some CI containers, "root" is the honest
|
||||||
|
// reason; accept that rather than asserting a false negative.)
|
||||||
|
t.Setenv("IN_DOCKER", "")
|
||||||
|
off, reason := disableSandbox()
|
||||||
|
if isRoot() || inContainer() {
|
||||||
|
if !off {
|
||||||
|
t.Errorf("disableSandbox() = false as root/container; want true")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if off {
|
||||||
|
t.Errorf("disableSandbox() = true (%q) on a normal host; want sandbox kept on", reason)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInContainerHonorsEnv(t *testing.T) {
|
||||||
|
t.Setenv("IN_DOCKER", "1")
|
||||||
|
if !inContainer() {
|
||||||
|
t.Errorf("inContainer() = false with IN_DOCKER=1; want true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDisableSandboxContainer(t *testing.T) {
|
||||||
|
t.Setenv("IN_DOCKER", "true")
|
||||||
|
if off, reason := disableSandbox(); !off || reason != "container" {
|
||||||
|
t.Errorf("in container: got (%v, %q); want (true, container)", off, reason)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRenderCapturesFinalDOM(t *testing.T) {
|
func TestRenderCapturesFinalDOM(t *testing.T) {
|
||||||
if testing.Short() {
|
if testing.Short() {
|
||||||
t.Skip("render test drives Chrome; skipped under -short")
|
t.Skip("render test drives Chrome; skipped under -short")
|
||||||
|
|||||||
@@ -186,9 +186,27 @@ func printSummary(res clone.Result) {
|
|||||||
styleAccent.Render("assets"), res.Assets)
|
styleAccent.Render("assets"), res.Assets)
|
||||||
if res.PageErrors+res.AssetErrors > 0 {
|
if res.PageErrors+res.AssetErrors > 0 {
|
||||||
fmt.Fprintf(os.Stderr, " %s %d\n", styleErr.Render("errors"), res.PageErrors+res.AssetErrors)
|
fmt.Fprintf(os.Stderr, " %s %d\n", styleErr.Render("errors"), res.PageErrors+res.AssetErrors)
|
||||||
|
printFailures(res)
|
||||||
}
|
}
|
||||||
if res.Skipped > 0 {
|
if res.Skipped > 0 {
|
||||||
fmt.Fprintf(os.Stderr, " %s %d\n", styleWarn.Render("skipped"), res.Skipped)
|
fmt.Fprintf(os.Stderr, " %s %d\n", styleWarn.Render("skipped"), res.Skipped)
|
||||||
}
|
}
|
||||||
fmt.Fprintf(os.Stderr, " open %s\n", styleAccent.Render("kage serve "+res.OutDir))
|
fmt.Fprintf(os.Stderr, " open %s\n", styleAccent.Render("kage serve "+res.OutDir))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// printFailures lists what went wrong, grouped reason and URL, so the error
|
||||||
|
// count is actionable instead of opaque. The list is capped during the crawl;
|
||||||
|
// when it overflows, say how many more there were.
|
||||||
|
func printFailures(res clone.Result) {
|
||||||
|
total := res.PageErrors + res.AssetErrors
|
||||||
|
for _, f := range res.Failures {
|
||||||
|
line := fmt.Sprintf(" %s %s", styleErr.Render(f.Reason), f.URL)
|
||||||
|
fmt.Fprintln(os.Stderr, line)
|
||||||
|
if f.Referer != "" {
|
||||||
|
fmt.Fprintln(os.Stderr, styleDim.Render(" referenced by "+f.Referer))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if more := total - int64(len(res.Failures)); more > 0 {
|
||||||
|
fmt.Fprintln(os.Stderr, styleDim.Render(fmt.Sprintf(" ... and %d more", more)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+48
-20
@@ -79,16 +79,22 @@ func runPack(mirrorArg string, f *packFlags) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
target := resolveTargetOS(f.base)
|
||||||
out := f.out
|
out := f.out
|
||||||
if out == "" {
|
if out == "" {
|
||||||
out = defaultBinaryName(dir, f.base)
|
out = defaultBinaryName(dir)
|
||||||
|
}
|
||||||
|
// A Windows viewer must end in .exe to run, whether the name came from
|
||||||
|
// --out or the default, so make sure it does.
|
||||||
|
if target == "windows" && !strings.HasSuffix(strings.ToLower(out), ".exe") {
|
||||||
|
out += ".exe"
|
||||||
}
|
}
|
||||||
path, size, err := pack.BuildBinary(zbytes, pack.BinaryOptions{Out: out, Base: f.base})
|
path, size, err := pack.BuildBinary(zbytes, pack.BinaryOptions{Out: out, Base: f.base})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
printPackResult(path, size)
|
printPackResult(path, size)
|
||||||
printRunHint(path)
|
printRunHint(path, target)
|
||||||
return nil
|
return nil
|
||||||
|
|
||||||
default:
|
default:
|
||||||
@@ -110,27 +116,32 @@ func resolveMirror(arg string) string {
|
|||||||
return arg
|
return arg
|
||||||
}
|
}
|
||||||
|
|
||||||
// defaultBinaryName derives a clean program name from the mirror's host: strip a
|
// defaultBinaryName derives a clean program name from the mirror's host by
|
||||||
// trailing dot-suffix (paulgraham.com -> paulgraham), and append .exe when the
|
// stripping a trailing dot-suffix (paulgraham.com -> paulgraham). The caller
|
||||||
// target is Windows. The target is the running OS unless --base names a binary
|
// appends .exe for Windows targets.
|
||||||
// that looks like it was built for Windows.
|
func defaultBinaryName(dir string) string {
|
||||||
func defaultBinaryName(dir, base string) string {
|
|
||||||
host := filepath.Base(dir)
|
host := filepath.Base(dir)
|
||||||
name := host
|
|
||||||
if i := strings.IndexByte(host, '.'); i > 0 {
|
if i := strings.IndexByte(host, '.'); i > 0 {
|
||||||
name = host[:i]
|
return host[:i]
|
||||||
}
|
}
|
||||||
if windowsTarget(base) {
|
return host
|
||||||
name += ".exe"
|
|
||||||
}
|
|
||||||
return name
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func windowsTarget(base string) bool {
|
// resolveTargetOS reports which OS the packed viewer will run on. With no
|
||||||
|
// --base it is this kage's OS; with one, we sniff the base's executable header
|
||||||
|
// so detection does not hinge on the file being named ".exe". If the header is
|
||||||
|
// unrecognised we fall back to that name heuristic.
|
||||||
|
func resolveTargetOS(base string) string {
|
||||||
if base == "" {
|
if base == "" {
|
||||||
return runtime.GOOS == "windows"
|
return runtime.GOOS
|
||||||
}
|
}
|
||||||
return strings.HasSuffix(strings.ToLower(base), ".exe")
|
if os := pack.SniffOS(base); os != "" {
|
||||||
|
return os
|
||||||
|
}
|
||||||
|
if strings.HasSuffix(strings.ToLower(base), ".exe") {
|
||||||
|
return "windows"
|
||||||
|
}
|
||||||
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
func printPackResult(path string, size int64) {
|
func printPackResult(path string, size int64) {
|
||||||
@@ -138,21 +149,38 @@ func printPackResult(path string, size int64) {
|
|||||||
fmt.Fprintf(os.Stderr, " %s %s\n", styleAccent.Render("size"), humanBytes(size))
|
fmt.Fprintf(os.Stderr, " %s %s\n", styleAccent.Render("size"), humanBytes(size))
|
||||||
}
|
}
|
||||||
|
|
||||||
func printRunHint(path string) {
|
func printRunHint(path, target string) {
|
||||||
rel := path
|
rel := path
|
||||||
if !strings.ContainsAny(path, "/\\") {
|
if !strings.ContainsAny(path, "/\\") {
|
||||||
rel = "./" + path
|
rel = "./" + path
|
||||||
}
|
}
|
||||||
if windowsTarget(path) && runtime.GOOS != "windows" {
|
// A viewer built for another OS cannot run here, so say where it goes
|
||||||
fmt.Fprintf(os.Stderr, " this is a Windows viewer; run %s on Windows\n", styleAccent.Render(filepath.Base(path)))
|
// instead of printing a run command that would not work.
|
||||||
|
if target != "" && target != runtime.GOOS {
|
||||||
|
fmt.Fprintf(os.Stderr, " this is a %s viewer; copy %s to that machine to run it\n",
|
||||||
|
osLabel(target), styleAccent.Render(filepath.Base(path)))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
fmt.Fprintf(os.Stderr, " run %s to view the site offline\n", styleAccent.Render(rel))
|
fmt.Fprintf(os.Stderr, " run %s to view the site offline\n", styleAccent.Render(rel))
|
||||||
if runtime.GOOS == "darwin" {
|
if target == "darwin" {
|
||||||
fmt.Fprintln(os.Stderr, styleDim.Render(" (macOS may quarantine it: xattr -d com.apple.quarantine "+rel+")"))
|
fmt.Fprintln(os.Stderr, styleDim.Render(" (macOS may quarantine it: xattr -d com.apple.quarantine "+rel+")"))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// osLabel turns a GOOS value into a friendly name for the run hint.
|
||||||
|
func osLabel(goos string) string {
|
||||||
|
switch goos {
|
||||||
|
case "windows":
|
||||||
|
return "Windows"
|
||||||
|
case "darwin":
|
||||||
|
return "macOS"
|
||||||
|
case "linux":
|
||||||
|
return "Linux"
|
||||||
|
default:
|
||||||
|
return goos
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// humanBytes renders a byte count in B, KiB, MiB, or GiB.
|
// humanBytes renders a byte count in B, KiB, MiB, or GiB.
|
||||||
func humanBytes(n int64) string {
|
func humanBytes(n int64) string {
|
||||||
const unit = 1024
|
const unit = 1024
|
||||||
|
|||||||
+50
-12
@@ -2,6 +2,7 @@ package clone
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -165,7 +166,7 @@ func (c *Cloner) Run(ctx context.Context) (Result, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
res := Result{Progress: c.stats.snapshot(), OutDir: c.outRoot}
|
res := Result{Progress: c.stats.snapshot(), OutDir: c.outRoot, Failures: c.stats.recordedFailures()}
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return res, ctx.Err()
|
return res, ctx.Err()
|
||||||
}
|
}
|
||||||
@@ -236,15 +237,13 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) {
|
|||||||
|
|
||||||
res, err := c.pool.Render(ctx, j.u.String())
|
res, err := c.pool.Render(ctx, j.u.String())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.stats.pageErrors.Add(1)
|
c.failPage(j.u.String(), fmt.Errorf("render: %w", err))
|
||||||
c.logf("page error %s: %v", j.u, err)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
root, err := html.Parse(strings.NewReader(res.HTML))
|
root, err := html.Parse(strings.NewReader(res.HTML))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.stats.pageErrors.Add(1)
|
c.failPage(j.u.String(), fmt.Errorf("parse: %w", err))
|
||||||
c.logf("parse error %s: %v", j.u, err)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -275,12 +274,11 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) {
|
|||||||
|
|
||||||
var buf strings.Builder
|
var buf strings.Builder
|
||||||
if err := html.Render(&buf, root); err != nil {
|
if err := html.Render(&buf, root); err != nil {
|
||||||
c.stats.pageErrors.Add(1)
|
c.failPage(j.u.String(), fmt.Errorf("render html: %w", err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := c.writeFile(localFile, []byte(buf.String())); err != nil {
|
if err := c.writeFile(localFile, []byte(buf.String())); err != nil {
|
||||||
c.stats.pageErrors.Add(1)
|
c.failPage(j.u.String(), fmt.Errorf("write %s: %w", localFile, err))
|
||||||
c.logf("write error %s: %v", localFile, err)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
c.front.markVisited(key)
|
c.front.markVisited(key)
|
||||||
@@ -295,8 +293,7 @@ func (c *Cloner) processAsset(ctx context.Context, j assetItem) {
|
|||||||
}
|
}
|
||||||
res, err := c.dl.Get(ctx, j.u, j.referer)
|
res, err := c.dl.Get(ctx, j.u, j.referer)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.stats.assetErrors.Add(1)
|
c.failAsset(j.u.String(), j.referer, err)
|
||||||
c.logf("asset error %s: %v", j.u, err)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -312,13 +309,54 @@ func (c *Cloner) processAsset(ctx context.Context, j assetItem) {
|
|||||||
body = asset.RewriteCSS(body, j.u, cssSink)
|
body = asset.RewriteCSS(body, j.u, cssSink)
|
||||||
}
|
}
|
||||||
if err := c.writeFile(localFile, body); err != nil {
|
if err := c.writeFile(localFile, body); err != nil {
|
||||||
c.stats.assetErrors.Add(1)
|
c.failAsset(j.u.String(), j.referer, fmt.Errorf("write %s: %w", localFile, err))
|
||||||
c.logf("write error %s: %v", localFile, err)
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
c.stats.assets.Add(1)
|
c.stats.assets.Add(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// failAsset records and logs a failed asset, naming the page that referenced it
|
||||||
|
// so a 403 or 404 is traceable back to where it came from. The reason is
|
||||||
|
// classified (HTTP status, timeout, or other) for a readable line.
|
||||||
|
func (c *Cloner) failAsset(u, referer string, err error) {
|
||||||
|
c.stats.assetErrors.Add(1)
|
||||||
|
reason := classifyError(err)
|
||||||
|
c.stats.recordFailure(Failure{Kind: "asset", URL: u, Referer: referer, Reason: reason})
|
||||||
|
if referer != "" {
|
||||||
|
c.logf("asset error: %s\n %s\n referenced by %s", reason, u, referer)
|
||||||
|
} else {
|
||||||
|
c.logf("asset error: %s\n %s", reason, u)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// failPage records and logs a failed page.
|
||||||
|
func (c *Cloner) failPage(u string, err error) {
|
||||||
|
c.stats.pageErrors.Add(1)
|
||||||
|
reason := classifyError(err)
|
||||||
|
c.stats.recordFailure(Failure{Kind: "page", URL: u, Reason: reason})
|
||||||
|
c.logf("page error: %s\n %s", reason, u)
|
||||||
|
}
|
||||||
|
|
||||||
|
// classifyError turns an error into a short, human-readable reason for the log
|
||||||
|
// and the final report: an HTTP status with its name, a timeout, a cancellation,
|
||||||
|
// or the underlying message otherwise.
|
||||||
|
func classifyError(err error) string {
|
||||||
|
if err == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
var se *asset.StatusError
|
||||||
|
if errors.As(err, &se) {
|
||||||
|
return se.Error()
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, context.DeadlineExceeded):
|
||||||
|
return "timed out"
|
||||||
|
case errors.Is(err, context.Canceled):
|
||||||
|
return "cancelled"
|
||||||
|
}
|
||||||
|
return err.Error()
|
||||||
|
}
|
||||||
|
|
||||||
// enqueuePage offers a page URL to the frontier, honouring the visited set, the
|
// enqueuePage offers a page URL to the frontier, honouring the visited set, the
|
||||||
// depth cap, and the page budget. It reports whether the page was newly queued.
|
// depth cap, and the page budget. It reports whether the page was newly queued.
|
||||||
func (c *Cloner) enqueuePage(ctx context.Context, u *url.URL, depth int) bool {
|
func (c *Cloner) enqueuePage(ctx context.Context, u *url.URL, depth int) bool {
|
||||||
|
|||||||
+39
-1
@@ -1,6 +1,14 @@
|
|||||||
package clone
|
package clone
|
||||||
|
|
||||||
import "sync/atomic"
|
import (
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
)
|
||||||
|
|
||||||
|
// maxRecordedFailures caps how many individual failures Run keeps for the final
|
||||||
|
// report, so a huge broken site cannot grow the slice without bound. The error
|
||||||
|
// counters still count every failure.
|
||||||
|
const maxRecordedFailures = 100
|
||||||
|
|
||||||
// stats are the live counters of a run, read by the CLI's progress ticker.
|
// stats are the live counters of a run, read by the CLI's progress ticker.
|
||||||
type stats struct {
|
type stats struct {
|
||||||
@@ -9,6 +17,34 @@ type stats struct {
|
|||||||
pageErrors atomic.Int64
|
pageErrors atomic.Int64
|
||||||
assetErrors atomic.Int64
|
assetErrors atomic.Int64
|
||||||
skipped atomic.Int64 // robots-disallowed or out of budget
|
skipped atomic.Int64 // robots-disallowed or out of budget
|
||||||
|
|
||||||
|
muFail sync.Mutex
|
||||||
|
failures []Failure
|
||||||
|
}
|
||||||
|
|
||||||
|
// Failure is one thing that went wrong, kept for the end-of-run report so the
|
||||||
|
// errors are visible as a list rather than only as a count.
|
||||||
|
type Failure struct {
|
||||||
|
Kind string // "page" or "asset"
|
||||||
|
URL string
|
||||||
|
Referer string // the page that referenced it, when known
|
||||||
|
Reason string // e.g. "HTTP 403 Forbidden"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stats) recordFailure(f Failure) {
|
||||||
|
s.muFail.Lock()
|
||||||
|
if len(s.failures) < maxRecordedFailures {
|
||||||
|
s.failures = append(s.failures, f)
|
||||||
|
}
|
||||||
|
s.muFail.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *stats) recordedFailures() []Failure {
|
||||||
|
s.muFail.Lock()
|
||||||
|
defer s.muFail.Unlock()
|
||||||
|
out := make([]Failure, len(s.failures))
|
||||||
|
copy(out, s.failures)
|
||||||
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// Progress is a snapshot of a run for display.
|
// Progress is a snapshot of a run for display.
|
||||||
@@ -34,4 +70,6 @@ func (s *stats) snapshot() Progress {
|
|||||||
type Result struct {
|
type Result struct {
|
||||||
Progress
|
Progress
|
||||||
OutDir string
|
OutDir string
|
||||||
|
// Failures is a capped sample of what went wrong, for the final report.
|
||||||
|
Failures []Failure
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -83,7 +83,7 @@ The window title comes from the archive's title. This build needs cgo and links
|
|||||||
|
|
||||||
### Build a viewer for another platform
|
### Build a viewer for another platform
|
||||||
|
|
||||||
The appended archive is platform-independent; only the base executable carries the architecture. Point `--base` at a kage binary built for another OS (download one from a kage release) to produce a viewer for that platform from your own machine:
|
The appended archive is platform-independent; only the base executable carries the architecture. Point `--base` at a kage binary built for another OS (download one from a kage release; every platform ships one) to produce a viewer for that platform from your own machine. kage reads the base's executable header to detect the target OS, so a Windows viewer automatically gets a `.exe` name and the run hint names the right platform:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# From macOS, build a Windows viewer
|
# From macOS, build a Windows viewer
|
||||||
|
|||||||
@@ -6,6 +6,15 @@ weight: 40
|
|||||||
|
|
||||||
The authoritative, commit-level history lives in [`CHANGELOG.md`](https://github.com/tamnd/kage/blob/main/CHANGELOG.md) and on the [releases page](https://github.com/tamnd/kage/releases). This page summarises each version.
|
The authoritative, commit-level history lives in [`CHANGELOG.md`](https://github.com/tamnd/kage/blob/main/CHANGELOG.md) and on the [releases page](https://github.com/tamnd/kage/releases). This page summarises each version.
|
||||||
|
|
||||||
|
## v0.1.2
|
||||||
|
|
||||||
|
A security fix for how kage launches Chrome, clearer crawl errors, and a container image that actually runs.
|
||||||
|
|
||||||
|
- **Chrome keeps its sandbox on by default.** Earlier versions launched Chrome with `--no-sandbox` on every run, which switched off the browser's main security boundary even on an ordinary desktop where the sandbox works fine ([#10](https://github.com/tamnd/kage/issues/10)). The sandbox now stays on, and is dropped only where it genuinely cannot start: inside a container (detected from `IN_DOCKER` or `/.dockerenv`) or when running as root. Whenever it is dropped, kage says so on stderr, so the choice is never silent.
|
||||||
|
- **Transient asset failures retry.** A download that hits a 403/429, a 5xx, or a network blip is retried with a short backoff, which recovers files that bot-protection rejects on the first request of a burst. Permanent failures like a 404 are not retried.
|
||||||
|
- **Clearer crawl errors.** Each failure now logs a classified reason (`HTTP 403 Forbidden`, `timed out`, ...), the URL, and the page that referenced it, and the end-of-run summary lists what went wrong instead of printing only a count.
|
||||||
|
- **The container image runs.** Chrome aborted in the image with `chrome_crashpad_handler: --database is required`, so the crash reporter is now disabled inside a container, and the `kage` user has a writable home (the mounted `/out` volume) so output, resume state, and Chrome's profile no longer fail with a permission error ([#7](https://github.com/tamnd/kage/issues/7)).
|
||||||
|
|
||||||
## v0.1.1
|
## v0.1.1
|
||||||
|
|
||||||
Packing, so a clone can travel as one file instead of a folder.
|
Packing, so a clone can travel as one file instead of a folder.
|
||||||
|
|||||||
@@ -0,0 +1,57 @@
|
|||||||
|
package pack
|
||||||
|
|
||||||
|
import "os"
|
||||||
|
|
||||||
|
// Executable-format magic numbers, enough to tell the three desktop targets
|
||||||
|
// apart by their first bytes. We only need the family (windows, darwin, linux),
|
||||||
|
// not the architecture, so the smallest distinguishing prefix is plenty.
|
||||||
|
var (
|
||||||
|
magicELF = []byte{0x7f, 'E', 'L', 'F'} // Linux, FreeBSD, and other ELF systems
|
||||||
|
magicPE = []byte{'M', 'Z'} // Windows PE/COFF (DOS stub header)
|
||||||
|
machOLE64 = []byte{0xcf, 0xfa, 0xed, 0xfe} // Mach-O 64-bit, little-endian (amd64, arm64)
|
||||||
|
machOLE32 = []byte{0xce, 0xfa, 0xed, 0xfe} // Mach-O 32-bit, little-endian
|
||||||
|
machOBE64 = []byte{0xfe, 0xed, 0xfa, 0xcf} // Mach-O 64-bit, big-endian
|
||||||
|
machOBE32 = []byte{0xfe, 0xed, 0xfa, 0xce} // Mach-O 32-bit, big-endian
|
||||||
|
machOFat = []byte{0xca, 0xfe, 0xba, 0xbe} // Mach-O universal (fat) binary
|
||||||
|
)
|
||||||
|
|
||||||
|
// SniffOS reads the first bytes of an executable and returns the GOOS family it
|
||||||
|
// was built for: "windows", "darwin", "linux", or "" when the bytes match none
|
||||||
|
// of them. It is how pack decides whether a cross-built viewer needs a .exe
|
||||||
|
// suffix and which run hint to print, without trusting the base's file name.
|
||||||
|
func SniffOS(path string) string {
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
defer func() { _ = f.Close() }()
|
||||||
|
|
||||||
|
head := make([]byte, 4)
|
||||||
|
if _, err := f.ReadAt(head, 0); err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case hasPrefix(head, magicPE):
|
||||||
|
return "windows"
|
||||||
|
case hasPrefix(head, magicELF):
|
||||||
|
return "linux"
|
||||||
|
case hasPrefix(head, machOLE64), hasPrefix(head, machOLE32),
|
||||||
|
hasPrefix(head, machOBE64), hasPrefix(head, machOBE32),
|
||||||
|
hasPrefix(head, machOFat):
|
||||||
|
return "darwin"
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasPrefix(b, prefix []byte) bool {
|
||||||
|
if len(b) < len(prefix) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for i := range prefix {
|
||||||
|
if b[i] != prefix[i] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
package pack
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSniffOS(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
head []byte
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"elf", []byte{0x7f, 'E', 'L', 'F', 0x02, 0x01}, "linux"},
|
||||||
|
{"pe", []byte{'M', 'Z', 0x90, 0x00}, "windows"},
|
||||||
|
{"macho-le64", []byte{0xcf, 0xfa, 0xed, 0xfe}, "darwin"},
|
||||||
|
{"macho-le32", []byte{0xce, 0xfa, 0xed, 0xfe}, "darwin"},
|
||||||
|
{"macho-be64", []byte{0xfe, 0xed, 0xfa, 0xcf}, "darwin"},
|
||||||
|
{"macho-fat", []byte{0xca, 0xfe, 0xba, 0xbe}, "darwin"},
|
||||||
|
{"unknown", []byte{0x00, 0x01, 0x02, 0x03}, ""},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
p := filepath.Join(t.TempDir(), tc.name)
|
||||||
|
if err := os.WriteFile(p, tc.head, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := SniffOS(p); got != tc.want {
|
||||||
|
t.Errorf("SniffOS(%s) = %q, want %q", tc.name, got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSniffOSMissingFile(t *testing.T) {
|
||||||
|
if got := SniffOS(filepath.Join(t.TempDir(), "nope")); got != "" {
|
||||||
|
t.Errorf("SniffOS(missing) = %q, want empty", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user