chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
Executable
+61
@@ -0,0 +1,61 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Script to empty the logs directory except for the last (most recent) log file.
|
||||
|
||||
LOG_DIR="logs"
|
||||
|
||||
# Check if the log directory exists
|
||||
if [ ! -d "$LOG_DIR" ]; then
|
||||
echo "Log directory '$LOG_DIR' not found."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Change to the log directory to make file operations simpler
|
||||
cd "$LOG_DIR" || exit 1
|
||||
|
||||
# Count the number of files in the directory
|
||||
# Using find to correctly handle filenames with spaces or special characters
|
||||
# and to only count regular files (-type f) directly in this directory (-maxdepth 1)
|
||||
num_files=$(find . -maxdepth 1 -type f -print | wc -l)
|
||||
|
||||
if [ "$num_files" -le 1 ]; then
|
||||
echo "No old log files to remove (found $num_files file(s))."
|
||||
# Change back to the original directory before exiting
|
||||
cd - > /dev/null
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# List all files, sort by modification time (newest first), then skip the first one (the newest)
|
||||
# and pass the rest to xargs for deletion.
|
||||
# `ls -t` sorts by modification time.
|
||||
# `tail -n +2` skips the first line.
|
||||
# `xargs -d '\n'` handles filenames with spaces.
|
||||
files_to_delete=$(ls -t | tail -n +2)
|
||||
|
||||
if [ -n "$files_to_delete" ]; then
|
||||
echo "Removing the following old log files:"
|
||||
# Loop through files to print them before deleting
|
||||
echo "$files_to_delete" | while IFS= read -r file_to_delete; do
|
||||
echo " - $file_to_delete"
|
||||
done
|
||||
# Actual deletion
|
||||
for file_to_delete in $files_to_delete; do
|
||||
# Check if the file exists before trying to delete it
|
||||
if [ -e "$file_to_delete" ]; then
|
||||
rm -f "$file_to_delete"
|
||||
else
|
||||
echo "Warning: File '$file_to_delete' does not exist."
|
||||
fi
|
||||
done
|
||||
|
||||
# Count how many were actually passed to xargs (can be tricky if names have newlines)
|
||||
# A simpler way is to count lines from the `files_to_delete` variable
|
||||
num_deleted=$(echo "$files_to_delete" | wc -l)
|
||||
echo "Successfully removed $num_deleted old log file(s)."
|
||||
else
|
||||
echo "No old log files to remove."
|
||||
fi
|
||||
|
||||
# Change back to the original directory
|
||||
cd - > /dev/null
|
||||
exit 0
|
||||
Executable
+84
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
|
||||
# Debug harness for the short-lived-process attribution race.
|
||||
#
|
||||
# Runs rustnet with debug logging IN THE FOREGROUND for ~20s (you will
|
||||
# see the TUI take over; it exits by itself), while a background loop
|
||||
# fires short-lived curl/dig processes. Captures the eBPF bpf_printk
|
||||
# stream from the kernel tracing pipe, then summarizes what the
|
||||
# process-lookup pipeline did for exactly those connections.
|
||||
#
|
||||
# Usage: scripts/debug-attribution.sh (will sudo)
|
||||
|
||||
OUT_DIR="/tmp/claude/attribution-debug"
|
||||
mkdir -p "${OUT_DIR}"
|
||||
rm -f "${OUT_DIR}"/*.log 2>/dev/null
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
sudo -v || exit 1
|
||||
|
||||
# tracefs location differs across distros.
|
||||
TRACE_PIPE="/sys/kernel/tracing/trace_pipe"
|
||||
[[ -e "${TRACE_PIPE}" ]] || TRACE_PIPE="/sys/kernel/debug/tracing/trace_pipe"
|
||||
|
||||
# Capture bpf_printk output (e.g. "map update failed") while the test runs.
|
||||
sudo sh -c "timeout 24 cat ${TRACE_PIPE}" > "${OUT_DIR}/bpf_trace.log" 2>"${OUT_DIR}/bpf_trace.err" &
|
||||
|
||||
# Background traffic: short-lived processes, started after capture warms up.
|
||||
(
|
||||
sleep 6
|
||||
for i in 1 2 3; do
|
||||
curl -s -o /dev/null --max-time 3 https://example.com
|
||||
echo "curl #$i done ($(date +%H:%M:%S.%3N))" >> "${OUT_DIR}/traffic.log"
|
||||
dig +short +time=1 +tries=1 "example.com" @1.1.1.1 >/dev/null 2>&1
|
||||
echo "dig #$i done ($(date +%H:%M:%S.%3N))" >> "${OUT_DIR}/traffic.log"
|
||||
sleep 2
|
||||
done
|
||||
) &
|
||||
TRAFFIC_PID=$!
|
||||
|
||||
echo "Starting rustnet for 20s (TUI will take over this terminal)..."
|
||||
sleep 1
|
||||
sudo sh -c 'timeout 20 target/release/rustnet -l debug' || true
|
||||
|
||||
# rustnet may have been killed mid-frame; restore the terminal.
|
||||
stty sane 2>/dev/null
|
||||
printf '\033[?1049l\033[?25h\033[?1000l\033[?1006l\n'
|
||||
|
||||
wait ${TRAFFIC_PID} 2>/dev/null
|
||||
|
||||
# The logs/ dir is created 0700 by the (privilege-dropped) rustnet
|
||||
# process, so globbing it needs sudo.
|
||||
LOG="$(sudo sh -c 'ls -t logs/rustnet_*.log 2>/dev/null' | head -1)"
|
||||
if [[ -z "${LOG}" ]]; then
|
||||
echo "ERROR: no rustnet log produced; see ${OUT_DIR}/"
|
||||
exit 1
|
||||
fi
|
||||
sudo cp "${LOG}" "${OUT_DIR}/rustnet-debug.log"
|
||||
sudo chown "$(id -u)" "${OUT_DIR}/rustnet-debug.log"
|
||||
LOG="${OUT_DIR}/rustnet-debug.log"
|
||||
|
||||
echo ""
|
||||
echo "=== Summary (${LOG}) ==="
|
||||
echo "-- lookup outcome counts --"
|
||||
echo " eBPF hits: $(grep -c 'Enhanced lookup: eBPF hit' "${LOG}")"
|
||||
echo " eBPF misses: $(grep -c 'Enhanced lookup: eBPF miss' "${LOG}")"
|
||||
echo " zero-source rescues: $(grep -c 'succeeded with zero source' "${LOG}")"
|
||||
echo " procfs name sets: $(grep -c 'Set process name' "${LOG}")"
|
||||
echo ""
|
||||
echo "-- :443 (curl) lookup trace (first 30) --"
|
||||
grep -E "Trying eBPF|eBPF lookup (successful|missed)|Set process name" "${LOG}" | grep ":443" | head -30
|
||||
echo ""
|
||||
echo "-- 1.1.1.1 (dig) lookup trace (first 20) --"
|
||||
grep -E "Trying eBPF|eBPF lookup (successful|missed)|Set process name" "${LOG}" | grep "1\.1\.1\.1" | head -20
|
||||
echo ""
|
||||
echo "-- bpf_printk (kernel side) --"
|
||||
grep -cE "map update failed" "${OUT_DIR}/bpf_trace.log" | sed 's/^/ map update failures: /'
|
||||
wc -l < "${OUT_DIR}/bpf_trace.log" | sed 's/^/ total trace lines: /'
|
||||
echo ""
|
||||
echo "-- traffic timeline --"
|
||||
cat "${OUT_DIR}/traffic.log" 2>/dev/null
|
||||
echo ""
|
||||
echo "Full logs preserved in ${OUT_DIR}/"
|
||||
Executable
+424
@@ -0,0 +1,424 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Enrich RustNet PCAP captures with process and GeoIP information from sidecar JSONL.
|
||||
|
||||
This script correlates packets in a PCAP file with process and GeoIP information
|
||||
from the accompanying .connections.jsonl file created by RustNet.
|
||||
|
||||
NOTE: If you captured using PKTAP on macOS (e.g., `--interface pktap,en0`),
|
||||
the process information is already embedded in the PCAP file itself.
|
||||
You can view it directly in Wireshark without using this script.
|
||||
This script is only needed for regular (non-PKTAP) captures.
|
||||
|
||||
Usage:
|
||||
# Show packets with process info
|
||||
python pcap_enrich.py capture.pcap
|
||||
|
||||
# Export to annotated PCAPNG (requires editcap from Wireshark)
|
||||
python pcap_enrich.py capture.pcap --output annotated.pcapng
|
||||
|
||||
# Generate TSV report
|
||||
python pcap_enrich.py capture.pcap --format tsv > report.tsv
|
||||
|
||||
Requirements:
|
||||
pip install scapy
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from scapy.all import rdpcap, IP, TCP, UDP, ICMP
|
||||
except ImportError:
|
||||
print("Error: scapy is required. Install with: pip install scapy", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def parse_systemtime(st) -> float | None:
|
||||
"""Parse a SystemTime serialized as {secs_since_epoch, nanos_since_epoch}."""
|
||||
if st is None:
|
||||
return None
|
||||
if isinstance(st, dict):
|
||||
secs = st.get("secs_since_epoch", 0)
|
||||
nanos = st.get("nanos_since_epoch", 0)
|
||||
return secs + nanos / 1e9
|
||||
# Fallback for other formats
|
||||
return None
|
||||
|
||||
|
||||
def load_connections(jsonl_path: Path) -> dict:
|
||||
"""Load connection-to-process mappings from JSONL file.
|
||||
|
||||
Returns a dict mapping (proto, local, remote) -> list of connection info dicts.
|
||||
Multiple connections can exist for the same tuple (port reuse over time).
|
||||
"""
|
||||
lookup = {}
|
||||
|
||||
if not jsonl_path.exists():
|
||||
print(f"Warning: Sidecar file not found: {jsonl_path}", file=sys.stderr)
|
||||
return lookup
|
||||
|
||||
with open(jsonl_path) as f:
|
||||
for line_num, line in enumerate(f, 1):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
c = json.loads(line)
|
||||
proto = c.get("protocol", "").upper()
|
||||
local = c.get("local_addr", "")
|
||||
remote = c.get("remote_addr", "")
|
||||
|
||||
if proto and local and remote:
|
||||
info = {
|
||||
"pid": c.get("pid"),
|
||||
"process_name": c.get("process_name"),
|
||||
"first_seen": parse_systemtime(c.get("first_seen")),
|
||||
"last_seen": parse_systemtime(c.get("last_seen")),
|
||||
"bytes_sent": c.get("bytes_sent", 0),
|
||||
"bytes_received": c.get("bytes_received", 0),
|
||||
# GeoIP information
|
||||
"geoip_country_code": c.get("geoip_country_code"),
|
||||
"geoip_country_name": c.get("geoip_country_name"),
|
||||
"geoip_asn": c.get("geoip_asn"),
|
||||
"geoip_as_org": c.get("geoip_as_org"),
|
||||
"geoip_city": c.get("geoip_city"),
|
||||
"geoip_postal_code": c.get("geoip_postal_code"),
|
||||
}
|
||||
|
||||
# Store both directions, as a list to handle port reuse
|
||||
for key in [(proto, local, remote), (proto, remote, local)]:
|
||||
if key not in lookup:
|
||||
lookup[key] = []
|
||||
lookup[key].append(info)
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"Warning: Invalid JSON at line {line_num}: {e}", file=sys.stderr)
|
||||
|
||||
return lookup
|
||||
|
||||
|
||||
def find_matching_connection(lookup: dict, pkt_tuple: tuple, pkt_time: float, slack: float) -> dict | None:
|
||||
"""Find the best matching connection for a packet based on tuple and timestamp.
|
||||
|
||||
Args:
|
||||
lookup: Connection lookup dict
|
||||
pkt_tuple: (proto, src, dst) tuple from packet
|
||||
pkt_time: Packet timestamp (seconds since epoch)
|
||||
slack: Allowed time slack in seconds
|
||||
|
||||
Returns:
|
||||
Best matching connection info dict, or None if no match
|
||||
"""
|
||||
connections = lookup.get(pkt_tuple, [])
|
||||
if not connections:
|
||||
return None
|
||||
|
||||
best_match = None
|
||||
best_score = float('inf')
|
||||
|
||||
for conn in connections:
|
||||
first_seen = conn.get("first_seen")
|
||||
last_seen = conn.get("last_seen")
|
||||
|
||||
# If no timestamps, fall back to simple match (first connection wins)
|
||||
if first_seen is None or last_seen is None:
|
||||
if best_match is None:
|
||||
best_match = conn
|
||||
continue
|
||||
|
||||
# Check if packet falls within connection time range (with slack)
|
||||
if first_seen - slack <= pkt_time <= last_seen + slack:
|
||||
# Score by how close the packet is to the connection's time range
|
||||
# Prefer connections where the packet is well within the range
|
||||
if pkt_time < first_seen:
|
||||
score = first_seen - pkt_time
|
||||
elif pkt_time > last_seen:
|
||||
score = pkt_time - last_seen
|
||||
else:
|
||||
score = 0 # Perfect match (within range)
|
||||
|
||||
if score < best_score:
|
||||
best_score = score
|
||||
best_match = conn
|
||||
|
||||
return best_match
|
||||
|
||||
|
||||
def get_packet_tuple(pkt) -> tuple:
|
||||
"""Extract connection tuple from packet."""
|
||||
if not pkt.haslayer(IP):
|
||||
return None
|
||||
|
||||
ip = pkt[IP]
|
||||
src_ip = ip.src
|
||||
dst_ip = ip.dst
|
||||
|
||||
if pkt.haslayer(TCP):
|
||||
tcp = pkt[TCP]
|
||||
return ("TCP", f"{src_ip}:{tcp.sport}", f"{dst_ip}:{tcp.dport}")
|
||||
elif pkt.haslayer(UDP):
|
||||
udp = pkt[UDP]
|
||||
return ("UDP", f"{src_ip}:{udp.sport}", f"{dst_ip}:{udp.dport}")
|
||||
elif pkt.haslayer(ICMP):
|
||||
return ("ICMP", src_ip, dst_ip)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def enrich_packets(pcap_path: Path, lookup: dict, slack: float):
|
||||
"""Yield enriched packet information."""
|
||||
packets = rdpcap(str(pcap_path))
|
||||
|
||||
for frame_num, pkt in enumerate(packets, 1):
|
||||
pkt_tuple = get_packet_tuple(pkt)
|
||||
pkt_time = float(pkt.time)
|
||||
|
||||
if not pkt_tuple:
|
||||
yield {
|
||||
"frame": frame_num,
|
||||
"time": pkt_time,
|
||||
"proto": "OTHER",
|
||||
"src": "",
|
||||
"dst": "",
|
||||
"pid": None,
|
||||
"process": None,
|
||||
"country": None,
|
||||
"city": None,
|
||||
"postal_code": None,
|
||||
"asn": None,
|
||||
"as_org": None,
|
||||
}
|
||||
continue
|
||||
|
||||
proto, src, dst = pkt_tuple
|
||||
info = find_matching_connection(lookup, pkt_tuple, pkt_time, slack) or {}
|
||||
|
||||
yield {
|
||||
"frame": frame_num,
|
||||
"time": pkt_time,
|
||||
"proto": proto,
|
||||
"src": src,
|
||||
"dst": dst,
|
||||
"pid": info.get("pid"),
|
||||
"process": info.get("process_name"),
|
||||
"bytes_sent": info.get("bytes_sent"),
|
||||
"bytes_received": info.get("bytes_received"),
|
||||
"country": info.get("geoip_country_code"),
|
||||
"city": info.get("geoip_city"),
|
||||
"postal_code": info.get("geoip_postal_code"),
|
||||
"asn": info.get("geoip_asn"),
|
||||
"as_org": info.get("geoip_as_org"),
|
||||
}
|
||||
|
||||
|
||||
def print_table(packets: list):
|
||||
"""Print enriched packets as a formatted table."""
|
||||
print(f"{'Frame':>6} {'Proto':<5} {'Source':<22} {'Destination':<22} {'Loc':<4} {'City':<15} {'ASN':>8} {'PID':>7} {'Process':<15}")
|
||||
print("-" * 116)
|
||||
|
||||
for p in packets:
|
||||
pid_str = str(p["pid"]) if p["pid"] else "-"
|
||||
proc_str = p["process"] or "-"
|
||||
if len(proc_str) > 15:
|
||||
proc_str = proc_str[:12] + "..."
|
||||
loc_str = p["country"] or "-"
|
||||
city_str = p["city"] or "-"
|
||||
if len(city_str) > 15:
|
||||
city_str = city_str[:12] + "..."
|
||||
asn_str = f"AS{p['asn']}" if p["asn"] else "-"
|
||||
src_str = p["src"][:22] if len(p["src"]) > 22 else p["src"]
|
||||
dst_str = p["dst"][:22] if len(p["dst"]) > 22 else p["dst"]
|
||||
print(f"{p['frame']:>6} {p['proto']:<5} {src_str:<22} {dst_str:<22} {loc_str:<4} {city_str:<15} {asn_str:>8} {pid_str:>7} {proc_str:<15}")
|
||||
|
||||
|
||||
def print_tsv(packets: list):
|
||||
"""Print enriched packets as TSV."""
|
||||
print("frame\ttime\tproto\tsrc\tdst\tcountry\tcity\tpostal_code\tasn\tas_org\tpid\tprocess")
|
||||
for p in packets:
|
||||
print(f"{p['frame']}\t{p['time']:.6f}\t{p['proto']}\t{p['src']}\t{p['dst']}\t{p['country'] or ''}\t{p['city'] or ''}\t{p['postal_code'] or ''}\t{p['asn'] or ''}\t{p['as_org'] or ''}\t{p['pid'] or ''}\t{p['process'] or ''}")
|
||||
|
||||
|
||||
def print_json(packets: list):
|
||||
"""Print enriched packets as JSON."""
|
||||
print(json.dumps(packets, indent=2))
|
||||
|
||||
|
||||
def create_pcapng(pcap_path: Path, packets: list, output_path: Path):
|
||||
"""Create annotated PCAPNG using editcap."""
|
||||
# Check if editcap is available
|
||||
try:
|
||||
subprocess.run(["editcap", "--version"], capture_output=True, check=True)
|
||||
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||
print("Error: editcap not found. Install Wireshark to get editcap.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# First convert to pcapng
|
||||
with tempfile.NamedTemporaryFile(suffix=".pcapng", delete=False) as tmp:
|
||||
tmp_path = Path(tmp.name)
|
||||
|
||||
subprocess.run(["editcap", "-F", "pcapng", str(pcap_path), str(tmp_path)], check=True)
|
||||
|
||||
# Build annotation commands
|
||||
# editcap -a "frame:comment" format
|
||||
annotations = []
|
||||
for p in packets:
|
||||
if p["pid"] or p["process"] or p["country"] or p["city"] or p["postal_code"] or p["asn"]:
|
||||
comment_parts = []
|
||||
if p["country"]:
|
||||
comment_parts.append(f"Loc:{p['country']}")
|
||||
if p["city"]:
|
||||
comment_parts.append(f"City:{p['city']}")
|
||||
if p["postal_code"]:
|
||||
comment_parts.append(f"ZIP:{p['postal_code']}")
|
||||
if p["asn"]:
|
||||
comment_parts.append(f"AS{p['asn']}")
|
||||
if p["pid"]:
|
||||
comment_parts.append(f"PID:{p['pid']}")
|
||||
if p["process"]:
|
||||
comment_parts.append(f"Process:{p['process']}")
|
||||
comment = " ".join(comment_parts)
|
||||
annotations.append(f"{p['frame']}:{comment}")
|
||||
|
||||
if not annotations:
|
||||
print("No process or GeoIP information found to annotate.", file=sys.stderr)
|
||||
# Just copy the pcapng as-is
|
||||
tmp_path.rename(output_path)
|
||||
return
|
||||
|
||||
# Apply annotations in batches (editcap has command line limits)
|
||||
current_input = tmp_path
|
||||
batch_size = 100
|
||||
|
||||
for i in range(0, len(annotations), batch_size):
|
||||
batch = annotations[i:i + batch_size]
|
||||
with tempfile.NamedTemporaryFile(suffix=".pcapng", delete=False) as tmp2:
|
||||
tmp2_path = Path(tmp2.name)
|
||||
|
||||
cmd = ["editcap"]
|
||||
for ann in batch:
|
||||
cmd.extend(["-a", ann])
|
||||
cmd.extend([str(current_input), str(tmp2_path)])
|
||||
|
||||
subprocess.run(cmd, check=True)
|
||||
|
||||
if current_input != tmp_path:
|
||||
current_input.unlink()
|
||||
current_input = tmp2_path
|
||||
|
||||
# Move final result to output
|
||||
current_input.rename(output_path)
|
||||
if tmp_path.exists():
|
||||
tmp_path.unlink()
|
||||
|
||||
print(f"Created annotated PCAPNG: {output_path}")
|
||||
print(f"Annotated {len(annotations)} packets with process/GeoIP information.")
|
||||
|
||||
|
||||
def count_unique_connections(lookup: dict) -> int:
|
||||
"""Count unique connections (accounting for bidirectional storage)."""
|
||||
seen = set()
|
||||
count = 0
|
||||
for key, conns in lookup.items():
|
||||
for conn in conns:
|
||||
# Create a unique identifier for each connection
|
||||
conn_id = (key, conn.get("first_seen"), conn.get("pid"))
|
||||
if conn_id not in seen:
|
||||
seen.add(conn_id)
|
||||
count += 1
|
||||
return count // 2 # Divide by 2 because we store both directions
|
||||
|
||||
|
||||
def print_summary(packets: list, lookup: dict):
|
||||
"""Print a summary of process information found."""
|
||||
total = len(packets)
|
||||
with_pid = sum(1 for p in packets if p["pid"])
|
||||
|
||||
# Group by process
|
||||
by_process = {}
|
||||
for p in packets:
|
||||
proc = p["process"] or "<unknown>"
|
||||
if proc not in by_process:
|
||||
by_process[proc] = {"count": 0, "pid": p["pid"]}
|
||||
by_process[proc]["count"] += 1
|
||||
|
||||
print(f"\nSummary:")
|
||||
print(f" Total packets: {total}")
|
||||
print(f" Packets with process info: {with_pid} ({100*with_pid/total:.1f}%)")
|
||||
print(f" Unique connections in sidecar: {count_unique_connections(lookup)}")
|
||||
print(f"\nPackets by process:")
|
||||
for proc, info in sorted(by_process.items(), key=lambda x: -x[1]["count"]):
|
||||
pid_str = f" (PID {info['pid']})" if info["pid"] else ""
|
||||
print(f" {proc}{pid_str}: {info['count']} packets")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Enrich RustNet PCAP captures with process and GeoIP information.",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
%(prog)s capture.pcap # Show packets with process info
|
||||
%(prog)s capture.pcap --format tsv # Output as TSV
|
||||
%(prog)s capture.pcap --format json # Output as JSON
|
||||
%(prog)s capture.pcap -o annotated.pcapng # Create annotated PCAPNG
|
||||
%(prog)s capture.pcap --summary # Show summary only
|
||||
%(prog)s capture.pcap --slack 5 # Use 5 second slack for timestamp matching
|
||||
"""
|
||||
)
|
||||
parser.add_argument("pcap", type=Path, help="Path to PCAP file")
|
||||
parser.add_argument("-j", "--jsonl", type=Path,
|
||||
help="Path to sidecar JSONL file (default: <pcap>.connections.jsonl)")
|
||||
parser.add_argument("-o", "--output", type=Path,
|
||||
help="Output annotated PCAPNG file")
|
||||
parser.add_argument("-f", "--format", choices=["table", "tsv", "json"], default="table",
|
||||
help="Output format (default: table)")
|
||||
parser.add_argument("-s", "--summary", action="store_true",
|
||||
help="Show summary only")
|
||||
parser.add_argument("-l", "--limit", type=int, default=0,
|
||||
help="Limit number of packets to process (0 = no limit)")
|
||||
parser.add_argument("--slack", type=float, default=2.0,
|
||||
help="Timestamp matching slack in seconds (default: 2.0)")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.pcap.exists():
|
||||
print(f"Error: PCAP file not found: {args.pcap}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Default sidecar path
|
||||
jsonl_path = args.jsonl or Path(f"{args.pcap}.connections.jsonl")
|
||||
|
||||
# Load connection mappings
|
||||
lookup = load_connections(jsonl_path)
|
||||
if lookup:
|
||||
print(f"Loaded {count_unique_connections(lookup)} connections from {jsonl_path}", file=sys.stderr)
|
||||
|
||||
# Process packets
|
||||
packets = list(enrich_packets(args.pcap, lookup, args.slack))
|
||||
if args.limit > 0:
|
||||
packets = packets[:args.limit]
|
||||
|
||||
if args.summary:
|
||||
print_summary(packets, lookup)
|
||||
return
|
||||
|
||||
if args.output:
|
||||
create_pcapng(args.pcap, packets, args.output)
|
||||
print_summary(packets, lookup)
|
||||
else:
|
||||
if args.format == "table":
|
||||
print_table(packets)
|
||||
print_summary(packets, lookup)
|
||||
elif args.format == "tsv":
|
||||
print_tsv(packets)
|
||||
elif args.format == "json":
|
||||
print_json(packets)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+179
@@ -0,0 +1,179 @@
|
||||
#!/usr/bin/env bash
|
||||
# Pre-release validation script for RustNet.
|
||||
# Run this before tagging a release to catch common issues.
|
||||
#
|
||||
# Usage: ./scripts/pre-release-check.sh [version]
|
||||
# e.g.: ./scripts/pre-release-check.sh 1.2.0
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
ERRORS=0
|
||||
WARNINGS=0
|
||||
|
||||
pass() { echo -e " ${GREEN}✓${NC} $1"; }
|
||||
fail() { echo -e " ${RED}✗${NC} $1"; ERRORS=$((ERRORS + 1)); }
|
||||
warn() { echo -e " ${YELLOW}!${NC} $1"; WARNINGS=$((WARNINGS + 1)); }
|
||||
|
||||
VERSION="${1:-}"
|
||||
|
||||
echo "RustNet Pre-Release Checks"
|
||||
echo "=========================="
|
||||
echo
|
||||
|
||||
# --- Version consistency ---
|
||||
echo "Version consistency:"
|
||||
|
||||
# Read the binary's [package] version, NOT the [workspace.package] version.
|
||||
# Since the workspace split, Cargo.toml has two `version =` lines and the
|
||||
# workspace one (0.x libraries) comes first, so `grep '^version' | head -1`
|
||||
# would read the wrong value. Anchor on the [package] section instead.
|
||||
CARGO_VERSION=$(awk -F'"' '/^\[package\]/{p=1} p && /^version = /{print $2; exit}' Cargo.toml)
|
||||
RPM_VERSION=$(grep '^Version:' rpm/rustnet.spec | awk '{print $2}')
|
||||
|
||||
if [ -n "$VERSION" ]; then
|
||||
if [ "$CARGO_VERSION" = "$VERSION" ]; then
|
||||
pass "Cargo.toml version: $CARGO_VERSION"
|
||||
else
|
||||
fail "Cargo.toml version is $CARGO_VERSION, expected $VERSION"
|
||||
fi
|
||||
if [ "$RPM_VERSION" = "$VERSION" ]; then
|
||||
pass "rpm/rustnet.spec version: $RPM_VERSION"
|
||||
else
|
||||
fail "rpm/rustnet.spec version is $RPM_VERSION, expected $VERSION"
|
||||
fi
|
||||
else
|
||||
if [ "$CARGO_VERSION" = "$RPM_VERSION" ]; then
|
||||
pass "Versions match: Cargo.toml=$CARGO_VERSION, rustnet.spec=$RPM_VERSION"
|
||||
else
|
||||
fail "Version mismatch: Cargo.toml=$CARGO_VERSION, rustnet.spec=$RPM_VERSION"
|
||||
fi
|
||||
VERSION="$CARGO_VERSION"
|
||||
fi
|
||||
|
||||
echo
|
||||
|
||||
# --- Changelog ---
|
||||
echo "Changelog:"
|
||||
|
||||
if grep -q "## \[$VERSION\]" CHANGELOG.md; then
|
||||
pass "CHANGELOG.md has entry for $VERSION"
|
||||
else
|
||||
fail "CHANGELOG.md missing entry for [$VERSION]"
|
||||
fi
|
||||
|
||||
if grep -q "\[$VERSION\]: https://github.com" CHANGELOG.md; then
|
||||
pass "CHANGELOG.md has comparison link for $VERSION"
|
||||
else
|
||||
fail "CHANGELOG.md missing comparison link for $VERSION"
|
||||
fi
|
||||
|
||||
UNRELEASED_CONTENT=$(awk '/^## \[Unreleased\]/{found=1; next} /^## \[/{found=0} found{print}' CHANGELOG.md | grep -v '^$' | head -1 || true)
|
||||
if [ -z "$UNRELEASED_CONTENT" ]; then
|
||||
pass "[Unreleased] section is empty (content moved to $VERSION)"
|
||||
else
|
||||
warn "[Unreleased] section still has content"
|
||||
fi
|
||||
|
||||
echo
|
||||
|
||||
# --- Build checks ---
|
||||
echo "Build checks:"
|
||||
|
||||
if cargo fmt --check > /dev/null 2>&1; then
|
||||
pass "cargo fmt"
|
||||
else
|
||||
fail "cargo fmt --check has formatting issues"
|
||||
fi
|
||||
|
||||
if cargo clippy -- -D warnings > /dev/null 2>&1; then
|
||||
pass "cargo clippy"
|
||||
else
|
||||
fail "cargo clippy has warnings"
|
||||
fi
|
||||
|
||||
if cargo test > /dev/null 2>&1; then
|
||||
pass "cargo test"
|
||||
else
|
||||
fail "cargo test failed"
|
||||
fi
|
||||
|
||||
echo
|
||||
|
||||
# --- Dockerfile ---
|
||||
echo "Dockerfile:"
|
||||
|
||||
CARGO_BENCHES=$(grep -c '^\[\[bench\]\]' Cargo.toml || true)
|
||||
if [ "$CARGO_BENCHES" -gt 0 ]; then
|
||||
if grep -q 'COPY benches' Dockerfile; then
|
||||
pass "Dockerfile copies benches/ directory"
|
||||
else
|
||||
fail "Cargo.toml defines $CARGO_BENCHES bench(es) but Dockerfile doesn't COPY benches/"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check that all include_bytes!()/include_str!() assets referenced at compile
|
||||
# time are present in the Dockerfile's COPY commands. Since the workspace split,
|
||||
# the baked-in assets (oui.gz, services) live under crates/rustnet-core/, not
|
||||
# src/, and the include path is relative to the file doing the include — so scan
|
||||
# both trees and resolve each path against its own source file's directory.
|
||||
ROOT=$(pwd)
|
||||
while IFS= read -r match; do
|
||||
[ -z "$match" ] && continue
|
||||
srcfile=${match%%:*} # path before the first ':'
|
||||
asset=$(printf '%s\n' "$match" | sed -E 's/.*include_(bytes|str)!\("//; s/"\).*//')
|
||||
srcdir=$(dirname "$srcfile")
|
||||
# Resolve the include path (relative to srcfile) into a repo-root-relative path
|
||||
# (portable, no GNU realpath needed).
|
||||
resolved=$(cd "$srcdir" && python3 -c "import os.path,sys; print(os.path.relpath(os.path.abspath(sys.argv[1]), sys.argv[2]))" "$asset" "$ROOT" 2>/dev/null || echo "$asset")
|
||||
if grep -q "$resolved\|$(basename "$resolved")" Dockerfile; then
|
||||
pass "Dockerfile includes compile-time asset: $resolved"
|
||||
else
|
||||
fail "Compile-time asset $resolved not found in Dockerfile COPY commands"
|
||||
fi
|
||||
done < <(grep -rno -E 'include_(bytes|str)!\("[^"]*"\)' src/ crates/*/src 2>/dev/null | sort -u || true)
|
||||
|
||||
if docker info > /dev/null 2>&1; then
|
||||
echo
|
||||
echo "Docker build test:"
|
||||
if docker build -t rustnet:pre-release-test . > /dev/null 2>&1; then
|
||||
pass "Docker image builds successfully"
|
||||
docker rmi rustnet:pre-release-test > /dev/null 2>&1 || true
|
||||
else
|
||||
fail "Docker build failed"
|
||||
fi
|
||||
else
|
||||
warn "Docker not available, skipping Docker build test"
|
||||
fi
|
||||
|
||||
echo
|
||||
|
||||
# --- Git status ---
|
||||
echo "Git status:"
|
||||
|
||||
if git diff --quiet Cargo.lock; then
|
||||
pass "Cargo.lock is up to date"
|
||||
else
|
||||
fail "Cargo.lock has uncommitted changes (run cargo build)"
|
||||
fi
|
||||
|
||||
if [ -z "$(git status --porcelain -- src/ Cargo.toml Cargo.lock CHANGELOG.md rpm/)" ]; then
|
||||
pass "No uncommitted changes in release files"
|
||||
else
|
||||
warn "Uncommitted changes in release files"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "=========================="
|
||||
if [ "$ERRORS" -gt 0 ]; then
|
||||
echo -e "${RED}$ERRORS error(s)${NC}, $WARNINGS warning(s) — fix errors before releasing"
|
||||
exit 1
|
||||
elif [ "$WARNINGS" -gt 0 ]; then
|
||||
echo -e "${GREEN}0 errors${NC}, ${YELLOW}$WARNINGS warning(s)${NC} — review warnings before releasing"
|
||||
else
|
||||
echo -e "${GREEN}All checks passed!${NC} Ready to tag v$VERSION"
|
||||
fi
|
||||
Executable
+177
@@ -0,0 +1,177 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Automate the rustnet VHS recording: produces assets/rustnet.gif (demo)
|
||||
# and assets/screenshots/*.png (README) in one pass.
|
||||
#
|
||||
# Prerequisites (Linux):
|
||||
# vhs # required (https://github.com/charmbracelet/vhs)
|
||||
# gifsicle # optional, for GIF size optimization
|
||||
# cargo build --release # auto-run if target/release/rustnet missing
|
||||
#
|
||||
# Usage:
|
||||
# scripts/record-rustnet-demo.sh
|
||||
#
|
||||
# What it does:
|
||||
# 1. Verifies vhs and cargo are installed.
|
||||
# 2. Builds target/release/rustnet if missing or stale vs Cargo.lock.
|
||||
# 3. Grants the binary capture capabilities via `sudo setcap`, so the
|
||||
# recording runs rustnet UNPRIVILEGED — the Security sidebar shows
|
||||
# the sandboxed non-root posture instead of a root warning.
|
||||
# 4. Spawns a background traffic generator (curl/dig/ping) so the
|
||||
# connection table is busy and sparklines move.
|
||||
# 5. Runs `vhs demo.tape` -> assets/rustnet.gif.
|
||||
# 6. Runs `vhs screenshots.tape` -> assets/screenshots/*.png.
|
||||
# 7. Optimizes the GIF with gifsicle -O3 --lossy=80 if available
|
||||
# (terminal flat colors are unaffected by the lossy pass).
|
||||
# 8. Cleans up the throwaway GIF and background processes.
|
||||
# 9. Prints a summary of the produced assets.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
RUSTNET_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
DEMO_TAPE="${RUSTNET_DIR}/demo.tape"
|
||||
SCREENSHOTS_TAPE="${RUSTNET_DIR}/screenshots.tape"
|
||||
GIF_FILE="${RUSTNET_DIR}/assets/rustnet.gif"
|
||||
SCREENSHOTS_DIR="${RUSTNET_DIR}/assets/screenshots"
|
||||
THROWAWAY_GIF="${SCREENSHOTS_DIR}/_throwaway.gif"
|
||||
RELEASE_BIN="${RUSTNET_DIR}/target/release/rustnet"
|
||||
|
||||
TRAFFIC_PID=""
|
||||
|
||||
# Background traffic generator: HTTPS, DNS, ICMP, multiple processes.
|
||||
# Started fresh for each tape with a delay so that every generated
|
||||
# connection is born AFTER the tape's rustnet instance (and its eBPF
|
||||
# hooks) is up — connections whose process exits before rustnet starts
|
||||
# can never be attributed and would show as dead "-" rows.
|
||||
start_traffic() {
|
||||
(
|
||||
sleep 6
|
||||
while true; do
|
||||
curl -s -o /dev/null --max-time 4 https://example.com || true
|
||||
curl -s -o /dev/null --max-time 4 https://github.com || true
|
||||
curl -s -o /dev/null --max-time 4 https://wikipedia.org || true
|
||||
curl -s -o /dev/null --max-time 4 https://ratatui.rs || true
|
||||
dig +short +time=2 +tries=1 example.com >/dev/null 2>&1 || true
|
||||
dig +short +time=2 +tries=1 github.com >/dev/null 2>&1 || true
|
||||
ping -c 1 -W 1 1.1.1.1 >/dev/null 2>&1 || true
|
||||
sleep 1
|
||||
done
|
||||
) &
|
||||
TRAFFIC_PID=$!
|
||||
}
|
||||
|
||||
stop_traffic() {
|
||||
if [[ -n "${TRAFFIC_PID}" ]] && kill -0 "${TRAFFIC_PID}" 2>/dev/null; then
|
||||
kill "${TRAFFIC_PID}" 2>/dev/null || true
|
||||
wait "${TRAFFIC_PID}" 2>/dev/null || true
|
||||
fi
|
||||
TRAFFIC_PID=""
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
stop_traffic
|
||||
rm -f "${THROWAWAY_GIF}"
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
# Kill stale ttyd / headless-Chrome processes left over from prior failed
|
||||
# vhs runs. VHS uses go-rod, which writes its Chrome profile to a
|
||||
# `rod/user-data` tmpdir; if a prior run died, the locked profile causes
|
||||
# the next run to fail with `could not open ttyd: ERR_CONNECTION_REFUSED`.
|
||||
preflight_cleanup_vhs() {
|
||||
local stale=0
|
||||
if pgrep -f "ttyd --port" >/dev/null 2>&1; then
|
||||
pkill -f "ttyd --port" 2>/dev/null || true
|
||||
stale=1
|
||||
fi
|
||||
if pgrep -f "rod/user-data" >/dev/null 2>&1; then
|
||||
pkill -f "rod/user-data" 2>/dev/null || true
|
||||
stale=1
|
||||
fi
|
||||
if [[ "${stale}" -eq 1 ]]; then
|
||||
echo "Cleaned up stale ttyd / Chrome processes from prior vhs run."
|
||||
sleep 1
|
||||
fi
|
||||
rm -rf "${TMPDIR:-/tmp}/rod" 2>/dev/null || true
|
||||
}
|
||||
preflight_cleanup_vhs
|
||||
|
||||
# Dependency checks
|
||||
for cmd in vhs cargo setcap; do
|
||||
if ! command -v "$cmd" &> /dev/null; then
|
||||
echo "Error: $cmd is not installed."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
if ! command -v gifsicle &> /dev/null; then
|
||||
echo "Note: gifsicle not found; skipping GIF optimization."
|
||||
HAS_GIFSICLE=0
|
||||
else
|
||||
HAS_GIFSICLE=1
|
||||
fi
|
||||
|
||||
# Tape files must exist
|
||||
[[ -f "${DEMO_TAPE}" ]] || { echo "Error: ${DEMO_TAPE} missing."; exit 1; }
|
||||
[[ -f "${SCREENSHOTS_TAPE}" ]] || { echo "Error: ${SCREENSHOTS_TAPE} missing."; exit 1; }
|
||||
|
||||
# Validate sudo up front (before the potentially long build) so the run
|
||||
# doesn't die at the setcap step minutes in. Honors SUDO_ASKPASS when
|
||||
# the caller exports one; otherwise prompts on the terminal as usual.
|
||||
SUDO=(sudo)
|
||||
[[ -n "${SUDO_ASKPASS:-}" ]] && SUDO=(sudo -A)
|
||||
echo "setcap needs sudo; validating credentials..."
|
||||
"${SUDO[@]}" -v || { echo "Error: sudo authentication failed."; exit 1; }
|
||||
|
||||
# Build release binary if missing or stale
|
||||
if [[ ! -x "${RELEASE_BIN}" ]] || [[ "${RUSTNET_DIR}/Cargo.lock" -nt "${RELEASE_BIN}" ]]; then
|
||||
echo "Building release binary..."
|
||||
(cd "${RUSTNET_DIR}" && cargo build --release)
|
||||
fi
|
||||
|
||||
# Grant capture capabilities so the tapes can run rustnet unprivileged.
|
||||
# File capabilities are dropped whenever cargo rewrites the binary, so
|
||||
# (re)apply them on every run.
|
||||
echo "Granting capture capabilities (sudo setcap)..."
|
||||
"${SUDO[@]}" setcap 'cap_net_raw,cap_bpf,cap_perfmon+eip' "${RELEASE_BIN}"
|
||||
|
||||
mkdir -p "${SCREENSHOTS_DIR}"
|
||||
|
||||
echo ""
|
||||
echo "Rendering demo GIF (vhs demo.tape)..."
|
||||
start_traffic
|
||||
(cd "${RUSTNET_DIR}" && vhs "${DEMO_TAPE}")
|
||||
stop_traffic
|
||||
|
||||
echo ""
|
||||
echo "Rendering screenshots (vhs screenshots.tape)..."
|
||||
start_traffic
|
||||
(cd "${RUSTNET_DIR}" && vhs "${SCREENSHOTS_TAPE}")
|
||||
stop_traffic
|
||||
|
||||
# Optimize the GIF. The lossy pass shrinks dithered/AA edges; flat
|
||||
# terminal colors are visually unaffected.
|
||||
if [[ "${HAS_GIFSICLE}" -eq 1 ]] && [[ -f "${GIF_FILE}" ]]; then
|
||||
echo ""
|
||||
echo "Optimizing GIF with gifsicle -O3 --lossy=80..."
|
||||
BEFORE_SIZE=$(stat -f%z "${GIF_FILE}" 2>/dev/null || stat -c%s "${GIF_FILE}")
|
||||
gifsicle -O3 --lossy=80 --batch "${GIF_FILE}"
|
||||
AFTER_SIZE=$(stat -f%z "${GIF_FILE}" 2>/dev/null || stat -c%s "${GIF_FILE}")
|
||||
echo " ${BEFORE_SIZE} -> ${AFTER_SIZE} bytes"
|
||||
fi
|
||||
|
||||
# Summary.
|
||||
echo ""
|
||||
echo "Done."
|
||||
echo ""
|
||||
echo "GIF:"
|
||||
ls -lh "${GIF_FILE}" 2>/dev/null | awk '{print " " $9 " (" $5 ")"}'
|
||||
echo ""
|
||||
echo "Screenshots:"
|
||||
for png in "${SCREENSHOTS_DIR}"/*.png; do
|
||||
[[ -f "${png}" ]] && ls -lh "${png}" | awk '{print " " $9 " (" $5 ")"}'
|
||||
done
|
||||
echo ""
|
||||
echo "Preview:"
|
||||
echo " open ${GIF_FILE}"
|
||||
echo " open ${SCREENSHOTS_DIR}"
|
||||
Executable
+66
@@ -0,0 +1,66 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
UBUNTU_RELEASE=${1:-noble}
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
|
||||
echo "Testing Debian package build for Ubuntu $UBUNTU_RELEASE"
|
||||
echo "=================================================="
|
||||
|
||||
# Build the Docker container
|
||||
docker build -t rustnet-deb-test:$UBUNTU_RELEASE -f - "$PROJECT_DIR" <<EOF
|
||||
FROM ubuntu:$UBUNTU_RELEASE
|
||||
|
||||
# Install build dependencies
|
||||
RUN apt-get update && apt-get install -y \\
|
||||
debhelper \\
|
||||
devscripts \\
|
||||
dpkg-dev \\
|
||||
rustup \\
|
||||
libpcap-dev \\
|
||||
libelf-dev \\
|
||||
elfutils \\
|
||||
zlib1g-dev \\
|
||||
clang \\
|
||||
llvm \\
|
||||
pkg-config \\
|
||||
lintian \\
|
||||
file
|
||||
|
||||
WORKDIR /build
|
||||
COPY . /build/
|
||||
|
||||
# Build the source package
|
||||
RUN echo "Building source package..." && \\
|
||||
debuild -S -sa -d -us -uc
|
||||
|
||||
# Build the binary package (simulates what Launchpad does)
|
||||
RUN echo "Building binary package..." && \\
|
||||
cd .. && \\
|
||||
dpkg-source -x rustnet-monitor_*.dsc extracted && \\
|
||||
cd extracted && \\
|
||||
dpkg-buildpackage -b -uc -us
|
||||
|
||||
# List the built packages
|
||||
RUN echo "Built packages:" && \\
|
||||
ls -lh /build/../*.deb || true
|
||||
|
||||
# Run lintian on the package
|
||||
RUN echo "Running lintian checks..." && \\
|
||||
lintian /build/../*.deb || true
|
||||
|
||||
# Test the package contents
|
||||
RUN echo "Package contents:" && \\
|
||||
dpkg-deb -c /build/../rustnet_*.deb
|
||||
|
||||
CMD ["/bin/bash"]
|
||||
EOF
|
||||
|
||||
echo ""
|
||||
echo "Build completed successfully!"
|
||||
echo ""
|
||||
echo "To extract the .deb file, run:"
|
||||
echo " docker create --name rustnet-deb-extract rustnet-deb-test:$UBUNTU_RELEASE"
|
||||
echo " docker cp rustnet-deb-extract:/build/../rustnet_*.deb ."
|
||||
echo " docker rm rustnet-deb-extract"
|
||||
Reference in New Issue
Block a user