dmtrkovalenko--fff.nvim/scripts/diagnose_lockmdb.py

#!/usr/bin/env python3
"""
Diagnose LMDB lock.mdb holders and waiters.

Usage: ./diagnose_lockmdb.py [path_to_lock.mdb]
Default: /home/neogoose/.cache/nvim/fff_nvim/lock.mdb

Shows:
1. Every process that has the lock.mdb inode in its mmap/fd table (potential holders)
2. Every process currently blocked on a futex at an address in the lock.mdb mmap
   (actually waiting on the reader/writer mutex)
3. The mutex state (__lock / __owner / __kind) for both reader and writer mutex
"""

import os
import struct
import sys
from pathlib import Path

LOCK_PATH = sys.argv[1] if len(sys.argv) > 1 else "/home/neogoose/.cache/nvim/fff_nvim/lock.mdb"

RMUTEX_OFFSET = 24   # MDB_txbody.mtb_rmutex
WMUTEX_OFFSET = 64   # MDB_txninfo.mt2.mt2_wmutex
MUTEX_SIZE = 40      # sizeof(pthread_mutex_t) on glibc x86_64


def mutex_state(data, off):
    """Decode pthread_mutex_t at offset `off`."""
    if len(data) < off + 20:
        return None
    return {
        "lock":   struct.unpack("i", data[off:off + 4])[0],
        "count":  struct.unpack("I", data[off + 4:off + 8])[0],
        "owner":  struct.unpack("i", data[off + 8:off + 12])[0],
        "nusers": struct.unpack("I", data[off + 12:off + 16])[0],
        "kind":   struct.unpack("i", data[off + 16:off + 20])[0],
    }


def fmt_mutex(m):
    if m is None:
        return "(too small)"
    robust = "ROBUST" if m["kind"] & 0x10 else "non-robust"
    pshared = "PSHARED" if m["kind"] & 0x80 else "private"
    state = {0: "unlocked", 1: "locked", 2: "locked+waiters"}.get(m["lock"], f"lock={m['lock']}")
    return (f"state={state} owner_tid={m['owner']} nusers={m['nusers']} "
            f"kind=0x{m['kind']:x} ({robust}, {pshared})")


def get_comm(pid):
    try:
        return Path(f"/proc/{pid}/comm").read_text().strip()
    except Exception:
        return "?"


def get_cmdline(pid):
    try:
        return Path(f"/proc/{pid}/cmdline").read_text().replace("\0", " ").strip()[:100]
    except Exception:
        return "?"


def get_wchan(pid_or_tid, path):
    try:
        return Path(f"{path}/wchan").read_text().strip()
    except Exception:
        return "?"


def get_syscall(tid_path):
    """Return (syscall_nr, arg0, arg1, ..., arg5) or None."""
    try:
        parts = Path(f"{tid_path}/syscall").read_text().split()
        if parts and parts[0] != "running":
            return [int(p, 0) for p in parts[:7]]
    except Exception:
        pass
    return None


def get_mmap_regions(pid, target_inode):
    """Return list of (start, end) virtual addresses where `target_inode` is mapped."""
    regions = []
    try:
        for line in Path(f"/proc/{pid}/maps").read_text().splitlines():
            parts = line.split()
            if len(parts) < 5:
                continue
            # inode is field index 4
            try:
                inode = int(parts[4])
            except ValueError:
                continue
            if inode != target_inode:
                continue
            addr_range = parts[0]
            try:
                start_s, end_s = addr_range.split("-")
                regions.append((int(start_s, 16), int(end_s, 16), line))
            except Exception:
                continue
    except Exception:
        pass
    return regions


def main():
    path = Path(LOCK_PATH)
    if not path.exists():
        print(f"ERROR: {path} does not exist")
        return 1

    st = path.stat()
    target_inode = st.st_ino
    size = st.st_size
    print(f"=== Target: {path} ===")
    print(f"   inode={target_inode}  size={size}\n")

    # Read mutex state from the on-disk file
    data = path.read_bytes()
    r_mutex = mutex_state(data, RMUTEX_OFFSET)
    w_mutex = mutex_state(data, WMUTEX_OFFSET)
    print("=== Mutex state on disk ===")
    print(f"  READER mutex (offset {RMUTEX_OFFSET}): {fmt_mutex(r_mutex)}")
    print(f"  WRITER mutex (offset {WMUTEX_OFFSET}): {fmt_mutex(w_mutex)}")
    print()

    # Scan all processes for mmap hits on this inode.
    holders = []  # list of (pid, start, end, deleted)
    for pid_dir in Path("/proc").iterdir():
        if not pid_dir.name.isdigit():
            continue
        pid = int(pid_dir.name)
        regions = get_mmap_regions(pid, target_inode)
        if not regions:
            # Also check fd table — inode may be open-but-not-mmap'd
            try:
                for fd in (pid_dir / "fd").iterdir():
                    try:
                        tgt = os.readlink(fd)
                    except OSError:
                        continue
                    if "lock.mdb" in tgt and f"[{target_inode}]" in os.stat(fd).st_dev.__str__():
                        # crude — we already caught mmap above, skip
                        pass
            except OSError:
                pass
            continue
        for (start, end, line) in regions:
            deleted = "(deleted)" in line
            holders.append((pid, start, end, deleted))

    print("=== Processes with lock.mdb mmap'd ===")
    if not holders:
        print("  (none — no live process holds this inode via mmap)")
    for (pid, start, end, deleted) in holders:
        marker = " (DELETED inode — stale mapping from previous file)" if deleted else ""
        print(f"  PID {pid} ({get_comm(pid)})  mmap @ 0x{start:x}-0x{end:x}{marker}")
        print(f"    cmdline: {get_cmdline(pid)}")

    print()

    # For each holder mmap, compute the futex addresses of each mutex,
    # then scan all threads in all processes for futex syscalls targeting those addresses.
    futex_addrs = {}  # address -> description
    for (pid, start, _, deleted) in holders:
        futex_addrs[start + RMUTEX_OFFSET] = (pid, "READER mutex", deleted)
        futex_addrs[start + WMUTEX_OFFSET] = (pid, "WRITER mutex", deleted)

    print("=== Threads currently blocked on mutex futex ===")
    found_any = False
    for pid_dir in Path("/proc").iterdir():
        if not pid_dir.name.isdigit():
            continue
        pid = int(pid_dir.name)
        task_dir = pid_dir / "task"
        if not task_dir.exists():
            continue
        try:
            tids = [t.name for t in task_dir.iterdir() if t.name.isdigit()]
        except OSError:
            continue
        for tid in tids:
            tid_path = task_dir / tid
            sc = get_syscall(tid_path)
            if sc is None:
                continue
            # syscall 202 = futex on x86_64
            if sc[0] != 202:
                continue
            futex_addr = sc[1]
            # Match: the blocked thread must have this address mapped into its own address space.
            # Since mmap addresses differ per process, compare only against THIS process's own holders.
            my_regions = get_mmap_regions(pid, target_inode)
            for (start, end, line) in my_regions:
                deleted = "(deleted)" in line
                r_addr = start + RMUTEX_OFFSET
                w_addr = start + WMUTEX_OFFSET
                if futex_addr == r_addr:
                    wchan = get_wchan(tid, tid_path)
                    dmark = " (DELETED)" if deleted else ""
                    print(f"  PID {pid} TID {tid} ({get_comm(pid)}) wchan={wchan}: "
                          f"blocked on READER mutex @ 0x{futex_addr:x}{dmark}")
                    found_any = True
                elif futex_addr == w_addr:
                    wchan = get_wchan(tid, tid_path)
                    dmark = " (DELETED)" if deleted else ""
                    print(f"  PID {pid} TID {tid} ({get_comm(pid)}) wchan={wchan}: "
                          f"blocked on WRITER mutex @ 0x{futex_addr:x}{dmark}")
                    found_any = True
    if not found_any:
        print("  (none)")

    print()
    print("=== /proc/locks fcntl/flock state ===")
    try:
        for line in Path("/proc/locks").read_text().splitlines():
            if str(target_inode) in line:
                # format: N: TYPE ACCESS KIND PID MAJ:MIN:INODE START END
                parts = line.split()
                if len(parts) >= 7 and parts[6].endswith(f":{target_inode}"):
                    pid = parts[4]
                    print(f"  {line}")
                    print(f"     -> PID {pid} ({get_comm(pid)})")
    except Exception as e:
        print(f"  error reading /proc/locks: {e}")

    return 0


if __name__ == "__main__":
    sys.exit(main())