first commit

cts/hang_watch.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+import glob
+import os
+import subprocess
+import sys
+import time
+from datetime import datetime
+from pathlib import Path
+
+
+def now_iso():
+    return datetime.now().astimezone().isoformat(timespec="seconds")
+
+
+def meminfo():
+    d = {}
+    with open("/proc/meminfo") as f:
+        for line in f:
+            k, v = line.split(":", 1)
+            d[k.strip()] = int(v.split()[0])
+    return d
+
+
+def main():
+    outdir = Path(sys.argv[1]) if len(sys.argv) > 1 else Path.cwd() / "hangs"
+    poll = int(sys.argv[2]) if len(sys.argv) > 2 else 10
+
+    err = ""
+    for c in sorted(glob.glob("/sys/class/drm/card*/error")):
+        err = c
+        break
+
+    if not err:
+        print("no /sys/class/drm/card*/error — is i915 loaded?", file=sys.stderr)
+    elif not os.access(err, os.R_OK):
+        print(f"cannot read {err} — rerun with sudo for GPU hang capture", file=sys.stderr)
+        err = ""
+
+    outdir.mkdir(parents=True, exist_ok=True)
+    hangs_txt = outdir / "hangs.txt"
+    mem_log = outdir / "mem.log"
+
+    print(f"watching {err or '(memory only)'} every {poll}s -> {outdir}")
+    with open(hangs_txt, "a") as f:
+        f.write(f"{now_iso()} watch started\n")
+
+    n = 0
+    while True:
+        if err and os.path.getsize(err) > 0:
+            n += 1
+            stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+            hang_f = outdir / f"hang-{stamp}.txt"
+            data = open(err, "rb").read()
+            hang_f.write_bytes(data)
+
+            with open(hangs_txt, "a") as hf:
+                for line in data.decode(errors="replace").splitlines():
+                    if line.startswith(("GPU HANG", "Kernel", "Time")):
+                        hf.write(line + "\n")
+                        break
+                msg = f"{now_iso()} hang #{n} -> {hang_f} ({hang_f.stat().st_size} B)"
+                hf.write(msg + "\n")
+            print(msg)
+
+            dmesg = subprocess.run(["dmesg"], capture_output=True, text=True).stdout
+            matched = [l for l in dmesg.splitlines()
+                       if any(k.lower() in l.lower() for k in ("GPU HANG", "reset", "hangcheck"))]
+            (outdir / f"dmesg-{stamp}.txt").write_text("\n".join(matched[-20:]) + ("\n" if matched else ""))
+
+            with open(err, "w") as ef:
+                ef.write("1")
+
+        mi = meminfo()
+        avail = mi.get("MemAvailable", 0)
+        swtot = mi.get("SwapTotal", 0)
+        swfree = mi.get("SwapFree", 0)
+        sunrecl = mi.get("SUnreclaim", 0)
+        ptbl = mi.get("PageTables", 0)
+
+        ps = subprocess.run(["ps", "-eo", "rss,comm", "--sort=-rss", "--no-headers"],
+                             capture_output=True, text=True).stdout
+        top3 = []
+        for line in ps.splitlines()[:3]:
+            parts = line.split(None, 1)
+            if len(parts) == 2:
+                rss, comm = parts
+                top3.append(f"{comm}:{int(rss) // 1024}M")
+        top3_str = " ".join(top3) + (" " if top3 else "")
+
+        line = (f"{now_iso()} avail={avail // 1024}M swapused={(swtot - swfree) // 1024}M "
+                f"sunreclaim={sunrecl // 1024}M pagetables={ptbl // 1024}M {top3_str}\n")
+        with open(mem_log, "a") as mf:
+            mf.write(line)
+        try:
+            subprocess.run(["sync", "-d", str(mem_log)], check=True,
+                            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        except subprocess.CalledProcessError:
+            subprocess.run(["sync"])
+
+        time.sleep(poll)
+
+
+if __name__ == "__main__":
+    main()

cts/mem_guard.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+import os
+import subprocess
+import sys
+import time
+from datetime import datetime
+
+
+def now_iso():
+    return datetime.now().astimezone().isoformat(timespec="seconds")
+
+
+def mem_available_mb():
+    with open("/proc/meminfo") as f:
+        for line in f:
+            if line.startswith("MemAvailable:"):
+                return int(line.split()[1]) // 1024
+    return None
+
+
+def proc_alive(name):
+    return subprocess.run(["pgrep", "-x", name],
+                           stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0
+
+
+def main():
+    threshold = int(sys.argv[1]) if len(sys.argv) > 1 else 1500
+    proc = sys.argv[2] if len(sys.argv) > 2 else "deqp-vk"
+    poll = int(sys.argv[3]) if len(sys.argv) > 3 else 3
+
+    print(f"guarding: kill {proc} when MemAvailable < {threshold}M (poll {poll}s)")
+
+    while True:
+        avail = mem_available_mb()
+        if avail is not None and avail < threshold:
+            print(f"{now_iso()} avail={avail}M below {threshold}M — terminating {proc}")
+            subprocess.run(["pkill", "-TERM", "-x", proc])
+            time.sleep(10)
+            if proc_alive(proc):
+                print("still alive, SIGKILL")
+                subprocess.run(["pkill", "-KILL", "-x", proc])
+
+            outdir = os.environ.get("OUTDIR", ".")
+            with open(os.path.join(outdir, "mem-guard.log"), "a") as f:
+                f.write(f"{now_iso()} killed {proc} at avail={avail}M\n")
+            subprocess.run(["sync"])
+            sys.exit(1)
+
+        time.sleep(poll)
+
+
+if __name__ == "__main__":
+    main()

cts/run_chunked.py

@@ -0,0 +1,240 @@
+#!/usr/bin/env python3
+import collections
+import glob
+import os
+import re
+import subprocess
+import sys
+import time
+from datetime import datetime
+from pathlib import Path
+
+
+def now_iso():
+    return datetime.now().astimezone().isoformat(timespec="seconds")
+
+
+def wc_l(path):
+    if not os.path.exists(path):
+        return 0
+    with open(path, "rb") as f:
+        return sum(1 for _ in f)
+
+
+def read_lines(path):
+    if not os.path.exists(path):
+        return []
+    with open(path) as f:
+        return [l.rstrip("\n") for l in f if l.strip()]
+
+
+def write_lines(path, lines):
+    with open(path, "w") as f:
+        for l in lines:
+            f.write(l + "\n")
+
+
+def begin_test_case_names(qpa_path):
+    names = set()
+    if not os.path.exists(qpa_path):
+        return names
+    txt = open(qpa_path, errors="replace").read()
+    for line in txt.splitlines():
+        if line.startswith("#beginTestCaseResult "):
+            names.add(line[len("#beginTestCaseResult "):].strip())
+    return names
+
+
+def main():
+    lab_root = Path(os.environ.get("LAB_ROOT", Path(__file__).resolve().parent.parent))
+    drivers_root = Path(os.environ.get("DRIVERS_ROOT", lab_root.parent))
+    mesa_src = Path(os.environ.get("MESA_SRC", drivers_root / "intel-hard-graphics"))
+    mesa_build = Path(os.environ.get("MESA_BUILD", mesa_src / "build"))
+    cts_dir = Path(os.environ.get("CTS_DIR", mesa_src / "VK-GL-CTS/build-cts/external/vulkancts/modules/vulkan"))
+    cts_mustpass = Path(os.environ.get("CTS_MUSTPASS", mesa_src / "VK-GL-CTS/external/vulkancts/mustpass/main/vk-default"))
+    icd = Path(os.environ.get("VK_DRIVER_FILES", mesa_build / "src/intel/vulkan_hasvk/intel_hasvk13_devenv_icd.x86_64.json"))
+    exclude = Path(os.environ.get("EXCLUDE", lab_root / "cts/cts-exclude-gen7.txt"))
+
+    scope = sys.argv[1] if len(sys.argv) > 1 else "mustpass"
+    chunk_size = int(sys.argv[2]) if len(sys.argv) > 2 else 100000
+    pause = int(sys.argv[3]) if len(sys.argv) > 3 else 30
+    outdir = Path(os.environ.get("OUTDIR", Path.cwd() / "cts-chunked"))
+
+    stop_on_hang = os.environ.get("STOP_ON_HANG", "0") == "1"
+    max_hangs = int(os.environ.get("MAX_HANGS", "3"))
+    max_chunks = int(os.environ.get("MAX_CHUNKS", "3"))
+    max_subruns = int(os.environ.get("MAX_SUBRUNS", "40"))
+    hangs = 0
+
+    deqp = cts_dir / "deqp-vk"
+    for p in (deqp, icd):
+        if not p.exists():
+            print(f"missing: {p}", file=sys.stderr)
+            sys.exit(1)
+
+    outdir.mkdir(parents=True, exist_ok=True)
+    os.chdir(cts_dir)
+
+    src_txt = outdir / "src.txt"
+    cases_txt = outdir / "cases.txt"
+    if not (src_txt.exists() and src_txt.stat().st_size > 0):
+        if scope == "all":
+            out = subprocess.run([str(deqp), "--deqp-runmode=stdout-caselist"],
+                                  capture_output=True, text=True).stdout
+            lines = [l[len("TEST: "):] for l in out.splitlines() if l.startswith("TEST: ")]
+            write_lines(src_txt, lines)
+        else:
+            lines = []
+            for f in sorted(glob.glob(str(cts_mustpass / "*.txt"))):
+                lines.extend(l for l in read_lines(f) if l.startswith("dEQP-VK"))
+            write_lines(src_txt, lines)
+
+    if exclude.exists() and exclude.stat().st_size > 0:
+        excl = set(read_lines(exclude))
+        write_lines(cases_txt, [l for l in read_lines(src_txt) if l not in excl])
+    else:
+        write_lines(cases_txt, read_lines(src_txt))
+
+    todo_txt = outdir / "todo.txt"
+    prev = sorted(glob.glob(str(outdir / "chunk-*.qpa")))
+    if prev:
+        done = set()
+        for f in prev:
+            done |= begin_test_case_names(f)
+        done_txt = outdir / "done.txt"
+        write_lines(done_txt, sorted(done))
+        cases = read_lines(cases_txt)
+        todo = [l for l in cases if l not in done]
+        write_lines(todo_txt, todo)
+        print(f"resume: {len(done)} recorded, {len(todo)} left")
+    else:
+        cases = read_lines(cases_txt)
+        write_lines(todo_txt, cases)
+        print(f"{len(cases)} cases to run")
+
+    if not (todo_txt.exists() and todo_txt.stat().st_size > 0):
+        print("nothing left to run — summarising")
+
+    todo = read_lines(todo_txt)
+    total = len(todo)
+    n = 0
+    start = 1
+    ran = 0
+    rc = 0
+    sub = 0
+
+    while todo and start <= total:
+        n += 1
+        while glob.glob(str(outdir / f"chunk-{n}.*.qpa")):
+            n += 1
+
+        list_path = outdir / f"chunk-{n}.txt"
+        chunk_lines = todo[start - 1: start - 1 + chunk_size]
+        write_lines(list_path, chunk_lines)
+        if not chunk_lines:
+            break
+
+        print(f"chunk {n}: {len(chunk_lines)} cases (from line {start} of {total})")
+        want = len(chunk_lines)
+        work_path = outdir / f"chunk-{n}.work"
+        write_lines(work_path, chunk_lines)
+        sub = 0
+
+        while work_path.exists() and work_path.stat().st_size > 0:
+            sub += 1
+            if sub > max_subruns:
+                print(f"  gave up after {max_subruns} subruns")
+                break
+
+            qpa = outdir / f"chunk-{n}.{sub}.qpa"
+            log = outdir / f"chunk-{n}.{sub}.log"
+            env = dict(os.environ, VK_DRIVER_FILES=str(icd))
+            with open(log, "w") as lf:
+                proc = subprocess.run(
+                    [str(deqp), f"--deqp-caselist-file={work_path}",
+                     f"--deqp-log-filename={qpa}",
+                     "--deqp-log-images=disable", "--deqp-log-shader-sources=disable"],
+                    stdout=lf, stderr=subprocess.STDOUT, env=env)
+            rc = proc.returncode
+
+            work_lines = read_lines(work_path)
+            left = len(work_lines)
+            got = sum(1 for l in open(qpa, errors="replace").read().splitlines()
+                      if l.startswith("#beginTestCaseResult")) if qpa.exists() else 0
+            if got >= left:
+                break
+
+            done_now = begin_test_case_names(qpa)
+            next_lines = [l for l in work_lines if l not in done_now]
+            if len(next_lines) == left:
+                culprit = work_lines[0] if work_lines else ""
+                with open(outdir / "crashers.txt", "a") as cf:
+                    cf.write(f"{now_iso()} crashed before recording: {culprit}\n")
+                next_lines = work_lines[1:]
+            write_lines(work_path, next_lines)
+            print(f"  subrun {sub}: rc={rc}, {got} done, {len(next_lines)} left — restarting")
+            time.sleep(pause)
+
+        for suffix in (".work", ".work.done", ".work.next"):
+            p = outdir / f"chunk-{n}{suffix}"
+            if p.exists():
+                p.unlink()
+
+        done_n = 0
+        bad_log = False
+        for f in sorted(glob.glob(str(outdir / f"chunk-{n}.*.qpa"))):
+            done_n += sum(1 for l in open(f, errors="replace").read().splitlines()
+                          if l.startswith("#beginTestCaseResult"))
+        for f in sorted(glob.glob(str(outdir / f"chunk-{n}.*.log"))):
+            if "DeviceLost" in open(f, errors="replace").read():
+                bad_log = True
+        print(f"  rc={rc}, {done_n}/{want} results ({sub} subrun)")
+
+        bad = done_n < want or bad_log
+        wait_s = pause
+        if bad:
+            hangs += 1
+            with open(outdir / "hangs.txt", "a") as hf:
+                hf.write(f"{now_iso()} chunk {n} ended badly (rc={rc})\n")
+            if stop_on_hang:
+                print("  chunk ended badly — stopping (STOP_ON_HANG=1)")
+                print("  rerun the same command to continue from here")
+                break
+            wait_s = pause * 4
+            print(f"  chunk ended badly (#{hangs}) — cooling down {wait_s}s")
+            if max_hangs > 0 and hangs >= max_hangs:
+                print(f"  {hangs} bad chunks — stopping (MAX_HANGS={max_hangs})")
+                break
+
+        start += chunk_size
+        ran += 1
+        if max_chunks > 0 and ran >= max_chunks and start <= total:
+            print()
+            print(f"{ran} chunks done this boot (MAX_CHUNKS={max_chunks}) — stopping here.")
+            print(f"REBOOT, then rerun the same command to continue from case {start}.")
+            break
+        if start <= total:
+            time.sleep(wait_s)
+
+    res = {}
+    for f in sorted(glob.glob(str(outdir / "chunk-*.qpa"))):
+        txt = open(f, errors="replace").read()
+        for block in re.split(r"#beginTestCaseResult ", txt)[1:]:
+            name = block.split("\n", 1)[0].strip()
+            m = re.search(r'<Result StatusCode="(\w+)">', block)
+            res[name] = m.group(1) if m else "Crash"
+
+    counts = collections.Counter(res.values())
+    print(f"\n{sum(counts.values())} results")
+    for k, v in counts.most_common():
+        print(f"  {k:16} {v}")
+
+    with open(outdir / "failures.txt", "w") as fh:
+        for name, status in sorted(res.items()):
+            if status not in ("Pass", "NotSupported"):
+                fh.write(f"{status}\t{name}\n")
+    print(f"\nfailures -> {outdir}/failures.txt")
+
+
+if __name__ == "__main__":
+    main()

cts/run_cts.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+import collections
+import glob
+import os
+import re
+import subprocess
+import sys
+import threading
+import time
+from datetime import datetime
+from pathlib import Path
+
+
+def now_iso():
+    return datetime.now().astimezone().isoformat(timespec="seconds")
+
+
+def read_lines(path):
+    if not os.path.exists(path):
+        return []
+    with open(path) as f:
+        return [l.rstrip("\n") for l in f if l.strip()]
+
+
+def write_lines(path, lines):
+    with open(path, "w") as f:
+        for l in lines:
+            f.write(l + "\n")
+
+
+def begin_test_case_names(qpa_path):
+    names = set()
+    if not os.path.exists(qpa_path):
+        return names
+    txt = open(qpa_path, errors="replace").read()
+    for line in txt.splitlines():
+        if line.startswith("#beginTestCaseResult "):
+            names.add(line[len("#beginTestCaseResult "):].strip())
+    return names
+
+
+def last_begin_test_case_name(qpa_path):
+    last = None
+    if not os.path.exists(qpa_path):
+        return last
+    for line in open(qpa_path, errors="replace"):
+        if line.startswith("#beginTestCaseResult "):
+            last = line[len("#beginTestCaseResult "):].strip()
+    return last
+
+
+def nproc():
+    return os.cpu_count() or 1
+
+
+def main():
+    lab_root = Path(os.environ.get("LAB_ROOT", Path(__file__).resolve().parent.parent))
+    drivers_root = Path(os.environ.get("DRIVERS_ROOT", lab_root.parent))
+    mesa_src = Path(os.environ.get("MESA_SRC", drivers_root / "intel-hard-graphics"))
+    mesa_build = Path(os.environ.get("MESA_BUILD", mesa_src / "build"))
+    cts_dir = Path(os.environ.get("CTS_DIR", mesa_src / "VK-GL-CTS/build-cts/external/vulkancts/modules/vulkan"))
+    cts_mustpass = Path(os.environ.get("CTS_MUSTPASS", mesa_src / "VK-GL-CTS/external/vulkancts/mustpass/main/vk-default"))
+    icd = Path(os.environ.get("VK_DRIVER_FILES", mesa_build / "src/intel/vulkan_hasvk/intel_hasvk13_devenv_icd.x86_64.json"))
+    exclude = lab_root / "cts/cts-exclude-gen7.txt"
+
+    scope = sys.argv[1] if len(sys.argv) > 1 else "mustpass"
+    shards = int(sys.argv[2]) if len(sys.argv) > 2 else nproc()
+    cooldown = int(os.environ.get("COOLDOWN", "60"))
+    outdir = Path(os.environ.get(
+        "OUTDIR", Path.cwd() / f"cts-run-{datetime.now().strftime('%Y%m%d-%H%M%S')}"))
+
+    deqp = cts_dir / "deqp-vk"
+    for p in (deqp, icd):
+        if not p.exists():
+            print(f"missing: {p}", file=sys.stderr)
+            sys.exit(1)
+
+    outdir.mkdir(parents=True, exist_ok=True)
+    os.chdir(cts_dir)
+
+    cases_txt = outdir / "cases.txt"
+    if scope == "all":
+        out = subprocess.run([str(deqp), "--deqp-runmode=stdout-caselist"],
+                              capture_output=True, text=True).stdout
+        lines = [l[len("TEST: "):] for l in out.splitlines() if l.startswith("TEST: ")]
+        src_txt = outdir / "all.txt"
+        write_lines(src_txt, lines)
+    else:
+        lines = []
+        for f in sorted(glob.glob(str(cts_mustpass / "*.txt"))):
+            lines.extend(l for l in read_lines(f) if l.startswith("dEQP-VK"))
+        src_txt = outdir / "mustpass.txt"
+        write_lines(src_txt, lines)
+
+    src_lines = read_lines(src_txt)
+    if exclude.exists() and exclude.stat().st_size > 0:
+        excl = set(read_lines(exclude))
+        case_lines = [l for l in src_lines if l not in excl]
+    else:
+        case_lines = list(src_lines)
+    write_lines(cases_txt, case_lines)
+    print(f"{len(src_lines)} cases, {len(src_lines) - len(case_lines)} excluded, {len(case_lines)} to run")
+
+    prev = sorted(glob.glob(str(outdir / "*.qpa"))) + sorted(glob.glob(str(outdir / "attempt-*/*.qpa")))
+    if prev:
+        done = set()
+        for f in prev:
+            done |= begin_test_case_names(f)
+        done_txt = outdir / "done.txt"
+        write_lines(done_txt, sorted(done))
+        if done:
+            case_lines = [l for l in read_lines(cases_txt) if l not in done]
+            write_lines(cases_txt, case_lines)
+            print(f"resume: {len(done)} already recorded, {len(case_lines)} left")
+
+        n = 1
+        while (outdir / f"attempt-{n}").exists():
+            n += 1
+        attempt_dir = outdir / f"attempt-{n}"
+        attempt_dir.mkdir(parents=True, exist_ok=True)
+        for f in list(glob.glob(str(outdir / "shard*.qpa"))) + list(glob.glob(str(outdir / "shard*.log"))):
+            os.rename(f, attempt_dir / os.path.basename(f))
+        print(f"previous attempt parked in attempt-{n}/")
+
+    case_lines = read_lines(cases_txt)
+    if not case_lines:
+        print("nothing left to run")
+        sys.exit(0)
+
+    def run_shard(i):
+        list_path = outdir / f"shard{i}.txt"
+        shard_lines = [l for j, l in enumerate(case_lines) if (j + 1) % shards == i]
+        write_lines(list_path, shard_lines)
+        pass_n = 1
+
+        while list_path.exists() and list_path.stat().st_size > 0:
+            qpa = outdir / f"shard{i}.pass{pass_n}.qpa"
+            log = outdir / f"shard{i}.pass{pass_n}.log"
+            env = dict(os.environ, VK_DRIVER_FILES=str(icd))
+            with open(log, "w") as lf:
+                subprocess.run(
+                    [str(deqp), f"--deqp-caselist-file={list_path}",
+                     f"--deqp-log-filename={qpa}",
+                     "--deqp-log-images=disable", "--deqp-log-shader-sources=disable"],
+                    stdout=lf, stderr=subprocess.STDOUT, env=env)
+
+            log_text = open(log, errors="replace").read() if log.exists() else ""
+            if "ABORTED" not in log_text:
+                break
+
+            last = last_begin_test_case_name(qpa)
+            if not last:
+                break
+
+            print(f"shard {i} aborted at {last} -- cooling down {cooldown}s", file=sys.stderr)
+            with open(outdir / "aborted.txt", "a") as af:
+                af.write(last + "\n")
+            time.sleep(cooldown)
+
+            shard_lines = read_lines(list_path)
+            seen = False
+            remaining = []
+            for l in shard_lines:
+                if seen:
+                    remaining.append(l)
+                if l == last:
+                    seen = True
+            write_lines(list_path, remaining)
+            pass_n += 1
+
+    threads = [threading.Thread(target=run_shard, args=(i,)) for i in range(shards)]
+    for t in threads:
+        t.start()
+    for t in threads:
+        t.join()
+
+    res = {}
+    for f in sorted(glob.glob(str(outdir / "shard*.qpa"))) + sorted(glob.glob(str(outdir / "attempt-*/shard*.qpa"))):
+        txt = open(f, errors="replace").read()
+        for block in re.split(r"#beginTestCaseResult ", txt)[1:]:
+            name = block.split("\n", 1)[0].strip()
+            m = re.search(r'<Result StatusCode="(\w+)">', block)
+            if m:
+                res[name] = m.group(1)
+
+    counts = collections.Counter(res.values())
+    print(f"\n{sum(counts.values())} results")
+    for k, v in counts.most_common():
+        print(f"  {k:16} {v}")
+
+    with open(outdir / "failures.txt", "w") as fh:
+        for name, status in sorted(res.items()):
+            if status not in ("Pass", "NotSupported"):
+                fh.write(f"{status}\t{name}\n")
+    print(f"\nfailures -> {outdir}/failures.txt")
+
+
+if __name__ == "__main__":
+    main()

cts/scan-2026-08-20/hangs.txt

@@ -0,0 +1,8 @@
+2026-08-20T16:08:40+03:00 chunk 1 ended badly (rc=1)
+2026-08-20T16:13:52+03:00 chunk 2 ended badly (rc=1)
+2026-08-20T16:24:23+03:00 chunk 3 ended badly (rc=1)
+2026-08-20T18:03:06+03:00 chunk 6 ended badly (rc=1)
+2026-08-20T18:48:18+03:00 chunk 10 ended badly (rc=139)
+2026-08-20T18:54:08+03:00 chunk 11 ended badly (rc=139)
+2026-08-20T20:28:44+03:00 chunk 9 ended badly (rc=0)
+2026-08-20T21:15:33+03:00 chunk 13 ended badly (rc=1)