init

bin/ci/.gitignore

@@ -0,0 +1,2 @@
+schema.graphql
+gitlab_gql.py.cache*

bin/ci/ci_gantt_chart.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+# Copyright © 2023 Collabora Ltd.
+# Authors:
+#   Helen Koike <[email protected]>
+#
+# For the dependencies, see the requirements.txt
+# SPDX-License-Identifier: MIT
+
+
+import argparse
+from datetime import datetime, timedelta, timezone
+from typing import Dict, List
+
+import plotly.express as px
+import plotly.graph_objs as go
+from gitlab import Gitlab, base
+from gitlab.v4.objects import ProjectPipeline
+from gitlab_common import (GITLAB_URL, get_gitlab_pipeline_from_url,
+                           get_token_from_default_dir, pretty_duration,
+                           read_token)
+
+
+def calculate_queued_at(job) -> datetime:
+    started_at = job.started_at.replace("Z", "+00:00")
+    return datetime.fromisoformat(started_at) - timedelta(seconds=job.queued_duration)
+
+
+def calculate_time_difference(time1, time2) -> str:
+    if type(time1) is str:
+        time1 = datetime.fromisoformat(time1.replace("Z", "+00:00"))
+    if type(time2) is str:
+        time2 = datetime.fromisoformat(time2.replace("Z", "+00:00"))
+
+    diff = time2 - time1
+    return pretty_duration(diff.seconds)
+
+
+def create_task_name(job) -> str:
+    status_color = {"success": "green", "failed": "red"}.get(job.status, "grey")
+    return f"{job.name}\t(<span style='color: {status_color}'>{job.status}</span>,<a href='{job.web_url}'>{job.id}</a>)"
+
+
+def add_gantt_bar(
+    job: base.RESTObject, tasks: List[Dict[str, str | datetime | timedelta]]
+) -> None:
+    queued_at = calculate_queued_at(job)
+    task_name = create_task_name(job)
+
+    tasks.append(
+        {
+            "Job": task_name,
+            "Start": job.created_at,
+            "Finish": queued_at,
+            "Duration": calculate_time_difference(job.created_at, queued_at),
+            "Phase": "Waiting dependencies",
+        }
+    )
+    tasks.append(
+        {
+            "Job": task_name,
+            "Start": queued_at,
+            "Finish": job.started_at,
+            "Duration": calculate_time_difference(queued_at, job.started_at),
+            "Phase": "Queued",
+        }
+    )
+
+    if job.finished_at:
+        tasks.append(
+            {
+                "Job": task_name,
+                "Start": job.started_at,
+                "Finish": job.finished_at,
+                "Duration": calculate_time_difference(job.started_at, job.finished_at),
+                "Phase": "Time spent running",
+            }
+        )
+    else:
+        current_time = datetime.now(timezone.utc).isoformat()
+        tasks.append(
+            {
+                "Job": task_name,
+                "Start": job.started_at,
+                "Finish": current_time,
+                "Duration": calculate_time_difference(job.started_at, current_time),
+                "Phase": "In-Progress",
+            }
+        )
+
+
+def generate_gantt_chart(
+    pipeline: ProjectPipeline, ci_timeout: float = 60
+) -> go.Figure:
+    if pipeline.yaml_errors:
+        raise ValueError("Pipeline YAML errors detected")
+
+    # Convert the data into a list of dictionaries for plotly
+    tasks: List[Dict[str, str | datetime | timedelta]] = []
+
+    for job in pipeline.jobs.list(all=True, include_retried=True):
+        # we can have queued_duration without started_at when a job is canceled
+        if not job.queued_duration or not job.started_at:
+            continue
+        add_gantt_bar(job, tasks)
+
+    # Make it easier to see retried jobs
+    tasks.sort(key=lambda x: x["Job"])
+
+    title = f"Gantt chart of jobs in pipeline <a href='{pipeline.web_url}'>{pipeline.web_url}</a>."
+    title += (
+        f" Total duration {str(timedelta(seconds=pipeline.duration))}"
+        if pipeline.duration
+        else ""
+    )
+
+    # Create a Gantt chart
+    default_colors = px.colors.qualitative.Plotly
+    fig: go.Figure = px.timeline(
+        tasks,
+        x_start="Start",
+        x_end="Finish",
+        y="Job",
+        color="Phase",
+        title=title,
+        hover_data=["Duration"],
+        color_discrete_map={
+            "In-Progress": default_colors[3],  # purple
+            "Waiting dependencies": default_colors[0],  # blue
+            "Queued": default_colors[1],  # red
+            "Time spent running": default_colors[2],  # green
+        },
+    )
+
+    # Calculate the height dynamically
+    fig.update_layout(height=len(tasks) * 10, yaxis_tickfont_size=14)
+
+    # Add a deadline line to the chart
+    created_at = datetime.fromisoformat(pipeline.created_at.replace("Z", "+00:00"))
+    timeout_at = created_at + timedelta(minutes=ci_timeout)
+    fig.add_vrect(
+        x0=timeout_at,
+        x1=timeout_at,
+        annotation_text=f"{int(ci_timeout)} min Timeout",
+        fillcolor="gray",
+        line_width=2,
+        line_color="gray",
+        line_dash="dash",
+        annotation_position="top left",
+        annotation_textangle=90,
+    )
+
+    return fig
+
+
+def main(
+    token: str | None,
+    pipeline_url: str,
+    output: str | None,
+    ci_timeout: float = 60,
+):
+    if token is None:
+        token = get_token_from_default_dir()
+
+    token = read_token(token)
+    gl = Gitlab(url=GITLAB_URL, private_token=token, retry_transient_errors=True)
+
+    pipeline, _ = get_gitlab_pipeline_from_url(gl, pipeline_url)
+    fig: go.Figure = generate_gantt_chart(pipeline, ci_timeout)
+    if output and "htm" in output:
+        fig.write_html(output)
+    elif output:
+        fig.update_layout(width=1000)
+        fig.write_image(output)
+    else:
+        fig.show()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Generate the Gantt chart from a given pipeline."
+    )
+    parser.add_argument("pipeline_url", type=str, help="URLs to the pipeline.")
+    parser.add_argument(
+        "-o",
+        "--output",
+        type=str,
+        help="Output file name. Use html or image suffixes to choose the format.",
+    )
+    parser.add_argument(
+        "--token",
+        metavar="token",
+        help="force GitLab token, otherwise it's read from ~/.config/gitlab-token",
+    )
+    parser.add_argument(
+        "--ci-timeout",
+        metavar="ci_timeout",
+        type=float,
+        default=60,
+        help="Time that marge-bot will wait for ci to finish. Defaults to one hour.",
+    )
+    args = parser.parse_args()
+    main(args.token, args.pipeline_url, args.output, args.ci_timeout)

bin/ci/ci_gantt_chart.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -eu
+
+this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
+readonly this_dir
+
+exec \
+  "$this_dir/../python-venv.sh" \
+  "$this_dir/requirements.txt" \
+  "$this_dir/ci_gantt_chart.py" "$@"

bin/ci/ci_post_gantt.py

@@ -0,0 +1,223 @@
+#!/usr/bin/env python3
+# Copyright © 2023 Collabora Ltd.
+# Authors:
+#   Helen Koike <[email protected]>
+#
+# For the dependencies, see the requirements.txt
+# SPDX-License-Identifier: MIT
+
+
+import argparse
+import logging as log
+import os
+import re
+import traceback
+from datetime import datetime, timedelta
+from typing import Any, Dict
+
+import gitlab
+import pytz
+from ci_gantt_chart import generate_gantt_chart
+from gitlab import Gitlab
+from gitlab.base import RESTObject
+from gitlab.v4.objects import Project, ProjectPipeline
+from gitlab_common import (GITLAB_URL, get_gitlab_pipeline_from_url,
+                           get_token_from_default_dir, read_token)
+
+
+class MockGanttExit(Exception):
+    pass
+
+
+LAST_MARGE_EVENT_FILE = os.path.expanduser("~/.config/last_marge_event")
+
+
+def read_last_event_date_from_file() -> str:
+    try:
+        with open(LAST_MARGE_EVENT_FILE, "r") as f:
+            last_event_date = f.read().strip()
+    except FileNotFoundError:
+        # 3 days ago
+        last_event_date = (datetime.now() - timedelta(days=3)).isoformat()
+    return last_event_date
+
+
+def pretty_time(time_str: str) -> str:
+    """Pretty print time"""
+    local_timezone = datetime.now().astimezone().tzinfo
+
+    time_d = datetime.fromisoformat(time_str.replace("Z", "+00:00")).astimezone(
+        local_timezone
+    )
+    return f'{time_str} ({time_d.strftime("%d %b %Y %Hh%Mm%Ss")} {local_timezone})'
+
+
+def compose_message(file_name: str, attachment_url: str) -> str:
+    return f"""
+[{file_name}]({attachment_url})
+
+<details>
+<summary>more info</summary>
+
+This message was generated by the ci_post_gantt.py script, which is running on a server at Collabora.
+</details>
+"""
+
+
+def gitlab_upload_file_get_url(gl: Gitlab, project_id: str, filepath: str) -> str:
+    project: Project = gl.projects.get(project_id)
+    uploaded_file: Dict[str, Any] = project.upload(filepath, filepath=filepath)
+    return uploaded_file["url"]
+
+
+def gitlab_post_reply_to_note(gl: Gitlab, event: RESTObject, reply_message: str):
+    """
+    Post a reply to a note in thread based on a GitLab event.
+
+    :param gl: The GitLab connection instance.
+    :param event: The event object containing the note details.
+    :param reply_message: The reply message.
+    """
+    try:
+        note_id = event.target_id
+        merge_request_iid = event.note["noteable_iid"]
+
+        project = gl.projects.get(event.project_id)
+        merge_request = project.mergerequests.get(merge_request_iid)
+
+        # Find the discussion to which the note belongs
+        discussions = merge_request.discussions.list(iterator=True)
+        target_discussion = next(
+            (
+                d
+                for d in discussions
+                if any(n["id"] == note_id for n in d.attributes["notes"])
+            ),
+            None,
+        )
+
+        if target_discussion is None:
+            raise ValueError("Discussion for the note not found.")
+
+        # Add a reply to the discussion
+        reply = target_discussion.notes.create({"body": reply_message})
+        return reply
+
+    except gitlab.exceptions.GitlabError as e:
+        log.error(f"Failed to post a reply to '{event.note['body']}': {e}")
+        return None
+
+
+def main(
+    token: str | None,
+    since: str | None,
+    marge_user_id: int = 9716,
+    project_ids: list[int] = [176],
+    ci_timeout: float = 60,
+):
+    log.basicConfig(level=log.INFO)
+    if token is None:
+        token = get_token_from_default_dir()
+
+    token = read_token(token)
+    gl = Gitlab(url=GITLAB_URL, private_token=token, retry_transient_errors=True)
+
+    user = gl.users.get(marge_user_id)
+    last_event_at = since if since else read_last_event_date_from_file()
+
+    log.info(f"Retrieving Marge messages since {pretty_time(last_event_at)}\n")
+
+    # the "after" only considers the "2023-10-24" part, it doesn't consider the time
+    events = user.events.list(
+        all=True,
+        target_type="note",
+        after=(datetime.now() - timedelta(days=3)).isoformat(),
+        sort="asc",
+    )
+
+    last_event_at_date = datetime.fromisoformat(
+        last_event_at.replace("Z", "+00:00")
+    ).replace(tzinfo=pytz.UTC)
+
+    for event in events:
+        if event.project_id not in project_ids:
+            continue
+        created_at_date = datetime.fromisoformat(
+            event.created_at.replace("Z", "+00:00")
+        ).replace(tzinfo=pytz.UTC)
+        if created_at_date <= last_event_at_date:
+            continue
+        last_event_at = event.created_at
+
+        escaped_gitlab_url = re.escape(GITLAB_URL)
+        match = re.search(rf"{escaped_gitlab_url}/[^\s<]+", event.note["body"])
+
+        if match:
+            try:
+                log.info(f"Found message: {event.note['body']}")
+                pipeline_url = match.group(0)[:-1]
+                pipeline: ProjectPipeline
+                pipeline, _ = get_gitlab_pipeline_from_url(gl, pipeline_url)
+                log.info("Generating gantt chart...")
+                fig = generate_gantt_chart(pipeline, ci_timeout)
+                file_name = f"{str(pipeline.id)}-Gantt.html"
+                fig.write_html(file_name)
+                log.info("Uploading gantt file...")
+                file_url = gitlab_upload_file_get_url(gl, event.project_id, file_name)
+                log.info("Posting reply ...")
+                message = compose_message(file_name, file_url)
+                gitlab_post_reply_to_note(gl, event, message)
+            except MockGanttExit:
+                pass  # Allow tests to exit early without printing a traceback
+            except Exception as e:
+                log.info(f"Failed to generate gantt chart, not posting reply.{e}")
+                traceback.print_exc()
+
+        if not since:
+            log.info(
+                f"Updating last event date to {pretty_time(last_event_at)} on {LAST_MARGE_EVENT_FILE}\n"
+            )
+            with open(LAST_MARGE_EVENT_FILE, "w") as f:
+                f.write(last_event_at)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Monitor rejected pipelines by Marge.")
+    parser.add_argument(
+        "--token",
+        metavar="token",
+        type=str,
+        default=None,
+        help="force GitLab token, otherwise it's read from ~/.config/gitlab-token",
+    )
+    parser.add_argument(
+        "--since",
+        metavar="since",
+        type=str,
+        default=None,
+        help="consider only events after this date (ISO format), otherwise it's read from ~/.config/last_marge_event",
+    )
+    parser.add_argument(
+        "--marge-user-id",
+        metavar="marge_user_id",
+        type=int,
+        default=9716,  # default https://gitlab.freedesktop.org/users/marge-bot/activity
+        help="GitLab user ID for marge-bot, defaults to 9716",
+    )
+    parser.add_argument(
+        "--project-id",
+        metavar="project_id",
+        type=int,
+        nargs="+",
+        default=[176],  # default is the mesa/mesa project id
+        help="GitLab project id(s) to analyze. Defaults to 176 i.e. mesa/mesa.",
+    )
+    parser.add_argument(
+        "--ci-timeout",
+        metavar="ci_timeout",
+        type=float,
+        default=60,
+        help="Time that marge-bot will wait for ci to finish. Defaults to one hour.",
+    )
+    args = parser.parse_args()
+    main(args.token, args.since, args.marge_user_id, args.project_id, args.ci_timeout)

bin/ci/ci_post_gantt.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -eu
+
+this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
+readonly this_dir
+
+exec \
+  "$this_dir/../python-venv.sh" \
+  "$this_dir/requirements.txt" \
+  "$this_dir/ci_post_gantt.py" "$@"

bin/ci/ci_run_n_monitor.py

@@ -0,0 +1,808 @@
+#!/usr/bin/env python3
+# Copyright © 2020 - 2022 Collabora Ltd.
+# Authors:
+#   Tomeu Vizoso <[email protected]>
+#   David Heidelberg <[email protected]>
+#
+# For the dependencies, see the requirements.txt
+# SPDX-License-Identifier: MIT
+
+"""
+Helper script to restrict running only required CI jobs
+and show the job(s) logs.
+"""
+
+import argparse
+import re
+import sys
+import time
+from collections import defaultdict, Counter
+from concurrent.futures import ThreadPoolExecutor
+from functools import partial
+from itertools import chain
+from subprocess import check_output, CalledProcessError
+from typing import Callable, Dict, TYPE_CHECKING, Iterable, Literal, Optional, Tuple, cast
+
+import gitlab
+import gitlab.v4.objects
+from gitlab_common import (
+    GITLAB_URL,
+    TOKEN_DIR,
+    get_gitlab_pipeline_from_url,
+    get_gitlab_project,
+    get_token_from_default_dir,
+    is_gitlab_job,
+    pretty_duration,
+    read_token,
+    wait_for_pipeline,
+)
+from gitlab_gql import GitlabGQL, create_job_needs_dag, filter_dag, print_dag, print_formatted_list
+from rich.console import Console
+
+if TYPE_CHECKING:
+    from gitlab_gql import Dag
+
+REFRESH_WAIT_LOG = 10
+REFRESH_WAIT_JOBS = 6
+MAX_ENABLE_JOB_ATTEMPTS = 3
+
+STATUS_COLORS = defaultdict(lambda: "", {
+    "running": "[blue]",
+    "success": "[green]",
+    "failed": "[red]",
+    "canceled": "[magenta]",
+    "canceling": "[magenta]",
+})
+
+COMPLETED_STATUSES = frozenset({"success", "failed"})
+RUNNING_STATUSES = frozenset({"created", "pending", "running"})
+
+if is_gitlab_job():
+    console = Console(highlight=False, no_color=False, color_system="truecolor", width=120)
+else:
+    console = Console(highlight=False)
+print = console.print
+
+
+def print_job_status(
+    job: gitlab.v4.objects.ProjectPipelineJob,
+    new_status: bool = False,
+) -> None:
+    """It prints a nice, colored job status with a link to the job."""
+    if job.status in {"canceled", "canceling"}:
+        return
+
+    if new_status and job.status == "created":
+        return
+
+    global type_field_pad
+    global name_field_pad
+    jtype = "🞋 job"
+    job_name = job.name
+    type_field_pad = len(jtype) if len(jtype) > type_field_pad else type_field_pad
+    name_field_pad = len(job_name) if len(job_name) > name_field_pad else name_field_pad
+
+    duration = job_duration(job)
+
+    print(
+        f"{STATUS_COLORS[job.status]}"
+        f"{jtype:{type_field_pad}} "  # U+1F78B Round target
+        f"{link2print(job.web_url, job.name, name_field_pad)} " +
+        (f" has new status: {job.status}" if new_status else f" {job.status}") +
+        (f" ({pretty_duration(duration)})" if job.started_at else "")
+    )
+
+
+def job_duration(job: gitlab.v4.objects.ProjectPipelineJob) -> float:
+    """
+    Given a job, report the time lapsed in execution.
+    :param job: Pipeline job
+    :return: Current time in execution
+    """
+    if job.duration:
+        return job.duration
+    elif job.started_at:
+        # Convert both times to UTC timestamps for consistent comparison
+        current_time = time.time()
+        start_time = job.started_at.timestamp()
+        return current_time - start_time
+    return 0.0
+
+
+def pretty_wait(sec: int) -> None:
+    """shows progressbar in dots"""
+    if is_gitlab_job():
+        time.sleep(sec)
+        return
+    for val in range(sec, 0, -1):
+        print(f"⏲  {val:2d} seconds", end="\r")  # U+23F2 Timer clock
+        time.sleep(1)
+
+
+def run_target_job(
+    job: gitlab.v4.objects.ProjectPipelineJob,
+    enable_job_fn: Callable,
+    stress: int,
+    execution_times: dict,
+    target_statuses: dict,
+) -> None:
+    execution_times[job.name][job.id] = (job_duration(job), job.status, job.web_url)
+    if stress and job.status in COMPLETED_STATUSES:
+        if (
+            stress < 0
+            or len(execution_times[job.name]) < stress
+        ):
+            enable_job_fn(job=job, action_type="retry")
+            # Wait for the next loop to get the updated job object
+            return
+    else:
+        enable_job_fn(job=job, action_type="target")
+
+    print_job_status(job, job.status not in target_statuses[job.name])
+    target_statuses[job.name] = job.status
+
+
+def monitor_pipeline(
+    project: gitlab.v4.objects.Project,
+    pipeline: gitlab.v4.objects.ProjectPipeline,
+    job_filter: callable,
+    dependencies: set[str],
+    stress: int,
+    inhibit_single_target_trace: int = False,
+    polling_period: int = REFRESH_WAIT_JOBS,
+) -> tuple[Optional[int], Optional[int], Dict[str, Dict[int, Tuple[float, str, str]]]]:
+    """Monitors pipeline and delegate canceling jobs"""
+    statuses: dict[str, str] = defaultdict(str)
+    target_statuses: dict[str, str] = defaultdict(str)
+    execution_times: dict[str, dict[str, tuple[float, str, str]]] = defaultdict(lambda: defaultdict(tuple))
+    target_id: int = -1
+    global type_field_pad
+    type_field_pad = 0
+    global name_field_pad
+    name_field_pad = len(max(dependencies, key=len))+2
+    # In a running pipeline, we can skip following job traces that are in these statuses.
+    skip_follow_statuses: frozenset[str] = (COMPLETED_STATUSES)
+
+    # Pre-populate the stress status counter for already completed target jobs.
+    if stress:
+        # When stress test, it is necessary to collect this information before start.
+        for job in pipeline.jobs.list(all=True, include_retried=True):
+            if job_filter(
+                job_name=job.name,
+                job_stage=job.stage,
+                job_tags=job.tag_list,
+            ) and job.status in COMPLETED_STATUSES:
+                execution_times[job.name][job.id] = (job_duration(job), job.status, job.web_url)
+
+    # jobs_waiting is a list of job names that are waiting for status update.
+    # It occurs when a job that we want to run depends on another job that is not yet finished.
+    jobs_waiting = []
+    # Dictionary to track the number of attempts made for each job for a given status
+    enable_attempts: dict[tuple[int, str], int] = {}
+    # FIXME: This function has too many parameters, consider refactoring.
+    enable_job_fn = partial(
+        enable_job,
+        project=project,
+        enable_attempts=enable_attempts,
+        jobs_waiting=jobs_waiting,
+    )
+    while True:
+        deps_failed = []
+        to_cancel = []
+        jobs_waiting.clear()
+        for job in sorted(pipeline.jobs.list(all=True), key=lambda j: j.name):
+            job = cast(gitlab.v4.objects.ProjectPipelineJob, job)
+            if job_filter(
+                job_name=job.name,
+                job_stage=job.stage,
+                job_tags=job.tag_list,
+            ):
+                run_target_job(
+                    job,
+                    enable_job_fn,
+                    stress,
+                    execution_times,
+                    target_statuses
+                )
+                target_id = job.id
+                continue
+            # all other non-target jobs
+            if job.status != statuses[job.name]:
+                print_job_status(job, True)
+                statuses[job.name] = job.status
+
+            # run dependencies and cancel the rest
+            if job.name in dependencies:
+                if not enable_job_fn(job=job, action_type="dep"):
+                    # Wait for the next loop to get the updated job object
+                    continue
+                if job.status == "failed":
+                    deps_failed.append(job.name)
+            else:
+                to_cancel.append(job)
+
+        cancel_jobs(project, to_cancel)
+
+        if stress:
+            enough = True
+            status_counters = {
+                name: Counter(info[1] for info in runs.values())
+                for name, runs in execution_times.items()
+            }
+            for job_name, counter in sorted(status_counters.items()):
+                n_succeed = counter.get("success", 0)
+                n_failed = counter.get("failed", 0)
+                n_total_completed = n_succeed + n_failed
+                n_total_seen = len(execution_times[job_name])
+                print(
+                    f"* {job_name:{name_field_pad}} succ: {n_succeed}; "
+                    f"fail: {n_failed}; "
+                    f"total: {n_total_seen} of {stress}",
+                )
+                if stress < 0 or n_total_completed < stress:
+                    enough = False
+
+            if not enough:
+                pretty_wait(polling_period)
+                continue
+
+        if jobs_waiting:
+            print(f"[yellow]Waiting for jobs to update status:")
+            print_formatted_list(jobs_waiting, indentation=8, color="[yellow]")
+            pretty_wait(polling_period)
+            continue
+
+        if (
+            not inhibit_single_target_trace
+            and stress in [0, 1]
+            and len(target_statuses) == 1
+            and RUNNING_STATUSES.intersection(target_statuses.values())
+        ):
+            return target_id, None, execution_times
+
+        if (
+            {"failed"}.intersection(target_statuses.values())
+            and not RUNNING_STATUSES.intersection(target_statuses.values())
+        ):
+            return None, 1, execution_times
+
+        if (
+            {"skipped"}.intersection(target_statuses.values())
+            and not RUNNING_STATUSES.intersection(target_statuses.values())
+        ):
+            print(
+                f"[red]Target in skipped state, aborting. Failed dependencies:{deps_failed}"
+            )
+            return None, 1, execution_times
+
+        if skip_follow_statuses.issuperset(target_statuses.values()):
+            return None, 0, execution_times
+
+        pretty_wait(polling_period)
+
+
+def enable_job(
+    project: gitlab.v4.objects.Project,
+    job: gitlab.v4.objects.ProjectPipelineJob,
+    enable_attempts: dict[tuple[int, str], int],
+    action_type: Literal["target", "dep", "retry"],
+    jobs_waiting: list[str] = list,
+) -> bool:
+    """
+    Enable a job to run.
+    :param project: The GitLab project.
+    :param job: The job to enable.
+    :param enable_attempts: A dictionary to track the number of attempts made for each job for a give status.
+    :param action_type: The type of action to perform.
+    :param jobs_waiting:
+    :return: True if the job was enabled, False otherwise.
+    """
+    # We want to run this job, but it is not ready to run yet, so let's try again in the next
+    # iteration.
+    if job.status == "created":
+        jobs_waiting.append(job.name)
+        return False
+
+    if (
+        (job.status in COMPLETED_STATUSES and action_type != "retry")
+        or job.status in {"skipped"} | RUNNING_STATUSES
+    ):
+        return False
+
+    # Get current attempt number
+    attempt_count = enable_attempts.get((job.id, job.status), 0)
+    # Check if we've exceeded max attempts to avoid infinite loop
+    if attempt_count == MAX_ENABLE_JOB_ATTEMPTS:
+        print(
+            f"[yellow]WARNING: "
+            f"Maximum enabling attempts ({MAX_ENABLE_JOB_ATTEMPTS}) reached for job {job.name} in {job.status} status"
+            f"({link2print(job.web_url, job.id)})."
+        )
+        enable_attempts[(job.id, job.status)] = attempt_count + 1
+        return False
+    elif attempt_count > MAX_ENABLE_JOB_ATTEMPTS:
+        return False
+
+    enable_attempts[(job.id, job.status)] = attempt_count + 1
+
+    pjob = project.jobs.get(job.id, lazy=True)
+
+    if job.status in {"success", "failed", "canceled", "canceling"}:
+        try:
+            pjob.retry()
+        except Exception as e:
+            print(f"Error retrying job {job.name}: {e}")
+            return False
+    else:
+        try:
+            pjob.play()
+        except Exception as e:
+            print(f"Error playing job {job.name}: {e}")
+            return False
+
+    if action_type == "target":
+        jtype = "🞋 target"  # U+1F78B Round target
+    elif action_type == "retry":
+        jtype = "↻ retrying"  # U+21BB Clockwise open circle arrow
+    else:
+        jtype = "↪ dependency"  # U+21AA Left Arrow Curving Right
+
+    global type_field_pad
+    global name_field_pad
+    job_name = job.name
+    type_field_pad = len(jtype) if len(jtype) > type_field_pad else type_field_pad
+    name_field_pad = len(job_name) if len(job_name) > name_field_pad else name_field_pad
+    print(
+        f"[magenta]{jtype:{type_field_pad}} {job.name:{name_field_pad}} manually enabled"
+    )
+
+    return True
+
+
+def cancel_job(
+    project: gitlab.v4.objects.Project,
+    pipeline_job: gitlab.v4.objects.ProjectPipelineJob
+) -> Optional[gitlab.v4.objects.ProjectPipelineJob]:
+    """
+    Cancel GitLab job
+    :param project: project from the pipeline job comes from
+    :param pipeline_job: job made from the pipeline list
+    :return the job object when cancel was called
+    """
+    if pipeline_job.status not in RUNNING_STATUSES:
+        return
+    try:
+        project_job = project.jobs.get(pipeline_job.id, lazy=True)
+        project_job.cancel()
+    except (gitlab.GitlabCancelError, gitlab.GitlabGetError):
+        # If the job failed to cancel, it will be retried in the monitor_pipeline() next iteration
+        return
+    return pipeline_job
+
+
+def cancel_jobs(
+    project: gitlab.v4.objects.Project,
+    to_cancel: list[gitlab.v4.objects.ProjectPipelineJob]
+) -> None:
+    """
+    Cancel unwanted GitLab jobs
+    :param project: project from where the pipeline comes
+    :param to_cancel: list of jobs to be cancelled
+    """
+    if not to_cancel:
+        return
+
+    with ThreadPoolExecutor(max_workers=6) as exe:
+        part = partial(cancel_job, project)
+        maybe_cancelled_job = exe.map(part, to_cancel)
+        cancelled_jobs = [f"🗙 {job.name}" for job in maybe_cancelled_job if job]  # U+1F5D9 Cancellation X
+
+    # The cancelled jobs are printed without a newline
+    if len(cancelled_jobs):
+        print(f"Cancelled {len(cancelled_jobs)} jobs:")
+        print_formatted_list(cancelled_jobs, indentation=8)
+
+
+def print_log(
+    project: gitlab.v4.objects.Project,
+    job_id: int
+) -> None:
+    """Print job log into output"""
+    printed_lines = 0
+    while True:
+        job = project.jobs.get(job_id)
+
+        # GitLab's REST API doesn't offer pagination for logs, so we have to refetch it all
+        lines = job.trace().decode().splitlines()
+        for line in lines[printed_lines:]:
+            print(line, markup=False)
+        printed_lines = len(lines)
+
+        if job.status in COMPLETED_STATUSES:
+            print(f"[green]Job finished: {job.web_url}")
+            return
+        pretty_wait(REFRESH_WAIT_LOG)
+
+
+def parse_args() -> argparse.Namespace:
+    """Parse args"""
+    parser = argparse.ArgumentParser(
+        description="Tool to trigger a subset of container jobs "
+        + "and monitor the progress of a test job",
+        epilog="Example: %(prog)s --rev $(git rev-parse HEAD) "
+        + '--target ".*traces" ',
+    )
+    parser.add_argument(
+        "--server",
+        metavar="gitlab-server",
+        type=str,
+        default=GITLAB_URL,
+        help=f"Specify the GitLab server work with (Default: {GITLAB_URL})",
+    )
+    parser.add_argument(
+        "--target",
+        metavar="target-job",
+        help="Target job regex. For multiple targets, pass multiple values, "
+             "eg. `--target foo bar`. Only jobs in the target stage(s) "
+             "supplied, and their dependencies, will be considered.",
+        required=True,
+        nargs=argparse.ONE_OR_MORE,
+    )
+    parser.add_argument(
+        "--include-stage",
+        metavar="include-stage",
+        help="Job stages to include when searching for target jobs. "
+             "For multiple targets, pass multiple values, eg. "
+             "`--include-stage foo bar`.",
+        default=[".*"],
+        nargs=argparse.ONE_OR_MORE,
+    )
+    parser.add_argument(
+        "--exclude-stage",
+        metavar="exclude-stage",
+        help="Job stages to exclude when searching for target jobs. "
+             "For multiple targets, pass multiple values, eg. "
+             "`--exclude-stage foo bar`. By default, performance and "
+             "nightly jobs are excluded; pass --exclude-stage '' to "
+             "include them for consideration.",
+        default=["performance", ".*-postmerge", ".*-nightly"],
+        nargs=argparse.ONE_OR_MORE,
+    )
+    parser.add_argument(
+        "--job-tags",
+        metavar="job-tags",
+        help="Job tags to require when searching for target jobs. If multiple "
+             "values are passed, eg. `--job-tags 'foo.*' 'bar'`, the job will "
+             "need to have a tag matching `foo.*` *and* a tag matching `bar` "
+             "to qualify. Passing `--job-tags '.*'` makes sure the job has "
+             "a tag defined, while not passing `--job-tags` also allows "
+             "untagged jobs.",
+        default=[],
+        nargs=argparse.ONE_OR_MORE,
+    )
+    parser.add_argument(
+        "--token",
+        metavar="token",
+        type=str,
+        default=get_token_from_default_dir(),
+        help="Use the provided GitLab token (with `api` scope) or token file, "
+             f"otherwise it's read from {TOKEN_DIR / 'gitlab-token'}",
+    )
+    parser.add_argument(
+        "--force-manual", action="store_true",
+        help="Deprecated argument; manual jobs are always force-enabled"
+    )
+    parser.add_argument(
+        "--stress",
+        metavar="n",
+        type=int,
+        default=0,
+        help="Stresstest job(s). Specify the number of times to rerun the selected jobs, "
+             "or use -1 for indefinite. Defaults to 0. If jobs have already been executed, "
+             "this will ensure the total run count respects the specified number.",
+    )
+    parser.add_argument(
+        "--project",
+        metavar="name",
+        type=str,
+        default="mesa",
+        help="GitLab project in the format <user>/<project> or just <project>",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Exit after printing target jobs and dependencies",
+    )
+    parser.add_argument(
+        "--no-job-log",
+        action="store_true",
+        help="When there is only one target job, inhibit the job trace output in the console.",
+    )
+    parser.add_argument(
+        "--polling-period",
+        type=int,
+        default=REFRESH_WAIT_JOBS,
+        help=f"Specify the waiting seconds between monitor loops. (Default: {REFRESH_WAIT_JOBS})",
+     )
+
+
+    mutex_group1 = parser.add_mutually_exclusive_group()
+    mutex_group1.add_argument(
+        "--rev",
+        metavar="id",
+        type=str,
+        default="HEAD",
+        help="Repository git commit-ish, tag or branch name (default: HEAD)",
+    )
+    mutex_group1.add_argument(
+        "--pipeline-url",
+        metavar="url",
+        type=str,
+        help="URL of the pipeline to use, instead of auto-detecting it.",
+    )
+    mutex_group1.add_argument(
+        "--mr",
+        metavar="id",
+        type=int,
+        help="ID of a merge request; the latest pipeline in that MR will be used.",
+    )
+
+    args = parser.parse_args()
+
+    # argparse doesn't support groups inside add_mutually_exclusive_group(),
+    # which means we can't just put `--project` and `--rev` in a group together,
+    # we have to do this by heand instead.
+    if args.pipeline_url and args.project != parser.get_default("project"):
+        # weird phrasing but it's the error add_mutually_exclusive_group() gives
+        parser.error("argument --project: not allowed with argument --pipeline-url")
+
+    return args
+
+
+def print_detected_jobs(
+    target_dep_dag: "Dag",
+    dependency_jobs: Iterable[str],
+    target_jobs: Iterable[str],
+) -> None:
+    def print_job_set(color: str, kind: str, job_set: Iterable[str]):
+        job_list = list(job_set)
+        print(f"{color}Running {len(job_list)} {kind} jobs:")
+        print_formatted_list(job_list, indentation=8, color=color)
+
+    print("[yellow]Detected target job and its dependencies:")
+    print_dag(target_dep_dag, indentation=8, color="[yellow]")
+    print_job_set("[magenta]", "dependency", dependency_jobs)
+    print_job_set("[blue]", "target", target_jobs)
+
+
+def find_dependencies(
+    server: str,
+    token: str | None,
+    job_filter: callable,
+    project_path: str,
+    iid: int
+) -> set[str]:
+    """
+    Find the dependencies of the target jobs in a GitLab pipeline.
+
+    This function uses the GitLab GraphQL API to fetch the job dependency graph
+    of a pipeline, filters the graph to only include the target jobs and their
+    dependencies, and returns the names of these jobs.
+
+    Args:
+        server (str): The url to the GitLab server.
+        token (str | None): The GitLab API token. If None, the API is accessed without
+                            authentication.
+        target_jobs_regex (re.Pattern): A regex pattern to match the names of the target jobs.
+        project_path (str): The path of the GitLab project.
+        iid (int): The internal ID of the pipeline.
+
+    Returns:
+        set[str]: A set of the names of the target jobs and their dependencies.
+
+    Raises:
+        SystemExit: If no target jobs are found in the pipeline.
+    """
+    gql_instance = GitlabGQL(
+        url=f"{server}/api/graphql",
+        token=token
+    )
+    dag = create_job_needs_dag(
+        gql_instance, {"projectPath": project_path.path_with_namespace, "iid": iid}
+    )
+
+    target_dep_dag = filter_dag(dag, job_filter)
+    if not target_dep_dag:
+        print("[red]The job(s) were not found in the pipeline.")
+        sys.exit(1)
+
+    dependency_jobs = set(chain.from_iterable(d["needs"] for d in target_dep_dag.values()))
+    target_jobs = set(target_dep_dag.keys())
+    print_detected_jobs(target_dep_dag, dependency_jobs, target_jobs)
+    return target_jobs.union(dependency_jobs)
+
+
+def print_monitor_summary(
+    execution_collection: Dict[str, Dict[int, Tuple[float, str, str]]],
+    t_start: float,
+) -> None:
+    """Summary of the test execution"""
+    t_end = time.perf_counter()
+    spend_minutes = (t_end - t_start) / 60
+    print(f"⏲ Duration of script execution: {spend_minutes:0.1f} minutes")  # U+23F2 Timer clock
+    if len(execution_collection) == 0:
+        return
+    print(f"⏲ Jobs execution times:")  # U+23F2 Timer clock
+    job_names = list(execution_collection.keys())
+    job_names.sort()
+    name_field_pad = len(max(job_names, key=len)) + 2
+    for name in job_names:
+        job_executions = execution_collection[name]
+        job_times = ', '.join([__job_duration_record(job_execution)
+                               for job_execution in sorted(job_executions.items())])
+        print(f"* {name:{name_field_pad}}: ({len(job_executions)}) {job_times}")
+
+
+def __job_duration_record(dict_item: tuple) -> str:
+    """
+    Format each pair of job and its duration.
+    :param job_execution: item of execution_collection[name][idn]: Dict[int, Tuple[float, str, str]]
+    """
+    job_id = f"{dict_item[0]}"  # dictionary key
+    job_duration, job_status, job_url = dict_item[1]  # dictionary value, the tuple
+    return (
+        f"{STATUS_COLORS[job_status]}"
+        f"{link2print(job_url, job_id)}: {pretty_duration(job_duration):>8}"
+    )
+
+
+def link2print(url: str, text: str, text_pad: int = 0) -> str:
+    text = str(text)
+    text_pad = len(text) if text_pad < 1 else text_pad
+    if console.is_terminal:
+        return f"[link={url}]{text:{text_pad}}[/link]"
+    else:
+        return f"{text:{text_pad}}"
+
+
+def main() -> None:
+    try:
+        t_start = time.perf_counter()
+
+        args = parse_args()
+
+        token = read_token(args.token)
+
+        gl = gitlab.Gitlab(url=args.server,
+                           private_token=token,
+                           retry_transient_errors=True)
+
+        REV: str = args.rev
+
+        if args.pipeline_url:
+            pipe, cur_project = get_gitlab_pipeline_from_url(gl, args.pipeline_url)
+            REV = pipe.sha
+        else:
+            mesa_project = gl.projects.get("mesa/mesa")
+            projects = [mesa_project]
+            if args.mr:
+                REV = mesa_project.mergerequests.get(args.mr).sha
+            else:
+                REV = check_output(['git', 'rev-parse', REV]).decode('ascii').strip()
+
+                if args.rev == 'HEAD':
+                    try:
+                        branch_name = check_output([
+                            'git', 'symbolic-ref', '-q', 'HEAD',
+                        ]).decode('ascii').strip()
+                    except CalledProcessError:
+                        branch_name = ""
+
+                    # Ignore detached heads
+                    if branch_name:
+                        tracked_remote = check_output([
+                            'git', 'for-each-ref', '--format=%(upstream)',
+                            branch_name,
+                        ]).decode('ascii').strip()
+
+                        # Ignore local branches that do not track any remote
+                        if tracked_remote:
+                            remote_rev = check_output([
+                                'git', 'rev-parse', tracked_remote,
+                            ]).decode('ascii').strip()
+
+                            if REV != remote_rev:
+                                print(
+                                    f"Local HEAD commit {REV[:10]} is different than "
+                                    f"tracked remote HEAD commit {remote_rev[:10]}"
+                                )
+                                print("Did you forget to `git push` ?")
+
+                projects.append(get_gitlab_project(gl, args.project))
+            (pipe, cur_project) = wait_for_pipeline(projects, REV)
+
+        print(f"Revision: {REV}")
+        print(f"Pipeline: {pipe.web_url}")
+
+        target = '|'.join(args.target)
+        target = target.strip()
+
+        print(f"🞋 target job: [blue]{target}")  # U+1F78B Round target
+
+        # Implicitly include `parallel:` jobs
+        target = f'({target})' + r'( \d+/\d+)?'
+
+        target_jobs_regex = re.compile(target)
+
+        include_stage = '|'.join(args.include_stage)
+        include_stage = include_stage.strip()
+
+        print(f"🞋 target from stages: [blue]{include_stage}")  # U+1F78B Round target
+
+        include_stage_regex = re.compile(include_stage)
+
+        exclude_stage = '|'.join(args.exclude_stage)
+        exclude_stage = exclude_stage.strip()
+
+        print(f"🞋 target excluding stages: [blue]{exclude_stage}")  # U+1F78B Round target
+
+        exclude_stage_regex = re.compile(exclude_stage)
+
+        print(f"🞋 target jobs with tags: [blue]{str(args.job_tags)}")  # U+1F78B Round target
+        job_tags_regexes = [re.compile(job_tag) for job_tag in args.job_tags]
+
+        def job_filter(
+            job_name: str,
+            job_stage: str,
+            job_tags: set[str],
+        ) -> bool:
+            """
+            Apply user-specified filters to a job, and return whether the
+            filters allow that job (True) or not (False).
+            """
+            if not target_jobs_regex.fullmatch(job_name):
+                return False
+            if not include_stage_regex.fullmatch(job_stage):
+                return False
+            if exclude_stage_regex.fullmatch(job_stage):
+                return False
+            if not all(
+                any(job_tags_regex.fullmatch(tag) for tag in job_tags)
+                for job_tags_regex in job_tags_regexes
+            ):
+                return False
+            return True
+
+        deps = find_dependencies(
+            server=args.server,
+            token=token,
+            job_filter=job_filter,
+            iid=pipe.iid,
+            project_path=cur_project
+        )
+
+        if args.dry_run:
+            sys.exit(0)
+
+        target_job_id, ret, exec_t = monitor_pipeline(
+            cur_project,
+            pipe,
+            job_filter,
+            deps,
+            args.stress,
+            args.no_job_log,
+            args.polling_period,
+        )
+
+        if target_job_id:
+            print_log(cur_project, target_job_id)
+
+        print_monitor_summary(exec_t, t_start)
+
+        sys.exit(ret)
+    except KeyboardInterrupt:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

bin/ci/ci_run_n_monitor.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -eu
+
+this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
+readonly this_dir
+
+exec \
+  "$this_dir/../python-venv.sh" \
+  "$this_dir/requirements.txt" \
+  "$this_dir/ci_run_n_monitor.py" "$@"

bin/ci/custom_logger.py

@@ -0,0 +1,353 @@
+import argparse
+import logging
+from datetime import datetime
+from pathlib import Path
+
+from structured_logger import StructuredLogger
+
+
+class CustomLogger:
+    def __init__(self, log_file):
+        self.log_file = log_file
+        self.logger = StructuredLogger(file_name=self.log_file)
+
+    def get_last_dut_job(self):
+        """
+        Gets the details of the most recent DUT job.
+
+        Returns:
+            dict: Details of the most recent DUT job.
+
+        Raises:
+            ValueError: If no DUT jobs are found in the logger's data.
+        """
+        try:
+            job = self.logger.data["dut_jobs"][-1]
+        except KeyError:
+            raise ValueError(
+                "No DUT jobs found. Please create a job via create_dut_job call."
+            )
+
+        return job
+
+    def update(self, **kwargs):
+        """
+        Updates the log file with provided key-value pairs.
+
+        Args:
+            **kwargs: Key-value pairs to be updated.
+
+        """
+        with self.logger.edit_context():
+            for key, value in kwargs.items():
+                self.logger.data[key] = value
+
+    def create_dut_job(self, **kwargs):
+        """
+        Creates a new DUT job with provided key-value pairs.
+
+        Args:
+            **kwargs: Key-value pairs for the new DUT job.
+
+        """
+        with self.logger.edit_context():
+            if "dut_jobs" not in self.logger.data:
+                self.logger.data["dut_jobs"] = []
+            new_job = {
+                "status": "",
+                "submitter_start_time": datetime.now().isoformat(),
+                "dut_submit_time": "",
+                "dut_start_time": "",
+                "dut_end_time": "",
+                "dut_duration_time": "",
+                "dut_name": "",
+                "dut_state": "pending",
+                "dut_job_phases": [],
+                **kwargs,
+            }
+            self.logger.data["dut_jobs"].append(new_job)
+
+    def update_dut_job(self, key, value):
+        """
+        Updates the last DUT job with a key-value pair.
+
+        Args:
+            key : The key to be updated.
+            value: The value to be assigned.
+
+        """
+        with self.logger.edit_context():
+            job = self.get_last_dut_job()
+            job[key] = value
+
+    def update_status_fail(self, reason=""):
+        """
+        Sets the status of the last DUT job to 'fail' and logs the failure reason.
+
+        Args:
+            reason (str, optional): The reason for the failure. Defaults to "".
+
+        """
+        with self.logger.edit_context():
+            job = self.get_last_dut_job()
+            job["status"] = "fail"
+            job["dut_job_fail_reason"] = reason
+
+    def create_job_phase(self, phase_name):
+        """
+        Creates a new job phase for the last DUT job.
+
+        Args:
+            phase_name : The name of the new job phase.
+
+        """
+        with self.logger.edit_context():
+            job = self.get_last_dut_job()
+            if job["dut_job_phases"] and job["dut_job_phases"][-1]["end_time"] == "":
+                # If the last phase exists and its end time is empty, set the end time
+                timestamp = datetime.now().isoformat()
+                job["dut_job_phases"][-1]["end_time"] = timestamp
+                job["dut_job_phases"][-1]["duration_time"] = self.get_duration_time(job["dut_job_phases"][-1]["start_time"], timestamp)
+
+            # Create a new phase
+            phase_data = {
+                "name": phase_name,
+                "start_time": datetime.now().isoformat(),
+                "end_time": "",
+                "duration_time": "",
+            }
+            job["dut_job_phases"].append(phase_data)
+
+    def check_dut_timings(self, job):
+        """
+        Check the timing sequence of a job to ensure logical consistency.
+
+        The function verifies that the job's submission time is not earlier than its start time and that
+        the job's end time is not earlier than its start time. If either of these conditions is found to be true,
+        an error is logged for each instance of inconsistency.
+
+        Args:
+        job (dict): A dictionary containing timing information of a job. Expected keys are 'dut_start_time',
+                    'dut_submit_time', and 'dut_end_time'.
+
+        Returns:
+        None: This function does not return a value; it logs errors if timing inconsistencies are detected.
+
+        The function checks the following:
+        - If 'dut_start_time' and 'dut_submit_time' are both present and correctly sequenced.
+        - If 'dut_start_time' and 'dut_end_time' are both present and correctly sequenced.
+        """
+
+        # Check if the start time and submit time exist
+        if job.get("dut_start_time") and job.get("dut_submit_time"):
+            # If they exist, check if the submission time is before the start time
+            if job["dut_start_time"] < job["dut_submit_time"]:
+                logging.error("Job submission is happening before job start.")
+
+        # Check if the start time and end time exist
+        if job.get("dut_start_time") and job.get("dut_end_time"):
+            # If they exist, check if the end time is after the start time
+            if job["dut_end_time"] < job["dut_start_time"]:
+                logging.error("Job ended before it started.")
+
+    def get_duration_time(self, start_time, end_time):
+        """
+        Computes duration time, in minutes and seconds.
+        """
+        try:
+            start = datetime.fromisoformat(start_time)
+            end = datetime.fromisoformat(end_time)
+            duration = end - start
+            return str(duration)
+        except ValueError:
+            return ""
+
+    # Method to update DUT start, submit and end time
+    def update_dut_time(self, value, custom_time):
+        """
+        Updates DUT start, submit, and end times.
+
+        Args:
+            value : Specifies which DUT time to update. Options: 'start', 'submit', 'end'.
+            custom_time : Custom time to set. If None, use current time.
+
+        Raises:
+            ValueError: If an invalid argument is provided for value.
+
+        """
+        with self.logger.edit_context():
+            job = self.get_last_dut_job()
+            timestamp = custom_time if custom_time else datetime.now().isoformat()
+            if value == "start":
+                job["dut_start_time"] = timestamp
+                job["dut_state"] = "running"
+            elif value == "submit":
+                job["dut_submit_time"] = timestamp
+                job["dut_state"] = "submitted"
+            elif value == "end":
+                job["dut_end_time"] = timestamp
+                job["dut_state"] = "finished"
+                job["dut_duration_time"] = self.get_duration_time(job["dut_start_time"], job["dut_end_time"])
+            else:
+                raise ValueError(
+                    "Error: Invalid argument provided for --update-dut-time. Use 'start', 'submit', 'end'."
+                )
+            # check the sanity of the partial structured log
+            self.check_dut_timings(job)
+
+    def close_dut_job(self):
+        """
+        Closes the most recent DUT (Device Under Test) job in the logger's data.
+
+        The method performs the following operations:
+        1. Validates if there are any DUT jobs in the logger's data.
+        2. If the last phase of the most recent DUT job has an empty end time, it sets the end time to the current time.
+
+        Raises:
+            ValueError: If no DUT jobs are found in the logger's data.
+        """
+        with self.logger.edit_context():
+            job = self.get_last_dut_job()
+            # Check if the last phase exists and its end time is empty, then set the end time
+            if job["dut_job_phases"] and job["dut_job_phases"][-1]["end_time"] == "":
+                timestamp = datetime.now().isoformat()
+                job["dut_job_phases"][-1]["end_time"] = timestamp
+                job["dut_job_phases"][-1]["duration_time"] = self.get_duration_time(job["dut_job_phases"][-1]["start_time"], timestamp)
+
+    def close(self):
+        """
+        Closes the most recent DUT (Device Under Test) job in the logger's data.
+
+        The method performs the following operations:
+        1. Determines the combined status of all DUT jobs.
+        2. Sets the submitter's end time to the current time.
+        3. Updates the DUT attempt counter to reflect the total number of DUT jobs.
+
+        """
+        with self.logger.edit_context():
+            job_status = []
+            for job in self.logger.data["dut_jobs"]:
+                if "status" in job:
+                    job_status.append(job["status"])
+
+            if not job_status:
+                job_combined_status = "null"
+            else:
+                # Get job_combined_status
+                if "pass" in job_status:
+                    job_combined_status = "pass"
+                else:
+                    job_combined_status = "fail"
+
+            self.logger.data["job_combined_status"] = job_combined_status
+            self.logger.data["dut_attempt_counter"] = len(self.logger.data["dut_jobs"])
+            job["submitter_end_time"] = datetime.now().isoformat()
+
+
+def process_args(args):
+    # Function to process key-value pairs and call corresponding logger methods
+    def process_key_value_pairs(args_list, action_func):
+        if not args_list:
+            raise ValueError(
+                f"No key-value pairs provided for {action_func.__name__.replace('_', '-')}"
+            )
+        if len(args_list) % 2 != 0:
+            raise ValueError(
+                f"Incomplete key-value pairs for {action_func.__name__.replace('_', '-')}"
+            )
+        kwargs = dict(zip(args_list[::2], args_list[1::2]))
+        action_func(**kwargs)
+
+    # Create a CustomLogger object with the specified log file path
+    custom_logger = CustomLogger(Path(args.log_file))
+
+    if args.update:
+        process_key_value_pairs(args.update, custom_logger.update)
+
+    if args.create_dut_job:
+        process_key_value_pairs(args.create_dut_job, custom_logger.create_dut_job)
+
+    if args.update_dut_job:
+        key, value = args.update_dut_job
+        custom_logger.update_dut_job(key, value)
+
+    if args.create_job_phase:
+        custom_logger.create_job_phase(args.create_job_phase)
+
+    if args.update_status_fail:
+        custom_logger.update_status_fail(args.update_status_fail)
+
+    if args.update_dut_time:
+        if len(args.update_dut_time) == 2:
+            action, custom_time = args.update_dut_time
+        elif len(args.update_dut_time) == 1:
+            action, custom_time = args.update_dut_time[0], None
+        else:
+            raise ValueError("Invalid number of values for --update-dut-time")
+
+        if action in ["start", "end", "submit"]:
+            custom_logger.update_dut_time(action, custom_time)
+        else:
+            raise ValueError(
+                "Error: Invalid argument provided for --update-dut-time. Use 'start', 'submit', 'end'."
+            )
+
+    if args.close_dut_job:
+        custom_logger.close_dut_job()
+
+    if args.close:
+        custom_logger.close()
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Custom Logger Command Line Tool")
+    parser.add_argument("log_file", help="Path to the log file")
+    parser.add_argument(
+        "--update",
+        nargs=argparse.ZERO_OR_MORE,
+        metavar=("key", "value"),
+        help="Update a key-value pair e.g., --update key1 value1 key2 value2)",
+    )
+    parser.add_argument(
+        "--create-dut-job",
+        nargs=argparse.ZERO_OR_MORE,
+        metavar=("key", "value"),
+        help="Create a new DUT job with key-value pairs (e.g., --create-dut-job key1 value1 key2 value2)",
+    )
+    parser.add_argument(
+        "--update-dut-job",
+        nargs=argparse.ZERO_OR_MORE,
+        metavar=("key", "value"),
+        help="Update a key-value pair in DUT job",
+    )
+    parser.add_argument(
+        "--create-job-phase",
+        help="Create a new job phase (e.g., --create-job-phase name)",
+    )
+    parser.add_argument(
+        "--update-status-fail",
+        help="Update fail as the status and log the failure reason (e.g., --update-status-fail reason)",
+    )
+    parser.add_argument(
+        "--update-dut-time",
+        nargs=argparse.ZERO_OR_MORE,
+        metavar=("action", "custom_time"),
+        help="Update DUT start and end time. Provide action ('start', 'submit', 'end') and custom_time (e.g., '2023-01-01T12:00:00')",
+    )
+    parser.add_argument(
+        "--close-dut-job",
+        action="store_true",
+        help="Close the dut job by updating end time of last dut job)",
+    )
+    parser.add_argument(
+        "--close",
+        action="store_true",
+        help="Updates combined status, submitter's end time and DUT attempt counter",
+    )
+    args = parser.parse_args()
+
+    process_args(args)
+
+
+if __name__ == "__main__":
+    main()

bin/ci/download_gl_schema.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+# Helper script to download the schema GraphQL from Gitlab to enable IDEs to
+# assist the developer to edit gql files
+
+SOURCE_DIR=$(dirname "$(realpath "$0")")
+
+(
+    cd $SOURCE_DIR || exit 1
+    gql-cli https://gitlab.freedesktop.org/api/graphql --print-schema > schema.graphql
+)

bin/ci/gitlab_common.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python3
+# Copyright © 2020 - 2022 Collabora Ltd.
+# Authors:
+#   Tomeu Vizoso <[email protected]>
+#   David Heidelberg <[email protected]>
+#   Guilherme Gallo <[email protected]>
+#
+# SPDX-License-Identifier: MIT
+'''Shared functions between the scripts.'''
+
+import logging
+import os
+import re
+import time
+from functools import cache
+from pathlib import Path
+
+GITLAB_URL = "https://gitlab.freedesktop.org"
+TOKEN_DIR = Path(os.environ.get("XDG_CONFIG_HOME", "")
+                 if os.environ.get("XDG_CONFIG_HOME", None)
+                 else Path.home() / ".config")
+
+# Known GitLab token prefixes: https://docs.gitlab.com/security/tokens/#token-prefixes
+TOKEN_PREFIXES: dict[str, str] = {
+    "Personal access token": "glpat-",
+    "OAuth Application Secret": "gloas-",
+    "Deploy token": "gldt-",
+    "Runner authentication token": "glrt-",
+    "CI/CD Job token": "glcbt-",
+    "Trigger token": "glptt-",
+    "Feed token": "glft-",
+    "Incoming mail token": "glimt-",
+    "GitLab Agent for Kubernetes token": "glagent-",
+    "SCIM Tokens": "glsoat-",
+}
+
+
+@cache
+def print_once(*args, **kwargs):
+    """Print without spamming the output"""
+    print(*args, **kwargs)
+
+
+def pretty_duration(seconds: int | float) -> str:
+    """Pretty print duration"""
+    seconds = int(seconds)
+    hours, rem = divmod(seconds, 3600)
+    minutes, seconds = divmod(rem, 60)
+    if hours:
+        return f"{hours:0.0f}h{minutes:02.0f}m{seconds:02.0f}s"
+    if minutes:
+        return f"{minutes:0.0f}m{seconds:02.0f}s"
+    return f"{seconds:0.0f}s"
+
+
+def get_gitlab_pipeline_from_url(gl, pipeline_url) -> tuple:
+    """
+    Extract the project and pipeline object from the url string
+    :param gl: Gitlab object
+    :param pipeline_url: string with a url to a pipeline
+    :return: ProjectPipeline, Project objects
+    """
+    pattern = rf"^{re.escape(GITLAB_URL)}/(.*)/-/pipelines/([0-9]+)$"
+    match = re.match(pattern, pipeline_url)
+    if not match:
+        raise AssertionError(f"url {pipeline_url} doesn't follow the pattern {pattern}")
+    namespace_with_project, pipeline_id = match.groups()
+    cur_project = gl.projects.get(namespace_with_project)
+    pipe = cur_project.pipelines.get(pipeline_id)
+    return pipe, cur_project
+
+
+def get_gitlab_project(glab, name: str):
+    """Finds a specified gitlab project for given user"""
+    if "/" in name:
+        project_path = name
+    else:
+        glab.auth()
+        username = glab.user.username
+        project_path = f"{username}/{name}"
+    return glab.projects.get(project_path)
+
+
+def get_token_from_default_dir() -> str:
+    """
+    Retrieves the GitLab token from the default directory.
+
+    Returns:
+        str: The path to the GitLab token file.
+
+    Raises:
+        FileNotFoundError: If the token file is not found.
+    """
+    token_file = TOKEN_DIR / "gitlab-token"
+    try:
+        return str(token_file.resolve())
+    except FileNotFoundError as ex:
+        print(
+            f"Could not find {token_file}, please provide a token file as an argument"
+        )
+        raise ex
+
+
+def validate_gitlab_token(token: str) -> bool:
+    # Match against recognised token prefixes
+    token_suffix = None
+    for token_type, token_prefix in TOKEN_PREFIXES.items():
+        if token.startswith(token_prefix):
+            logging.info(f"Found probable token type: {token_type}")
+            token_suffix = token[len(token_prefix):]
+            break
+
+    if not token_suffix:
+        return False
+
+    # Basic validation of the token suffix based on:
+    # https://gitlab.com/gitlab-org/gitlab/-/blob/master/gems/gitlab-secret_detection/lib/gitleaks.toml
+    if not re.match(r"(\w+-)?[0-9a-zA-Z_\-]{20,64}", token_suffix):
+        return False
+
+    return True
+
+
+def get_token_from_arg(token_arg: str | Path | None) -> str | None:
+    if not token_arg:
+        logging.info("No token provided.")
+        return None
+
+    token_path = Path(token_arg)
+    if token_path.is_file():
+        return read_token_from_file(token_path)
+
+    return handle_direct_token(token_path, token_arg)
+
+
+def read_token_from_file(token_path: Path) -> str:
+    token = token_path.read_text().strip()
+    logging.info(f"Token read from file: {token_path}")
+    return token
+
+
+def handle_direct_token(token_path: Path, token_arg: str | Path) -> str | None:
+    if token_path == Path(get_token_from_default_dir()):
+        logging.warning(
+            f"The default token file {token_path} was not found. "
+            "Please provide a token file or a token directly via --token arg."
+        )
+        return None
+    logging.info("Token provided directly as an argument.")
+    return str(token_arg)
+
+
+def read_token(token_arg: str | Path | None) -> str | None:
+    token = get_token_from_arg(token_arg)
+    if token and not validate_gitlab_token(token):
+        logging.warning("The provided token is either an old token or does not seem to "
+                        "be a valid token.")
+        logging.warning("Newer tokens are the ones created from a Gitlab 14.5+ instance.")
+        logging.warning("See https://about.gitlab.com/releases/2021/11/22/"
+                        "gitlab-14-5-released/"
+                        "#new-gitlab-access-token-prefix-and-detection")
+    return token
+
+
+def wait_for_pipeline(projects, sha: str, timeout=None):
+    """await until pipeline appears in Gitlab"""
+    project_names = [project.path_with_namespace for project in projects]
+    print(f"⏲ for the pipeline to appear in {project_names}..", end="")
+    start_time = time.time()
+    while True:
+        for project in projects:
+            pipelines = project.pipelines.list(sha=sha)
+            if pipelines:
+                print("", flush=True)
+                return (pipelines[0], project)
+        print("", end=".", flush=True)
+        if timeout and time.time() - start_time > timeout:
+            print(" not found", flush=True)
+            return (None, None)
+        time.sleep(1)
+
+@cache
+def is_gitlab_job() -> bool:
+    return os.getenv("CI_JOB_ID") is not None

bin/ci/gitlab_gql.py

@@ -0,0 +1,641 @@
+#!/usr/bin/env python3
+# For the dependencies, see the requirements.txt
+
+import logging
+import re
+import sys
+import traceback
+from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser, Namespace
+from collections import OrderedDict
+from copy import deepcopy
+from dataclasses import dataclass, field
+from itertools import accumulate
+from os import get_terminal_size, getenv
+from pathlib import Path
+from subprocess import check_output
+from textwrap import dedent
+from typing import Any, Iterable, Optional, TypedDict, Union
+
+import yaml
+from filecache import DAY, filecache
+from gitlab_common import get_token_from_default_dir, is_gitlab_job
+from gql import Client, gql
+from gql.transport.requests import RequestsHTTPTransport
+from graphql import DocumentNode
+from rich.console import Console
+
+DEFAULT_TERMINAL_SIZE: int = 80  # columns
+
+
+class DagNode(TypedDict):
+    needs: set[str]
+    stage: str
+    tags: set[str]
+    # `name` is redundant but is here for retro-compatibility
+    name: str
+
+
+# see create_job_needs_dag function for more details
+Dag = dict[str, DagNode]
+
+
+StageSeq = OrderedDict[str, set[str]]
+
+if is_gitlab_job():
+    console = Console(highlight=False, no_color=False, color_system="truecolor", width=120)
+else:
+    console = Console(highlight=False)
+print = console.print
+
+
+def get_project_root_dir():
+    root_path = Path(__file__).parent.parent.parent.resolve()
+    gitlab_file = root_path / ".gitlab-ci.yml"
+    assert gitlab_file.exists()
+
+    return root_path
+
+
+@dataclass
+class GitlabGQL:
+    _transport: Any = field(init=False)
+    client: Client = field(init=False)
+    url: str = "https://gitlab.freedesktop.org/api/graphql"
+    token: Optional[str] = None
+
+    def __post_init__(self) -> None:
+        self._setup_gitlab_gql_client()
+
+    def _setup_gitlab_gql_client(self) -> None:
+        # Select your transport with a defined url endpoint
+        headers = {}
+        if self.token:
+            headers["Authorization"] = f"Bearer {self.token}"
+        self._transport = RequestsHTTPTransport(url=self.url, headers=headers)
+
+        # Create a GraphQL client using the defined transport
+        self.client = Client(transport=self._transport, fetch_schema_from_transport=True)
+
+    def query(
+        self,
+        gql_file: Union[Path, str],
+        params: dict[str, Any] = {},
+        operation_name: Optional[str] = None,
+        paginated_key_loc: Iterable[str] = [],
+        disable_cache: bool = False,
+    ) -> dict[str, Any]:
+        def run_uncached() -> dict[str, Any]:
+            if paginated_key_loc:
+                return self._sweep_pages(gql_file, params, operation_name, paginated_key_loc)
+            return self._query(gql_file, params, operation_name)
+
+        if disable_cache:
+            return run_uncached()
+
+        try:
+            # Create an auxiliary variable to deliver a cached result and enable catching exceptions
+            # Decorate the query to be cached
+            if paginated_key_loc:
+                result = self._sweep_pages_cached(
+                    gql_file, params, operation_name, paginated_key_loc
+                )
+            else:
+                result = self._query_cached(gql_file, params, operation_name)
+            return result  # type: ignore
+        except Exception as ex:
+            logging.error(f"Cached query failed with {ex}")
+            # print exception traceback
+            traceback_str = "".join(traceback.format_exception(ex))
+            logging.error(traceback_str)
+            self.invalidate_query_cache()
+            logging.error("Cache invalidated, retrying without cache")
+            return run_uncached()
+
+    def _query(
+        self,
+        gql_file: Union[Path, str],
+        params: dict[str, Any] = {},
+        operation_name: Optional[str] = None,
+    ) -> dict[str, Any]:
+        # Provide a GraphQL query
+        source_path: Path = Path(__file__).parent
+        pipeline_query_file: Path = source_path / gql_file
+
+        query: DocumentNode
+        with open(pipeline_query_file, "r") as f:
+            pipeline_query = f.read()
+            query = gql(pipeline_query)
+
+        # Execute the query on the transport
+        return self.client.execute_sync(
+            query, variable_values=params, operation_name=operation_name
+        )
+
+    @filecache(DAY)
+    def _sweep_pages_cached(self, *args, **kwargs):
+        return self._sweep_pages(*args, **kwargs)
+
+    @filecache(DAY)
+    def _query_cached(self, *args, **kwargs):
+        return self._query(*args, **kwargs)
+
+    def _sweep_pages(
+        self, query, params, operation_name=None, paginated_key_loc: Iterable[str] = []
+    ) -> dict[str, Any]:
+        """
+        Retrieve paginated data from a GraphQL API and concatenate the results into a single
+        response.
+
+        Args:
+            query: represents a filepath with the GraphQL query to be executed.
+            params: a dictionary that contains the parameters to be passed to the query. These
+                parameters can be used to filter or modify the results of the query.
+            operation_name: The `operation_name` parameter is an optional parameter that specifies
+                the name of the GraphQL operation to be executed. It is used when making a GraphQL
+                query to specify which operation to execute if there are multiple operations defined
+                in the GraphQL schema. If not provided, the default operation will be executed.
+            paginated_key_loc (Iterable[str]): The `paginated_key_loc` parameter is an iterable of
+                strings that represents the location of the paginated field within the response. It
+                is used to extract the paginated field from the response and append it to the final
+                result. The node has to be a list of objects with a `pageInfo` field that contains
+                at least the `hasNextPage` and `endCursor` fields.
+
+        Returns:
+            a dictionary containing the response from the query with the paginated field
+            concatenated.
+        """
+
+        def fetch_page(cursor: str | None = None) -> dict[str, Any]:
+            if cursor:
+                params["cursor"] = cursor
+                logging.info(
+                    f"Found more than 100 elements, paginating. "
+                    f"Current cursor at {cursor}"
+                )
+
+            return self._query(query, params, operation_name)
+
+        # Execute the initial query
+        response: dict[str, Any] = fetch_page()
+
+        # Initialize an empty list to store the final result
+        final_partial_field: list[dict[str, Any]] = []
+
+        # Loop until all pages have been retrieved
+        while True:
+            # Get the partial field to be appended to the final result
+            partial_field = response
+            for key in paginated_key_loc:
+                partial_field = partial_field[key]
+
+            # Append the partial field to the final result
+            final_partial_field += partial_field["nodes"]
+
+            # Check if there are more pages to retrieve
+            page_info = partial_field["pageInfo"]
+            if not page_info["hasNextPage"]:
+                break
+
+            # Execute the query with the updated cursor parameter
+            response = fetch_page(page_info["endCursor"])
+
+        # Replace the "nodes" field in the original response with the final result
+        partial_field["nodes"] = final_partial_field
+        return response
+
+    def invalidate_query_cache(self) -> None:
+        logging.warning("Invalidating query cache")
+        try:
+            self._sweep_pages._db.clear()
+            self._query._db.clear()
+        except AttributeError as ex:
+            logging.warning(f"Could not invalidate cache, maybe it was not used in {ex.args}?")
+
+
+def insert_early_stage_jobs(stage_sequence: StageSeq, jobs_metadata: Dag) -> Dag:
+    pre_processed_dag: dict[str, set[str]] = {}
+    jobs_from_early_stages = list(accumulate(stage_sequence.values(), set.union))
+    for job_name, metadata in jobs_metadata.items():
+        final_needs: set[str] = deepcopy(metadata["needs"])
+        # Pre-process jobs that are not based on needs field
+        # e.g. sanity job in mesa MR pipelines
+        if not final_needs:
+            job_stage: str = jobs_metadata[job_name]["stage"]
+            stage_index: int = list(stage_sequence.keys()).index(job_stage)
+            if stage_index > 0:
+                final_needs |= jobs_from_early_stages[stage_index - 1]
+        pre_processed_dag[job_name] = final_needs
+
+    for job_name, needs in pre_processed_dag.items():
+        jobs_metadata[job_name]["needs"] = needs
+
+    return jobs_metadata
+
+
+def traverse_dag_needs(jobs_metadata: Dag) -> None:
+    created_jobs = set(jobs_metadata.keys())
+    for job, metadata in jobs_metadata.items():
+        final_needs: set = deepcopy(metadata["needs"]) & created_jobs
+        # Post process jobs that are based on needs field
+        partial = True
+
+        while partial:
+            next_depth: set[str] = {n for dn in final_needs if dn in jobs_metadata for n in jobs_metadata[dn]["needs"]}
+            partial: bool = not final_needs.issuperset(next_depth)
+            final_needs = final_needs.union(next_depth)
+
+        jobs_metadata[job]["needs"] = final_needs
+
+
+def extract_stages_and_job_needs(
+    pipeline_jobs: dict[str, Any], pipeline_stages: dict[str, Any]
+) -> tuple[StageSeq, Dag]:
+    jobs_metadata = Dag()
+    # Record the stage sequence to post process deps that are not based on needs
+    # field, for example: sanity job
+    stage_sequence: OrderedDict[str, set[str]] = OrderedDict()
+    for stage in pipeline_stages["nodes"]:
+        stage_sequence[stage["name"]] = set()
+
+    for job in pipeline_jobs["nodes"]:
+        stage_sequence[job["stage"]["name"]].add(job["name"])
+        dag_job: DagNode = {
+            "name": job["name"],
+            "tags": set(job["tags"] or []),  # jobs with no tags defined return None here
+            "stage": job["stage"]["name"],
+            "needs": set([j["node"]["name"] for j in job["needs"]["edges"]]),
+        }
+        jobs_metadata[job["name"]] = dag_job
+
+    return stage_sequence, jobs_metadata
+
+
+def create_job_needs_dag(gl_gql: GitlabGQL, params, disable_cache: bool = True) -> Dag:
+    """
+    This function creates a Directed Acyclic Graph (DAG) to represent a sequence of jobs, where each
+    job has a set of jobs that it depends on (its "needs") and belongs to a certain "stage".
+    The "name" of the job is used as the key in the dictionary.
+
+    For example, consider the following DAG:
+
+        1. build stage: job1 -> job2 -> job3
+        2. test stage: job2 -> job4
+
+    - The job needs for job3 are: job1, job2
+    - The job needs for job4 are: job2
+    - The job2 needs to wait all jobs from build stage to finish.
+
+    The resulting DAG would look like this:
+
+        dag = {
+            "job1": {"needs": set(), "stage": "build", "name": "job1"},
+            "job2": {"needs": {"job1", "job2", job3"}, "stage": "test", "name": "job2"},
+            "job3": {"needs": {"job1", "job2"}, "stage": "build", "name": "job3"},
+            "job4": {"needs": {"job2"}, "stage": "test", "name": "job4"},
+        }
+
+    To access the job needs, one can do:
+
+        dag["job3"]["needs"]
+
+    This will return the set of jobs that job3 needs: {"job1", "job2"}
+
+    Args:
+        gl_gql (GitlabGQL): The `gl_gql` parameter is an instance of the `GitlabGQL` class, which is
+            used to make GraphQL queries to the GitLab API.
+        params (dict): The `params` parameter is a dictionary that contains the necessary parameters
+            for the GraphQL query. It is used to specify the details of the pipeline for which the
+            job needs DAG is being created.
+            The specific keys and values in the `params` dictionary will depend on
+            the requirements of the GraphQL query being executed
+        disable_cache (bool): The `disable_cache` parameter is a boolean that specifies whether the
+
+    Returns:
+        The final DAG (Directed Acyclic Graph) representing the job dependencies sourced from needs
+        or stages rule.
+    """
+    stages_jobs_gql = gl_gql.query(
+        "pipeline_details.gql",
+        params=params,
+        paginated_key_loc=["project", "pipeline", "jobs"],
+        disable_cache=disable_cache,
+    )
+    pipeline_data = stages_jobs_gql["project"]["pipeline"]
+    if not pipeline_data:
+        raise RuntimeError(f"Could not find any pipelines for {params}")
+
+    stage_sequence, jobs_metadata = extract_stages_and_job_needs(
+        pipeline_data["jobs"], pipeline_data["stages"]
+    )
+    # Fill the DAG with the job needs from stages that don't have any needs but still need to wait
+    # for previous stages
+    final_dag = insert_early_stage_jobs(stage_sequence, jobs_metadata)
+    # Now that each job has its direct needs filled correctly, update the "needs" field for each job
+    # in the DAG by performing a topological traversal
+    traverse_dag_needs(final_dag)
+
+    return final_dag
+
+
+def filter_dag(dag: Dag, job_filter: callable) -> Dag:
+    return Dag({
+        job: data
+        for job, data in dag.items()
+        if job_filter(
+            job_name=job,
+            job_stage=data["stage"],
+            job_tags=data["tags"],
+        )
+    })
+
+
+def print_dag(dag: Dag, indentation: int = 0, color: str = "") -> None:
+    for job, data in sorted(dag.items()):
+        print(f"{color}{' '*indentation}{job}:")
+        print_formatted_list(list(data['needs']), indentation=indentation+8, color=color)
+
+
+def print_formatted_list(elements: list[str], indentation: int = 0, color: str = "") -> None:
+    """
+    When a list of elements is going to be printed, if it is longer than one line, reformat it to be multiple
+    lines with a 'ls' command style.
+    :param elements: list of elements to be printed
+    :param indentation: number of spaces to be injected in front of each line
+    """
+    if len(elements) == 0:
+        return
+    elements.sort()
+    try:
+        h_size = get_terminal_size().columns if sys.stdin.isatty() else DEFAULT_TERMINAL_SIZE
+    except OSError:
+        h_size = DEFAULT_TERMINAL_SIZE
+    if indentation + sum(len(element) for element in elements) + (len(elements)*2) < h_size:  # fits in one line
+        print(f"{color}{' '*indentation}{', '.join([element for element in elements])}")
+        return
+    column_separator_size = 2
+    column_width: int = len(max(elements, key=len)) + column_separator_size
+    n_columns: int = max((h_size - indentation) // column_width, 1)
+    step = (len(elements) // n_columns) + 1
+    rows = [elements[i::step] for i in range(step)]
+    for line in rows:
+        print(' '*indentation, end='')
+        for column in range(len(line)):
+            if line[column] is not None:
+                print(f"{color}{line[column]:<{column_width}}", end='')
+        print()
+
+
+def fetch_merged_yaml(gl_gql: GitlabGQL, params) -> dict[str, Any]:
+    params["content"] = dedent("""\
+    include:
+      - local: .gitlab-ci.yml
+    """)
+    raw_response = gl_gql.query("job_details.gql", params)
+    ci_config = raw_response["ciConfig"]
+    if merged_yaml := ci_config["mergedYaml"]:
+        return yaml.safe_load(merged_yaml)
+    if "errors" in ci_config:
+        for error in ci_config["errors"]:
+            print(error)
+
+    gl_gql.invalidate_query_cache()
+    raise ValueError(
+        """
+    Could not fetch any content for merged YAML,
+    please verify if the git SHA exists in remote.
+    Maybe you forgot to `git push`?  """
+    )
+
+
+def recursive_fill(job, relationship_field, target_data, acc_data: dict, merged_yaml):
+    if relatives := job.get(relationship_field):
+        if isinstance(relatives, str):
+            relatives = [relatives]
+
+        for relative in relatives:
+            parent_job = merged_yaml[relative]
+            acc_data = recursive_fill(parent_job, acc_data, merged_yaml)  # type: ignore
+
+    acc_data |= job.get(target_data, {})
+
+    return acc_data
+
+
+def get_variables(job, merged_yaml, project_path, sha) -> dict[str, str]:
+    p = get_project_root_dir() / ".gitlab-ci" / "image-tags.yml"
+    image_tags = yaml.safe_load(p.read_text())
+
+    variables = image_tags["variables"]
+    variables |= merged_yaml["variables"]
+    variables |= job["variables"]
+    variables["CI_PROJECT_PATH"] = project_path
+    variables["CI_PROJECT_NAME"] = project_path.split("/")[1]
+    variables["CI_REGISTRY_IMAGE"] = "registry.freedesktop.org/${CI_PROJECT_PATH}"
+    variables["CI_COMMIT_SHA"] = sha
+
+    while recurse_among_variables_space(variables):
+        pass
+
+    return variables
+
+
+# Based on: https://stackoverflow.com/a/2158532/1079223
+def flatten(xs):
+    for x in xs:
+        if isinstance(x, Iterable) and not isinstance(x, (str, bytes)):
+            yield from flatten(x)
+        else:
+            yield x
+
+
+def get_full_script(job) -> list[str]:
+    script = []
+    for script_part in ("before_script", "script", "after_script"):
+        script.append(f"# {script_part}")
+        lines = flatten(job.get(script_part, []))
+        script.extend(lines)
+        script.append("")
+
+    return script
+
+
+def recurse_among_variables_space(var_graph) -> bool:
+    updated = False
+    for var, value in var_graph.items():
+        value = str(value)
+        dep_vars = []
+        if match := re.findall(r"(\$[{]?[\w\d_]*[}]?)", value):
+            all_dep_vars = [v.lstrip("${").rstrip("}") for v in match]
+            # print(value, match, all_dep_vars)
+            dep_vars = [v for v in all_dep_vars if v in var_graph]
+
+        for dep_var in dep_vars:
+            dep_value = str(var_graph[dep_var])
+            new_value = var_graph[var]
+            new_value = new_value.replace(f"${{{dep_var}}}", dep_value)
+            new_value = new_value.replace(f"${dep_var}", dep_value)
+            var_graph[var] = new_value
+            updated |= dep_value != new_value
+
+    return updated
+
+
+def print_job_final_definition(job_name, merged_yaml, project_path, sha):
+    job = merged_yaml[job_name]
+    variables = get_variables(job, merged_yaml, project_path, sha)
+
+    print("# --------- variables ---------------")
+    for var, value in sorted(variables.items()):
+        print(f"export {var}={value!r}")
+
+    # TODO: Recurse into needs to get full script
+    # TODO: maybe create a extra yaml file to avoid too much rework
+    script = get_full_script(job)
+    print()
+    print()
+    print("# --------- full script ---------------")
+    print("\n".join(script))
+
+    if image := variables.get("MESA_IMAGE"):
+        print()
+        print()
+        print("# --------- container image ---------------")
+        print(image)
+
+
+def from_sha_to_pipeline_iid(gl_gql: GitlabGQL, params) -> str:
+    result = gl_gql.query("pipeline_utils.gql", params)
+
+    return result["project"]["pipelines"]["nodes"][0]["iid"]
+
+
+def parse_args() -> Namespace:
+    parser = ArgumentParser(
+        formatter_class=ArgumentDefaultsHelpFormatter,
+        description="CLI and library with utility functions to debug jobs via Gitlab GraphQL",
+        epilog=f"""Example:
+        {Path(__file__).name} --print-dag""",
+    )
+    parser.add_argument("-pp", "--project-path", type=str, default="mesa/mesa")
+    parser.add_argument("--sha", "--rev", type=str, default='HEAD')
+    parser.add_argument(
+        "--regex",
+        type=str,
+        required=False,
+        default=".*",
+        help="Regex pattern for the job name to be considered",
+    )
+    parser.add_argument(
+        "--include-stage",
+        type=str,
+        required=False,
+        default=".*",
+        help="Regex pattern for the stage name to be considered",
+    )
+    parser.add_argument(
+        "--exclude-stage",
+        type=str,
+        required=False,
+        default="^$",
+        help="Regex pattern for the stage name to be excluded",
+    )
+    parser.add_argument(
+        "--job-tags",
+        metavar="job-tags",
+        help="Job tags to require when searching for target jobs. If multiple "
+             "values are passed, eg. `--job-tags 'foo.*' 'bar'`, the job will "
+             "need to have a tag matching `foo.*` *and* a tag matching `bar` "
+             "to qualify. Passing `--job-tags '.*'` makes sure the job has "
+             "a tag defined, while not passing `--job-tags` also allows "
+             "untagged jobs.",
+        default=[],
+        nargs='+',
+    )
+    mutex_group_print = parser.add_mutually_exclusive_group()
+    mutex_group_print.add_argument(
+        "--print-dag",
+        action="store_true",
+        help="Print job needs DAG",
+    )
+    mutex_group_print.add_argument(
+        "--print-merged-yaml",
+        action="store_true",
+        help="Print the resulting YAML for the specific SHA",
+    )
+    mutex_group_print.add_argument(
+        "--print-job-manifest",
+        metavar='JOB_NAME',
+        type=str,
+        help="Print the resulting job data"
+    )
+    parser.add_argument(
+        "--gitlab-token-file",
+        type=str,
+        default=get_token_from_default_dir(),
+        help="force GitLab token, otherwise it's read from $XDG_CONFIG_HOME/gitlab-token",
+    )
+
+    args = parser.parse_args()
+    args.gitlab_token = Path(args.gitlab_token_file).read_text().strip()
+    return args
+
+
+def main():
+    args = parse_args()
+    gl_gql = GitlabGQL(token=args.gitlab_token)
+
+    sha = check_output(['git', 'rev-parse', args.sha]).decode('ascii').strip()
+
+    if args.print_dag:
+        iid = from_sha_to_pipeline_iid(gl_gql, {"projectPath": args.project_path, "sha": sha})
+        dag = create_job_needs_dag(
+            gl_gql, {"projectPath": args.project_path, "iid": iid}, disable_cache=True
+        )
+
+        job_name_regex = re.compile(args.regex)
+        include_stage_regex = re.compile(args.include_stage)
+        exclude_stage_regex = re.compile(args.exclude_stage)
+        job_tags_regexes = [re.compile(job_tag) for job_tag in args.job_tags]
+
+        def job_filter(
+            job_name: str,
+            job_stage: str,
+            job_tags: set[str],
+        ) -> bool:
+            """
+            Apply user-specified filters to a job, and return whether the
+            filters allow that job (True) or not (False).
+            """
+            if not job_name_regex.fullmatch(job_name):
+                return False
+            if not include_stage_regex.fullmatch(job_stage):
+                return False
+            if exclude_stage_regex.fullmatch(job_stage):
+                return False
+            if not all(
+                any(job_tags_regex.fullmatch(tag) for tag in job_tags)
+                for job_tags_regex in job_tags_regexes
+            ):
+                return False
+            return True
+
+        dag = filter_dag(dag, job_filter)
+
+        print_dag(dag)
+
+    if args.print_merged_yaml or args.print_job_manifest:
+        merged_yaml = fetch_merged_yaml(
+            gl_gql, {"projectPath": args.project_path, "sha": sha}
+        )
+
+        if args.print_merged_yaml:
+            print(yaml.dump(merged_yaml, indent=2))
+
+        if args.print_job_manifest:
+            print_job_final_definition(
+                args.print_job_manifest, merged_yaml, args.project_path, sha
+            )
+
+
+if __name__ == "__main__":
+    main()

bin/ci/gitlab_gql.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -eu
+
+this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
+readonly this_dir
+
+exec \
+  "$this_dir/../python-venv.sh" \
+  "$this_dir/requirements.txt" \
+  "$this_dir/gitlab_gql.py" "$@"

bin/ci/job_details.gql

@@ -0,0 +1,7 @@
+query getCiConfigData($projectPath: ID!, $sha: String, $content: String!) {
+  ciConfig(projectPath: $projectPath, sha: $sha, content: $content) {
+    errors
+    mergedYaml
+    __typename
+  }
+}

bin/ci/marge_queue.py

@@ -0,0 +1,236 @@
+#!/usr/bin/env python3
+# Copyright © 2020 - 2023 Collabora Ltd.
+# Authors:
+#   David Heidelberg <[email protected]>
+#
+# SPDX-License-Identifier: MIT
+
+"""
+Monitors Marge-bot and return number of assigned MRs.
+"""
+
+import argparse
+import time
+import sys
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta, timezone
+from dateutil import parser
+from traceback import print_exc
+from typing import Optional
+
+import gitlab
+from gitlab.base import RESTObjectList
+from gitlab.exceptions import GitlabAuthenticationError
+from gitlab.v4.objects import Project, ProjectMergeRequest
+from gitlab_common import get_token_from_default_dir, read_token, pretty_duration
+from rich.console import Console
+
+REFRESH_WAIT = 30
+MARGE_BOT_USER_ID = 9716
+ASSIGNED_TO_MARGE = "assigned to @marge-bot"
+ASSIGNED_AT_PADDING = 19
+WAITING_PADDING = 8
+MR_ID_PADDING = 5
+
+console = Console(highlight=False)
+print = console.print
+
+
+def parse_args() -> argparse.Namespace:
+    """Parse args"""
+    parse = argparse.ArgumentParser(
+        description="Tool to show merge requests assigned to the marge-bot",
+    )
+    parse.add_argument(
+        "--wait", action="store_true", help="wait until CI is free",
+    )
+    parse.add_argument(
+        "--token",
+        metavar="token",
+        default=get_token_from_default_dir(),
+        help="force GitLab token, otherwise it's read from ~/.config/gitlab-token",
+    )
+    return parse.parse_args()
+
+
+@dataclass
+class MargeMergeRequest:
+    """Represent a Merge Request assigned to Marge"""
+    id: int = field(init=False)
+    mr: ProjectMergeRequest = field(repr=False)
+    updated_at: datetime | None = field(init=False, repr=False)
+    assigned_at: datetime | None = field(init=False, repr=False)
+
+    def __post_init__(self):
+        self.id = self.mr.iid
+        self.updated_at = parser.parse(self.mr.updated_at)
+        self.assigned_at = self.__find_last_assign_to_marge()
+
+    def __find_last_assign_to_marge(self) -> Optional[datetime]:
+        for note in self.__get_mr_notes_iterator():
+            if note.body.startswith(ASSIGNED_TO_MARGE):
+                return parser.parse(note.created_at)
+
+    def __eq__(self, other: "MargeMergeRequest") -> bool:
+        return self.id == other.id
+
+    @property
+    def time_enqueued(self) -> timedelta:
+        if self.assigned_at is None:
+            raise ValueError("Assign to marge timestamp not defined")
+        return datetime.now(timezone.utc) - self.assigned_at
+
+    @property
+    def web_url(self) -> str:
+        return self.mr.web_url
+
+    @property
+    def title(self) -> str:
+        return self.mr.title
+
+    def __get_mr_notes_iterator(self) -> RESTObjectList:
+        return self.mr.notes.list(
+            iterator=True, order_by="updated_at", sort="desc"
+        )  # start with the most recent
+
+
+@dataclass
+class MargeQueue:
+    """
+    Collect and sort the merge requests assigned to marge
+    """
+    elements_sorted: dict[datetime, MargeMergeRequest] = field(
+        init=False, repr=False, default_factory=dict
+    )
+    undetermined: list[MargeMergeRequest] = field(
+        init=False, repr=False, default_factory=list
+    )
+
+    def append(self, mr: MargeMergeRequest) -> None:
+        if mr.assigned_at is not None:
+            self.elements_sorted[mr.assigned_at] = mr
+        else:
+            self.undetermined.append(mr)
+
+    @property
+    def n_merge_requests_enqueued(self) -> int:
+        return len(self.elements_sorted) + len(self.undetermined)
+
+    @property
+    def sorted_queue(self) -> list[MargeMergeRequest]:
+        """
+        Provide a list of the elements that can be sorted based on the
+        assignment to Marge.
+        """
+        return list(dict(sorted(self.elements_sorted.items())).values())
+
+    @property
+    def all_assigned(
+        self
+    ) -> list[MargeMergeRequest]:
+        """
+        Provide a single list, but in sorted, of all the elements found
+        assigned to Marge. This include the elements in elements_sorted
+        and the ones undetermined (expected to be empty).
+        """
+        return self.sorted_queue + self.undetermined
+
+
+def get_merge_queue(project: Project) -> MargeQueue:
+    queue = MargeQueue()
+    for mr in __get_project_marge_merge_requests(project):
+        marge_merge_request = MargeMergeRequest(mr)
+        queue.append(marge_merge_request)
+    for mr in queue.sorted_queue:
+        assigned_at = f"{mr.assigned_at:%Y-%m-%d %H:%M:%S}"
+        time_enqueued = f"{pretty_duration(mr.time_enqueued.total_seconds())}"
+        print(
+            f"⛭ {assigned_at:<{ASSIGNED_AT_PADDING}} "
+            f"{time_enqueued:>{WAITING_PADDING}} "
+            f"{link2print(mr.web_url, mr.id, MR_ID_PADDING)} {mr.title}"
+        )
+    if queue.undetermined:
+        print(
+            "Unable to determine when they where assigned to Marge:"
+        )
+        for mr in queue.undetermined:
+            updated_at = f"{mr.updated_at:%Y-%m-%d %H:%M:%S}"
+            print(
+                f"⛭ {updated_at:<{ASSIGNED_AT_PADDING}} "
+                f"{" ":{WAITING_PADDING}} "
+                f"{link2print(mr.web_url, mr.id, MR_ID_PADDING)} {mr.title}"
+            )
+    return queue
+
+
+def __get_gitlab_object(token: str) -> gitlab.Gitlab:
+    return gitlab.Gitlab(url="https://gitlab.freedesktop.org", private_token=token, retry_transient_errors=True)
+
+
+def __get_gitlab_project(gl: gitlab.Gitlab) -> Project:
+    return gl.projects.get("mesa/mesa")
+
+
+def __get_project_marge_merge_requests(
+    project: Project
+) -> list[ProjectMergeRequest]:
+    return project.mergerequests.list(
+        assignee_id=MARGE_BOT_USER_ID,
+        scope="all",
+        state="opened",
+        get_all=True,
+    )
+
+
+def link2print(url: str, text: str, text_pad: int = 0) -> str:
+    text = str(text)
+    text_pad = len(text) if text_pad < 1 else text_pad
+    return f"[link={url}]{text:{text_pad}}[/link]"
+
+
+def main():
+    args = parse_args()
+    token = read_token(args.token)
+    gl = __get_gitlab_object(token)
+
+    project = __get_gitlab_project(gl)
+
+    assigned_at = "Assigned at"
+    waiting = "Waiting"
+    mr = "MR"
+    title = "Title"
+    print(
+        f"  {assigned_at:<{ASSIGNED_AT_PADDING}} "
+        f"{waiting:<{WAITING_PADDING}} "
+        f"{mr:<{MR_ID_PADDING}} {title}"
+    )
+    while True:
+        try:
+            n_mrs = get_merge_queue(project).n_merge_requests_enqueued
+        except GitlabAuthenticationError as autherror:
+            if autherror.response_code == 401:  # 401 Unauthorized
+                print(f"Alert! Tool action requires authentication. Use the --token argument")
+            else:
+                print(f"Alert! Something went wrong: {autherror.responce_body}")
+                print_exc()
+            sys.exit(-1)
+
+        print(f"Job waiting: {n_mrs}")
+
+        if n_mrs == 0:
+            sys.exit(0)
+        if not args.wait:
+            sys.exit(min(n_mrs, 127))
+
+        try:
+            time.sleep(REFRESH_WAIT)
+        except KeyboardInterrupt:
+            print(
+                "Sleep interrupted from keyboard, "
+                "doing one last iteration before finish."
+            )
+            args.wait = False
+
+
+if __name__ == "__main__":
+    main()

bin/ci/marge_queue.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -eu
+
+this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
+readonly this_dir
+
+exec \
+  "$this_dir/../python-venv.sh" \
+  "$this_dir/requirements.txt" \
+  "$this_dir/marge_queue.py" "$@"

bin/ci/nightly_compare.py

@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+# Copyright © 2020 - 2024 Collabora Ltd.
+# Authors:
+#   David Heidelberg <[email protected]>
+#   Sergi Blanch Torne <[email protected]>
+# SPDX-License-Identifier: MIT
+
+"""
+Compare the two latest scheduled pipelines and provide information
+about the jobs you're interested in.
+"""
+
+import argparse
+import csv
+import re
+import requests
+import io
+from tabulate import tabulate
+
+import gitlab
+from gitlab_common import read_token
+from rich import print
+
+
+MARGE_BOT_USER_ID = 9716
+
+
+def print_failures_csv(id):
+    url = "https://gitlab.freedesktop.org/mesa/mesa"\
+        f"/-/jobs/{id}/artifacts/raw/results/failures.csv"
+    missing: int = 0
+    MAX_MISS: int = 20
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        csv_content = io.StringIO(response.text)
+        csv_reader = csv.reader(csv_content)
+        data = list(csv_reader)
+
+        for line in data[:]:
+            if line[1] == "UnexpectedImprovement(Pass)":
+                line[1] = f"[green]{line[1]}[/green]"
+            elif line[1] == "UnexpectedImprovement(Fail)":
+                line[1] = f"[yellow]{line[1]}[/yellow]"
+            elif line[1] == "Crash" or line[1] == "Fail":
+                line[1] = f" [red]{line[1]}[/red]"
+            elif line[1] == "Missing":
+                if missing > MAX_MISS:
+                    data.remove(line)
+                    continue
+                missing += 1
+                line[1] = f"[yellow]{line[1]}[/yellow]"
+            elif line[1] == "Fail":
+                line[1] = f"[red]{line[1]}[/red]"
+            else:
+                line[1] = f"[white]{line[1]}[/white]"
+
+        if missing > MAX_MISS:
+            data.append(
+                [
+                    f"[red]... more than {MAX_MISS} missing tests, "
+                    "something crashed?[/red]",
+                    "[red]Missing[/red]"
+                ]
+            )
+        headers = [f"Test{"":<75}", "Result"]
+        print(tabulate(data, headers, tablefmt="plain"))
+    except Exception:
+        pass
+
+
+def job_failed_before(old_jobs, job):
+    for old_job in old_jobs:
+        if job.name == old_job.name:
+            return old_job
+
+
+def parse_args() -> None:
+    """Parse args"""
+    parser = argparse.ArgumentParser(
+        description="Tool to show merge requests assigned to the marge-bot",
+    )
+    parser.add_argument(
+        "--target",
+        metavar="target-job",
+        help="Target job regex. For multiple targets, pass multiple values, "
+        "eg. `--target foo bar`.",
+        required=False,
+        nargs=argparse.ONE_OR_MORE,
+    )
+    parser.add_argument(
+        "--token",
+        metavar="token",
+        help="force GitLab token, "
+        "otherwise it's read from ~/.config/gitlab-token",
+    )
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    token = read_token(args.token)
+    gl = gitlab.Gitlab(
+        url="https://gitlab.freedesktop.org",
+        private_token=token,
+    )
+
+    project = gl.projects.get("mesa/mesa")
+
+    print(
+        "[link=https://gitlab.freedesktop.org/mesa/mesa/-/pipelines?"
+        "page=1&scope=all&source=schedule]Scheduled pipelines overview[/link]"
+    )
+    pipelines = project.pipelines.list(
+        source="schedule",
+        ordered_by="created_at",
+        sort="desc",
+        page=1,
+        per_page=2,
+    )
+    print(
+        "Old pipeline:"
+        f" {pipelines[1].created_at}"
+        f"\t[link={pipelines[1].web_url}]{pipelines[1].status}[/link]"
+        f"\t{pipelines[1].sha}"
+    )
+    print(
+        "New pipeline:"
+        f" {pipelines[0].created_at}"
+        f"\t[link={pipelines[0].web_url}]{pipelines[0].status}[/link]"
+        f"\t{pipelines[0].sha}"
+    )
+    print(
+        "\nWebUI visual compare: "
+        "https://gitlab.freedesktop.org/mesa/mesa/-/compare/"
+        f"{pipelines[1].sha}...{pipelines[0].sha}\n"
+    )
+
+    # regex part
+    if args.target:
+        target = "|".join(args.target)
+        target = target.strip()
+        print(f"🞋 jobs: [blue]{target}[/blue]")
+
+        target = f"({target})" + r"( \d+/\d+)?"
+    else:
+        target = ".*"
+
+    target_jobs_regex: re.Pattern = re.compile(target)
+
+    old_failed_jobs = []
+    for job in pipelines[1].jobs.list(all=True):
+        if (
+            job.status != "failed"
+            or target_jobs_regex
+            and not target_jobs_regex.fullmatch(job.name)
+        ):
+            continue
+        old_failed_jobs.append(job)
+
+    job_failed = False
+    for job in pipelines[0].jobs.list(all=True):
+        if (
+            job.status != "failed"
+            or target_jobs_regex
+            and not target_jobs_regex.fullmatch(job.name)
+        ):
+            continue
+
+        job_failed = True
+
+        previously_failed_job = job_failed_before(old_failed_jobs, job)
+        if previously_failed_job:
+            print(
+                f"[yellow]"
+                f" :: [link={job.web_url}]{job.name}[/link][/yellow]"
+                f"[magenta]"
+                f" [link={previously_failed_job.web_url}](previous run)[/link]"
+            )
+        else:
+            print(
+                f"[red]:: [link={job.web_url}]{job.name}[/link]"
+            )
+        print_failures_csv(job.id)
+
+    if not job_failed:
+        exit(0)
+
+    print("Commits between nightly pipelines:")
+    commit = project.commits.get(pipelines[0].sha)
+    while True:
+        print(
+            f"{commit.id}  [link={commit.web_url}]{commit.title}[/link]"
+        )
+        if commit.id == pipelines[1].sha:
+            break
+        commit = project.commits.get(commit.parent_ids[0])

bin/ci/nightly_compare.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -eu
+
+this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
+readonly this_dir
+
+exec \
+  "$this_dir/../python-venv.sh" \
+  "$this_dir/requirements.txt" \
+  "$this_dir/nightly_compare.py" "$@"
+

bin/ci/pipeline_details.gql

@@ -0,0 +1,36 @@
+query jobs($projectPath: ID!, $iid: ID!, $cursor: String) {
+  project(fullPath: $projectPath) {
+    id
+    pipeline(iid: $iid) {
+      id
+      iid
+      complete
+      stages {
+        nodes {
+          name
+        }
+      }
+      jobs(after: $cursor) {
+        pageInfo {
+          hasNextPage
+          endCursor
+        }
+        count
+        nodes {
+          name
+          needs {
+            edges {
+              node {
+                name
+              }
+            }
+          }
+          stage {
+            name
+          }
+          tags
+        }
+      }
+    }
+  }
+}

bin/ci/pipeline_message.py

@@ -0,0 +1 @@
+../../.marge/hooks/pipeline_message.py

bin/ci/pipeline_message.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -eu
+
+this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
+readonly this_dir
+
+exec \
+  "$this_dir/../python-venv.sh" \
+  "$this_dir/requirements.txt" \
+  "$this_dir/pipeline_message.py" "$@"

bin/ci/pipeline_utils.gql

@@ -0,0 +1,9 @@
+query sha2pipelineIID($projectPath: ID!, $sha: String!) {
+  project(fullPath: $projectPath) {
+    pipelines(last: 1, sha:$sha){
+      nodes {
+        iid
+      }
+    }
+  }
+}

bin/ci/requirements.txt

@@ -0,0 +1,14 @@
+PyYAML==6.*
+filecache==0.81
+flake8==7.*
+gql==3.*
+kaleido==0.2.*
+pandas==2.*
+plotly==5.*
+python-dateutil==2.*
+python-gitlab==5.*
+rich==14.1.*
+ruamel.yaml.clib==0.2.*
+ruamel.yaml==0.17.*
+tabulate==0.9.*
+tomledit==1.*

bin/ci/structured_logger.py

@@ -0,0 +1,294 @@
+"""
+A structured logging utility supporting multiple data formats such as CSV, JSON,
+and YAML.
+
+The main purpose of this script, besides having relevant information available
+in a condensed and deserialized.
+
+This script defines a protocol for different file handling strategies and provides
+implementations for CSV, JSON, and YAML formats. The main class, StructuredLogger,
+allows for easy interaction with log data, enabling users to load, save, increment,
+set, and append fields in the log. The script also includes context managers for
+file locking and editing log data to ensure data integrity and avoid race conditions.
+"""
+
+import json
+import os
+from collections.abc import MutableMapping, MutableSequence
+from contextlib import contextmanager
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Protocol
+
+import fire
+from filelock import FileLock
+
+try:
+    import polars as pl
+
+    CSV_LIB_EXCEPTION = None
+except ImportError as e:
+    CSV_LIB_EXCEPTION: ImportError = e
+
+try:
+    from ruamel.yaml import YAML
+
+    YAML_LIB_EXCEPTION = None
+except ImportError as e:
+    YAML_LIB_EXCEPTION: ImportError = e
+
+
+class ContainerProxy:
+    """
+    A proxy class that wraps a mutable container object (such as a dictionary or
+    a list) and calls a provided save_callback function whenever the container
+    or its contents
+    are changed.
+    """
+    def __init__(self, container, save_callback):
+        self.container = container
+        self.save_callback = save_callback
+
+    def __getitem__(self, key):
+        value = self.container[key]
+        if isinstance(value, (MutableMapping, MutableSequence)):
+            return ContainerProxy(value, self.save_callback)
+        return value
+
+    def __setitem__(self, key, value):
+        self.container[key] = value
+        self.save_callback()
+
+    def __delitem__(self, key):
+        del self.container[key]
+        self.save_callback()
+
+    def __getattr__(self, name):
+        attr = getattr(self.container, name)
+
+        if callable(attr):
+            def wrapper(*args, **kwargs):
+                result = attr(*args, **kwargs)
+                self.save_callback()
+                return result
+
+            return wrapper
+        return attr
+
+    def __iter__(self):
+        return iter(self.container)
+
+    def __len__(self):
+        return len(self.container)
+
+    def __repr__(self):
+        return repr(self.container)
+
+
+class AutoSaveDict(dict):
+    """
+    A subclass of the built-in dict class with additional functionality to
+    automatically save changes to the dictionary. It maintains a timestamp of
+    the last modification and automatically wraps nested mutable containers
+    using ContainerProxy.
+    """
+    timestamp_key = "_timestamp"
+
+    def __init__(self, *args, save_callback, register_timestamp=True, **kwargs):
+        self.save_callback = save_callback
+        self.__register_timestamp = register_timestamp
+        self.__heartbeat()
+        super().__init__(*args, **kwargs)
+        self.__wrap_dictionaries()
+
+    def __heartbeat(self):
+        if self.__register_timestamp:
+            self[AutoSaveDict.timestamp_key] = datetime.now().isoformat()
+
+    def __save(self):
+        self.__heartbeat()
+        self.save_callback()
+
+    def __wrap_dictionaries(self):
+        for key, value in self.items():
+            if isinstance(value, MutableMapping) and not isinstance(
+                value, AutoSaveDict
+            ):
+                self[key] = AutoSaveDict(
+                    value, save_callback=self.save_callback, register_timestamp=False
+                )
+
+    def __setitem__(self, key, value):
+        if isinstance(value, MutableMapping) and not isinstance(value, AutoSaveDict):
+            value = AutoSaveDict(
+                value, save_callback=self.save_callback, register_timestamp=False
+            )
+        super().__setitem__(key, value)
+
+        if self.__register_timestamp and key == AutoSaveDict.timestamp_key:
+            return
+        self.__save()
+
+    def __getitem__(self, key):
+        value = super().__getitem__(key)
+        if isinstance(value, (MutableMapping, MutableSequence)):
+            return ContainerProxy(value, self.__save)
+        return value
+
+    def __delitem__(self, key):
+        super().__delitem__(key)
+        self.__save()
+
+    def pop(self, *args, **kwargs):
+        result = super().pop(*args, **kwargs)
+        self.__save()
+        return result
+
+    def update(self, *args, **kwargs):
+        super().update(*args, **kwargs)
+        self.__wrap_dictionaries()
+        self.__save()
+
+
+class StructuredLoggerStrategy(Protocol):
+    def load_data(self, file_path: Path) -> dict:
+        pass
+
+    def save_data(self, file_path: Path, data: dict) -> None:
+        pass
+
+
+class CSVStrategy:
+    def __init__(self) -> None:
+        if CSV_LIB_EXCEPTION:
+            raise RuntimeError(
+                "Can't parse CSV files. Missing library"
+            ) from CSV_LIB_EXCEPTION
+
+    def load_data(self, file_path: Path) -> dict:
+        dicts: list[dict[str, Any]] = pl.read_csv(
+            file_path, try_parse_dates=True
+        ).to_dicts()
+        data = {}
+        for d in dicts:
+            for k, v in d.items():
+                if k != AutoSaveDict.timestamp_key and k in data:
+                    if isinstance(data[k], list):
+                        data[k].append(v)
+                        continue
+                    data[k] = [data[k], v]
+                else:
+                    data[k] = v
+        return data
+
+    def save_data(self, file_path: Path, data: dict) -> None:
+        pl.DataFrame(data).write_csv(file_path)
+
+
+class JSONStrategy:
+    def load_data(self, file_path: Path) -> dict:
+        return json.loads(file_path.read_text())
+
+    def save_data(self, file_path: Path, data: dict) -> None:
+        with open(file_path, "w") as f:
+            json.dump(data, f, indent=2)
+
+
+class YAMLStrategy:
+    def __init__(self):
+        if YAML_LIB_EXCEPTION:
+            raise RuntimeError(
+                "Can't parse YAML files. Missing library"
+            ) from YAML_LIB_EXCEPTION
+        self.yaml = YAML()
+        self.yaml.indent(sequence=4, offset=2)
+        self.yaml.default_flow_style = False
+        self.yaml.representer.add_representer(AutoSaveDict, self.represent_dict)
+
+    @classmethod
+    def represent_dict(cls, dumper, data):
+        return dumper.represent_mapping("tag:yaml.org,2002:map", data)
+
+    def load_data(self, file_path: Path) -> dict:
+        return self.yaml.load(file_path.read_text())
+
+    def save_data(self, file_path: Path, data: dict) -> None:
+        with open(file_path, "w") as f:
+            self.yaml.dump(data, f)
+
+
+class StructuredLogger:
+    def __init__(
+        self, file_name: str, strategy: StructuredLoggerStrategy = None, truncate=False
+    ):
+        self.file_name: str = file_name
+        self.file_path = Path(self.file_name)
+        self._data: AutoSaveDict = AutoSaveDict(save_callback=self.save_data)
+
+        if strategy is None:
+            self.strategy: StructuredLoggerStrategy = self.guess_strategy_from_file(
+                self.file_path
+            )
+        else:
+            self.strategy = strategy
+
+        if not self.file_path.exists():
+            Path.mkdir(self.file_path.parent, exist_ok=True)
+            self.save_data()
+            return
+
+        if truncate:
+            with self.get_lock():
+                os.truncate(self.file_path, 0)
+                self.save_data()
+
+    def load_data(self):
+        self._data = self.strategy.load_data(self.file_path)
+
+    def save_data(self):
+        self.strategy.save_data(self.file_path, self._data)
+
+    @property
+    def data(self) -> AutoSaveDict:
+        return self._data
+
+    @contextmanager
+    def get_lock(self):
+        with FileLock(f"{self.file_path}.lock", timeout=10):
+            yield
+
+    @contextmanager
+    def edit_context(self):
+        """
+        Context manager that ensures proper loading and saving of log data when
+        performing multiple modifications.
+        """
+        with self.get_lock():
+            try:
+                self.load_data()
+                yield
+            finally:
+                self.save_data()
+
+    @staticmethod
+    def guess_strategy_from_file(file_path: Path) -> StructuredLoggerStrategy:
+        file_extension = file_path.suffix.lower().lstrip(".")
+        return StructuredLogger.get_strategy(file_extension)
+
+    @staticmethod
+    def get_strategy(strategy_name: str) -> StructuredLoggerStrategy:
+        strategies = {
+            "csv": CSVStrategy,
+            "json": JSONStrategy,
+            "yaml": YAMLStrategy,
+            "yml": YAMLStrategy,
+        }
+
+        try:
+            return strategies[strategy_name]()
+        except KeyError as e:
+            raise ValueError(f"Unknown strategy for: {strategy_name}") from e
+
+
+if __name__ == "__main__":
+    fire.Fire(StructuredLogger)

bin/ci/test/requirements.txt

@@ -0,0 +1,10 @@
+-r ../requirements.txt
+aiohttp==3.13.*
+filelock==3.12.4
+fire==0.7.0
+freezegun==1.5.1
+mock==5.1.0
+polars==0.19.3
+pytest==7.4.2
+pytest-asyncio==0.21.0
+pytest-cov==3.0.0

bin/ci/test/test_custom_logger.py

@@ -0,0 +1,746 @@
+import logging
+import subprocess
+from datetime import datetime
+
+import pytest
+from custom_logger import CustomLogger
+
+
+@pytest.fixture
+def tmp_log_file(tmp_path):
+    return tmp_path / "test_log.json"
+
+
+@pytest.fixture
+def custom_logger(tmp_log_file):
+    return CustomLogger(tmp_log_file)
+
+
+def run_script_with_args(args):
+    import custom_logger
+
+    script_path = custom_logger.__file__
+    return subprocess.run(
+        ["python3", str(script_path), *args], capture_output=True, text=True
+    )
+
+
+# Test case for missing log file
+@pytest.mark.parametrize(
+    "key, value", [("dut_attempt_counter", "1"), ("job_combined_status", "pass")]
+)
+def test_missing_log_file_argument(key, value):
+    result = run_script_with_args(["--update", "key", "value"])
+    assert result.returncode != 0
+
+
+# Parametrize test case for valid update arguments
+@pytest.mark.parametrize(
+    "key, value", [("dut_attempt_counter", "1"), ("job_combined_status", "pass")]
+)
+def test_update_argument_valid(custom_logger, tmp_log_file, key, value):
+    result = run_script_with_args([str(tmp_log_file), "--update", key, value])
+    assert result.returncode == 0
+
+
+# Test case for passing only the key without a value
+def test_update_argument_key_only(custom_logger, tmp_log_file):
+    key = "dut_attempt_counter"
+    result = run_script_with_args([str(tmp_log_file), "--update", key])
+    assert result.returncode != 0
+
+
+# Test case for not passing any key-value pair
+def test_update_argument_no_values(custom_logger, tmp_log_file):
+    result = run_script_with_args([str(tmp_log_file), "--update"])
+    assert result.returncode == 0
+
+
+# Parametrize test case for valid arguments
+@pytest.mark.parametrize(
+    "key, value", [("dut_attempt_counter", "1"), ("job_combined_status", "pass")]
+)
+def test_create_argument_valid(custom_logger, tmp_log_file, key, value):
+    result = run_script_with_args([str(tmp_log_file), "--create-dut-job", key, value])
+    assert result.returncode == 0
+
+
+# Test case for passing only the key without a value
+def test_create_argument_key_only(custom_logger, tmp_log_file):
+    key = "dut_attempt_counter"
+    result = run_script_with_args([str(tmp_log_file), "--create-dut-job", key])
+    assert result.returncode != 0
+
+
+# Test case for not passing any key-value pair
+def test_create_argument_no_values(custom_logger, tmp_log_file):
+    result = run_script_with_args([str(tmp_log_file), "--create-dut-job"])
+    assert result.returncode == 0
+
+
+# Test case for updating a DUT job
+@pytest.mark.parametrize(
+    "key, value", [("status", "hung"), ("dut_state", "Canceling"), ("dut_name", "asus")]
+)
+def test_update_dut_job(custom_logger, tmp_log_file, key, value):
+    result = run_script_with_args([str(tmp_log_file), "--update-dut-job", key, value])
+    assert result.returncode != 0
+
+    result = run_script_with_args([str(tmp_log_file), "--create-dut-job", key, value])
+    assert result.returncode == 0
+
+    result = run_script_with_args([str(tmp_log_file), "--update-dut-job", key, value])
+    assert result.returncode == 0
+
+
+# Test case for updating last DUT job
+def test_update_dut_multiple_job(custom_logger, tmp_log_file):
+    # Create the first DUT job with the first key
+    result = run_script_with_args(
+        [str(tmp_log_file), "--create-dut-job", "status", "hung"]
+    )
+    assert result.returncode == 0
+
+    # Create the second DUT job with the second key
+    result = run_script_with_args(
+        [str(tmp_log_file), "--create-dut-job", "dut_state", "Canceling"]
+    )
+    assert result.returncode == 0
+
+    result = run_script_with_args(
+        [str(tmp_log_file), "--update-dut-job", "dut_name", "asus"]
+    )
+    assert result.returncode == 0
+
+
+# Parametrize test case for valid phase arguments
+@pytest.mark.parametrize(
+    "phase_name",
+    [("Phase1"), ("Phase2"), ("Phase3")],
+)
+def test_create_job_phase_valid(custom_logger, tmp_log_file, phase_name):
+    custom_logger.create_dut_job(status="pass")
+
+    result = run_script_with_args([str(tmp_log_file), "--create-job-phase", phase_name])
+    assert result.returncode == 0
+
+
+# Test case for not passing any arguments for create-job-phase
+def test_create_job_phase_no_arguments(custom_logger, tmp_log_file):
+    custom_logger.create_dut_job(status="pass")
+
+    result = run_script_with_args([str(tmp_log_file), "--create-job-phase"])
+    assert result.returncode != 0
+
+
+# Test case for trying to create a phase job without an existing DUT job
+def test_create_job_phase_no_dut_job(custom_logger, tmp_log_file):
+    phase_name = "Phase1"
+
+    result = run_script_with_args([str(tmp_log_file), "--create-job-phase", phase_name])
+    assert result.returncode != 0
+
+
+# Combined test cases for valid scenarios
+def test_valid_scenarios(custom_logger, tmp_log_file):
+    valid_update_args = [("dut_attempt_counter", "1"), ("job_combined_status", "pass")]
+    for key, value in valid_update_args:
+        result = run_script_with_args([str(tmp_log_file), "--update", key, value])
+        assert result.returncode == 0
+
+    valid_create_args = [
+        ("status", "hung"),
+        ("dut_state", "Canceling"),
+        ("dut_name", "asus"),
+        ("phase_name", "Bootloader"),
+    ]
+    for key, value in valid_create_args:
+        result = run_script_with_args(
+            [str(tmp_log_file), "--create-dut-job", key, value]
+        )
+        assert result.returncode == 0
+
+    result = run_script_with_args(
+        [str(tmp_log_file), "--create-dut-job", "status", "hung"]
+    )
+    assert result.returncode == 0
+
+    result = run_script_with_args(
+        [str(tmp_log_file), "--update-dut-job", "dut_name", "asus"]
+    )
+    assert result.returncode == 0
+
+    result = run_script_with_args(
+        [
+            str(tmp_log_file),
+            "--create-job-phase",
+            "phase_name",
+        ]
+    )
+    assert result.returncode == 0
+
+
+# Parametrize test case for valid update arguments
+@pytest.mark.parametrize(
+    "key, value", [("dut_attempt_counter", "1"), ("job_combined_status", "pass")]
+)
+def test_update(custom_logger, key, value):
+    custom_logger.update(**{key: value})
+    logger_data = custom_logger.logger.data
+
+    assert key in logger_data
+    assert logger_data[key] == value
+
+
+# Test case for updating with a key that already exists
+def test_update_existing_key(custom_logger):
+    key = "status"
+    value = "new_value"
+    custom_logger.logger.data[key] = "old_value"
+    custom_logger.update(**{key: value})
+    logger_data = custom_logger.logger.data
+
+    assert key in logger_data
+    assert logger_data[key] == value
+
+
+# Test case for updating "dut_jobs"
+def test_update_dut_jobs(custom_logger):
+    key1 = "status"
+    value1 = "fail"
+    key2 = "state"
+    value2 = "hung"
+
+    custom_logger.create_dut_job(**{key1: value1})
+    logger_data = custom_logger.logger.data
+
+    job1 = logger_data["dut_jobs"][0]
+    assert key1 in job1
+    assert job1[key1] == value1
+
+    custom_logger.update_dut_job(key2, value2)
+    logger_data = custom_logger.logger.data
+
+    job2 = logger_data["dut_jobs"][0]
+    assert key2 in job2
+    assert job2[key2] == value2
+
+
+# Test case for creating and updating DUT job
+def test_create_dut_job(custom_logger):
+    key = "status"
+    value1 = "pass"
+    value2 = "fail"
+    value3 = "hung"
+
+    reason = "job_combined_status"
+    result = "Finished"
+
+    custom_logger.update(**{reason: result})
+    logger_data = custom_logger.logger.data
+
+    assert reason in logger_data
+    assert logger_data[reason] == result
+
+    # Create the first DUT job
+    custom_logger.create_dut_job(**{key: value1})
+    logger_data = custom_logger.logger.data
+
+    assert "dut_jobs" in logger_data
+    assert isinstance(logger_data["dut_jobs"], list)
+    assert len(logger_data["dut_jobs"]) == 1
+    assert isinstance(logger_data["dut_jobs"][0], dict)
+
+    # Check the values of the keys in the created first DUT job
+    job1 = logger_data["dut_jobs"][0]
+    assert key in job1
+    assert job1[key] == value1
+
+    # Create the second DUT job
+    custom_logger.create_dut_job(**{key: value2})
+    logger_data = custom_logger.logger.data
+
+    assert "dut_jobs" in logger_data
+    assert isinstance(logger_data["dut_jobs"], list)
+    assert len(logger_data["dut_jobs"]) == 2
+    assert isinstance(logger_data["dut_jobs"][1], dict)
+
+    # Check the values of the keys in the created second DUT job
+    job2 = logger_data["dut_jobs"][1]
+    assert key in job2
+    assert job2[key] == value2
+
+    # Update the second DUT job with value3
+    custom_logger.update_dut_job(key, value3)
+    logger_data = custom_logger.logger.data
+
+    # Check the updated value in the second DUT job
+    job2 = logger_data["dut_jobs"][1]
+    assert key in job2
+    assert job2[key] == value3
+
+    # Find the index of the last DUT job
+    last_job_index = len(logger_data["dut_jobs"]) - 1
+
+    # Update the last DUT job
+    custom_logger.update_dut_job("dut_name", "asus")
+    logger_data = custom_logger.logger.data
+
+    # Check the updated value in the last DUT job
+    job2 = logger_data["dut_jobs"][last_job_index]
+    assert "dut_name" in job2
+    assert job2["dut_name"] == "asus"
+
+    # Check that "dut_name" is not present in other DUT jobs
+    for idx, job in enumerate(logger_data["dut_jobs"]):
+        if idx != last_job_index:
+            assert job.get("dut_name") == ""
+
+
+# Test case for updating with missing "dut_jobs" key
+def test_update_dut_job_missing_dut_jobs(custom_logger):
+    key = "status"
+    value = "fail"
+
+    # Attempt to update a DUT job when "dut_jobs" is missing
+    with pytest.raises(ValueError, match="No DUT jobs found."):
+        custom_logger.update_dut_job(key, value)
+
+
+# Test case for creating a job phase
+def test_create_job_phase(custom_logger):
+    custom_logger.create_dut_job(status="pass")
+    phase_name = "Phase1"
+
+    custom_logger.create_job_phase(phase_name)
+    logger_data = custom_logger.logger.data
+
+    assert "dut_jobs" in logger_data
+    assert isinstance(logger_data["dut_jobs"], list)
+    assert len(logger_data["dut_jobs"]) == 1
+
+    job = logger_data["dut_jobs"][0]
+    assert "dut_job_phases" in job
+    assert isinstance(job["dut_job_phases"], list)
+    assert len(job["dut_job_phases"]) == 1
+
+    phase = job["dut_job_phases"][0]
+    assert phase["name"] == phase_name
+    try:
+        datetime.fromisoformat(phase["start_time"])
+        assert True
+    except ValueError:
+        assert False
+    assert phase["end_time"] == ""
+
+
+# Test case for creating multiple phase jobs
+def test_create_multiple_phase_jobs(custom_logger):
+    custom_logger.create_dut_job(status="pass")
+
+    phase_data = [
+        {
+            "phase_name": "Phase1",
+        },
+        {
+            "phase_name": "Phase2",
+        },
+        {
+            "phase_name": "Phase3",
+        },
+    ]
+
+    for data in phase_data:
+        phase_name = data["phase_name"]
+
+        custom_logger.create_job_phase(phase_name)
+
+    logger_data = custom_logger.logger.data
+
+    assert "dut_jobs" in logger_data
+    assert isinstance(logger_data["dut_jobs"], list)
+    assert len(logger_data["dut_jobs"]) == 1
+
+    job = logger_data["dut_jobs"][0]
+    assert "dut_job_phases" in job
+    assert isinstance(job["dut_job_phases"], list)
+    assert len(job["dut_job_phases"]) == len(phase_data)
+
+    for data in phase_data:
+        phase_name = data["phase_name"]
+
+        phase = job["dut_job_phases"][phase_data.index(data)]
+
+        assert phase["name"] == phase_name
+        try:
+            datetime.fromisoformat(phase["start_time"])
+            assert True
+        except ValueError:
+            assert False
+
+        if phase_data.index(data) != len(phase_data) - 1:
+            try:
+                datetime.fromisoformat(phase["end_time"])
+                assert True
+            except ValueError:
+                assert False
+
+    # Check if the end_time of the last phase is an empty string
+    last_phase = job["dut_job_phases"][-1]
+    assert last_phase["end_time"] == ""
+
+
+# Test case for creating multiple dut jobs and updating phase job for last dut job
+def test_create_two_dut_jobs_and_add_phase(custom_logger):
+    # Create the first DUT job
+    custom_logger.create_dut_job(status="pass")
+
+    # Create the second DUT job
+    custom_logger.create_dut_job(status="fail")
+
+    logger_data = custom_logger.logger.data
+
+    assert "dut_jobs" in logger_data
+    assert isinstance(logger_data["dut_jobs"], list)
+    assert len(logger_data["dut_jobs"]) == 2
+
+    first_dut_job = logger_data["dut_jobs"][0]
+    second_dut_job = logger_data["dut_jobs"][1]
+
+    # Add a phase to the second DUT job
+    custom_logger.create_job_phase("Phase1")
+
+    logger_data = custom_logger.logger.data
+
+    assert "dut_jobs" in logger_data
+    assert isinstance(logger_data["dut_jobs"], list)
+    assert len(logger_data["dut_jobs"]) == 2
+
+    first_dut_job = logger_data["dut_jobs"][0]
+    second_dut_job = logger_data["dut_jobs"][1]
+
+    # Check first DUT job does not have a phase
+    assert not first_dut_job.get("dut_job_phases")
+
+    # Check second DUT job has a phase
+    assert second_dut_job.get("dut_job_phases")
+    assert isinstance(second_dut_job["dut_job_phases"], list)
+    assert len(second_dut_job["dut_job_phases"]) == 1
+
+
+# Test case for updating DUT start time
+def test_update_dut_start_time(custom_logger):
+    custom_logger.create_dut_job(status="pass")
+    custom_logger.update_dut_time("start", None)
+
+    logger_data = custom_logger.logger.data
+    assert "dut_jobs" in logger_data
+    assert len(logger_data["dut_jobs"]) == 1
+
+    dut_job = logger_data["dut_jobs"][0]
+    assert "dut_start_time" in dut_job
+    assert dut_job["dut_start_time"] != ""
+
+    try:
+        datetime.fromisoformat(dut_job["dut_start_time"])
+        assert True
+    except ValueError:
+        assert False
+
+
+# Test case for updating DUT submit time
+def test_update_dut_submit_time(custom_logger):
+    custom_time = "2023-11-09T02:37:06Z"
+    custom_logger.create_dut_job(status="pass")
+    custom_logger.update_dut_time("submit", custom_time)
+
+    logger_data = custom_logger.logger.data
+    assert "dut_jobs" in logger_data
+    assert len(logger_data["dut_jobs"]) == 1
+
+    dut_job = logger_data["dut_jobs"][0]
+    assert "dut_submit_time" in dut_job
+
+    try:
+        datetime.fromisoformat(dut_job["dut_submit_time"])
+        assert True
+    except ValueError:
+        assert False
+
+
+# Test case for updating DUT end time
+def test_update_dut_end_time(custom_logger):
+    custom_logger.create_dut_job(status="pass")
+    custom_logger.update_dut_time("end", None)
+
+    logger_data = custom_logger.logger.data
+    assert "dut_jobs" in logger_data
+    assert len(logger_data["dut_jobs"]) == 1
+
+    dut_job = logger_data["dut_jobs"][0]
+    assert "dut_end_time" in dut_job
+
+    try:
+        datetime.fromisoformat(dut_job["dut_end_time"])
+        assert True
+    except ValueError:
+        assert False
+
+
+# Test case for updating DUT time with invalid value
+def test_update_dut_time_invalid_value(custom_logger):
+    custom_logger.create_dut_job(status="pass")
+    with pytest.raises(
+        ValueError,
+        match="Error: Invalid argument provided for --update-dut-time. Use 'start', 'submit', 'end'.",
+    ):
+        custom_logger.update_dut_time("invalid_value", None)
+
+
+# Test case for close_dut_job
+def test_close_dut_job(custom_logger):
+    custom_logger.create_dut_job(status="pass")
+
+    custom_logger.create_job_phase("Phase1")
+    custom_logger.create_job_phase("Phase2")
+
+    custom_logger.close_dut_job()
+
+    logger_data = custom_logger.logger.data
+    assert "dut_jobs" in logger_data
+    assert len(logger_data["dut_jobs"]) == 1
+
+    dut_job = logger_data["dut_jobs"][0]
+    assert "dut_job_phases" in dut_job
+    dut_job_phases = dut_job["dut_job_phases"]
+
+    phase1 = dut_job_phases[0]
+    assert phase1["name"] == "Phase1"
+
+    try:
+        datetime.fromisoformat(phase1["start_time"])
+        assert True
+    except ValueError:
+        assert False
+
+    try:
+        datetime.fromisoformat(phase1["end_time"])
+        assert True
+    except ValueError:
+        assert False
+
+    phase2 = dut_job_phases[1]
+    assert phase2["name"] == "Phase2"
+
+    try:
+        datetime.fromisoformat(phase2["start_time"])
+        assert True
+    except ValueError:
+        assert False
+
+    try:
+        datetime.fromisoformat(phase2["end_time"])
+        assert True
+    except ValueError:
+        assert False
+
+
+# Test case for close
+def test_close(custom_logger):
+    custom_logger.create_dut_job(status="pass")
+
+    custom_logger.close()
+
+    logger_data = custom_logger.logger.data
+    assert "dut_jobs" in logger_data
+    assert len(logger_data["dut_jobs"]) == 1
+    assert "dut_attempt_counter" in logger_data
+    assert logger_data["dut_attempt_counter"] == len(logger_data["dut_jobs"])
+    assert "job_combined_status" in logger_data
+    assert logger_data["job_combined_status"] != ""
+
+    dut_job = logger_data["dut_jobs"][0]
+    assert "submitter_end_time" in dut_job
+    try:
+        datetime.fromisoformat(dut_job["submitter_end_time"])
+        assert True
+    except ValueError:
+        assert False
+
+
+# Test case for updating status to fail with a reason
+def test_update_status_fail_with_reason(custom_logger):
+    custom_logger.create_dut_job()
+
+    reason = "kernel panic"
+    custom_logger.update_status_fail(reason)
+
+    logger_data = custom_logger.logger.data
+    assert "dut_jobs" in logger_data
+    assert len(logger_data["dut_jobs"]) == 1
+
+    dut_job = logger_data["dut_jobs"][0]
+    assert "status" in dut_job
+    assert dut_job["status"] == "fail"
+    assert "dut_job_fail_reason" in dut_job
+    assert dut_job["dut_job_fail_reason"] == reason
+
+
+# Test case for updating status to fail without providing a reason
+def test_update_status_fail_without_reason(custom_logger):
+    custom_logger.create_dut_job()
+
+    custom_logger.update_status_fail()
+
+    # Check if the status is updated and fail reason is empty
+    logger_data = custom_logger.logger.data
+    assert "dut_jobs" in logger_data
+    assert len(logger_data["dut_jobs"]) == 1
+
+    dut_job = logger_data["dut_jobs"][0]
+    assert "status" in dut_job
+    assert dut_job["status"] == "fail"
+    assert "dut_job_fail_reason" in dut_job
+    assert dut_job["dut_job_fail_reason"] == ""
+
+
+# Test case for check_dut_timings with submission time earlier than start time
+def test_check_dut_timings_submission_earlier_than_start(custom_logger, caplog):
+    custom_logger.create_dut_job()
+
+    # Set submission time to be earlier than start time
+    custom_logger.update_dut_time("start", "2023-01-01T11:00:00")
+    custom_logger.update_dut_time("submit", "2023-01-01T12:00:00")
+
+    logger_data = custom_logger.logger.data
+    assert "dut_jobs" in logger_data
+    assert len(logger_data["dut_jobs"]) == 1
+
+    job = logger_data["dut_jobs"][0]
+
+    # Call check_dut_timings
+    custom_logger.check_dut_timings(job)
+
+    # Check if an error message is logged
+    assert "Job submission is happening before job start." in caplog.text
+
+
+# Test case for check_dut_timings with end time earlier than start time
+def test_check_dut_timings_end_earlier_than_start(custom_logger, caplog):
+    custom_logger.create_dut_job()
+
+    # Set end time to be earlier than start time
+    custom_logger.update_dut_time("end", "2023-01-01T11:00:00")
+    custom_logger.update_dut_time("start", "2023-01-01T12:00:00")
+
+    logger_data = custom_logger.logger.data
+    assert "dut_jobs" in logger_data
+    assert len(logger_data["dut_jobs"]) == 1
+
+    job = logger_data["dut_jobs"][0]
+
+    # Call check_dut_timings
+    custom_logger.check_dut_timings(job)
+
+    # Check if an error message is logged
+    assert "Job ended before it started." in caplog.text
+
+
+# Test case for check_dut_timings with valid timing sequence
+def test_check_dut_timings_valid_timing_sequence(custom_logger, caplog):
+    custom_logger.create_dut_job()
+
+    # Set valid timing sequence
+    custom_logger.update_dut_time("submit", "2023-01-01T12:00:00")
+    custom_logger.update_dut_time("start", "2023-01-01T12:30:00")
+    custom_logger.update_dut_time("end", "2023-01-01T13:00:00")
+
+    logger_data = custom_logger.logger.data
+    assert "dut_jobs" in logger_data
+    assert len(logger_data["dut_jobs"]) == 1
+
+    job = logger_data["dut_jobs"][0]
+
+    # Call check_dut_timings
+    custom_logger.check_dut_timings(job)
+
+    # Check that no error messages are logged
+    assert "Job submission is happening before job start." not in caplog.text
+    assert "Job ended before it started." not in caplog.text
+
+
+@pytest.mark.parametrize("start_time,end_time,expected", [
+    ("2023-01-01T12:00:00", "2023-01-01T12:30:00", "0:30:00"),
+    ("invalid", "2023-01-01T12:30:00", ""),
+    ("", "", ""),
+])
+def test_get_duration_time(custom_logger, start_time, end_time, expected):
+    duration = custom_logger.get_duration_time(start_time, end_time)
+    assert duration == expected
+
+
+# Test case for DUT job duration tracking
+def test_dut_job_duration(custom_logger):
+    custom_logger.create_dut_job(status="pass")
+
+    # Set start time
+    custom_logger.update_dut_time("start", "2023-01-01T12:00:00")
+    logger_data = custom_logger.logger.data
+    dut_job = logger_data["dut_jobs"][0]
+    assert dut_job["dut_duration_time"] == ""
+
+    # Set end time
+    custom_logger.update_dut_time("end", "2023-01-01T12:30:00")
+    logger_data = custom_logger.logger.data
+    dut_job = logger_data["dut_jobs"][0]
+    assert dut_job["dut_duration_time"] == "0:30:00"
+
+
+# Test case for job phase duration tracking
+def test_job_phase_duration(custom_logger):
+    custom_logger.create_dut_job(status="pass")
+
+    # Create first phase
+    custom_logger.create_job_phase("Phase1")
+    logger_data = custom_logger.logger.data
+    job = logger_data["dut_jobs"][0]
+    phase = job["dut_job_phases"][0]
+    assert phase["duration_time"] == ""
+
+    # Create second phase (which should close the first phase)
+    custom_logger.create_job_phase("Phase2")
+    logger_data = custom_logger.logger.data
+    job = logger_data["dut_jobs"][0]
+
+    # First phase should have duration
+    phase1 = job["dut_job_phases"][0]
+    assert phase1["duration_time"] != ""
+
+    # Verify duration format
+    start = datetime.fromisoformat(phase1["start_time"])
+    end = datetime.fromisoformat(phase1["end_time"])
+    duration = end - start
+    assert phase1["duration_time"] == str(duration)
+
+    # Second phase should have empty duration
+    phase2 = job["dut_job_phases"][1]
+    assert phase2["duration_time"] == ""
+
+
+# Test case for closing job with phases
+def test_close_job_with_phases_duration(custom_logger):
+    custom_logger.create_dut_job(status="pass")
+    custom_logger.create_job_phase("Phase1")
+    custom_logger.close_dut_job()
+
+    logger_data = custom_logger.logger.data
+    job = logger_data["dut_jobs"][0]
+    phase = job["dut_job_phases"][0]
+
+    assert phase["duration_time"] != ""
+
+    # Verify duration format
+    start = datetime.fromisoformat(phase["start_time"])
+    end = datetime.fromisoformat(phase["end_time"])
+    duration = end - start
+    assert phase["duration_time"] == str(duration)

bin/ci/test/test_gantt_chart.py

@@ -0,0 +1,201 @@
+from contextlib import suppress
+from datetime import datetime, timedelta
+from unittest import mock
+from unittest.mock import MagicMock, patch
+
+import ci_post_gantt
+import pytest
+from ci_gantt_chart import generate_gantt_chart
+from ci_post_gantt import Gitlab, MockGanttExit
+
+
+def create_mock_job(
+    name, id, status, created_at, queued_duration, started_at, finished_at=None
+):
+    mock_job = MagicMock()
+    mock_job.name = name
+    mock_job.status = status
+    mock_job.id = id
+    mock_job.created_at = created_at
+    mock_job.queued_duration = queued_duration
+    mock_job.started_at = started_at
+    mock_job.finished_at = finished_at
+    return mock_job
+
+
+@pytest.fixture
+def fake_pipeline():
+    current_time = datetime.fromisoformat("2024-12-17 23:54:13.940091+00:00")
+    created_at = current_time - timedelta(minutes=10)
+
+    job1 = create_mock_job(
+        name="job1",
+        id="1",
+        status="success",
+        created_at=created_at.isoformat(),
+        queued_duration=1,  # seconds
+        started_at=(created_at + timedelta(seconds=2)).isoformat(),
+        finished_at=(created_at + timedelta(minutes=1)).isoformat(),
+    )
+
+    mock_pipeline = MagicMock()
+    mock_pipeline.web_url = "https://gitlab.freedesktop.org/mesa/mesa/-/pipelines/9999"
+    mock_pipeline.duration = 600  # Total pipeline duration in seconds
+    mock_pipeline.created_at = created_at.isoformat()
+    mock_pipeline.yaml_errors = False
+    mock_pipeline.jobs.list.return_value = [job1]
+    return mock_pipeline
+
+
+def test_generate_gantt_chart(fake_pipeline):
+    fig = generate_gantt_chart(fake_pipeline)
+
+    fig_dict = fig.to_dict()
+    assert "data" in fig_dict
+
+    # Extract all job names from the "y" axis in the Gantt chart data
+    all_job_names = set()
+    for trace in fig_dict["data"]:
+        if "y" in trace:
+            all_job_names.update(trace["y"])
+
+    assert any(
+        "job1" in job for job in all_job_names
+    ), "job1 should be present in the Gantt chart"
+
+
+def test_ci_timeout(fake_pipeline):
+    fig = generate_gantt_chart(fake_pipeline, ci_timeout=1)
+
+    fig_dict = fig.to_dict()
+
+    timeout_line = None
+    for shape in fig_dict.get("layout", {}).get("shapes", []):
+        if shape.get("line", {}).get("dash") == "dash":
+            timeout_line = shape
+            break
+
+    assert timeout_line is not None, "Timeout line should exist in the Gantt chart"
+    timeout_x = timeout_line["x0"]
+
+    # Check that the timeout line is 1 minute after the pipeline creation time
+    pipeline_created_at = datetime.fromisoformat(fake_pipeline.created_at)
+    expected_timeout = pipeline_created_at + timedelta(minutes=1)
+    assert (
+        timeout_x == expected_timeout
+    ), f"Timeout should be at {expected_timeout}, got {timeout_x}"
+
+
+def test_marge_bot_user_id():
+    with patch("ci_post_gantt.Gitlab") as MockGitlab:
+        mock_gitlab_instance = MagicMock(spec=Gitlab)
+        mock_gitlab_instance.users = MagicMock()
+        MockGitlab.return_value = mock_gitlab_instance
+
+        marge_bot_user_id = 12345
+        ci_post_gantt.main("fake_token", None, marge_bot_user_id)
+        mock_gitlab_instance.users.get.assert_called_once_with(marge_bot_user_id)
+
+
+def test_project_ids():
+    current_time = datetime.now()
+    project_id_1 = 176
+    event_1 = MagicMock()
+    event_1.project_id = project_id_1
+    event_1.created_at = (current_time - timedelta(days=1)).isoformat()
+    event_1.note = {"body": f"Event for project {project_id_1}"}
+
+    project_id_2 = 166
+    event_2 = MagicMock()
+    event_2.project_id = project_id_2
+    event_2.created_at = (current_time - timedelta(days=2)).isoformat()
+    event_2.note = {"body": f"Event for project {project_id_2}"}
+
+    with patch("ci_post_gantt.Gitlab") as MockGitlab:
+        mock_user = MagicMock()
+        mock_user.events = MagicMock()
+        mock_user.events.list.return_value = [event_1, event_2]
+
+        mock_gitlab_instance = MagicMock(spec=Gitlab)
+        mock_gitlab_instance.users = MagicMock()
+        mock_gitlab_instance.users.get.return_value = mock_user
+        MockGitlab.return_value = mock_gitlab_instance
+
+        last_event_date = (current_time - timedelta(days=3)).isoformat()
+
+        # Test a single project id
+        ci_post_gantt.main("fake_token", last_event_date)
+        marge_bot_single_project_scope = [
+            event.note["body"]
+            for event in mock_user.events.list.return_value
+            if event.project_id == project_id_1
+        ]
+        assert f"Event for project {project_id_1}" in marge_bot_single_project_scope
+        assert f"Event for project {project_id_2}" not in marge_bot_single_project_scope
+
+        # Test multiple project ids
+        ci_post_gantt.main(
+            "fake_token", last_event_date, 9716, [project_id_1, project_id_2]
+        )
+
+        marge_bot_multiple_project_scope = [
+            event.note["body"] for event in mock_user.events.list.return_value
+        ]
+        assert f"Event for project {project_id_1}" in marge_bot_multiple_project_scope
+        assert f"Event for project {project_id_2}" in marge_bot_multiple_project_scope
+
+
+def test_add_gantt_after_pipeline_message():
+    current_time = datetime.now()
+
+    plain_url = "https://gitlab.freedesktop.org/mesa/mesa/-/pipelines/12345"
+    plain_message = (
+        f"I couldn't merge this branch: CI failed! See pipeline {plain_url}."
+    )
+    event_plain = MagicMock()
+    event_plain.project_id = 176
+    event_plain.created_at = (current_time - timedelta(days=1)).isoformat()
+    event_plain.note = {"body": plain_message}
+
+    summary_url = "https://gitlab.freedesktop.org/mesa/mesa/-/pipelines/99999"
+    summary_message = (
+        "I couldn't merge this branch: "
+        f"CI failed! See pipeline {summary_url}.<br>There were problems with job:"
+        "[lavapipe](https://gitlab.freedesktop.org/mesa/mesa/-/jobs/68141218)<details><summary> "
+        "3 crashed tests</summary>dEQP-VK.ray_query.builtin.instancecustomindex.frag.aabbs,Crash<br>dEQP"
+        "-VK.ray_query.builtin.objecttoworld.frag.aabbs,Crash<br>dEQP-VK.sparse_resources.shader_intrinsics."
+        "2d_array_sparse_fetch.g16_b16r16_2plane_444_unorm.11_37_3_nontemporal,Crash<br></details>"
+    )
+    event_with_summary = MagicMock()
+    event_with_summary.project_id = 176
+    event_with_summary.created_at = (current_time - timedelta(days=1)).isoformat()
+    event_with_summary.note = {"body": summary_message}
+
+    with patch("ci_post_gantt.Gitlab") as MockGitlab, patch(
+        "ci_post_gantt.get_gitlab_pipeline_from_url", return_value=None
+    ) as mock_get_gitlab_pipeline_from_url:
+
+        def safe_mock(*args, **kwargs):
+            with suppress(TypeError):
+                raise MockGanttExit("Exiting for test purposes")
+
+        mock_get_gitlab_pipeline_from_url.side_effect = safe_mock
+
+        mock_user = MagicMock()
+        mock_user.events = MagicMock()
+        mock_user.events.list.return_value = [event_plain, event_with_summary]
+
+        mock_gitlab_instance = MagicMock(spec=Gitlab)
+        mock_gitlab_instance.users = MagicMock()
+        mock_gitlab_instance.users.get.return_value = mock_user
+        MockGitlab.return_value = mock_gitlab_instance
+
+        last_event_date = (current_time - timedelta(days=3)).isoformat()
+        ci_post_gantt.main("fake_token", last_event_date, 12345)
+        mock_get_gitlab_pipeline_from_url.assert_has_calls(
+            [
+                mock.call(mock_gitlab_instance, plain_url),
+                mock.call(mock_gitlab_instance, summary_url),
+            ],
+            any_order=True,
+        )

bin/ci/test/test_marge_queue.py

@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+# Copyright © 2025 Collabora Ltd.
+# Authors:
+#   Sergi Blanch Torne <[email protected]>
+#
+# SPDX-License-Identifier: MIT
+
+from dataclasses import dataclass
+from datetime import timedelta
+
+from unittest import TestCase
+from typing import Iterable
+
+from marge_queue import MargeMergeRequest, MargeQueue, get_merge_queue
+
+
+DATA = {
+    37564: {
+        # sample of a Merge Request rejected by marge and later reassigned to marge
+        "updated_at": "2025-09-25T10:59:27.144Z",
+        "title": "etnaviv: Fix util_blitter_save_so_targets(..) call",
+        "notes": [
+            # Those notes must be listed in reverse order
+            # because the list is requested sort="desc"
+            {
+                "body": "assigned to @marge-bot and unassigned @austriancoder",
+                "created_at": "2025-09-25T11:44:01.672Z",
+            },
+            {
+                "body": "added 1 commit",
+                "created_at": "2025-09-25T10:43:43.490Z",
+            },
+            {
+                "body": "This branch couldn't be merged:",
+                "created_at": "2025-09-25T10:42:27.817Z",
+            },
+            {
+                "body": "assigned to @austriancoder and unassigned @marge-bot",
+                "created_at": "2025-09-25T10:42:15.404Z",
+            },
+            {
+                "body": "added 3 commits",
+                "created_at": "2025-09-25T10:38:57.332Z",
+            },
+            {
+                "body": "assigned to @marge-bot",
+                "created_at": "2025-09-25T10:38:03.993Z",
+            },
+        ],
+    },
+    37560: {
+        # sample of a Merge Request assigned to marge between the two
+        # assignments of the previous sample. This is meant to show that
+        # this goes first in the queue, and it doesn't confuses the sequence.
+        "updated_at": "2025-09-25T11:38:08.420Z",
+        "title": "tu: limit query pool types logged into RMV",
+        "notes": [
+            {
+                "body": "assigned to @marge-bot",
+                "created_at": "2025-09-25T10:47:40.040Z",
+            },
+            {
+                "body": "added 1 commit",
+                "created_at": "2025-09-25T10:46:42.113Z",
+            },
+            {
+                "body": "mentioned in issue #13970",
+                "created_at": "2025-09-25T10:45:35.566Z",
+            },
+            {
+                "body": "Rb",
+                "created_at": "2025-09-25T08:25:51.527Z",
+            },
+        ],
+    },
+}
+
+
+@dataclass
+class ProjectMergeRequestNote:
+    body: str
+    created_at: str
+
+
+@dataclass
+class ProjectMergeRequestNoteManager:
+    """
+    Mock the gitlab.v4.objects.ProjectMergeRequestNoteManager
+    """
+    iid: int
+
+    def list(self, *args, **kwargs) -> Iterable:
+        for note in DATA[self.iid]["notes"]:
+            yield ProjectMergeRequestNote(**note)
+
+
+@dataclass
+class ProjectMergeRequest:
+    """
+    Mock the gitlab.v4.objects.ProjectMergeRequest
+    """
+    iid: int
+
+    @property
+    def updated_at(self) -> str:
+        return DATA[self.iid]["updated_at"]
+
+    @property
+    def web_url(self) -> str:
+        return f"https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/{self.iid}"
+
+    @property
+    def title(self) -> str:
+        return DATA[self.iid]["title"]
+
+    @property
+    def notes(self) -> ProjectMergeRequestNoteManager:
+        return ProjectMergeRequestNoteManager(self.iid)
+
+
+@dataclass
+class ProjectMergeRequestManager:
+    """
+    Mock the gitlab.v4.objects.ProjectMergeRequestManager
+    """
+    def list(self, *args, **kwargs) -> Iterable:
+        for mr in DATA.keys():
+            yield ProjectMergeRequest(mr)
+
+
+@dataclass
+class Project:
+    """
+    Mock the gitlab.v4.objects.Project
+    """
+    @property
+    def mergerequests(self) -> ProjectMergeRequestManager:
+        return ProjectMergeRequestManager()
+
+
+class MargeMergeTestCase(TestCase):
+    def test_queue(self) -> None:
+        queue = MargeQueue()
+        for mr_id in DATA.keys():
+            gl_mr_mock_obj = ProjectMergeRequest(mr_id)
+            marge_mr = MargeMergeRequest(gl_mr_mock_obj)
+
+            # for each element in the data sample, test the MargeMergeRequest construction
+            self.assertEqual(marge_mr.title, gl_mr_mock_obj.title)
+            self.assertEqual(marge_mr.web_url, gl_mr_mock_obj.web_url)
+            self.assertIsInstance(marge_mr.time_enqueued, timedelta)
+
+            queue.append(marge_mr)
+
+        self.assertEqual(queue.n_merge_requests_enqueued, 2)
+
+
+    def test_get_merge_queue(self) -> None:
+        project = Project()
+        queue = get_merge_queue(project)
+
+        self.assertEqual(queue.n_merge_requests_enqueued, 2)
+        self.assertEqual(len(queue.sorted_queue), 2)
+        # They are in reverse order because of the reassignment to marge
+        self.assertEqual(queue.sorted_queue[0].id, list(DATA.keys())[1])
+        self.assertEqual(queue.sorted_queue[1].id, list(DATA.keys())[0])
+        self.assertEqual(len(queue.undetermined), 0)
+
+        for marge_mr in queue.sorted_queue:
+            mr_id = marge_mr.id
+            self.assertEqual(marge_mr.title, DATA[mr_id]["title"])
+            self.assertEqual(
+                marge_mr.web_url,
+                f"https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/{mr_id}"
+            )
+            self.assertIsInstance(marge_mr.time_enqueued, timedelta)

bin/ci/test/test_pipeline_message.py

@@ -0,0 +1,309 @@
+import logging
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from pipeline_message import (
+    get_failed_test_summary_message,
+    get_problem_jobs,
+    get_trace_failures,
+    main,
+    process_problem_jobs,
+    search_job_log_for_errors,
+    sort_failed_tests_by_status,
+    unexpected_improvements,
+)
+
+
+def test_get_problem_jobs():
+    jobs = [
+        {"stage": "build", "status": "failed"},
+        {"stage": "test", "status": "canceled"},
+        {"stage": "postmerge", "status": "failed"},
+        {"stage": "performance", "status": "failed"},
+        {"stage": "deploy", "status": "failed"},
+    ]
+
+    problem_jobs = get_problem_jobs(jobs)
+
+    assert len(problem_jobs) == 3
+    assert problem_jobs[0]["stage"] == "build"
+    assert problem_jobs[1]["stage"] == "test"
+    assert problem_jobs[2]["stage"] == "deploy"
+
+
+def test_sort_failed_tests_by_status():
+    failures_csv = """\
+Test1,UnexpectedImprovement
+Test2,Fail
+Test3,Crash
+Test4,Timeout
+Test5,Fail
+Test6,UnexpectedImprovement
+"""
+    sorted_tests = sort_failed_tests_by_status(failures_csv)
+
+    assert len(sorted_tests["unexpected_improvements"]) == 2
+    assert len(sorted_tests["fails"]) == 2
+    assert len(sorted_tests["crashes"]) == 1
+    assert len(sorted_tests["timeouts"]) == 1
+
+    assert sorted_tests["unexpected_improvements"] == [
+        "Test1,UnexpectedImprovement",
+        "Test6,UnexpectedImprovement",
+    ]
+    assert sorted_tests["fails"] == ["Test2,Fail", "Test5,Fail"]
+    assert sorted_tests["crashes"] == ["Test3,Crash"]
+    assert sorted_tests["timeouts"] == ["Test4,Timeout"]
+
+
+def test_get_failed_test_summary_message():
+    failed_test_array = {
+        "unexpected_improvements": [
+            "test1 UnexpectedImprovement",
+            "test2 UnexpectedImprovement",
+        ],
+        "fails": ["test3 Fail", "test4 Fail", "test5 Fail"],
+        "crashes": ["test6 Crash"],
+        "timeouts": [],
+    }
+
+    summary_message = get_failed_test_summary_message(failed_test_array)
+
+    assert "<summary>" in summary_message
+    assert "2 improved tests" in summary_message
+    assert "3 failed tests" in summary_message
+    assert "1 crashed test" in summary_message
+    assert "</summary>" in summary_message
+
+
+def test_unexpected_improvements():
+    message = "<summary>"
+    failed_test_array = {
+        "unexpected_improvements": ["test_improvement_1", "test_improvement_2"],
+        "fails": [],
+        "crashes": [],
+        "timeouts": [],
+    }
+    result = unexpected_improvements(failed_test_array)
+    assert result == " 2 improved tests", f"Unexpected result: {result}"
+
+
+@pytest.mark.asyncio
+@patch("pipeline_message.get_pipeline_status", new_callable=AsyncMock)
+async def test_gitlab_api_failure(mock_get_pipeline_status):
+    mock_get_pipeline_status.side_effect = Exception("GitLab API not responding")
+    message = await main("1234567")
+    assert message == ""
+
+
+@pytest.mark.asyncio
+async def test_no_message_when_pipeline_not_failed():
+    project_id = "176"
+    pipeline_id = "12345"
+
+    with patch(
+        "pipeline_message.get_pipeline_status", new_callable=AsyncMock
+    ) as mock_get_pipeline_status:
+        mock_get_pipeline_status.return_value = "success"
+
+        message = await main(pipeline_id, project_id)
+        assert (
+            message == ""
+        ), f"Expected no message for successful pipeline, but got: {message}"
+
+
+@pytest.mark.asyncio
+async def test_single_problem_job_not_summarized():
+    session = AsyncMock()
+    project_id = "176"
+    problem_jobs = [
+        {
+            "id": 1234,
+            "name": "test-job",
+            "web_url": "http://example.com/job/1234",
+            "status": "canceled",
+        }
+    ]
+
+    mock_response = AsyncMock()
+    mock_response.status = 200
+    mock_response.text.return_value = ""  # Empty CSV response for test
+    session.get.return_value = mock_response
+
+    message = await process_problem_jobs(session, project_id, problem_jobs)
+
+    assert "summary" not in message
+    assert "[test-job](http://example.com/job/1234)" in message
+
+
+@pytest.mark.asyncio
+@patch("pipeline_message.get_project_json", new_callable=AsyncMock)
+@patch("pipeline_message.aiohttp.ClientSession", autospec=True)
+async def test_get_trace_failures_no_response(
+    mock_client_session_cls, mock_get_project_json, caplog
+):
+    caplog.set_level(logging.DEBUG)
+    namespace = "mesa"
+    mock_get_project_json.return_value = {"path": namespace}
+
+    mock_get = AsyncMock()
+    mock_get.status = 404
+
+    mock_session_instance = mock_client_session_cls.return_value
+    mock_session_instance.get.return_value = mock_get
+
+    job_id = 12345678
+    job = {"id": job_id}
+    url = await get_trace_failures(mock_session_instance, "176", job)
+
+    assert url == ""
+
+    expected_log_message = f"No response from: https://mesa.pages.freedesktop.org/-/{namespace}/-/jobs/{job_id}/artifacts/results/summary/problems.html"
+    assert any(expected_log_message in record.message for record in caplog.records)
+
+
+@pytest.mark.asyncio
+@patch("pipeline_message.get_job_log", new_callable=AsyncMock)
+async def test_search_job_log_for_errors(mock_get_job_log):
+    session = AsyncMock()
+    project_id = "176"
+    job = {"id": 12345}
+
+    job_log = r"""
+error_msg: something useful
+[0m15:41:36.102:                GL_KHR_no_error GL_KHR_texture_compression_astc_sliced_3d
+1 error generated
+3 errors generated.
+-- Looking for strerror_r - found
+-- Looking for strerror_s - not found
+[49/176] Building CXX object lib/Support/CMakeFiles/LLVMSupport.dir/ErrorHandling.cpp.o
+[127/2034] Building C object lib/Support/CMakeFiles/LLVMSupport.dir/regerror.c.o
+-- Performing Test HAS_WERROR_GLOBAL_CTORS
+-- Performing Test C_SUPPORTS_WERROR_UNGUARDED_AVAILABILITY_NEW - Success
+-- Performing Test LLVM_LIBSTDCXX_SOFT_ERROR
+error aborting
+error_msg      : None
+error_type     : Job
+[0Ksection_end:1734694783:job_data
+[0K
+[0m11:39:43.438: [1mFinished executing LAVA job in the attempt #3 [0m
+[0Ksection_end:1734694783:lava_submit
+[0K
+[0;31m[01:54] ERROR: lava_submit: ret code: 1 [0m
+
+[0;31m[01:54] ERROR: unknown-section: ret code: 1 [0m
+section_end:1734694783:step_script
+[0Ksection_start:1734694783:after_script
+[0K[0K[36;1mRunning after_script[0;m[0;m
+[32;1mRunning after script...[0;m
+[32;1m$ curl -L --retry 4 -f --retry-all-errors --retry-delay 60 -s "https://" | tar --warning=no-timestamp --zstd -x[0;m
+zstd: /*stdin*\: unexpected end of file # noqa: W605
+tar: Child returned status 1
+tar: Error is not recoverable: exiting now
+section_end:1734695025:after_script
+[0K[0;33mWARNING: after_script failed, but job will continue unaffected: exit code 1[0;m
+section_start:1734695025:upload_artifacts_on_failure
+[0K[0K[36;1mUploading artifacts for failed job[0;m[0;m
+[32;1mUploading artifacts...[0;m
+results/: found 11 matching artifact files and directories[0;m
+Uploading artifacts as "archive" to coordinator... 201 Created[0;m  id[0;m=68509685 responseStatus[0;m=201 Created token[0;m=glcbt-64
+[32;1mUploading artifacts...[0;m
+[0;33mWARNING: results/junit.xml: no matching files. Ensure that the artifact path is relative to the working directory (/builds/mesa/mesa)[0;m
+[31;1mERROR: No files to upload                         [0;m
+section_end:1734695027:upload_artifacts_on_failure
+[0Ksection_start:1734695027:cleanup_file_variables
+[0K[0K[36;1mCleaning up project directory and file based variables[0;m[0;m
+section_end:1734695027:cleanup_file_variables
+[0K[31;1mERROR: Job failed: exit code 1
+[0;m
+[0;m
+    """
+
+    mock_get_job_log.return_value = job_log
+
+    error_message = await search_job_log_for_errors(session, project_id, job)
+    assert "something useful" in error_message
+
+
+@pytest.mark.asyncio
+@patch("pipeline_message.get_job_log", new_callable=AsyncMock)
+async def test_search_job_log_for_fatal_errors(mock_get_job_log):
+    session = AsyncMock()
+    project_id = "176"
+    job = {"id": 12345}
+
+    job_log = r"""
+[0m15:41:36.105: [15:41:31.951] fatal: something fatal
+Uploading artifacts as "archive" to coordinator... 201 Created[0;m  id[0;m=68509685 responseStatus[0;m=201 Created token[0;m=glcbt-64
+[32;1mUploading artifacts...[0;m
+[0;33mWARNING: results/junit.xml: no matching files. Ensure that the artifact path is relative to the working directory (/builds/mesa/mesa)[0;m
+[31;1mERROR: No files to upload                         [0;m
+section_end:1734695027:upload_artifacts_on_failure
+[0Ksection_start:1734695027:cleanup_file_variables
+[0K[0K[36;1mCleaning up project directory and file based variables[0;m[0;m
+section_end:1734695027:cleanup_file_variables
+[0K[31;1mERROR: Job failed: exit code 1
+[0;m
+[0;m
+    """
+
+    mock_get_job_log.return_value = job_log
+
+    error_message = await search_job_log_for_errors(session, project_id, job)
+    assert "something fatal" in error_message
+
+
+@pytest.mark.asyncio
+@patch("pipeline_message.get_job_log", new_callable=AsyncMock)
+async def test_search_job_log_for_errors_but_find_none(mock_get_job_log):
+    session = AsyncMock()
+    project_id = "176"
+    job = {"id": 12345}
+
+    job_log = r"""
+[0KRunning with gitlab-runner 17.4.0 (b92ee590)[0;m
+[0K  on fdo-equinix-m3l-30-placeholder_63 XmDXAt7xd, system ID: s_785ae19292ea[0;m
+section_start:1734736110:prepare_executor
+[0K[0K[36;1mPreparing the "docker" executor[0;m[0;m
+[0KUsing Docker executor with image registry.freedesktop.org/mesa/mesa/debian
+[0KAuthenticating with credentials from job payload (GitLab Registry)[0;m
+[0KPulling docker image registry.freedesktop.org/mesa/mesa/debian/x86_64_pyuti
+[0KUsing docker image sha256:ebc7b3fe89be4d390775303adddb33539c235a2663165d78d
+[0Ksection_start:1734736124:prepare_script
+[0K[0K[36;1mPreparing environment[0;m[0;m
+Running on runner-xmdxat7xd-project-23076-concurrent-1 via fdo-equinix-m3l-30...
+section_end:1734736125:prepare_script
+[0Ksection_start:1734736125:get_sources
+[0K[0K[36;1mGetting source from Git repository[0;m[0;m
+[32;1m$ /host/bin/curl -s -L --cacert /host/ca-certificates.crt --retry 4 -f --retry-delay 60 https://gitlab.
+Checking if the user of the pipeline is allowed...
+Checking if the job's project is part of a well-known group...
+Checking if the job is part of an official MR pipeline...
+Thank you for contributing to freedesktop.org
+Running pre-clone script: 'set -o xtrace
+wget -q -O download-git-cache.sh https://gitlab.freedesktop.org/mesa/mesa/-/raw/0d43b4cba639b809ad0e08a065ce01846e262249/.gitlab-ci/download-git-cache.sh
+bash download-git-cache.sh
+rm download-git-cache.sh
+[31;1m errors
+[0K[31;1mERROR:
+[31;1m error
+[31;1m Here is a blank error:
+/builds/mesa/mesa/bin/ci/test/test_pipeline_message.py:162: AssertionError
+Uploading artifacts as "archive" to coordinator... 201 Created[0;m  id[0;m=68509685 responseStatus[0;m=201 Created token[0;m=glcbt-64
+[32;1mUploading artifacts...[0;m
+[0;33mWARNING: results/junit.xml: no matching files. Ensure that the artifact path is relative to the working directory (/builds/mesa/mesa)[0;m
+[31;1mERROR: No files to upload                         [0;m
+section_end:1734695027:upload_artifacts_on_failure
+[0Ksection_start:1734695027:cleanup_file_variables
+[0K[0K[36;1mCleaning up project directory and file based variables[0;m[0;m
+section_end:1734695027:cleanup_file_variables
+[0K[31;1mERROR: Job failed: exit code 1
+[0;m
+[0;m
+    """
+
+    mock_get_job_log.return_value = job_log
+
+    error_message = await search_job_log_for_errors(session, project_id, job)
+    assert error_message == "", f"Unexpected error message: {error_message}"

bin/ci/test/test_structured_logger.py

@@ -0,0 +1,182 @@
+import json
+from pathlib import Path
+
+import pytest
+from mock import MagicMock, patch
+from structured_logger import (
+    AutoSaveDict,
+    CSVStrategy,
+    JSONStrategy,
+    StructuredLogger,
+    YAMLStrategy,
+)
+
+
+@pytest.fixture(params=[CSVStrategy, JSONStrategy, YAMLStrategy])
+def strategy(request):
+    return request.param
+
+
+@pytest.fixture
+def file_extension(strategy):
+    if strategy == CSVStrategy:
+        return "csv"
+    elif strategy == JSONStrategy:
+        return "json"
+    elif strategy == YAMLStrategy:
+        return "yaml"
+
+
+@pytest.fixture
+def tmp_file(tmp_path):
+    return tmp_path / "test.json"
+
+
+def test_guess_strategy_from_file(tmp_path, strategy, file_extension):
+    file_name = tmp_path / f"test_guess.{file_extension}"
+    Path(file_name).touch()
+    guessed_strategy = StructuredLogger.guess_strategy_from_file(file_name)
+    assert isinstance(guessed_strategy, strategy)
+
+
+def test_get_strategy(strategy, file_extension):
+    result = StructuredLogger.get_strategy(file_extension)
+    assert isinstance(result, strategy)
+
+
+def test_invalid_file_extension(tmp_path):
+    file_name = tmp_path / "test_invalid.xyz"
+    Path(file_name).touch()
+
+    with pytest.raises(ValueError, match="Unknown strategy for: xyz"):
+        StructuredLogger.guess_strategy_from_file(file_name)
+
+
+def test_non_existent_file(tmp_path, strategy, file_extension):
+    file_name = tmp_path / f"non_existent.{file_extension}"
+    logger = StructuredLogger(file_name, strategy())
+
+    assert logger.file_path.exists()
+    assert "_timestamp" in logger._data
+
+
+@pytest.fixture
+def structured_logger_module():
+    with patch.dict("sys.modules", {"polars": None, "ruamel.yaml": None}):
+        import importlib
+
+        import structured_logger
+
+        importlib.reload(structured_logger)
+        yield structured_logger
+
+
+def test_missing_csv_library(tmp_path, structured_logger_module):
+    with pytest.raises(RuntimeError, match="Can't parse CSV files. Missing library"):
+        structured_logger_module.CSVStrategy()
+
+
+def test_missing_yaml_library(tmp_path, structured_logger_module):
+    with pytest.raises(RuntimeError, match="Can't parse YAML files. Missing library"):
+        structured_logger_module.YAMLStrategy()
+
+
+def test_autosavedict_setitem():
+    save_callback = MagicMock()
+    d = AutoSaveDict(save_callback=save_callback)
+    d["key"] = "value"
+    assert d["key"] == "value"
+    save_callback.assert_called_once()
+
+
+def test_autosavedict_delitem():
+    save_callback = MagicMock()
+    d = AutoSaveDict({"key": "value"}, save_callback=save_callback)
+    del d["key"]
+    assert "key" not in d
+    save_callback.assert_called_once()
+
+
+def test_autosavedict_pop():
+    save_callback = MagicMock()
+    d = AutoSaveDict({"key": "value"}, save_callback=save_callback)
+    result = d.pop("key")
+    assert result == "value"
+    assert "key" not in d
+    save_callback.assert_called_once()
+
+
+def test_autosavedict_update():
+    save_callback = MagicMock()
+    d = AutoSaveDict({"key": "old_value"}, save_callback=save_callback)
+    d.update({"key": "new_value"})
+    assert d["key"] == "new_value"
+    save_callback.assert_called_once()
+
+
+def test_structured_logger_setitem(tmp_file):
+    logger = StructuredLogger(tmp_file, JSONStrategy())
+    logger.data["field"] = "value"
+
+    with open(tmp_file, "r") as f:
+        data = json.load(f)
+
+    assert data["field"] == "value"
+
+
+def test_structured_logger_set_recursive(tmp_file):
+    logger = StructuredLogger(tmp_file, JSONStrategy())
+    logger.data["field"] = {"test": True}
+    other = logger.data["field"]
+    other["late"] = True
+
+    with open(tmp_file, "r") as f:
+        data = json.load(f)
+
+    assert data["field"]["test"]
+    assert data["field"]["late"]
+
+
+def test_structured_logger_set_list(tmp_file):
+    logger = StructuredLogger(tmp_file, JSONStrategy())
+    logger.data["field"] = [True]
+    other = logger.data["field"]
+    other.append(True)
+
+    with open(tmp_file, "r") as f:
+        data = json.load(f)
+
+    assert data["field"][0]
+    assert data["field"][1]
+
+
+def test_structured_logger_delitem(tmp_file):
+    logger = StructuredLogger(tmp_file, JSONStrategy())
+    logger.data["field"] = "value"
+    del logger.data["field"]
+
+    with open(tmp_file, "r") as f:
+        data = json.load(f)
+
+    assert "field" not in data
+
+
+def test_structured_logger_pop(tmp_file):
+    logger = StructuredLogger(tmp_file, JSONStrategy())
+    logger.data["field"] = "value"
+    logger.data.pop("field")
+
+    with open(tmp_file, "r") as f:
+        data = json.load(f)
+
+    assert "field" not in data
+
+
+def test_structured_logger_update(tmp_file):
+    logger = StructuredLogger(tmp_file, JSONStrategy())
+    logger.data.update({"field": "value"})
+
+    with open(tmp_file, "r") as f:
+        data = json.load(f)
+
+    assert data["field"] == "value"

bin/ci/test/test_update_tag.py

@@ -0,0 +1,391 @@
+import sys
+import subprocess
+import yaml
+import pytest
+
+from textwrap import dedent
+from unittest.mock import patch
+
+from bin.ci.update_tag import (
+    from_component_to_build_tag,
+    filter_components,
+    from_component_to_tag_var,
+    from_script_name_to_component,
+    from_script_name_to_tag_var,
+    load_image_tags_yaml,
+    update_image_tag_in_yaml,
+    run_build_script,
+    main,
+    PATHS,
+)
+
+
+@pytest.fixture
+def temp_image_tags_file(tmp_path, monkeypatch):
+    """
+    Fixture that creates a temporary YAML file (to simulate conditional_tags)
+    and updates the global variable accordingly.
+    """
+    # Create a temporary test file
+    temp_file = tmp_path / "conditional-build-image-tags.yml"
+    temp_file.write_text(yaml.dump({"variables": {}}))
+
+    # Create a copy of the original PATHS values
+    original_conditional_tags = PATHS.conditional_tags
+
+    # Patch the PATHS instance with our test file
+    PATHS.conditional_tags = temp_file
+
+    # Yield the test file for the test to use
+    yield temp_file
+
+    # Restore the original value after the test
+    PATHS.conditional_tags = original_conditional_tags
+
+
+@pytest.fixture
+def temp_container_ci_file(tmp_path, monkeypatch):
+    """
+    Fixture that creates a temporary container CI file (to simulate container_ci)
+    and updates the global variable accordingly.
+    """
+    # Create a temporary test file
+    temp_file = tmp_path / "container-ci.yml"
+    temp_file.touch()
+
+    # Create a copy of the original PATHS values
+    original_container_ci = PATHS.container_ci
+
+    # Patch the PATHS instance with our test file
+    PATHS.container_ci = temp_file
+
+    # Yield the test file for the test to use
+    yield temp_file
+
+    # Restore the original value after the test
+    PATHS.container_ci = original_container_ci
+
+
+@pytest.fixture
+def mock_container_dir(tmp_path, monkeypatch):
+    """
+    Fixture that creates a dummy container directory and build script.
+    """
+    # Create a dummy container directory
+    container_dir = tmp_path / "container"
+    container_dir.mkdir()
+
+    # Create a dummy setup-test-env.sh file
+    dummy_setup_path = tmp_path / "setup-test-env.sh"
+    dummy_setup_path.write_text("echo Setup")
+
+    # Save original values
+    original_container_dir = PATHS.container_dir
+    original_setup_test_env = PATHS.setup_test_env
+
+    # Patch the PATHS instance
+    PATHS.container_dir = container_dir
+    PATHS.setup_test_env = dummy_setup_path
+
+    # Yield the directory for the test to use
+    yield container_dir
+
+    # Restore original values
+    PATHS.container_dir = original_container_dir
+    PATHS.setup_test_env = original_setup_test_env
+
+
+@pytest.fixture
+def mock_build_script(mock_container_dir):
+    """
+    Fixture that creates a dummy build script in the container directory.
+    """
+    build_script = mock_container_dir / "build-foo.sh"
+    return build_script
+
+
+###############################################################################
+# Tests for argument parsing and helper functions
+###############################################################################
+
+
+@pytest.mark.parametrize(
+    "component, expected_tag_var",
+    [
+        ("foo", "FOO_TAG"),
+        ("my-component", "MY_COMPONENT_TAG"),
+        ("foo-bar-baz", "FOO_BAR_BAZ_TAG"),
+    ],
+)
+def test_from_component_to_tag_var(component, expected_tag_var):
+    """
+    Test that from_component_to_tag_var returns the correct tag variable name.
+    """
+    assert from_component_to_tag_var(component) == expected_tag_var
+
+
+@pytest.mark.parametrize(
+    "component, expected_build_tag",
+    [
+        ("foo", "CONDITIONAL_BUILD_FOO_TAG"),
+        ("my-component", "CONDITIONAL_BUILD_MY_COMPONENT_TAG"),
+        ("foo-bar-baz", "CONDITIONAL_BUILD_FOO_BAR_BAZ_TAG"),
+    ],
+)
+def test_from_component_to_build_tag(component, expected_build_tag):
+    """
+    Test that from_component_to_build_tag returns the correct build tag name.
+    """
+    assert from_component_to_build_tag(component) == expected_build_tag
+
+
+@pytest.mark.parametrize(
+    "script_name, expected_component",
+    [
+        ("build-foo.sh", "foo"),
+        ("build-my-component.sh", "my-component"),
+        ("build-foo-bar-baz.sh", "foo-bar-baz"),
+    ],
+)
+def test_from_script_name_to_component(script_name, expected_component):
+    """
+    Test that from_script_name_to_component returns the correct component name.
+    """
+    assert from_script_name_to_component(script_name) == expected_component
+
+
+@pytest.mark.parametrize(
+    "script_name, expected_tag_var",
+    [
+        ("build-foo.sh", "FOO_TAG"),
+        ("build-my-component.sh", "MY_COMPONENT_TAG"),
+        ("build-foo-bar-baz.sh", "FOO_BAR_BAZ_TAG"),
+    ],
+)
+def test_from_script_name_to_tag_var(script_name, expected_tag_var):
+    """
+    Test that from_script_name_to_tag_var returns the correct tag variable name.
+    """
+    assert from_script_name_to_tag_var(script_name) == expected_tag_var
+
+
+def test_filter_components():
+    """
+    Test that filter_components returns only components that match an include regex
+    and do not match any exclude regex. If includes is empty, an empty list is returned.
+    """
+    components = ["alpha", "beta", "gamma", "delta"]
+
+    # When includes is empty, should return an empty list.
+    assert filter_components(components, [], []) == []
+
+    # Test includes only.
+    # Components that start with 'a' or 'b'
+    expected = ["alpha", "beta"]
+    result = filter_components(components, ["^[a-b].*$"], [])
+    assert result == expected
+
+    # Test with an exclude that filters out "alpha" (which matches "lph").
+    result = filter_components(components, ["^[a-b].*$"], ["^.*lph.*$"])
+    # "alpha" is removed.
+    assert result == ["beta"]
+
+
+@pytest.mark.parametrize(
+    "component, tag",
+    [
+        # Basic case
+        ("test", "new_tag"),
+        # Component with hyphen
+        ("my-component", "v1.2.3"),
+        # Uppercase component name
+        ("UpperCase", "123"),
+        # Numbers in component name
+        ("123service", "build-456"),
+    ],
+)
+def test_update_image_tag_in_yaml(component, tag, temp_image_tags_file):
+    """
+    Test multiple updates with different component names and tags, verifying:
+    1. Correct variable name generation
+    2. Proper tag value storage
+    3. Maintained sorting of variables
+    """
+    # Initial update
+    update_image_tag_in_yaml(component, tag)
+
+    data = load_image_tags_yaml()
+    expected_var = from_component_to_build_tag(component)
+    assert data["variables"][expected_var] == tag
+
+    # Add second component and verify both exist
+    update_image_tag_in_yaml("another_component", "secondary_tag")
+    updated_data = load_image_tags_yaml()
+
+    assert from_component_to_build_tag("another_component") in updated_data["variables"]
+    assert updated_data["variables"][expected_var] == tag  # Original value remains
+
+    # Verify sorting
+    variables = list(data["variables"].keys())
+    assert len(updated_data["variables"]) == len(variables) + 1
+    assert variables == sorted(variables), "Variables are not alphabetically sorted"
+
+
+def test_if_run_extracts_the_tag_from_stdout(monkeypatch, mock_build_script):
+    """
+    Test that run_build_script returns the tag (last stdout line) when the build
+    script executes successfully.
+    """
+    mock_build_script.write_text("#!/bin/bash\necho Build script\necho new_tag")
+    # Create a fake subprocess.CompletedProcess to simulate a successful build.
+    fake_result = subprocess.CompletedProcess(
+        args=[], returncode=0, stdout="line1\nnew_tag", stderr=""
+    )
+    monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: fake_result)
+
+    tag = run_build_script("foo", check_only=False)
+    assert tag == "new_tag"
+
+
+def test_running_real_process_works(monkeypatch, mock_build_script):
+    """
+    Test that run_build_script returns the tag (last stdout line) when the build
+    script executes successfully.
+    """
+    mock_build_script.write_text("echo Build script\necho new_tag")
+    tag = run_build_script("foo", check_only=False)
+    assert tag == "new_tag"
+
+
+###############################################################################
+# Tests for main() argument features
+###############################################################################
+
+
+def setup_build_script_and_container_ci_file(container_dir, container_ci_file, comp):
+    tag_var = from_component_to_build_tag(comp)
+    build_check_var = from_component_to_tag_var(comp)
+    build_script_path = container_dir / f"build-{comp}.sh"
+    build_script_path.write_text(
+        dedent(
+            f"""
+        #!/bin/bash
+        ci_tag_build_time_check {tag_var}
+        echo "new_tag"
+        """
+        )
+    )
+    container_ci_file.write_text(
+        container_ci_file.read_text()
+        + dedent(
+            f"""
+        .container-builds-{comp}:
+            variables:
+                {tag_var}: "${{{build_check_var}}}"
+        """
+        )
+    )
+
+
+def test_main_list(monkeypatch, capsys, mock_container_dir, temp_container_ci_file):
+    """
+    Test that when the --list argument is provided, main() prints all detected
+    components
+    """
+    # Monkeypatch find_components to return a known list.
+    for comp in ["comp-a", "comp-b"]:
+        setup_build_script_and_container_ci_file(mock_container_dir, temp_container_ci_file, comp)
+    # Set sys.argv to simulate passing --list.
+    monkeypatch.setattr(sys, "argv", ["bin/ci/update_tag.py", "--list"])
+
+    main()
+
+    captured = capsys.readouterr().out
+    assert "Detected components:" in captured
+    assert "comp-a" in captured
+    assert "comp-b" in captured
+
+
+def test_main_check(monkeypatch, temp_image_tags_file, temp_container_ci_file):
+    """
+    Test that when --check is provided, main() calls run_build_script in check mode.
+    """
+
+    with patch(
+        "bin.ci.update_tag.run_build_script", return_value="dummy_tag"
+    ) as mock_run_build_script:
+        # Simulate command line: --check compX
+        monkeypatch.setattr(sys, "argv", ["bin/ci/update_tag.py", "--check", "comp-x"])
+
+        main()
+    # Verify run_build_script was called with check_only=True.
+    assert mock_run_build_script.call_count == 1
+    assert "comp-x" in mock_run_build_script.call_args.args[0]
+    assert mock_run_build_script.call_args.kwargs["check_only"] is True
+
+
+EXIT_CODE_SCENARIOS = {
+    "unbound_variable": (
+        "line 2: UNDEFINED_VARIABLE: unbound variable",
+        127,
+        3,
+        "Please set the variable UNDEFINED_VARIABLE",
+    ),
+    "build script error": (
+        "",
+        50,
+        50,
+        "",
+    ),
+    "tag_mismatch": (
+        "Tag mismatch for foo.",
+        2,
+        2,
+        "Tag mismatch for foo.",
+    ),
+}
+
+
+@pytest.mark.parametrize(
+    "stderr_content, script_returncode, expected_exit_code, expected_error_message",
+    EXIT_CODE_SCENARIOS.values(),
+    ids=list(EXIT_CODE_SCENARIOS.keys()),
+)
+def test_build_script_error_exit_codes(
+    monkeypatch,
+    mock_build_script,
+    capsys,
+    stderr_content,
+    script_returncode,
+    expected_exit_code,
+    expected_error_message,
+):
+    """
+    Test that build script errors generate appropriate exit codes:
+    1 - Unhandled error in this script
+    2 - Tag mismatch when using --check
+    3 - Unbound variable error in build script
+    x - Build script failed with return code x
+    """
+    # Create a build script that will fail
+    mock_build_script.write_text("#!/bin/bash\necho 'This will fail'")
+
+    # Create a fake subprocess.CompletedProcess to simulate a failed build
+    fake_result = subprocess.CompletedProcess(
+        args=[], returncode=script_returncode, stdout="", stderr=stderr_content
+    )
+    monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: fake_result)
+    monkeypatch.setattr(sys, "argv", ["bin/ci/update_tag.py", "--check", "foo"])
+    monkeypatch.setattr("bin.ci.update_tag.get_current_tag_value", lambda *args: "current_tag")
+
+    # Mock sys.exit to capture the exit code instead of exiting the test
+    with pytest.raises(SystemExit) as e:
+        main()
+
+        # Check for expected error message if one is specified
+        if expected_error_message:
+            captured = capsys.readouterr()
+            assert expected_error_message in captured.err
+
+    # Verify correct exit code
+    assert e.value.code == expected_exit_code

bin/ci/update_tag.py

@@ -0,0 +1,447 @@
+#!/usr/bin/env python3
+import logging
+import os
+import re
+import sys
+import argparse
+import subprocess
+import yaml
+from typing import Optional, Set, Any
+from dataclasses import dataclass, field
+
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+# Very simple type alias for GitLab YAML data structure
+# It is composed by a dictionary of job names, each with a dictionary of fields
+# (e.g., script, stage, rules, etc.)
+YamlData = dict[str, dict[str, Any]]
+
+# Dummy environment vars to make build scripts happy
+# To avoid set -u errors in build scripts
+DUMMY_ENV_VARS: dict[str, str] = {
+    # setup-test-env.sh
+    "CI_JOB_STARTED_AT": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S%z"),
+    # build-deqp.sh
+    "DEQP_API": "dummy",
+    "DEQP_TARGET": "dummy",
+}
+
+
+@dataclass
+class ProjectPaths:
+    """Manages all project-related paths"""
+
+    root: Path = field(default_factory=lambda: Path(), init=False)
+    setup_test_env: Path = field(default_factory=lambda: Path(), init=False)
+    conditional_tags: Path = field(default_factory=lambda: Path(), init=False)
+    container_dir: Path = field(default_factory=lambda: Path(), init=False)
+    container_ci: Path = field(default_factory=lambda: Path(), init=False)
+
+    def __post_init__(self):
+        self.root = self.find_root()
+        self.setup_test_env = self.root / ".gitlab-ci" / "setup-test-env.sh"
+        self.conditional_tags = self.root / ".gitlab-ci" / "conditional-build-image-tags.yml"
+        self.container_dir = self.root / ".gitlab-ci" / "container"
+        self.container_ci = self.container_dir / "gitlab-ci-inc.yml"
+
+    def find_root(self) -> Path:
+        """Find git repository root or fallback to other methods"""
+        try:
+            logging.debug("Getting git repo root using git rev-parse")
+            return Path(
+                subprocess.check_output(
+                    ["git", "rev-parse", "--show-toplevel"],
+                    text=True,
+                    stderr=subprocess.PIPE,
+                ).strip()
+            )
+        # FileNotFoundError is raised if git is not installed
+        except (subprocess.CalledProcessError, FileNotFoundError):
+            # Fallback to CI_PROJECT_DIR or path-based method
+            if ci_dir := os.environ.get("CI_PROJECT_DIR"):
+                logging.debug(f"Getting git repo root using CI_PROJECT_DIR: {ci_dir}")
+                return Path(ci_dir)
+
+            # Last resort, use the path to this script
+            logging.debug("Getting git repo root using path to this script")
+            return Path(__file__).resolve().parent.parent.parent
+
+
+PATHS: ProjectPaths = ProjectPaths()
+
+
+def from_component_to_build_tag(component: str) -> str:
+    # e.g., "angle" -> "CONDITIONAL_BUILD_ANGLE_TAG"
+    return "CONDITIONAL_BUILD_" + re.sub(r"-", "_", component.upper()) + "_TAG"
+
+
+def from_component_to_tag_var(component: str) -> str:
+    # e.g., "angle" -> "ANGLE_TAG"
+    return re.sub(r"-", "_", component.upper()) + "_TAG"
+
+
+def from_script_name_to_component(script_name: str) -> str:
+    # e.g., "build-angle.sh" -> "angle"
+    return re.sub(r"^build-([a-z0-9_-]+)\.sh$", r"\1", script_name)
+
+
+def from_script_name_to_tag_var(script_name: str) -> str:
+    # e.g., "build-angle.sh" -> "ANGLE_TAG"
+    return re.sub(r"^build-([a-z0-9_-]+)\.sh$", r"\1_TAG", script_name).replace("-", "_").upper()
+
+
+def prepare_setup_env_script() -> Path:
+    """
+    Sets up dummy environment variables to mimic the script in the CI repo.
+    Returns the path to the setup-test-env.sh script.
+    """
+    if not PATHS.setup_test_env.exists():
+        logging.error(".gitlab-ci/setup-test-env.sh not found. Exiting.")
+        sys.exit(1)
+
+    # Dummy environment vars to mimic the script
+    for key, value in DUMMY_ENV_VARS.items():
+        os.environ[key] = value
+
+    os.environ["CI_PROJECT_DIR"] = str(PATHS.root)
+
+    return PATHS.setup_test_env
+
+
+def validate_build_script(script_filename: str) -> bool:
+    """
+    Returns True if the build script for the given component uses the structured tag variable.
+    """
+    build_script = PATHS.container_dir / script_filename
+    with open(build_script, "r", encoding="utf-8") as f:
+        script_content = f.read()
+
+    tag_var = from_script_name_to_tag_var(script_filename)
+    if not re.search(tag_var, script_content, re.IGNORECASE):
+        logging.debug(
+            f"Skipping {build_script} because it doesn't use {tag_var}",
+        )
+        return False
+    return True
+
+
+def load_container_yaml() -> YamlData:
+    if not PATHS.container_ci.is_file():
+        logging.error(f"File not found: {PATHS.container_ci}")
+        sys.exit(1)
+
+    # Ignore !reference and other custom GitLab tags, we just want to know the
+    # job names and fields
+    yaml.SafeLoader.add_multi_constructor("", lambda loader, suffix, node: None)
+    with open(str(PATHS.container_ci), "r", encoding="utf-8") as f:
+        data = yaml.load(f, Loader=yaml.SafeLoader)
+        if not isinstance(data, dict):
+            return {"variables": {}}
+        return data
+
+
+def find_candidate_components() -> list[str]:
+    """
+    1) Reads .gitlab-ci/container/gitlab-ci.yml to find component links:
+        lines matching '.*.container-builds-<component>'
+    2) Looks for matching build-<component>.sh in .gitlab-ci/container/
+    3) Returns a sorted list of components in the intersection of these sets.
+    """
+    container_yaml = load_container_yaml()
+    # Extract patterns like `container-builds-foo` from job names
+    candidates: Set[str] = set()
+    for job_name in container_yaml:
+        if match := re.search(r"\.container-builds-([a-z0-9_-]+)$", str(job_name)):
+            candidates.add(match.group(1))
+
+    if not candidates:
+        logging.error(
+            f"No viable build components found in {PATHS.container_ci}. "
+            "Please check the file for valid component names. "
+            "They should be named like '.container-builds-<component>'."
+        )
+        return []
+
+    # Find build scripts named build-<component>.sh
+    build_scripts: list[str] = []
+    for path in PATHS.container_dir.glob("build-*.sh"):
+        if validate_build_script(path.name):
+            logging.info(f"Found build script: {path.name}")
+            component = from_script_name_to_component(path.name)
+            build_scripts.append(component)
+
+    # Return sorted intersection of components found in build scripts and candidates
+    return sorted(candidates.intersection(build_scripts))
+
+
+def filter_components(components: list[str], includes: list[str], excludes: list[str]) -> list[str]:
+    """
+    Returns components that match at least one `includes` regex and none of the `excludes` regex.
+    If includes is empty, returns an empty list (unless user explicitly does --all or --include).
+    """
+    if not includes:
+        return []
+
+    filtered = []
+    for comp in components:
+        # Must match at least one "include"
+        if not any(re.fullmatch(inc, comp) for inc in includes):
+            logging.debug(f"Excluding {comp}, no matches in includes.")
+            continue
+        # Must not match any "exclude"
+        if any(re.fullmatch(exc, comp) for exc in excludes):
+            logging.debug(f"Excluding {comp}, matched exclude pattern.")
+            continue
+        filtered.append(comp)
+    return filtered
+
+
+def run_build_script(component: str, check_only: bool = False) -> Optional[str]:
+    """
+    Runs .gitlab-ci/container/build-<component>.sh to produce a new tag (last line of stdout).
+    If check_only=True, we skip updates to the YAML (but do the build to see if it passes).
+    Returns the extracted tag (string) on success, or None on failure.
+    """
+    # 1) Set up environment
+    setup_env_script = prepare_setup_env_script()
+
+    build_script = PATHS.container_dir / f"build-{component}.sh"
+    if not build_script.is_file():
+        logging.error(f"Build script not found for {component}: {build_script}")
+        return None
+
+    # Tag var should appear in the script, e.g., ANGLE_TAG for 'angle'
+    tag_var = from_component_to_tag_var(component)
+
+    # We set up environment for the child process
+    child_env: dict[str, str] = {}
+    child_env["NEW_TAG_DRY_RUN"] = "1"
+    if check_only:
+        # For checking only
+        child_env.pop("NEW_TAG_DRY_RUN", None)
+        child_env["CI_NOT_BUILDING_ANYTHING"] = "1"
+        if tag_value := get_current_tag_value(component):
+            child_env[tag_var] = tag_value
+        else:
+            logging.error(f"No current tag value for {component}")
+            return None
+
+    logging.debug(
+        f"Running build for {component} with "
+        f"{tag_var}={child_env.get(tag_var)} "
+        f"(check_only={check_only})"
+    )
+
+    # Run the build script
+    result = subprocess.run(
+        ["bash", "-c", f"source {setup_env_script} && source {build_script}"],
+        env=os.environ | child_env,
+        capture_output=True,
+        text=True,
+    )
+    logging.debug(f"{' '.join(result.args)}")
+
+    # Tag check succeeded
+    if result.returncode == 0:
+        # Tag is assumed to be the last line of stdout
+        lines = result.stdout.strip().splitlines()
+        return lines[-1] if lines else ""
+
+    # Tag check failed, let's dissect the error
+
+    if result.returncode == 2:
+        logging.error(f"Tag mismatch for {component}.")
+        logging.error(result.stdout)
+        return None
+
+    # Check if there's an unbound variable error
+    err_output = result.stderr
+    unbound_match = re.search(r"([A-Z_]+)(?=: unbound variable)", err_output)
+    if unbound_match:
+        var_name = unbound_match.group(1)
+        logging.fatal(f"Please set the variable {var_name} in {build_script}.")
+        sys.exit(3)
+
+    # Unexpected error in the build script, propagate the exit code
+    logging.fatal(f"Build script for {component} failed with return code {result.returncode}")
+    logging.error(result.stdout)
+    sys.exit(result.returncode)
+
+
+def load_image_tags_yaml() -> YamlData:
+    if not PATHS.conditional_tags.is_file():
+        raise FileNotFoundError(
+            f"Conditional build image tags file not found: {PATHS.conditional_tags}"
+        )
+
+    with open(str(PATHS.conditional_tags), "r", encoding="utf-8") as f:
+        data = yaml.safe_load(f)
+        if not isinstance(data, dict):
+            return {"variables": {}}
+        if "variables" not in data:
+            data["variables"] = {}
+        return data
+
+
+def get_current_tag_value(component: str) -> Optional[str]:
+    # If the CI job already set the tag, use it
+    tag_var = from_component_to_tag_var(component)
+    if os.getenv(tag_var):
+        logging.debug(f"Using tag from environment variable {tag_var}")
+        return os.getenv(tag_var)
+
+    # Otherwise, look in the YAML file, which normally occurs when updating tags locally
+    full_tag_var = from_component_to_build_tag(component)
+    data = load_image_tags_yaml()
+    variables = data.get("variables", {})
+    if not isinstance(variables, dict):
+        return None
+    logging.debug(f"Using tag from YAML file {full_tag_var}")
+    return variables.get(full_tag_var)
+
+
+def update_image_tag_in_yaml(component: str, tag_value: str) -> None:
+    """
+    Uses PyYAML to edit the YAML file at IMAGE_TAGS_FILE, setting the environment variable
+    for the given component. Maintains sorted keys.
+    """
+    full_tag_var = from_component_to_build_tag(component)
+    data = load_image_tags_yaml()
+
+    # Ensure we have a variables dictionary
+    if "variables" not in data:
+        data["variables"] = {}
+    elif not isinstance(data["variables"], dict):
+        data["variables"] = {}
+
+    # Update the tag
+    data["variables"][full_tag_var] = tag_value
+
+    # Sort the variables
+    data["variables"] = dict(sorted(data["variables"].items()))
+
+    # Write back to file
+    with open(str(PATHS.conditional_tags), "w", encoding="utf-8") as f:
+        yaml.dump(data, f, sort_keys=False)  # Don't sort top-level keys
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description="Manage container image tags for CI builds with regex-based includes/excludes.",
+        epilog="""
+Exit codes:
+  0 - Success
+  1 - Unhandled error in this script
+  2 - Tag mismatch when using --check
+  3 - Unbound variable error in build script
+  x - Build script failed with return code x
+        """,
+    )
+
+    parser.add_argument(
+        "--include",
+        "-i",
+        action="append",
+        default=[],
+        help="Full match regex pattern for components to include.",
+    )
+    parser.add_argument(
+        "--exclude",
+        "-x",
+        action="append",
+        default=[],
+        help="Full match regex pattern for components to exclude.",
+    )
+    parser.add_argument("--all", action="store_true", help="Equivalent to --include '.*'")
+    parser.add_argument(
+        "--check",
+        "-c",
+        action="append",
+        default=[],
+        help="Check matching components instead of updating YAML. "
+        "If any component fails, exit with a non-zero exit code.",
+    )
+    parser.add_argument(
+        "--list",
+        "-l",
+        action="store_true",
+        help="List all available components and exit.",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="count",
+        default=0,
+        help="Increase verbosity level (-v for info, -vv for debug)",
+    )
+
+    if len(sys.argv) == 1:
+        parser.print_help()
+        sys.exit(0)
+
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+
+    # Configure logging based on verbosity level
+    if args.verbose == 1:
+        log_level = logging.INFO
+    elif args.verbose == 2:
+        log_level = logging.DEBUG
+    else:
+        log_level = logging.WARNING
+
+    logging.basicConfig(level=log_level, format="%(levelname)s: %(message)s")
+
+    # 1) If checking, just run build scripts in check mode and propagate errors
+    if args.check:
+        tag_mismatch = False
+        for comp in args.check:
+            try:
+                if run_build_script(comp, check_only=True) is None:
+                    # The tag is invalid
+                    tag_mismatch = True
+            except SystemExit as e:
+                # Let custom exit codes propagate
+                raise e
+            except Exception as e:
+                logging.error(f"Internal error: {e}")
+                sys.exit(1)
+        # If any component failed, exit with code 1
+        if tag_mismatch:
+            sys.exit(2)
+        return
+
+    # Convert --all into a wildcard include
+    if args.all:
+        args.include.append(".*")
+
+    # 2) If --list, just show all discovered components
+    all_components = find_candidate_components()
+    if args.list:
+        print("Detected components:", ", ".join(all_components))
+        return
+
+    # 3) Filter components
+    final_components = filter_components(all_components, args.include, args.exclude)
+
+    if args.verbose:
+        logging.debug(f"Found components: {all_components}")
+        logging.debug(f"Filtered components: {final_components}")
+
+    for comp in final_components:
+        logging.info(f"Updating {comp}...")
+        new_tag = run_build_script(comp, check_only=False)
+        if new_tag is not None:
+            update_image_tag_in_yaml(comp, new_tag)
+            if args.verbose:
+                logging.debug(f"Updated {comp} with tag: {new_tag}")
+
+
+if __name__ == "__main__":
+    main()

bin/ci/update_traces_checksum.py

@@ -0,0 +1,217 @@
+#!/usr/bin/env python3
+# Copyright © 2022 Collabora Ltd.
+# Authors:
+#   David Heidelberg <[email protected]>
+#
+# For the dependencies, see the requirements.txt
+# SPDX-License-Identifier: MIT
+
+"""
+Helper script to update traces checksums
+"""
+
+import argparse
+import bz2
+import glob
+import re
+import json
+import sys
+from ruamel.yaml import YAML
+from subprocess import check_output
+from tomledit import Document
+
+import gitlab
+from gitlab_common import (get_gitlab_project, read_token, wait_for_pipeline,
+                           get_gitlab_pipeline_from_url, TOKEN_DIR, get_token_from_default_dir)
+from rich import print
+
+
+DESCRIPTION_FILE = "export PIGLIT_REPLAY_DESCRIPTION_FILE=.*/install/(.*)$"
+DEVICE_NAME = "(?:declare -x|export) PIGLIT_REPLAY_DEVICE_NAME='?([^']*)'?$"
+
+def gather_job_results_yml(cur_job):
+    log: list[str] = cur_job.trace().decode().splitlines()
+    filename: str = ''
+    dev_name: str = ''
+    for logline in log:
+        desc_file = re.search(DESCRIPTION_FILE, logline)
+        device_name = re.search(DEVICE_NAME, logline)
+        if desc_file:
+            filename = desc_file.group(1)
+        if device_name:
+            dev_name = device_name.group(1)
+
+    if not filename or not dev_name:
+        print("[red]Couldn't find device name or YML file in the logs!")
+        return
+
+    print(f"👁 Found {dev_name} and file {filename}")
+
+    # find filename in Mesa source
+    traces_file = glob.glob('./**/' + filename, recursive=True)
+    # write into it
+    with open(traces_file[0], 'r', encoding='utf-8') as target_file:
+        yaml = YAML()
+        yaml.compact(seq_seq=False, seq_map=False)
+        yaml.version = 1,2
+        yaml.width = 2048  # do not break the text fields
+        yaml.default_flow_style = None
+        target = yaml.load(target_file)
+
+        # parse artifact
+        results_json_bz2 = cur_job.artifact("results/results.json.bz2")
+        results_json = bz2.decompress(results_json_bz2).decode("utf-8", errors="replace")
+        results = json.loads(results_json)
+
+        for _, value in results["tests"].items():
+            if (
+                not value['images'] or
+                not value['images'][0] or
+                "image_desc" not in value['images'][0]
+            ):
+                continue
+
+            trace: str = value['images'][0]['image_desc']
+            checksum: str = value['images'][0]['checksum_render']
+
+            if not checksum:
+                print(f"[red]{dev_name}: {trace}: checksum is missing! Crash?")
+                continue
+
+            if checksum == "error":
+                print(f"[red]{dev_name}: {trace}: crashed")
+                continue
+
+            if target['traces'][trace][dev_name].get('checksum') == checksum:
+                continue
+
+            if "label" in target['traces'][trace][dev_name]:
+                print(
+                    f"{dev_name}: {trace}: please verify that label "
+                    f"[blue]{target['traces'][trace][dev_name]['label']}[/blue] "
+                    "is still valid"
+                )
+
+            print(f"[green]{dev_name}: {trace}: checksum updated")
+            target['traces'][trace][dev_name]['checksum'] = checksum
+
+    with open(traces_file[0], 'w', encoding='utf-8') as target_file:
+        yaml.dump(target, target_file)
+
+
+def update_toml_checksum(toml_file, trace_path, device, old_checksum, new_checksum):
+    with open(toml_file, 'r', encoding='utf-8') as f:
+        doc = Document.parse(f.read())
+
+    for table in doc["traces"]:
+        if table["path"].value != trace_path:
+            continue
+        try:
+            table["devices"][device]["checksum"] = new_checksum
+        except KeyError:
+            return False
+        with open(toml_file, 'w', encoding='utf-8') as f:
+            f.write(doc.as_toml())
+        return True
+
+    return False
+
+
+def gather_job_results_toml(cur_job) -> None:
+    """Process checksum-changes.json artifact (gpu-trace-perf replay format)"""
+    toml_files = glob.glob('./**/traces-*.toml', recursive=True)
+
+    changes_json = cur_job.artifact("results/checksum-changes.json")
+    changes = json.loads(changes_json.decode("utf-8"))
+
+    for change in changes:
+        device = change['device']
+        trace = change['trace']
+        old_checksum = change['old_checksum']
+        new_checksum = change['new_checksum']
+
+        if not new_checksum:
+            print(f"[red]{device}: {trace}: no new checksum (crash?)")
+            continue
+
+        updated = False
+        for toml_file in toml_files:
+            if update_toml_checksum(toml_file, trace, device, old_checksum, new_checksum):
+                print(f"[green]{device}: {trace}: checksum updated")
+                updated = True
+                break
+
+        if not updated:
+            print(f"[red]{device}: {trace}: not found in any traces TOML file")
+
+
+def gather_results(
+    project,
+    pipeline,
+) -> None:
+    """Gather results"""
+
+    target_jobs_regex = re.compile(".*-traces(-restricted)?([:].*)?$")
+
+    for job in pipeline.jobs.list(all=True, sort="desc"):
+        if target_jobs_regex.match(job.name) and job.status == "failed":
+            cur_job = project.jobs.get(job.id)
+            # get variables
+            print(f"👁  {job.name}...")
+            try:
+                gather_job_results_toml(cur_job)
+            except Exception:
+                gather_job_results_yml(cur_job)
+
+
+def parse_args() -> None:
+    """Parse args"""
+    parser = argparse.ArgumentParser(
+        description="Tool to generate patch from checksums ",
+        epilog="Example: update_traces_checksum.py --rev $(git rev-parse HEAD) "
+    )
+    parser.add_argument(
+        "--rev", metavar="revision", help="repository git revision", default='HEAD'
+    )
+    parser.add_argument(
+        "--token",
+        metavar="token",
+        type=str,
+        default=get_token_from_default_dir(),
+        help="Use the provided GitLab token or token file, "
+             f"otherwise it's read from {TOKEN_DIR / 'gitlab-token'}",
+    )
+    parser.add_argument(
+        "--pipeline-url",
+        metavar="pipeline_url",
+        help="specify a pipeline url",
+    )
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    try:
+        args = parse_args()
+
+        token = read_token(args.token)
+
+        gl = gitlab.Gitlab(url="https://gitlab.freedesktop.org", private_token=token)
+
+        cur_project = get_gitlab_project(gl, "mesa")
+
+        if args.pipeline_url:
+            pipe, cur_project = get_gitlab_pipeline_from_url(gl, args.pipeline_url)
+            REV = pipe.sha
+        else:
+            if not args.rev:
+                print('error: the following arguments are required: --rev')
+                sys.exit(1)
+            sha = check_output(['git', 'rev-parse', args.rev]).decode('ascii').strip()
+            print(f"Revision: {sha}")
+            (pipe, cur_project) = wait_for_pipeline([cur_project], sha)
+        print(f"Pipeline: {pipe.web_url}")
+        gather_results(cur_project, pipe)
+
+        sys.exit()
+    except KeyboardInterrupt:
+        sys.exit(1)

bin/ci/update_traces_checksum.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -eu
+
+this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
+readonly this_dir
+
+exec \
+  "$this_dir/../python-venv.sh" \
+  "$this_dir/requirements.txt" \
+  "$this_dir/update_traces_checksum.py" "$@"