init

bin/ci/test/requirements.txt

@@ -0,0 +1,10 @@
+-r ../requirements.txt
+aiohttp==3.13.*
+filelock==3.12.4
+fire==0.7.0
+freezegun==1.5.1
+mock==5.1.0
+polars==0.19.3
+pytest==7.4.2
+pytest-asyncio==0.21.0
+pytest-cov==3.0.0

bin/ci/test/test_custom_logger.py

@@ -0,0 +1,746 @@
+import logging
+import subprocess
+from datetime import datetime
+
+import pytest
+from custom_logger import CustomLogger
+
+
+@pytest.fixture
+def tmp_log_file(tmp_path):
+    return tmp_path / "test_log.json"
+
+
+@pytest.fixture
+def custom_logger(tmp_log_file):
+    return CustomLogger(tmp_log_file)
+
+
+def run_script_with_args(args):
+    import custom_logger
+
+    script_path = custom_logger.__file__
+    return subprocess.run(
+        ["python3", str(script_path), *args], capture_output=True, text=True
+    )
+
+
+# Test case for missing log file
+@pytest.mark.parametrize(
+    "key, value", [("dut_attempt_counter", "1"), ("job_combined_status", "pass")]
+)
+def test_missing_log_file_argument(key, value):
+    result = run_script_with_args(["--update", "key", "value"])
+    assert result.returncode != 0
+
+
+# Parametrize test case for valid update arguments
+@pytest.mark.parametrize(
+    "key, value", [("dut_attempt_counter", "1"), ("job_combined_status", "pass")]
+)
+def test_update_argument_valid(custom_logger, tmp_log_file, key, value):
+    result = run_script_with_args([str(tmp_log_file), "--update", key, value])
+    assert result.returncode == 0
+
+
+# Test case for passing only the key without a value
+def test_update_argument_key_only(custom_logger, tmp_log_file):
+    key = "dut_attempt_counter"
+    result = run_script_with_args([str(tmp_log_file), "--update", key])
+    assert result.returncode != 0
+
+
+# Test case for not passing any key-value pair
+def test_update_argument_no_values(custom_logger, tmp_log_file):
+    result = run_script_with_args([str(tmp_log_file), "--update"])
+    assert result.returncode == 0
+
+
+# Parametrize test case for valid arguments
+@pytest.mark.parametrize(
+    "key, value", [("dut_attempt_counter", "1"), ("job_combined_status", "pass")]
+)
+def test_create_argument_valid(custom_logger, tmp_log_file, key, value):
+    result = run_script_with_args([str(tmp_log_file), "--create-dut-job", key, value])
+    assert result.returncode == 0
+
+
+# Test case for passing only the key without a value
+def test_create_argument_key_only(custom_logger, tmp_log_file):
+    key = "dut_attempt_counter"
+    result = run_script_with_args([str(tmp_log_file), "--create-dut-job", key])
+    assert result.returncode != 0
+
+
+# Test case for not passing any key-value pair
+def test_create_argument_no_values(custom_logger, tmp_log_file):
+    result = run_script_with_args([str(tmp_log_file), "--create-dut-job"])
+    assert result.returncode == 0
+
+
+# Test case for updating a DUT job
+@pytest.mark.parametrize(
+    "key, value", [("status", "hung"), ("dut_state", "Canceling"), ("dut_name", "asus")]
+)
+def test_update_dut_job(custom_logger, tmp_log_file, key, value):
+    result = run_script_with_args([str(tmp_log_file), "--update-dut-job", key, value])
+    assert result.returncode != 0
+
+    result = run_script_with_args([str(tmp_log_file), "--create-dut-job", key, value])
+    assert result.returncode == 0
+
+    result = run_script_with_args([str(tmp_log_file), "--update-dut-job", key, value])
+    assert result.returncode == 0
+
+
+# Test case for updating last DUT job
+def test_update_dut_multiple_job(custom_logger, tmp_log_file):
+    # Create the first DUT job with the first key
+    result = run_script_with_args(
+        [str(tmp_log_file), "--create-dut-job", "status", "hung"]
+    )
+    assert result.returncode == 0
+
+    # Create the second DUT job with the second key
+    result = run_script_with_args(
+        [str(tmp_log_file), "--create-dut-job", "dut_state", "Canceling"]
+    )
+    assert result.returncode == 0
+
+    result = run_script_with_args(
+        [str(tmp_log_file), "--update-dut-job", "dut_name", "asus"]
+    )
+    assert result.returncode == 0
+
+
+# Parametrize test case for valid phase arguments
+@pytest.mark.parametrize(
+    "phase_name",
+    [("Phase1"), ("Phase2"), ("Phase3")],
+)
+def test_create_job_phase_valid(custom_logger, tmp_log_file, phase_name):
+    custom_logger.create_dut_job(status="pass")
+
+    result = run_script_with_args([str(tmp_log_file), "--create-job-phase", phase_name])
+    assert result.returncode == 0
+
+
+# Test case for not passing any arguments for create-job-phase
+def test_create_job_phase_no_arguments(custom_logger, tmp_log_file):
+    custom_logger.create_dut_job(status="pass")
+
+    result = run_script_with_args([str(tmp_log_file), "--create-job-phase"])
+    assert result.returncode != 0
+
+
+# Test case for trying to create a phase job without an existing DUT job
+def test_create_job_phase_no_dut_job(custom_logger, tmp_log_file):
+    phase_name = "Phase1"
+
+    result = run_script_with_args([str(tmp_log_file), "--create-job-phase", phase_name])
+    assert result.returncode != 0
+
+
+# Combined test cases for valid scenarios
+def test_valid_scenarios(custom_logger, tmp_log_file):
+    valid_update_args = [("dut_attempt_counter", "1"), ("job_combined_status", "pass")]
+    for key, value in valid_update_args:
+        result = run_script_with_args([str(tmp_log_file), "--update", key, value])
+        assert result.returncode == 0
+
+    valid_create_args = [
+        ("status", "hung"),
+        ("dut_state", "Canceling"),
+        ("dut_name", "asus"),
+        ("phase_name", "Bootloader"),
+    ]
+    for key, value in valid_create_args:
+        result = run_script_with_args(
+            [str(tmp_log_file), "--create-dut-job", key, value]
+        )
+        assert result.returncode == 0
+
+    result = run_script_with_args(
+        [str(tmp_log_file), "--create-dut-job", "status", "hung"]
+    )
+    assert result.returncode == 0
+
+    result = run_script_with_args(
+        [str(tmp_log_file), "--update-dut-job", "dut_name", "asus"]
+    )
+    assert result.returncode == 0
+
+    result = run_script_with_args(
+        [
+            str(tmp_log_file),
+            "--create-job-phase",
+            "phase_name",
+        ]
+    )
+    assert result.returncode == 0
+
+
+# Parametrize test case for valid update arguments
+@pytest.mark.parametrize(
+    "key, value", [("dut_attempt_counter", "1"), ("job_combined_status", "pass")]
+)
+def test_update(custom_logger, key, value):
+    custom_logger.update(**{key: value})
+    logger_data = custom_logger.logger.data
+
+    assert key in logger_data
+    assert logger_data[key] == value
+
+
+# Test case for updating with a key that already exists
+def test_update_existing_key(custom_logger):
+    key = "status"
+    value = "new_value"
+    custom_logger.logger.data[key] = "old_value"
+    custom_logger.update(**{key: value})
+    logger_data = custom_logger.logger.data
+
+    assert key in logger_data
+    assert logger_data[key] == value
+
+
+# Test case for updating "dut_jobs"
+def test_update_dut_jobs(custom_logger):
+    key1 = "status"
+    value1 = "fail"
+    key2 = "state"
+    value2 = "hung"
+
+    custom_logger.create_dut_job(**{key1: value1})
+    logger_data = custom_logger.logger.data
+
+    job1 = logger_data["dut_jobs"][0]
+    assert key1 in job1
+    assert job1[key1] == value1
+
+    custom_logger.update_dut_job(key2, value2)
+    logger_data = custom_logger.logger.data
+
+    job2 = logger_data["dut_jobs"][0]
+    assert key2 in job2
+    assert job2[key2] == value2
+
+
+# Test case for creating and updating DUT job
+def test_create_dut_job(custom_logger):
+    key = "status"
+    value1 = "pass"
+    value2 = "fail"
+    value3 = "hung"
+
+    reason = "job_combined_status"
+    result = "Finished"
+
+    custom_logger.update(**{reason: result})
+    logger_data = custom_logger.logger.data
+
+    assert reason in logger_data
+    assert logger_data[reason] == result
+
+    # Create the first DUT job
+    custom_logger.create_dut_job(**{key: value1})
+    logger_data = custom_logger.logger.data
+
+    assert "dut_jobs" in logger_data
+    assert isinstance(logger_data["dut_jobs"], list)
+    assert len(logger_data["dut_jobs"]) == 1
+    assert isinstance(logger_data["dut_jobs"][0], dict)
+
+    # Check the values of the keys in the created first DUT job
+    job1 = logger_data["dut_jobs"][0]
+    assert key in job1
+    assert job1[key] == value1
+
+    # Create the second DUT job
+    custom_logger.create_dut_job(**{key: value2})
+    logger_data = custom_logger.logger.data
+
+    assert "dut_jobs" in logger_data
+    assert isinstance(logger_data["dut_jobs"], list)
+    assert len(logger_data["dut_jobs"]) == 2
+    assert isinstance(logger_data["dut_jobs"][1], dict)
+
+    # Check the values of the keys in the created second DUT job
+    job2 = logger_data["dut_jobs"][1]
+    assert key in job2
+    assert job2[key] == value2
+
+    # Update the second DUT job with value3
+    custom_logger.update_dut_job(key, value3)
+    logger_data = custom_logger.logger.data
+
+    # Check the updated value in the second DUT job
+    job2 = logger_data["dut_jobs"][1]
+    assert key in job2
+    assert job2[key] == value3
+
+    # Find the index of the last DUT job
+    last_job_index = len(logger_data["dut_jobs"]) - 1
+
+    # Update the last DUT job
+    custom_logger.update_dut_job("dut_name", "asus")
+    logger_data = custom_logger.logger.data
+
+    # Check the updated value in the last DUT job
+    job2 = logger_data["dut_jobs"][last_job_index]
+    assert "dut_name" in job2
+    assert job2["dut_name"] == "asus"
+
+    # Check that "dut_name" is not present in other DUT jobs
+    for idx, job in enumerate(logger_data["dut_jobs"]):
+        if idx != last_job_index:
+            assert job.get("dut_name") == ""
+
+
+# Test case for updating with missing "dut_jobs" key
+def test_update_dut_job_missing_dut_jobs(custom_logger):
+    key = "status"
+    value = "fail"
+
+    # Attempt to update a DUT job when "dut_jobs" is missing
+    with pytest.raises(ValueError, match="No DUT jobs found."):
+        custom_logger.update_dut_job(key, value)
+
+
+# Test case for creating a job phase
+def test_create_job_phase(custom_logger):
+    custom_logger.create_dut_job(status="pass")
+    phase_name = "Phase1"
+
+    custom_logger.create_job_phase(phase_name)
+    logger_data = custom_logger.logger.data
+
+    assert "dut_jobs" in logger_data
+    assert isinstance(logger_data["dut_jobs"], list)
+    assert len(logger_data["dut_jobs"]) == 1
+
+    job = logger_data["dut_jobs"][0]
+    assert "dut_job_phases" in job
+    assert isinstance(job["dut_job_phases"], list)
+    assert len(job["dut_job_phases"]) == 1
+
+    phase = job["dut_job_phases"][0]
+    assert phase["name"] == phase_name
+    try:
+        datetime.fromisoformat(phase["start_time"])
+        assert True
+    except ValueError:
+        assert False
+    assert phase["end_time"] == ""
+
+
+# Test case for creating multiple phase jobs
+def test_create_multiple_phase_jobs(custom_logger):
+    custom_logger.create_dut_job(status="pass")
+
+    phase_data = [
+        {
+            "phase_name": "Phase1",
+        },
+        {
+            "phase_name": "Phase2",
+        },
+        {
+            "phase_name": "Phase3",
+        },
+    ]
+
+    for data in phase_data:
+        phase_name = data["phase_name"]
+
+        custom_logger.create_job_phase(phase_name)
+
+    logger_data = custom_logger.logger.data
+
+    assert "dut_jobs" in logger_data
+    assert isinstance(logger_data["dut_jobs"], list)
+    assert len(logger_data["dut_jobs"]) == 1
+
+    job = logger_data["dut_jobs"][0]
+    assert "dut_job_phases" in job
+    assert isinstance(job["dut_job_phases"], list)
+    assert len(job["dut_job_phases"]) == len(phase_data)
+
+    for data in phase_data:
+        phase_name = data["phase_name"]
+
+        phase = job["dut_job_phases"][phase_data.index(data)]
+
+        assert phase["name"] == phase_name
+        try:
+            datetime.fromisoformat(phase["start_time"])
+            assert True
+        except ValueError:
+            assert False
+
+        if phase_data.index(data) != len(phase_data) - 1:
+            try:
+                datetime.fromisoformat(phase["end_time"])
+                assert True
+            except ValueError:
+                assert False
+
+    # Check if the end_time of the last phase is an empty string
+    last_phase = job["dut_job_phases"][-1]
+    assert last_phase["end_time"] == ""
+
+
+# Test case for creating multiple dut jobs and updating phase job for last dut job
+def test_create_two_dut_jobs_and_add_phase(custom_logger):
+    # Create the first DUT job
+    custom_logger.create_dut_job(status="pass")
+
+    # Create the second DUT job
+    custom_logger.create_dut_job(status="fail")
+
+    logger_data = custom_logger.logger.data
+
+    assert "dut_jobs" in logger_data
+    assert isinstance(logger_data["dut_jobs"], list)
+    assert len(logger_data["dut_jobs"]) == 2
+
+    first_dut_job = logger_data["dut_jobs"][0]
+    second_dut_job = logger_data["dut_jobs"][1]
+
+    # Add a phase to the second DUT job
+    custom_logger.create_job_phase("Phase1")
+
+    logger_data = custom_logger.logger.data
+
+    assert "dut_jobs" in logger_data
+    assert isinstance(logger_data["dut_jobs"], list)
+    assert len(logger_data["dut_jobs"]) == 2
+
+    first_dut_job = logger_data["dut_jobs"][0]
+    second_dut_job = logger_data["dut_jobs"][1]
+
+    # Check first DUT job does not have a phase
+    assert not first_dut_job.get("dut_job_phases")
+
+    # Check second DUT job has a phase
+    assert second_dut_job.get("dut_job_phases")
+    assert isinstance(second_dut_job["dut_job_phases"], list)
+    assert len(second_dut_job["dut_job_phases"]) == 1
+
+
+# Test case for updating DUT start time
+def test_update_dut_start_time(custom_logger):
+    custom_logger.create_dut_job(status="pass")
+    custom_logger.update_dut_time("start", None)
+
+    logger_data = custom_logger.logger.data
+    assert "dut_jobs" in logger_data
+    assert len(logger_data["dut_jobs"]) == 1
+
+    dut_job = logger_data["dut_jobs"][0]
+    assert "dut_start_time" in dut_job
+    assert dut_job["dut_start_time"] != ""
+
+    try:
+        datetime.fromisoformat(dut_job["dut_start_time"])
+        assert True
+    except ValueError:
+        assert False
+
+
+# Test case for updating DUT submit time
+def test_update_dut_submit_time(custom_logger):
+    custom_time = "2023-11-09T02:37:06Z"
+    custom_logger.create_dut_job(status="pass")
+    custom_logger.update_dut_time("submit", custom_time)
+
+    logger_data = custom_logger.logger.data
+    assert "dut_jobs" in logger_data
+    assert len(logger_data["dut_jobs"]) == 1
+
+    dut_job = logger_data["dut_jobs"][0]
+    assert "dut_submit_time" in dut_job
+
+    try:
+        datetime.fromisoformat(dut_job["dut_submit_time"])
+        assert True
+    except ValueError:
+        assert False
+
+
+# Test case for updating DUT end time
+def test_update_dut_end_time(custom_logger):
+    custom_logger.create_dut_job(status="pass")
+    custom_logger.update_dut_time("end", None)
+
+    logger_data = custom_logger.logger.data
+    assert "dut_jobs" in logger_data
+    assert len(logger_data["dut_jobs"]) == 1
+
+    dut_job = logger_data["dut_jobs"][0]
+    assert "dut_end_time" in dut_job
+
+    try:
+        datetime.fromisoformat(dut_job["dut_end_time"])
+        assert True
+    except ValueError:
+        assert False
+
+
+# Test case for updating DUT time with invalid value
+def test_update_dut_time_invalid_value(custom_logger):
+    custom_logger.create_dut_job(status="pass")
+    with pytest.raises(
+        ValueError,
+        match="Error: Invalid argument provided for --update-dut-time. Use 'start', 'submit', 'end'.",
+    ):
+        custom_logger.update_dut_time("invalid_value", None)
+
+
+# Test case for close_dut_job
+def test_close_dut_job(custom_logger):
+    custom_logger.create_dut_job(status="pass")
+
+    custom_logger.create_job_phase("Phase1")
+    custom_logger.create_job_phase("Phase2")
+
+    custom_logger.close_dut_job()
+
+    logger_data = custom_logger.logger.data
+    assert "dut_jobs" in logger_data
+    assert len(logger_data["dut_jobs"]) == 1
+
+    dut_job = logger_data["dut_jobs"][0]
+    assert "dut_job_phases" in dut_job
+    dut_job_phases = dut_job["dut_job_phases"]
+
+    phase1 = dut_job_phases[0]
+    assert phase1["name"] == "Phase1"
+
+    try:
+        datetime.fromisoformat(phase1["start_time"])
+        assert True
+    except ValueError:
+        assert False
+
+    try:
+        datetime.fromisoformat(phase1["end_time"])
+        assert True
+    except ValueError:
+        assert False
+
+    phase2 = dut_job_phases[1]
+    assert phase2["name"] == "Phase2"
+
+    try:
+        datetime.fromisoformat(phase2["start_time"])
+        assert True
+    except ValueError:
+        assert False
+
+    try:
+        datetime.fromisoformat(phase2["end_time"])
+        assert True
+    except ValueError:
+        assert False
+
+
+# Test case for close
+def test_close(custom_logger):
+    custom_logger.create_dut_job(status="pass")
+
+    custom_logger.close()
+
+    logger_data = custom_logger.logger.data
+    assert "dut_jobs" in logger_data
+    assert len(logger_data["dut_jobs"]) == 1
+    assert "dut_attempt_counter" in logger_data
+    assert logger_data["dut_attempt_counter"] == len(logger_data["dut_jobs"])
+    assert "job_combined_status" in logger_data
+    assert logger_data["job_combined_status"] != ""
+
+    dut_job = logger_data["dut_jobs"][0]
+    assert "submitter_end_time" in dut_job
+    try:
+        datetime.fromisoformat(dut_job["submitter_end_time"])
+        assert True
+    except ValueError:
+        assert False
+
+
+# Test case for updating status to fail with a reason
+def test_update_status_fail_with_reason(custom_logger):
+    custom_logger.create_dut_job()
+
+    reason = "kernel panic"
+    custom_logger.update_status_fail(reason)
+
+    logger_data = custom_logger.logger.data
+    assert "dut_jobs" in logger_data
+    assert len(logger_data["dut_jobs"]) == 1
+
+    dut_job = logger_data["dut_jobs"][0]
+    assert "status" in dut_job
+    assert dut_job["status"] == "fail"
+    assert "dut_job_fail_reason" in dut_job
+    assert dut_job["dut_job_fail_reason"] == reason
+
+
+# Test case for updating status to fail without providing a reason
+def test_update_status_fail_without_reason(custom_logger):
+    custom_logger.create_dut_job()
+
+    custom_logger.update_status_fail()
+
+    # Check if the status is updated and fail reason is empty
+    logger_data = custom_logger.logger.data
+    assert "dut_jobs" in logger_data
+    assert len(logger_data["dut_jobs"]) == 1
+
+    dut_job = logger_data["dut_jobs"][0]
+    assert "status" in dut_job
+    assert dut_job["status"] == "fail"
+    assert "dut_job_fail_reason" in dut_job
+    assert dut_job["dut_job_fail_reason"] == ""
+
+
+# Test case for check_dut_timings with submission time earlier than start time
+def test_check_dut_timings_submission_earlier_than_start(custom_logger, caplog):
+    custom_logger.create_dut_job()
+
+    # Set submission time to be earlier than start time
+    custom_logger.update_dut_time("start", "2023-01-01T11:00:00")
+    custom_logger.update_dut_time("submit", "2023-01-01T12:00:00")
+
+    logger_data = custom_logger.logger.data
+    assert "dut_jobs" in logger_data
+    assert len(logger_data["dut_jobs"]) == 1
+
+    job = logger_data["dut_jobs"][0]
+
+    # Call check_dut_timings
+    custom_logger.check_dut_timings(job)
+
+    # Check if an error message is logged
+    assert "Job submission is happening before job start." in caplog.text
+
+
+# Test case for check_dut_timings with end time earlier than start time
+def test_check_dut_timings_end_earlier_than_start(custom_logger, caplog):
+    custom_logger.create_dut_job()
+
+    # Set end time to be earlier than start time
+    custom_logger.update_dut_time("end", "2023-01-01T11:00:00")
+    custom_logger.update_dut_time("start", "2023-01-01T12:00:00")
+
+    logger_data = custom_logger.logger.data
+    assert "dut_jobs" in logger_data
+    assert len(logger_data["dut_jobs"]) == 1
+
+    job = logger_data["dut_jobs"][0]
+
+    # Call check_dut_timings
+    custom_logger.check_dut_timings(job)
+
+    # Check if an error message is logged
+    assert "Job ended before it started." in caplog.text
+
+
+# Test case for check_dut_timings with valid timing sequence
+def test_check_dut_timings_valid_timing_sequence(custom_logger, caplog):
+    custom_logger.create_dut_job()
+
+    # Set valid timing sequence
+    custom_logger.update_dut_time("submit", "2023-01-01T12:00:00")
+    custom_logger.update_dut_time("start", "2023-01-01T12:30:00")
+    custom_logger.update_dut_time("end", "2023-01-01T13:00:00")
+
+    logger_data = custom_logger.logger.data
+    assert "dut_jobs" in logger_data
+    assert len(logger_data["dut_jobs"]) == 1
+
+    job = logger_data["dut_jobs"][0]
+
+    # Call check_dut_timings
+    custom_logger.check_dut_timings(job)
+
+    # Check that no error messages are logged
+    assert "Job submission is happening before job start." not in caplog.text
+    assert "Job ended before it started." not in caplog.text
+
+
+@pytest.mark.parametrize("start_time,end_time,expected", [
+    ("2023-01-01T12:00:00", "2023-01-01T12:30:00", "0:30:00"),
+    ("invalid", "2023-01-01T12:30:00", ""),
+    ("", "", ""),
+])
+def test_get_duration_time(custom_logger, start_time, end_time, expected):
+    duration = custom_logger.get_duration_time(start_time, end_time)
+    assert duration == expected
+
+
+# Test case for DUT job duration tracking
+def test_dut_job_duration(custom_logger):
+    custom_logger.create_dut_job(status="pass")
+
+    # Set start time
+    custom_logger.update_dut_time("start", "2023-01-01T12:00:00")
+    logger_data = custom_logger.logger.data
+    dut_job = logger_data["dut_jobs"][0]
+    assert dut_job["dut_duration_time"] == ""
+
+    # Set end time
+    custom_logger.update_dut_time("end", "2023-01-01T12:30:00")
+    logger_data = custom_logger.logger.data
+    dut_job = logger_data["dut_jobs"][0]
+    assert dut_job["dut_duration_time"] == "0:30:00"
+
+
+# Test case for job phase duration tracking
+def test_job_phase_duration(custom_logger):
+    custom_logger.create_dut_job(status="pass")
+
+    # Create first phase
+    custom_logger.create_job_phase("Phase1")
+    logger_data = custom_logger.logger.data
+    job = logger_data["dut_jobs"][0]
+    phase = job["dut_job_phases"][0]
+    assert phase["duration_time"] == ""
+
+    # Create second phase (which should close the first phase)
+    custom_logger.create_job_phase("Phase2")
+    logger_data = custom_logger.logger.data
+    job = logger_data["dut_jobs"][0]
+
+    # First phase should have duration
+    phase1 = job["dut_job_phases"][0]
+    assert phase1["duration_time"] != ""
+
+    # Verify duration format
+    start = datetime.fromisoformat(phase1["start_time"])
+    end = datetime.fromisoformat(phase1["end_time"])
+    duration = end - start
+    assert phase1["duration_time"] == str(duration)
+
+    # Second phase should have empty duration
+    phase2 = job["dut_job_phases"][1]
+    assert phase2["duration_time"] == ""
+
+
+# Test case for closing job with phases
+def test_close_job_with_phases_duration(custom_logger):
+    custom_logger.create_dut_job(status="pass")
+    custom_logger.create_job_phase("Phase1")
+    custom_logger.close_dut_job()
+
+    logger_data = custom_logger.logger.data
+    job = logger_data["dut_jobs"][0]
+    phase = job["dut_job_phases"][0]
+
+    assert phase["duration_time"] != ""
+
+    # Verify duration format
+    start = datetime.fromisoformat(phase["start_time"])
+    end = datetime.fromisoformat(phase["end_time"])
+    duration = end - start
+    assert phase["duration_time"] == str(duration)

bin/ci/test/test_gantt_chart.py

@@ -0,0 +1,201 @@
+from contextlib import suppress
+from datetime import datetime, timedelta
+from unittest import mock
+from unittest.mock import MagicMock, patch
+
+import ci_post_gantt
+import pytest
+from ci_gantt_chart import generate_gantt_chart
+from ci_post_gantt import Gitlab, MockGanttExit
+
+
+def create_mock_job(
+    name, id, status, created_at, queued_duration, started_at, finished_at=None
+):
+    mock_job = MagicMock()
+    mock_job.name = name
+    mock_job.status = status
+    mock_job.id = id
+    mock_job.created_at = created_at
+    mock_job.queued_duration = queued_duration
+    mock_job.started_at = started_at
+    mock_job.finished_at = finished_at
+    return mock_job
+
+
+@pytest.fixture
+def fake_pipeline():
+    current_time = datetime.fromisoformat("2024-12-17 23:54:13.940091+00:00")
+    created_at = current_time - timedelta(minutes=10)
+
+    job1 = create_mock_job(
+        name="job1",
+        id="1",
+        status="success",
+        created_at=created_at.isoformat(),
+        queued_duration=1,  # seconds
+        started_at=(created_at + timedelta(seconds=2)).isoformat(),
+        finished_at=(created_at + timedelta(minutes=1)).isoformat(),
+    )
+
+    mock_pipeline = MagicMock()
+    mock_pipeline.web_url = "https://gitlab.freedesktop.org/mesa/mesa/-/pipelines/9999"
+    mock_pipeline.duration = 600  # Total pipeline duration in seconds
+    mock_pipeline.created_at = created_at.isoformat()
+    mock_pipeline.yaml_errors = False
+    mock_pipeline.jobs.list.return_value = [job1]
+    return mock_pipeline
+
+
+def test_generate_gantt_chart(fake_pipeline):
+    fig = generate_gantt_chart(fake_pipeline)
+
+    fig_dict = fig.to_dict()
+    assert "data" in fig_dict
+
+    # Extract all job names from the "y" axis in the Gantt chart data
+    all_job_names = set()
+    for trace in fig_dict["data"]:
+        if "y" in trace:
+            all_job_names.update(trace["y"])
+
+    assert any(
+        "job1" in job for job in all_job_names
+    ), "job1 should be present in the Gantt chart"
+
+
+def test_ci_timeout(fake_pipeline):
+    fig = generate_gantt_chart(fake_pipeline, ci_timeout=1)
+
+    fig_dict = fig.to_dict()
+
+    timeout_line = None
+    for shape in fig_dict.get("layout", {}).get("shapes", []):
+        if shape.get("line", {}).get("dash") == "dash":
+            timeout_line = shape
+            break
+
+    assert timeout_line is not None, "Timeout line should exist in the Gantt chart"
+    timeout_x = timeout_line["x0"]
+
+    # Check that the timeout line is 1 minute after the pipeline creation time
+    pipeline_created_at = datetime.fromisoformat(fake_pipeline.created_at)
+    expected_timeout = pipeline_created_at + timedelta(minutes=1)
+    assert (
+        timeout_x == expected_timeout
+    ), f"Timeout should be at {expected_timeout}, got {timeout_x}"
+
+
+def test_marge_bot_user_id():
+    with patch("ci_post_gantt.Gitlab") as MockGitlab:
+        mock_gitlab_instance = MagicMock(spec=Gitlab)
+        mock_gitlab_instance.users = MagicMock()
+        MockGitlab.return_value = mock_gitlab_instance
+
+        marge_bot_user_id = 12345
+        ci_post_gantt.main("fake_token", None, marge_bot_user_id)
+        mock_gitlab_instance.users.get.assert_called_once_with(marge_bot_user_id)
+
+
+def test_project_ids():
+    current_time = datetime.now()
+    project_id_1 = 176
+    event_1 = MagicMock()
+    event_1.project_id = project_id_1
+    event_1.created_at = (current_time - timedelta(days=1)).isoformat()
+    event_1.note = {"body": f"Event for project {project_id_1}"}
+
+    project_id_2 = 166
+    event_2 = MagicMock()
+    event_2.project_id = project_id_2
+    event_2.created_at = (current_time - timedelta(days=2)).isoformat()
+    event_2.note = {"body": f"Event for project {project_id_2}"}
+
+    with patch("ci_post_gantt.Gitlab") as MockGitlab:
+        mock_user = MagicMock()
+        mock_user.events = MagicMock()
+        mock_user.events.list.return_value = [event_1, event_2]
+
+        mock_gitlab_instance = MagicMock(spec=Gitlab)
+        mock_gitlab_instance.users = MagicMock()
+        mock_gitlab_instance.users.get.return_value = mock_user
+        MockGitlab.return_value = mock_gitlab_instance
+
+        last_event_date = (current_time - timedelta(days=3)).isoformat()
+
+        # Test a single project id
+        ci_post_gantt.main("fake_token", last_event_date)
+        marge_bot_single_project_scope = [
+            event.note["body"]
+            for event in mock_user.events.list.return_value
+            if event.project_id == project_id_1
+        ]
+        assert f"Event for project {project_id_1}" in marge_bot_single_project_scope
+        assert f"Event for project {project_id_2}" not in marge_bot_single_project_scope
+
+        # Test multiple project ids
+        ci_post_gantt.main(
+            "fake_token", last_event_date, 9716, [project_id_1, project_id_2]
+        )
+
+        marge_bot_multiple_project_scope = [
+            event.note["body"] for event in mock_user.events.list.return_value
+        ]
+        assert f"Event for project {project_id_1}" in marge_bot_multiple_project_scope
+        assert f"Event for project {project_id_2}" in marge_bot_multiple_project_scope
+
+
+def test_add_gantt_after_pipeline_message():
+    current_time = datetime.now()
+
+    plain_url = "https://gitlab.freedesktop.org/mesa/mesa/-/pipelines/12345"
+    plain_message = (
+        f"I couldn't merge this branch: CI failed! See pipeline {plain_url}."
+    )
+    event_plain = MagicMock()
+    event_plain.project_id = 176
+    event_plain.created_at = (current_time - timedelta(days=1)).isoformat()
+    event_plain.note = {"body": plain_message}
+
+    summary_url = "https://gitlab.freedesktop.org/mesa/mesa/-/pipelines/99999"
+    summary_message = (
+        "I couldn't merge this branch: "
+        f"CI failed! See pipeline {summary_url}.<br>There were problems with job:"
+        "[lavapipe](https://gitlab.freedesktop.org/mesa/mesa/-/jobs/68141218)<details><summary> "
+        "3 crashed tests</summary>dEQP-VK.ray_query.builtin.instancecustomindex.frag.aabbs,Crash<br>dEQP"
+        "-VK.ray_query.builtin.objecttoworld.frag.aabbs,Crash<br>dEQP-VK.sparse_resources.shader_intrinsics."
+        "2d_array_sparse_fetch.g16_b16r16_2plane_444_unorm.11_37_3_nontemporal,Crash<br></details>"
+    )
+    event_with_summary = MagicMock()
+    event_with_summary.project_id = 176
+    event_with_summary.created_at = (current_time - timedelta(days=1)).isoformat()
+    event_with_summary.note = {"body": summary_message}
+
+    with patch("ci_post_gantt.Gitlab") as MockGitlab, patch(
+        "ci_post_gantt.get_gitlab_pipeline_from_url", return_value=None
+    ) as mock_get_gitlab_pipeline_from_url:
+
+        def safe_mock(*args, **kwargs):
+            with suppress(TypeError):
+                raise MockGanttExit("Exiting for test purposes")
+
+        mock_get_gitlab_pipeline_from_url.side_effect = safe_mock
+
+        mock_user = MagicMock()
+        mock_user.events = MagicMock()
+        mock_user.events.list.return_value = [event_plain, event_with_summary]
+
+        mock_gitlab_instance = MagicMock(spec=Gitlab)
+        mock_gitlab_instance.users = MagicMock()
+        mock_gitlab_instance.users.get.return_value = mock_user
+        MockGitlab.return_value = mock_gitlab_instance
+
+        last_event_date = (current_time - timedelta(days=3)).isoformat()
+        ci_post_gantt.main("fake_token", last_event_date, 12345)
+        mock_get_gitlab_pipeline_from_url.assert_has_calls(
+            [
+                mock.call(mock_gitlab_instance, plain_url),
+                mock.call(mock_gitlab_instance, summary_url),
+            ],
+            any_order=True,
+        )

bin/ci/test/test_marge_queue.py

@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+# Copyright © 2025 Collabora Ltd.
+# Authors:
+#   Sergi Blanch Torne <[email protected]>
+#
+# SPDX-License-Identifier: MIT
+
+from dataclasses import dataclass
+from datetime import timedelta
+
+from unittest import TestCase
+from typing import Iterable
+
+from marge_queue import MargeMergeRequest, MargeQueue, get_merge_queue
+
+
+DATA = {
+    37564: {
+        # sample of a Merge Request rejected by marge and later reassigned to marge
+        "updated_at": "2025-09-25T10:59:27.144Z",
+        "title": "etnaviv: Fix util_blitter_save_so_targets(..) call",
+        "notes": [
+            # Those notes must be listed in reverse order
+            # because the list is requested sort="desc"
+            {
+                "body": "assigned to @marge-bot and unassigned @austriancoder",
+                "created_at": "2025-09-25T11:44:01.672Z",
+            },
+            {
+                "body": "added 1 commit",
+                "created_at": "2025-09-25T10:43:43.490Z",
+            },
+            {
+                "body": "This branch couldn't be merged:",
+                "created_at": "2025-09-25T10:42:27.817Z",
+            },
+            {
+                "body": "assigned to @austriancoder and unassigned @marge-bot",
+                "created_at": "2025-09-25T10:42:15.404Z",
+            },
+            {
+                "body": "added 3 commits",
+                "created_at": "2025-09-25T10:38:57.332Z",
+            },
+            {
+                "body": "assigned to @marge-bot",
+                "created_at": "2025-09-25T10:38:03.993Z",
+            },
+        ],
+    },
+    37560: {
+        # sample of a Merge Request assigned to marge between the two
+        # assignments of the previous sample. This is meant to show that
+        # this goes first in the queue, and it doesn't confuses the sequence.
+        "updated_at": "2025-09-25T11:38:08.420Z",
+        "title": "tu: limit query pool types logged into RMV",
+        "notes": [
+            {
+                "body": "assigned to @marge-bot",
+                "created_at": "2025-09-25T10:47:40.040Z",
+            },
+            {
+                "body": "added 1 commit",
+                "created_at": "2025-09-25T10:46:42.113Z",
+            },
+            {
+                "body": "mentioned in issue #13970",
+                "created_at": "2025-09-25T10:45:35.566Z",
+            },
+            {
+                "body": "Rb",
+                "created_at": "2025-09-25T08:25:51.527Z",
+            },
+        ],
+    },
+}
+
+
+@dataclass
+class ProjectMergeRequestNote:
+    body: str
+    created_at: str
+
+
+@dataclass
+class ProjectMergeRequestNoteManager:
+    """
+    Mock the gitlab.v4.objects.ProjectMergeRequestNoteManager
+    """
+    iid: int
+
+    def list(self, *args, **kwargs) -> Iterable:
+        for note in DATA[self.iid]["notes"]:
+            yield ProjectMergeRequestNote(**note)
+
+
+@dataclass
+class ProjectMergeRequest:
+    """
+    Mock the gitlab.v4.objects.ProjectMergeRequest
+    """
+    iid: int
+
+    @property
+    def updated_at(self) -> str:
+        return DATA[self.iid]["updated_at"]
+
+    @property
+    def web_url(self) -> str:
+        return f"https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/{self.iid}"
+
+    @property
+    def title(self) -> str:
+        return DATA[self.iid]["title"]
+
+    @property
+    def notes(self) -> ProjectMergeRequestNoteManager:
+        return ProjectMergeRequestNoteManager(self.iid)
+
+
+@dataclass
+class ProjectMergeRequestManager:
+    """
+    Mock the gitlab.v4.objects.ProjectMergeRequestManager
+    """
+    def list(self, *args, **kwargs) -> Iterable:
+        for mr in DATA.keys():
+            yield ProjectMergeRequest(mr)
+
+
+@dataclass
+class Project:
+    """
+    Mock the gitlab.v4.objects.Project
+    """
+    @property
+    def mergerequests(self) -> ProjectMergeRequestManager:
+        return ProjectMergeRequestManager()
+
+
+class MargeMergeTestCase(TestCase):
+    def test_queue(self) -> None:
+        queue = MargeQueue()
+        for mr_id in DATA.keys():
+            gl_mr_mock_obj = ProjectMergeRequest(mr_id)
+            marge_mr = MargeMergeRequest(gl_mr_mock_obj)
+
+            # for each element in the data sample, test the MargeMergeRequest construction
+            self.assertEqual(marge_mr.title, gl_mr_mock_obj.title)
+            self.assertEqual(marge_mr.web_url, gl_mr_mock_obj.web_url)
+            self.assertIsInstance(marge_mr.time_enqueued, timedelta)
+
+            queue.append(marge_mr)
+
+        self.assertEqual(queue.n_merge_requests_enqueued, 2)
+
+
+    def test_get_merge_queue(self) -> None:
+        project = Project()
+        queue = get_merge_queue(project)
+
+        self.assertEqual(queue.n_merge_requests_enqueued, 2)
+        self.assertEqual(len(queue.sorted_queue), 2)
+        # They are in reverse order because of the reassignment to marge
+        self.assertEqual(queue.sorted_queue[0].id, list(DATA.keys())[1])
+        self.assertEqual(queue.sorted_queue[1].id, list(DATA.keys())[0])
+        self.assertEqual(len(queue.undetermined), 0)
+
+        for marge_mr in queue.sorted_queue:
+            mr_id = marge_mr.id
+            self.assertEqual(marge_mr.title, DATA[mr_id]["title"])
+            self.assertEqual(
+                marge_mr.web_url,
+                f"https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/{mr_id}"
+            )
+            self.assertIsInstance(marge_mr.time_enqueued, timedelta)

bin/ci/test/test_pipeline_message.py

@@ -0,0 +1,309 @@
+import logging
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from pipeline_message import (
+    get_failed_test_summary_message,
+    get_problem_jobs,
+    get_trace_failures,
+    main,
+    process_problem_jobs,
+    search_job_log_for_errors,
+    sort_failed_tests_by_status,
+    unexpected_improvements,
+)
+
+
+def test_get_problem_jobs():
+    jobs = [
+        {"stage": "build", "status": "failed"},
+        {"stage": "test", "status": "canceled"},
+        {"stage": "postmerge", "status": "failed"},
+        {"stage": "performance", "status": "failed"},
+        {"stage": "deploy", "status": "failed"},
+    ]
+
+    problem_jobs = get_problem_jobs(jobs)
+
+    assert len(problem_jobs) == 3
+    assert problem_jobs[0]["stage"] == "build"
+    assert problem_jobs[1]["stage"] == "test"
+    assert problem_jobs[2]["stage"] == "deploy"
+
+
+def test_sort_failed_tests_by_status():
+    failures_csv = """\
+Test1,UnexpectedImprovement
+Test2,Fail
+Test3,Crash
+Test4,Timeout
+Test5,Fail
+Test6,UnexpectedImprovement
+"""
+    sorted_tests = sort_failed_tests_by_status(failures_csv)
+
+    assert len(sorted_tests["unexpected_improvements"]) == 2
+    assert len(sorted_tests["fails"]) == 2
+    assert len(sorted_tests["crashes"]) == 1
+    assert len(sorted_tests["timeouts"]) == 1
+
+    assert sorted_tests["unexpected_improvements"] == [
+        "Test1,UnexpectedImprovement",
+        "Test6,UnexpectedImprovement",
+    ]
+    assert sorted_tests["fails"] == ["Test2,Fail", "Test5,Fail"]
+    assert sorted_tests["crashes"] == ["Test3,Crash"]
+    assert sorted_tests["timeouts"] == ["Test4,Timeout"]
+
+
+def test_get_failed_test_summary_message():
+    failed_test_array = {
+        "unexpected_improvements": [
+            "test1 UnexpectedImprovement",
+            "test2 UnexpectedImprovement",
+        ],
+        "fails": ["test3 Fail", "test4 Fail", "test5 Fail"],
+        "crashes": ["test6 Crash"],
+        "timeouts": [],
+    }
+
+    summary_message = get_failed_test_summary_message(failed_test_array)
+
+    assert "<summary>" in summary_message
+    assert "2 improved tests" in summary_message
+    assert "3 failed tests" in summary_message
+    assert "1 crashed test" in summary_message
+    assert "</summary>" in summary_message
+
+
+def test_unexpected_improvements():
+    message = "<summary>"
+    failed_test_array = {
+        "unexpected_improvements": ["test_improvement_1", "test_improvement_2"],
+        "fails": [],
+        "crashes": [],
+        "timeouts": [],
+    }
+    result = unexpected_improvements(failed_test_array)
+    assert result == " 2 improved tests", f"Unexpected result: {result}"
+
+
+@pytest.mark.asyncio
+@patch("pipeline_message.get_pipeline_status", new_callable=AsyncMock)
+async def test_gitlab_api_failure(mock_get_pipeline_status):
+    mock_get_pipeline_status.side_effect = Exception("GitLab API not responding")
+    message = await main("1234567")
+    assert message == ""
+
+
+@pytest.mark.asyncio
+async def test_no_message_when_pipeline_not_failed():
+    project_id = "176"
+    pipeline_id = "12345"
+
+    with patch(
+        "pipeline_message.get_pipeline_status", new_callable=AsyncMock
+    ) as mock_get_pipeline_status:
+        mock_get_pipeline_status.return_value = "success"
+
+        message = await main(pipeline_id, project_id)
+        assert (
+            message == ""
+        ), f"Expected no message for successful pipeline, but got: {message}"
+
+
+@pytest.mark.asyncio
+async def test_single_problem_job_not_summarized():
+    session = AsyncMock()
+    project_id = "176"
+    problem_jobs = [
+        {
+            "id": 1234,
+            "name": "test-job",
+            "web_url": "http://example.com/job/1234",
+            "status": "canceled",
+        }
+    ]
+
+    mock_response = AsyncMock()
+    mock_response.status = 200
+    mock_response.text.return_value = ""  # Empty CSV response for test
+    session.get.return_value = mock_response
+
+    message = await process_problem_jobs(session, project_id, problem_jobs)
+
+    assert "summary" not in message
+    assert "[test-job](http://example.com/job/1234)" in message
+
+
+@pytest.mark.asyncio
+@patch("pipeline_message.get_project_json", new_callable=AsyncMock)
+@patch("pipeline_message.aiohttp.ClientSession", autospec=True)
+async def test_get_trace_failures_no_response(
+    mock_client_session_cls, mock_get_project_json, caplog
+):
+    caplog.set_level(logging.DEBUG)
+    namespace = "mesa"
+    mock_get_project_json.return_value = {"path": namespace}
+
+    mock_get = AsyncMock()
+    mock_get.status = 404
+
+    mock_session_instance = mock_client_session_cls.return_value
+    mock_session_instance.get.return_value = mock_get
+
+    job_id = 12345678
+    job = {"id": job_id}
+    url = await get_trace_failures(mock_session_instance, "176", job)
+
+    assert url == ""
+
+    expected_log_message = f"No response from: https://mesa.pages.freedesktop.org/-/{namespace}/-/jobs/{job_id}/artifacts/results/summary/problems.html"
+    assert any(expected_log_message in record.message for record in caplog.records)
+
+
+@pytest.mark.asyncio
+@patch("pipeline_message.get_job_log", new_callable=AsyncMock)
+async def test_search_job_log_for_errors(mock_get_job_log):
+    session = AsyncMock()
+    project_id = "176"
+    job = {"id": 12345}
+
+    job_log = r"""
+error_msg: something useful
+[0m15:41:36.102:                GL_KHR_no_error GL_KHR_texture_compression_astc_sliced_3d
+1 error generated
+3 errors generated.
+-- Looking for strerror_r - found
+-- Looking for strerror_s - not found
+[49/176] Building CXX object lib/Support/CMakeFiles/LLVMSupport.dir/ErrorHandling.cpp.o
+[127/2034] Building C object lib/Support/CMakeFiles/LLVMSupport.dir/regerror.c.o
+-- Performing Test HAS_WERROR_GLOBAL_CTORS
+-- Performing Test C_SUPPORTS_WERROR_UNGUARDED_AVAILABILITY_NEW - Success
+-- Performing Test LLVM_LIBSTDCXX_SOFT_ERROR
+error aborting
+error_msg      : None
+error_type     : Job
+[0Ksection_end:1734694783:job_data
+[0K
+[0m11:39:43.438: [1mFinished executing LAVA job in the attempt #3 [0m
+[0Ksection_end:1734694783:lava_submit
+[0K
+[0;31m[01:54] ERROR: lava_submit: ret code: 1 [0m
+
+[0;31m[01:54] ERROR: unknown-section: ret code: 1 [0m
+section_end:1734694783:step_script
+[0Ksection_start:1734694783:after_script
+[0K[0K[36;1mRunning after_script[0;m[0;m
+[32;1mRunning after script...[0;m
+[32;1m$ curl -L --retry 4 -f --retry-all-errors --retry-delay 60 -s "https://" | tar --warning=no-timestamp --zstd -x[0;m
+zstd: /*stdin*\: unexpected end of file # noqa: W605
+tar: Child returned status 1
+tar: Error is not recoverable: exiting now
+section_end:1734695025:after_script
+[0K[0;33mWARNING: after_script failed, but job will continue unaffected: exit code 1[0;m
+section_start:1734695025:upload_artifacts_on_failure
+[0K[0K[36;1mUploading artifacts for failed job[0;m[0;m
+[32;1mUploading artifacts...[0;m
+results/: found 11 matching artifact files and directories[0;m
+Uploading artifacts as "archive" to coordinator... 201 Created[0;m  id[0;m=68509685 responseStatus[0;m=201 Created token[0;m=glcbt-64
+[32;1mUploading artifacts...[0;m
+[0;33mWARNING: results/junit.xml: no matching files. Ensure that the artifact path is relative to the working directory (/builds/mesa/mesa)[0;m
+[31;1mERROR: No files to upload                         [0;m
+section_end:1734695027:upload_artifacts_on_failure
+[0Ksection_start:1734695027:cleanup_file_variables
+[0K[0K[36;1mCleaning up project directory and file based variables[0;m[0;m
+section_end:1734695027:cleanup_file_variables
+[0K[31;1mERROR: Job failed: exit code 1
+[0;m
+[0;m
+    """
+
+    mock_get_job_log.return_value = job_log
+
+    error_message = await search_job_log_for_errors(session, project_id, job)
+    assert "something useful" in error_message
+
+
+@pytest.mark.asyncio
+@patch("pipeline_message.get_job_log", new_callable=AsyncMock)
+async def test_search_job_log_for_fatal_errors(mock_get_job_log):
+    session = AsyncMock()
+    project_id = "176"
+    job = {"id": 12345}
+
+    job_log = r"""
+[0m15:41:36.105: [15:41:31.951] fatal: something fatal
+Uploading artifacts as "archive" to coordinator... 201 Created[0;m  id[0;m=68509685 responseStatus[0;m=201 Created token[0;m=glcbt-64
+[32;1mUploading artifacts...[0;m
+[0;33mWARNING: results/junit.xml: no matching files. Ensure that the artifact path is relative to the working directory (/builds/mesa/mesa)[0;m
+[31;1mERROR: No files to upload                         [0;m
+section_end:1734695027:upload_artifacts_on_failure
+[0Ksection_start:1734695027:cleanup_file_variables
+[0K[0K[36;1mCleaning up project directory and file based variables[0;m[0;m
+section_end:1734695027:cleanup_file_variables
+[0K[31;1mERROR: Job failed: exit code 1
+[0;m
+[0;m
+    """
+
+    mock_get_job_log.return_value = job_log
+
+    error_message = await search_job_log_for_errors(session, project_id, job)
+    assert "something fatal" in error_message
+
+
+@pytest.mark.asyncio
+@patch("pipeline_message.get_job_log", new_callable=AsyncMock)
+async def test_search_job_log_for_errors_but_find_none(mock_get_job_log):
+    session = AsyncMock()
+    project_id = "176"
+    job = {"id": 12345}
+
+    job_log = r"""
+[0KRunning with gitlab-runner 17.4.0 (b92ee590)[0;m
+[0K  on fdo-equinix-m3l-30-placeholder_63 XmDXAt7xd, system ID: s_785ae19292ea[0;m
+section_start:1734736110:prepare_executor
+[0K[0K[36;1mPreparing the "docker" executor[0;m[0;m
+[0KUsing Docker executor with image registry.freedesktop.org/mesa/mesa/debian
+[0KAuthenticating with credentials from job payload (GitLab Registry)[0;m
+[0KPulling docker image registry.freedesktop.org/mesa/mesa/debian/x86_64_pyuti
+[0KUsing docker image sha256:ebc7b3fe89be4d390775303adddb33539c235a2663165d78d
+[0Ksection_start:1734736124:prepare_script
+[0K[0K[36;1mPreparing environment[0;m[0;m
+Running on runner-xmdxat7xd-project-23076-concurrent-1 via fdo-equinix-m3l-30...
+section_end:1734736125:prepare_script
+[0Ksection_start:1734736125:get_sources
+[0K[0K[36;1mGetting source from Git repository[0;m[0;m
+[32;1m$ /host/bin/curl -s -L --cacert /host/ca-certificates.crt --retry 4 -f --retry-delay 60 https://gitlab.
+Checking if the user of the pipeline is allowed...
+Checking if the job's project is part of a well-known group...
+Checking if the job is part of an official MR pipeline...
+Thank you for contributing to freedesktop.org
+Running pre-clone script: 'set -o xtrace
+wget -q -O download-git-cache.sh https://gitlab.freedesktop.org/mesa/mesa/-/raw/0d43b4cba639b809ad0e08a065ce01846e262249/.gitlab-ci/download-git-cache.sh
+bash download-git-cache.sh
+rm download-git-cache.sh
+[31;1m errors
+[0K[31;1mERROR:
+[31;1m error
+[31;1m Here is a blank error:
+/builds/mesa/mesa/bin/ci/test/test_pipeline_message.py:162: AssertionError
+Uploading artifacts as "archive" to coordinator... 201 Created[0;m  id[0;m=68509685 responseStatus[0;m=201 Created token[0;m=glcbt-64
+[32;1mUploading artifacts...[0;m
+[0;33mWARNING: results/junit.xml: no matching files. Ensure that the artifact path is relative to the working directory (/builds/mesa/mesa)[0;m
+[31;1mERROR: No files to upload                         [0;m
+section_end:1734695027:upload_artifacts_on_failure
+[0Ksection_start:1734695027:cleanup_file_variables
+[0K[0K[36;1mCleaning up project directory and file based variables[0;m[0;m
+section_end:1734695027:cleanup_file_variables
+[0K[31;1mERROR: Job failed: exit code 1
+[0;m
+[0;m
+    """
+
+    mock_get_job_log.return_value = job_log
+
+    error_message = await search_job_log_for_errors(session, project_id, job)
+    assert error_message == "", f"Unexpected error message: {error_message}"

bin/ci/test/test_structured_logger.py

@@ -0,0 +1,182 @@
+import json
+from pathlib import Path
+
+import pytest
+from mock import MagicMock, patch
+from structured_logger import (
+    AutoSaveDict,
+    CSVStrategy,
+    JSONStrategy,
+    StructuredLogger,
+    YAMLStrategy,
+)
+
+
+@pytest.fixture(params=[CSVStrategy, JSONStrategy, YAMLStrategy])
+def strategy(request):
+    return request.param
+
+
+@pytest.fixture
+def file_extension(strategy):
+    if strategy == CSVStrategy:
+        return "csv"
+    elif strategy == JSONStrategy:
+        return "json"
+    elif strategy == YAMLStrategy:
+        return "yaml"
+
+
+@pytest.fixture
+def tmp_file(tmp_path):
+    return tmp_path / "test.json"
+
+
+def test_guess_strategy_from_file(tmp_path, strategy, file_extension):
+    file_name = tmp_path / f"test_guess.{file_extension}"
+    Path(file_name).touch()
+    guessed_strategy = StructuredLogger.guess_strategy_from_file(file_name)
+    assert isinstance(guessed_strategy, strategy)
+
+
+def test_get_strategy(strategy, file_extension):
+    result = StructuredLogger.get_strategy(file_extension)
+    assert isinstance(result, strategy)
+
+
+def test_invalid_file_extension(tmp_path):
+    file_name = tmp_path / "test_invalid.xyz"
+    Path(file_name).touch()
+
+    with pytest.raises(ValueError, match="Unknown strategy for: xyz"):
+        StructuredLogger.guess_strategy_from_file(file_name)
+
+
+def test_non_existent_file(tmp_path, strategy, file_extension):
+    file_name = tmp_path / f"non_existent.{file_extension}"
+    logger = StructuredLogger(file_name, strategy())
+
+    assert logger.file_path.exists()
+    assert "_timestamp" in logger._data
+
+
+@pytest.fixture
+def structured_logger_module():
+    with patch.dict("sys.modules", {"polars": None, "ruamel.yaml": None}):
+        import importlib
+
+        import structured_logger
+
+        importlib.reload(structured_logger)
+        yield structured_logger
+
+
+def test_missing_csv_library(tmp_path, structured_logger_module):
+    with pytest.raises(RuntimeError, match="Can't parse CSV files. Missing library"):
+        structured_logger_module.CSVStrategy()
+
+
+def test_missing_yaml_library(tmp_path, structured_logger_module):
+    with pytest.raises(RuntimeError, match="Can't parse YAML files. Missing library"):
+        structured_logger_module.YAMLStrategy()
+
+
+def test_autosavedict_setitem():
+    save_callback = MagicMock()
+    d = AutoSaveDict(save_callback=save_callback)
+    d["key"] = "value"
+    assert d["key"] == "value"
+    save_callback.assert_called_once()
+
+
+def test_autosavedict_delitem():
+    save_callback = MagicMock()
+    d = AutoSaveDict({"key": "value"}, save_callback=save_callback)
+    del d["key"]
+    assert "key" not in d
+    save_callback.assert_called_once()
+
+
+def test_autosavedict_pop():
+    save_callback = MagicMock()
+    d = AutoSaveDict({"key": "value"}, save_callback=save_callback)
+    result = d.pop("key")
+    assert result == "value"
+    assert "key" not in d
+    save_callback.assert_called_once()
+
+
+def test_autosavedict_update():
+    save_callback = MagicMock()
+    d = AutoSaveDict({"key": "old_value"}, save_callback=save_callback)
+    d.update({"key": "new_value"})
+    assert d["key"] == "new_value"
+    save_callback.assert_called_once()
+
+
+def test_structured_logger_setitem(tmp_file):
+    logger = StructuredLogger(tmp_file, JSONStrategy())
+    logger.data["field"] = "value"
+
+    with open(tmp_file, "r") as f:
+        data = json.load(f)
+
+    assert data["field"] == "value"
+
+
+def test_structured_logger_set_recursive(tmp_file):
+    logger = StructuredLogger(tmp_file, JSONStrategy())
+    logger.data["field"] = {"test": True}
+    other = logger.data["field"]
+    other["late"] = True
+
+    with open(tmp_file, "r") as f:
+        data = json.load(f)
+
+    assert data["field"]["test"]
+    assert data["field"]["late"]
+
+
+def test_structured_logger_set_list(tmp_file):
+    logger = StructuredLogger(tmp_file, JSONStrategy())
+    logger.data["field"] = [True]
+    other = logger.data["field"]
+    other.append(True)
+
+    with open(tmp_file, "r") as f:
+        data = json.load(f)
+
+    assert data["field"][0]
+    assert data["field"][1]
+
+
+def test_structured_logger_delitem(tmp_file):
+    logger = StructuredLogger(tmp_file, JSONStrategy())
+    logger.data["field"] = "value"
+    del logger.data["field"]
+
+    with open(tmp_file, "r") as f:
+        data = json.load(f)
+
+    assert "field" not in data
+
+
+def test_structured_logger_pop(tmp_file):
+    logger = StructuredLogger(tmp_file, JSONStrategy())
+    logger.data["field"] = "value"
+    logger.data.pop("field")
+
+    with open(tmp_file, "r") as f:
+        data = json.load(f)
+
+    assert "field" not in data
+
+
+def test_structured_logger_update(tmp_file):
+    logger = StructuredLogger(tmp_file, JSONStrategy())
+    logger.data.update({"field": "value"})
+
+    with open(tmp_file, "r") as f:
+        data = json.load(f)
+
+    assert data["field"] == "value"

bin/ci/test/test_update_tag.py

@@ -0,0 +1,391 @@
+import sys
+import subprocess
+import yaml
+import pytest
+
+from textwrap import dedent
+from unittest.mock import patch
+
+from bin.ci.update_tag import (
+    from_component_to_build_tag,
+    filter_components,
+    from_component_to_tag_var,
+    from_script_name_to_component,
+    from_script_name_to_tag_var,
+    load_image_tags_yaml,
+    update_image_tag_in_yaml,
+    run_build_script,
+    main,
+    PATHS,
+)
+
+
+@pytest.fixture
+def temp_image_tags_file(tmp_path, monkeypatch):
+    """
+    Fixture that creates a temporary YAML file (to simulate conditional_tags)
+    and updates the global variable accordingly.
+    """
+    # Create a temporary test file
+    temp_file = tmp_path / "conditional-build-image-tags.yml"
+    temp_file.write_text(yaml.dump({"variables": {}}))
+
+    # Create a copy of the original PATHS values
+    original_conditional_tags = PATHS.conditional_tags
+
+    # Patch the PATHS instance with our test file
+    PATHS.conditional_tags = temp_file
+
+    # Yield the test file for the test to use
+    yield temp_file
+
+    # Restore the original value after the test
+    PATHS.conditional_tags = original_conditional_tags
+
+
+@pytest.fixture
+def temp_container_ci_file(tmp_path, monkeypatch):
+    """
+    Fixture that creates a temporary container CI file (to simulate container_ci)
+    and updates the global variable accordingly.
+    """
+    # Create a temporary test file
+    temp_file = tmp_path / "container-ci.yml"
+    temp_file.touch()
+
+    # Create a copy of the original PATHS values
+    original_container_ci = PATHS.container_ci
+
+    # Patch the PATHS instance with our test file
+    PATHS.container_ci = temp_file
+
+    # Yield the test file for the test to use
+    yield temp_file
+
+    # Restore the original value after the test
+    PATHS.container_ci = original_container_ci
+
+
+@pytest.fixture
+def mock_container_dir(tmp_path, monkeypatch):
+    """
+    Fixture that creates a dummy container directory and build script.
+    """
+    # Create a dummy container directory
+    container_dir = tmp_path / "container"
+    container_dir.mkdir()
+
+    # Create a dummy setup-test-env.sh file
+    dummy_setup_path = tmp_path / "setup-test-env.sh"
+    dummy_setup_path.write_text("echo Setup")
+
+    # Save original values
+    original_container_dir = PATHS.container_dir
+    original_setup_test_env = PATHS.setup_test_env
+
+    # Patch the PATHS instance
+    PATHS.container_dir = container_dir
+    PATHS.setup_test_env = dummy_setup_path
+
+    # Yield the directory for the test to use
+    yield container_dir
+
+    # Restore original values
+    PATHS.container_dir = original_container_dir
+    PATHS.setup_test_env = original_setup_test_env
+
+
+@pytest.fixture
+def mock_build_script(mock_container_dir):
+    """
+    Fixture that creates a dummy build script in the container directory.
+    """
+    build_script = mock_container_dir / "build-foo.sh"
+    return build_script
+
+
+###############################################################################
+# Tests for argument parsing and helper functions
+###############################################################################
+
+
+@pytest.mark.parametrize(
+    "component, expected_tag_var",
+    [
+        ("foo", "FOO_TAG"),
+        ("my-component", "MY_COMPONENT_TAG"),
+        ("foo-bar-baz", "FOO_BAR_BAZ_TAG"),
+    ],
+)
+def test_from_component_to_tag_var(component, expected_tag_var):
+    """
+    Test that from_component_to_tag_var returns the correct tag variable name.
+    """
+    assert from_component_to_tag_var(component) == expected_tag_var
+
+
+@pytest.mark.parametrize(
+    "component, expected_build_tag",
+    [
+        ("foo", "CONDITIONAL_BUILD_FOO_TAG"),
+        ("my-component", "CONDITIONAL_BUILD_MY_COMPONENT_TAG"),
+        ("foo-bar-baz", "CONDITIONAL_BUILD_FOO_BAR_BAZ_TAG"),
+    ],
+)
+def test_from_component_to_build_tag(component, expected_build_tag):
+    """
+    Test that from_component_to_build_tag returns the correct build tag name.
+    """
+    assert from_component_to_build_tag(component) == expected_build_tag
+
+
+@pytest.mark.parametrize(
+    "script_name, expected_component",
+    [
+        ("build-foo.sh", "foo"),
+        ("build-my-component.sh", "my-component"),
+        ("build-foo-bar-baz.sh", "foo-bar-baz"),
+    ],
+)
+def test_from_script_name_to_component(script_name, expected_component):
+    """
+    Test that from_script_name_to_component returns the correct component name.
+    """
+    assert from_script_name_to_component(script_name) == expected_component
+
+
+@pytest.mark.parametrize(
+    "script_name, expected_tag_var",
+    [
+        ("build-foo.sh", "FOO_TAG"),
+        ("build-my-component.sh", "MY_COMPONENT_TAG"),
+        ("build-foo-bar-baz.sh", "FOO_BAR_BAZ_TAG"),
+    ],
+)
+def test_from_script_name_to_tag_var(script_name, expected_tag_var):
+    """
+    Test that from_script_name_to_tag_var returns the correct tag variable name.
+    """
+    assert from_script_name_to_tag_var(script_name) == expected_tag_var
+
+
+def test_filter_components():
+    """
+    Test that filter_components returns only components that match an include regex
+    and do not match any exclude regex. If includes is empty, an empty list is returned.
+    """
+    components = ["alpha", "beta", "gamma", "delta"]
+
+    # When includes is empty, should return an empty list.
+    assert filter_components(components, [], []) == []
+
+    # Test includes only.
+    # Components that start with 'a' or 'b'
+    expected = ["alpha", "beta"]
+    result = filter_components(components, ["^[a-b].*$"], [])
+    assert result == expected
+
+    # Test with an exclude that filters out "alpha" (which matches "lph").
+    result = filter_components(components, ["^[a-b].*$"], ["^.*lph.*$"])
+    # "alpha" is removed.
+    assert result == ["beta"]
+
+
+@pytest.mark.parametrize(
+    "component, tag",
+    [
+        # Basic case
+        ("test", "new_tag"),
+        # Component with hyphen
+        ("my-component", "v1.2.3"),
+        # Uppercase component name
+        ("UpperCase", "123"),
+        # Numbers in component name
+        ("123service", "build-456"),
+    ],
+)
+def test_update_image_tag_in_yaml(component, tag, temp_image_tags_file):
+    """
+    Test multiple updates with different component names and tags, verifying:
+    1. Correct variable name generation
+    2. Proper tag value storage
+    3. Maintained sorting of variables
+    """
+    # Initial update
+    update_image_tag_in_yaml(component, tag)
+
+    data = load_image_tags_yaml()
+    expected_var = from_component_to_build_tag(component)
+    assert data["variables"][expected_var] == tag
+
+    # Add second component and verify both exist
+    update_image_tag_in_yaml("another_component", "secondary_tag")
+    updated_data = load_image_tags_yaml()
+
+    assert from_component_to_build_tag("another_component") in updated_data["variables"]
+    assert updated_data["variables"][expected_var] == tag  # Original value remains
+
+    # Verify sorting
+    variables = list(data["variables"].keys())
+    assert len(updated_data["variables"]) == len(variables) + 1
+    assert variables == sorted(variables), "Variables are not alphabetically sorted"
+
+
+def test_if_run_extracts_the_tag_from_stdout(monkeypatch, mock_build_script):
+    """
+    Test that run_build_script returns the tag (last stdout line) when the build
+    script executes successfully.
+    """
+    mock_build_script.write_text("#!/bin/bash\necho Build script\necho new_tag")
+    # Create a fake subprocess.CompletedProcess to simulate a successful build.
+    fake_result = subprocess.CompletedProcess(
+        args=[], returncode=0, stdout="line1\nnew_tag", stderr=""
+    )
+    monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: fake_result)
+
+    tag = run_build_script("foo", check_only=False)
+    assert tag == "new_tag"
+
+
+def test_running_real_process_works(monkeypatch, mock_build_script):
+    """
+    Test that run_build_script returns the tag (last stdout line) when the build
+    script executes successfully.
+    """
+    mock_build_script.write_text("echo Build script\necho new_tag")
+    tag = run_build_script("foo", check_only=False)
+    assert tag == "new_tag"
+
+
+###############################################################################
+# Tests for main() argument features
+###############################################################################
+
+
+def setup_build_script_and_container_ci_file(container_dir, container_ci_file, comp):
+    tag_var = from_component_to_build_tag(comp)
+    build_check_var = from_component_to_tag_var(comp)
+    build_script_path = container_dir / f"build-{comp}.sh"
+    build_script_path.write_text(
+        dedent(
+            f"""
+        #!/bin/bash
+        ci_tag_build_time_check {tag_var}
+        echo "new_tag"
+        """
+        )
+    )
+    container_ci_file.write_text(
+        container_ci_file.read_text()
+        + dedent(
+            f"""
+        .container-builds-{comp}:
+            variables:
+                {tag_var}: "${{{build_check_var}}}"
+        """
+        )
+    )
+
+
+def test_main_list(monkeypatch, capsys, mock_container_dir, temp_container_ci_file):
+    """
+    Test that when the --list argument is provided, main() prints all detected
+    components
+    """
+    # Monkeypatch find_components to return a known list.
+    for comp in ["comp-a", "comp-b"]:
+        setup_build_script_and_container_ci_file(mock_container_dir, temp_container_ci_file, comp)
+    # Set sys.argv to simulate passing --list.
+    monkeypatch.setattr(sys, "argv", ["bin/ci/update_tag.py", "--list"])
+
+    main()
+
+    captured = capsys.readouterr().out
+    assert "Detected components:" in captured
+    assert "comp-a" in captured
+    assert "comp-b" in captured
+
+
+def test_main_check(monkeypatch, temp_image_tags_file, temp_container_ci_file):
+    """
+    Test that when --check is provided, main() calls run_build_script in check mode.
+    """
+
+    with patch(
+        "bin.ci.update_tag.run_build_script", return_value="dummy_tag"
+    ) as mock_run_build_script:
+        # Simulate command line: --check compX
+        monkeypatch.setattr(sys, "argv", ["bin/ci/update_tag.py", "--check", "comp-x"])
+
+        main()
+    # Verify run_build_script was called with check_only=True.
+    assert mock_run_build_script.call_count == 1
+    assert "comp-x" in mock_run_build_script.call_args.args[0]
+    assert mock_run_build_script.call_args.kwargs["check_only"] is True
+
+
+EXIT_CODE_SCENARIOS = {
+    "unbound_variable": (
+        "line 2: UNDEFINED_VARIABLE: unbound variable",
+        127,
+        3,
+        "Please set the variable UNDEFINED_VARIABLE",
+    ),
+    "build script error": (
+        "",
+        50,
+        50,
+        "",
+    ),
+    "tag_mismatch": (
+        "Tag mismatch for foo.",
+        2,
+        2,
+        "Tag mismatch for foo.",
+    ),
+}
+
+
+@pytest.mark.parametrize(
+    "stderr_content, script_returncode, expected_exit_code, expected_error_message",
+    EXIT_CODE_SCENARIOS.values(),
+    ids=list(EXIT_CODE_SCENARIOS.keys()),
+)
+def test_build_script_error_exit_codes(
+    monkeypatch,
+    mock_build_script,
+    capsys,
+    stderr_content,
+    script_returncode,
+    expected_exit_code,
+    expected_error_message,
+):
+    """
+    Test that build script errors generate appropriate exit codes:
+    1 - Unhandled error in this script
+    2 - Tag mismatch when using --check
+    3 - Unbound variable error in build script
+    x - Build script failed with return code x
+    """
+    # Create a build script that will fail
+    mock_build_script.write_text("#!/bin/bash\necho 'This will fail'")
+
+    # Create a fake subprocess.CompletedProcess to simulate a failed build
+    fake_result = subprocess.CompletedProcess(
+        args=[], returncode=script_returncode, stdout="", stderr=stderr_content
+    )
+    monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: fake_result)
+    monkeypatch.setattr(sys, "argv", ["bin/ci/update_tag.py", "--check", "foo"])
+    monkeypatch.setattr("bin.ci.update_tag.get_current_tag_value", lambda *args: "current_tag")
+
+    # Mock sys.exit to capture the exit code instead of exiting the test
+    with pytest.raises(SystemExit) as e:
+        main()
+
+        # Check for expected error message if one is specified
+        if expected_error_message:
+            captured = capsys.readouterr()
+            assert expected_error_message in captured.err
+
+    # Verify correct exit code
+    assert e.value.code == expected_exit_code