mirror of
https://github.com/elwrcl/intel-hard-graphics.git
synced 2026-10-31 02:48:03 +03:00
init
This commit is contained in:
12511 files changed
+6156641
No files matched your search
@@ -0,0 +1,2 @@
|
||||
[*.sh]
|
||||
indent_style = tab
|
||||
Whitespace-only changes.
@@ -0,0 +1,2 @@
|
||||
schema.graphql
|
||||
gitlab_gql.py.cache*
|
||||
Executable
+202
@@ -0,0 +1,202 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright © 2023 Collabora Ltd.
|
||||
# Authors:
|
||||
# Helen Koike <[email protected]>
|
||||
#
|
||||
# For the dependencies, see the requirements.txt
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
|
||||
import argparse
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Dict, List
|
||||
|
||||
import plotly.express as px
|
||||
import plotly.graph_objs as go
|
||||
from gitlab import Gitlab, base
|
||||
from gitlab.v4.objects import ProjectPipeline
|
||||
from gitlab_common import (GITLAB_URL, get_gitlab_pipeline_from_url,
|
||||
get_token_from_default_dir, pretty_duration,
|
||||
read_token)
|
||||
|
||||
|
||||
def calculate_queued_at(job) -> datetime:
|
||||
started_at = job.started_at.replace("Z", "+00:00")
|
||||
return datetime.fromisoformat(started_at) - timedelta(seconds=job.queued_duration)
|
||||
|
||||
|
||||
def calculate_time_difference(time1, time2) -> str:
|
||||
if type(time1) is str:
|
||||
time1 = datetime.fromisoformat(time1.replace("Z", "+00:00"))
|
||||
if type(time2) is str:
|
||||
time2 = datetime.fromisoformat(time2.replace("Z", "+00:00"))
|
||||
|
||||
diff = time2 - time1
|
||||
return pretty_duration(diff.seconds)
|
||||
|
||||
|
||||
def create_task_name(job) -> str:
|
||||
status_color = {"success": "green", "failed": "red"}.get(job.status, "grey")
|
||||
return f"{job.name}\t(<span style='color: {status_color}'>{job.status}</span>,<a href='{job.web_url}'>{job.id}</a>)"
|
||||
|
||||
|
||||
def add_gantt_bar(
|
||||
job: base.RESTObject, tasks: List[Dict[str, str | datetime | timedelta]]
|
||||
) -> None:
|
||||
queued_at = calculate_queued_at(job)
|
||||
task_name = create_task_name(job)
|
||||
|
||||
tasks.append(
|
||||
{
|
||||
"Job": task_name,
|
||||
"Start": job.created_at,
|
||||
"Finish": queued_at,
|
||||
"Duration": calculate_time_difference(job.created_at, queued_at),
|
||||
"Phase": "Waiting dependencies",
|
||||
}
|
||||
)
|
||||
tasks.append(
|
||||
{
|
||||
"Job": task_name,
|
||||
"Start": queued_at,
|
||||
"Finish": job.started_at,
|
||||
"Duration": calculate_time_difference(queued_at, job.started_at),
|
||||
"Phase": "Queued",
|
||||
}
|
||||
)
|
||||
|
||||
if job.finished_at:
|
||||
tasks.append(
|
||||
{
|
||||
"Job": task_name,
|
||||
"Start": job.started_at,
|
||||
"Finish": job.finished_at,
|
||||
"Duration": calculate_time_difference(job.started_at, job.finished_at),
|
||||
"Phase": "Time spent running",
|
||||
}
|
||||
)
|
||||
else:
|
||||
current_time = datetime.now(timezone.utc).isoformat()
|
||||
tasks.append(
|
||||
{
|
||||
"Job": task_name,
|
||||
"Start": job.started_at,
|
||||
"Finish": current_time,
|
||||
"Duration": calculate_time_difference(job.started_at, current_time),
|
||||
"Phase": "In-Progress",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def generate_gantt_chart(
|
||||
pipeline: ProjectPipeline, ci_timeout: float = 60
|
||||
) -> go.Figure:
|
||||
if pipeline.yaml_errors:
|
||||
raise ValueError("Pipeline YAML errors detected")
|
||||
|
||||
# Convert the data into a list of dictionaries for plotly
|
||||
tasks: List[Dict[str, str | datetime | timedelta]] = []
|
||||
|
||||
for job in pipeline.jobs.list(all=True, include_retried=True):
|
||||
# we can have queued_duration without started_at when a job is canceled
|
||||
if not job.queued_duration or not job.started_at:
|
||||
continue
|
||||
add_gantt_bar(job, tasks)
|
||||
|
||||
# Make it easier to see retried jobs
|
||||
tasks.sort(key=lambda x: x["Job"])
|
||||
|
||||
title = f"Gantt chart of jobs in pipeline <a href='{pipeline.web_url}'>{pipeline.web_url}</a>."
|
||||
title += (
|
||||
f" Total duration {str(timedelta(seconds=pipeline.duration))}"
|
||||
if pipeline.duration
|
||||
else ""
|
||||
)
|
||||
|
||||
# Create a Gantt chart
|
||||
default_colors = px.colors.qualitative.Plotly
|
||||
fig: go.Figure = px.timeline(
|
||||
tasks,
|
||||
x_start="Start",
|
||||
x_end="Finish",
|
||||
y="Job",
|
||||
color="Phase",
|
||||
title=title,
|
||||
hover_data=["Duration"],
|
||||
color_discrete_map={
|
||||
"In-Progress": default_colors[3], # purple
|
||||
"Waiting dependencies": default_colors[0], # blue
|
||||
"Queued": default_colors[1], # red
|
||||
"Time spent running": default_colors[2], # green
|
||||
},
|
||||
)
|
||||
|
||||
# Calculate the height dynamically
|
||||
fig.update_layout(height=len(tasks) * 10, yaxis_tickfont_size=14)
|
||||
|
||||
# Add a deadline line to the chart
|
||||
created_at = datetime.fromisoformat(pipeline.created_at.replace("Z", "+00:00"))
|
||||
timeout_at = created_at + timedelta(minutes=ci_timeout)
|
||||
fig.add_vrect(
|
||||
x0=timeout_at,
|
||||
x1=timeout_at,
|
||||
annotation_text=f"{int(ci_timeout)} min Timeout",
|
||||
fillcolor="gray",
|
||||
line_width=2,
|
||||
line_color="gray",
|
||||
line_dash="dash",
|
||||
annotation_position="top left",
|
||||
annotation_textangle=90,
|
||||
)
|
||||
|
||||
return fig
|
||||
|
||||
|
||||
def main(
|
||||
token: str | None,
|
||||
pipeline_url: str,
|
||||
output: str | None,
|
||||
ci_timeout: float = 60,
|
||||
):
|
||||
if token is None:
|
||||
token = get_token_from_default_dir()
|
||||
|
||||
token = read_token(token)
|
||||
gl = Gitlab(url=GITLAB_URL, private_token=token, retry_transient_errors=True)
|
||||
|
||||
pipeline, _ = get_gitlab_pipeline_from_url(gl, pipeline_url)
|
||||
fig: go.Figure = generate_gantt_chart(pipeline, ci_timeout)
|
||||
if output and "htm" in output:
|
||||
fig.write_html(output)
|
||||
elif output:
|
||||
fig.update_layout(width=1000)
|
||||
fig.write_image(output)
|
||||
else:
|
||||
fig.show()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate the Gantt chart from a given pipeline."
|
||||
)
|
||||
parser.add_argument("pipeline_url", type=str, help="URLs to the pipeline.")
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
type=str,
|
||||
help="Output file name. Use html or image suffixes to choose the format.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--token",
|
||||
metavar="token",
|
||||
help="force GitLab token, otherwise it's read from ~/.config/gitlab-token",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ci-timeout",
|
||||
metavar="ci_timeout",
|
||||
type=float,
|
||||
default=60,
|
||||
help="Time that marge-bot will wait for ci to finish. Defaults to one hour.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
main(args.token, args.pipeline_url, args.output, args.ci_timeout)
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
|
||||
this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
|
||||
readonly this_dir
|
||||
|
||||
exec \
|
||||
"$this_dir/../python-venv.sh" \
|
||||
"$this_dir/requirements.txt" \
|
||||
"$this_dir/ci_gantt_chart.py" "$@"
|
||||
Executable
+223
@@ -0,0 +1,223 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright © 2023 Collabora Ltd.
|
||||
# Authors:
|
||||
# Helen Koike <[email protected]>
|
||||
#
|
||||
# For the dependencies, see the requirements.txt
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
|
||||
import argparse
|
||||
import logging as log
|
||||
import os
|
||||
import re
|
||||
import traceback
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict
|
||||
|
||||
import gitlab
|
||||
import pytz
|
||||
from ci_gantt_chart import generate_gantt_chart
|
||||
from gitlab import Gitlab
|
||||
from gitlab.base import RESTObject
|
||||
from gitlab.v4.objects import Project, ProjectPipeline
|
||||
from gitlab_common import (GITLAB_URL, get_gitlab_pipeline_from_url,
|
||||
get_token_from_default_dir, read_token)
|
||||
|
||||
|
||||
class MockGanttExit(Exception):
|
||||
pass
|
||||
|
||||
|
||||
LAST_MARGE_EVENT_FILE = os.path.expanduser("~/.config/last_marge_event")
|
||||
|
||||
|
||||
def read_last_event_date_from_file() -> str:
|
||||
try:
|
||||
with open(LAST_MARGE_EVENT_FILE, "r") as f:
|
||||
last_event_date = f.read().strip()
|
||||
except FileNotFoundError:
|
||||
# 3 days ago
|
||||
last_event_date = (datetime.now() - timedelta(days=3)).isoformat()
|
||||
return last_event_date
|
||||
|
||||
|
||||
def pretty_time(time_str: str) -> str:
|
||||
"""Pretty print time"""
|
||||
local_timezone = datetime.now().astimezone().tzinfo
|
||||
|
||||
time_d = datetime.fromisoformat(time_str.replace("Z", "+00:00")).astimezone(
|
||||
local_timezone
|
||||
)
|
||||
return f'{time_str} ({time_d.strftime("%d %b %Y %Hh%Mm%Ss")} {local_timezone})'
|
||||
|
||||
|
||||
def compose_message(file_name: str, attachment_url: str) -> str:
|
||||
return f"""
|
||||
[{file_name}]({attachment_url})
|
||||
|
||||
<details>
|
||||
<summary>more info</summary>
|
||||
|
||||
This message was generated by the ci_post_gantt.py script, which is running on a server at Collabora.
|
||||
</details>
|
||||
"""
|
||||
|
||||
|
||||
def gitlab_upload_file_get_url(gl: Gitlab, project_id: str, filepath: str) -> str:
|
||||
project: Project = gl.projects.get(project_id)
|
||||
uploaded_file: Dict[str, Any] = project.upload(filepath, filepath=filepath)
|
||||
return uploaded_file["url"]
|
||||
|
||||
|
||||
def gitlab_post_reply_to_note(gl: Gitlab, event: RESTObject, reply_message: str):
|
||||
"""
|
||||
Post a reply to a note in thread based on a GitLab event.
|
||||
|
||||
:param gl: The GitLab connection instance.
|
||||
:param event: The event object containing the note details.
|
||||
:param reply_message: The reply message.
|
||||
"""
|
||||
try:
|
||||
note_id = event.target_id
|
||||
merge_request_iid = event.note["noteable_iid"]
|
||||
|
||||
project = gl.projects.get(event.project_id)
|
||||
merge_request = project.mergerequests.get(merge_request_iid)
|
||||
|
||||
# Find the discussion to which the note belongs
|
||||
discussions = merge_request.discussions.list(iterator=True)
|
||||
target_discussion = next(
|
||||
(
|
||||
d
|
||||
for d in discussions
|
||||
if any(n["id"] == note_id for n in d.attributes["notes"])
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
if target_discussion is None:
|
||||
raise ValueError("Discussion for the note not found.")
|
||||
|
||||
# Add a reply to the discussion
|
||||
reply = target_discussion.notes.create({"body": reply_message})
|
||||
return reply
|
||||
|
||||
except gitlab.exceptions.GitlabError as e:
|
||||
log.error(f"Failed to post a reply to '{event.note['body']}': {e}")
|
||||
return None
|
||||
|
||||
|
||||
def main(
|
||||
token: str | None,
|
||||
since: str | None,
|
||||
marge_user_id: int = 9716,
|
||||
project_ids: list[int] = [176],
|
||||
ci_timeout: float = 60,
|
||||
):
|
||||
log.basicConfig(level=log.INFO)
|
||||
if token is None:
|
||||
token = get_token_from_default_dir()
|
||||
|
||||
token = read_token(token)
|
||||
gl = Gitlab(url=GITLAB_URL, private_token=token, retry_transient_errors=True)
|
||||
|
||||
user = gl.users.get(marge_user_id)
|
||||
last_event_at = since if since else read_last_event_date_from_file()
|
||||
|
||||
log.info(f"Retrieving Marge messages since {pretty_time(last_event_at)}\n")
|
||||
|
||||
# the "after" only considers the "2023-10-24" part, it doesn't consider the time
|
||||
events = user.events.list(
|
||||
all=True,
|
||||
target_type="note",
|
||||
after=(datetime.now() - timedelta(days=3)).isoformat(),
|
||||
sort="asc",
|
||||
)
|
||||
|
||||
last_event_at_date = datetime.fromisoformat(
|
||||
last_event_at.replace("Z", "+00:00")
|
||||
).replace(tzinfo=pytz.UTC)
|
||||
|
||||
for event in events:
|
||||
if event.project_id not in project_ids:
|
||||
continue
|
||||
created_at_date = datetime.fromisoformat(
|
||||
event.created_at.replace("Z", "+00:00")
|
||||
).replace(tzinfo=pytz.UTC)
|
||||
if created_at_date <= last_event_at_date:
|
||||
continue
|
||||
last_event_at = event.created_at
|
||||
|
||||
escaped_gitlab_url = re.escape(GITLAB_URL)
|
||||
match = re.search(rf"{escaped_gitlab_url}/[^\s<]+", event.note["body"])
|
||||
|
||||
if match:
|
||||
try:
|
||||
log.info(f"Found message: {event.note['body']}")
|
||||
pipeline_url = match.group(0)[:-1]
|
||||
pipeline: ProjectPipeline
|
||||
pipeline, _ = get_gitlab_pipeline_from_url(gl, pipeline_url)
|
||||
log.info("Generating gantt chart...")
|
||||
fig = generate_gantt_chart(pipeline, ci_timeout)
|
||||
file_name = f"{str(pipeline.id)}-Gantt.html"
|
||||
fig.write_html(file_name)
|
||||
log.info("Uploading gantt file...")
|
||||
file_url = gitlab_upload_file_get_url(gl, event.project_id, file_name)
|
||||
log.info("Posting reply ...")
|
||||
message = compose_message(file_name, file_url)
|
||||
gitlab_post_reply_to_note(gl, event, message)
|
||||
except MockGanttExit:
|
||||
pass # Allow tests to exit early without printing a traceback
|
||||
except Exception as e:
|
||||
log.info(f"Failed to generate gantt chart, not posting reply.{e}")
|
||||
traceback.print_exc()
|
||||
|
||||
if not since:
|
||||
log.info(
|
||||
f"Updating last event date to {pretty_time(last_event_at)} on {LAST_MARGE_EVENT_FILE}\n"
|
||||
)
|
||||
with open(LAST_MARGE_EVENT_FILE, "w") as f:
|
||||
f.write(last_event_at)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Monitor rejected pipelines by Marge.")
|
||||
parser.add_argument(
|
||||
"--token",
|
||||
metavar="token",
|
||||
type=str,
|
||||
default=None,
|
||||
help="force GitLab token, otherwise it's read from ~/.config/gitlab-token",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--since",
|
||||
metavar="since",
|
||||
type=str,
|
||||
default=None,
|
||||
help="consider only events after this date (ISO format), otherwise it's read from ~/.config/last_marge_event",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--marge-user-id",
|
||||
metavar="marge_user_id",
|
||||
type=int,
|
||||
default=9716, # default https://gitlab.freedesktop.org/users/marge-bot/activity
|
||||
help="GitLab user ID for marge-bot, defaults to 9716",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--project-id",
|
||||
metavar="project_id",
|
||||
type=int,
|
||||
nargs="+",
|
||||
default=[176], # default is the mesa/mesa project id
|
||||
help="GitLab project id(s) to analyze. Defaults to 176 i.e. mesa/mesa.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ci-timeout",
|
||||
metavar="ci_timeout",
|
||||
type=float,
|
||||
default=60,
|
||||
help="Time that marge-bot will wait for ci to finish. Defaults to one hour.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
main(args.token, args.since, args.marge_user_id, args.project_id, args.ci_timeout)
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
|
||||
this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
|
||||
readonly this_dir
|
||||
|
||||
exec \
|
||||
"$this_dir/../python-venv.sh" \
|
||||
"$this_dir/requirements.txt" \
|
||||
"$this_dir/ci_post_gantt.py" "$@"
|
||||
Executable
+808
@@ -0,0 +1,808 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright © 2020 - 2022 Collabora Ltd.
|
||||
# Authors:
|
||||
# Tomeu Vizoso <[email protected]>
|
||||
# David Heidelberg <[email protected]>
|
||||
#
|
||||
# For the dependencies, see the requirements.txt
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
"""
|
||||
Helper script to restrict running only required CI jobs
|
||||
and show the job(s) logs.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from collections import defaultdict, Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from functools import partial
|
||||
from itertools import chain
|
||||
from subprocess import check_output, CalledProcessError
|
||||
from typing import Callable, Dict, TYPE_CHECKING, Iterable, Literal, Optional, Tuple, cast
|
||||
|
||||
import gitlab
|
||||
import gitlab.v4.objects
|
||||
from gitlab_common import (
|
||||
GITLAB_URL,
|
||||
TOKEN_DIR,
|
||||
get_gitlab_pipeline_from_url,
|
||||
get_gitlab_project,
|
||||
get_token_from_default_dir,
|
||||
is_gitlab_job,
|
||||
pretty_duration,
|
||||
read_token,
|
||||
wait_for_pipeline,
|
||||
)
|
||||
from gitlab_gql import GitlabGQL, create_job_needs_dag, filter_dag, print_dag, print_formatted_list
|
||||
from rich.console import Console
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from gitlab_gql import Dag
|
||||
|
||||
REFRESH_WAIT_LOG = 10
|
||||
REFRESH_WAIT_JOBS = 6
|
||||
MAX_ENABLE_JOB_ATTEMPTS = 3
|
||||
|
||||
STATUS_COLORS = defaultdict(lambda: "", {
|
||||
"running": "[blue]",
|
||||
"success": "[green]",
|
||||
"failed": "[red]",
|
||||
"canceled": "[magenta]",
|
||||
"canceling": "[magenta]",
|
||||
})
|
||||
|
||||
COMPLETED_STATUSES = frozenset({"success", "failed"})
|
||||
RUNNING_STATUSES = frozenset({"created", "pending", "running"})
|
||||
|
||||
if is_gitlab_job():
|
||||
console = Console(highlight=False, no_color=False, color_system="truecolor", width=120)
|
||||
else:
|
||||
console = Console(highlight=False)
|
||||
print = console.print
|
||||
|
||||
|
||||
def print_job_status(
|
||||
job: gitlab.v4.objects.ProjectPipelineJob,
|
||||
new_status: bool = False,
|
||||
) -> None:
|
||||
"""It prints a nice, colored job status with a link to the job."""
|
||||
if job.status in {"canceled", "canceling"}:
|
||||
return
|
||||
|
||||
if new_status and job.status == "created":
|
||||
return
|
||||
|
||||
global type_field_pad
|
||||
global name_field_pad
|
||||
jtype = "🞋 job"
|
||||
job_name = job.name
|
||||
type_field_pad = len(jtype) if len(jtype) > type_field_pad else type_field_pad
|
||||
name_field_pad = len(job_name) if len(job_name) > name_field_pad else name_field_pad
|
||||
|
||||
duration = job_duration(job)
|
||||
|
||||
print(
|
||||
f"{STATUS_COLORS[job.status]}"
|
||||
f"{jtype:{type_field_pad}} " # U+1F78B Round target
|
||||
f"{link2print(job.web_url, job.name, name_field_pad)} " +
|
||||
(f" has new status: {job.status}" if new_status else f" {job.status}") +
|
||||
(f" ({pretty_duration(duration)})" if job.started_at else "")
|
||||
)
|
||||
|
||||
|
||||
def job_duration(job: gitlab.v4.objects.ProjectPipelineJob) -> float:
|
||||
"""
|
||||
Given a job, report the time lapsed in execution.
|
||||
:param job: Pipeline job
|
||||
:return: Current time in execution
|
||||
"""
|
||||
if job.duration:
|
||||
return job.duration
|
||||
elif job.started_at:
|
||||
# Convert both times to UTC timestamps for consistent comparison
|
||||
current_time = time.time()
|
||||
start_time = job.started_at.timestamp()
|
||||
return current_time - start_time
|
||||
return 0.0
|
||||
|
||||
|
||||
def pretty_wait(sec: int) -> None:
|
||||
"""shows progressbar in dots"""
|
||||
if is_gitlab_job():
|
||||
time.sleep(sec)
|
||||
return
|
||||
for val in range(sec, 0, -1):
|
||||
print(f"⏲ {val:2d} seconds", end="\r") # U+23F2 Timer clock
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def run_target_job(
|
||||
job: gitlab.v4.objects.ProjectPipelineJob,
|
||||
enable_job_fn: Callable,
|
||||
stress: int,
|
||||
execution_times: dict,
|
||||
target_statuses: dict,
|
||||
) -> None:
|
||||
execution_times[job.name][job.id] = (job_duration(job), job.status, job.web_url)
|
||||
if stress and job.status in COMPLETED_STATUSES:
|
||||
if (
|
||||
stress < 0
|
||||
or len(execution_times[job.name]) < stress
|
||||
):
|
||||
enable_job_fn(job=job, action_type="retry")
|
||||
# Wait for the next loop to get the updated job object
|
||||
return
|
||||
else:
|
||||
enable_job_fn(job=job, action_type="target")
|
||||
|
||||
print_job_status(job, job.status not in target_statuses[job.name])
|
||||
target_statuses[job.name] = job.status
|
||||
|
||||
|
||||
def monitor_pipeline(
|
||||
project: gitlab.v4.objects.Project,
|
||||
pipeline: gitlab.v4.objects.ProjectPipeline,
|
||||
job_filter: callable,
|
||||
dependencies: set[str],
|
||||
stress: int,
|
||||
inhibit_single_target_trace: int = False,
|
||||
polling_period: int = REFRESH_WAIT_JOBS,
|
||||
) -> tuple[Optional[int], Optional[int], Dict[str, Dict[int, Tuple[float, str, str]]]]:
|
||||
"""Monitors pipeline and delegate canceling jobs"""
|
||||
statuses: dict[str, str] = defaultdict(str)
|
||||
target_statuses: dict[str, str] = defaultdict(str)
|
||||
execution_times: dict[str, dict[str, tuple[float, str, str]]] = defaultdict(lambda: defaultdict(tuple))
|
||||
target_id: int = -1
|
||||
global type_field_pad
|
||||
type_field_pad = 0
|
||||
global name_field_pad
|
||||
name_field_pad = len(max(dependencies, key=len))+2
|
||||
# In a running pipeline, we can skip following job traces that are in these statuses.
|
||||
skip_follow_statuses: frozenset[str] = (COMPLETED_STATUSES)
|
||||
|
||||
# Pre-populate the stress status counter for already completed target jobs.
|
||||
if stress:
|
||||
# When stress test, it is necessary to collect this information before start.
|
||||
for job in pipeline.jobs.list(all=True, include_retried=True):
|
||||
if job_filter(
|
||||
job_name=job.name,
|
||||
job_stage=job.stage,
|
||||
job_tags=job.tag_list,
|
||||
) and job.status in COMPLETED_STATUSES:
|
||||
execution_times[job.name][job.id] = (job_duration(job), job.status, job.web_url)
|
||||
|
||||
# jobs_waiting is a list of job names that are waiting for status update.
|
||||
# It occurs when a job that we want to run depends on another job that is not yet finished.
|
||||
jobs_waiting = []
|
||||
# Dictionary to track the number of attempts made for each job for a given status
|
||||
enable_attempts: dict[tuple[int, str], int] = {}
|
||||
# FIXME: This function has too many parameters, consider refactoring.
|
||||
enable_job_fn = partial(
|
||||
enable_job,
|
||||
project=project,
|
||||
enable_attempts=enable_attempts,
|
||||
jobs_waiting=jobs_waiting,
|
||||
)
|
||||
while True:
|
||||
deps_failed = []
|
||||
to_cancel = []
|
||||
jobs_waiting.clear()
|
||||
for job in sorted(pipeline.jobs.list(all=True), key=lambda j: j.name):
|
||||
job = cast(gitlab.v4.objects.ProjectPipelineJob, job)
|
||||
if job_filter(
|
||||
job_name=job.name,
|
||||
job_stage=job.stage,
|
||||
job_tags=job.tag_list,
|
||||
):
|
||||
run_target_job(
|
||||
job,
|
||||
enable_job_fn,
|
||||
stress,
|
||||
execution_times,
|
||||
target_statuses
|
||||
)
|
||||
target_id = job.id
|
||||
continue
|
||||
# all other non-target jobs
|
||||
if job.status != statuses[job.name]:
|
||||
print_job_status(job, True)
|
||||
statuses[job.name] = job.status
|
||||
|
||||
# run dependencies and cancel the rest
|
||||
if job.name in dependencies:
|
||||
if not enable_job_fn(job=job, action_type="dep"):
|
||||
# Wait for the next loop to get the updated job object
|
||||
continue
|
||||
if job.status == "failed":
|
||||
deps_failed.append(job.name)
|
||||
else:
|
||||
to_cancel.append(job)
|
||||
|
||||
cancel_jobs(project, to_cancel)
|
||||
|
||||
if stress:
|
||||
enough = True
|
||||
status_counters = {
|
||||
name: Counter(info[1] for info in runs.values())
|
||||
for name, runs in execution_times.items()
|
||||
}
|
||||
for job_name, counter in sorted(status_counters.items()):
|
||||
n_succeed = counter.get("success", 0)
|
||||
n_failed = counter.get("failed", 0)
|
||||
n_total_completed = n_succeed + n_failed
|
||||
n_total_seen = len(execution_times[job_name])
|
||||
print(
|
||||
f"* {job_name:{name_field_pad}} succ: {n_succeed}; "
|
||||
f"fail: {n_failed}; "
|
||||
f"total: {n_total_seen} of {stress}",
|
||||
)
|
||||
if stress < 0 or n_total_completed < stress:
|
||||
enough = False
|
||||
|
||||
if not enough:
|
||||
pretty_wait(polling_period)
|
||||
continue
|
||||
|
||||
if jobs_waiting:
|
||||
print(f"[yellow]Waiting for jobs to update status:")
|
||||
print_formatted_list(jobs_waiting, indentation=8, color="[yellow]")
|
||||
pretty_wait(polling_period)
|
||||
continue
|
||||
|
||||
if (
|
||||
not inhibit_single_target_trace
|
||||
and stress in [0, 1]
|
||||
and len(target_statuses) == 1
|
||||
and RUNNING_STATUSES.intersection(target_statuses.values())
|
||||
):
|
||||
return target_id, None, execution_times
|
||||
|
||||
if (
|
||||
{"failed"}.intersection(target_statuses.values())
|
||||
and not RUNNING_STATUSES.intersection(target_statuses.values())
|
||||
):
|
||||
return None, 1, execution_times
|
||||
|
||||
if (
|
||||
{"skipped"}.intersection(target_statuses.values())
|
||||
and not RUNNING_STATUSES.intersection(target_statuses.values())
|
||||
):
|
||||
print(
|
||||
f"[red]Target in skipped state, aborting. Failed dependencies:{deps_failed}"
|
||||
)
|
||||
return None, 1, execution_times
|
||||
|
||||
if skip_follow_statuses.issuperset(target_statuses.values()):
|
||||
return None, 0, execution_times
|
||||
|
||||
pretty_wait(polling_period)
|
||||
|
||||
|
||||
def enable_job(
|
||||
project: gitlab.v4.objects.Project,
|
||||
job: gitlab.v4.objects.ProjectPipelineJob,
|
||||
enable_attempts: dict[tuple[int, str], int],
|
||||
action_type: Literal["target", "dep", "retry"],
|
||||
jobs_waiting: list[str] = list,
|
||||
) -> bool:
|
||||
"""
|
||||
Enable a job to run.
|
||||
:param project: The GitLab project.
|
||||
:param job: The job to enable.
|
||||
:param enable_attempts: A dictionary to track the number of attempts made for each job for a give status.
|
||||
:param action_type: The type of action to perform.
|
||||
:param jobs_waiting:
|
||||
:return: True if the job was enabled, False otherwise.
|
||||
"""
|
||||
# We want to run this job, but it is not ready to run yet, so let's try again in the next
|
||||
# iteration.
|
||||
if job.status == "created":
|
||||
jobs_waiting.append(job.name)
|
||||
return False
|
||||
|
||||
if (
|
||||
(job.status in COMPLETED_STATUSES and action_type != "retry")
|
||||
or job.status in {"skipped"} | RUNNING_STATUSES
|
||||
):
|
||||
return False
|
||||
|
||||
# Get current attempt number
|
||||
attempt_count = enable_attempts.get((job.id, job.status), 0)
|
||||
# Check if we've exceeded max attempts to avoid infinite loop
|
||||
if attempt_count == MAX_ENABLE_JOB_ATTEMPTS:
|
||||
print(
|
||||
f"[yellow]WARNING: "
|
||||
f"Maximum enabling attempts ({MAX_ENABLE_JOB_ATTEMPTS}) reached for job {job.name} in {job.status} status"
|
||||
f"({link2print(job.web_url, job.id)})."
|
||||
)
|
||||
enable_attempts[(job.id, job.status)] = attempt_count + 1
|
||||
return False
|
||||
elif attempt_count > MAX_ENABLE_JOB_ATTEMPTS:
|
||||
return False
|
||||
|
||||
enable_attempts[(job.id, job.status)] = attempt_count + 1
|
||||
|
||||
pjob = project.jobs.get(job.id, lazy=True)
|
||||
|
||||
if job.status in {"success", "failed", "canceled", "canceling"}:
|
||||
try:
|
||||
pjob.retry()
|
||||
except Exception as e:
|
||||
print(f"Error retrying job {job.name}: {e}")
|
||||
return False
|
||||
else:
|
||||
try:
|
||||
pjob.play()
|
||||
except Exception as e:
|
||||
print(f"Error playing job {job.name}: {e}")
|
||||
return False
|
||||
|
||||
if action_type == "target":
|
||||
jtype = "🞋 target" # U+1F78B Round target
|
||||
elif action_type == "retry":
|
||||
jtype = "↻ retrying" # U+21BB Clockwise open circle arrow
|
||||
else:
|
||||
jtype = "↪ dependency" # U+21AA Left Arrow Curving Right
|
||||
|
||||
global type_field_pad
|
||||
global name_field_pad
|
||||
job_name = job.name
|
||||
type_field_pad = len(jtype) if len(jtype) > type_field_pad else type_field_pad
|
||||
name_field_pad = len(job_name) if len(job_name) > name_field_pad else name_field_pad
|
||||
print(
|
||||
f"[magenta]{jtype:{type_field_pad}} {job.name:{name_field_pad}} manually enabled"
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def cancel_job(
|
||||
project: gitlab.v4.objects.Project,
|
||||
pipeline_job: gitlab.v4.objects.ProjectPipelineJob
|
||||
) -> Optional[gitlab.v4.objects.ProjectPipelineJob]:
|
||||
"""
|
||||
Cancel GitLab job
|
||||
:param project: project from the pipeline job comes from
|
||||
:param pipeline_job: job made from the pipeline list
|
||||
:return the job object when cancel was called
|
||||
"""
|
||||
if pipeline_job.status not in RUNNING_STATUSES:
|
||||
return
|
||||
try:
|
||||
project_job = project.jobs.get(pipeline_job.id, lazy=True)
|
||||
project_job.cancel()
|
||||
except (gitlab.GitlabCancelError, gitlab.GitlabGetError):
|
||||
# If the job failed to cancel, it will be retried in the monitor_pipeline() next iteration
|
||||
return
|
||||
return pipeline_job
|
||||
|
||||
|
||||
def cancel_jobs(
|
||||
project: gitlab.v4.objects.Project,
|
||||
to_cancel: list[gitlab.v4.objects.ProjectPipelineJob]
|
||||
) -> None:
|
||||
"""
|
||||
Cancel unwanted GitLab jobs
|
||||
:param project: project from where the pipeline comes
|
||||
:param to_cancel: list of jobs to be cancelled
|
||||
"""
|
||||
if not to_cancel:
|
||||
return
|
||||
|
||||
with ThreadPoolExecutor(max_workers=6) as exe:
|
||||
part = partial(cancel_job, project)
|
||||
maybe_cancelled_job = exe.map(part, to_cancel)
|
||||
cancelled_jobs = [f"🗙 {job.name}" for job in maybe_cancelled_job if job] # U+1F5D9 Cancellation X
|
||||
|
||||
# The cancelled jobs are printed without a newline
|
||||
if len(cancelled_jobs):
|
||||
print(f"Cancelled {len(cancelled_jobs)} jobs:")
|
||||
print_formatted_list(cancelled_jobs, indentation=8)
|
||||
|
||||
|
||||
def print_log(
|
||||
project: gitlab.v4.objects.Project,
|
||||
job_id: int
|
||||
) -> None:
|
||||
"""Print job log into output"""
|
||||
printed_lines = 0
|
||||
while True:
|
||||
job = project.jobs.get(job_id)
|
||||
|
||||
# GitLab's REST API doesn't offer pagination for logs, so we have to refetch it all
|
||||
lines = job.trace().decode().splitlines()
|
||||
for line in lines[printed_lines:]:
|
||||
print(line, markup=False)
|
||||
printed_lines = len(lines)
|
||||
|
||||
if job.status in COMPLETED_STATUSES:
|
||||
print(f"[green]Job finished: {job.web_url}")
|
||||
return
|
||||
pretty_wait(REFRESH_WAIT_LOG)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
"""Parse args"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Tool to trigger a subset of container jobs "
|
||||
+ "and monitor the progress of a test job",
|
||||
epilog="Example: %(prog)s --rev $(git rev-parse HEAD) "
|
||||
+ '--target ".*traces" ',
|
||||
)
|
||||
parser.add_argument(
|
||||
"--server",
|
||||
metavar="gitlab-server",
|
||||
type=str,
|
||||
default=GITLAB_URL,
|
||||
help=f"Specify the GitLab server work with (Default: {GITLAB_URL})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--target",
|
||||
metavar="target-job",
|
||||
help="Target job regex. For multiple targets, pass multiple values, "
|
||||
"eg. `--target foo bar`. Only jobs in the target stage(s) "
|
||||
"supplied, and their dependencies, will be considered.",
|
||||
required=True,
|
||||
nargs=argparse.ONE_OR_MORE,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--include-stage",
|
||||
metavar="include-stage",
|
||||
help="Job stages to include when searching for target jobs. "
|
||||
"For multiple targets, pass multiple values, eg. "
|
||||
"`--include-stage foo bar`.",
|
||||
default=[".*"],
|
||||
nargs=argparse.ONE_OR_MORE,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--exclude-stage",
|
||||
metavar="exclude-stage",
|
||||
help="Job stages to exclude when searching for target jobs. "
|
||||
"For multiple targets, pass multiple values, eg. "
|
||||
"`--exclude-stage foo bar`. By default, performance and "
|
||||
"nightly jobs are excluded; pass --exclude-stage '' to "
|
||||
"include them for consideration.",
|
||||
default=["performance", ".*-postmerge", ".*-nightly"],
|
||||
nargs=argparse.ONE_OR_MORE,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--job-tags",
|
||||
metavar="job-tags",
|
||||
help="Job tags to require when searching for target jobs. If multiple "
|
||||
"values are passed, eg. `--job-tags 'foo.*' 'bar'`, the job will "
|
||||
"need to have a tag matching `foo.*` *and* a tag matching `bar` "
|
||||
"to qualify. Passing `--job-tags '.*'` makes sure the job has "
|
||||
"a tag defined, while not passing `--job-tags` also allows "
|
||||
"untagged jobs.",
|
||||
default=[],
|
||||
nargs=argparse.ONE_OR_MORE,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--token",
|
||||
metavar="token",
|
||||
type=str,
|
||||
default=get_token_from_default_dir(),
|
||||
help="Use the provided GitLab token (with `api` scope) or token file, "
|
||||
f"otherwise it's read from {TOKEN_DIR / 'gitlab-token'}",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--force-manual", action="store_true",
|
||||
help="Deprecated argument; manual jobs are always force-enabled"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stress",
|
||||
metavar="n",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Stresstest job(s). Specify the number of times to rerun the selected jobs, "
|
||||
"or use -1 for indefinite. Defaults to 0. If jobs have already been executed, "
|
||||
"this will ensure the total run count respects the specified number.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--project",
|
||||
metavar="name",
|
||||
type=str,
|
||||
default="mesa",
|
||||
help="GitLab project in the format <user>/<project> or just <project>",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Exit after printing target jobs and dependencies",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-job-log",
|
||||
action="store_true",
|
||||
help="When there is only one target job, inhibit the job trace output in the console.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--polling-period",
|
||||
type=int,
|
||||
default=REFRESH_WAIT_JOBS,
|
||||
help=f"Specify the waiting seconds between monitor loops. (Default: {REFRESH_WAIT_JOBS})",
|
||||
)
|
||||
|
||||
|
||||
mutex_group1 = parser.add_mutually_exclusive_group()
|
||||
mutex_group1.add_argument(
|
||||
"--rev",
|
||||
metavar="id",
|
||||
type=str,
|
||||
default="HEAD",
|
||||
help="Repository git commit-ish, tag or branch name (default: HEAD)",
|
||||
)
|
||||
mutex_group1.add_argument(
|
||||
"--pipeline-url",
|
||||
metavar="url",
|
||||
type=str,
|
||||
help="URL of the pipeline to use, instead of auto-detecting it.",
|
||||
)
|
||||
mutex_group1.add_argument(
|
||||
"--mr",
|
||||
metavar="id",
|
||||
type=int,
|
||||
help="ID of a merge request; the latest pipeline in that MR will be used.",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# argparse doesn't support groups inside add_mutually_exclusive_group(),
|
||||
# which means we can't just put `--project` and `--rev` in a group together,
|
||||
# we have to do this by heand instead.
|
||||
if args.pipeline_url and args.project != parser.get_default("project"):
|
||||
# weird phrasing but it's the error add_mutually_exclusive_group() gives
|
||||
parser.error("argument --project: not allowed with argument --pipeline-url")
|
||||
|
||||
return args
|
||||
|
||||
|
||||
def print_detected_jobs(
|
||||
target_dep_dag: "Dag",
|
||||
dependency_jobs: Iterable[str],
|
||||
target_jobs: Iterable[str],
|
||||
) -> None:
|
||||
def print_job_set(color: str, kind: str, job_set: Iterable[str]):
|
||||
job_list = list(job_set)
|
||||
print(f"{color}Running {len(job_list)} {kind} jobs:")
|
||||
print_formatted_list(job_list, indentation=8, color=color)
|
||||
|
||||
print("[yellow]Detected target job and its dependencies:")
|
||||
print_dag(target_dep_dag, indentation=8, color="[yellow]")
|
||||
print_job_set("[magenta]", "dependency", dependency_jobs)
|
||||
print_job_set("[blue]", "target", target_jobs)
|
||||
|
||||
|
||||
def find_dependencies(
|
||||
server: str,
|
||||
token: str | None,
|
||||
job_filter: callable,
|
||||
project_path: str,
|
||||
iid: int
|
||||
) -> set[str]:
|
||||
"""
|
||||
Find the dependencies of the target jobs in a GitLab pipeline.
|
||||
|
||||
This function uses the GitLab GraphQL API to fetch the job dependency graph
|
||||
of a pipeline, filters the graph to only include the target jobs and their
|
||||
dependencies, and returns the names of these jobs.
|
||||
|
||||
Args:
|
||||
server (str): The url to the GitLab server.
|
||||
token (str | None): The GitLab API token. If None, the API is accessed without
|
||||
authentication.
|
||||
target_jobs_regex (re.Pattern): A regex pattern to match the names of the target jobs.
|
||||
project_path (str): The path of the GitLab project.
|
||||
iid (int): The internal ID of the pipeline.
|
||||
|
||||
Returns:
|
||||
set[str]: A set of the names of the target jobs and their dependencies.
|
||||
|
||||
Raises:
|
||||
SystemExit: If no target jobs are found in the pipeline.
|
||||
"""
|
||||
gql_instance = GitlabGQL(
|
||||
url=f"{server}/api/graphql",
|
||||
token=token
|
||||
)
|
||||
dag = create_job_needs_dag(
|
||||
gql_instance, {"projectPath": project_path.path_with_namespace, "iid": iid}
|
||||
)
|
||||
|
||||
target_dep_dag = filter_dag(dag, job_filter)
|
||||
if not target_dep_dag:
|
||||
print("[red]The job(s) were not found in the pipeline.")
|
||||
sys.exit(1)
|
||||
|
||||
dependency_jobs = set(chain.from_iterable(d["needs"] for d in target_dep_dag.values()))
|
||||
target_jobs = set(target_dep_dag.keys())
|
||||
print_detected_jobs(target_dep_dag, dependency_jobs, target_jobs)
|
||||
return target_jobs.union(dependency_jobs)
|
||||
|
||||
|
||||
def print_monitor_summary(
|
||||
execution_collection: Dict[str, Dict[int, Tuple[float, str, str]]],
|
||||
t_start: float,
|
||||
) -> None:
|
||||
"""Summary of the test execution"""
|
||||
t_end = time.perf_counter()
|
||||
spend_minutes = (t_end - t_start) / 60
|
||||
print(f"⏲ Duration of script execution: {spend_minutes:0.1f} minutes") # U+23F2 Timer clock
|
||||
if len(execution_collection) == 0:
|
||||
return
|
||||
print(f"⏲ Jobs execution times:") # U+23F2 Timer clock
|
||||
job_names = list(execution_collection.keys())
|
||||
job_names.sort()
|
||||
name_field_pad = len(max(job_names, key=len)) + 2
|
||||
for name in job_names:
|
||||
job_executions = execution_collection[name]
|
||||
job_times = ', '.join([__job_duration_record(job_execution)
|
||||
for job_execution in sorted(job_executions.items())])
|
||||
print(f"* {name:{name_field_pad}}: ({len(job_executions)}) {job_times}")
|
||||
|
||||
|
||||
def __job_duration_record(dict_item: tuple) -> str:
|
||||
"""
|
||||
Format each pair of job and its duration.
|
||||
:param job_execution: item of execution_collection[name][idn]: Dict[int, Tuple[float, str, str]]
|
||||
"""
|
||||
job_id = f"{dict_item[0]}" # dictionary key
|
||||
job_duration, job_status, job_url = dict_item[1] # dictionary value, the tuple
|
||||
return (
|
||||
f"{STATUS_COLORS[job_status]}"
|
||||
f"{link2print(job_url, job_id)}: {pretty_duration(job_duration):>8}"
|
||||
)
|
||||
|
||||
|
||||
def link2print(url: str, text: str, text_pad: int = 0) -> str:
|
||||
text = str(text)
|
||||
text_pad = len(text) if text_pad < 1 else text_pad
|
||||
if console.is_terminal:
|
||||
return f"[link={url}]{text:{text_pad}}[/link]"
|
||||
else:
|
||||
return f"{text:{text_pad}}"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
try:
|
||||
t_start = time.perf_counter()
|
||||
|
||||
args = parse_args()
|
||||
|
||||
token = read_token(args.token)
|
||||
|
||||
gl = gitlab.Gitlab(url=args.server,
|
||||
private_token=token,
|
||||
retry_transient_errors=True)
|
||||
|
||||
REV: str = args.rev
|
||||
|
||||
if args.pipeline_url:
|
||||
pipe, cur_project = get_gitlab_pipeline_from_url(gl, args.pipeline_url)
|
||||
REV = pipe.sha
|
||||
else:
|
||||
mesa_project = gl.projects.get("mesa/mesa")
|
||||
projects = [mesa_project]
|
||||
if args.mr:
|
||||
REV = mesa_project.mergerequests.get(args.mr).sha
|
||||
else:
|
||||
REV = check_output(['git', 'rev-parse', REV]).decode('ascii').strip()
|
||||
|
||||
if args.rev == 'HEAD':
|
||||
try:
|
||||
branch_name = check_output([
|
||||
'git', 'symbolic-ref', '-q', 'HEAD',
|
||||
]).decode('ascii').strip()
|
||||
except CalledProcessError:
|
||||
branch_name = ""
|
||||
|
||||
# Ignore detached heads
|
||||
if branch_name:
|
||||
tracked_remote = check_output([
|
||||
'git', 'for-each-ref', '--format=%(upstream)',
|
||||
branch_name,
|
||||
]).decode('ascii').strip()
|
||||
|
||||
# Ignore local branches that do not track any remote
|
||||
if tracked_remote:
|
||||
remote_rev = check_output([
|
||||
'git', 'rev-parse', tracked_remote,
|
||||
]).decode('ascii').strip()
|
||||
|
||||
if REV != remote_rev:
|
||||
print(
|
||||
f"Local HEAD commit {REV[:10]} is different than "
|
||||
f"tracked remote HEAD commit {remote_rev[:10]}"
|
||||
)
|
||||
print("Did you forget to `git push` ?")
|
||||
|
||||
projects.append(get_gitlab_project(gl, args.project))
|
||||
(pipe, cur_project) = wait_for_pipeline(projects, REV)
|
||||
|
||||
print(f"Revision: {REV}")
|
||||
print(f"Pipeline: {pipe.web_url}")
|
||||
|
||||
target = '|'.join(args.target)
|
||||
target = target.strip()
|
||||
|
||||
print(f"🞋 target job: [blue]{target}") # U+1F78B Round target
|
||||
|
||||
# Implicitly include `parallel:` jobs
|
||||
target = f'({target})' + r'( \d+/\d+)?'
|
||||
|
||||
target_jobs_regex = re.compile(target)
|
||||
|
||||
include_stage = '|'.join(args.include_stage)
|
||||
include_stage = include_stage.strip()
|
||||
|
||||
print(f"🞋 target from stages: [blue]{include_stage}") # U+1F78B Round target
|
||||
|
||||
include_stage_regex = re.compile(include_stage)
|
||||
|
||||
exclude_stage = '|'.join(args.exclude_stage)
|
||||
exclude_stage = exclude_stage.strip()
|
||||
|
||||
print(f"🞋 target excluding stages: [blue]{exclude_stage}") # U+1F78B Round target
|
||||
|
||||
exclude_stage_regex = re.compile(exclude_stage)
|
||||
|
||||
print(f"🞋 target jobs with tags: [blue]{str(args.job_tags)}") # U+1F78B Round target
|
||||
job_tags_regexes = [re.compile(job_tag) for job_tag in args.job_tags]
|
||||
|
||||
def job_filter(
|
||||
job_name: str,
|
||||
job_stage: str,
|
||||
job_tags: set[str],
|
||||
) -> bool:
|
||||
"""
|
||||
Apply user-specified filters to a job, and return whether the
|
||||
filters allow that job (True) or not (False).
|
||||
"""
|
||||
if not target_jobs_regex.fullmatch(job_name):
|
||||
return False
|
||||
if not include_stage_regex.fullmatch(job_stage):
|
||||
return False
|
||||
if exclude_stage_regex.fullmatch(job_stage):
|
||||
return False
|
||||
if not all(
|
||||
any(job_tags_regex.fullmatch(tag) for tag in job_tags)
|
||||
for job_tags_regex in job_tags_regexes
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
deps = find_dependencies(
|
||||
server=args.server,
|
||||
token=token,
|
||||
job_filter=job_filter,
|
||||
iid=pipe.iid,
|
||||
project_path=cur_project
|
||||
)
|
||||
|
||||
if args.dry_run:
|
||||
sys.exit(0)
|
||||
|
||||
target_job_id, ret, exec_t = monitor_pipeline(
|
||||
cur_project,
|
||||
pipe,
|
||||
job_filter,
|
||||
deps,
|
||||
args.stress,
|
||||
args.no_job_log,
|
||||
args.polling_period,
|
||||
)
|
||||
|
||||
if target_job_id:
|
||||
print_log(cur_project, target_job_id)
|
||||
|
||||
print_monitor_summary(exec_t, t_start)
|
||||
|
||||
sys.exit(ret)
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
|
||||
this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
|
||||
readonly this_dir
|
||||
|
||||
exec \
|
||||
"$this_dir/../python-venv.sh" \
|
||||
"$this_dir/requirements.txt" \
|
||||
"$this_dir/ci_run_n_monitor.py" "$@"
|
||||
@@ -0,0 +1,353 @@
|
||||
import argparse
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from structured_logger import StructuredLogger
|
||||
|
||||
|
||||
class CustomLogger:
|
||||
def __init__(self, log_file):
|
||||
self.log_file = log_file
|
||||
self.logger = StructuredLogger(file_name=self.log_file)
|
||||
|
||||
def get_last_dut_job(self):
|
||||
"""
|
||||
Gets the details of the most recent DUT job.
|
||||
|
||||
Returns:
|
||||
dict: Details of the most recent DUT job.
|
||||
|
||||
Raises:
|
||||
ValueError: If no DUT jobs are found in the logger's data.
|
||||
"""
|
||||
try:
|
||||
job = self.logger.data["dut_jobs"][-1]
|
||||
except KeyError:
|
||||
raise ValueError(
|
||||
"No DUT jobs found. Please create a job via create_dut_job call."
|
||||
)
|
||||
|
||||
return job
|
||||
|
||||
def update(self, **kwargs):
|
||||
"""
|
||||
Updates the log file with provided key-value pairs.
|
||||
|
||||
Args:
|
||||
**kwargs: Key-value pairs to be updated.
|
||||
|
||||
"""
|
||||
with self.logger.edit_context():
|
||||
for key, value in kwargs.items():
|
||||
self.logger.data[key] = value
|
||||
|
||||
def create_dut_job(self, **kwargs):
|
||||
"""
|
||||
Creates a new DUT job with provided key-value pairs.
|
||||
|
||||
Args:
|
||||
**kwargs: Key-value pairs for the new DUT job.
|
||||
|
||||
"""
|
||||
with self.logger.edit_context():
|
||||
if "dut_jobs" not in self.logger.data:
|
||||
self.logger.data["dut_jobs"] = []
|
||||
new_job = {
|
||||
"status": "",
|
||||
"submitter_start_time": datetime.now().isoformat(),
|
||||
"dut_submit_time": "",
|
||||
"dut_start_time": "",
|
||||
"dut_end_time": "",
|
||||
"dut_duration_time": "",
|
||||
"dut_name": "",
|
||||
"dut_state": "pending",
|
||||
"dut_job_phases": [],
|
||||
**kwargs,
|
||||
}
|
||||
self.logger.data["dut_jobs"].append(new_job)
|
||||
|
||||
def update_dut_job(self, key, value):
|
||||
"""
|
||||
Updates the last DUT job with a key-value pair.
|
||||
|
||||
Args:
|
||||
key : The key to be updated.
|
||||
value: The value to be assigned.
|
||||
|
||||
"""
|
||||
with self.logger.edit_context():
|
||||
job = self.get_last_dut_job()
|
||||
job[key] = value
|
||||
|
||||
def update_status_fail(self, reason=""):
|
||||
"""
|
||||
Sets the status of the last DUT job to 'fail' and logs the failure reason.
|
||||
|
||||
Args:
|
||||
reason (str, optional): The reason for the failure. Defaults to "".
|
||||
|
||||
"""
|
||||
with self.logger.edit_context():
|
||||
job = self.get_last_dut_job()
|
||||
job["status"] = "fail"
|
||||
job["dut_job_fail_reason"] = reason
|
||||
|
||||
def create_job_phase(self, phase_name):
|
||||
"""
|
||||
Creates a new job phase for the last DUT job.
|
||||
|
||||
Args:
|
||||
phase_name : The name of the new job phase.
|
||||
|
||||
"""
|
||||
with self.logger.edit_context():
|
||||
job = self.get_last_dut_job()
|
||||
if job["dut_job_phases"] and job["dut_job_phases"][-1]["end_time"] == "":
|
||||
# If the last phase exists and its end time is empty, set the end time
|
||||
timestamp = datetime.now().isoformat()
|
||||
job["dut_job_phases"][-1]["end_time"] = timestamp
|
||||
job["dut_job_phases"][-1]["duration_time"] = self.get_duration_time(job["dut_job_phases"][-1]["start_time"], timestamp)
|
||||
|
||||
# Create a new phase
|
||||
phase_data = {
|
||||
"name": phase_name,
|
||||
"start_time": datetime.now().isoformat(),
|
||||
"end_time": "",
|
||||
"duration_time": "",
|
||||
}
|
||||
job["dut_job_phases"].append(phase_data)
|
||||
|
||||
def check_dut_timings(self, job):
|
||||
"""
|
||||
Check the timing sequence of a job to ensure logical consistency.
|
||||
|
||||
The function verifies that the job's submission time is not earlier than its start time and that
|
||||
the job's end time is not earlier than its start time. If either of these conditions is found to be true,
|
||||
an error is logged for each instance of inconsistency.
|
||||
|
||||
Args:
|
||||
job (dict): A dictionary containing timing information of a job. Expected keys are 'dut_start_time',
|
||||
'dut_submit_time', and 'dut_end_time'.
|
||||
|
||||
Returns:
|
||||
None: This function does not return a value; it logs errors if timing inconsistencies are detected.
|
||||
|
||||
The function checks the following:
|
||||
- If 'dut_start_time' and 'dut_submit_time' are both present and correctly sequenced.
|
||||
- If 'dut_start_time' and 'dut_end_time' are both present and correctly sequenced.
|
||||
"""
|
||||
|
||||
# Check if the start time and submit time exist
|
||||
if job.get("dut_start_time") and job.get("dut_submit_time"):
|
||||
# If they exist, check if the submission time is before the start time
|
||||
if job["dut_start_time"] < job["dut_submit_time"]:
|
||||
logging.error("Job submission is happening before job start.")
|
||||
|
||||
# Check if the start time and end time exist
|
||||
if job.get("dut_start_time") and job.get("dut_end_time"):
|
||||
# If they exist, check if the end time is after the start time
|
||||
if job["dut_end_time"] < job["dut_start_time"]:
|
||||
logging.error("Job ended before it started.")
|
||||
|
||||
def get_duration_time(self, start_time, end_time):
|
||||
"""
|
||||
Computes duration time, in minutes and seconds.
|
||||
"""
|
||||
try:
|
||||
start = datetime.fromisoformat(start_time)
|
||||
end = datetime.fromisoformat(end_time)
|
||||
duration = end - start
|
||||
return str(duration)
|
||||
except ValueError:
|
||||
return ""
|
||||
|
||||
# Method to update DUT start, submit and end time
|
||||
def update_dut_time(self, value, custom_time):
|
||||
"""
|
||||
Updates DUT start, submit, and end times.
|
||||
|
||||
Args:
|
||||
value : Specifies which DUT time to update. Options: 'start', 'submit', 'end'.
|
||||
custom_time : Custom time to set. If None, use current time.
|
||||
|
||||
Raises:
|
||||
ValueError: If an invalid argument is provided for value.
|
||||
|
||||
"""
|
||||
with self.logger.edit_context():
|
||||
job = self.get_last_dut_job()
|
||||
timestamp = custom_time if custom_time else datetime.now().isoformat()
|
||||
if value == "start":
|
||||
job["dut_start_time"] = timestamp
|
||||
job["dut_state"] = "running"
|
||||
elif value == "submit":
|
||||
job["dut_submit_time"] = timestamp
|
||||
job["dut_state"] = "submitted"
|
||||
elif value == "end":
|
||||
job["dut_end_time"] = timestamp
|
||||
job["dut_state"] = "finished"
|
||||
job["dut_duration_time"] = self.get_duration_time(job["dut_start_time"], job["dut_end_time"])
|
||||
else:
|
||||
raise ValueError(
|
||||
"Error: Invalid argument provided for --update-dut-time. Use 'start', 'submit', 'end'."
|
||||
)
|
||||
# check the sanity of the partial structured log
|
||||
self.check_dut_timings(job)
|
||||
|
||||
def close_dut_job(self):
|
||||
"""
|
||||
Closes the most recent DUT (Device Under Test) job in the logger's data.
|
||||
|
||||
The method performs the following operations:
|
||||
1. Validates if there are any DUT jobs in the logger's data.
|
||||
2. If the last phase of the most recent DUT job has an empty end time, it sets the end time to the current time.
|
||||
|
||||
Raises:
|
||||
ValueError: If no DUT jobs are found in the logger's data.
|
||||
"""
|
||||
with self.logger.edit_context():
|
||||
job = self.get_last_dut_job()
|
||||
# Check if the last phase exists and its end time is empty, then set the end time
|
||||
if job["dut_job_phases"] and job["dut_job_phases"][-1]["end_time"] == "":
|
||||
timestamp = datetime.now().isoformat()
|
||||
job["dut_job_phases"][-1]["end_time"] = timestamp
|
||||
job["dut_job_phases"][-1]["duration_time"] = self.get_duration_time(job["dut_job_phases"][-1]["start_time"], timestamp)
|
||||
|
||||
def close(self):
|
||||
"""
|
||||
Closes the most recent DUT (Device Under Test) job in the logger's data.
|
||||
|
||||
The method performs the following operations:
|
||||
1. Determines the combined status of all DUT jobs.
|
||||
2. Sets the submitter's end time to the current time.
|
||||
3. Updates the DUT attempt counter to reflect the total number of DUT jobs.
|
||||
|
||||
"""
|
||||
with self.logger.edit_context():
|
||||
job_status = []
|
||||
for job in self.logger.data["dut_jobs"]:
|
||||
if "status" in job:
|
||||
job_status.append(job["status"])
|
||||
|
||||
if not job_status:
|
||||
job_combined_status = "null"
|
||||
else:
|
||||
# Get job_combined_status
|
||||
if "pass" in job_status:
|
||||
job_combined_status = "pass"
|
||||
else:
|
||||
job_combined_status = "fail"
|
||||
|
||||
self.logger.data["job_combined_status"] = job_combined_status
|
||||
self.logger.data["dut_attempt_counter"] = len(self.logger.data["dut_jobs"])
|
||||
job["submitter_end_time"] = datetime.now().isoformat()
|
||||
|
||||
|
||||
def process_args(args):
|
||||
# Function to process key-value pairs and call corresponding logger methods
|
||||
def process_key_value_pairs(args_list, action_func):
|
||||
if not args_list:
|
||||
raise ValueError(
|
||||
f"No key-value pairs provided for {action_func.__name__.replace('_', '-')}"
|
||||
)
|
||||
if len(args_list) % 2 != 0:
|
||||
raise ValueError(
|
||||
f"Incomplete key-value pairs for {action_func.__name__.replace('_', '-')}"
|
||||
)
|
||||
kwargs = dict(zip(args_list[::2], args_list[1::2]))
|
||||
action_func(**kwargs)
|
||||
|
||||
# Create a CustomLogger object with the specified log file path
|
||||
custom_logger = CustomLogger(Path(args.log_file))
|
||||
|
||||
if args.update:
|
||||
process_key_value_pairs(args.update, custom_logger.update)
|
||||
|
||||
if args.create_dut_job:
|
||||
process_key_value_pairs(args.create_dut_job, custom_logger.create_dut_job)
|
||||
|
||||
if args.update_dut_job:
|
||||
key, value = args.update_dut_job
|
||||
custom_logger.update_dut_job(key, value)
|
||||
|
||||
if args.create_job_phase:
|
||||
custom_logger.create_job_phase(args.create_job_phase)
|
||||
|
||||
if args.update_status_fail:
|
||||
custom_logger.update_status_fail(args.update_status_fail)
|
||||
|
||||
if args.update_dut_time:
|
||||
if len(args.update_dut_time) == 2:
|
||||
action, custom_time = args.update_dut_time
|
||||
elif len(args.update_dut_time) == 1:
|
||||
action, custom_time = args.update_dut_time[0], None
|
||||
else:
|
||||
raise ValueError("Invalid number of values for --update-dut-time")
|
||||
|
||||
if action in ["start", "end", "submit"]:
|
||||
custom_logger.update_dut_time(action, custom_time)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Error: Invalid argument provided for --update-dut-time. Use 'start', 'submit', 'end'."
|
||||
)
|
||||
|
||||
if args.close_dut_job:
|
||||
custom_logger.close_dut_job()
|
||||
|
||||
if args.close:
|
||||
custom_logger.close()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Custom Logger Command Line Tool")
|
||||
parser.add_argument("log_file", help="Path to the log file")
|
||||
parser.add_argument(
|
||||
"--update",
|
||||
nargs=argparse.ZERO_OR_MORE,
|
||||
metavar=("key", "value"),
|
||||
help="Update a key-value pair e.g., --update key1 value1 key2 value2)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--create-dut-job",
|
||||
nargs=argparse.ZERO_OR_MORE,
|
||||
metavar=("key", "value"),
|
||||
help="Create a new DUT job with key-value pairs (e.g., --create-dut-job key1 value1 key2 value2)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--update-dut-job",
|
||||
nargs=argparse.ZERO_OR_MORE,
|
||||
metavar=("key", "value"),
|
||||
help="Update a key-value pair in DUT job",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--create-job-phase",
|
||||
help="Create a new job phase (e.g., --create-job-phase name)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--update-status-fail",
|
||||
help="Update fail as the status and log the failure reason (e.g., --update-status-fail reason)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--update-dut-time",
|
||||
nargs=argparse.ZERO_OR_MORE,
|
||||
metavar=("action", "custom_time"),
|
||||
help="Update DUT start and end time. Provide action ('start', 'submit', 'end') and custom_time (e.g., '2023-01-01T12:00:00')",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--close-dut-job",
|
||||
action="store_true",
|
||||
help="Close the dut job by updating end time of last dut job)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--close",
|
||||
action="store_true",
|
||||
help="Updates combined status, submitter's end time and DUT attempt counter",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
process_args(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
|
||||
# Helper script to download the schema GraphQL from Gitlab to enable IDEs to
|
||||
# assist the developer to edit gql files
|
||||
|
||||
SOURCE_DIR=$(dirname "$(realpath "$0")")
|
||||
|
||||
(
|
||||
cd $SOURCE_DIR || exit 1
|
||||
gql-cli https://gitlab.freedesktop.org/api/graphql --print-schema > schema.graphql
|
||||
)
|
||||
@@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright © 2020 - 2022 Collabora Ltd.
|
||||
# Authors:
|
||||
# Tomeu Vizoso <[email protected]>
|
||||
# David Heidelberg <[email protected]>
|
||||
# Guilherme Gallo <[email protected]>
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
'''Shared functions between the scripts.'''
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from functools import cache
|
||||
from pathlib import Path
|
||||
|
||||
GITLAB_URL = "https://gitlab.freedesktop.org"
|
||||
TOKEN_DIR = Path(os.environ.get("XDG_CONFIG_HOME", "")
|
||||
if os.environ.get("XDG_CONFIG_HOME", None)
|
||||
else Path.home() / ".config")
|
||||
|
||||
# Known GitLab token prefixes: https://docs.gitlab.com/security/tokens/#token-prefixes
|
||||
TOKEN_PREFIXES: dict[str, str] = {
|
||||
"Personal access token": "glpat-",
|
||||
"OAuth Application Secret": "gloas-",
|
||||
"Deploy token": "gldt-",
|
||||
"Runner authentication token": "glrt-",
|
||||
"CI/CD Job token": "glcbt-",
|
||||
"Trigger token": "glptt-",
|
||||
"Feed token": "glft-",
|
||||
"Incoming mail token": "glimt-",
|
||||
"GitLab Agent for Kubernetes token": "glagent-",
|
||||
"SCIM Tokens": "glsoat-",
|
||||
}
|
||||
|
||||
|
||||
@cache
|
||||
def print_once(*args, **kwargs):
|
||||
"""Print without spamming the output"""
|
||||
print(*args, **kwargs)
|
||||
|
||||
|
||||
def pretty_duration(seconds: int | float) -> str:
|
||||
"""Pretty print duration"""
|
||||
seconds = int(seconds)
|
||||
hours, rem = divmod(seconds, 3600)
|
||||
minutes, seconds = divmod(rem, 60)
|
||||
if hours:
|
||||
return f"{hours:0.0f}h{minutes:02.0f}m{seconds:02.0f}s"
|
||||
if minutes:
|
||||
return f"{minutes:0.0f}m{seconds:02.0f}s"
|
||||
return f"{seconds:0.0f}s"
|
||||
|
||||
|
||||
def get_gitlab_pipeline_from_url(gl, pipeline_url) -> tuple:
|
||||
"""
|
||||
Extract the project and pipeline object from the url string
|
||||
:param gl: Gitlab object
|
||||
:param pipeline_url: string with a url to a pipeline
|
||||
:return: ProjectPipeline, Project objects
|
||||
"""
|
||||
pattern = rf"^{re.escape(GITLAB_URL)}/(.*)/-/pipelines/([0-9]+)$"
|
||||
match = re.match(pattern, pipeline_url)
|
||||
if not match:
|
||||
raise AssertionError(f"url {pipeline_url} doesn't follow the pattern {pattern}")
|
||||
namespace_with_project, pipeline_id = match.groups()
|
||||
cur_project = gl.projects.get(namespace_with_project)
|
||||
pipe = cur_project.pipelines.get(pipeline_id)
|
||||
return pipe, cur_project
|
||||
|
||||
|
||||
def get_gitlab_project(glab, name: str):
|
||||
"""Finds a specified gitlab project for given user"""
|
||||
if "/" in name:
|
||||
project_path = name
|
||||
else:
|
||||
glab.auth()
|
||||
username = glab.user.username
|
||||
project_path = f"{username}/{name}"
|
||||
return glab.projects.get(project_path)
|
||||
|
||||
|
||||
def get_token_from_default_dir() -> str:
|
||||
"""
|
||||
Retrieves the GitLab token from the default directory.
|
||||
|
||||
Returns:
|
||||
str: The path to the GitLab token file.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If the token file is not found.
|
||||
"""
|
||||
token_file = TOKEN_DIR / "gitlab-token"
|
||||
try:
|
||||
return str(token_file.resolve())
|
||||
except FileNotFoundError as ex:
|
||||
print(
|
||||
f"Could not find {token_file}, please provide a token file as an argument"
|
||||
)
|
||||
raise ex
|
||||
|
||||
|
||||
def validate_gitlab_token(token: str) -> bool:
|
||||
# Match against recognised token prefixes
|
||||
token_suffix = None
|
||||
for token_type, token_prefix in TOKEN_PREFIXES.items():
|
||||
if token.startswith(token_prefix):
|
||||
logging.info(f"Found probable token type: {token_type}")
|
||||
token_suffix = token[len(token_prefix):]
|
||||
break
|
||||
|
||||
if not token_suffix:
|
||||
return False
|
||||
|
||||
# Basic validation of the token suffix based on:
|
||||
# https://gitlab.com/gitlab-org/gitlab/-/blob/master/gems/gitlab-secret_detection/lib/gitleaks.toml
|
||||
if not re.match(r"(\w+-)?[0-9a-zA-Z_\-]{20,64}", token_suffix):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def get_token_from_arg(token_arg: str | Path | None) -> str | None:
|
||||
if not token_arg:
|
||||
logging.info("No token provided.")
|
||||
return None
|
||||
|
||||
token_path = Path(token_arg)
|
||||
if token_path.is_file():
|
||||
return read_token_from_file(token_path)
|
||||
|
||||
return handle_direct_token(token_path, token_arg)
|
||||
|
||||
|
||||
def read_token_from_file(token_path: Path) -> str:
|
||||
token = token_path.read_text().strip()
|
||||
logging.info(f"Token read from file: {token_path}")
|
||||
return token
|
||||
|
||||
|
||||
def handle_direct_token(token_path: Path, token_arg: str | Path) -> str | None:
|
||||
if token_path == Path(get_token_from_default_dir()):
|
||||
logging.warning(
|
||||
f"The default token file {token_path} was not found. "
|
||||
"Please provide a token file or a token directly via --token arg."
|
||||
)
|
||||
return None
|
||||
logging.info("Token provided directly as an argument.")
|
||||
return str(token_arg)
|
||||
|
||||
|
||||
def read_token(token_arg: str | Path | None) -> str | None:
|
||||
token = get_token_from_arg(token_arg)
|
||||
if token and not validate_gitlab_token(token):
|
||||
logging.warning("The provided token is either an old token or does not seem to "
|
||||
"be a valid token.")
|
||||
logging.warning("Newer tokens are the ones created from a Gitlab 14.5+ instance.")
|
||||
logging.warning("See https://about.gitlab.com/releases/2021/11/22/"
|
||||
"gitlab-14-5-released/"
|
||||
"#new-gitlab-access-token-prefix-and-detection")
|
||||
return token
|
||||
|
||||
|
||||
def wait_for_pipeline(projects, sha: str, timeout=None):
|
||||
"""await until pipeline appears in Gitlab"""
|
||||
project_names = [project.path_with_namespace for project in projects]
|
||||
print(f"⏲ for the pipeline to appear in {project_names}..", end="")
|
||||
start_time = time.time()
|
||||
while True:
|
||||
for project in projects:
|
||||
pipelines = project.pipelines.list(sha=sha)
|
||||
if pipelines:
|
||||
print("", flush=True)
|
||||
return (pipelines[0], project)
|
||||
print("", end=".", flush=True)
|
||||
if timeout and time.time() - start_time > timeout:
|
||||
print(" not found", flush=True)
|
||||
return (None, None)
|
||||
time.sleep(1)
|
||||
|
||||
@cache
|
||||
def is_gitlab_job() -> bool:
|
||||
return os.getenv("CI_JOB_ID") is not None
|
||||
Executable
+641
@@ -0,0 +1,641 @@
|
||||
#!/usr/bin/env python3
|
||||
# For the dependencies, see the requirements.txt
|
||||
|
||||
import logging
|
||||
import re
|
||||
import sys
|
||||
import traceback
|
||||
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser, Namespace
|
||||
from collections import OrderedDict
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass, field
|
||||
from itertools import accumulate
|
||||
from os import get_terminal_size, getenv
|
||||
from pathlib import Path
|
||||
from subprocess import check_output
|
||||
from textwrap import dedent
|
||||
from typing import Any, Iterable, Optional, TypedDict, Union
|
||||
|
||||
import yaml
|
||||
from filecache import DAY, filecache
|
||||
from gitlab_common import get_token_from_default_dir, is_gitlab_job
|
||||
from gql import Client, gql
|
||||
from gql.transport.requests import RequestsHTTPTransport
|
||||
from graphql import DocumentNode
|
||||
from rich.console import Console
|
||||
|
||||
DEFAULT_TERMINAL_SIZE: int = 80 # columns
|
||||
|
||||
|
||||
class DagNode(TypedDict):
|
||||
needs: set[str]
|
||||
stage: str
|
||||
tags: set[str]
|
||||
# `name` is redundant but is here for retro-compatibility
|
||||
name: str
|
||||
|
||||
|
||||
# see create_job_needs_dag function for more details
|
||||
Dag = dict[str, DagNode]
|
||||
|
||||
|
||||
StageSeq = OrderedDict[str, set[str]]
|
||||
|
||||
if is_gitlab_job():
|
||||
console = Console(highlight=False, no_color=False, color_system="truecolor", width=120)
|
||||
else:
|
||||
console = Console(highlight=False)
|
||||
print = console.print
|
||||
|
||||
|
||||
def get_project_root_dir():
|
||||
root_path = Path(__file__).parent.parent.parent.resolve()
|
||||
gitlab_file = root_path / ".gitlab-ci.yml"
|
||||
assert gitlab_file.exists()
|
||||
|
||||
return root_path
|
||||
|
||||
|
||||
@dataclass
|
||||
class GitlabGQL:
|
||||
_transport: Any = field(init=False)
|
||||
client: Client = field(init=False)
|
||||
url: str = "https://gitlab.freedesktop.org/api/graphql"
|
||||
token: Optional[str] = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self._setup_gitlab_gql_client()
|
||||
|
||||
def _setup_gitlab_gql_client(self) -> None:
|
||||
# Select your transport with a defined url endpoint
|
||||
headers = {}
|
||||
if self.token:
|
||||
headers["Authorization"] = f"Bearer {self.token}"
|
||||
self._transport = RequestsHTTPTransport(url=self.url, headers=headers)
|
||||
|
||||
# Create a GraphQL client using the defined transport
|
||||
self.client = Client(transport=self._transport, fetch_schema_from_transport=True)
|
||||
|
||||
def query(
|
||||
self,
|
||||
gql_file: Union[Path, str],
|
||||
params: dict[str, Any] = {},
|
||||
operation_name: Optional[str] = None,
|
||||
paginated_key_loc: Iterable[str] = [],
|
||||
disable_cache: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
def run_uncached() -> dict[str, Any]:
|
||||
if paginated_key_loc:
|
||||
return self._sweep_pages(gql_file, params, operation_name, paginated_key_loc)
|
||||
return self._query(gql_file, params, operation_name)
|
||||
|
||||
if disable_cache:
|
||||
return run_uncached()
|
||||
|
||||
try:
|
||||
# Create an auxiliary variable to deliver a cached result and enable catching exceptions
|
||||
# Decorate the query to be cached
|
||||
if paginated_key_loc:
|
||||
result = self._sweep_pages_cached(
|
||||
gql_file, params, operation_name, paginated_key_loc
|
||||
)
|
||||
else:
|
||||
result = self._query_cached(gql_file, params, operation_name)
|
||||
return result # type: ignore
|
||||
except Exception as ex:
|
||||
logging.error(f"Cached query failed with {ex}")
|
||||
# print exception traceback
|
||||
traceback_str = "".join(traceback.format_exception(ex))
|
||||
logging.error(traceback_str)
|
||||
self.invalidate_query_cache()
|
||||
logging.error("Cache invalidated, retrying without cache")
|
||||
return run_uncached()
|
||||
|
||||
def _query(
|
||||
self,
|
||||
gql_file: Union[Path, str],
|
||||
params: dict[str, Any] = {},
|
||||
operation_name: Optional[str] = None,
|
||||
) -> dict[str, Any]:
|
||||
# Provide a GraphQL query
|
||||
source_path: Path = Path(__file__).parent
|
||||
pipeline_query_file: Path = source_path / gql_file
|
||||
|
||||
query: DocumentNode
|
||||
with open(pipeline_query_file, "r") as f:
|
||||
pipeline_query = f.read()
|
||||
query = gql(pipeline_query)
|
||||
|
||||
# Execute the query on the transport
|
||||
return self.client.execute_sync(
|
||||
query, variable_values=params, operation_name=operation_name
|
||||
)
|
||||
|
||||
@filecache(DAY)
|
||||
def _sweep_pages_cached(self, *args, **kwargs):
|
||||
return self._sweep_pages(*args, **kwargs)
|
||||
|
||||
@filecache(DAY)
|
||||
def _query_cached(self, *args, **kwargs):
|
||||
return self._query(*args, **kwargs)
|
||||
|
||||
def _sweep_pages(
|
||||
self, query, params, operation_name=None, paginated_key_loc: Iterable[str] = []
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Retrieve paginated data from a GraphQL API and concatenate the results into a single
|
||||
response.
|
||||
|
||||
Args:
|
||||
query: represents a filepath with the GraphQL query to be executed.
|
||||
params: a dictionary that contains the parameters to be passed to the query. These
|
||||
parameters can be used to filter or modify the results of the query.
|
||||
operation_name: The `operation_name` parameter is an optional parameter that specifies
|
||||
the name of the GraphQL operation to be executed. It is used when making a GraphQL
|
||||
query to specify which operation to execute if there are multiple operations defined
|
||||
in the GraphQL schema. If not provided, the default operation will be executed.
|
||||
paginated_key_loc (Iterable[str]): The `paginated_key_loc` parameter is an iterable of
|
||||
strings that represents the location of the paginated field within the response. It
|
||||
is used to extract the paginated field from the response and append it to the final
|
||||
result. The node has to be a list of objects with a `pageInfo` field that contains
|
||||
at least the `hasNextPage` and `endCursor` fields.
|
||||
|
||||
Returns:
|
||||
a dictionary containing the response from the query with the paginated field
|
||||
concatenated.
|
||||
"""
|
||||
|
||||
def fetch_page(cursor: str | None = None) -> dict[str, Any]:
|
||||
if cursor:
|
||||
params["cursor"] = cursor
|
||||
logging.info(
|
||||
f"Found more than 100 elements, paginating. "
|
||||
f"Current cursor at {cursor}"
|
||||
)
|
||||
|
||||
return self._query(query, params, operation_name)
|
||||
|
||||
# Execute the initial query
|
||||
response: dict[str, Any] = fetch_page()
|
||||
|
||||
# Initialize an empty list to store the final result
|
||||
final_partial_field: list[dict[str, Any]] = []
|
||||
|
||||
# Loop until all pages have been retrieved
|
||||
while True:
|
||||
# Get the partial field to be appended to the final result
|
||||
partial_field = response
|
||||
for key in paginated_key_loc:
|
||||
partial_field = partial_field[key]
|
||||
|
||||
# Append the partial field to the final result
|
||||
final_partial_field += partial_field["nodes"]
|
||||
|
||||
# Check if there are more pages to retrieve
|
||||
page_info = partial_field["pageInfo"]
|
||||
if not page_info["hasNextPage"]:
|
||||
break
|
||||
|
||||
# Execute the query with the updated cursor parameter
|
||||
response = fetch_page(page_info["endCursor"])
|
||||
|
||||
# Replace the "nodes" field in the original response with the final result
|
||||
partial_field["nodes"] = final_partial_field
|
||||
return response
|
||||
|
||||
def invalidate_query_cache(self) -> None:
|
||||
logging.warning("Invalidating query cache")
|
||||
try:
|
||||
self._sweep_pages._db.clear()
|
||||
self._query._db.clear()
|
||||
except AttributeError as ex:
|
||||
logging.warning(f"Could not invalidate cache, maybe it was not used in {ex.args}?")
|
||||
|
||||
|
||||
def insert_early_stage_jobs(stage_sequence: StageSeq, jobs_metadata: Dag) -> Dag:
|
||||
pre_processed_dag: dict[str, set[str]] = {}
|
||||
jobs_from_early_stages = list(accumulate(stage_sequence.values(), set.union))
|
||||
for job_name, metadata in jobs_metadata.items():
|
||||
final_needs: set[str] = deepcopy(metadata["needs"])
|
||||
# Pre-process jobs that are not based on needs field
|
||||
# e.g. sanity job in mesa MR pipelines
|
||||
if not final_needs:
|
||||
job_stage: str = jobs_metadata[job_name]["stage"]
|
||||
stage_index: int = list(stage_sequence.keys()).index(job_stage)
|
||||
if stage_index > 0:
|
||||
final_needs |= jobs_from_early_stages[stage_index - 1]
|
||||
pre_processed_dag[job_name] = final_needs
|
||||
|
||||
for job_name, needs in pre_processed_dag.items():
|
||||
jobs_metadata[job_name]["needs"] = needs
|
||||
|
||||
return jobs_metadata
|
||||
|
||||
|
||||
def traverse_dag_needs(jobs_metadata: Dag) -> None:
|
||||
created_jobs = set(jobs_metadata.keys())
|
||||
for job, metadata in jobs_metadata.items():
|
||||
final_needs: set = deepcopy(metadata["needs"]) & created_jobs
|
||||
# Post process jobs that are based on needs field
|
||||
partial = True
|
||||
|
||||
while partial:
|
||||
next_depth: set[str] = {n for dn in final_needs if dn in jobs_metadata for n in jobs_metadata[dn]["needs"]}
|
||||
partial: bool = not final_needs.issuperset(next_depth)
|
||||
final_needs = final_needs.union(next_depth)
|
||||
|
||||
jobs_metadata[job]["needs"] = final_needs
|
||||
|
||||
|
||||
def extract_stages_and_job_needs(
|
||||
pipeline_jobs: dict[str, Any], pipeline_stages: dict[str, Any]
|
||||
) -> tuple[StageSeq, Dag]:
|
||||
jobs_metadata = Dag()
|
||||
# Record the stage sequence to post process deps that are not based on needs
|
||||
# field, for example: sanity job
|
||||
stage_sequence: OrderedDict[str, set[str]] = OrderedDict()
|
||||
for stage in pipeline_stages["nodes"]:
|
||||
stage_sequence[stage["name"]] = set()
|
||||
|
||||
for job in pipeline_jobs["nodes"]:
|
||||
stage_sequence[job["stage"]["name"]].add(job["name"])
|
||||
dag_job: DagNode = {
|
||||
"name": job["name"],
|
||||
"tags": set(job["tags"] or []), # jobs with no tags defined return None here
|
||||
"stage": job["stage"]["name"],
|
||||
"needs": set([j["node"]["name"] for j in job["needs"]["edges"]]),
|
||||
}
|
||||
jobs_metadata[job["name"]] = dag_job
|
||||
|
||||
return stage_sequence, jobs_metadata
|
||||
|
||||
|
||||
def create_job_needs_dag(gl_gql: GitlabGQL, params, disable_cache: bool = True) -> Dag:
|
||||
"""
|
||||
This function creates a Directed Acyclic Graph (DAG) to represent a sequence of jobs, where each
|
||||
job has a set of jobs that it depends on (its "needs") and belongs to a certain "stage".
|
||||
The "name" of the job is used as the key in the dictionary.
|
||||
|
||||
For example, consider the following DAG:
|
||||
|
||||
1. build stage: job1 -> job2 -> job3
|
||||
2. test stage: job2 -> job4
|
||||
|
||||
- The job needs for job3 are: job1, job2
|
||||
- The job needs for job4 are: job2
|
||||
- The job2 needs to wait all jobs from build stage to finish.
|
||||
|
||||
The resulting DAG would look like this:
|
||||
|
||||
dag = {
|
||||
"job1": {"needs": set(), "stage": "build", "name": "job1"},
|
||||
"job2": {"needs": {"job1", "job2", job3"}, "stage": "test", "name": "job2"},
|
||||
"job3": {"needs": {"job1", "job2"}, "stage": "build", "name": "job3"},
|
||||
"job4": {"needs": {"job2"}, "stage": "test", "name": "job4"},
|
||||
}
|
||||
|
||||
To access the job needs, one can do:
|
||||
|
||||
dag["job3"]["needs"]
|
||||
|
||||
This will return the set of jobs that job3 needs: {"job1", "job2"}
|
||||
|
||||
Args:
|
||||
gl_gql (GitlabGQL): The `gl_gql` parameter is an instance of the `GitlabGQL` class, which is
|
||||
used to make GraphQL queries to the GitLab API.
|
||||
params (dict): The `params` parameter is a dictionary that contains the necessary parameters
|
||||
for the GraphQL query. It is used to specify the details of the pipeline for which the
|
||||
job needs DAG is being created.
|
||||
The specific keys and values in the `params` dictionary will depend on
|
||||
the requirements of the GraphQL query being executed
|
||||
disable_cache (bool): The `disable_cache` parameter is a boolean that specifies whether the
|
||||
|
||||
Returns:
|
||||
The final DAG (Directed Acyclic Graph) representing the job dependencies sourced from needs
|
||||
or stages rule.
|
||||
"""
|
||||
stages_jobs_gql = gl_gql.query(
|
||||
"pipeline_details.gql",
|
||||
params=params,
|
||||
paginated_key_loc=["project", "pipeline", "jobs"],
|
||||
disable_cache=disable_cache,
|
||||
)
|
||||
pipeline_data = stages_jobs_gql["project"]["pipeline"]
|
||||
if not pipeline_data:
|
||||
raise RuntimeError(f"Could not find any pipelines for {params}")
|
||||
|
||||
stage_sequence, jobs_metadata = extract_stages_and_job_needs(
|
||||
pipeline_data["jobs"], pipeline_data["stages"]
|
||||
)
|
||||
# Fill the DAG with the job needs from stages that don't have any needs but still need to wait
|
||||
# for previous stages
|
||||
final_dag = insert_early_stage_jobs(stage_sequence, jobs_metadata)
|
||||
# Now that each job has its direct needs filled correctly, update the "needs" field for each job
|
||||
# in the DAG by performing a topological traversal
|
||||
traverse_dag_needs(final_dag)
|
||||
|
||||
return final_dag
|
||||
|
||||
|
||||
def filter_dag(dag: Dag, job_filter: callable) -> Dag:
|
||||
return Dag({
|
||||
job: data
|
||||
for job, data in dag.items()
|
||||
if job_filter(
|
||||
job_name=job,
|
||||
job_stage=data["stage"],
|
||||
job_tags=data["tags"],
|
||||
)
|
||||
})
|
||||
|
||||
|
||||
def print_dag(dag: Dag, indentation: int = 0, color: str = "") -> None:
|
||||
for job, data in sorted(dag.items()):
|
||||
print(f"{color}{' '*indentation}{job}:")
|
||||
print_formatted_list(list(data['needs']), indentation=indentation+8, color=color)
|
||||
|
||||
|
||||
def print_formatted_list(elements: list[str], indentation: int = 0, color: str = "") -> None:
|
||||
"""
|
||||
When a list of elements is going to be printed, if it is longer than one line, reformat it to be multiple
|
||||
lines with a 'ls' command style.
|
||||
:param elements: list of elements to be printed
|
||||
:param indentation: number of spaces to be injected in front of each line
|
||||
"""
|
||||
if len(elements) == 0:
|
||||
return
|
||||
elements.sort()
|
||||
try:
|
||||
h_size = get_terminal_size().columns if sys.stdin.isatty() else DEFAULT_TERMINAL_SIZE
|
||||
except OSError:
|
||||
h_size = DEFAULT_TERMINAL_SIZE
|
||||
if indentation + sum(len(element) for element in elements) + (len(elements)*2) < h_size: # fits in one line
|
||||
print(f"{color}{' '*indentation}{', '.join([element for element in elements])}")
|
||||
return
|
||||
column_separator_size = 2
|
||||
column_width: int = len(max(elements, key=len)) + column_separator_size
|
||||
n_columns: int = max((h_size - indentation) // column_width, 1)
|
||||
step = (len(elements) // n_columns) + 1
|
||||
rows = [elements[i::step] for i in range(step)]
|
||||
for line in rows:
|
||||
print(' '*indentation, end='')
|
||||
for column in range(len(line)):
|
||||
if line[column] is not None:
|
||||
print(f"{color}{line[column]:<{column_width}}", end='')
|
||||
print()
|
||||
|
||||
|
||||
def fetch_merged_yaml(gl_gql: GitlabGQL, params) -> dict[str, Any]:
|
||||
params["content"] = dedent("""\
|
||||
include:
|
||||
- local: .gitlab-ci.yml
|
||||
""")
|
||||
raw_response = gl_gql.query("job_details.gql", params)
|
||||
ci_config = raw_response["ciConfig"]
|
||||
if merged_yaml := ci_config["mergedYaml"]:
|
||||
return yaml.safe_load(merged_yaml)
|
||||
if "errors" in ci_config:
|
||||
for error in ci_config["errors"]:
|
||||
print(error)
|
||||
|
||||
gl_gql.invalidate_query_cache()
|
||||
raise ValueError(
|
||||
"""
|
||||
Could not fetch any content for merged YAML,
|
||||
please verify if the git SHA exists in remote.
|
||||
Maybe you forgot to `git push`? """
|
||||
)
|
||||
|
||||
|
||||
def recursive_fill(job, relationship_field, target_data, acc_data: dict, merged_yaml):
|
||||
if relatives := job.get(relationship_field):
|
||||
if isinstance(relatives, str):
|
||||
relatives = [relatives]
|
||||
|
||||
for relative in relatives:
|
||||
parent_job = merged_yaml[relative]
|
||||
acc_data = recursive_fill(parent_job, acc_data, merged_yaml) # type: ignore
|
||||
|
||||
acc_data |= job.get(target_data, {})
|
||||
|
||||
return acc_data
|
||||
|
||||
|
||||
def get_variables(job, merged_yaml, project_path, sha) -> dict[str, str]:
|
||||
p = get_project_root_dir() / ".gitlab-ci" / "image-tags.yml"
|
||||
image_tags = yaml.safe_load(p.read_text())
|
||||
|
||||
variables = image_tags["variables"]
|
||||
variables |= merged_yaml["variables"]
|
||||
variables |= job["variables"]
|
||||
variables["CI_PROJECT_PATH"] = project_path
|
||||
variables["CI_PROJECT_NAME"] = project_path.split("/")[1]
|
||||
variables["CI_REGISTRY_IMAGE"] = "registry.freedesktop.org/${CI_PROJECT_PATH}"
|
||||
variables["CI_COMMIT_SHA"] = sha
|
||||
|
||||
while recurse_among_variables_space(variables):
|
||||
pass
|
||||
|
||||
return variables
|
||||
|
||||
|
||||
# Based on: https://stackoverflow.com/a/2158532/1079223
|
||||
def flatten(xs):
|
||||
for x in xs:
|
||||
if isinstance(x, Iterable) and not isinstance(x, (str, bytes)):
|
||||
yield from flatten(x)
|
||||
else:
|
||||
yield x
|
||||
|
||||
|
||||
def get_full_script(job) -> list[str]:
|
||||
script = []
|
||||
for script_part in ("before_script", "script", "after_script"):
|
||||
script.append(f"# {script_part}")
|
||||
lines = flatten(job.get(script_part, []))
|
||||
script.extend(lines)
|
||||
script.append("")
|
||||
|
||||
return script
|
||||
|
||||
|
||||
def recurse_among_variables_space(var_graph) -> bool:
|
||||
updated = False
|
||||
for var, value in var_graph.items():
|
||||
value = str(value)
|
||||
dep_vars = []
|
||||
if match := re.findall(r"(\$[{]?[\w\d_]*[}]?)", value):
|
||||
all_dep_vars = [v.lstrip("${").rstrip("}") for v in match]
|
||||
# print(value, match, all_dep_vars)
|
||||
dep_vars = [v for v in all_dep_vars if v in var_graph]
|
||||
|
||||
for dep_var in dep_vars:
|
||||
dep_value = str(var_graph[dep_var])
|
||||
new_value = var_graph[var]
|
||||
new_value = new_value.replace(f"${{{dep_var}}}", dep_value)
|
||||
new_value = new_value.replace(f"${dep_var}", dep_value)
|
||||
var_graph[var] = new_value
|
||||
updated |= dep_value != new_value
|
||||
|
||||
return updated
|
||||
|
||||
|
||||
def print_job_final_definition(job_name, merged_yaml, project_path, sha):
|
||||
job = merged_yaml[job_name]
|
||||
variables = get_variables(job, merged_yaml, project_path, sha)
|
||||
|
||||
print("# --------- variables ---------------")
|
||||
for var, value in sorted(variables.items()):
|
||||
print(f"export {var}={value!r}")
|
||||
|
||||
# TODO: Recurse into needs to get full script
|
||||
# TODO: maybe create a extra yaml file to avoid too much rework
|
||||
script = get_full_script(job)
|
||||
print()
|
||||
print()
|
||||
print("# --------- full script ---------------")
|
||||
print("\n".join(script))
|
||||
|
||||
if image := variables.get("MESA_IMAGE"):
|
||||
print()
|
||||
print()
|
||||
print("# --------- container image ---------------")
|
||||
print(image)
|
||||
|
||||
|
||||
def from_sha_to_pipeline_iid(gl_gql: GitlabGQL, params) -> str:
|
||||
result = gl_gql.query("pipeline_utils.gql", params)
|
||||
|
||||
return result["project"]["pipelines"]["nodes"][0]["iid"]
|
||||
|
||||
|
||||
def parse_args() -> Namespace:
|
||||
parser = ArgumentParser(
|
||||
formatter_class=ArgumentDefaultsHelpFormatter,
|
||||
description="CLI and library with utility functions to debug jobs via Gitlab GraphQL",
|
||||
epilog=f"""Example:
|
||||
{Path(__file__).name} --print-dag""",
|
||||
)
|
||||
parser.add_argument("-pp", "--project-path", type=str, default="mesa/mesa")
|
||||
parser.add_argument("--sha", "--rev", type=str, default='HEAD')
|
||||
parser.add_argument(
|
||||
"--regex",
|
||||
type=str,
|
||||
required=False,
|
||||
default=".*",
|
||||
help="Regex pattern for the job name to be considered",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--include-stage",
|
||||
type=str,
|
||||
required=False,
|
||||
default=".*",
|
||||
help="Regex pattern for the stage name to be considered",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--exclude-stage",
|
||||
type=str,
|
||||
required=False,
|
||||
default="^$",
|
||||
help="Regex pattern for the stage name to be excluded",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--job-tags",
|
||||
metavar="job-tags",
|
||||
help="Job tags to require when searching for target jobs. If multiple "
|
||||
"values are passed, eg. `--job-tags 'foo.*' 'bar'`, the job will "
|
||||
"need to have a tag matching `foo.*` *and* a tag matching `bar` "
|
||||
"to qualify. Passing `--job-tags '.*'` makes sure the job has "
|
||||
"a tag defined, while not passing `--job-tags` also allows "
|
||||
"untagged jobs.",
|
||||
default=[],
|
||||
nargs='+',
|
||||
)
|
||||
mutex_group_print = parser.add_mutually_exclusive_group()
|
||||
mutex_group_print.add_argument(
|
||||
"--print-dag",
|
||||
action="store_true",
|
||||
help="Print job needs DAG",
|
||||
)
|
||||
mutex_group_print.add_argument(
|
||||
"--print-merged-yaml",
|
||||
action="store_true",
|
||||
help="Print the resulting YAML for the specific SHA",
|
||||
)
|
||||
mutex_group_print.add_argument(
|
||||
"--print-job-manifest",
|
||||
metavar='JOB_NAME',
|
||||
type=str,
|
||||
help="Print the resulting job data"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--gitlab-token-file",
|
||||
type=str,
|
||||
default=get_token_from_default_dir(),
|
||||
help="force GitLab token, otherwise it's read from $XDG_CONFIG_HOME/gitlab-token",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
args.gitlab_token = Path(args.gitlab_token_file).read_text().strip()
|
||||
return args
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
gl_gql = GitlabGQL(token=args.gitlab_token)
|
||||
|
||||
sha = check_output(['git', 'rev-parse', args.sha]).decode('ascii').strip()
|
||||
|
||||
if args.print_dag:
|
||||
iid = from_sha_to_pipeline_iid(gl_gql, {"projectPath": args.project_path, "sha": sha})
|
||||
dag = create_job_needs_dag(
|
||||
gl_gql, {"projectPath": args.project_path, "iid": iid}, disable_cache=True
|
||||
)
|
||||
|
||||
job_name_regex = re.compile(args.regex)
|
||||
include_stage_regex = re.compile(args.include_stage)
|
||||
exclude_stage_regex = re.compile(args.exclude_stage)
|
||||
job_tags_regexes = [re.compile(job_tag) for job_tag in args.job_tags]
|
||||
|
||||
def job_filter(
|
||||
job_name: str,
|
||||
job_stage: str,
|
||||
job_tags: set[str],
|
||||
) -> bool:
|
||||
"""
|
||||
Apply user-specified filters to a job, and return whether the
|
||||
filters allow that job (True) or not (False).
|
||||
"""
|
||||
if not job_name_regex.fullmatch(job_name):
|
||||
return False
|
||||
if not include_stage_regex.fullmatch(job_stage):
|
||||
return False
|
||||
if exclude_stage_regex.fullmatch(job_stage):
|
||||
return False
|
||||
if not all(
|
||||
any(job_tags_regex.fullmatch(tag) for tag in job_tags)
|
||||
for job_tags_regex in job_tags_regexes
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
dag = filter_dag(dag, job_filter)
|
||||
|
||||
print_dag(dag)
|
||||
|
||||
if args.print_merged_yaml or args.print_job_manifest:
|
||||
merged_yaml = fetch_merged_yaml(
|
||||
gl_gql, {"projectPath": args.project_path, "sha": sha}
|
||||
)
|
||||
|
||||
if args.print_merged_yaml:
|
||||
print(yaml.dump(merged_yaml, indent=2))
|
||||
|
||||
if args.print_job_manifest:
|
||||
print_job_final_definition(
|
||||
args.print_job_manifest, merged_yaml, args.project_path, sha
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
|
||||
this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
|
||||
readonly this_dir
|
||||
|
||||
exec \
|
||||
"$this_dir/../python-venv.sh" \
|
||||
"$this_dir/requirements.txt" \
|
||||
"$this_dir/gitlab_gql.py" "$@"
|
||||
@@ -0,0 +1,7 @@
|
||||
query getCiConfigData($projectPath: ID!, $sha: String, $content: String!) {
|
||||
ciConfig(projectPath: $projectPath, sha: $sha, content: $content) {
|
||||
errors
|
||||
mergedYaml
|
||||
__typename
|
||||
}
|
||||
}
|
||||
Executable
+236
@@ -0,0 +1,236 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright © 2020 - 2023 Collabora Ltd.
|
||||
# Authors:
|
||||
# David Heidelberg <[email protected]>
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
"""
|
||||
Monitors Marge-bot and return number of assigned MRs.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import time
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from dateutil import parser
|
||||
from traceback import print_exc
|
||||
from typing import Optional
|
||||
|
||||
import gitlab
|
||||
from gitlab.base import RESTObjectList
|
||||
from gitlab.exceptions import GitlabAuthenticationError
|
||||
from gitlab.v4.objects import Project, ProjectMergeRequest
|
||||
from gitlab_common import get_token_from_default_dir, read_token, pretty_duration
|
||||
from rich.console import Console
|
||||
|
||||
REFRESH_WAIT = 30
|
||||
MARGE_BOT_USER_ID = 9716
|
||||
ASSIGNED_TO_MARGE = "assigned to @marge-bot"
|
||||
ASSIGNED_AT_PADDING = 19
|
||||
WAITING_PADDING = 8
|
||||
MR_ID_PADDING = 5
|
||||
|
||||
console = Console(highlight=False)
|
||||
print = console.print
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
"""Parse args"""
|
||||
parse = argparse.ArgumentParser(
|
||||
description="Tool to show merge requests assigned to the marge-bot",
|
||||
)
|
||||
parse.add_argument(
|
||||
"--wait", action="store_true", help="wait until CI is free",
|
||||
)
|
||||
parse.add_argument(
|
||||
"--token",
|
||||
metavar="token",
|
||||
default=get_token_from_default_dir(),
|
||||
help="force GitLab token, otherwise it's read from ~/.config/gitlab-token",
|
||||
)
|
||||
return parse.parse_args()
|
||||
|
||||
|
||||
@dataclass
|
||||
class MargeMergeRequest:
|
||||
"""Represent a Merge Request assigned to Marge"""
|
||||
id: int = field(init=False)
|
||||
mr: ProjectMergeRequest = field(repr=False)
|
||||
updated_at: datetime | None = field(init=False, repr=False)
|
||||
assigned_at: datetime | None = field(init=False, repr=False)
|
||||
|
||||
def __post_init__(self):
|
||||
self.id = self.mr.iid
|
||||
self.updated_at = parser.parse(self.mr.updated_at)
|
||||
self.assigned_at = self.__find_last_assign_to_marge()
|
||||
|
||||
def __find_last_assign_to_marge(self) -> Optional[datetime]:
|
||||
for note in self.__get_mr_notes_iterator():
|
||||
if note.body.startswith(ASSIGNED_TO_MARGE):
|
||||
return parser.parse(note.created_at)
|
||||
|
||||
def __eq__(self, other: "MargeMergeRequest") -> bool:
|
||||
return self.id == other.id
|
||||
|
||||
@property
|
||||
def time_enqueued(self) -> timedelta:
|
||||
if self.assigned_at is None:
|
||||
raise ValueError("Assign to marge timestamp not defined")
|
||||
return datetime.now(timezone.utc) - self.assigned_at
|
||||
|
||||
@property
|
||||
def web_url(self) -> str:
|
||||
return self.mr.web_url
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
return self.mr.title
|
||||
|
||||
def __get_mr_notes_iterator(self) -> RESTObjectList:
|
||||
return self.mr.notes.list(
|
||||
iterator=True, order_by="updated_at", sort="desc"
|
||||
) # start with the most recent
|
||||
|
||||
|
||||
@dataclass
|
||||
class MargeQueue:
|
||||
"""
|
||||
Collect and sort the merge requests assigned to marge
|
||||
"""
|
||||
elements_sorted: dict[datetime, MargeMergeRequest] = field(
|
||||
init=False, repr=False, default_factory=dict
|
||||
)
|
||||
undetermined: list[MargeMergeRequest] = field(
|
||||
init=False, repr=False, default_factory=list
|
||||
)
|
||||
|
||||
def append(self, mr: MargeMergeRequest) -> None:
|
||||
if mr.assigned_at is not None:
|
||||
self.elements_sorted[mr.assigned_at] = mr
|
||||
else:
|
||||
self.undetermined.append(mr)
|
||||
|
||||
@property
|
||||
def n_merge_requests_enqueued(self) -> int:
|
||||
return len(self.elements_sorted) + len(self.undetermined)
|
||||
|
||||
@property
|
||||
def sorted_queue(self) -> list[MargeMergeRequest]:
|
||||
"""
|
||||
Provide a list of the elements that can be sorted based on the
|
||||
assignment to Marge.
|
||||
"""
|
||||
return list(dict(sorted(self.elements_sorted.items())).values())
|
||||
|
||||
@property
|
||||
def all_assigned(
|
||||
self
|
||||
) -> list[MargeMergeRequest]:
|
||||
"""
|
||||
Provide a single list, but in sorted, of all the elements found
|
||||
assigned to Marge. This include the elements in elements_sorted
|
||||
and the ones undetermined (expected to be empty).
|
||||
"""
|
||||
return self.sorted_queue + self.undetermined
|
||||
|
||||
|
||||
def get_merge_queue(project: Project) -> MargeQueue:
|
||||
queue = MargeQueue()
|
||||
for mr in __get_project_marge_merge_requests(project):
|
||||
marge_merge_request = MargeMergeRequest(mr)
|
||||
queue.append(marge_merge_request)
|
||||
for mr in queue.sorted_queue:
|
||||
assigned_at = f"{mr.assigned_at:%Y-%m-%d %H:%M:%S}"
|
||||
time_enqueued = f"{pretty_duration(mr.time_enqueued.total_seconds())}"
|
||||
print(
|
||||
f"⛭ {assigned_at:<{ASSIGNED_AT_PADDING}} "
|
||||
f"{time_enqueued:>{WAITING_PADDING}} "
|
||||
f"{link2print(mr.web_url, mr.id, MR_ID_PADDING)} {mr.title}"
|
||||
)
|
||||
if queue.undetermined:
|
||||
print(
|
||||
"Unable to determine when they where assigned to Marge:"
|
||||
)
|
||||
for mr in queue.undetermined:
|
||||
updated_at = f"{mr.updated_at:%Y-%m-%d %H:%M:%S}"
|
||||
print(
|
||||
f"⛭ {updated_at:<{ASSIGNED_AT_PADDING}} "
|
||||
f"{" ":{WAITING_PADDING}} "
|
||||
f"{link2print(mr.web_url, mr.id, MR_ID_PADDING)} {mr.title}"
|
||||
)
|
||||
return queue
|
||||
|
||||
|
||||
def __get_gitlab_object(token: str) -> gitlab.Gitlab:
|
||||
return gitlab.Gitlab(url="https://gitlab.freedesktop.org", private_token=token, retry_transient_errors=True)
|
||||
|
||||
|
||||
def __get_gitlab_project(gl: gitlab.Gitlab) -> Project:
|
||||
return gl.projects.get("mesa/mesa")
|
||||
|
||||
|
||||
def __get_project_marge_merge_requests(
|
||||
project: Project
|
||||
) -> list[ProjectMergeRequest]:
|
||||
return project.mergerequests.list(
|
||||
assignee_id=MARGE_BOT_USER_ID,
|
||||
scope="all",
|
||||
state="opened",
|
||||
get_all=True,
|
||||
)
|
||||
|
||||
|
||||
def link2print(url: str, text: str, text_pad: int = 0) -> str:
|
||||
text = str(text)
|
||||
text_pad = len(text) if text_pad < 1 else text_pad
|
||||
return f"[link={url}]{text:{text_pad}}[/link]"
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
token = read_token(args.token)
|
||||
gl = __get_gitlab_object(token)
|
||||
|
||||
project = __get_gitlab_project(gl)
|
||||
|
||||
assigned_at = "Assigned at"
|
||||
waiting = "Waiting"
|
||||
mr = "MR"
|
||||
title = "Title"
|
||||
print(
|
||||
f" {assigned_at:<{ASSIGNED_AT_PADDING}} "
|
||||
f"{waiting:<{WAITING_PADDING}} "
|
||||
f"{mr:<{MR_ID_PADDING}} {title}"
|
||||
)
|
||||
while True:
|
||||
try:
|
||||
n_mrs = get_merge_queue(project).n_merge_requests_enqueued
|
||||
except GitlabAuthenticationError as autherror:
|
||||
if autherror.response_code == 401: # 401 Unauthorized
|
||||
print(f"Alert! Tool action requires authentication. Use the --token argument")
|
||||
else:
|
||||
print(f"Alert! Something went wrong: {autherror.responce_body}")
|
||||
print_exc()
|
||||
sys.exit(-1)
|
||||
|
||||
print(f"Job waiting: {n_mrs}")
|
||||
|
||||
if n_mrs == 0:
|
||||
sys.exit(0)
|
||||
if not args.wait:
|
||||
sys.exit(min(n_mrs, 127))
|
||||
|
||||
try:
|
||||
time.sleep(REFRESH_WAIT)
|
||||
except KeyboardInterrupt:
|
||||
print(
|
||||
"Sleep interrupted from keyboard, "
|
||||
"doing one last iteration before finish."
|
||||
)
|
||||
args.wait = False
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
|
||||
this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
|
||||
readonly this_dir
|
||||
|
||||
exec \
|
||||
"$this_dir/../python-venv.sh" \
|
||||
"$this_dir/requirements.txt" \
|
||||
"$this_dir/marge_queue.py" "$@"
|
||||
Executable
+197
@@ -0,0 +1,197 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright © 2020 - 2024 Collabora Ltd.
|
||||
# Authors:
|
||||
# David Heidelberg <[email protected]>
|
||||
# Sergi Blanch Torne <[email protected]>
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
"""
|
||||
Compare the two latest scheduled pipelines and provide information
|
||||
about the jobs you're interested in.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import re
|
||||
import requests
|
||||
import io
|
||||
from tabulate import tabulate
|
||||
|
||||
import gitlab
|
||||
from gitlab_common import read_token
|
||||
from rich import print
|
||||
|
||||
|
||||
MARGE_BOT_USER_ID = 9716
|
||||
|
||||
|
||||
def print_failures_csv(id):
|
||||
url = "https://gitlab.freedesktop.org/mesa/mesa"\
|
||||
f"/-/jobs/{id}/artifacts/raw/results/failures.csv"
|
||||
missing: int = 0
|
||||
MAX_MISS: int = 20
|
||||
try:
|
||||
response = requests.get(url)
|
||||
response.raise_for_status()
|
||||
csv_content = io.StringIO(response.text)
|
||||
csv_reader = csv.reader(csv_content)
|
||||
data = list(csv_reader)
|
||||
|
||||
for line in data[:]:
|
||||
if line[1] == "UnexpectedImprovement(Pass)":
|
||||
line[1] = f"[green]{line[1]}[/green]"
|
||||
elif line[1] == "UnexpectedImprovement(Fail)":
|
||||
line[1] = f"[yellow]{line[1]}[/yellow]"
|
||||
elif line[1] == "Crash" or line[1] == "Fail":
|
||||
line[1] = f" [red]{line[1]}[/red]"
|
||||
elif line[1] == "Missing":
|
||||
if missing > MAX_MISS:
|
||||
data.remove(line)
|
||||
continue
|
||||
missing += 1
|
||||
line[1] = f"[yellow]{line[1]}[/yellow]"
|
||||
elif line[1] == "Fail":
|
||||
line[1] = f"[red]{line[1]}[/red]"
|
||||
else:
|
||||
line[1] = f"[white]{line[1]}[/white]"
|
||||
|
||||
if missing > MAX_MISS:
|
||||
data.append(
|
||||
[
|
||||
f"[red]... more than {MAX_MISS} missing tests, "
|
||||
"something crashed?[/red]",
|
||||
"[red]Missing[/red]"
|
||||
]
|
||||
)
|
||||
headers = [f"Test{"":<75}", "Result"]
|
||||
print(tabulate(data, headers, tablefmt="plain"))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def job_failed_before(old_jobs, job):
|
||||
for old_job in old_jobs:
|
||||
if job.name == old_job.name:
|
||||
return old_job
|
||||
|
||||
|
||||
def parse_args() -> None:
|
||||
"""Parse args"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Tool to show merge requests assigned to the marge-bot",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--target",
|
||||
metavar="target-job",
|
||||
help="Target job regex. For multiple targets, pass multiple values, "
|
||||
"eg. `--target foo bar`.",
|
||||
required=False,
|
||||
nargs=argparse.ONE_OR_MORE,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--token",
|
||||
metavar="token",
|
||||
help="force GitLab token, "
|
||||
"otherwise it's read from ~/.config/gitlab-token",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parse_args()
|
||||
token = read_token(args.token)
|
||||
gl = gitlab.Gitlab(
|
||||
url="https://gitlab.freedesktop.org",
|
||||
private_token=token,
|
||||
)
|
||||
|
||||
project = gl.projects.get("mesa/mesa")
|
||||
|
||||
print(
|
||||
"[link=https://gitlab.freedesktop.org/mesa/mesa/-/pipelines?"
|
||||
"page=1&scope=all&source=schedule]Scheduled pipelines overview[/link]"
|
||||
)
|
||||
pipelines = project.pipelines.list(
|
||||
source="schedule",
|
||||
ordered_by="created_at",
|
||||
sort="desc",
|
||||
page=1,
|
||||
per_page=2,
|
||||
)
|
||||
print(
|
||||
"Old pipeline:"
|
||||
f" {pipelines[1].created_at}"
|
||||
f"\t[link={pipelines[1].web_url}]{pipelines[1].status}[/link]"
|
||||
f"\t{pipelines[1].sha}"
|
||||
)
|
||||
print(
|
||||
"New pipeline:"
|
||||
f" {pipelines[0].created_at}"
|
||||
f"\t[link={pipelines[0].web_url}]{pipelines[0].status}[/link]"
|
||||
f"\t{pipelines[0].sha}"
|
||||
)
|
||||
print(
|
||||
"\nWebUI visual compare: "
|
||||
"https://gitlab.freedesktop.org/mesa/mesa/-/compare/"
|
||||
f"{pipelines[1].sha}...{pipelines[0].sha}\n"
|
||||
)
|
||||
|
||||
# regex part
|
||||
if args.target:
|
||||
target = "|".join(args.target)
|
||||
target = target.strip()
|
||||
print(f"🞋 jobs: [blue]{target}[/blue]")
|
||||
|
||||
target = f"({target})" + r"( \d+/\d+)?"
|
||||
else:
|
||||
target = ".*"
|
||||
|
||||
target_jobs_regex: re.Pattern = re.compile(target)
|
||||
|
||||
old_failed_jobs = []
|
||||
for job in pipelines[1].jobs.list(all=True):
|
||||
if (
|
||||
job.status != "failed"
|
||||
or target_jobs_regex
|
||||
and not target_jobs_regex.fullmatch(job.name)
|
||||
):
|
||||
continue
|
||||
old_failed_jobs.append(job)
|
||||
|
||||
job_failed = False
|
||||
for job in pipelines[0].jobs.list(all=True):
|
||||
if (
|
||||
job.status != "failed"
|
||||
or target_jobs_regex
|
||||
and not target_jobs_regex.fullmatch(job.name)
|
||||
):
|
||||
continue
|
||||
|
||||
job_failed = True
|
||||
|
||||
previously_failed_job = job_failed_before(old_failed_jobs, job)
|
||||
if previously_failed_job:
|
||||
print(
|
||||
f"[yellow]"
|
||||
f" :: [link={job.web_url}]{job.name}[/link][/yellow]"
|
||||
f"[magenta]"
|
||||
f" [link={previously_failed_job.web_url}](previous run)[/link]"
|
||||
)
|
||||
else:
|
||||
print(
|
||||
f"[red]:: [link={job.web_url}]{job.name}[/link]"
|
||||
)
|
||||
print_failures_csv(job.id)
|
||||
|
||||
if not job_failed:
|
||||
exit(0)
|
||||
|
||||
print("Commits between nightly pipelines:")
|
||||
commit = project.commits.get(pipelines[0].sha)
|
||||
while True:
|
||||
print(
|
||||
f"{commit.id} [link={commit.web_url}]{commit.title}[/link]"
|
||||
)
|
||||
if commit.id == pipelines[1].sha:
|
||||
break
|
||||
commit = project.commits.get(commit.parent_ids[0])
|
||||
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
|
||||
this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
|
||||
readonly this_dir
|
||||
|
||||
exec \
|
||||
"$this_dir/../python-venv.sh" \
|
||||
"$this_dir/requirements.txt" \
|
||||
"$this_dir/nightly_compare.py" "$@"
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
query jobs($projectPath: ID!, $iid: ID!, $cursor: String) {
|
||||
project(fullPath: $projectPath) {
|
||||
id
|
||||
pipeline(iid: $iid) {
|
||||
id
|
||||
iid
|
||||
complete
|
||||
stages {
|
||||
nodes {
|
||||
name
|
||||
}
|
||||
}
|
||||
jobs(after: $cursor) {
|
||||
pageInfo {
|
||||
hasNextPage
|
||||
endCursor
|
||||
}
|
||||
count
|
||||
nodes {
|
||||
name
|
||||
needs {
|
||||
edges {
|
||||
node {
|
||||
name
|
||||
}
|
||||
}
|
||||
}
|
||||
stage {
|
||||
name
|
||||
}
|
||||
tags
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
../../.marge/hooks/pipeline_message.py
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
|
||||
this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
|
||||
readonly this_dir
|
||||
|
||||
exec \
|
||||
"$this_dir/../python-venv.sh" \
|
||||
"$this_dir/requirements.txt" \
|
||||
"$this_dir/pipeline_message.py" "$@"
|
||||
@@ -0,0 +1,9 @@
|
||||
query sha2pipelineIID($projectPath: ID!, $sha: String!) {
|
||||
project(fullPath: $projectPath) {
|
||||
pipelines(last: 1, sha:$sha){
|
||||
nodes {
|
||||
iid
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
PyYAML==6.*
|
||||
filecache==0.81
|
||||
flake8==7.*
|
||||
gql==3.*
|
||||
kaleido==0.2.*
|
||||
pandas==2.*
|
||||
plotly==5.*
|
||||
python-dateutil==2.*
|
||||
python-gitlab==5.*
|
||||
rich==14.1.*
|
||||
ruamel.yaml.clib==0.2.*
|
||||
ruamel.yaml==0.17.*
|
||||
tabulate==0.9.*
|
||||
tomledit==1.*
|
||||
@@ -0,0 +1,294 @@
|
||||
"""
|
||||
A structured logging utility supporting multiple data formats such as CSV, JSON,
|
||||
and YAML.
|
||||
|
||||
The main purpose of this script, besides having relevant information available
|
||||
in a condensed and deserialized.
|
||||
|
||||
This script defines a protocol for different file handling strategies and provides
|
||||
implementations for CSV, JSON, and YAML formats. The main class, StructuredLogger,
|
||||
allows for easy interaction with log data, enabling users to load, save, increment,
|
||||
set, and append fields in the log. The script also includes context managers for
|
||||
file locking and editing log data to ensure data integrity and avoid race conditions.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from collections.abc import MutableMapping, MutableSequence
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
import fire
|
||||
from filelock import FileLock
|
||||
|
||||
try:
|
||||
import polars as pl
|
||||
|
||||
CSV_LIB_EXCEPTION = None
|
||||
except ImportError as e:
|
||||
CSV_LIB_EXCEPTION: ImportError = e
|
||||
|
||||
try:
|
||||
from ruamel.yaml import YAML
|
||||
|
||||
YAML_LIB_EXCEPTION = None
|
||||
except ImportError as e:
|
||||
YAML_LIB_EXCEPTION: ImportError = e
|
||||
|
||||
|
||||
class ContainerProxy:
|
||||
"""
|
||||
A proxy class that wraps a mutable container object (such as a dictionary or
|
||||
a list) and calls a provided save_callback function whenever the container
|
||||
or its contents
|
||||
are changed.
|
||||
"""
|
||||
def __init__(self, container, save_callback):
|
||||
self.container = container
|
||||
self.save_callback = save_callback
|
||||
|
||||
def __getitem__(self, key):
|
||||
value = self.container[key]
|
||||
if isinstance(value, (MutableMapping, MutableSequence)):
|
||||
return ContainerProxy(value, self.save_callback)
|
||||
return value
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
self.container[key] = value
|
||||
self.save_callback()
|
||||
|
||||
def __delitem__(self, key):
|
||||
del self.container[key]
|
||||
self.save_callback()
|
||||
|
||||
def __getattr__(self, name):
|
||||
attr = getattr(self.container, name)
|
||||
|
||||
if callable(attr):
|
||||
def wrapper(*args, **kwargs):
|
||||
result = attr(*args, **kwargs)
|
||||
self.save_callback()
|
||||
return result
|
||||
|
||||
return wrapper
|
||||
return attr
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self.container)
|
||||
|
||||
def __len__(self):
|
||||
return len(self.container)
|
||||
|
||||
def __repr__(self):
|
||||
return repr(self.container)
|
||||
|
||||
|
||||
class AutoSaveDict(dict):
|
||||
"""
|
||||
A subclass of the built-in dict class with additional functionality to
|
||||
automatically save changes to the dictionary. It maintains a timestamp of
|
||||
the last modification and automatically wraps nested mutable containers
|
||||
using ContainerProxy.
|
||||
"""
|
||||
timestamp_key = "_timestamp"
|
||||
|
||||
def __init__(self, *args, save_callback, register_timestamp=True, **kwargs):
|
||||
self.save_callback = save_callback
|
||||
self.__register_timestamp = register_timestamp
|
||||
self.__heartbeat()
|
||||
super().__init__(*args, **kwargs)
|
||||
self.__wrap_dictionaries()
|
||||
|
||||
def __heartbeat(self):
|
||||
if self.__register_timestamp:
|
||||
self[AutoSaveDict.timestamp_key] = datetime.now().isoformat()
|
||||
|
||||
def __save(self):
|
||||
self.__heartbeat()
|
||||
self.save_callback()
|
||||
|
||||
def __wrap_dictionaries(self):
|
||||
for key, value in self.items():
|
||||
if isinstance(value, MutableMapping) and not isinstance(
|
||||
value, AutoSaveDict
|
||||
):
|
||||
self[key] = AutoSaveDict(
|
||||
value, save_callback=self.save_callback, register_timestamp=False
|
||||
)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
if isinstance(value, MutableMapping) and not isinstance(value, AutoSaveDict):
|
||||
value = AutoSaveDict(
|
||||
value, save_callback=self.save_callback, register_timestamp=False
|
||||
)
|
||||
super().__setitem__(key, value)
|
||||
|
||||
if self.__register_timestamp and key == AutoSaveDict.timestamp_key:
|
||||
return
|
||||
self.__save()
|
||||
|
||||
def __getitem__(self, key):
|
||||
value = super().__getitem__(key)
|
||||
if isinstance(value, (MutableMapping, MutableSequence)):
|
||||
return ContainerProxy(value, self.__save)
|
||||
return value
|
||||
|
||||
def __delitem__(self, key):
|
||||
super().__delitem__(key)
|
||||
self.__save()
|
||||
|
||||
def pop(self, *args, **kwargs):
|
||||
result = super().pop(*args, **kwargs)
|
||||
self.__save()
|
||||
return result
|
||||
|
||||
def update(self, *args, **kwargs):
|
||||
super().update(*args, **kwargs)
|
||||
self.__wrap_dictionaries()
|
||||
self.__save()
|
||||
|
||||
|
||||
class StructuredLoggerStrategy(Protocol):
|
||||
def load_data(self, file_path: Path) -> dict:
|
||||
pass
|
||||
|
||||
def save_data(self, file_path: Path, data: dict) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class CSVStrategy:
|
||||
def __init__(self) -> None:
|
||||
if CSV_LIB_EXCEPTION:
|
||||
raise RuntimeError(
|
||||
"Can't parse CSV files. Missing library"
|
||||
) from CSV_LIB_EXCEPTION
|
||||
|
||||
def load_data(self, file_path: Path) -> dict:
|
||||
dicts: list[dict[str, Any]] = pl.read_csv(
|
||||
file_path, try_parse_dates=True
|
||||
).to_dicts()
|
||||
data = {}
|
||||
for d in dicts:
|
||||
for k, v in d.items():
|
||||
if k != AutoSaveDict.timestamp_key and k in data:
|
||||
if isinstance(data[k], list):
|
||||
data[k].append(v)
|
||||
continue
|
||||
data[k] = [data[k], v]
|
||||
else:
|
||||
data[k] = v
|
||||
return data
|
||||
|
||||
def save_data(self, file_path: Path, data: dict) -> None:
|
||||
pl.DataFrame(data).write_csv(file_path)
|
||||
|
||||
|
||||
class JSONStrategy:
|
||||
def load_data(self, file_path: Path) -> dict:
|
||||
return json.loads(file_path.read_text())
|
||||
|
||||
def save_data(self, file_path: Path, data: dict) -> None:
|
||||
with open(file_path, "w") as f:
|
||||
json.dump(data, f, indent=2)
|
||||
|
||||
|
||||
class YAMLStrategy:
|
||||
def __init__(self):
|
||||
if YAML_LIB_EXCEPTION:
|
||||
raise RuntimeError(
|
||||
"Can't parse YAML files. Missing library"
|
||||
) from YAML_LIB_EXCEPTION
|
||||
self.yaml = YAML()
|
||||
self.yaml.indent(sequence=4, offset=2)
|
||||
self.yaml.default_flow_style = False
|
||||
self.yaml.representer.add_representer(AutoSaveDict, self.represent_dict)
|
||||
|
||||
@classmethod
|
||||
def represent_dict(cls, dumper, data):
|
||||
return dumper.represent_mapping("tag:yaml.org,2002:map", data)
|
||||
|
||||
def load_data(self, file_path: Path) -> dict:
|
||||
return self.yaml.load(file_path.read_text())
|
||||
|
||||
def save_data(self, file_path: Path, data: dict) -> None:
|
||||
with open(file_path, "w") as f:
|
||||
self.yaml.dump(data, f)
|
||||
|
||||
|
||||
class StructuredLogger:
|
||||
def __init__(
|
||||
self, file_name: str, strategy: StructuredLoggerStrategy = None, truncate=False
|
||||
):
|
||||
self.file_name: str = file_name
|
||||
self.file_path = Path(self.file_name)
|
||||
self._data: AutoSaveDict = AutoSaveDict(save_callback=self.save_data)
|
||||
|
||||
if strategy is None:
|
||||
self.strategy: StructuredLoggerStrategy = self.guess_strategy_from_file(
|
||||
self.file_path
|
||||
)
|
||||
else:
|
||||
self.strategy = strategy
|
||||
|
||||
if not self.file_path.exists():
|
||||
Path.mkdir(self.file_path.parent, exist_ok=True)
|
||||
self.save_data()
|
||||
return
|
||||
|
||||
if truncate:
|
||||
with self.get_lock():
|
||||
os.truncate(self.file_path, 0)
|
||||
self.save_data()
|
||||
|
||||
def load_data(self):
|
||||
self._data = self.strategy.load_data(self.file_path)
|
||||
|
||||
def save_data(self):
|
||||
self.strategy.save_data(self.file_path, self._data)
|
||||
|
||||
@property
|
||||
def data(self) -> AutoSaveDict:
|
||||
return self._data
|
||||
|
||||
@contextmanager
|
||||
def get_lock(self):
|
||||
with FileLock(f"{self.file_path}.lock", timeout=10):
|
||||
yield
|
||||
|
||||
@contextmanager
|
||||
def edit_context(self):
|
||||
"""
|
||||
Context manager that ensures proper loading and saving of log data when
|
||||
performing multiple modifications.
|
||||
"""
|
||||
with self.get_lock():
|
||||
try:
|
||||
self.load_data()
|
||||
yield
|
||||
finally:
|
||||
self.save_data()
|
||||
|
||||
@staticmethod
|
||||
def guess_strategy_from_file(file_path: Path) -> StructuredLoggerStrategy:
|
||||
file_extension = file_path.suffix.lower().lstrip(".")
|
||||
return StructuredLogger.get_strategy(file_extension)
|
||||
|
||||
@staticmethod
|
||||
def get_strategy(strategy_name: str) -> StructuredLoggerStrategy:
|
||||
strategies = {
|
||||
"csv": CSVStrategy,
|
||||
"json": JSONStrategy,
|
||||
"yaml": YAMLStrategy,
|
||||
"yml": YAMLStrategy,
|
||||
}
|
||||
|
||||
try:
|
||||
return strategies[strategy_name]()
|
||||
except KeyError as e:
|
||||
raise ValueError(f"Unknown strategy for: {strategy_name}") from e
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
fire.Fire(StructuredLogger)
|
||||
@@ -0,0 +1,10 @@
|
||||
-r ../requirements.txt
|
||||
aiohttp==3.13.*
|
||||
filelock==3.12.4
|
||||
fire==0.7.0
|
||||
freezegun==1.5.1
|
||||
mock==5.1.0
|
||||
polars==0.19.3
|
||||
pytest==7.4.2
|
||||
pytest-asyncio==0.21.0
|
||||
pytest-cov==3.0.0
|
||||
@@ -0,0 +1,746 @@
|
||||
import logging
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
from custom_logger import CustomLogger
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tmp_log_file(tmp_path):
|
||||
return tmp_path / "test_log.json"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def custom_logger(tmp_log_file):
|
||||
return CustomLogger(tmp_log_file)
|
||||
|
||||
|
||||
def run_script_with_args(args):
|
||||
import custom_logger
|
||||
|
||||
script_path = custom_logger.__file__
|
||||
return subprocess.run(
|
||||
["python3", str(script_path), *args], capture_output=True, text=True
|
||||
)
|
||||
|
||||
|
||||
# Test case for missing log file
|
||||
@pytest.mark.parametrize(
|
||||
"key, value", [("dut_attempt_counter", "1"), ("job_combined_status", "pass")]
|
||||
)
|
||||
def test_missing_log_file_argument(key, value):
|
||||
result = run_script_with_args(["--update", "key", "value"])
|
||||
assert result.returncode != 0
|
||||
|
||||
|
||||
# Parametrize test case for valid update arguments
|
||||
@pytest.mark.parametrize(
|
||||
"key, value", [("dut_attempt_counter", "1"), ("job_combined_status", "pass")]
|
||||
)
|
||||
def test_update_argument_valid(custom_logger, tmp_log_file, key, value):
|
||||
result = run_script_with_args([str(tmp_log_file), "--update", key, value])
|
||||
assert result.returncode == 0
|
||||
|
||||
|
||||
# Test case for passing only the key without a value
|
||||
def test_update_argument_key_only(custom_logger, tmp_log_file):
|
||||
key = "dut_attempt_counter"
|
||||
result = run_script_with_args([str(tmp_log_file), "--update", key])
|
||||
assert result.returncode != 0
|
||||
|
||||
|
||||
# Test case for not passing any key-value pair
|
||||
def test_update_argument_no_values(custom_logger, tmp_log_file):
|
||||
result = run_script_with_args([str(tmp_log_file), "--update"])
|
||||
assert result.returncode == 0
|
||||
|
||||
|
||||
# Parametrize test case for valid arguments
|
||||
@pytest.mark.parametrize(
|
||||
"key, value", [("dut_attempt_counter", "1"), ("job_combined_status", "pass")]
|
||||
)
|
||||
def test_create_argument_valid(custom_logger, tmp_log_file, key, value):
|
||||
result = run_script_with_args([str(tmp_log_file), "--create-dut-job", key, value])
|
||||
assert result.returncode == 0
|
||||
|
||||
|
||||
# Test case for passing only the key without a value
|
||||
def test_create_argument_key_only(custom_logger, tmp_log_file):
|
||||
key = "dut_attempt_counter"
|
||||
result = run_script_with_args([str(tmp_log_file), "--create-dut-job", key])
|
||||
assert result.returncode != 0
|
||||
|
||||
|
||||
# Test case for not passing any key-value pair
|
||||
def test_create_argument_no_values(custom_logger, tmp_log_file):
|
||||
result = run_script_with_args([str(tmp_log_file), "--create-dut-job"])
|
||||
assert result.returncode == 0
|
||||
|
||||
|
||||
# Test case for updating a DUT job
|
||||
@pytest.mark.parametrize(
|
||||
"key, value", [("status", "hung"), ("dut_state", "Canceling"), ("dut_name", "asus")]
|
||||
)
|
||||
def test_update_dut_job(custom_logger, tmp_log_file, key, value):
|
||||
result = run_script_with_args([str(tmp_log_file), "--update-dut-job", key, value])
|
||||
assert result.returncode != 0
|
||||
|
||||
result = run_script_with_args([str(tmp_log_file), "--create-dut-job", key, value])
|
||||
assert result.returncode == 0
|
||||
|
||||
result = run_script_with_args([str(tmp_log_file), "--update-dut-job", key, value])
|
||||
assert result.returncode == 0
|
||||
|
||||
|
||||
# Test case for updating last DUT job
|
||||
def test_update_dut_multiple_job(custom_logger, tmp_log_file):
|
||||
# Create the first DUT job with the first key
|
||||
result = run_script_with_args(
|
||||
[str(tmp_log_file), "--create-dut-job", "status", "hung"]
|
||||
)
|
||||
assert result.returncode == 0
|
||||
|
||||
# Create the second DUT job with the second key
|
||||
result = run_script_with_args(
|
||||
[str(tmp_log_file), "--create-dut-job", "dut_state", "Canceling"]
|
||||
)
|
||||
assert result.returncode == 0
|
||||
|
||||
result = run_script_with_args(
|
||||
[str(tmp_log_file), "--update-dut-job", "dut_name", "asus"]
|
||||
)
|
||||
assert result.returncode == 0
|
||||
|
||||
|
||||
# Parametrize test case for valid phase arguments
|
||||
@pytest.mark.parametrize(
|
||||
"phase_name",
|
||||
[("Phase1"), ("Phase2"), ("Phase3")],
|
||||
)
|
||||
def test_create_job_phase_valid(custom_logger, tmp_log_file, phase_name):
|
||||
custom_logger.create_dut_job(status="pass")
|
||||
|
||||
result = run_script_with_args([str(tmp_log_file), "--create-job-phase", phase_name])
|
||||
assert result.returncode == 0
|
||||
|
||||
|
||||
# Test case for not passing any arguments for create-job-phase
|
||||
def test_create_job_phase_no_arguments(custom_logger, tmp_log_file):
|
||||
custom_logger.create_dut_job(status="pass")
|
||||
|
||||
result = run_script_with_args([str(tmp_log_file), "--create-job-phase"])
|
||||
assert result.returncode != 0
|
||||
|
||||
|
||||
# Test case for trying to create a phase job without an existing DUT job
|
||||
def test_create_job_phase_no_dut_job(custom_logger, tmp_log_file):
|
||||
phase_name = "Phase1"
|
||||
|
||||
result = run_script_with_args([str(tmp_log_file), "--create-job-phase", phase_name])
|
||||
assert result.returncode != 0
|
||||
|
||||
|
||||
# Combined test cases for valid scenarios
|
||||
def test_valid_scenarios(custom_logger, tmp_log_file):
|
||||
valid_update_args = [("dut_attempt_counter", "1"), ("job_combined_status", "pass")]
|
||||
for key, value in valid_update_args:
|
||||
result = run_script_with_args([str(tmp_log_file), "--update", key, value])
|
||||
assert result.returncode == 0
|
||||
|
||||
valid_create_args = [
|
||||
("status", "hung"),
|
||||
("dut_state", "Canceling"),
|
||||
("dut_name", "asus"),
|
||||
("phase_name", "Bootloader"),
|
||||
]
|
||||
for key, value in valid_create_args:
|
||||
result = run_script_with_args(
|
||||
[str(tmp_log_file), "--create-dut-job", key, value]
|
||||
)
|
||||
assert result.returncode == 0
|
||||
|
||||
result = run_script_with_args(
|
||||
[str(tmp_log_file), "--create-dut-job", "status", "hung"]
|
||||
)
|
||||
assert result.returncode == 0
|
||||
|
||||
result = run_script_with_args(
|
||||
[str(tmp_log_file), "--update-dut-job", "dut_name", "asus"]
|
||||
)
|
||||
assert result.returncode == 0
|
||||
|
||||
result = run_script_with_args(
|
||||
[
|
||||
str(tmp_log_file),
|
||||
"--create-job-phase",
|
||||
"phase_name",
|
||||
]
|
||||
)
|
||||
assert result.returncode == 0
|
||||
|
||||
|
||||
# Parametrize test case for valid update arguments
|
||||
@pytest.mark.parametrize(
|
||||
"key, value", [("dut_attempt_counter", "1"), ("job_combined_status", "pass")]
|
||||
)
|
||||
def test_update(custom_logger, key, value):
|
||||
custom_logger.update(**{key: value})
|
||||
logger_data = custom_logger.logger.data
|
||||
|
||||
assert key in logger_data
|
||||
assert logger_data[key] == value
|
||||
|
||||
|
||||
# Test case for updating with a key that already exists
|
||||
def test_update_existing_key(custom_logger):
|
||||
key = "status"
|
||||
value = "new_value"
|
||||
custom_logger.logger.data[key] = "old_value"
|
||||
custom_logger.update(**{key: value})
|
||||
logger_data = custom_logger.logger.data
|
||||
|
||||
assert key in logger_data
|
||||
assert logger_data[key] == value
|
||||
|
||||
|
||||
# Test case for updating "dut_jobs"
|
||||
def test_update_dut_jobs(custom_logger):
|
||||
key1 = "status"
|
||||
value1 = "fail"
|
||||
key2 = "state"
|
||||
value2 = "hung"
|
||||
|
||||
custom_logger.create_dut_job(**{key1: value1})
|
||||
logger_data = custom_logger.logger.data
|
||||
|
||||
job1 = logger_data["dut_jobs"][0]
|
||||
assert key1 in job1
|
||||
assert job1[key1] == value1
|
||||
|
||||
custom_logger.update_dut_job(key2, value2)
|
||||
logger_data = custom_logger.logger.data
|
||||
|
||||
job2 = logger_data["dut_jobs"][0]
|
||||
assert key2 in job2
|
||||
assert job2[key2] == value2
|
||||
|
||||
|
||||
# Test case for creating and updating DUT job
|
||||
def test_create_dut_job(custom_logger):
|
||||
key = "status"
|
||||
value1 = "pass"
|
||||
value2 = "fail"
|
||||
value3 = "hung"
|
||||
|
||||
reason = "job_combined_status"
|
||||
result = "Finished"
|
||||
|
||||
custom_logger.update(**{reason: result})
|
||||
logger_data = custom_logger.logger.data
|
||||
|
||||
assert reason in logger_data
|
||||
assert logger_data[reason] == result
|
||||
|
||||
# Create the first DUT job
|
||||
custom_logger.create_dut_job(**{key: value1})
|
||||
logger_data = custom_logger.logger.data
|
||||
|
||||
assert "dut_jobs" in logger_data
|
||||
assert isinstance(logger_data["dut_jobs"], list)
|
||||
assert len(logger_data["dut_jobs"]) == 1
|
||||
assert isinstance(logger_data["dut_jobs"][0], dict)
|
||||
|
||||
# Check the values of the keys in the created first DUT job
|
||||
job1 = logger_data["dut_jobs"][0]
|
||||
assert key in job1
|
||||
assert job1[key] == value1
|
||||
|
||||
# Create the second DUT job
|
||||
custom_logger.create_dut_job(**{key: value2})
|
||||
logger_data = custom_logger.logger.data
|
||||
|
||||
assert "dut_jobs" in logger_data
|
||||
assert isinstance(logger_data["dut_jobs"], list)
|
||||
assert len(logger_data["dut_jobs"]) == 2
|
||||
assert isinstance(logger_data["dut_jobs"][1], dict)
|
||||
|
||||
# Check the values of the keys in the created second DUT job
|
||||
job2 = logger_data["dut_jobs"][1]
|
||||
assert key in job2
|
||||
assert job2[key] == value2
|
||||
|
||||
# Update the second DUT job with value3
|
||||
custom_logger.update_dut_job(key, value3)
|
||||
logger_data = custom_logger.logger.data
|
||||
|
||||
# Check the updated value in the second DUT job
|
||||
job2 = logger_data["dut_jobs"][1]
|
||||
assert key in job2
|
||||
assert job2[key] == value3
|
||||
|
||||
# Find the index of the last DUT job
|
||||
last_job_index = len(logger_data["dut_jobs"]) - 1
|
||||
|
||||
# Update the last DUT job
|
||||
custom_logger.update_dut_job("dut_name", "asus")
|
||||
logger_data = custom_logger.logger.data
|
||||
|
||||
# Check the updated value in the last DUT job
|
||||
job2 = logger_data["dut_jobs"][last_job_index]
|
||||
assert "dut_name" in job2
|
||||
assert job2["dut_name"] == "asus"
|
||||
|
||||
# Check that "dut_name" is not present in other DUT jobs
|
||||
for idx, job in enumerate(logger_data["dut_jobs"]):
|
||||
if idx != last_job_index:
|
||||
assert job.get("dut_name") == ""
|
||||
|
||||
|
||||
# Test case for updating with missing "dut_jobs" key
|
||||
def test_update_dut_job_missing_dut_jobs(custom_logger):
|
||||
key = "status"
|
||||
value = "fail"
|
||||
|
||||
# Attempt to update a DUT job when "dut_jobs" is missing
|
||||
with pytest.raises(ValueError, match="No DUT jobs found."):
|
||||
custom_logger.update_dut_job(key, value)
|
||||
|
||||
|
||||
# Test case for creating a job phase
|
||||
def test_create_job_phase(custom_logger):
|
||||
custom_logger.create_dut_job(status="pass")
|
||||
phase_name = "Phase1"
|
||||
|
||||
custom_logger.create_job_phase(phase_name)
|
||||
logger_data = custom_logger.logger.data
|
||||
|
||||
assert "dut_jobs" in logger_data
|
||||
assert isinstance(logger_data["dut_jobs"], list)
|
||||
assert len(logger_data["dut_jobs"]) == 1
|
||||
|
||||
job = logger_data["dut_jobs"][0]
|
||||
assert "dut_job_phases" in job
|
||||
assert isinstance(job["dut_job_phases"], list)
|
||||
assert len(job["dut_job_phases"]) == 1
|
||||
|
||||
phase = job["dut_job_phases"][0]
|
||||
assert phase["name"] == phase_name
|
||||
try:
|
||||
datetime.fromisoformat(phase["start_time"])
|
||||
assert True
|
||||
except ValueError:
|
||||
assert False
|
||||
assert phase["end_time"] == ""
|
||||
|
||||
|
||||
# Test case for creating multiple phase jobs
|
||||
def test_create_multiple_phase_jobs(custom_logger):
|
||||
custom_logger.create_dut_job(status="pass")
|
||||
|
||||
phase_data = [
|
||||
{
|
||||
"phase_name": "Phase1",
|
||||
},
|
||||
{
|
||||
"phase_name": "Phase2",
|
||||
},
|
||||
{
|
||||
"phase_name": "Phase3",
|
||||
},
|
||||
]
|
||||
|
||||
for data in phase_data:
|
||||
phase_name = data["phase_name"]
|
||||
|
||||
custom_logger.create_job_phase(phase_name)
|
||||
|
||||
logger_data = custom_logger.logger.data
|
||||
|
||||
assert "dut_jobs" in logger_data
|
||||
assert isinstance(logger_data["dut_jobs"], list)
|
||||
assert len(logger_data["dut_jobs"]) == 1
|
||||
|
||||
job = logger_data["dut_jobs"][0]
|
||||
assert "dut_job_phases" in job
|
||||
assert isinstance(job["dut_job_phases"], list)
|
||||
assert len(job["dut_job_phases"]) == len(phase_data)
|
||||
|
||||
for data in phase_data:
|
||||
phase_name = data["phase_name"]
|
||||
|
||||
phase = job["dut_job_phases"][phase_data.index(data)]
|
||||
|
||||
assert phase["name"] == phase_name
|
||||
try:
|
||||
datetime.fromisoformat(phase["start_time"])
|
||||
assert True
|
||||
except ValueError:
|
||||
assert False
|
||||
|
||||
if phase_data.index(data) != len(phase_data) - 1:
|
||||
try:
|
||||
datetime.fromisoformat(phase["end_time"])
|
||||
assert True
|
||||
except ValueError:
|
||||
assert False
|
||||
|
||||
# Check if the end_time of the last phase is an empty string
|
||||
last_phase = job["dut_job_phases"][-1]
|
||||
assert last_phase["end_time"] == ""
|
||||
|
||||
|
||||
# Test case for creating multiple dut jobs and updating phase job for last dut job
|
||||
def test_create_two_dut_jobs_and_add_phase(custom_logger):
|
||||
# Create the first DUT job
|
||||
custom_logger.create_dut_job(status="pass")
|
||||
|
||||
# Create the second DUT job
|
||||
custom_logger.create_dut_job(status="fail")
|
||||
|
||||
logger_data = custom_logger.logger.data
|
||||
|
||||
assert "dut_jobs" in logger_data
|
||||
assert isinstance(logger_data["dut_jobs"], list)
|
||||
assert len(logger_data["dut_jobs"]) == 2
|
||||
|
||||
first_dut_job = logger_data["dut_jobs"][0]
|
||||
second_dut_job = logger_data["dut_jobs"][1]
|
||||
|
||||
# Add a phase to the second DUT job
|
||||
custom_logger.create_job_phase("Phase1")
|
||||
|
||||
logger_data = custom_logger.logger.data
|
||||
|
||||
assert "dut_jobs" in logger_data
|
||||
assert isinstance(logger_data["dut_jobs"], list)
|
||||
assert len(logger_data["dut_jobs"]) == 2
|
||||
|
||||
first_dut_job = logger_data["dut_jobs"][0]
|
||||
second_dut_job = logger_data["dut_jobs"][1]
|
||||
|
||||
# Check first DUT job does not have a phase
|
||||
assert not first_dut_job.get("dut_job_phases")
|
||||
|
||||
# Check second DUT job has a phase
|
||||
assert second_dut_job.get("dut_job_phases")
|
||||
assert isinstance(second_dut_job["dut_job_phases"], list)
|
||||
assert len(second_dut_job["dut_job_phases"]) == 1
|
||||
|
||||
|
||||
# Test case for updating DUT start time
|
||||
def test_update_dut_start_time(custom_logger):
|
||||
custom_logger.create_dut_job(status="pass")
|
||||
custom_logger.update_dut_time("start", None)
|
||||
|
||||
logger_data = custom_logger.logger.data
|
||||
assert "dut_jobs" in logger_data
|
||||
assert len(logger_data["dut_jobs"]) == 1
|
||||
|
||||
dut_job = logger_data["dut_jobs"][0]
|
||||
assert "dut_start_time" in dut_job
|
||||
assert dut_job["dut_start_time"] != ""
|
||||
|
||||
try:
|
||||
datetime.fromisoformat(dut_job["dut_start_time"])
|
||||
assert True
|
||||
except ValueError:
|
||||
assert False
|
||||
|
||||
|
||||
# Test case for updating DUT submit time
|
||||
def test_update_dut_submit_time(custom_logger):
|
||||
custom_time = "2023-11-09T02:37:06Z"
|
||||
custom_logger.create_dut_job(status="pass")
|
||||
custom_logger.update_dut_time("submit", custom_time)
|
||||
|
||||
logger_data = custom_logger.logger.data
|
||||
assert "dut_jobs" in logger_data
|
||||
assert len(logger_data["dut_jobs"]) == 1
|
||||
|
||||
dut_job = logger_data["dut_jobs"][0]
|
||||
assert "dut_submit_time" in dut_job
|
||||
|
||||
try:
|
||||
datetime.fromisoformat(dut_job["dut_submit_time"])
|
||||
assert True
|
||||
except ValueError:
|
||||
assert False
|
||||
|
||||
|
||||
# Test case for updating DUT end time
|
||||
def test_update_dut_end_time(custom_logger):
|
||||
custom_logger.create_dut_job(status="pass")
|
||||
custom_logger.update_dut_time("end", None)
|
||||
|
||||
logger_data = custom_logger.logger.data
|
||||
assert "dut_jobs" in logger_data
|
||||
assert len(logger_data["dut_jobs"]) == 1
|
||||
|
||||
dut_job = logger_data["dut_jobs"][0]
|
||||
assert "dut_end_time" in dut_job
|
||||
|
||||
try:
|
||||
datetime.fromisoformat(dut_job["dut_end_time"])
|
||||
assert True
|
||||
except ValueError:
|
||||
assert False
|
||||
|
||||
|
||||
# Test case for updating DUT time with invalid value
|
||||
def test_update_dut_time_invalid_value(custom_logger):
|
||||
custom_logger.create_dut_job(status="pass")
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Error: Invalid argument provided for --update-dut-time. Use 'start', 'submit', 'end'.",
|
||||
):
|
||||
custom_logger.update_dut_time("invalid_value", None)
|
||||
|
||||
|
||||
# Test case for close_dut_job
|
||||
def test_close_dut_job(custom_logger):
|
||||
custom_logger.create_dut_job(status="pass")
|
||||
|
||||
custom_logger.create_job_phase("Phase1")
|
||||
custom_logger.create_job_phase("Phase2")
|
||||
|
||||
custom_logger.close_dut_job()
|
||||
|
||||
logger_data = custom_logger.logger.data
|
||||
assert "dut_jobs" in logger_data
|
||||
assert len(logger_data["dut_jobs"]) == 1
|
||||
|
||||
dut_job = logger_data["dut_jobs"][0]
|
||||
assert "dut_job_phases" in dut_job
|
||||
dut_job_phases = dut_job["dut_job_phases"]
|
||||
|
||||
phase1 = dut_job_phases[0]
|
||||
assert phase1["name"] == "Phase1"
|
||||
|
||||
try:
|
||||
datetime.fromisoformat(phase1["start_time"])
|
||||
assert True
|
||||
except ValueError:
|
||||
assert False
|
||||
|
||||
try:
|
||||
datetime.fromisoformat(phase1["end_time"])
|
||||
assert True
|
||||
except ValueError:
|
||||
assert False
|
||||
|
||||
phase2 = dut_job_phases[1]
|
||||
assert phase2["name"] == "Phase2"
|
||||
|
||||
try:
|
||||
datetime.fromisoformat(phase2["start_time"])
|
||||
assert True
|
||||
except ValueError:
|
||||
assert False
|
||||
|
||||
try:
|
||||
datetime.fromisoformat(phase2["end_time"])
|
||||
assert True
|
||||
except ValueError:
|
||||
assert False
|
||||
|
||||
|
||||
# Test case for close
|
||||
def test_close(custom_logger):
|
||||
custom_logger.create_dut_job(status="pass")
|
||||
|
||||
custom_logger.close()
|
||||
|
||||
logger_data = custom_logger.logger.data
|
||||
assert "dut_jobs" in logger_data
|
||||
assert len(logger_data["dut_jobs"]) == 1
|
||||
assert "dut_attempt_counter" in logger_data
|
||||
assert logger_data["dut_attempt_counter"] == len(logger_data["dut_jobs"])
|
||||
assert "job_combined_status" in logger_data
|
||||
assert logger_data["job_combined_status"] != ""
|
||||
|
||||
dut_job = logger_data["dut_jobs"][0]
|
||||
assert "submitter_end_time" in dut_job
|
||||
try:
|
||||
datetime.fromisoformat(dut_job["submitter_end_time"])
|
||||
assert True
|
||||
except ValueError:
|
||||
assert False
|
||||
|
||||
|
||||
# Test case for updating status to fail with a reason
|
||||
def test_update_status_fail_with_reason(custom_logger):
|
||||
custom_logger.create_dut_job()
|
||||
|
||||
reason = "kernel panic"
|
||||
custom_logger.update_status_fail(reason)
|
||||
|
||||
logger_data = custom_logger.logger.data
|
||||
assert "dut_jobs" in logger_data
|
||||
assert len(logger_data["dut_jobs"]) == 1
|
||||
|
||||
dut_job = logger_data["dut_jobs"][0]
|
||||
assert "status" in dut_job
|
||||
assert dut_job["status"] == "fail"
|
||||
assert "dut_job_fail_reason" in dut_job
|
||||
assert dut_job["dut_job_fail_reason"] == reason
|
||||
|
||||
|
||||
# Test case for updating status to fail without providing a reason
|
||||
def test_update_status_fail_without_reason(custom_logger):
|
||||
custom_logger.create_dut_job()
|
||||
|
||||
custom_logger.update_status_fail()
|
||||
|
||||
# Check if the status is updated and fail reason is empty
|
||||
logger_data = custom_logger.logger.data
|
||||
assert "dut_jobs" in logger_data
|
||||
assert len(logger_data["dut_jobs"]) == 1
|
||||
|
||||
dut_job = logger_data["dut_jobs"][0]
|
||||
assert "status" in dut_job
|
||||
assert dut_job["status"] == "fail"
|
||||
assert "dut_job_fail_reason" in dut_job
|
||||
assert dut_job["dut_job_fail_reason"] == ""
|
||||
|
||||
|
||||
# Test case for check_dut_timings with submission time earlier than start time
|
||||
def test_check_dut_timings_submission_earlier_than_start(custom_logger, caplog):
|
||||
custom_logger.create_dut_job()
|
||||
|
||||
# Set submission time to be earlier than start time
|
||||
custom_logger.update_dut_time("start", "2023-01-01T11:00:00")
|
||||
custom_logger.update_dut_time("submit", "2023-01-01T12:00:00")
|
||||
|
||||
logger_data = custom_logger.logger.data
|
||||
assert "dut_jobs" in logger_data
|
||||
assert len(logger_data["dut_jobs"]) == 1
|
||||
|
||||
job = logger_data["dut_jobs"][0]
|
||||
|
||||
# Call check_dut_timings
|
||||
custom_logger.check_dut_timings(job)
|
||||
|
||||
# Check if an error message is logged
|
||||
assert "Job submission is happening before job start." in caplog.text
|
||||
|
||||
|
||||
# Test case for check_dut_timings with end time earlier than start time
|
||||
def test_check_dut_timings_end_earlier_than_start(custom_logger, caplog):
|
||||
custom_logger.create_dut_job()
|
||||
|
||||
# Set end time to be earlier than start time
|
||||
custom_logger.update_dut_time("end", "2023-01-01T11:00:00")
|
||||
custom_logger.update_dut_time("start", "2023-01-01T12:00:00")
|
||||
|
||||
logger_data = custom_logger.logger.data
|
||||
assert "dut_jobs" in logger_data
|
||||
assert len(logger_data["dut_jobs"]) == 1
|
||||
|
||||
job = logger_data["dut_jobs"][0]
|
||||
|
||||
# Call check_dut_timings
|
||||
custom_logger.check_dut_timings(job)
|
||||
|
||||
# Check if an error message is logged
|
||||
assert "Job ended before it started." in caplog.text
|
||||
|
||||
|
||||
# Test case for check_dut_timings with valid timing sequence
|
||||
def test_check_dut_timings_valid_timing_sequence(custom_logger, caplog):
|
||||
custom_logger.create_dut_job()
|
||||
|
||||
# Set valid timing sequence
|
||||
custom_logger.update_dut_time("submit", "2023-01-01T12:00:00")
|
||||
custom_logger.update_dut_time("start", "2023-01-01T12:30:00")
|
||||
custom_logger.update_dut_time("end", "2023-01-01T13:00:00")
|
||||
|
||||
logger_data = custom_logger.logger.data
|
||||
assert "dut_jobs" in logger_data
|
||||
assert len(logger_data["dut_jobs"]) == 1
|
||||
|
||||
job = logger_data["dut_jobs"][0]
|
||||
|
||||
# Call check_dut_timings
|
||||
custom_logger.check_dut_timings(job)
|
||||
|
||||
# Check that no error messages are logged
|
||||
assert "Job submission is happening before job start." not in caplog.text
|
||||
assert "Job ended before it started." not in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("start_time,end_time,expected", [
|
||||
("2023-01-01T12:00:00", "2023-01-01T12:30:00", "0:30:00"),
|
||||
("invalid", "2023-01-01T12:30:00", ""),
|
||||
("", "", ""),
|
||||
])
|
||||
def test_get_duration_time(custom_logger, start_time, end_time, expected):
|
||||
duration = custom_logger.get_duration_time(start_time, end_time)
|
||||
assert duration == expected
|
||||
|
||||
|
||||
# Test case for DUT job duration tracking
|
||||
def test_dut_job_duration(custom_logger):
|
||||
custom_logger.create_dut_job(status="pass")
|
||||
|
||||
# Set start time
|
||||
custom_logger.update_dut_time("start", "2023-01-01T12:00:00")
|
||||
logger_data = custom_logger.logger.data
|
||||
dut_job = logger_data["dut_jobs"][0]
|
||||
assert dut_job["dut_duration_time"] == ""
|
||||
|
||||
# Set end time
|
||||
custom_logger.update_dut_time("end", "2023-01-01T12:30:00")
|
||||
logger_data = custom_logger.logger.data
|
||||
dut_job = logger_data["dut_jobs"][0]
|
||||
assert dut_job["dut_duration_time"] == "0:30:00"
|
||||
|
||||
|
||||
# Test case for job phase duration tracking
|
||||
def test_job_phase_duration(custom_logger):
|
||||
custom_logger.create_dut_job(status="pass")
|
||||
|
||||
# Create first phase
|
||||
custom_logger.create_job_phase("Phase1")
|
||||
logger_data = custom_logger.logger.data
|
||||
job = logger_data["dut_jobs"][0]
|
||||
phase = job["dut_job_phases"][0]
|
||||
assert phase["duration_time"] == ""
|
||||
|
||||
# Create second phase (which should close the first phase)
|
||||
custom_logger.create_job_phase("Phase2")
|
||||
logger_data = custom_logger.logger.data
|
||||
job = logger_data["dut_jobs"][0]
|
||||
|
||||
# First phase should have duration
|
||||
phase1 = job["dut_job_phases"][0]
|
||||
assert phase1["duration_time"] != ""
|
||||
|
||||
# Verify duration format
|
||||
start = datetime.fromisoformat(phase1["start_time"])
|
||||
end = datetime.fromisoformat(phase1["end_time"])
|
||||
duration = end - start
|
||||
assert phase1["duration_time"] == str(duration)
|
||||
|
||||
# Second phase should have empty duration
|
||||
phase2 = job["dut_job_phases"][1]
|
||||
assert phase2["duration_time"] == ""
|
||||
|
||||
|
||||
# Test case for closing job with phases
|
||||
def test_close_job_with_phases_duration(custom_logger):
|
||||
custom_logger.create_dut_job(status="pass")
|
||||
custom_logger.create_job_phase("Phase1")
|
||||
custom_logger.close_dut_job()
|
||||
|
||||
logger_data = custom_logger.logger.data
|
||||
job = logger_data["dut_jobs"][0]
|
||||
phase = job["dut_job_phases"][0]
|
||||
|
||||
assert phase["duration_time"] != ""
|
||||
|
||||
# Verify duration format
|
||||
start = datetime.fromisoformat(phase["start_time"])
|
||||
end = datetime.fromisoformat(phase["end_time"])
|
||||
duration = end - start
|
||||
assert phase["duration_time"] == str(duration)
|
||||
@@ -0,0 +1,201 @@
|
||||
from contextlib import suppress
|
||||
from datetime import datetime, timedelta
|
||||
from unittest import mock
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import ci_post_gantt
|
||||
import pytest
|
||||
from ci_gantt_chart import generate_gantt_chart
|
||||
from ci_post_gantt import Gitlab, MockGanttExit
|
||||
|
||||
|
||||
def create_mock_job(
|
||||
name, id, status, created_at, queued_duration, started_at, finished_at=None
|
||||
):
|
||||
mock_job = MagicMock()
|
||||
mock_job.name = name
|
||||
mock_job.status = status
|
||||
mock_job.id = id
|
||||
mock_job.created_at = created_at
|
||||
mock_job.queued_duration = queued_duration
|
||||
mock_job.started_at = started_at
|
||||
mock_job.finished_at = finished_at
|
||||
return mock_job
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_pipeline():
|
||||
current_time = datetime.fromisoformat("2024-12-17 23:54:13.940091+00:00")
|
||||
created_at = current_time - timedelta(minutes=10)
|
||||
|
||||
job1 = create_mock_job(
|
||||
name="job1",
|
||||
id="1",
|
||||
status="success",
|
||||
created_at=created_at.isoformat(),
|
||||
queued_duration=1, # seconds
|
||||
started_at=(created_at + timedelta(seconds=2)).isoformat(),
|
||||
finished_at=(created_at + timedelta(minutes=1)).isoformat(),
|
||||
)
|
||||
|
||||
mock_pipeline = MagicMock()
|
||||
mock_pipeline.web_url = "https://gitlab.freedesktop.org/mesa/mesa/-/pipelines/9999"
|
||||
mock_pipeline.duration = 600 # Total pipeline duration in seconds
|
||||
mock_pipeline.created_at = created_at.isoformat()
|
||||
mock_pipeline.yaml_errors = False
|
||||
mock_pipeline.jobs.list.return_value = [job1]
|
||||
return mock_pipeline
|
||||
|
||||
|
||||
def test_generate_gantt_chart(fake_pipeline):
|
||||
fig = generate_gantt_chart(fake_pipeline)
|
||||
|
||||
fig_dict = fig.to_dict()
|
||||
assert "data" in fig_dict
|
||||
|
||||
# Extract all job names from the "y" axis in the Gantt chart data
|
||||
all_job_names = set()
|
||||
for trace in fig_dict["data"]:
|
||||
if "y" in trace:
|
||||
all_job_names.update(trace["y"])
|
||||
|
||||
assert any(
|
||||
"job1" in job for job in all_job_names
|
||||
), "job1 should be present in the Gantt chart"
|
||||
|
||||
|
||||
def test_ci_timeout(fake_pipeline):
|
||||
fig = generate_gantt_chart(fake_pipeline, ci_timeout=1)
|
||||
|
||||
fig_dict = fig.to_dict()
|
||||
|
||||
timeout_line = None
|
||||
for shape in fig_dict.get("layout", {}).get("shapes", []):
|
||||
if shape.get("line", {}).get("dash") == "dash":
|
||||
timeout_line = shape
|
||||
break
|
||||
|
||||
assert timeout_line is not None, "Timeout line should exist in the Gantt chart"
|
||||
timeout_x = timeout_line["x0"]
|
||||
|
||||
# Check that the timeout line is 1 minute after the pipeline creation time
|
||||
pipeline_created_at = datetime.fromisoformat(fake_pipeline.created_at)
|
||||
expected_timeout = pipeline_created_at + timedelta(minutes=1)
|
||||
assert (
|
||||
timeout_x == expected_timeout
|
||||
), f"Timeout should be at {expected_timeout}, got {timeout_x}"
|
||||
|
||||
|
||||
def test_marge_bot_user_id():
|
||||
with patch("ci_post_gantt.Gitlab") as MockGitlab:
|
||||
mock_gitlab_instance = MagicMock(spec=Gitlab)
|
||||
mock_gitlab_instance.users = MagicMock()
|
||||
MockGitlab.return_value = mock_gitlab_instance
|
||||
|
||||
marge_bot_user_id = 12345
|
||||
ci_post_gantt.main("fake_token", None, marge_bot_user_id)
|
||||
mock_gitlab_instance.users.get.assert_called_once_with(marge_bot_user_id)
|
||||
|
||||
|
||||
def test_project_ids():
|
||||
current_time = datetime.now()
|
||||
project_id_1 = 176
|
||||
event_1 = MagicMock()
|
||||
event_1.project_id = project_id_1
|
||||
event_1.created_at = (current_time - timedelta(days=1)).isoformat()
|
||||
event_1.note = {"body": f"Event for project {project_id_1}"}
|
||||
|
||||
project_id_2 = 166
|
||||
event_2 = MagicMock()
|
||||
event_2.project_id = project_id_2
|
||||
event_2.created_at = (current_time - timedelta(days=2)).isoformat()
|
||||
event_2.note = {"body": f"Event for project {project_id_2}"}
|
||||
|
||||
with patch("ci_post_gantt.Gitlab") as MockGitlab:
|
||||
mock_user = MagicMock()
|
||||
mock_user.events = MagicMock()
|
||||
mock_user.events.list.return_value = [event_1, event_2]
|
||||
|
||||
mock_gitlab_instance = MagicMock(spec=Gitlab)
|
||||
mock_gitlab_instance.users = MagicMock()
|
||||
mock_gitlab_instance.users.get.return_value = mock_user
|
||||
MockGitlab.return_value = mock_gitlab_instance
|
||||
|
||||
last_event_date = (current_time - timedelta(days=3)).isoformat()
|
||||
|
||||
# Test a single project id
|
||||
ci_post_gantt.main("fake_token", last_event_date)
|
||||
marge_bot_single_project_scope = [
|
||||
event.note["body"]
|
||||
for event in mock_user.events.list.return_value
|
||||
if event.project_id == project_id_1
|
||||
]
|
||||
assert f"Event for project {project_id_1}" in marge_bot_single_project_scope
|
||||
assert f"Event for project {project_id_2}" not in marge_bot_single_project_scope
|
||||
|
||||
# Test multiple project ids
|
||||
ci_post_gantt.main(
|
||||
"fake_token", last_event_date, 9716, [project_id_1, project_id_2]
|
||||
)
|
||||
|
||||
marge_bot_multiple_project_scope = [
|
||||
event.note["body"] for event in mock_user.events.list.return_value
|
||||
]
|
||||
assert f"Event for project {project_id_1}" in marge_bot_multiple_project_scope
|
||||
assert f"Event for project {project_id_2}" in marge_bot_multiple_project_scope
|
||||
|
||||
|
||||
def test_add_gantt_after_pipeline_message():
|
||||
current_time = datetime.now()
|
||||
|
||||
plain_url = "https://gitlab.freedesktop.org/mesa/mesa/-/pipelines/12345"
|
||||
plain_message = (
|
||||
f"I couldn't merge this branch: CI failed! See pipeline {plain_url}."
|
||||
)
|
||||
event_plain = MagicMock()
|
||||
event_plain.project_id = 176
|
||||
event_plain.created_at = (current_time - timedelta(days=1)).isoformat()
|
||||
event_plain.note = {"body": plain_message}
|
||||
|
||||
summary_url = "https://gitlab.freedesktop.org/mesa/mesa/-/pipelines/99999"
|
||||
summary_message = (
|
||||
"I couldn't merge this branch: "
|
||||
f"CI failed! See pipeline {summary_url}.<br>There were problems with job:"
|
||||
"[lavapipe](https://gitlab.freedesktop.org/mesa/mesa/-/jobs/68141218)<details><summary> "
|
||||
"3 crashed tests</summary>dEQP-VK.ray_query.builtin.instancecustomindex.frag.aabbs,Crash<br>dEQP"
|
||||
"-VK.ray_query.builtin.objecttoworld.frag.aabbs,Crash<br>dEQP-VK.sparse_resources.shader_intrinsics."
|
||||
"2d_array_sparse_fetch.g16_b16r16_2plane_444_unorm.11_37_3_nontemporal,Crash<br></details>"
|
||||
)
|
||||
event_with_summary = MagicMock()
|
||||
event_with_summary.project_id = 176
|
||||
event_with_summary.created_at = (current_time - timedelta(days=1)).isoformat()
|
||||
event_with_summary.note = {"body": summary_message}
|
||||
|
||||
with patch("ci_post_gantt.Gitlab") as MockGitlab, patch(
|
||||
"ci_post_gantt.get_gitlab_pipeline_from_url", return_value=None
|
||||
) as mock_get_gitlab_pipeline_from_url:
|
||||
|
||||
def safe_mock(*args, **kwargs):
|
||||
with suppress(TypeError):
|
||||
raise MockGanttExit("Exiting for test purposes")
|
||||
|
||||
mock_get_gitlab_pipeline_from_url.side_effect = safe_mock
|
||||
|
||||
mock_user = MagicMock()
|
||||
mock_user.events = MagicMock()
|
||||
mock_user.events.list.return_value = [event_plain, event_with_summary]
|
||||
|
||||
mock_gitlab_instance = MagicMock(spec=Gitlab)
|
||||
mock_gitlab_instance.users = MagicMock()
|
||||
mock_gitlab_instance.users.get.return_value = mock_user
|
||||
MockGitlab.return_value = mock_gitlab_instance
|
||||
|
||||
last_event_date = (current_time - timedelta(days=3)).isoformat()
|
||||
ci_post_gantt.main("fake_token", last_event_date, 12345)
|
||||
mock_get_gitlab_pipeline_from_url.assert_has_calls(
|
||||
[
|
||||
mock.call(mock_gitlab_instance, plain_url),
|
||||
mock.call(mock_gitlab_instance, summary_url),
|
||||
],
|
||||
any_order=True,
|
||||
)
|
||||
@@ -0,0 +1,176 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright © 2025 Collabora Ltd.
|
||||
# Authors:
|
||||
# Sergi Blanch Torne <[email protected]>
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import timedelta
|
||||
|
||||
from unittest import TestCase
|
||||
from typing import Iterable
|
||||
|
||||
from marge_queue import MargeMergeRequest, MargeQueue, get_merge_queue
|
||||
|
||||
|
||||
DATA = {
|
||||
37564: {
|
||||
# sample of a Merge Request rejected by marge and later reassigned to marge
|
||||
"updated_at": "2025-09-25T10:59:27.144Z",
|
||||
"title": "etnaviv: Fix util_blitter_save_so_targets(..) call",
|
||||
"notes": [
|
||||
# Those notes must be listed in reverse order
|
||||
# because the list is requested sort="desc"
|
||||
{
|
||||
"body": "assigned to @marge-bot and unassigned @austriancoder",
|
||||
"created_at": "2025-09-25T11:44:01.672Z",
|
||||
},
|
||||
{
|
||||
"body": "added 1 commit",
|
||||
"created_at": "2025-09-25T10:43:43.490Z",
|
||||
},
|
||||
{
|
||||
"body": "This branch couldn't be merged:",
|
||||
"created_at": "2025-09-25T10:42:27.817Z",
|
||||
},
|
||||
{
|
||||
"body": "assigned to @austriancoder and unassigned @marge-bot",
|
||||
"created_at": "2025-09-25T10:42:15.404Z",
|
||||
},
|
||||
{
|
||||
"body": "added 3 commits",
|
||||
"created_at": "2025-09-25T10:38:57.332Z",
|
||||
},
|
||||
{
|
||||
"body": "assigned to @marge-bot",
|
||||
"created_at": "2025-09-25T10:38:03.993Z",
|
||||
},
|
||||
],
|
||||
},
|
||||
37560: {
|
||||
# sample of a Merge Request assigned to marge between the two
|
||||
# assignments of the previous sample. This is meant to show that
|
||||
# this goes first in the queue, and it doesn't confuses the sequence.
|
||||
"updated_at": "2025-09-25T11:38:08.420Z",
|
||||
"title": "tu: limit query pool types logged into RMV",
|
||||
"notes": [
|
||||
{
|
||||
"body": "assigned to @marge-bot",
|
||||
"created_at": "2025-09-25T10:47:40.040Z",
|
||||
},
|
||||
{
|
||||
"body": "added 1 commit",
|
||||
"created_at": "2025-09-25T10:46:42.113Z",
|
||||
},
|
||||
{
|
||||
"body": "mentioned in issue #13970",
|
||||
"created_at": "2025-09-25T10:45:35.566Z",
|
||||
},
|
||||
{
|
||||
"body": "Rb",
|
||||
"created_at": "2025-09-25T08:25:51.527Z",
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProjectMergeRequestNote:
|
||||
body: str
|
||||
created_at: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProjectMergeRequestNoteManager:
|
||||
"""
|
||||
Mock the gitlab.v4.objects.ProjectMergeRequestNoteManager
|
||||
"""
|
||||
iid: int
|
||||
|
||||
def list(self, *args, **kwargs) -> Iterable:
|
||||
for note in DATA[self.iid]["notes"]:
|
||||
yield ProjectMergeRequestNote(**note)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProjectMergeRequest:
|
||||
"""
|
||||
Mock the gitlab.v4.objects.ProjectMergeRequest
|
||||
"""
|
||||
iid: int
|
||||
|
||||
@property
|
||||
def updated_at(self) -> str:
|
||||
return DATA[self.iid]["updated_at"]
|
||||
|
||||
@property
|
||||
def web_url(self) -> str:
|
||||
return f"https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/{self.iid}"
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
return DATA[self.iid]["title"]
|
||||
|
||||
@property
|
||||
def notes(self) -> ProjectMergeRequestNoteManager:
|
||||
return ProjectMergeRequestNoteManager(self.iid)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProjectMergeRequestManager:
|
||||
"""
|
||||
Mock the gitlab.v4.objects.ProjectMergeRequestManager
|
||||
"""
|
||||
def list(self, *args, **kwargs) -> Iterable:
|
||||
for mr in DATA.keys():
|
||||
yield ProjectMergeRequest(mr)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Project:
|
||||
"""
|
||||
Mock the gitlab.v4.objects.Project
|
||||
"""
|
||||
@property
|
||||
def mergerequests(self) -> ProjectMergeRequestManager:
|
||||
return ProjectMergeRequestManager()
|
||||
|
||||
|
||||
class MargeMergeTestCase(TestCase):
|
||||
def test_queue(self) -> None:
|
||||
queue = MargeQueue()
|
||||
for mr_id in DATA.keys():
|
||||
gl_mr_mock_obj = ProjectMergeRequest(mr_id)
|
||||
marge_mr = MargeMergeRequest(gl_mr_mock_obj)
|
||||
|
||||
# for each element in the data sample, test the MargeMergeRequest construction
|
||||
self.assertEqual(marge_mr.title, gl_mr_mock_obj.title)
|
||||
self.assertEqual(marge_mr.web_url, gl_mr_mock_obj.web_url)
|
||||
self.assertIsInstance(marge_mr.time_enqueued, timedelta)
|
||||
|
||||
queue.append(marge_mr)
|
||||
|
||||
self.assertEqual(queue.n_merge_requests_enqueued, 2)
|
||||
|
||||
|
||||
def test_get_merge_queue(self) -> None:
|
||||
project = Project()
|
||||
queue = get_merge_queue(project)
|
||||
|
||||
self.assertEqual(queue.n_merge_requests_enqueued, 2)
|
||||
self.assertEqual(len(queue.sorted_queue), 2)
|
||||
# They are in reverse order because of the reassignment to marge
|
||||
self.assertEqual(queue.sorted_queue[0].id, list(DATA.keys())[1])
|
||||
self.assertEqual(queue.sorted_queue[1].id, list(DATA.keys())[0])
|
||||
self.assertEqual(len(queue.undetermined), 0)
|
||||
|
||||
for marge_mr in queue.sorted_queue:
|
||||
mr_id = marge_mr.id
|
||||
self.assertEqual(marge_mr.title, DATA[mr_id]["title"])
|
||||
self.assertEqual(
|
||||
marge_mr.web_url,
|
||||
f"https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/{mr_id}"
|
||||
)
|
||||
self.assertIsInstance(marge_mr.time_enqueued, timedelta)
|
||||
@@ -0,0 +1,309 @@
|
||||
import logging
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from pipeline_message import (
|
||||
get_failed_test_summary_message,
|
||||
get_problem_jobs,
|
||||
get_trace_failures,
|
||||
main,
|
||||
process_problem_jobs,
|
||||
search_job_log_for_errors,
|
||||
sort_failed_tests_by_status,
|
||||
unexpected_improvements,
|
||||
)
|
||||
|
||||
|
||||
def test_get_problem_jobs():
|
||||
jobs = [
|
||||
{"stage": "build", "status": "failed"},
|
||||
{"stage": "test", "status": "canceled"},
|
||||
{"stage": "postmerge", "status": "failed"},
|
||||
{"stage": "performance", "status": "failed"},
|
||||
{"stage": "deploy", "status": "failed"},
|
||||
]
|
||||
|
||||
problem_jobs = get_problem_jobs(jobs)
|
||||
|
||||
assert len(problem_jobs) == 3
|
||||
assert problem_jobs[0]["stage"] == "build"
|
||||
assert problem_jobs[1]["stage"] == "test"
|
||||
assert problem_jobs[2]["stage"] == "deploy"
|
||||
|
||||
|
||||
def test_sort_failed_tests_by_status():
|
||||
failures_csv = """\
|
||||
Test1,UnexpectedImprovement
|
||||
Test2,Fail
|
||||
Test3,Crash
|
||||
Test4,Timeout
|
||||
Test5,Fail
|
||||
Test6,UnexpectedImprovement
|
||||
"""
|
||||
sorted_tests = sort_failed_tests_by_status(failures_csv)
|
||||
|
||||
assert len(sorted_tests["unexpected_improvements"]) == 2
|
||||
assert len(sorted_tests["fails"]) == 2
|
||||
assert len(sorted_tests["crashes"]) == 1
|
||||
assert len(sorted_tests["timeouts"]) == 1
|
||||
|
||||
assert sorted_tests["unexpected_improvements"] == [
|
||||
"Test1,UnexpectedImprovement",
|
||||
"Test6,UnexpectedImprovement",
|
||||
]
|
||||
assert sorted_tests["fails"] == ["Test2,Fail", "Test5,Fail"]
|
||||
assert sorted_tests["crashes"] == ["Test3,Crash"]
|
||||
assert sorted_tests["timeouts"] == ["Test4,Timeout"]
|
||||
|
||||
|
||||
def test_get_failed_test_summary_message():
|
||||
failed_test_array = {
|
||||
"unexpected_improvements": [
|
||||
"test1 UnexpectedImprovement",
|
||||
"test2 UnexpectedImprovement",
|
||||
],
|
||||
"fails": ["test3 Fail", "test4 Fail", "test5 Fail"],
|
||||
"crashes": ["test6 Crash"],
|
||||
"timeouts": [],
|
||||
}
|
||||
|
||||
summary_message = get_failed_test_summary_message(failed_test_array)
|
||||
|
||||
assert "<summary>" in summary_message
|
||||
assert "2 improved tests" in summary_message
|
||||
assert "3 failed tests" in summary_message
|
||||
assert "1 crashed test" in summary_message
|
||||
assert "</summary>" in summary_message
|
||||
|
||||
|
||||
def test_unexpected_improvements():
|
||||
message = "<summary>"
|
||||
failed_test_array = {
|
||||
"unexpected_improvements": ["test_improvement_1", "test_improvement_2"],
|
||||
"fails": [],
|
||||
"crashes": [],
|
||||
"timeouts": [],
|
||||
}
|
||||
result = unexpected_improvements(failed_test_array)
|
||||
assert result == " 2 improved tests", f"Unexpected result: {result}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("pipeline_message.get_pipeline_status", new_callable=AsyncMock)
|
||||
async def test_gitlab_api_failure(mock_get_pipeline_status):
|
||||
mock_get_pipeline_status.side_effect = Exception("GitLab API not responding")
|
||||
message = await main("1234567")
|
||||
assert message == ""
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_message_when_pipeline_not_failed():
|
||||
project_id = "176"
|
||||
pipeline_id = "12345"
|
||||
|
||||
with patch(
|
||||
"pipeline_message.get_pipeline_status", new_callable=AsyncMock
|
||||
) as mock_get_pipeline_status:
|
||||
mock_get_pipeline_status.return_value = "success"
|
||||
|
||||
message = await main(pipeline_id, project_id)
|
||||
assert (
|
||||
message == ""
|
||||
), f"Expected no message for successful pipeline, but got: {message}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_problem_job_not_summarized():
|
||||
session = AsyncMock()
|
||||
project_id = "176"
|
||||
problem_jobs = [
|
||||
{
|
||||
"id": 1234,
|
||||
"name": "test-job",
|
||||
"web_url": "http://example.com/job/1234",
|
||||
"status": "canceled",
|
||||
}
|
||||
]
|
||||
|
||||
mock_response = AsyncMock()
|
||||
mock_response.status = 200
|
||||
mock_response.text.return_value = "" # Empty CSV response for test
|
||||
session.get.return_value = mock_response
|
||||
|
||||
message = await process_problem_jobs(session, project_id, problem_jobs)
|
||||
|
||||
assert "summary" not in message
|
||||
assert "[test-job](http://example.com/job/1234)" in message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("pipeline_message.get_project_json", new_callable=AsyncMock)
|
||||
@patch("pipeline_message.aiohttp.ClientSession", autospec=True)
|
||||
async def test_get_trace_failures_no_response(
|
||||
mock_client_session_cls, mock_get_project_json, caplog
|
||||
):
|
||||
caplog.set_level(logging.DEBUG)
|
||||
namespace = "mesa"
|
||||
mock_get_project_json.return_value = {"path": namespace}
|
||||
|
||||
mock_get = AsyncMock()
|
||||
mock_get.status = 404
|
||||
|
||||
mock_session_instance = mock_client_session_cls.return_value
|
||||
mock_session_instance.get.return_value = mock_get
|
||||
|
||||
job_id = 12345678
|
||||
job = {"id": job_id}
|
||||
url = await get_trace_failures(mock_session_instance, "176", job)
|
||||
|
||||
assert url == ""
|
||||
|
||||
expected_log_message = f"No response from: https://mesa.pages.freedesktop.org/-/{namespace}/-/jobs/{job_id}/artifacts/results/summary/problems.html"
|
||||
assert any(expected_log_message in record.message for record in caplog.records)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("pipeline_message.get_job_log", new_callable=AsyncMock)
|
||||
async def test_search_job_log_for_errors(mock_get_job_log):
|
||||
session = AsyncMock()
|
||||
project_id = "176"
|
||||
job = {"id": 12345}
|
||||
|
||||
job_log = r"""
|
||||
error_msg: something useful
|
||||
[0m15:41:36.102: GL_KHR_no_error GL_KHR_texture_compression_astc_sliced_3d
|
||||
1 error generated
|
||||
3 errors generated.
|
||||
-- Looking for strerror_r - found
|
||||
-- Looking for strerror_s - not found
|
||||
[49/176] Building CXX object lib/Support/CMakeFiles/LLVMSupport.dir/ErrorHandling.cpp.o
|
||||
[127/2034] Building C object lib/Support/CMakeFiles/LLVMSupport.dir/regerror.c.o
|
||||
-- Performing Test HAS_WERROR_GLOBAL_CTORS
|
||||
-- Performing Test C_SUPPORTS_WERROR_UNGUARDED_AVAILABILITY_NEW - Success
|
||||
-- Performing Test LLVM_LIBSTDCXX_SOFT_ERROR
|
||||
error aborting
|
||||
error_msg : None
|
||||
error_type : Job
|
||||
[0Ksection_end:1734694783:job_data
|
||||
[0K
|
||||
[0m11:39:43.438: [1mFinished executing LAVA job in the attempt #3 [0m
|
||||
[0Ksection_end:1734694783:lava_submit
|
||||
[0K
|
||||
[0;31m[01:54] ERROR: lava_submit: ret code: 1 [0m
|
||||
|
||||
[0;31m[01:54] ERROR: unknown-section: ret code: 1 [0m
|
||||
section_end:1734694783:step_script
|
||||
[0Ksection_start:1734694783:after_script
|
||||
[0K[0K[36;1mRunning after_script[0;m[0;m
|
||||
[32;1mRunning after script...[0;m
|
||||
[32;1m$ curl -L --retry 4 -f --retry-all-errors --retry-delay 60 -s "https://" | tar --warning=no-timestamp --zstd -x[0;m
|
||||
zstd: /*stdin*\: unexpected end of file # noqa: W605
|
||||
tar: Child returned status 1
|
||||
tar: Error is not recoverable: exiting now
|
||||
section_end:1734695025:after_script
|
||||
[0K[0;33mWARNING: after_script failed, but job will continue unaffected: exit code 1[0;m
|
||||
section_start:1734695025:upload_artifacts_on_failure
|
||||
[0K[0K[36;1mUploading artifacts for failed job[0;m[0;m
|
||||
[32;1mUploading artifacts...[0;m
|
||||
results/: found 11 matching artifact files and directories[0;m
|
||||
Uploading artifacts as "archive" to coordinator... 201 Created[0;m id[0;m=68509685 responseStatus[0;m=201 Created token[0;m=glcbt-64
|
||||
[32;1mUploading artifacts...[0;m
|
||||
[0;33mWARNING: results/junit.xml: no matching files. Ensure that the artifact path is relative to the working directory (/builds/mesa/mesa)[0;m
|
||||
[31;1mERROR: No files to upload [0;m
|
||||
section_end:1734695027:upload_artifacts_on_failure
|
||||
[0Ksection_start:1734695027:cleanup_file_variables
|
||||
[0K[0K[36;1mCleaning up project directory and file based variables[0;m[0;m
|
||||
section_end:1734695027:cleanup_file_variables
|
||||
[0K[31;1mERROR: Job failed: exit code 1
|
||||
[0;m
|
||||
[0;m
|
||||
"""
|
||||
|
||||
mock_get_job_log.return_value = job_log
|
||||
|
||||
error_message = await search_job_log_for_errors(session, project_id, job)
|
||||
assert "something useful" in error_message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("pipeline_message.get_job_log", new_callable=AsyncMock)
|
||||
async def test_search_job_log_for_fatal_errors(mock_get_job_log):
|
||||
session = AsyncMock()
|
||||
project_id = "176"
|
||||
job = {"id": 12345}
|
||||
|
||||
job_log = r"""
|
||||
[0m15:41:36.105: [15:41:31.951] fatal: something fatal
|
||||
Uploading artifacts as "archive" to coordinator... 201 Created[0;m id[0;m=68509685 responseStatus[0;m=201 Created token[0;m=glcbt-64
|
||||
[32;1mUploading artifacts...[0;m
|
||||
[0;33mWARNING: results/junit.xml: no matching files. Ensure that the artifact path is relative to the working directory (/builds/mesa/mesa)[0;m
|
||||
[31;1mERROR: No files to upload [0;m
|
||||
section_end:1734695027:upload_artifacts_on_failure
|
||||
[0Ksection_start:1734695027:cleanup_file_variables
|
||||
[0K[0K[36;1mCleaning up project directory and file based variables[0;m[0;m
|
||||
section_end:1734695027:cleanup_file_variables
|
||||
[0K[31;1mERROR: Job failed: exit code 1
|
||||
[0;m
|
||||
[0;m
|
||||
"""
|
||||
|
||||
mock_get_job_log.return_value = job_log
|
||||
|
||||
error_message = await search_job_log_for_errors(session, project_id, job)
|
||||
assert "something fatal" in error_message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("pipeline_message.get_job_log", new_callable=AsyncMock)
|
||||
async def test_search_job_log_for_errors_but_find_none(mock_get_job_log):
|
||||
session = AsyncMock()
|
||||
project_id = "176"
|
||||
job = {"id": 12345}
|
||||
|
||||
job_log = r"""
|
||||
[0KRunning with gitlab-runner 17.4.0 (b92ee590)[0;m
|
||||
[0K on fdo-equinix-m3l-30-placeholder_63 XmDXAt7xd, system ID: s_785ae19292ea[0;m
|
||||
section_start:1734736110:prepare_executor
|
||||
[0K[0K[36;1mPreparing the "docker" executor[0;m[0;m
|
||||
[0KUsing Docker executor with image registry.freedesktop.org/mesa/mesa/debian
|
||||
[0KAuthenticating with credentials from job payload (GitLab Registry)[0;m
|
||||
[0KPulling docker image registry.freedesktop.org/mesa/mesa/debian/x86_64_pyuti
|
||||
[0KUsing docker image sha256:ebc7b3fe89be4d390775303adddb33539c235a2663165d78d
|
||||
[0Ksection_start:1734736124:prepare_script
|
||||
[0K[0K[36;1mPreparing environment[0;m[0;m
|
||||
Running on runner-xmdxat7xd-project-23076-concurrent-1 via fdo-equinix-m3l-30...
|
||||
section_end:1734736125:prepare_script
|
||||
[0Ksection_start:1734736125:get_sources
|
||||
[0K[0K[36;1mGetting source from Git repository[0;m[0;m
|
||||
[32;1m$ /host/bin/curl -s -L --cacert /host/ca-certificates.crt --retry 4 -f --retry-delay 60 https://gitlab.
|
||||
Checking if the user of the pipeline is allowed...
|
||||
Checking if the job's project is part of a well-known group...
|
||||
Checking if the job is part of an official MR pipeline...
|
||||
Thank you for contributing to freedesktop.org
|
||||
Running pre-clone script: 'set -o xtrace
|
||||
wget -q -O download-git-cache.sh https://gitlab.freedesktop.org/mesa/mesa/-/raw/0d43b4cba639b809ad0e08a065ce01846e262249/.gitlab-ci/download-git-cache.sh
|
||||
bash download-git-cache.sh
|
||||
rm download-git-cache.sh
|
||||
[31;1m errors
|
||||
[0K[31;1mERROR:
|
||||
[31;1m error
|
||||
[31;1m Here is a blank error:
|
||||
/builds/mesa/mesa/bin/ci/test/test_pipeline_message.py:162: AssertionError
|
||||
Uploading artifacts as "archive" to coordinator... 201 Created[0;m id[0;m=68509685 responseStatus[0;m=201 Created token[0;m=glcbt-64
|
||||
[32;1mUploading artifacts...[0;m
|
||||
[0;33mWARNING: results/junit.xml: no matching files. Ensure that the artifact path is relative to the working directory (/builds/mesa/mesa)[0;m
|
||||
[31;1mERROR: No files to upload [0;m
|
||||
section_end:1734695027:upload_artifacts_on_failure
|
||||
[0Ksection_start:1734695027:cleanup_file_variables
|
||||
[0K[0K[36;1mCleaning up project directory and file based variables[0;m[0;m
|
||||
section_end:1734695027:cleanup_file_variables
|
||||
[0K[31;1mERROR: Job failed: exit code 1
|
||||
[0;m
|
||||
[0;m
|
||||
"""
|
||||
|
||||
mock_get_job_log.return_value = job_log
|
||||
|
||||
error_message = await search_job_log_for_errors(session, project_id, job)
|
||||
assert error_message == "", f"Unexpected error message: {error_message}"
|
||||
@@ -0,0 +1,182 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from mock import MagicMock, patch
|
||||
from structured_logger import (
|
||||
AutoSaveDict,
|
||||
CSVStrategy,
|
||||
JSONStrategy,
|
||||
StructuredLogger,
|
||||
YAMLStrategy,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(params=[CSVStrategy, JSONStrategy, YAMLStrategy])
|
||||
def strategy(request):
|
||||
return request.param
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def file_extension(strategy):
|
||||
if strategy == CSVStrategy:
|
||||
return "csv"
|
||||
elif strategy == JSONStrategy:
|
||||
return "json"
|
||||
elif strategy == YAMLStrategy:
|
||||
return "yaml"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tmp_file(tmp_path):
|
||||
return tmp_path / "test.json"
|
||||
|
||||
|
||||
def test_guess_strategy_from_file(tmp_path, strategy, file_extension):
|
||||
file_name = tmp_path / f"test_guess.{file_extension}"
|
||||
Path(file_name).touch()
|
||||
guessed_strategy = StructuredLogger.guess_strategy_from_file(file_name)
|
||||
assert isinstance(guessed_strategy, strategy)
|
||||
|
||||
|
||||
def test_get_strategy(strategy, file_extension):
|
||||
result = StructuredLogger.get_strategy(file_extension)
|
||||
assert isinstance(result, strategy)
|
||||
|
||||
|
||||
def test_invalid_file_extension(tmp_path):
|
||||
file_name = tmp_path / "test_invalid.xyz"
|
||||
Path(file_name).touch()
|
||||
|
||||
with pytest.raises(ValueError, match="Unknown strategy for: xyz"):
|
||||
StructuredLogger.guess_strategy_from_file(file_name)
|
||||
|
||||
|
||||
def test_non_existent_file(tmp_path, strategy, file_extension):
|
||||
file_name = tmp_path / f"non_existent.{file_extension}"
|
||||
logger = StructuredLogger(file_name, strategy())
|
||||
|
||||
assert logger.file_path.exists()
|
||||
assert "_timestamp" in logger._data
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def structured_logger_module():
|
||||
with patch.dict("sys.modules", {"polars": None, "ruamel.yaml": None}):
|
||||
import importlib
|
||||
|
||||
import structured_logger
|
||||
|
||||
importlib.reload(structured_logger)
|
||||
yield structured_logger
|
||||
|
||||
|
||||
def test_missing_csv_library(tmp_path, structured_logger_module):
|
||||
with pytest.raises(RuntimeError, match="Can't parse CSV files. Missing library"):
|
||||
structured_logger_module.CSVStrategy()
|
||||
|
||||
|
||||
def test_missing_yaml_library(tmp_path, structured_logger_module):
|
||||
with pytest.raises(RuntimeError, match="Can't parse YAML files. Missing library"):
|
||||
structured_logger_module.YAMLStrategy()
|
||||
|
||||
|
||||
def test_autosavedict_setitem():
|
||||
save_callback = MagicMock()
|
||||
d = AutoSaveDict(save_callback=save_callback)
|
||||
d["key"] = "value"
|
||||
assert d["key"] == "value"
|
||||
save_callback.assert_called_once()
|
||||
|
||||
|
||||
def test_autosavedict_delitem():
|
||||
save_callback = MagicMock()
|
||||
d = AutoSaveDict({"key": "value"}, save_callback=save_callback)
|
||||
del d["key"]
|
||||
assert "key" not in d
|
||||
save_callback.assert_called_once()
|
||||
|
||||
|
||||
def test_autosavedict_pop():
|
||||
save_callback = MagicMock()
|
||||
d = AutoSaveDict({"key": "value"}, save_callback=save_callback)
|
||||
result = d.pop("key")
|
||||
assert result == "value"
|
||||
assert "key" not in d
|
||||
save_callback.assert_called_once()
|
||||
|
||||
|
||||
def test_autosavedict_update():
|
||||
save_callback = MagicMock()
|
||||
d = AutoSaveDict({"key": "old_value"}, save_callback=save_callback)
|
||||
d.update({"key": "new_value"})
|
||||
assert d["key"] == "new_value"
|
||||
save_callback.assert_called_once()
|
||||
|
||||
|
||||
def test_structured_logger_setitem(tmp_file):
|
||||
logger = StructuredLogger(tmp_file, JSONStrategy())
|
||||
logger.data["field"] = "value"
|
||||
|
||||
with open(tmp_file, "r") as f:
|
||||
data = json.load(f)
|
||||
|
||||
assert data["field"] == "value"
|
||||
|
||||
|
||||
def test_structured_logger_set_recursive(tmp_file):
|
||||
logger = StructuredLogger(tmp_file, JSONStrategy())
|
||||
logger.data["field"] = {"test": True}
|
||||
other = logger.data["field"]
|
||||
other["late"] = True
|
||||
|
||||
with open(tmp_file, "r") as f:
|
||||
data = json.load(f)
|
||||
|
||||
assert data["field"]["test"]
|
||||
assert data["field"]["late"]
|
||||
|
||||
|
||||
def test_structured_logger_set_list(tmp_file):
|
||||
logger = StructuredLogger(tmp_file, JSONStrategy())
|
||||
logger.data["field"] = [True]
|
||||
other = logger.data["field"]
|
||||
other.append(True)
|
||||
|
||||
with open(tmp_file, "r") as f:
|
||||
data = json.load(f)
|
||||
|
||||
assert data["field"][0]
|
||||
assert data["field"][1]
|
||||
|
||||
|
||||
def test_structured_logger_delitem(tmp_file):
|
||||
logger = StructuredLogger(tmp_file, JSONStrategy())
|
||||
logger.data["field"] = "value"
|
||||
del logger.data["field"]
|
||||
|
||||
with open(tmp_file, "r") as f:
|
||||
data = json.load(f)
|
||||
|
||||
assert "field" not in data
|
||||
|
||||
|
||||
def test_structured_logger_pop(tmp_file):
|
||||
logger = StructuredLogger(tmp_file, JSONStrategy())
|
||||
logger.data["field"] = "value"
|
||||
logger.data.pop("field")
|
||||
|
||||
with open(tmp_file, "r") as f:
|
||||
data = json.load(f)
|
||||
|
||||
assert "field" not in data
|
||||
|
||||
|
||||
def test_structured_logger_update(tmp_file):
|
||||
logger = StructuredLogger(tmp_file, JSONStrategy())
|
||||
logger.data.update({"field": "value"})
|
||||
|
||||
with open(tmp_file, "r") as f:
|
||||
data = json.load(f)
|
||||
|
||||
assert data["field"] == "value"
|
||||
@@ -0,0 +1,391 @@
|
||||
import sys
|
||||
import subprocess
|
||||
import yaml
|
||||
import pytest
|
||||
|
||||
from textwrap import dedent
|
||||
from unittest.mock import patch
|
||||
|
||||
from bin.ci.update_tag import (
|
||||
from_component_to_build_tag,
|
||||
filter_components,
|
||||
from_component_to_tag_var,
|
||||
from_script_name_to_component,
|
||||
from_script_name_to_tag_var,
|
||||
load_image_tags_yaml,
|
||||
update_image_tag_in_yaml,
|
||||
run_build_script,
|
||||
main,
|
||||
PATHS,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def temp_image_tags_file(tmp_path, monkeypatch):
|
||||
"""
|
||||
Fixture that creates a temporary YAML file (to simulate conditional_tags)
|
||||
and updates the global variable accordingly.
|
||||
"""
|
||||
# Create a temporary test file
|
||||
temp_file = tmp_path / "conditional-build-image-tags.yml"
|
||||
temp_file.write_text(yaml.dump({"variables": {}}))
|
||||
|
||||
# Create a copy of the original PATHS values
|
||||
original_conditional_tags = PATHS.conditional_tags
|
||||
|
||||
# Patch the PATHS instance with our test file
|
||||
PATHS.conditional_tags = temp_file
|
||||
|
||||
# Yield the test file for the test to use
|
||||
yield temp_file
|
||||
|
||||
# Restore the original value after the test
|
||||
PATHS.conditional_tags = original_conditional_tags
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def temp_container_ci_file(tmp_path, monkeypatch):
|
||||
"""
|
||||
Fixture that creates a temporary container CI file (to simulate container_ci)
|
||||
and updates the global variable accordingly.
|
||||
"""
|
||||
# Create a temporary test file
|
||||
temp_file = tmp_path / "container-ci.yml"
|
||||
temp_file.touch()
|
||||
|
||||
# Create a copy of the original PATHS values
|
||||
original_container_ci = PATHS.container_ci
|
||||
|
||||
# Patch the PATHS instance with our test file
|
||||
PATHS.container_ci = temp_file
|
||||
|
||||
# Yield the test file for the test to use
|
||||
yield temp_file
|
||||
|
||||
# Restore the original value after the test
|
||||
PATHS.container_ci = original_container_ci
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_container_dir(tmp_path, monkeypatch):
|
||||
"""
|
||||
Fixture that creates a dummy container directory and build script.
|
||||
"""
|
||||
# Create a dummy container directory
|
||||
container_dir = tmp_path / "container"
|
||||
container_dir.mkdir()
|
||||
|
||||
# Create a dummy setup-test-env.sh file
|
||||
dummy_setup_path = tmp_path / "setup-test-env.sh"
|
||||
dummy_setup_path.write_text("echo Setup")
|
||||
|
||||
# Save original values
|
||||
original_container_dir = PATHS.container_dir
|
||||
original_setup_test_env = PATHS.setup_test_env
|
||||
|
||||
# Patch the PATHS instance
|
||||
PATHS.container_dir = container_dir
|
||||
PATHS.setup_test_env = dummy_setup_path
|
||||
|
||||
# Yield the directory for the test to use
|
||||
yield container_dir
|
||||
|
||||
# Restore original values
|
||||
PATHS.container_dir = original_container_dir
|
||||
PATHS.setup_test_env = original_setup_test_env
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_build_script(mock_container_dir):
|
||||
"""
|
||||
Fixture that creates a dummy build script in the container directory.
|
||||
"""
|
||||
build_script = mock_container_dir / "build-foo.sh"
|
||||
return build_script
|
||||
|
||||
|
||||
###############################################################################
|
||||
# Tests for argument parsing and helper functions
|
||||
###############################################################################
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"component, expected_tag_var",
|
||||
[
|
||||
("foo", "FOO_TAG"),
|
||||
("my-component", "MY_COMPONENT_TAG"),
|
||||
("foo-bar-baz", "FOO_BAR_BAZ_TAG"),
|
||||
],
|
||||
)
|
||||
def test_from_component_to_tag_var(component, expected_tag_var):
|
||||
"""
|
||||
Test that from_component_to_tag_var returns the correct tag variable name.
|
||||
"""
|
||||
assert from_component_to_tag_var(component) == expected_tag_var
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"component, expected_build_tag",
|
||||
[
|
||||
("foo", "CONDITIONAL_BUILD_FOO_TAG"),
|
||||
("my-component", "CONDITIONAL_BUILD_MY_COMPONENT_TAG"),
|
||||
("foo-bar-baz", "CONDITIONAL_BUILD_FOO_BAR_BAZ_TAG"),
|
||||
],
|
||||
)
|
||||
def test_from_component_to_build_tag(component, expected_build_tag):
|
||||
"""
|
||||
Test that from_component_to_build_tag returns the correct build tag name.
|
||||
"""
|
||||
assert from_component_to_build_tag(component) == expected_build_tag
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"script_name, expected_component",
|
||||
[
|
||||
("build-foo.sh", "foo"),
|
||||
("build-my-component.sh", "my-component"),
|
||||
("build-foo-bar-baz.sh", "foo-bar-baz"),
|
||||
],
|
||||
)
|
||||
def test_from_script_name_to_component(script_name, expected_component):
|
||||
"""
|
||||
Test that from_script_name_to_component returns the correct component name.
|
||||
"""
|
||||
assert from_script_name_to_component(script_name) == expected_component
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"script_name, expected_tag_var",
|
||||
[
|
||||
("build-foo.sh", "FOO_TAG"),
|
||||
("build-my-component.sh", "MY_COMPONENT_TAG"),
|
||||
("build-foo-bar-baz.sh", "FOO_BAR_BAZ_TAG"),
|
||||
],
|
||||
)
|
||||
def test_from_script_name_to_tag_var(script_name, expected_tag_var):
|
||||
"""
|
||||
Test that from_script_name_to_tag_var returns the correct tag variable name.
|
||||
"""
|
||||
assert from_script_name_to_tag_var(script_name) == expected_tag_var
|
||||
|
||||
|
||||
def test_filter_components():
|
||||
"""
|
||||
Test that filter_components returns only components that match an include regex
|
||||
and do not match any exclude regex. If includes is empty, an empty list is returned.
|
||||
"""
|
||||
components = ["alpha", "beta", "gamma", "delta"]
|
||||
|
||||
# When includes is empty, should return an empty list.
|
||||
assert filter_components(components, [], []) == []
|
||||
|
||||
# Test includes only.
|
||||
# Components that start with 'a' or 'b'
|
||||
expected = ["alpha", "beta"]
|
||||
result = filter_components(components, ["^[a-b].*$"], [])
|
||||
assert result == expected
|
||||
|
||||
# Test with an exclude that filters out "alpha" (which matches "lph").
|
||||
result = filter_components(components, ["^[a-b].*$"], ["^.*lph.*$"])
|
||||
# "alpha" is removed.
|
||||
assert result == ["beta"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"component, tag",
|
||||
[
|
||||
# Basic case
|
||||
("test", "new_tag"),
|
||||
# Component with hyphen
|
||||
("my-component", "v1.2.3"),
|
||||
# Uppercase component name
|
||||
("UpperCase", "123"),
|
||||
# Numbers in component name
|
||||
("123service", "build-456"),
|
||||
],
|
||||
)
|
||||
def test_update_image_tag_in_yaml(component, tag, temp_image_tags_file):
|
||||
"""
|
||||
Test multiple updates with different component names and tags, verifying:
|
||||
1. Correct variable name generation
|
||||
2. Proper tag value storage
|
||||
3. Maintained sorting of variables
|
||||
"""
|
||||
# Initial update
|
||||
update_image_tag_in_yaml(component, tag)
|
||||
|
||||
data = load_image_tags_yaml()
|
||||
expected_var = from_component_to_build_tag(component)
|
||||
assert data["variables"][expected_var] == tag
|
||||
|
||||
# Add second component and verify both exist
|
||||
update_image_tag_in_yaml("another_component", "secondary_tag")
|
||||
updated_data = load_image_tags_yaml()
|
||||
|
||||
assert from_component_to_build_tag("another_component") in updated_data["variables"]
|
||||
assert updated_data["variables"][expected_var] == tag # Original value remains
|
||||
|
||||
# Verify sorting
|
||||
variables = list(data["variables"].keys())
|
||||
assert len(updated_data["variables"]) == len(variables) + 1
|
||||
assert variables == sorted(variables), "Variables are not alphabetically sorted"
|
||||
|
||||
|
||||
def test_if_run_extracts_the_tag_from_stdout(monkeypatch, mock_build_script):
|
||||
"""
|
||||
Test that run_build_script returns the tag (last stdout line) when the build
|
||||
script executes successfully.
|
||||
"""
|
||||
mock_build_script.write_text("#!/bin/bash\necho Build script\necho new_tag")
|
||||
# Create a fake subprocess.CompletedProcess to simulate a successful build.
|
||||
fake_result = subprocess.CompletedProcess(
|
||||
args=[], returncode=0, stdout="line1\nnew_tag", stderr=""
|
||||
)
|
||||
monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: fake_result)
|
||||
|
||||
tag = run_build_script("foo", check_only=False)
|
||||
assert tag == "new_tag"
|
||||
|
||||
|
||||
def test_running_real_process_works(monkeypatch, mock_build_script):
|
||||
"""
|
||||
Test that run_build_script returns the tag (last stdout line) when the build
|
||||
script executes successfully.
|
||||
"""
|
||||
mock_build_script.write_text("echo Build script\necho new_tag")
|
||||
tag = run_build_script("foo", check_only=False)
|
||||
assert tag == "new_tag"
|
||||
|
||||
|
||||
###############################################################################
|
||||
# Tests for main() argument features
|
||||
###############################################################################
|
||||
|
||||
|
||||
def setup_build_script_and_container_ci_file(container_dir, container_ci_file, comp):
|
||||
tag_var = from_component_to_build_tag(comp)
|
||||
build_check_var = from_component_to_tag_var(comp)
|
||||
build_script_path = container_dir / f"build-{comp}.sh"
|
||||
build_script_path.write_text(
|
||||
dedent(
|
||||
f"""
|
||||
#!/bin/bash
|
||||
ci_tag_build_time_check {tag_var}
|
||||
echo "new_tag"
|
||||
"""
|
||||
)
|
||||
)
|
||||
container_ci_file.write_text(
|
||||
container_ci_file.read_text()
|
||||
+ dedent(
|
||||
f"""
|
||||
.container-builds-{comp}:
|
||||
variables:
|
||||
{tag_var}: "${{{build_check_var}}}"
|
||||
"""
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_main_list(monkeypatch, capsys, mock_container_dir, temp_container_ci_file):
|
||||
"""
|
||||
Test that when the --list argument is provided, main() prints all detected
|
||||
components
|
||||
"""
|
||||
# Monkeypatch find_components to return a known list.
|
||||
for comp in ["comp-a", "comp-b"]:
|
||||
setup_build_script_and_container_ci_file(mock_container_dir, temp_container_ci_file, comp)
|
||||
# Set sys.argv to simulate passing --list.
|
||||
monkeypatch.setattr(sys, "argv", ["bin/ci/update_tag.py", "--list"])
|
||||
|
||||
main()
|
||||
|
||||
captured = capsys.readouterr().out
|
||||
assert "Detected components:" in captured
|
||||
assert "comp-a" in captured
|
||||
assert "comp-b" in captured
|
||||
|
||||
|
||||
def test_main_check(monkeypatch, temp_image_tags_file, temp_container_ci_file):
|
||||
"""
|
||||
Test that when --check is provided, main() calls run_build_script in check mode.
|
||||
"""
|
||||
|
||||
with patch(
|
||||
"bin.ci.update_tag.run_build_script", return_value="dummy_tag"
|
||||
) as mock_run_build_script:
|
||||
# Simulate command line: --check compX
|
||||
monkeypatch.setattr(sys, "argv", ["bin/ci/update_tag.py", "--check", "comp-x"])
|
||||
|
||||
main()
|
||||
# Verify run_build_script was called with check_only=True.
|
||||
assert mock_run_build_script.call_count == 1
|
||||
assert "comp-x" in mock_run_build_script.call_args.args[0]
|
||||
assert mock_run_build_script.call_args.kwargs["check_only"] is True
|
||||
|
||||
|
||||
EXIT_CODE_SCENARIOS = {
|
||||
"unbound_variable": (
|
||||
"line 2: UNDEFINED_VARIABLE: unbound variable",
|
||||
127,
|
||||
3,
|
||||
"Please set the variable UNDEFINED_VARIABLE",
|
||||
),
|
||||
"build script error": (
|
||||
"",
|
||||
50,
|
||||
50,
|
||||
"",
|
||||
),
|
||||
"tag_mismatch": (
|
||||
"Tag mismatch for foo.",
|
||||
2,
|
||||
2,
|
||||
"Tag mismatch for foo.",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"stderr_content, script_returncode, expected_exit_code, expected_error_message",
|
||||
EXIT_CODE_SCENARIOS.values(),
|
||||
ids=list(EXIT_CODE_SCENARIOS.keys()),
|
||||
)
|
||||
def test_build_script_error_exit_codes(
|
||||
monkeypatch,
|
||||
mock_build_script,
|
||||
capsys,
|
||||
stderr_content,
|
||||
script_returncode,
|
||||
expected_exit_code,
|
||||
expected_error_message,
|
||||
):
|
||||
"""
|
||||
Test that build script errors generate appropriate exit codes:
|
||||
1 - Unhandled error in this script
|
||||
2 - Tag mismatch when using --check
|
||||
3 - Unbound variable error in build script
|
||||
x - Build script failed with return code x
|
||||
"""
|
||||
# Create a build script that will fail
|
||||
mock_build_script.write_text("#!/bin/bash\necho 'This will fail'")
|
||||
|
||||
# Create a fake subprocess.CompletedProcess to simulate a failed build
|
||||
fake_result = subprocess.CompletedProcess(
|
||||
args=[], returncode=script_returncode, stdout="", stderr=stderr_content
|
||||
)
|
||||
monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: fake_result)
|
||||
monkeypatch.setattr(sys, "argv", ["bin/ci/update_tag.py", "--check", "foo"])
|
||||
monkeypatch.setattr("bin.ci.update_tag.get_current_tag_value", lambda *args: "current_tag")
|
||||
|
||||
# Mock sys.exit to capture the exit code instead of exiting the test
|
||||
with pytest.raises(SystemExit) as e:
|
||||
main()
|
||||
|
||||
# Check for expected error message if one is specified
|
||||
if expected_error_message:
|
||||
captured = capsys.readouterr()
|
||||
assert expected_error_message in captured.err
|
||||
|
||||
# Verify correct exit code
|
||||
assert e.value.code == expected_exit_code
|
||||
Executable
+447
@@ -0,0 +1,447 @@
|
||||
#!/usr/bin/env python3
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import argparse
|
||||
import subprocess
|
||||
import yaml
|
||||
from typing import Optional, Set, Any
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
# Very simple type alias for GitLab YAML data structure
|
||||
# It is composed by a dictionary of job names, each with a dictionary of fields
|
||||
# (e.g., script, stage, rules, etc.)
|
||||
YamlData = dict[str, dict[str, Any]]
|
||||
|
||||
# Dummy environment vars to make build scripts happy
|
||||
# To avoid set -u errors in build scripts
|
||||
DUMMY_ENV_VARS: dict[str, str] = {
|
||||
# setup-test-env.sh
|
||||
"CI_JOB_STARTED_AT": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||||
# build-deqp.sh
|
||||
"DEQP_API": "dummy",
|
||||
"DEQP_TARGET": "dummy",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProjectPaths:
|
||||
"""Manages all project-related paths"""
|
||||
|
||||
root: Path = field(default_factory=lambda: Path(), init=False)
|
||||
setup_test_env: Path = field(default_factory=lambda: Path(), init=False)
|
||||
conditional_tags: Path = field(default_factory=lambda: Path(), init=False)
|
||||
container_dir: Path = field(default_factory=lambda: Path(), init=False)
|
||||
container_ci: Path = field(default_factory=lambda: Path(), init=False)
|
||||
|
||||
def __post_init__(self):
|
||||
self.root = self.find_root()
|
||||
self.setup_test_env = self.root / ".gitlab-ci" / "setup-test-env.sh"
|
||||
self.conditional_tags = self.root / ".gitlab-ci" / "conditional-build-image-tags.yml"
|
||||
self.container_dir = self.root / ".gitlab-ci" / "container"
|
||||
self.container_ci = self.container_dir / "gitlab-ci-inc.yml"
|
||||
|
||||
def find_root(self) -> Path:
|
||||
"""Find git repository root or fallback to other methods"""
|
||||
try:
|
||||
logging.debug("Getting git repo root using git rev-parse")
|
||||
return Path(
|
||||
subprocess.check_output(
|
||||
["git", "rev-parse", "--show-toplevel"],
|
||||
text=True,
|
||||
stderr=subprocess.PIPE,
|
||||
).strip()
|
||||
)
|
||||
# FileNotFoundError is raised if git is not installed
|
||||
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||
# Fallback to CI_PROJECT_DIR or path-based method
|
||||
if ci_dir := os.environ.get("CI_PROJECT_DIR"):
|
||||
logging.debug(f"Getting git repo root using CI_PROJECT_DIR: {ci_dir}")
|
||||
return Path(ci_dir)
|
||||
|
||||
# Last resort, use the path to this script
|
||||
logging.debug("Getting git repo root using path to this script")
|
||||
return Path(__file__).resolve().parent.parent.parent
|
||||
|
||||
|
||||
PATHS: ProjectPaths = ProjectPaths()
|
||||
|
||||
|
||||
def from_component_to_build_tag(component: str) -> str:
|
||||
# e.g., "angle" -> "CONDITIONAL_BUILD_ANGLE_TAG"
|
||||
return "CONDITIONAL_BUILD_" + re.sub(r"-", "_", component.upper()) + "_TAG"
|
||||
|
||||
|
||||
def from_component_to_tag_var(component: str) -> str:
|
||||
# e.g., "angle" -> "ANGLE_TAG"
|
||||
return re.sub(r"-", "_", component.upper()) + "_TAG"
|
||||
|
||||
|
||||
def from_script_name_to_component(script_name: str) -> str:
|
||||
# e.g., "build-angle.sh" -> "angle"
|
||||
return re.sub(r"^build-([a-z0-9_-]+)\.sh$", r"\1", script_name)
|
||||
|
||||
|
||||
def from_script_name_to_tag_var(script_name: str) -> str:
|
||||
# e.g., "build-angle.sh" -> "ANGLE_TAG"
|
||||
return re.sub(r"^build-([a-z0-9_-]+)\.sh$", r"\1_TAG", script_name).replace("-", "_").upper()
|
||||
|
||||
|
||||
def prepare_setup_env_script() -> Path:
|
||||
"""
|
||||
Sets up dummy environment variables to mimic the script in the CI repo.
|
||||
Returns the path to the setup-test-env.sh script.
|
||||
"""
|
||||
if not PATHS.setup_test_env.exists():
|
||||
logging.error(".gitlab-ci/setup-test-env.sh not found. Exiting.")
|
||||
sys.exit(1)
|
||||
|
||||
# Dummy environment vars to mimic the script
|
||||
for key, value in DUMMY_ENV_VARS.items():
|
||||
os.environ[key] = value
|
||||
|
||||
os.environ["CI_PROJECT_DIR"] = str(PATHS.root)
|
||||
|
||||
return PATHS.setup_test_env
|
||||
|
||||
|
||||
def validate_build_script(script_filename: str) -> bool:
|
||||
"""
|
||||
Returns True if the build script for the given component uses the structured tag variable.
|
||||
"""
|
||||
build_script = PATHS.container_dir / script_filename
|
||||
with open(build_script, "r", encoding="utf-8") as f:
|
||||
script_content = f.read()
|
||||
|
||||
tag_var = from_script_name_to_tag_var(script_filename)
|
||||
if not re.search(tag_var, script_content, re.IGNORECASE):
|
||||
logging.debug(
|
||||
f"Skipping {build_script} because it doesn't use {tag_var}",
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def load_container_yaml() -> YamlData:
|
||||
if not PATHS.container_ci.is_file():
|
||||
logging.error(f"File not found: {PATHS.container_ci}")
|
||||
sys.exit(1)
|
||||
|
||||
# Ignore !reference and other custom GitLab tags, we just want to know the
|
||||
# job names and fields
|
||||
yaml.SafeLoader.add_multi_constructor("", lambda loader, suffix, node: None)
|
||||
with open(str(PATHS.container_ci), "r", encoding="utf-8") as f:
|
||||
data = yaml.load(f, Loader=yaml.SafeLoader)
|
||||
if not isinstance(data, dict):
|
||||
return {"variables": {}}
|
||||
return data
|
||||
|
||||
|
||||
def find_candidate_components() -> list[str]:
|
||||
"""
|
||||
1) Reads .gitlab-ci/container/gitlab-ci.yml to find component links:
|
||||
lines matching '.*.container-builds-<component>'
|
||||
2) Looks for matching build-<component>.sh in .gitlab-ci/container/
|
||||
3) Returns a sorted list of components in the intersection of these sets.
|
||||
"""
|
||||
container_yaml = load_container_yaml()
|
||||
# Extract patterns like `container-builds-foo` from job names
|
||||
candidates: Set[str] = set()
|
||||
for job_name in container_yaml:
|
||||
if match := re.search(r"\.container-builds-([a-z0-9_-]+)$", str(job_name)):
|
||||
candidates.add(match.group(1))
|
||||
|
||||
if not candidates:
|
||||
logging.error(
|
||||
f"No viable build components found in {PATHS.container_ci}. "
|
||||
"Please check the file for valid component names. "
|
||||
"They should be named like '.container-builds-<component>'."
|
||||
)
|
||||
return []
|
||||
|
||||
# Find build scripts named build-<component>.sh
|
||||
build_scripts: list[str] = []
|
||||
for path in PATHS.container_dir.glob("build-*.sh"):
|
||||
if validate_build_script(path.name):
|
||||
logging.info(f"Found build script: {path.name}")
|
||||
component = from_script_name_to_component(path.name)
|
||||
build_scripts.append(component)
|
||||
|
||||
# Return sorted intersection of components found in build scripts and candidates
|
||||
return sorted(candidates.intersection(build_scripts))
|
||||
|
||||
|
||||
def filter_components(components: list[str], includes: list[str], excludes: list[str]) -> list[str]:
|
||||
"""
|
||||
Returns components that match at least one `includes` regex and none of the `excludes` regex.
|
||||
If includes is empty, returns an empty list (unless user explicitly does --all or --include).
|
||||
"""
|
||||
if not includes:
|
||||
return []
|
||||
|
||||
filtered = []
|
||||
for comp in components:
|
||||
# Must match at least one "include"
|
||||
if not any(re.fullmatch(inc, comp) for inc in includes):
|
||||
logging.debug(f"Excluding {comp}, no matches in includes.")
|
||||
continue
|
||||
# Must not match any "exclude"
|
||||
if any(re.fullmatch(exc, comp) for exc in excludes):
|
||||
logging.debug(f"Excluding {comp}, matched exclude pattern.")
|
||||
continue
|
||||
filtered.append(comp)
|
||||
return filtered
|
||||
|
||||
|
||||
def run_build_script(component: str, check_only: bool = False) -> Optional[str]:
|
||||
"""
|
||||
Runs .gitlab-ci/container/build-<component>.sh to produce a new tag (last line of stdout).
|
||||
If check_only=True, we skip updates to the YAML (but do the build to see if it passes).
|
||||
Returns the extracted tag (string) on success, or None on failure.
|
||||
"""
|
||||
# 1) Set up environment
|
||||
setup_env_script = prepare_setup_env_script()
|
||||
|
||||
build_script = PATHS.container_dir / f"build-{component}.sh"
|
||||
if not build_script.is_file():
|
||||
logging.error(f"Build script not found for {component}: {build_script}")
|
||||
return None
|
||||
|
||||
# Tag var should appear in the script, e.g., ANGLE_TAG for 'angle'
|
||||
tag_var = from_component_to_tag_var(component)
|
||||
|
||||
# We set up environment for the child process
|
||||
child_env: dict[str, str] = {}
|
||||
child_env["NEW_TAG_DRY_RUN"] = "1"
|
||||
if check_only:
|
||||
# For checking only
|
||||
child_env.pop("NEW_TAG_DRY_RUN", None)
|
||||
child_env["CI_NOT_BUILDING_ANYTHING"] = "1"
|
||||
if tag_value := get_current_tag_value(component):
|
||||
child_env[tag_var] = tag_value
|
||||
else:
|
||||
logging.error(f"No current tag value for {component}")
|
||||
return None
|
||||
|
||||
logging.debug(
|
||||
f"Running build for {component} with "
|
||||
f"{tag_var}={child_env.get(tag_var)} "
|
||||
f"(check_only={check_only})"
|
||||
)
|
||||
|
||||
# Run the build script
|
||||
result = subprocess.run(
|
||||
["bash", "-c", f"source {setup_env_script} && source {build_script}"],
|
||||
env=os.environ | child_env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
logging.debug(f"{' '.join(result.args)}")
|
||||
|
||||
# Tag check succeeded
|
||||
if result.returncode == 0:
|
||||
# Tag is assumed to be the last line of stdout
|
||||
lines = result.stdout.strip().splitlines()
|
||||
return lines[-1] if lines else ""
|
||||
|
||||
# Tag check failed, let's dissect the error
|
||||
|
||||
if result.returncode == 2:
|
||||
logging.error(f"Tag mismatch for {component}.")
|
||||
logging.error(result.stdout)
|
||||
return None
|
||||
|
||||
# Check if there's an unbound variable error
|
||||
err_output = result.stderr
|
||||
unbound_match = re.search(r"([A-Z_]+)(?=: unbound variable)", err_output)
|
||||
if unbound_match:
|
||||
var_name = unbound_match.group(1)
|
||||
logging.fatal(f"Please set the variable {var_name} in {build_script}.")
|
||||
sys.exit(3)
|
||||
|
||||
# Unexpected error in the build script, propagate the exit code
|
||||
logging.fatal(f"Build script for {component} failed with return code {result.returncode}")
|
||||
logging.error(result.stdout)
|
||||
sys.exit(result.returncode)
|
||||
|
||||
|
||||
def load_image_tags_yaml() -> YamlData:
|
||||
if not PATHS.conditional_tags.is_file():
|
||||
raise FileNotFoundError(
|
||||
f"Conditional build image tags file not found: {PATHS.conditional_tags}"
|
||||
)
|
||||
|
||||
with open(str(PATHS.conditional_tags), "r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f)
|
||||
if not isinstance(data, dict):
|
||||
return {"variables": {}}
|
||||
if "variables" not in data:
|
||||
data["variables"] = {}
|
||||
return data
|
||||
|
||||
|
||||
def get_current_tag_value(component: str) -> Optional[str]:
|
||||
# If the CI job already set the tag, use it
|
||||
tag_var = from_component_to_tag_var(component)
|
||||
if os.getenv(tag_var):
|
||||
logging.debug(f"Using tag from environment variable {tag_var}")
|
||||
return os.getenv(tag_var)
|
||||
|
||||
# Otherwise, look in the YAML file, which normally occurs when updating tags locally
|
||||
full_tag_var = from_component_to_build_tag(component)
|
||||
data = load_image_tags_yaml()
|
||||
variables = data.get("variables", {})
|
||||
if not isinstance(variables, dict):
|
||||
return None
|
||||
logging.debug(f"Using tag from YAML file {full_tag_var}")
|
||||
return variables.get(full_tag_var)
|
||||
|
||||
|
||||
def update_image_tag_in_yaml(component: str, tag_value: str) -> None:
|
||||
"""
|
||||
Uses PyYAML to edit the YAML file at IMAGE_TAGS_FILE, setting the environment variable
|
||||
for the given component. Maintains sorted keys.
|
||||
"""
|
||||
full_tag_var = from_component_to_build_tag(component)
|
||||
data = load_image_tags_yaml()
|
||||
|
||||
# Ensure we have a variables dictionary
|
||||
if "variables" not in data:
|
||||
data["variables"] = {}
|
||||
elif not isinstance(data["variables"], dict):
|
||||
data["variables"] = {}
|
||||
|
||||
# Update the tag
|
||||
data["variables"][full_tag_var] = tag_value
|
||||
|
||||
# Sort the variables
|
||||
data["variables"] = dict(sorted(data["variables"].items()))
|
||||
|
||||
# Write back to file
|
||||
with open(str(PATHS.conditional_tags), "w", encoding="utf-8") as f:
|
||||
yaml.dump(data, f, sort_keys=False) # Don't sort top-level keys
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
description="Manage container image tags for CI builds with regex-based includes/excludes.",
|
||||
epilog="""
|
||||
Exit codes:
|
||||
0 - Success
|
||||
1 - Unhandled error in this script
|
||||
2 - Tag mismatch when using --check
|
||||
3 - Unbound variable error in build script
|
||||
x - Build script failed with return code x
|
||||
""",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--include",
|
||||
"-i",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Full match regex pattern for components to include.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--exclude",
|
||||
"-x",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Full match regex pattern for components to exclude.",
|
||||
)
|
||||
parser.add_argument("--all", action="store_true", help="Equivalent to --include '.*'")
|
||||
parser.add_argument(
|
||||
"--check",
|
||||
"-c",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Check matching components instead of updating YAML. "
|
||||
"If any component fails, exit with a non-zero exit code.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--list",
|
||||
"-l",
|
||||
action="store_true",
|
||||
help="List all available components and exit.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-v",
|
||||
"--verbose",
|
||||
action="count",
|
||||
default=0,
|
||||
help="Increase verbosity level (-v for info, -vv for debug)",
|
||||
)
|
||||
|
||||
if len(sys.argv) == 1:
|
||||
parser.print_help()
|
||||
sys.exit(0)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
|
||||
# Configure logging based on verbosity level
|
||||
if args.verbose == 1:
|
||||
log_level = logging.INFO
|
||||
elif args.verbose == 2:
|
||||
log_level = logging.DEBUG
|
||||
else:
|
||||
log_level = logging.WARNING
|
||||
|
||||
logging.basicConfig(level=log_level, format="%(levelname)s: %(message)s")
|
||||
|
||||
# 1) If checking, just run build scripts in check mode and propagate errors
|
||||
if args.check:
|
||||
tag_mismatch = False
|
||||
for comp in args.check:
|
||||
try:
|
||||
if run_build_script(comp, check_only=True) is None:
|
||||
# The tag is invalid
|
||||
tag_mismatch = True
|
||||
except SystemExit as e:
|
||||
# Let custom exit codes propagate
|
||||
raise e
|
||||
except Exception as e:
|
||||
logging.error(f"Internal error: {e}")
|
||||
sys.exit(1)
|
||||
# If any component failed, exit with code 1
|
||||
if tag_mismatch:
|
||||
sys.exit(2)
|
||||
return
|
||||
|
||||
# Convert --all into a wildcard include
|
||||
if args.all:
|
||||
args.include.append(".*")
|
||||
|
||||
# 2) If --list, just show all discovered components
|
||||
all_components = find_candidate_components()
|
||||
if args.list:
|
||||
print("Detected components:", ", ".join(all_components))
|
||||
return
|
||||
|
||||
# 3) Filter components
|
||||
final_components = filter_components(all_components, args.include, args.exclude)
|
||||
|
||||
if args.verbose:
|
||||
logging.debug(f"Found components: {all_components}")
|
||||
logging.debug(f"Filtered components: {final_components}")
|
||||
|
||||
for comp in final_components:
|
||||
logging.info(f"Updating {comp}...")
|
||||
new_tag = run_build_script(comp, check_only=False)
|
||||
if new_tag is not None:
|
||||
update_image_tag_in_yaml(comp, new_tag)
|
||||
if args.verbose:
|
||||
logging.debug(f"Updated {comp} with tag: {new_tag}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+217
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright © 2022 Collabora Ltd.
|
||||
# Authors:
|
||||
# David Heidelberg <[email protected]>
|
||||
#
|
||||
# For the dependencies, see the requirements.txt
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
"""
|
||||
Helper script to update traces checksums
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import bz2
|
||||
import glob
|
||||
import re
|
||||
import json
|
||||
import sys
|
||||
from ruamel.yaml import YAML
|
||||
from subprocess import check_output
|
||||
from tomledit import Document
|
||||
|
||||
import gitlab
|
||||
from gitlab_common import (get_gitlab_project, read_token, wait_for_pipeline,
|
||||
get_gitlab_pipeline_from_url, TOKEN_DIR, get_token_from_default_dir)
|
||||
from rich import print
|
||||
|
||||
|
||||
DESCRIPTION_FILE = "export PIGLIT_REPLAY_DESCRIPTION_FILE=.*/install/(.*)$"
|
||||
DEVICE_NAME = "(?:declare -x|export) PIGLIT_REPLAY_DEVICE_NAME='?([^']*)'?$"
|
||||
|
||||
def gather_job_results_yml(cur_job):
|
||||
log: list[str] = cur_job.trace().decode().splitlines()
|
||||
filename: str = ''
|
||||
dev_name: str = ''
|
||||
for logline in log:
|
||||
desc_file = re.search(DESCRIPTION_FILE, logline)
|
||||
device_name = re.search(DEVICE_NAME, logline)
|
||||
if desc_file:
|
||||
filename = desc_file.group(1)
|
||||
if device_name:
|
||||
dev_name = device_name.group(1)
|
||||
|
||||
if not filename or not dev_name:
|
||||
print("[red]Couldn't find device name or YML file in the logs!")
|
||||
return
|
||||
|
||||
print(f"👁 Found {dev_name} and file {filename}")
|
||||
|
||||
# find filename in Mesa source
|
||||
traces_file = glob.glob('./**/' + filename, recursive=True)
|
||||
# write into it
|
||||
with open(traces_file[0], 'r', encoding='utf-8') as target_file:
|
||||
yaml = YAML()
|
||||
yaml.compact(seq_seq=False, seq_map=False)
|
||||
yaml.version = 1,2
|
||||
yaml.width = 2048 # do not break the text fields
|
||||
yaml.default_flow_style = None
|
||||
target = yaml.load(target_file)
|
||||
|
||||
# parse artifact
|
||||
results_json_bz2 = cur_job.artifact("results/results.json.bz2")
|
||||
results_json = bz2.decompress(results_json_bz2).decode("utf-8", errors="replace")
|
||||
results = json.loads(results_json)
|
||||
|
||||
for _, value in results["tests"].items():
|
||||
if (
|
||||
not value['images'] or
|
||||
not value['images'][0] or
|
||||
"image_desc" not in value['images'][0]
|
||||
):
|
||||
continue
|
||||
|
||||
trace: str = value['images'][0]['image_desc']
|
||||
checksum: str = value['images'][0]['checksum_render']
|
||||
|
||||
if not checksum:
|
||||
print(f"[red]{dev_name}: {trace}: checksum is missing! Crash?")
|
||||
continue
|
||||
|
||||
if checksum == "error":
|
||||
print(f"[red]{dev_name}: {trace}: crashed")
|
||||
continue
|
||||
|
||||
if target['traces'][trace][dev_name].get('checksum') == checksum:
|
||||
continue
|
||||
|
||||
if "label" in target['traces'][trace][dev_name]:
|
||||
print(
|
||||
f"{dev_name}: {trace}: please verify that label "
|
||||
f"[blue]{target['traces'][trace][dev_name]['label']}[/blue] "
|
||||
"is still valid"
|
||||
)
|
||||
|
||||
print(f"[green]{dev_name}: {trace}: checksum updated")
|
||||
target['traces'][trace][dev_name]['checksum'] = checksum
|
||||
|
||||
with open(traces_file[0], 'w', encoding='utf-8') as target_file:
|
||||
yaml.dump(target, target_file)
|
||||
|
||||
|
||||
def update_toml_checksum(toml_file, trace_path, device, old_checksum, new_checksum):
|
||||
with open(toml_file, 'r', encoding='utf-8') as f:
|
||||
doc = Document.parse(f.read())
|
||||
|
||||
for table in doc["traces"]:
|
||||
if table["path"].value != trace_path:
|
||||
continue
|
||||
try:
|
||||
table["devices"][device]["checksum"] = new_checksum
|
||||
except KeyError:
|
||||
return False
|
||||
with open(toml_file, 'w', encoding='utf-8') as f:
|
||||
f.write(doc.as_toml())
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def gather_job_results_toml(cur_job) -> None:
|
||||
"""Process checksum-changes.json artifact (gpu-trace-perf replay format)"""
|
||||
toml_files = glob.glob('./**/traces-*.toml', recursive=True)
|
||||
|
||||
changes_json = cur_job.artifact("results/checksum-changes.json")
|
||||
changes = json.loads(changes_json.decode("utf-8"))
|
||||
|
||||
for change in changes:
|
||||
device = change['device']
|
||||
trace = change['trace']
|
||||
old_checksum = change['old_checksum']
|
||||
new_checksum = change['new_checksum']
|
||||
|
||||
if not new_checksum:
|
||||
print(f"[red]{device}: {trace}: no new checksum (crash?)")
|
||||
continue
|
||||
|
||||
updated = False
|
||||
for toml_file in toml_files:
|
||||
if update_toml_checksum(toml_file, trace, device, old_checksum, new_checksum):
|
||||
print(f"[green]{device}: {trace}: checksum updated")
|
||||
updated = True
|
||||
break
|
||||
|
||||
if not updated:
|
||||
print(f"[red]{device}: {trace}: not found in any traces TOML file")
|
||||
|
||||
|
||||
def gather_results(
|
||||
project,
|
||||
pipeline,
|
||||
) -> None:
|
||||
"""Gather results"""
|
||||
|
||||
target_jobs_regex = re.compile(".*-traces(-restricted)?([:].*)?$")
|
||||
|
||||
for job in pipeline.jobs.list(all=True, sort="desc"):
|
||||
if target_jobs_regex.match(job.name) and job.status == "failed":
|
||||
cur_job = project.jobs.get(job.id)
|
||||
# get variables
|
||||
print(f"👁 {job.name}...")
|
||||
try:
|
||||
gather_job_results_toml(cur_job)
|
||||
except Exception:
|
||||
gather_job_results_yml(cur_job)
|
||||
|
||||
|
||||
def parse_args() -> None:
|
||||
"""Parse args"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Tool to generate patch from checksums ",
|
||||
epilog="Example: update_traces_checksum.py --rev $(git rev-parse HEAD) "
|
||||
)
|
||||
parser.add_argument(
|
||||
"--rev", metavar="revision", help="repository git revision", default='HEAD'
|
||||
)
|
||||
parser.add_argument(
|
||||
"--token",
|
||||
metavar="token",
|
||||
type=str,
|
||||
default=get_token_from_default_dir(),
|
||||
help="Use the provided GitLab token or token file, "
|
||||
f"otherwise it's read from {TOKEN_DIR / 'gitlab-token'}",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pipeline-url",
|
||||
metavar="pipeline_url",
|
||||
help="specify a pipeline url",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
args = parse_args()
|
||||
|
||||
token = read_token(args.token)
|
||||
|
||||
gl = gitlab.Gitlab(url="https://gitlab.freedesktop.org", private_token=token)
|
||||
|
||||
cur_project = get_gitlab_project(gl, "mesa")
|
||||
|
||||
if args.pipeline_url:
|
||||
pipe, cur_project = get_gitlab_pipeline_from_url(gl, args.pipeline_url)
|
||||
REV = pipe.sha
|
||||
else:
|
||||
if not args.rev:
|
||||
print('error: the following arguments are required: --rev')
|
||||
sys.exit(1)
|
||||
sha = check_output(['git', 'rev-parse', args.rev]).decode('ascii').strip()
|
||||
print(f"Revision: {sha}")
|
||||
(pipe, cur_project) = wait_for_pipeline([cur_project], sha)
|
||||
print(f"Pipeline: {pipe.web_url}")
|
||||
gather_results(cur_project, pipe)
|
||||
|
||||
sys.exit()
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(1)
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
|
||||
this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
|
||||
readonly this_dir
|
||||
|
||||
exec \
|
||||
"$this_dir/../python-venv.sh" \
|
||||
"$this_dir/requirements.txt" \
|
||||
"$this_dir/update_traces_checksum.py" "$@"
|
||||
Executable
+141
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def print_(args: argparse.Namespace, success: bool, message: str) -> None:
|
||||
"""
|
||||
Print function with extra coloring when supported and/or requested,
|
||||
and with a "quiet" switch
|
||||
"""
|
||||
|
||||
COLOR_SUCCESS = '\033[32m'
|
||||
COLOR_FAILURE = '\033[31m'
|
||||
COLOR_RESET = '\033[0m'
|
||||
|
||||
if args.quiet:
|
||||
return
|
||||
|
||||
if args.color == 'auto':
|
||||
use_colors = sys.stdout.isatty()
|
||||
else:
|
||||
use_colors = args.color == 'always'
|
||||
|
||||
s = ''
|
||||
if use_colors:
|
||||
if success:
|
||||
s += COLOR_SUCCESS
|
||||
else:
|
||||
s += COLOR_FAILURE
|
||||
|
||||
s += message
|
||||
|
||||
if use_colors:
|
||||
s += COLOR_RESET
|
||||
|
||||
print(s)
|
||||
|
||||
|
||||
def is_commit_valid(commit: str) -> bool:
|
||||
ret = subprocess.call(['git', 'cat-file', '-e', commit],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL)
|
||||
return ret == 0
|
||||
|
||||
|
||||
def branch_has_commit(upstream_branch: str, commit: str) -> bool:
|
||||
"""
|
||||
Returns True if the commit is actually present in the branch
|
||||
"""
|
||||
ret = subprocess.call(['git', 'merge-base', '--is-ancestor',
|
||||
commit, upstream_branch],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL)
|
||||
return ret == 0
|
||||
|
||||
|
||||
def branch_has_backport_of_commit(upstream_branch: str, commit: str) -> str:
|
||||
"""
|
||||
Returns the commit hash if the commit has been backported to the branch,
|
||||
or an empty string if is hasn't
|
||||
"""
|
||||
upstream, _ = upstream_branch.split('/', 1)
|
||||
|
||||
out = subprocess.check_output(['git', 'log', '--format=%H',
|
||||
upstream + '..' + upstream_branch,
|
||||
'--grep', 'cherry picked from commit ' + commit],
|
||||
stderr=subprocess.DEVNULL)
|
||||
return out.decode().strip()
|
||||
|
||||
|
||||
def canonicalize_commit(commit: str) -> str:
|
||||
"""
|
||||
Takes a commit-ish and returns a commit sha1 if the commit exists
|
||||
"""
|
||||
|
||||
# Make sure input is valid first
|
||||
if not is_commit_valid(commit):
|
||||
raise argparse.ArgumentTypeError('invalid commit identifier: ' + commit)
|
||||
|
||||
out = subprocess.check_output(['git', 'rev-parse', commit],
|
||||
stderr=subprocess.DEVNULL)
|
||||
return out.decode().strip()
|
||||
|
||||
|
||||
def validate_branch(branch: str) -> str:
|
||||
if '/' not in branch:
|
||||
raise argparse.ArgumentTypeError('must be in the form `remote/branch`')
|
||||
|
||||
out = subprocess.check_output(['git', 'remote', '--verbose'],
|
||||
stderr=subprocess.DEVNULL)
|
||||
remotes = out.decode().splitlines()
|
||||
upstream, _ = branch.split('/', 1)
|
||||
valid_remote = False
|
||||
for line in remotes:
|
||||
if line.startswith(upstream + '\t'):
|
||||
valid_remote = True
|
||||
|
||||
if not valid_remote:
|
||||
raise argparse.ArgumentTypeError('Invalid remote: ' + upstream)
|
||||
|
||||
if not is_commit_valid(branch):
|
||||
raise argparse.ArgumentTypeError('Invalid branch: ' + branch)
|
||||
|
||||
return branch
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="""
|
||||
Returns 0 if the commit is present in the branch,
|
||||
1 if it's not,
|
||||
and 2 if it couldn't be determined (eg. invalid commit)
|
||||
""")
|
||||
parser.add_argument('commit',
|
||||
type=canonicalize_commit,
|
||||
help='commit sha1')
|
||||
parser.add_argument('branch',
|
||||
type=validate_branch,
|
||||
help='branch to check, in the form `remote/branch`')
|
||||
parser.add_argument('--quiet',
|
||||
action='store_true',
|
||||
help='suppress all output; exit code can still be used')
|
||||
parser.add_argument('--color',
|
||||
choices=['auto', 'always', 'never'],
|
||||
default='auto',
|
||||
help='colorize output (default: true if stdout is a terminal)')
|
||||
args = parser.parse_args()
|
||||
|
||||
if branch_has_commit(args.branch, args.commit):
|
||||
print_(args, True, 'Commit ' + args.commit + ' is in branch ' + args.branch)
|
||||
exit(0)
|
||||
|
||||
backport = branch_has_backport_of_commit(args.branch, args.commit)
|
||||
if backport:
|
||||
print_(args, True,
|
||||
'Commit ' + args.commit + ' was backported to branch ' + args.branch + ' as commit ' + backport)
|
||||
exit(0)
|
||||
|
||||
print_(args, False, 'Commit ' + args.commit + ' is NOT in branch ' + args.branch)
|
||||
exit(1)
|
||||
@@ -0,0 +1,118 @@
|
||||
import argparse
|
||||
import pytest # type: ignore
|
||||
import subprocess
|
||||
|
||||
from .commit_in_branch import (
|
||||
is_commit_valid,
|
||||
branch_has_commit,
|
||||
branch_has_backport_of_commit,
|
||||
canonicalize_commit,
|
||||
validate_branch,
|
||||
)
|
||||
|
||||
|
||||
def get_upstream() -> str:
|
||||
# Let's assume main is bound to the upstream remote and not a fork
|
||||
out = subprocess.check_output(['git', 'for-each-ref',
|
||||
'--format=%(upstream)',
|
||||
'refs/heads/main'],
|
||||
stderr=subprocess.DEVNULL)
|
||||
return out.decode().strip().split('/')[2]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'commit, expected',
|
||||
[
|
||||
('20.1-branchpoint', True),
|
||||
('main', True),
|
||||
('e58a10af640ba58b6001f5c5ad750b782547da76', True),
|
||||
('d043d24654c851f0be57dbbf48274b5373dea42b', True),
|
||||
('dd2bd68fa69124c86cd008b256d06f44fab8e6cd', True),
|
||||
('0000000000000000000000000000000000000000', False),
|
||||
('not-even-a-valid-commit-format', False),
|
||||
])
|
||||
def test_canonicalize_commit(commit: str, expected: bool) -> None:
|
||||
if expected:
|
||||
assert canonicalize_commit(commit)
|
||||
else:
|
||||
try:
|
||||
assert canonicalize_commit(commit)
|
||||
except argparse.ArgumentTypeError:
|
||||
return
|
||||
assert False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'commit, expected',
|
||||
[
|
||||
(get_upstream() + '/20.1', True),
|
||||
(get_upstream() + '/staging/20.1', True),
|
||||
(get_upstream() + '/main', True),
|
||||
('20.1', False),
|
||||
('main', False),
|
||||
('e58a10af640ba58b6001f5c5ad750b782547da76', False),
|
||||
('d043d24654c851f0be57dbbf48274b5373dea42b', False),
|
||||
('dd2bd68fa69124c86cd008b256d06f44fab8e6cd', False),
|
||||
('0000000000000000000000000000000000000000', False),
|
||||
('not-even-a-valid-commit-format', False),
|
||||
])
|
||||
def test_validate_branch(commit: str, expected: bool) -> None:
|
||||
if expected:
|
||||
assert validate_branch(commit)
|
||||
else:
|
||||
try:
|
||||
assert validate_branch(commit)
|
||||
except argparse.ArgumentTypeError:
|
||||
return
|
||||
assert False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'commit, expected',
|
||||
[
|
||||
('main', True),
|
||||
('20.1-branchpoint', True),
|
||||
('20.1', False),
|
||||
(get_upstream() + '/20.1', True),
|
||||
(get_upstream() + '/staging/20.1', True),
|
||||
('e58a10af640ba58b6001f5c5ad750b782547da76', True),
|
||||
('d043d24654c851f0be57dbbf48274b5373dea42b', True),
|
||||
('dd2bd68fa69124c86cd008b256d06f44fab8e6cd', True),
|
||||
('0000000000000000000000000000000000000000', False),
|
||||
('not-even-a-valid-commit-format', False),
|
||||
])
|
||||
def test_is_commit_valid(commit: str, expected: bool) -> None:
|
||||
assert is_commit_valid(commit) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'branch, commit, expected',
|
||||
[
|
||||
(get_upstream() + '/20.1', '20.1-branchpoint', True),
|
||||
(get_upstream() + '/20.1', '20.0', False),
|
||||
(get_upstream() + '/20.1', 'main', False),
|
||||
(get_upstream() + '/20.1', 'e58a10af640ba58b6001f5c5ad750b782547da76', True),
|
||||
(get_upstream() + '/20.1', 'd043d24654c851f0be57dbbf48274b5373dea42b', True),
|
||||
(get_upstream() + '/staging/20.1', 'd043d24654c851f0be57dbbf48274b5373dea42b', True),
|
||||
(get_upstream() + '/20.1', 'dd2bd68fa69124c86cd008b256d06f44fab8e6cd', False),
|
||||
(get_upstream() + '/main', 'dd2bd68fa69124c86cd008b256d06f44fab8e6cd', True),
|
||||
(get_upstream() + '/20.0', 'd043d24654c851f0be57dbbf48274b5373dea42b', False),
|
||||
])
|
||||
def test_branch_has_commit(branch: str, commit: str, expected: bool) -> None:
|
||||
assert branch_has_commit(branch, commit) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'branch, commit, expected',
|
||||
[
|
||||
(get_upstream() + '/20.1', 'dd2bd68fa69124c86cd008b256d06f44fab8e6cd', 'd043d24654c851f0be57dbbf48274b5373dea42b'),
|
||||
(get_upstream() + '/staging/20.1', 'dd2bd68fa69124c86cd008b256d06f44fab8e6cd', 'd043d24654c851f0be57dbbf48274b5373dea42b'),
|
||||
(get_upstream() + '/20.1', '20.1-branchpoint', ''),
|
||||
(get_upstream() + '/20.1', '20.0', ''),
|
||||
(get_upstream() + '/20.1', '20.2', 'abac4859618e02aea00f705b841a7c5c5007ad1a'),
|
||||
(get_upstream() + '/20.1', 'main', ''),
|
||||
(get_upstream() + '/20.1', 'd043d24654c851f0be57dbbf48274b5373dea42b', ''),
|
||||
(get_upstream() + '/20.0', 'dd2bd68fa69124c86cd008b256d06f44fab8e6cd', '8cd4f57381cefe69019a3282d457d5bda3644030'),
|
||||
])
|
||||
def test_branch_has_backport_of_commit(branch: str, commit: str, expected: bool) -> None:
|
||||
assert branch_has_backport_of_commit(branch, commit) == expected
|
||||
@@ -0,0 +1,142 @@
|
||||
#
|
||||
# Copyright 2024 Autodesk, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
#
|
||||
|
||||
import argparse
|
||||
from bisect import bisect_left, bisect_right
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
|
||||
@dataclass
|
||||
class Instruction:
|
||||
address: int
|
||||
assembly: str
|
||||
samples: int = 0
|
||||
|
||||
|
||||
def mapping_address_key(mapping: tuple[int, int, str]):
|
||||
return mapping[0]
|
||||
|
||||
|
||||
def instruction_address_key(instruction: Instruction):
|
||||
return instruction.address
|
||||
|
||||
|
||||
def parse_mappings(map_file_path: Path):
|
||||
mappings: list[tuple[int, int, str]] = []
|
||||
with open(map_file_path) as map_file:
|
||||
for mapping in map_file:
|
||||
address_hex, size_hex, name = mapping.split(' ')
|
||||
address = int(address_hex, base=16)
|
||||
mappings.append((address, address + int(size_hex, base=16), name.strip()))
|
||||
|
||||
mappings.sort(key=mapping_address_key)
|
||||
return mappings
|
||||
|
||||
|
||||
def parse_traces(trace_file_path: Path):
|
||||
pattern = re.compile(r'((?:[^;]+;)*?[^;]+) (\d+)\n')
|
||||
|
||||
traces: list[tuple[list[str], int]] = []
|
||||
with open(trace_file_path) as trace_file:
|
||||
for trace in trace_file:
|
||||
match = pattern.fullmatch(trace)
|
||||
traces.append((match.group(1).split(';'), int(match.group(2))))
|
||||
|
||||
return traces
|
||||
|
||||
|
||||
def parse_asm(asm_file_path: Path):
|
||||
symbol_pattern = re.compile(r'(\w+) ([0-9a-fA-F]+):\n')
|
||||
instruction_pattern = re.compile(r' *([0-9a-fA-F]+):\t(.*?)\n')
|
||||
|
||||
asm: dict[tuple[int, str], list[Instruction]] = {}
|
||||
with open(asm_file_path) as asm_file:
|
||||
current_instructions = None
|
||||
for line in asm_file:
|
||||
if match := symbol_pattern.fullmatch(line):
|
||||
symbol = (int(match.group(2), base=16), match.group(1))
|
||||
current_instructions = asm[symbol] = []
|
||||
elif match := instruction_pattern.fullmatch(line):
|
||||
current_instructions.append(Instruction(int(match.group(1), base=16), match.group(2)))
|
||||
|
||||
return asm
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Map LLVMPipe JIT addresses in FlameGraph style '
|
||||
'collapsed stack traces to their symbol name. Also optionally '
|
||||
'annotate JIT assembly dumps with sample counts.')
|
||||
parser.add_argument('jit_symbol_map', type=Path, help='JIT symbol map from LLVMPipe')
|
||||
parser.add_argument('collapsed_traces', type=Path)
|
||||
parser.add_argument('-a', '--asm', type=Path, nargs='?', const='', metavar='asm_path',
|
||||
help='JIT assembly dump from LLVMPipe. Defaults to "<jit_symbol_map>.asm"')
|
||||
parser.add_argument('-o', '--out', type=Path, metavar='out_path')
|
||||
arguments = parser.parse_args()
|
||||
|
||||
mappings = parse_mappings(arguments.jit_symbol_map)
|
||||
traces = parse_traces(arguments.collapsed_traces)
|
||||
|
||||
asm = {}
|
||||
asm_file_path: Path | None = arguments.asm
|
||||
if asm_file_path:
|
||||
if len(asm_file_path.parts) <= 0:
|
||||
asm_file_path = Path(str(arguments.jit_symbol_map) + '.asm')
|
||||
if asm_file_path.exists():
|
||||
asm = parse_asm(asm_file_path)
|
||||
else:
|
||||
asm = parse_asm(asm_file_path)
|
||||
|
||||
merged_traces: dict[str, int] = {}
|
||||
for stack, count in traces:
|
||||
for i, function in enumerate(stack):
|
||||
if not function.startswith('0x'):
|
||||
continue
|
||||
|
||||
address = int(function, base=16)
|
||||
mapping = mappings[bisect_right(mappings, address, key=mapping_address_key) - 1]
|
||||
if address < mapping[0] or address >= mapping[1]:
|
||||
continue
|
||||
|
||||
stack[i] = f'lp`{mapping[2]}@{mapping[0]:x}'
|
||||
|
||||
symbol = (mapping[0], mapping[2])
|
||||
if symbol in asm:
|
||||
instructions = asm[symbol]
|
||||
instruction_address = address - symbol[0]
|
||||
index = bisect_left(instructions, instruction_address, key=instruction_address_key)
|
||||
if index < len(instructions) and instructions[index].address == instruction_address:
|
||||
instructions[index].samples += count
|
||||
|
||||
stack_key = ';'.join(stack)
|
||||
if stack_key in merged_traces:
|
||||
merged_traces[stack_key] += count
|
||||
else:
|
||||
merged_traces[stack_key] = count
|
||||
|
||||
out_file_path: Path | None = arguments.out
|
||||
if not out_file_path:
|
||||
out_file_path = arguments.collapsed_traces.with_stem(f'{arguments.collapsed_traces.stem}_mapped')
|
||||
with open(out_file_path, 'w') as out:
|
||||
for t, c in merged_traces.items():
|
||||
print(f'{t} {c}', file=out)
|
||||
|
||||
if asm:
|
||||
annotated_asm_file_path = asm_file_path.with_stem(f'{asm_file_path.stem}_annotated')
|
||||
with open(annotated_asm_file_path, 'w') as out:
|
||||
for symbol, instructions in asm.items():
|
||||
print(f'{symbol[1]}: ;{symbol[0]:x}', file=out)
|
||||
for instruction in instructions:
|
||||
print(f'\t{instruction.assembly}', end='', file=out)
|
||||
if instruction.samples:
|
||||
print(f' ;s {instruction.samples}', file=out)
|
||||
else:
|
||||
print(file=out)
|
||||
print(file=out)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+253
@@ -0,0 +1,253 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
# Copyright © 2021 Intel Corporation
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
"""Helper script for manipulating the release calendar."""
|
||||
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
import csv
|
||||
import contextlib
|
||||
import datetime
|
||||
import pathlib
|
||||
import subprocess
|
||||
import typing
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
import _csv
|
||||
from typing_extensions import Protocol
|
||||
|
||||
class RCArguments(Protocol):
|
||||
"""Typing information for release-candidate command arguments."""
|
||||
|
||||
manager: str
|
||||
|
||||
class FinalArguments(Protocol):
|
||||
"""Typing information for release command arguments."""
|
||||
|
||||
series: str
|
||||
manager: str
|
||||
zero_released: bool
|
||||
|
||||
class ExtendArguments(Protocol):
|
||||
"""Typing information for extend command arguments."""
|
||||
|
||||
series: str
|
||||
count: int
|
||||
|
||||
|
||||
CalendarRowType = typing.Tuple[typing.Optional[str], str, str, str, typing.Optional[str]]
|
||||
|
||||
|
||||
_ROOT = pathlib.Path(__file__).parent.parent
|
||||
CALENDAR_CSV = _ROOT / 'docs' / 'release-calendar.csv'
|
||||
VERSION = _ROOT / 'VERSION'
|
||||
LAST_RELEASE = 'This is the last planned release of the {}.x series.'
|
||||
OR_FINAL = 'Or {}.0 final.'
|
||||
|
||||
|
||||
def read_calendar() -> typing.List[CalendarRowType]:
|
||||
"""Read the calendar and return a list of it's rows."""
|
||||
with CALENDAR_CSV.open('r') as f:
|
||||
return [typing.cast('CalendarRowType', tuple(r)) for r in csv.reader(f)]
|
||||
|
||||
|
||||
def commit(message: str) -> None:
|
||||
"""Commit the changes the the release-calendar.csv file."""
|
||||
subprocess.run(['git', 'commit', str(CALENDAR_CSV), '--message', message])
|
||||
|
||||
|
||||
|
||||
def _calculate_release_start(major: str, minor: str) -> datetime.date:
|
||||
"""Calculate the start of the release for release candidates.
|
||||
|
||||
This is quarterly, on the second wednesday, in January, April, July, and October.
|
||||
"""
|
||||
quarter = datetime.date.fromisoformat(f'20{major}-0{[1, 4, 7, 10][int(minor)]}-01')
|
||||
|
||||
# Wednesday is 3
|
||||
day = quarter.isoweekday()
|
||||
if day > 3:
|
||||
# this will walk back into the previous month, it's much simpler to
|
||||
# duplicate the 14 than handle the calculations for the month and year
|
||||
# changing.
|
||||
return quarter.replace(day=quarter.day - day + 3 + 14)
|
||||
elif day < 3:
|
||||
quarter = quarter.replace(day=quarter.day + 3 - day)
|
||||
return quarter.replace(day=quarter.day + 14)
|
||||
|
||||
|
||||
def release_candidate(args: RCArguments) -> None:
|
||||
"""Add release candidate entries."""
|
||||
with VERSION.open('r') as f:
|
||||
version = f.read().rstrip('-devel')
|
||||
major, minor, _ = version.split('.')
|
||||
date = _calculate_release_start(major, minor)
|
||||
|
||||
data = read_calendar()
|
||||
|
||||
with CALENDAR_CSV.open('w', newline='') as f:
|
||||
writer = csv.writer(f)
|
||||
writer.writerows(data)
|
||||
|
||||
writer.writerow([f'{major}.{minor}', date.isoformat(), f'{major}.{minor}.0-rc1', args.manager])
|
||||
for row in range(2, 4):
|
||||
date = date + datetime.timedelta(days=7)
|
||||
writer.writerow([None, date.isoformat(), f'{major}.{minor}.0-rc{row}', args.manager])
|
||||
date = date + datetime.timedelta(days=7)
|
||||
writer.writerow([None, date.isoformat(), f'{major}.{minor}.0-rc4', args.manager, OR_FINAL.format(f'{major}.{minor}')])
|
||||
|
||||
commit(f'docs: Add calendar entries for {major}.{minor} release candidates.')
|
||||
|
||||
|
||||
def _calculate_next_release_date(next_is_zero: bool) -> datetime.date:
|
||||
"""Calculate the date of the next release.
|
||||
|
||||
If the next is .0, we have the release in seven days, if the next is .1,
|
||||
then it's in 14
|
||||
"""
|
||||
date = datetime.date.today()
|
||||
day = date.isoweekday()
|
||||
if day < 3:
|
||||
delta = 3 - day
|
||||
elif day > 3:
|
||||
# this will walk back into the previous month, it's much simpler to
|
||||
# duplicate the 14 than handle the calculations for the month and year
|
||||
# changing.
|
||||
delta = (3 - day)
|
||||
else:
|
||||
delta = 0
|
||||
delta += 7
|
||||
if not next_is_zero:
|
||||
delta += 7
|
||||
return date + datetime.timedelta(days=delta)
|
||||
|
||||
|
||||
def final_release(args: FinalArguments) -> None:
|
||||
"""Add final release entries."""
|
||||
data = read_calendar()
|
||||
date = _calculate_next_release_date(not args.zero_released)
|
||||
|
||||
with CALENDAR_CSV.open('w', newline='') as f:
|
||||
writer = csv.writer(f)
|
||||
writer.writerows(data)
|
||||
|
||||
base = 1 if args.zero_released else 0
|
||||
|
||||
writer.writerow([args.series, date.isoformat(), f'{args.series}.{base}', args.manager])
|
||||
for row in range(base + 1, 3):
|
||||
date = date + datetime.timedelta(days=14)
|
||||
writer.writerow([None, date.isoformat(), f'{args.series}.{row}', args.manager])
|
||||
date = date + datetime.timedelta(days=14)
|
||||
writer.writerow([None, date.isoformat(), f'{args.series}.3', args.manager, LAST_RELEASE.format(args.series)])
|
||||
|
||||
commit(f'docs: Add calendar entries for {args.series} release.')
|
||||
|
||||
|
||||
def extend(args: ExtendArguments) -> None:
|
||||
"""Extend a release."""
|
||||
@contextlib.contextmanager
|
||||
def write_existing(writer: _csv._writer, current: typing.List[CalendarRowType]) -> typing.Iterator[CalendarRowType]:
|
||||
"""Write the orinal file, yield to insert new entries.
|
||||
|
||||
This is a bit clever, basically what happens it writes out the
|
||||
original csv file until it reaches the start of the release after the
|
||||
one we're appending, then it yields the last row. When control is
|
||||
returned it writes out the rest of the original calendar data.
|
||||
"""
|
||||
last_row: typing.Optional[CalendarRowType] = None
|
||||
in_wanted = False
|
||||
for row in current:
|
||||
if in_wanted and row[0]:
|
||||
in_wanted = False
|
||||
assert last_row is not None
|
||||
yield last_row
|
||||
if row[0] == args.series:
|
||||
in_wanted = True
|
||||
if in_wanted and len(row) >= 5 and row[4] in {LAST_RELEASE.format(args.series), OR_FINAL.format(args.series)}:
|
||||
# If this was the last planned release and we're adding more,
|
||||
# then we need to remove that message and add it elsewhere
|
||||
r = list(row)
|
||||
r[4] = None
|
||||
# Mypy can't figure this out…
|
||||
row = typing.cast('CalendarRowType', tuple(r))
|
||||
last_row = row
|
||||
writer.writerow(row)
|
||||
# If this is the only entry we can hit a case where the contextmanager
|
||||
# hasn't yielded
|
||||
if in_wanted:
|
||||
yield row
|
||||
|
||||
current = read_calendar()
|
||||
|
||||
with CALENDAR_CSV.open('w', newline='') as f:
|
||||
writer = csv.writer(f)
|
||||
with write_existing(writer, current) as row:
|
||||
# Get rid of -rcX as well
|
||||
if '-rc' in row[2]:
|
||||
first_point = int(row[2].split('rc')[-1]) + 1
|
||||
template = '{}.0-rc{}'
|
||||
days = 7
|
||||
else:
|
||||
first_point = int(row[2].split('-')[0].split('.')[-1]) + 1
|
||||
template = '{}.{}'
|
||||
days = 14
|
||||
|
||||
date = datetime.date.fromisoformat(row[1])
|
||||
for i in range(first_point, first_point + args.count):
|
||||
date = date + datetime.timedelta(days=days)
|
||||
r = [None, date.isoformat(), template.format(args.series, i), row[3], None]
|
||||
if i == first_point + args.count - 1:
|
||||
if days == 14:
|
||||
r[4] = LAST_RELEASE.format(args.series)
|
||||
else:
|
||||
r[4] = OR_FINAL.format(args.series)
|
||||
writer.writerow(r)
|
||||
|
||||
commit(f'docs: Extend calendar entries for {args.series} by {args.count} releases.')
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
sub = parser.add_subparsers()
|
||||
|
||||
rc = sub.add_parser('release-candidate', aliases=['rc'], help='Generate calendar entries for a release candidate.')
|
||||
rc.add_argument('manager', help="the name of the person managing the release.")
|
||||
rc.set_defaults(func=release_candidate)
|
||||
|
||||
fr = sub.add_parser('release', help='Generate calendar entries for a final release.')
|
||||
fr.add_argument('manager', help="the name of the person managing the release.")
|
||||
fr.add_argument('series', help='The series to extend, such as "29.3" or "30.0".')
|
||||
fr.add_argument('--zero-released', action='store_true', help='The .0 release was today, the next release is .1')
|
||||
fr.set_defaults(func=final_release)
|
||||
|
||||
ex = sub.add_parser('extend', help='Generate additional entries for a release.')
|
||||
ex.add_argument('series', help='The series to extend, such as "29.3" or "30.0".')
|
||||
ex.add_argument('count', type=int, help='The number of new entries to add.')
|
||||
ex.set_defaults(func=extend)
|
||||
|
||||
args = parser.parse_args()
|
||||
args.func(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,319 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
# Copyright © 2021 Intel Corporation
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
from __future__ import annotations
|
||||
from unittest import mock
|
||||
import argparse
|
||||
import csv
|
||||
import contextlib
|
||||
import datetime
|
||||
import tempfile
|
||||
import os
|
||||
import pathlib
|
||||
import typing
|
||||
|
||||
import pytest
|
||||
|
||||
from . import gen_calendar_entries
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def mock_csv(data: typing.List[gen_calendar_entries.CalendarRowType]) -> typing.Iterator[None]:
|
||||
"""Replace the actual CSV data with our test data."""
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
c = os.path.join(d, 'calendar.csv')
|
||||
with open(c, 'w') as f:
|
||||
writer = csv.writer(f)
|
||||
writer.writerows(data)
|
||||
|
||||
with mock.patch('bin.gen_calendar_entries.CALENDAR_CSV', pathlib.Path(c)):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True, scope='module')
|
||||
def disable_git_commits() -> None:
|
||||
"""Mock out the commit function so no git commits are made during testing."""
|
||||
with mock.patch('bin.gen_calendar_entries.commit', mock.Mock()):
|
||||
yield
|
||||
|
||||
|
||||
class TestReleaseStart:
|
||||
|
||||
def test_first_is_wednesday(self) -> None:
|
||||
d = gen_calendar_entries._calculate_release_start('20', '0')
|
||||
assert d.day == 15
|
||||
assert d.month == 1
|
||||
assert d.year == 2020
|
||||
|
||||
def test_first_is_before_wednesday(self) -> None:
|
||||
d = gen_calendar_entries._calculate_release_start('19', '0')
|
||||
assert d.day == 16
|
||||
assert d.month == 1
|
||||
assert d.year == 2019
|
||||
|
||||
def test_first_is_after_wednesday(self) -> None:
|
||||
d = gen_calendar_entries._calculate_release_start('21', '0')
|
||||
assert d.day == 13
|
||||
assert d.month == 1
|
||||
assert d.year == 2021
|
||||
|
||||
|
||||
class TestNextReleaseDate:
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _patch_date(date: datetime.date) -> typing.Iterator[None]:
|
||||
mdate = mock.Mock()
|
||||
mdate.today = mock.Mock(return_value=date)
|
||||
with mock.patch('bin.gen_calendar_entries.datetime.date', mdate):
|
||||
yield
|
||||
|
||||
class TestIsWeds:
|
||||
|
||||
@pytest.fixture(scope='class', autouse=True)
|
||||
def data(self) -> None:
|
||||
date = datetime.date(2021, 1, 6)
|
||||
with TestNextReleaseDate._patch_date(date):
|
||||
yield
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'is_zero, expected',
|
||||
[
|
||||
(True, 13),
|
||||
(False, 20),
|
||||
],
|
||||
)
|
||||
def test(self, is_zero: bool, expected: int) -> None:
|
||||
date = gen_calendar_entries._calculate_next_release_date(is_zero)
|
||||
assert date.day == expected
|
||||
|
||||
class TestBeforeWeds:
|
||||
|
||||
@pytest.fixture(scope='class', autouse=True)
|
||||
def data(self) -> None:
|
||||
date = datetime.date(2021, 1, 5)
|
||||
with TestNextReleaseDate._patch_date(date):
|
||||
yield
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'is_zero, expected',
|
||||
[
|
||||
(True, 13),
|
||||
(False, 20),
|
||||
],
|
||||
)
|
||||
def test(self, is_zero: bool, expected: int) -> None:
|
||||
date = gen_calendar_entries._calculate_next_release_date(is_zero)
|
||||
assert date.day == expected
|
||||
|
||||
class TestAfterWeds:
|
||||
|
||||
@pytest.fixture(scope='class', autouse=True)
|
||||
def data(self) -> None:
|
||||
date = datetime.date(2021, 1, 8)
|
||||
with TestNextReleaseDate._patch_date(date):
|
||||
yield
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'is_zero, expected',
|
||||
[
|
||||
(True, 13),
|
||||
(False, 20),
|
||||
],
|
||||
)
|
||||
def test(self, is_zero: bool, expected: int) -> None:
|
||||
date = gen_calendar_entries._calculate_next_release_date(is_zero)
|
||||
assert date.day == expected
|
||||
|
||||
|
||||
class TestRC:
|
||||
|
||||
ORIGINAL_DATA = [
|
||||
('20.3', '2021-01-13', '20.3.3', 'Dylan Baker', ''),
|
||||
('', '2021-01-27', '20.3.4', 'Dylan Baker', 'Last planned release of the 20.3.x series'),
|
||||
]
|
||||
|
||||
@pytest.fixture(autouse=True, scope='class')
|
||||
def mock_version(self) -> None:
|
||||
"""Keep the version set at a specific value."""
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
v = os.path.join(d, 'version')
|
||||
with open(v, 'w') as f:
|
||||
f.write('21.0.0-devel\n')
|
||||
|
||||
with mock.patch('bin.gen_calendar_entries.VERSION', pathlib.Path(v)):
|
||||
yield
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def csv(self) -> None:
|
||||
"""inject our test data.."""
|
||||
with mock_csv(self.ORIGINAL_DATA):
|
||||
yield
|
||||
|
||||
def test_basic(self) -> None:
|
||||
args: gen_calendar_entries.RCArguments = argparse.Namespace()
|
||||
args.manager = "Dylan Baker"
|
||||
gen_calendar_entries.release_candidate(args)
|
||||
|
||||
expected = self.ORIGINAL_DATA.copy()
|
||||
expected.append(('21.0', '2021-01-13', f'21.0.0-rc1', 'Dylan Baker'))
|
||||
expected.append(( '', '2021-01-20', f'21.0.0-rc2', 'Dylan Baker'))
|
||||
expected.append(( '', '2021-01-27', f'21.0.0-rc3', 'Dylan Baker'))
|
||||
expected.append(( '', '2021-02-03', f'21.0.0-rc4', 'Dylan Baker', 'Or 21.0.0 final.'))
|
||||
|
||||
actual = gen_calendar_entries.read_calendar()
|
||||
|
||||
assert actual == expected
|
||||
|
||||
|
||||
class TestExtend:
|
||||
|
||||
def test_one_release(self) -> None:
|
||||
data = [
|
||||
('20.3', '2021-01-13', '20.3.3', 'Dylan Baker', ''),
|
||||
('', '2021-01-27', '20.3.4', 'Dylan Baker', 'This is the last planned release of the 20.3.x series.'),
|
||||
]
|
||||
|
||||
args: gen_calendar_entries.ExtendArguments = argparse.Namespace()
|
||||
args.series = '20.3'
|
||||
args.count = 2
|
||||
|
||||
with mock_csv(data):
|
||||
gen_calendar_entries.extend(args)
|
||||
actual = gen_calendar_entries.read_calendar()
|
||||
|
||||
expected = [
|
||||
data[0],
|
||||
('', '2021-01-27', '20.3.4', 'Dylan Baker', ''),
|
||||
('', '2021-02-10', '20.3.5', 'Dylan Baker', ''),
|
||||
('', '2021-02-24', '20.3.6', 'Dylan Baker', 'This is the last planned release of the 20.3.x series.'),
|
||||
]
|
||||
|
||||
assert actual == expected
|
||||
def test_one_release(self) -> None:
|
||||
data = [
|
||||
('20.3', '2021-01-13', '20.3.3', 'Dylan Baker', ''),
|
||||
('', '2021-01-27', '20.3.4', 'Dylan Baker', 'This is the last planned release of the 20.3.x series.'),
|
||||
('21.0', '2021-01-13', '21.0.1', 'Dylan Baker', ''),
|
||||
('', '2021-01-27', '21.0.2', 'Dylan Baker', ''),
|
||||
('', '2021-02-10', '21.0.3', 'Dylan Baker', ''),
|
||||
('', '2021-02-24', '21.0.4', 'Dylan Baker', 'This is the last planned release of the 21.0.x series.'),
|
||||
]
|
||||
|
||||
args: gen_calendar_entries.ExtendArguments = argparse.Namespace()
|
||||
args.series = '21.0'
|
||||
args.count = 1
|
||||
|
||||
with mock_csv(data):
|
||||
gen_calendar_entries.extend(args)
|
||||
actual = gen_calendar_entries.read_calendar()
|
||||
|
||||
expected = data.copy()
|
||||
d = list(data[-1])
|
||||
d[-1] = ''
|
||||
expected[-1] = tuple(d)
|
||||
expected.extend([
|
||||
('', '2021-03-10', '21.0.5', 'Dylan Baker', 'This is the last planned release of the 21.0.x series.'),
|
||||
])
|
||||
|
||||
assert actual == expected
|
||||
|
||||
def test_rc(self) -> None:
|
||||
data = [
|
||||
('20.3', '2021-01-13', '20.3.3', 'Dylan Baker', ''),
|
||||
('', '2021-01-27', '20.3.4', 'Dylan Baker', 'This is the last planned release of the 20.3.x series.'),
|
||||
('21.0', '2021-01-13', '21.0.0-rc1', 'Dylan Baker', ''),
|
||||
('', '2021-01-20', '21.0.0-rc2', 'Dylan Baker', gen_calendar_entries.OR_FINAL.format('21.0')),
|
||||
]
|
||||
|
||||
args: gen_calendar_entries.ExtendArguments = argparse.Namespace()
|
||||
args.series = '21.0'
|
||||
args.count = 2
|
||||
|
||||
with mock_csv(data):
|
||||
gen_calendar_entries.extend(args)
|
||||
actual = gen_calendar_entries.read_calendar()
|
||||
|
||||
expected = data.copy()
|
||||
d = list(expected[-1])
|
||||
d[-1] = ''
|
||||
expected[-1] = tuple(d)
|
||||
expected.extend([
|
||||
('', '2021-01-27', '21.0.0-rc3', 'Dylan Baker', ''),
|
||||
('', '2021-02-03', '21.0.0-rc4', 'Dylan Baker', gen_calendar_entries.OR_FINAL.format('21.0')),
|
||||
])
|
||||
|
||||
assert actual == expected
|
||||
|
||||
|
||||
class TestFinal:
|
||||
|
||||
@pytest.fixture(autouse=True, scope='class')
|
||||
def _patch_date(self) -> typing.Iterator[None]:
|
||||
mdate = mock.Mock()
|
||||
mdate.today = mock.Mock(return_value=datetime.date(2021, 1, 6))
|
||||
with mock.patch('bin.gen_calendar_entries.datetime.date', mdate):
|
||||
yield
|
||||
|
||||
ORIGINAL_DATA = [
|
||||
('20.3', '2021-01-13', '20.3.3', 'Dylan Baker', ''),
|
||||
('', '2021-01-27', '20.3.4', 'Dylan Baker', 'Last planned release of the 20.3.x series'),
|
||||
]
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def csv(self) -> None:
|
||||
"""inject our test data.."""
|
||||
with mock_csv(self.ORIGINAL_DATA):
|
||||
yield
|
||||
|
||||
def test_zero_released(self) -> None:
|
||||
args: gen_calendar_entries.FinalArguments = argparse.Namespace()
|
||||
args.manager = "Dylan Baker"
|
||||
args.zero_released = True
|
||||
args.series = '21.0'
|
||||
gen_calendar_entries.final_release(args)
|
||||
|
||||
expected = self.ORIGINAL_DATA.copy()
|
||||
expected.append(('21.0', '2021-01-20', f'21.0.1', 'Dylan Baker'))
|
||||
expected.append(( '', '2021-02-03', f'21.0.2', 'Dylan Baker'))
|
||||
expected.append(( '', '2021-02-17', f'21.0.3', 'Dylan Baker', gen_calendar_entries.LAST_RELEASE.format(args.series)))
|
||||
|
||||
actual = gen_calendar_entries.read_calendar()
|
||||
|
||||
assert actual == expected
|
||||
|
||||
def test_zero_not_released(self) -> None:
|
||||
args: gen_calendar_entries.FinalArguments = argparse.Namespace()
|
||||
args.manager = "Dylan Baker"
|
||||
args.zero_released = False
|
||||
args.series = '21.0'
|
||||
gen_calendar_entries.final_release(args)
|
||||
|
||||
expected = self.ORIGINAL_DATA.copy()
|
||||
expected.append(('21.0', '2021-01-13', f'21.0.0', 'Dylan Baker'))
|
||||
expected.append(( '', '2021-01-27', f'21.0.1', 'Dylan Baker'))
|
||||
expected.append(( '', '2021-02-10', f'21.0.2', 'Dylan Baker'))
|
||||
expected.append(( '', '2021-02-24', f'21.0.3', 'Dylan Baker', gen_calendar_entries.LAST_RELEASE.format(args.series)))
|
||||
|
||||
actual = gen_calendar_entries.read_calendar()
|
||||
|
||||
assert actual == expected
|
||||
Executable
+395
@@ -0,0 +1,395 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright © 2019-2020 Intel Corporation
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
"""Generates release notes for a given version of mesa."""
|
||||
|
||||
import asyncio
|
||||
import datetime
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
import typing
|
||||
import urllib.parse
|
||||
|
||||
import aiohttp
|
||||
from mako.template import Template
|
||||
from mako import exceptions
|
||||
|
||||
import docutils.utils
|
||||
import docutils.parsers.rst.states as states
|
||||
|
||||
CURRENT_GL_VERSION = '4.6'
|
||||
CURRENT_CL_VERSION = '3.1'
|
||||
CURRENT_VK_VERSION = '1.4'
|
||||
|
||||
TEMPLATE = Template(textwrap.dedent("""\
|
||||
${header}
|
||||
${header_underline}
|
||||
|
||||
%if not bugfix:
|
||||
Mesa ${this_version} is a new development release. People who are concerned
|
||||
with stability and reliability should stick with a previous release or
|
||||
wait for Mesa ${this_version[:-1]}1.
|
||||
%else:
|
||||
Mesa ${this_version} is a bug fix release which fixes bugs found since the ${previous_version} release.
|
||||
%endif
|
||||
|
||||
Mesa ${this_version} implements the OpenGL ${gl_version} API, but the version reported by
|
||||
glGetString(GL_VERSION) or glGetIntegerv(GL_MAJOR_VERSION) /
|
||||
glGetIntegerv(GL_MINOR_VERSION) depends on the particular driver being used.
|
||||
Some drivers don't support all the features required in OpenGL ${gl_version}. OpenGL
|
||||
${gl_version} is **only** available if requested at context creation.
|
||||
Compatibility contexts may report a lower version depending on each driver.
|
||||
|
||||
Mesa ${this_version} implements the OpenCL ${cl_version} API, but the version reported by
|
||||
the CL_DEVICE_VERSION, CL_DEVICE_NUMERIC_VERSION and CL_DEVICE_OPENCL_C_ALL_VERSIONS
|
||||
clGetDeviceInfo queries depends on the particular driver being used.
|
||||
|
||||
Mesa ${this_version} implements the Vulkan ${vk_version} API, but the version reported by
|
||||
the apiVersion property of the VkPhysicalDeviceProperties struct
|
||||
depends on the particular driver being used.
|
||||
|
||||
SHA checksums
|
||||
-------------
|
||||
|
||||
::
|
||||
|
||||
TBD.
|
||||
|
||||
|
||||
New features
|
||||
------------
|
||||
|
||||
%for f in features:
|
||||
- ${rst_escape(f)}
|
||||
%endfor
|
||||
|
||||
|
||||
Bug fixes
|
||||
---------
|
||||
|
||||
%for b in bugs:
|
||||
- ${rst_escape(b)}
|
||||
%endfor
|
||||
|
||||
|
||||
Changes
|
||||
-------
|
||||
%for c, author_line in changes:
|
||||
%if author_line:
|
||||
|
||||
${rst_escape(c)}
|
||||
|
||||
%else:
|
||||
- ${rst_escape(c)}
|
||||
%endif
|
||||
%endfor
|
||||
"""))
|
||||
|
||||
|
||||
# copied from https://docutils.sourceforge.io/sandbox/xml2rst/xml2rstlib/markup.py
|
||||
class Inliner(states.Inliner):
|
||||
"""
|
||||
Recognizer for inline markup. Derive this from the original inline
|
||||
markup parser for best results.
|
||||
"""
|
||||
|
||||
# Copy static attributes from super class
|
||||
vars().update(vars(states.Inliner))
|
||||
|
||||
def quoteInline(self, text):
|
||||
"""
|
||||
`text`: ``str``
|
||||
Return `text` with inline markup quoted.
|
||||
"""
|
||||
# Method inspired by `states.Inliner.parse`
|
||||
self.document = docutils.utils.new_document("<string>")
|
||||
self.document.settings.trim_footnote_reference_space = False
|
||||
self.document.settings.character_level_inline_markup = False
|
||||
self.document.settings.pep_references = False
|
||||
self.document.settings.rfc_references = False
|
||||
|
||||
self.init_customizations(self.document.settings)
|
||||
|
||||
self.reporter = self.document.reporter
|
||||
self.reporter.stream = None
|
||||
self.language = None
|
||||
self.parent = self.document
|
||||
remaining = docutils.utils.escape2null(text)
|
||||
checked = ""
|
||||
processed = []
|
||||
unprocessed = []
|
||||
messages = []
|
||||
while remaining:
|
||||
original = remaining
|
||||
match = self.patterns.initial.search(remaining)
|
||||
if match:
|
||||
groups = match.groupdict()
|
||||
method = self.dispatch[groups['start'] or groups['backquote']
|
||||
or groups['refend'] or groups['fnend']]
|
||||
before, inlines, remaining, sysmessages = method(self, match, 0)
|
||||
checked += before
|
||||
if inlines:
|
||||
assert len(inlines) == 1, "More than one inline found"
|
||||
inline = original[len(before)
|
||||
:len(original) - len(remaining)]
|
||||
rolePfx = re.search("^:" + self.simplename + ":(?=`)",
|
||||
inline)
|
||||
refSfx = re.search("_+$", inline)
|
||||
if rolePfx:
|
||||
# Prefixed roles need to be quoted in the middle
|
||||
checked += (inline[:rolePfx.end()] + "\\"
|
||||
+ inline[rolePfx.end():])
|
||||
elif refSfx and not re.search("^`", inline):
|
||||
# Pure reference markup needs to be quoted at the end
|
||||
checked += (inline[:refSfx.start()] + "\\"
|
||||
+ inline[refSfx.start():])
|
||||
else:
|
||||
# Quote other inlines by prefixing
|
||||
checked += "\\" + inline
|
||||
else:
|
||||
checked += remaining
|
||||
break
|
||||
# Quote all original backslashes
|
||||
checked = re.sub('\x00', "\\\x00", checked)
|
||||
checked = re.sub('@', '\\@', checked)
|
||||
return docutils.utils.unescape(checked, 1)
|
||||
|
||||
inliner = Inliner();
|
||||
|
||||
|
||||
async def gather_commits(version: str) -> str:
|
||||
p = await asyncio.create_subprocess_exec(
|
||||
'git', 'log', '--oneline', f'mesa-{version}..', '-i', '--grep', r'\(Closes\|Fixes\): \(https\|#\).*',
|
||||
stdout=asyncio.subprocess.PIPE)
|
||||
out, _ = await p.communicate()
|
||||
assert p.returncode == 0, f"git log didn't work: {version}"
|
||||
return out.decode().strip()
|
||||
|
||||
|
||||
async def parse_issues(commits: str) -> typing.List[str]:
|
||||
issues: typing.List[str] = []
|
||||
for commit in commits.split('\n'):
|
||||
sha, message = commit.split(maxsplit=1)
|
||||
p = await asyncio.create_subprocess_exec(
|
||||
'git', 'log', '--max-count', '1', r'--format=%b', sha,
|
||||
stdout=asyncio.subprocess.PIPE)
|
||||
_out, _ = await p.communicate()
|
||||
out = _out.decode().split('\n')
|
||||
|
||||
for line in reversed(out):
|
||||
if not line.lower().startswith(('closes:', 'fixes:')):
|
||||
continue
|
||||
bug = line.split(':', 1)[1].strip()
|
||||
if (bug.startswith('https://gitlab.freedesktop.org/mesa/mesa')
|
||||
# Avoid parsing "merge_requests" URL. Note that a valid issue
|
||||
# URL may or may not contain the "/-/" text, so we check if
|
||||
# the word "issues" is contained in URL.
|
||||
and '/issues' in bug):
|
||||
# This means we have a bug in the form "Closes: https://..."
|
||||
issues.append(os.path.basename(urllib.parse.urlparse(bug).path))
|
||||
elif ',' in bug:
|
||||
multiple_bugs = [b.strip().lstrip('#') for b in bug.split(',')]
|
||||
if not all(b.isdigit() for b in multiple_bugs):
|
||||
# this is likely a "Fixes" tag that refers to a commit name
|
||||
continue
|
||||
issues.extend(multiple_bugs)
|
||||
elif bug.startswith('#'):
|
||||
issues.append(bug.lstrip('#'))
|
||||
|
||||
return issues
|
||||
|
||||
|
||||
async def gather_bugs(version: str) -> typing.List[str]:
|
||||
commits = await gather_commits(version)
|
||||
if commits:
|
||||
issues = await parse_issues(commits)
|
||||
else:
|
||||
issues = []
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
async with aiohttp.ClientSession(loop=loop) as session:
|
||||
results = await asyncio.gather(*[get_bug(session, i) for i in issues])
|
||||
# Remove duplicates.
|
||||
bugs = sorted(set(results))
|
||||
if not bugs:
|
||||
bugs = ['None']
|
||||
return bugs
|
||||
|
||||
|
||||
async def get_bug(session: aiohttp.ClientSession, bug_id: str) -> str:
|
||||
"""Query gitlab to get the name of the issue that was closed."""
|
||||
# Mesa's gitlab id is 176,
|
||||
url = 'https://gitlab.freedesktop.org/api/v4/projects/176/issues'
|
||||
params = {'iids[]': bug_id}
|
||||
async with session.get(url, params=params) as response:
|
||||
content = await response.json()
|
||||
if not content:
|
||||
# issues marked as "confidential" look like "404" page for
|
||||
# unauthorized users
|
||||
return f'Confidential issue #{bug_id}'
|
||||
else:
|
||||
return content[0]['title']
|
||||
|
||||
|
||||
async def get_shortlog(version: str) -> str:
|
||||
"""Call git shortlog."""
|
||||
p = await asyncio.create_subprocess_exec('git', 'shortlog', f'mesa-{version}..',
|
||||
stdout=asyncio.subprocess.PIPE)
|
||||
out, _ = await p.communicate()
|
||||
assert p.returncode == 0, 'error getting shortlog'
|
||||
assert out is not None, 'just for mypy'
|
||||
return out.decode()
|
||||
|
||||
|
||||
def walk_shortlog(log: str) -> typing.Generator[typing.Tuple[str, bool], None, None]:
|
||||
for l in log.split('\n'):
|
||||
if l.startswith(' '): # this means we have a patch description
|
||||
yield l.lstrip(), False
|
||||
elif l.strip():
|
||||
yield l, True
|
||||
|
||||
|
||||
def calculate_next_version(version: str, is_point: bool) -> str:
|
||||
"""Calculate the version about to be released."""
|
||||
if '-' in version:
|
||||
version = version.split('-')[0]
|
||||
if is_point:
|
||||
base = version.split('.')
|
||||
base[2] = str(int(base[2]) + 1)
|
||||
return '.'.join(base)
|
||||
return version
|
||||
|
||||
|
||||
def calculate_previous_version(version: str, is_point: bool) -> str:
|
||||
"""Calculate the previous version to compare to.
|
||||
|
||||
In the case of -rc to final that version is the previous .0 release,
|
||||
(19.3.0 in the case of 20.0.0, for example). for point releases that is
|
||||
the last point release. This value will be the same as the input value
|
||||
for a point release, but different for a major release.
|
||||
"""
|
||||
if '-' in version:
|
||||
version = version.split('-')[0]
|
||||
if is_point:
|
||||
return version
|
||||
base = version.split('.')
|
||||
if base[1] == '0':
|
||||
base[0] = str(int(base[0]) - 1)
|
||||
base[1] = '3'
|
||||
else:
|
||||
base[1] = str(int(base[1]) - 1)
|
||||
return '.'.join(base)
|
||||
|
||||
|
||||
def get_features(is_point_release: bool) -> typing.Generator[str, None, None]:
|
||||
p = pathlib.Path('docs') / 'relnotes' / 'new_features.txt'
|
||||
if p.exists() and p.stat().st_size > 0:
|
||||
if is_point_release:
|
||||
print("WARNING: new features being introduced in a point release", file=sys.stderr)
|
||||
with p.open('rt') as f:
|
||||
for line in f:
|
||||
yield line.rstrip()
|
||||
p.unlink()
|
||||
subprocess.run(['git', 'add', p])
|
||||
else:
|
||||
yield "None"
|
||||
|
||||
|
||||
def update_release_notes_index(version: str) -> None:
|
||||
relnotes_index_path = pathlib.Path('docs') / 'relnotes.rst'
|
||||
|
||||
with relnotes_index_path.open('r') as f:
|
||||
relnotes = f.readlines()
|
||||
|
||||
new_relnotes = []
|
||||
first_list = True
|
||||
second_list = True
|
||||
for line in relnotes:
|
||||
if first_list and line.startswith('-'):
|
||||
first_list = False
|
||||
new_relnotes.append(f'- :doc:`{version} release notes <relnotes/{version}>`\n')
|
||||
if (not first_list and second_list and
|
||||
re.match(r' \d+.\d+(.\d+)? <relnotes/\d+.\d+(.\d+)?>', line)):
|
||||
second_list = False
|
||||
new_relnotes.append(f' {version} <relnotes/{version}>\n')
|
||||
new_relnotes.append(line)
|
||||
|
||||
with relnotes_index_path.open('w', encoding='utf-8') as f:
|
||||
for line in new_relnotes:
|
||||
f.write(line)
|
||||
|
||||
subprocess.run(['git', 'add', relnotes_index_path])
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
v = pathlib.Path('VERSION')
|
||||
with v.open('rt') as f:
|
||||
raw_version = f.read().strip()
|
||||
is_point_release = '-rc' not in raw_version
|
||||
assert '-devel' not in raw_version, 'Do not run this script on -devel'
|
||||
version = raw_version.split('-')[0]
|
||||
previous_version = calculate_previous_version(version, is_point_release)
|
||||
this_version = calculate_next_version(version, is_point_release)
|
||||
today = datetime.date.today()
|
||||
header = f'Mesa {this_version} Release Notes / {today}'
|
||||
header_underline = '=' * len(header)
|
||||
|
||||
shortlog, bugs = await asyncio.gather(
|
||||
get_shortlog(previous_version),
|
||||
gather_bugs(previous_version),
|
||||
)
|
||||
|
||||
final = pathlib.Path('docs') / 'relnotes' / f'{this_version}.rst'
|
||||
with final.open('wt', encoding='utf-8') as f:
|
||||
try:
|
||||
f.write(TEMPLATE.render(
|
||||
bugfix=is_point_release,
|
||||
bugs=bugs,
|
||||
changes=walk_shortlog(shortlog),
|
||||
features=get_features(is_point_release),
|
||||
gl_version=CURRENT_GL_VERSION,
|
||||
cl_version=CURRENT_CL_VERSION,
|
||||
this_version=this_version,
|
||||
header=header,
|
||||
header_underline=header_underline,
|
||||
previous_version=previous_version,
|
||||
vk_version=CURRENT_VK_VERSION,
|
||||
rst_escape=inliner.quoteInline,
|
||||
))
|
||||
except:
|
||||
print(exceptions.text_error_template().render())
|
||||
return
|
||||
|
||||
subprocess.run(['git', 'add', final])
|
||||
|
||||
update_release_notes_index(this_version)
|
||||
|
||||
subprocess.run(['git', 'commit', '-m',
|
||||
f'docs: add release notes for {this_version}'])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
loop = asyncio.new_event_loop()
|
||||
loop.run_until_complete(main())
|
||||
@@ -0,0 +1,226 @@
|
||||
# Copyright © 2019,2021 Intel Corporation
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
import sys
|
||||
import textwrap
|
||||
import typing
|
||||
|
||||
import pytest
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
import aiohttp
|
||||
|
||||
# AsyncMock is new in 3.8, so if we're using an older version we need the
|
||||
# backported version of mock
|
||||
if sys.version_info >= (3, 8):
|
||||
from unittest import mock
|
||||
else:
|
||||
import mock
|
||||
|
||||
from .gen_release_notes import *
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'current, is_point, expected',
|
||||
[
|
||||
('19.2.0', True, '19.2.1'),
|
||||
('19.3.6', True, '19.3.7'),
|
||||
('20.0.0-rc4', False, '20.0.0'),
|
||||
])
|
||||
def test_next_version(current: str, is_point: bool, expected: str) -> None:
|
||||
assert calculate_next_version(current, is_point) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'current, is_point, expected',
|
||||
[
|
||||
('19.3.6', True, '19.3.6'),
|
||||
('20.0.0-rc4', False, '19.3.0'),
|
||||
])
|
||||
def test_previous_version(current: str, is_point: bool, expected: str) -> None:
|
||||
assert calculate_previous_version(current, is_point) == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_shortlog():
|
||||
# Certainly not perfect, but it's something
|
||||
version = '19.2.0'
|
||||
out = await get_shortlog(version)
|
||||
assert out
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gather_commits():
|
||||
# Certainly not perfect, but it's something
|
||||
version = '19.2.0'
|
||||
out = await gather_commits(version)
|
||||
assert out
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
'content, bugs',
|
||||
[
|
||||
# It is important to have the title on a new line, as
|
||||
# textwrap.dedent won't work otherwise.
|
||||
|
||||
# Test the `Closes: #N` syntax
|
||||
(
|
||||
'''\
|
||||
A commit
|
||||
|
||||
It has a message in it
|
||||
|
||||
Closes: #1
|
||||
''',
|
||||
['1'],
|
||||
),
|
||||
|
||||
# Test the Full url
|
||||
(
|
||||
'''\
|
||||
A commit with no body
|
||||
|
||||
Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/3456
|
||||
''',
|
||||
['3456'],
|
||||
),
|
||||
|
||||
# Test projects that are not mesa
|
||||
(
|
||||
'''\
|
||||
A commit for libdrm
|
||||
|
||||
Closes: https://gitlab.freedesktop.org/mesa/drm/-/3456
|
||||
''',
|
||||
[],
|
||||
),
|
||||
(
|
||||
'''\
|
||||
A commit for for something else completely
|
||||
|
||||
Closes: https://github.com/Organization/project/1234
|
||||
''',
|
||||
[],
|
||||
),
|
||||
|
||||
# Test multiple issues on one line
|
||||
(
|
||||
'''\
|
||||
Fix many bugs
|
||||
|
||||
Closes: #1, #2
|
||||
''',
|
||||
['1', '2'],
|
||||
),
|
||||
|
||||
# Test multiple closes
|
||||
(
|
||||
'''\
|
||||
Fix many bugs
|
||||
|
||||
Closes: #1
|
||||
Closes: #2
|
||||
''',
|
||||
['1', '2'],
|
||||
),
|
||||
(
|
||||
'''\
|
||||
With long form
|
||||
|
||||
Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/3456
|
||||
Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/3457
|
||||
Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/3458
|
||||
''',
|
||||
['3456', '3457', '3458'],
|
||||
),
|
||||
(
|
||||
'''\
|
||||
Without /-/
|
||||
|
||||
Closes: https://gitlab.freedesktop.org/mesa/mesa/issues/36
|
||||
''',
|
||||
['36'],
|
||||
),
|
||||
(
|
||||
'''\
|
||||
Ignore merge_requests
|
||||
|
||||
Closes: https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/20241
|
||||
''',
|
||||
[],
|
||||
),
|
||||
(
|
||||
'''\
|
||||
Parse "Fixes:" tag too
|
||||
|
||||
Fixes: https://gitlab.freedesktop.org/mesa/mesa/issues/36
|
||||
Fixes: 142565a3bc2
|
||||
Fixes: 142565a3bc2 ("docs: do something very useful")
|
||||
Fixes: 142565a3bc2 ("docs: fix #1234, have a comma")
|
||||
Fixes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/37
|
||||
''',
|
||||
['36', '37'],
|
||||
),
|
||||
(
|
||||
'''\
|
||||
Parse Fixes/Closes in weird cases
|
||||
|
||||
fixes: https://gitlab.freedesktop.org/mesa/mesa/issues/36
|
||||
fiXES: https://gitlab.freedesktop.org/mesa/mesa/issues/37
|
||||
closes: https://gitlab.freedesktop.org/mesa/mesa/issues/38
|
||||
cloSES: https://gitlab.freedesktop.org/mesa/mesa/issues/39
|
||||
''',
|
||||
['36', '37', '38', '39'],
|
||||
),
|
||||
])
|
||||
async def test_parse_issues(content: str, bugs: typing.List[str]) -> None:
|
||||
mock_com = mock.AsyncMock(return_value=(textwrap.dedent(content).encode(), ''))
|
||||
mock_p = mock.Mock()
|
||||
mock_p.communicate = mock_com
|
||||
mock_exec = mock.AsyncMock(return_value=mock_p)
|
||||
|
||||
with mock.patch('bin.gen_release_notes.asyncio.create_subprocess_exec', mock_exec), \
|
||||
mock.patch('bin.gen_release_notes.gather_commits', mock.AsyncMock(return_value='sha\n')):
|
||||
ids = await parse_issues('1234 not used')
|
||||
assert set(ids) == set(bugs)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rst_escape():
|
||||
out = inliner.quoteInline('foo@bar')
|
||||
assert out == 'foo\@bar'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gather_bugs_duplicates():
|
||||
mock_gc = mock.AsyncMock(return_value='something')
|
||||
mock_pi = mock.AsyncMock(return_value=['a', 'b', 'a', 'a', 'c', 'b'])
|
||||
|
||||
async def get_bug(session: 'aiohttp.ClientSession', bug_id: str) -> str:
|
||||
return bug_id
|
||||
|
||||
with mock.patch('bin.gen_release_notes.gather_commits', mock_gc), \
|
||||
mock.patch('bin.gen_release_notes.parse_issues', mock_pi), \
|
||||
mock.patch('bin.gen_release_notes.get_bug', get_bug):
|
||||
bugs = await gather_bugs('')
|
||||
|
||||
assert bugs == ['a', 'b', 'c']
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright © 2021-2021 Yonggang Luo
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
gen_help = """Generates visual studio module definition file."""
|
||||
|
||||
import argparse
|
||||
|
||||
"""
|
||||
For input template definition file
|
||||
For gcc/x64,gcc/arm64,visual studio
|
||||
`wglMakeCurrent@8 @357` => `wglMakeCurrent @357`
|
||||
`DrvCopyContext@12` => `DrvCopyContext`
|
||||
`stw_get_device` => `stw_get_device`
|
||||
For gcc/x86,gcc/arm
|
||||
`wglMakeCurrent@8 @357` => `wglMakeCurrent@8 @357 == wglMakeCurrent`
|
||||
`DrvCopyContext@12` => `DrvCopyContext@12 == DrvCopyContext`
|
||||
`stw_get_device` => `stw_get_device`
|
||||
|
||||
"""
|
||||
def gen_vs_module_def(in_file: str, out_file: str, compiler_abi: str, compiler_id: str, cpu_family: str) -> None:
|
||||
out_file_lines = ['EXPORTS']
|
||||
with open(in_file, 'r', encoding='utf-8') as f:
|
||||
lines = f.readlines()
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
tokens = line.split(';')
|
||||
if not tokens:
|
||||
continue
|
||||
def_infos = [x for x in tokens[0].split(' ') if len(x) > 0]
|
||||
if not def_infos:
|
||||
if line:
|
||||
out_file_lines.append('\t' + line)
|
||||
else:
|
||||
out_file_lines.append('')
|
||||
continue
|
||||
name_infos = def_infos[0].split('@')
|
||||
if not name_infos:
|
||||
out_file_lines.append('\t;' + line)
|
||||
continue
|
||||
order_info = '' if len(def_infos) <= 1 else def_infos[1]
|
||||
if def_infos[0] != name_infos[0] and \
|
||||
(compiler_abi == 'gcc' and compiler_id != 'clang') and (cpu_family not in {'x86_64', 'aarch64'}):
|
||||
if order_info:
|
||||
out_file_lines.append('\t' + def_infos[0] + ' ' + order_info + ' == ' + name_infos[0])
|
||||
else:
|
||||
out_file_lines.append('\t' + def_infos[0] + ' == ' + name_infos[0])
|
||||
else:
|
||||
if order_info:
|
||||
out_file_lines.append('\t' + name_infos[0] + ' ' + order_info)
|
||||
else:
|
||||
out_file_lines.append('\t' + name_infos[0])
|
||||
with open(out_file, 'wb') as f:
|
||||
out_file_content = '\n'.join(out_file_lines) + '\n'
|
||||
f.write(out_file_content.encode('utf-8'))
|
||||
'''
|
||||
python ./bin/gen_vs_module_defs.py --in_file src/gallium/targets/libgl-gdi/opengl32.def.in --out_file src/gallium/targets/libgl-gdi/opengl32.def --compiler_abi gcc --cpu_family x86_64
|
||||
python ./bin/gen_vs_module_defs.py --in_file src/gallium/targets/libgl-gdi/opengl32.def.in --out_file src/gallium/targets/libgl-gdi/opengl32.mingw.def --compiler_abi gcc --cpu_family x86
|
||||
|
||||
python ./bin/gen_vs_module_defs.py --in_file src/gallium/targets/wgl/gallium_wgl.def.in --out_file src/gallium/targets/wgl/gallium_wgl.def --compiler_abi gcc --cpu_family x86_64
|
||||
python ./bin/gen_vs_module_defs.py --in_file src/gallium/targets/wgl/gallium_wgl.def.in --out_file src/gallium/targets/wgl/gallium_wgl.mingw.def --compiler_abi gcc --cpu_family x86
|
||||
|
||||
python ./bin/gen_vs_module_defs.py --in_file src/egl/main/egl.def.in --out_file src/egl/main/egl.def --compiler_abi gcc --cpu_family x86_64
|
||||
python ./bin/gen_vs_module_defs.py --in_file src/egl/main/egl.def.in --out_file src/egl/main/egl.mingw.def --compiler_abi gcc --cpu_family x86
|
||||
|
||||
python ./bin/gen_vs_module_defs.py --in_file src/gallium/targets/lavapipe/vulkan_lvp.def.in --out_file src/gallium/targets/lavapipe/vulkan_lvp.def --compiler_abi gcc --cpu_family x86_64
|
||||
python ./bin/gen_vs_module_defs.py --in_file src/gallium/targets/lavapipe/vulkan_lvp.def.in --out_file src/gallium/targets/lavapipe/vulkan_lvp.mingw.def --compiler_abi gcc --cpu_family x86
|
||||
|
||||
'''
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=gen_help)
|
||||
parser.add_argument('--in_file', help='input template module definition file')
|
||||
parser.add_argument('--out_file', help='output module definition file')
|
||||
parser.add_argument('--compiler_abi', help='compiler abi')
|
||||
parser.add_argument('--compiler_id', help='compiler id')
|
||||
parser.add_argument('--cpu_family', help='cpu family')
|
||||
args = parser.parse_args()
|
||||
# print(args)
|
||||
gen_vs_module_def(args.in_file, args.out_file, args.compiler_abi, args.compiler_id, args.cpu_family)
|
||||
@@ -0,0 +1,50 @@
|
||||
"""
|
||||
Generate the contents of the git_sha1.h file.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import os.path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def get_git_sha1():
|
||||
"""Try to get the git SHA1 with git rev-parse."""
|
||||
git_dir = os.path.join(os.path.dirname(sys.argv[0]), '..', '.git')
|
||||
try:
|
||||
git_sha1 = subprocess.check_output([
|
||||
'git',
|
||||
'--git-dir=' + git_dir,
|
||||
'rev-parse',
|
||||
'HEAD',
|
||||
], stderr=open(os.devnull, 'w')).decode("ascii")
|
||||
except Exception:
|
||||
# don't print anything if it fails
|
||||
git_sha1 = ''
|
||||
return git_sha1
|
||||
|
||||
|
||||
def write_if_different(contents):
|
||||
"""
|
||||
Avoid touching the output file if it doesn't need modifications
|
||||
Useful to avoid triggering rebuilds when nothing has changed.
|
||||
"""
|
||||
if os.path.isfile(args.output):
|
||||
with open(args.output, 'r') as file:
|
||||
if file.read() == contents:
|
||||
return
|
||||
with open(args.output, 'w') as file:
|
||||
file.write(contents)
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--output', help='File to write the #define in',
|
||||
required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
git_sha1 = os.environ.get('MESA_GIT_SHA1_OVERRIDE', get_git_sha1())[:10]
|
||||
if git_sha1:
|
||||
write_if_different('#define MESA_GIT_SHA1 " (git-' + git_sha1 + ')"')
|
||||
else:
|
||||
write_if_different('#define MESA_GIT_SHA1 ""')
|
||||
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python3
|
||||
# encoding=utf-8
|
||||
# Copyright 2017-2018 Intel Corporation
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
"""Script to install megadriver symlinks for meson."""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
|
||||
|
||||
def resolve_libdir(libdir):
|
||||
if os.path.isabs(libdir):
|
||||
destdir = os.environ.get('DESTDIR')
|
||||
if destdir:
|
||||
return os.path.join(destdir, libdir[1:])
|
||||
else:
|
||||
return libdir
|
||||
return os.path.join(os.environ['MESON_INSTALL_DESTDIR_PREFIX'], libdir)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('megadriver')
|
||||
parser.add_argument('libdir')
|
||||
parser.add_argument('drivers', nargs='+')
|
||||
parser.add_argument('--megadriver-libdir')
|
||||
parser.add_argument('--libname-suffix', required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
# Not neccesarily at the end, there might be a version suffix, but let's
|
||||
# make sure that the same suffix is in the megadriver lib name.
|
||||
assert '.' + args.libname_suffix in args.megadriver
|
||||
|
||||
to = resolve_libdir(args.libdir)
|
||||
if args.megadriver_libdir:
|
||||
md_to = resolve_libdir(args.megadriver_libdir)
|
||||
else:
|
||||
md_to = to
|
||||
|
||||
basename = os.path.basename(args.megadriver)
|
||||
master = os.path.join(to, basename)
|
||||
|
||||
if not os.path.exists(to):
|
||||
if os.path.lexists(to):
|
||||
os.unlink(to)
|
||||
os.makedirs(to)
|
||||
|
||||
for driver in args.drivers:
|
||||
abs_driver = os.path.join(to, driver)
|
||||
|
||||
if os.path.lexists(abs_driver):
|
||||
os.unlink(abs_driver)
|
||||
|
||||
symlink = os.path.relpath(os.path.join(md_to, basename), start=to)
|
||||
|
||||
print(f'Installing symlink pointing to {symlink} to {abs_driver}')
|
||||
os.symlink(symlink, abs_driver)
|
||||
|
||||
try:
|
||||
ret = os.getcwd()
|
||||
os.chdir(to)
|
||||
|
||||
name, ext = os.path.splitext(driver)
|
||||
while ext != '.' + args.libname_suffix:
|
||||
if os.path.lexists(name):
|
||||
os.unlink(name)
|
||||
os.symlink(driver, name)
|
||||
name, ext = os.path.splitext(name)
|
||||
finally:
|
||||
os.chdir(ret)
|
||||
|
||||
# Remove meson-created symlinks
|
||||
name, ext = os.path.splitext(master)
|
||||
while ext != '.' + args.libname_suffix:
|
||||
if os.path.lexists(name):
|
||||
os.unlink(name)
|
||||
name, ext = os.path.splitext(name)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+286
@@ -0,0 +1,286 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import pathlib
|
||||
import requests
|
||||
import subprocess
|
||||
import typing
|
||||
|
||||
|
||||
def error(msg: str) -> None:
|
||||
print('\033[31m' + msg + '\033[0m')
|
||||
|
||||
|
||||
class Source:
|
||||
def __init__(self, filename: str, url: typing.Optional[str],
|
||||
template: typing.Optional[str] = None, remove:
|
||||
typing.Optional[str] = None):
|
||||
self.file = pathlib.Path(filename)
|
||||
self.url = url
|
||||
self.template = template
|
||||
self.remove = remove
|
||||
|
||||
def sync(self) -> None:
|
||||
if self.url is None:
|
||||
return
|
||||
|
||||
print('Syncing {}...'.format(self.file), end=' ', flush=True)
|
||||
req = requests.get(self.url)
|
||||
|
||||
if not req.ok:
|
||||
error('Failed to retrieve file: {} {}'.format(req.status_code, req.reason))
|
||||
return
|
||||
|
||||
# Gitiles returns base64-encoded strings.
|
||||
# Google has been resisting for years to the idea of allowing plain text: https://github.com/google/gitiles/issues/7
|
||||
if 'format=TEXT' in self.url:
|
||||
content = base64.b64decode(req.content)
|
||||
else:
|
||||
content = req.content
|
||||
|
||||
content = str(content, encoding='utf-8')
|
||||
if self.remove is not None:
|
||||
content = content.replace(self.remove, '')
|
||||
if self.template is not None:
|
||||
content = self.template % content
|
||||
|
||||
with open(self.file, 'w') as f:
|
||||
f.write(content)
|
||||
|
||||
print('Done')
|
||||
|
||||
|
||||
VK_ANDROID_NATIVE_BUFFER_TEMPLATE = """\
|
||||
/*
|
||||
* MESA: buffer_handle_t is defined by all Mesa builds, even if
|
||||
* one is building for a non-Android target. This avoids unnecessary
|
||||
* conditionals in driver code.
|
||||
*
|
||||
* We don't need to define buffer_handle_t locally when (__ANDROID__)
|
||||
* or ANDROID are set. Here's the distinction between the two:
|
||||
*
|
||||
* - AOSP always defines ANDROID, since it just means one is using the
|
||||
* AOSP tree. It means the build environment is Android, roughly.
|
||||
* - __ANDROID__ is defined by the toolchain. This typically means the
|
||||
* build target is Android.
|
||||
*
|
||||
* If the build environment is Android, AOSP can provide common Android
|
||||
* headers, such as <cutils/native_handle.h>. This allows one to build
|
||||
* and test certain aspects of Android window system code, on the host
|
||||
* system rather the build target.
|
||||
*/
|
||||
|
||||
#if defined(__ANDROID__) || defined(ANDROID)
|
||||
|
||||
#include <cutils/native_handle.h>
|
||||
#if ANDROID_API_LEVEL < 28
|
||||
/* buffer_handle_t was defined in the deprecated system/window.h */
|
||||
typedef const native_handle_t *buffer_handle_t;
|
||||
#endif
|
||||
|
||||
#else
|
||||
|
||||
typedef void *buffer_handle_t;
|
||||
|
||||
#endif
|
||||
|
||||
%s\
|
||||
"""
|
||||
|
||||
# a URL of `None` means there is no upstream, because *we* are the upstream
|
||||
SOURCES = [
|
||||
{
|
||||
'api': 'khr',
|
||||
'inc_folder': 'KHR',
|
||||
'sources': [
|
||||
Source('include/KHR/khrplatform.h', 'https://github.com/KhronosGroup/EGL-Registry/raw/main/api/KHR/khrplatform.h'),
|
||||
],
|
||||
},
|
||||
|
||||
{
|
||||
'api': 'egl',
|
||||
'inc_folder': 'EGL',
|
||||
'sources': [
|
||||
Source('src/egl/generate/egl.xml', 'https://github.com/KhronosGroup/EGL-Registry/raw/main/api/egl.xml'),
|
||||
Source('include/EGL/egl.h', 'https://github.com/KhronosGroup/EGL-Registry/raw/main/api/EGL/egl.h'),
|
||||
Source('include/EGL/eglplatform.h', 'https://github.com/KhronosGroup/EGL-Registry/raw/main/api/EGL/eglplatform.h'),
|
||||
Source('include/EGL/eglext.h', 'https://github.com/KhronosGroup/EGL-Registry/raw/main/api/EGL/eglext.h'),
|
||||
Source('include/EGL/eglext_angle.h', 'https://chromium.googlesource.com/angle/angle/+/refs/heads/main/include/EGL/eglext_angle.h?format=TEXT'),
|
||||
Source('include/EGL/eglmesaext.h', None),
|
||||
],
|
||||
},
|
||||
|
||||
{
|
||||
'api': 'gl',
|
||||
'inc_folder': 'GL',
|
||||
'sources': [
|
||||
Source('src/mesa/glapi/glapi/registry/gl.xml', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/xml/gl.xml'),
|
||||
Source('include/GL/glcorearb.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GL/glcorearb.h'),
|
||||
Source('include/GL/glext.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GL/glext.h'),
|
||||
Source('include/GL/glxext.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GL/glxext.h'),
|
||||
Source('include/GL/wglext.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GL/wglext.h'),
|
||||
Source('include/GL/gl.h', None), # FIXME: I don't know what the canonical source is
|
||||
Source('include/GL/glx.h', None), # FIXME: I don't know what the canonical source is
|
||||
Source('include/GL/internal/', None),
|
||||
Source('include/GL/mesa_glinterop.h', None),
|
||||
],
|
||||
},
|
||||
|
||||
{
|
||||
'api': 'gles1',
|
||||
'inc_folder': 'GLES',
|
||||
'sources': [
|
||||
Source('include/GLES/gl.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GLES/gl.h'),
|
||||
Source('include/GLES/glplatform.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GLES/glplatform.h'),
|
||||
Source('include/GLES/glext.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GLES/glext.h'),
|
||||
Source('include/GLES/egl.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GLES/egl.h'),
|
||||
],
|
||||
},
|
||||
|
||||
{
|
||||
'api': 'gles2',
|
||||
'inc_folder': 'GLES2',
|
||||
'sources': [
|
||||
Source('include/GLES2/gl2.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GLES2/gl2.h'),
|
||||
Source('include/GLES2/gl2platform.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GLES2/gl2platform.h'),
|
||||
Source('include/GLES2/gl2ext.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GLES2/gl2ext.h'),
|
||||
],
|
||||
},
|
||||
|
||||
{
|
||||
'api': 'gles3',
|
||||
'inc_folder': 'GLES3',
|
||||
'sources': [
|
||||
Source('include/GLES3/gl3.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GLES3/gl3.h'),
|
||||
Source('include/GLES3/gl31.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GLES3/gl31.h'),
|
||||
Source('include/GLES3/gl32.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GLES3/gl32.h'),
|
||||
Source('include/GLES3/gl3platform.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GLES3/gl3platform.h'),
|
||||
Source('include/GLES3/gl3ext.h', None), # FIXME: I don't know what the canonical source is
|
||||
],
|
||||
},
|
||||
|
||||
{
|
||||
'api': 'opencl',
|
||||
'inc_folder': 'CL',
|
||||
'sources': [
|
||||
Source('include/CL/cl.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl.h'),
|
||||
Source('include/CL/cl_d3d10.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_d3d10.h'),
|
||||
Source('include/CL/cl_d3d11.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_d3d11.h'),
|
||||
Source('include/CL/cl_dx9_media_sharing.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_dx9_media_sharing.h'),
|
||||
Source('include/CL/cl_dx9_media_sharing_intel.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_dx9_media_sharing_intel.h'),
|
||||
Source('include/CL/cl_egl.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_egl.h'),
|
||||
Source('include/CL/cl_ext.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_ext.h'),
|
||||
Source('include/CL/cl_ext_intel.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_ext_intel.h'),
|
||||
Source('include/CL/cl_function_types.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_function_types.h'),
|
||||
Source('include/CL/cl_gl.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_gl.h'),
|
||||
Source('include/CL/cl_gl_ext.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_gl_ext.h'),
|
||||
Source('include/CL/cl_half.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_half.h'),
|
||||
Source('include/CL/cl_icd.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_icd.h'),
|
||||
Source('include/CL/cl_layer.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_layer.h'),
|
||||
Source('include/CL/cl_platform.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_platform.h'),
|
||||
Source('include/CL/cl_va_api_media_sharing_intel.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_va_api_media_sharing_intel.h'),
|
||||
Source('include/CL/cl_version.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_version.h'),
|
||||
Source('include/CL/opencl.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/opencl.h'),
|
||||
],
|
||||
},
|
||||
|
||||
{
|
||||
'api': 'opencl_intel',
|
||||
'inc_folder': 'CL_intel',
|
||||
'sources': [
|
||||
Source('include/CL_intel/cl_ext_private.h', 'https://github.com/intel/compute-runtime/raw/master/opencl/extensions/public/cl_ext_private.h'),
|
||||
],
|
||||
},
|
||||
|
||||
{
|
||||
'api': 'spirv',
|
||||
'sources': [
|
||||
Source('src/compiler/spirv/NonSemanticShaderDebugInfo100.h', 'https://github.com/KhronosGroup/SPIRV-Headers/raw/main/include/spirv/unified1/NonSemanticShaderDebugInfo100.h'),
|
||||
Source('src/compiler/spirv/spirv.h', 'https://github.com/KhronosGroup/SPIRV-Headers/raw/main/include/spirv/unified1/spirv.h'),
|
||||
Source('src/compiler/spirv/spirv.core.grammar.json', 'https://github.com/KhronosGroup/SPIRV-Headers/raw/main/include/spirv/unified1/spirv.core.grammar.json'),
|
||||
Source('src/compiler/spirv/OpenCL.std.h', 'https://github.com/KhronosGroup/SPIRV-Headers/raw/main/include/spirv/unified1/OpenCL.std.h'),
|
||||
Source('src/compiler/spirv/GLSL.std.450.h', 'https://github.com/KhronosGroup/SPIRV-Headers/raw/main/include/spirv/unified1/GLSL.std.450.h'),
|
||||
Source('src/compiler/spirv/GLSL.ext.AMD.h', 'https://github.com/KhronosGroup/glslang/raw/main/SPIRV/GLSL.ext.AMD.h'), # FIXME: is this the canonical source?
|
||||
],
|
||||
},
|
||||
|
||||
{
|
||||
'api': 'vulkan',
|
||||
'inc_folder': 'vulkan',
|
||||
'sources': [
|
||||
Source('src/vulkan/registry/vk.xml', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/registry/vk.xml'),
|
||||
Source('include/vulkan/vulkan.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan.h'),
|
||||
Source('include/vulkan/vulkan_core.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_core.h'),
|
||||
Source('include/vulkan/vulkan_beta.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_beta.h'),
|
||||
Source('include/vulkan/vk_icd.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vk_icd.h'),
|
||||
Source('include/vulkan/vk_layer.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vk_layer.h'),
|
||||
Source('include/vulkan/vk_platform.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vk_platform.h'),
|
||||
Source('include/vulkan/vulkan_android.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_android.h'),
|
||||
Source('include/vulkan/vulkan_directfb.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_directfb.h'),
|
||||
Source('include/vulkan/vulkan_fuchsia.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_fuchsia.h'),
|
||||
Source('include/vulkan/vulkan_ggp.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_ggp.h'),
|
||||
Source('include/vulkan/vulkan_ios.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_ios.h'),
|
||||
Source('include/vulkan/vulkan_macos.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_macos.h'),
|
||||
Source('include/vulkan/vulkan_metal.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_metal.h'),
|
||||
Source('include/vulkan/vulkan_ohos.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_ohos.h'),
|
||||
Source('include/vulkan/vulkan_screen.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_screen.h'),
|
||||
Source('include/vulkan/vulkan_vi.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_vi.h'),
|
||||
Source('include/vulkan/vulkan_wayland.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_wayland.h'),
|
||||
Source('include/vulkan/vulkan_win32.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_win32.h'),
|
||||
Source('include/vulkan/vulkan_xcb.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_xcb.h'),
|
||||
Source('include/vulkan/vulkan_xlib.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_xlib.h'),
|
||||
Source('include/vulkan/vulkan_xlib_xrandr.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_xlib_xrandr.h'),
|
||||
Source('include/vulkan/vk_android_native_buffer.h', 'https://android.googlesource.com/platform/frameworks/native/+/master/vulkan/include/vulkan/vk_android_native_buffer.h?format=TEXT',
|
||||
template=VK_ANDROID_NATIVE_BUFFER_TEMPLATE, remove='#include <cutils/native_handle.h>\n'),
|
||||
Source('include/vk_video/vulkan_video_codec_av1std.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codec_av1std.h'),
|
||||
Source('include/vk_video/vulkan_video_codec_av1std_decode.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codec_av1std_decode.h'),
|
||||
Source('include/vk_video/vulkan_video_codec_av1std_encode.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codec_av1std_encode.h'),
|
||||
Source('include/vk_video/vulkan_video_codec_h264std.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codec_h264std.h'),
|
||||
Source('include/vk_video/vulkan_video_codec_h264std_decode.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codec_h264std_decode.h'),
|
||||
Source('include/vk_video/vulkan_video_codec_h264std_encode.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codec_h264std_encode.h'),
|
||||
Source('include/vk_video/vulkan_video_codec_h265std.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codec_h265std.h'),
|
||||
Source('include/vk_video/vulkan_video_codec_h265std_decode.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codec_h265std_decode.h'),
|
||||
Source('include/vk_video/vulkan_video_codec_h265std_encode.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codec_h265std_encode.h'),
|
||||
Source('include/vk_video/vulkan_video_codec_vp9std.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codec_vp9std.h'),
|
||||
Source('include/vk_video/vulkan_video_codec_vp9std_decode.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codec_vp9std_decode.h'),
|
||||
Source('include/vk_video/vulkan_video_codecs_common.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codecs_common.h'),
|
||||
Source('include/vulkan/.editorconfig', None),
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
git_toplevel = subprocess.check_output(['git', 'rev-parse', '--show-toplevel'],
|
||||
stderr=subprocess.DEVNULL).decode("ascii").strip()
|
||||
if not pathlib.Path(git_toplevel).resolve() == pathlib.Path('.').resolve():
|
||||
error('Please run this script from the root folder ({})'.format(git_toplevel))
|
||||
exit(1)
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('apis', nargs='*',
|
||||
# the `[[]]` here is a workaround for python bug 9625
|
||||
# where having `choices` breaks `nargs='*'`:
|
||||
# https://bugs.python.org/issue9625
|
||||
choices=[group['api'] for group in SOURCES] + [[]],
|
||||
help='Only update the APIs specified.')
|
||||
args = parser.parse_args()
|
||||
|
||||
# These APIs all depend on the KHR header
|
||||
depend_on_khr = set(['egl', 'gl', 'gles', 'gles2', 'gles3'])
|
||||
if args.apis and 'khr' not in args.apis and depend_on_khr.intersection(set(args.apis)):
|
||||
args.apis = ['khr'] + args.apis
|
||||
|
||||
for group in SOURCES:
|
||||
if args.apis and group['api'] not in args.apis:
|
||||
continue
|
||||
|
||||
for source in group['sources']:
|
||||
source.sync()
|
||||
|
||||
# Make sure all the API files are handled by this script
|
||||
if 'inc_folder' in group:
|
||||
for file in pathlib.Path('include/' + group['inc_folder']).iterdir():
|
||||
if file not in [source.file for source in group['sources']]:
|
||||
error('{} is unknown, please add it to SOURCES'.format(file))
|
||||
Executable
+88
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright © 2019 Intel Corporation
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
"""This script reads a meson build directory and gives back the command line it
|
||||
was configured with.
|
||||
|
||||
This only works for meson 0.49.0 and newer.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import configparser
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
"""Parse arguments."""
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
'build_dir',
|
||||
help='Path the meson build directory')
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
|
||||
def load_config(path: pathlib.Path) -> configparser.ConfigParser:
|
||||
"""Load config file."""
|
||||
conf = configparser.ConfigParser()
|
||||
with path.open() as f:
|
||||
conf.read_file(f)
|
||||
return conf
|
||||
|
||||
|
||||
def build_cmd(conf: configparser.ConfigParser) -> str:
|
||||
"""Rebuild the command line."""
|
||||
args = []
|
||||
for k, v in conf['options'].items():
|
||||
if ' ' in v:
|
||||
args.append(f'-D{k}="{v}"')
|
||||
else:
|
||||
args.append(f'-D{k}={v}')
|
||||
|
||||
cf = conf['properties'].get('cross_file')
|
||||
if cf:
|
||||
args.append('--cross-file={}'.format(cf))
|
||||
nf = conf['properties'].get('native_file')
|
||||
if nf:
|
||||
# this will be in the form "['str', 'str']", so use ast.literal_eval to
|
||||
# convert it to a list of strings.
|
||||
nf = ast.literal_eval(nf)
|
||||
args.extend(['--native-file={}'.format(f) for f in nf])
|
||||
return ' '.join(args)
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
path = pathlib.Path(args.build_dir, 'meson-private', 'cmd_line.txt')
|
||||
if not path.exists():
|
||||
print('Cannot find the necessary file to rebuild command line. '
|
||||
'Is your meson version >= 0.49.0?', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
conf = load_config(path)
|
||||
cmd = build_cmd(conf)
|
||||
print(cmd)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,17 @@
|
||||
# Copyright © 2017 Eric Engestrom
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
git_sha1_gen_py = files('git_sha1_gen.py')
|
||||
gen_vs_module_defs_py = files('gen_vs_module_defs.py')
|
||||
gen_vs_module_defs_normal_command = [
|
||||
prog_python, gen_vs_module_defs_py,
|
||||
'--in_file', '@INPUT@', '--out_file', '@OUTPUT@',
|
||||
'--compiler_abi', cc.get_argument_syntax(),
|
||||
'--compiler_id', cc.get_id(), '--cpu_family', host_machine.cpu_family()
|
||||
]
|
||||
symbols_check = find_program('symbols-check.py')
|
||||
install_megadrivers_py = find_program('install_megadrivers.py')
|
||||
install_megadrivers = [
|
||||
install_megadrivers_py.full_path(),
|
||||
'--libname-suffix', libname_suffix,
|
||||
]
|
||||
Executable
+182
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright © 2024 Valve Corporation
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
import argparse
|
||||
import collections
|
||||
import subprocess
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
|
||||
def trim_blank_lines(string, trailing):
|
||||
lines = string.split('\n')
|
||||
string = ''
|
||||
empty_line = True
|
||||
for i in range(len(lines)):
|
||||
line_index = len(lines) - 1 - i if trailing else i
|
||||
if empty_line and lines[line_index].strip() == '':
|
||||
continue
|
||||
|
||||
write_newline = not empty_line if trailing else line_index < len(lines) - 1
|
||||
newline = '\n' if write_newline else ''
|
||||
if trailing:
|
||||
string = lines[line_index] + newline + string
|
||||
else:
|
||||
string += lines[line_index] + newline
|
||||
|
||||
empty_line = False
|
||||
|
||||
return string
|
||||
|
||||
class TestFileChange:
|
||||
def __init__(self, expected, result):
|
||||
self.expected = expected
|
||||
|
||||
# Apply the indentation of the expectation to the result
|
||||
indentation = 1000
|
||||
for expected_line in expected.split('\n'):
|
||||
if match := re.match(r'^(\s*)\S', expected_line):
|
||||
line_indentation = len(match.group(1))
|
||||
if indentation > line_indentation:
|
||||
indentation = line_indentation
|
||||
|
||||
self.result = ''
|
||||
result = result.split('\n')
|
||||
for i in range(len(result)):
|
||||
result_line = result[i]
|
||||
indentation_str = '' if result_line.strip() == '' else ' '*indentation
|
||||
self.result += indentation_str + result_line + ('\n' if i < len(result) - 1 else '')
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--build-dir', '-B', required=False)
|
||||
parser.add_argument('--test-filter', '-f', required=False)
|
||||
parser.add_argument('--update-all', '-u', action='store_true')
|
||||
args = parser.parse_args()
|
||||
|
||||
bin_path = 'src/compiler/nir/nir_tests'
|
||||
if args.build_dir:
|
||||
bin_path = args.build_dir + '/' + bin_path
|
||||
|
||||
if not os.path.isfile(bin_path):
|
||||
print(f'{bin_path} \033[91m does not exist!\033[0m')
|
||||
exit(1)
|
||||
|
||||
build_args = ['meson', 'compile']
|
||||
if args.build_dir:
|
||||
build_args.append(f'-C{args.build_dir}')
|
||||
subprocess.run(build_args)
|
||||
|
||||
test_args = [bin_path]
|
||||
if args.test_filter:
|
||||
test_args.append(f'--gtest_filter={args.test_filter}')
|
||||
|
||||
env = os.environ.copy()
|
||||
if args.update_all:
|
||||
env['NIR_TEST_DUMP_SHADERS'] = 'true'
|
||||
|
||||
output = subprocess.run(test_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, env=env)
|
||||
|
||||
expected_pattern = re.compile(r'BEGIN EXPECTED\(([\d\w\W/.-_]+)\)')
|
||||
|
||||
expectations = collections.defaultdict(list)
|
||||
|
||||
current_file = None
|
||||
current_result = None
|
||||
current_expected = None
|
||||
inside_result = False
|
||||
inside_expected = False
|
||||
|
||||
# Parse the output of the test binary and gather the changed shaders.
|
||||
for output_line in output.stdout.split('\n'):
|
||||
if output_line.startswith('BEGIN RESULT'):
|
||||
inside_result = True
|
||||
current_result = ''
|
||||
continue
|
||||
|
||||
if output_line.startswith('BEGIN EXPECTED'):
|
||||
match = expected_pattern.match(output_line)
|
||||
current_file = match.group(1).removeprefix('../')
|
||||
inside_expected = True
|
||||
current_expected = ''
|
||||
continue
|
||||
|
||||
if output_line.startswith('END'):
|
||||
if current_result is not None and current_expected is not None:
|
||||
# remove trailing and leading blank lines
|
||||
current_result = trim_blank_lines(current_result, True)
|
||||
current_result = trim_blank_lines(current_result, False)
|
||||
current_expected = trim_blank_lines(current_expected, True)
|
||||
current_expected = trim_blank_lines(current_expected, False)
|
||||
|
||||
expectations[current_file].append(TestFileChange(current_expected, current_result))
|
||||
|
||||
current_result = None
|
||||
current_expected = None
|
||||
|
||||
inside_result = False
|
||||
inside_expected = False
|
||||
continue
|
||||
|
||||
if inside_result:
|
||||
current_result += output_line + '\n'
|
||||
|
||||
if inside_expected:
|
||||
current_expected += output_line + '\n'
|
||||
|
||||
patches = []
|
||||
|
||||
# Generate patches for the changed shaders.
|
||||
for file in expectations:
|
||||
changes = expectations[file]
|
||||
|
||||
updated_test_file = ''
|
||||
|
||||
with open(file) as test_file:
|
||||
updated_test_file = str(test_file.read())
|
||||
|
||||
for change in changes:
|
||||
updated_test_file = updated_test_file.replace(change.expected, change.result)
|
||||
|
||||
# change.expected == change.result can be the case when using NIR_TEST_DUMP_SHADERS.
|
||||
if change.expected in updated_test_file and change.expected != change.result:
|
||||
print(f'Duplicate test case in {file}!')
|
||||
exit(1)
|
||||
|
||||
with tempfile.NamedTemporaryFile(delete_on_close=False) as tmp:
|
||||
tmp.write(bytes(updated_test_file, encoding="utf-8"))
|
||||
tmp.close()
|
||||
|
||||
diff = subprocess.run(
|
||||
['git', 'diff', '--no-index', file, tmp.name],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True,
|
||||
)
|
||||
patch = diff.stdout.replace(tmp.name, '/' + file)
|
||||
|
||||
print(patch)
|
||||
|
||||
patches.append(patch)
|
||||
|
||||
if len(patches) != 0:
|
||||
sys.stdout.write('\033[96mApply the changes listed above?\033[0m [Y/n]')
|
||||
response = None
|
||||
try:
|
||||
response = input()
|
||||
except KeyboardInterrupt:
|
||||
print()
|
||||
sys.exit(1)
|
||||
|
||||
if response in ['', 'y', 'Y']:
|
||||
for patch in patches:
|
||||
apply = subprocess.Popen(
|
||||
['git', 'apply', '--allow-empty'],
|
||||
stdin=subprocess.PIPE,
|
||||
)
|
||||
apply.communicate(input=bytes(patch, encoding="utf-8"))
|
||||
+140
@@ -0,0 +1,140 @@
|
||||
Aaron Ruby, [email protected], aruby
|
||||
Adam Jackson, [email protected], ajax
|
||||
Alejandro Piñeiro, [email protected], apinheiro
|
||||
Aleksi Sapon, [email protected], DDoSQc
|
||||
Alyssa Rosenzweig, [email protected], alyssa
|
||||
Andrew Wolfers, [email protected], aswolfers
|
||||
Antonio Ospite, [email protected], ao2
|
||||
Benjamin Cheng, [email protected], bcheng
|
||||
Benjamin ROBIN, [email protected], benjarobin
|
||||
Boris Brezillon, [email protected], bbrezillon
|
||||
Brian Paul, [email protected], brianp
|
||||
Caio Oliveira, [email protected], cmarcelo
|
||||
Caleb Callaway, [email protected], ccallawa
|
||||
Casey Bowman, [email protected], cgbowman
|
||||
Caterina Shablia, [email protected], nanokatze
|
||||
Chia-I Wu, [email protected], olv
|
||||
Christian Gmeiner, [email protected], austriancoder
|
||||
Christopher Michael, [email protected], cpmichael
|
||||
Christoph Pillmayer, [email protected], chrpil
|
||||
Colin Marc, [email protected], colinmarc
|
||||
Connor Abbott, [email protected], cwabbott0
|
||||
Corentin Noël, [email protected], tintou
|
||||
Daniel Schürmann, [email protected], daniel-schuermann
|
||||
Daniel Stone, [email protected], daniels
|
||||
Danylo Piliaiev, [email protected], Danil
|
||||
Dave Airlie, [email protected], airlied
|
||||
David Heidelberg, [email protected], dh
|
||||
David Rosca, [email protected], nowrep
|
||||
David Tobolik, [email protected], dtobolik
|
||||
Deborah Brouwer, [email protected], dbrouwer
|
||||
Detlev Casanova, [email protected], detlev.c
|
||||
Dmitry Baryshkov, [email protected], lumag
|
||||
Dudemanguy, [email protected], Dudemanguy
|
||||
duncan.hopkins, [email protected], duncan.hopkins
|
||||
Dylan Baker, [email protected], dbaker
|
||||
Emma Anholt, [email protected], anholt
|
||||
Emmanuel Gil Peyrot, [email protected], linkmauve
|
||||
Eric Engestrom, [email protected], eric
|
||||
Erico Nunes, [email protected], enunes
|
||||
Eric R. Smith, [email protected], ericsmith
|
||||
Erik Faye-Lund, [email protected], kusma
|
||||
Ernst Persson, [email protected], ernstp
|
||||
Faith Ekstrand, [email protected], gfxstrand
|
||||
Felix DeGrood, [email protected], fjdegroo
|
||||
Feng Jiang, [email protected], flynnjiang
|
||||
Francisco Jerez, [email protected], currojerez
|
||||
Georg Lehmann, [email protected], DadSchoorse
|
||||
Guilherme Gallo, [email protected], gallo
|
||||
Gurchetan Singh, [email protected], gurchetansingh
|
||||
Hans-Kristian Arntzen, [email protected], themaister
|
||||
Hyunjun Ko, [email protected], zzoon
|
||||
Iago Toral Quiroga, [email protected], itoral
|
||||
Ian Romanick, [email protected], idr
|
||||
Icenowy Zheng, [email protected], icenowy
|
||||
Igor Torrente, [email protected], igor.torrente
|
||||
Ivan Avdeev, [email protected], provod
|
||||
Iván Briano, [email protected], ibriano
|
||||
James Hogan, [email protected], jameshogan
|
||||
Janne Grunau, [email protected], jannau
|
||||
Jason Macnak, [email protected], jmacnak
|
||||
Jesse Natalie, [email protected], jenatali
|
||||
Job Noorman, [email protected], jnoorman
|
||||
John Anthony, [email protected], jna
|
||||
Jordan Justen, [email protected], jljusten
|
||||
Jose Maria Casanova Crespo, [email protected], chema
|
||||
José Roberto de Souza, [email protected], zehortigoza
|
||||
Juan A. Suarez Romero, [email protected], jasuarez
|
||||
Julia Zhang, [email protected], Julia
|
||||
Jung-uk Kim, [email protected], jkim
|
||||
Juston Li, [email protected], justonli
|
||||
Karmjit Mahil, [email protected], CreativeCylon
|
||||
Karol Herbst, [email protected], karolherbst
|
||||
Kenneth Graunke, [email protected], kwg
|
||||
Kevin Chuang, [email protected], kaiwenjon
|
||||
Konstantin Seurer, [email protected], KonstantinSeurer
|
||||
Lars-Ivar Hesselberg Simonsen, [email protected], larsivsi
|
||||
lcagustini, [email protected], lcagustini
|
||||
Leonard Göhrs, [email protected], hnez
|
||||
Lionel Landwerlin, [email protected], llandwerlin
|
||||
liuqiang, [email protected], genhoayi
|
||||
Lorenzo Rossi, [email protected], SnowyCoder
|
||||
Lucas De Marchi, [email protected], demarchi
|
||||
Lucas Stach, [email protected], lynxeye
|
||||
Lukas Lipp, [email protected], fknfilewalker
|
||||
Maíra Canal, [email protected], mairacanal
|
||||
Manuel Dun, [email protected], manueldun
|
||||
Marc Herbert, [email protected], marc-hb
|
||||
Marek Olšák, [email protected], mareko
|
||||
Mark Collins, [email protected], PixelyIon
|
||||
Martin Krastev, [email protected], blu
|
||||
Martin Roukala (né Peres), [email protected], mupuf
|
||||
Mary Guillemard, [email protected], marysaka
|
||||
Matthew Brost, [email protected], mbrost
|
||||
Matt Turner, [email protected], mattst88
|
||||
Mauro Rossi, [email protected], issor.oruam
|
||||
Mel Henning, [email protected], mhenning
|
||||
Michael Cheng, [email protected], michaelonchrome
|
||||
Michel Dänzer, [email protected], daenzer
|
||||
Mike Blumenkrantz, [email protected], zmike
|
||||
Mohamed Ahmed, [email protected], mohamexiety
|
||||
Nanley Chery, [email protected], nchery
|
||||
Natalie Vock, [email protected], pixelcluster
|
||||
Olivia Lee, [email protected], olivia
|
||||
Patrick Lerda, [email protected], noblock
|
||||
Paulo Zanoni, [email protected], pzanoni
|
||||
Pavel Ondračka, [email protected], ondracka
|
||||
Peyton Lee, [email protected], peytolee
|
||||
Pierre-Eric Pelloux-Prayer, [email protected], pepp
|
||||
Pohsiang (John) Hsu, [email protected], pohhsu_microsoft
|
||||
Qiang Yu, [email protected], yuq825
|
||||
Rhys Perry, [email protected], pendingchaos
|
||||
Rob Clark, [email protected], robclark
|
||||
Roland Scheidegger, [email protected], sroland
|
||||
Ruijing Dong, [email protected], rdong
|
||||
Ryan Mckeever, [email protected], rmckeever
|
||||
Sagar Ghuge, [email protected], sagarghuge
|
||||
Samuel Pitoiset, [email protected], hakzsam
|
||||
Saroj Kumar, [email protected], sarojk
|
||||
Sathishkumar S, [email protected], sathish
|
||||
Serdar Kocdemir, [email protected], kocdemir
|
||||
Sergi Blanch Torne, [email protected], sergi
|
||||
Sil Vilerino, [email protected], sivileri
|
||||
Simon Ser, [email protected], emersion
|
||||
Sonny Jiang, [email protected], sonny
|
||||
Stéphane Cerveau, [email protected], dabrain34
|
||||
Tapani Pälli, [email protected], tpalli
|
||||
Thomas H.P. Andersen, [email protected], phomes
|
||||
Tim Keller, tjkeller.xyz, tjk
|
||||
Timothy Arceri, [email protected], tarceri
|
||||
Timur Kristóf, [email protected], Venemo
|
||||
Valentine Burley, [email protected], Valentine
|
||||
Vasily Khoruzhick, [email protected], anarsoul
|
||||
Vignesh Raman, [email protected], vigneshraman
|
||||
Vinson Lee, [email protected], vlee
|
||||
Xaver Hugl, [email protected], Zamundaaa
|
||||
Yiwei Zhang, [email protected], zzyiwei
|
||||
Yogesh Mohan Marimuthu, [email protected], yogeshmohan
|
||||
Yonggang Luo, [email protected], lygstate
|
||||
Zan Dobersek, [email protected], zdobersek
|
||||
Zoltán Böszörményi, [email protected], zboszor
|
||||
|
Executable
+251
@@ -0,0 +1,251 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# Copyright 2012 VMware Inc
|
||||
# Copyright 2008-2009 Jose Fonseca
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
# THE SOFTWARE.
|
||||
#
|
||||
|
||||
"""Perf annotate for JIT code.
|
||||
|
||||
Linux `perf annotate` does not work with JIT code. This script takes the data
|
||||
produced by `perf script` command, plus the diassemblies outputted by gallivm
|
||||
into /tmp/perf-XXXXX.map.asm and produces output similar to `perf annotate`.
|
||||
|
||||
See docs/llvmpipe.rst for usage instructions.
|
||||
|
||||
The `perf script` output parser was derived from the gprof2dot.py script.
|
||||
"""
|
||||
|
||||
|
||||
import sys
|
||||
import os.path
|
||||
import re
|
||||
import optparse
|
||||
import subprocess
|
||||
|
||||
|
||||
class Parser:
|
||||
"""Parser interface."""
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def parse(self):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class LineParser(Parser):
|
||||
"""Base class for parsers that read line-based formats."""
|
||||
|
||||
def __init__(self, file):
|
||||
Parser.__init__(self)
|
||||
self._file = file
|
||||
self.__line = None
|
||||
self.__eof = False
|
||||
self.line_no = 0
|
||||
|
||||
def readline(self):
|
||||
line = self._file.readline()
|
||||
if not line:
|
||||
self.__line = ''
|
||||
self.__eof = True
|
||||
else:
|
||||
self.line_no += 1
|
||||
self.__line = line.rstrip('\r\n')
|
||||
|
||||
def lookahead(self):
|
||||
assert self.__line is not None
|
||||
return self.__line
|
||||
|
||||
def consume(self):
|
||||
assert self.__line is not None
|
||||
line = self.__line
|
||||
self.readline()
|
||||
return line
|
||||
|
||||
def eof(self):
|
||||
assert self.__line is not None
|
||||
return self.__eof
|
||||
|
||||
|
||||
mapFile = None
|
||||
|
||||
def lookupMap(filename, matchSymbol):
|
||||
global mapFile
|
||||
mapFile = filename
|
||||
stream = open(filename, 'rt')
|
||||
for line in stream:
|
||||
start, length, symbol = line.split()
|
||||
|
||||
start = int(start, 16)
|
||||
length = int(length,16)
|
||||
|
||||
if symbol == matchSymbol:
|
||||
return start
|
||||
|
||||
return None
|
||||
|
||||
def lookupAsm(filename, desiredFunction):
|
||||
stream = open(filename + '.asm', 'rt')
|
||||
while not stream.readline().startswith(desiredFunction + ' '):
|
||||
pass
|
||||
|
||||
asm = []
|
||||
line = stream.readline().strip()
|
||||
while line:
|
||||
addr, instr = line.split(':', 1)
|
||||
addr = int(addr)
|
||||
asm.append((addr, instr))
|
||||
line = stream.readline().strip()
|
||||
|
||||
return asm
|
||||
|
||||
|
||||
|
||||
samples = {}
|
||||
|
||||
|
||||
class PerfParser(LineParser):
|
||||
"""Parser for linux perf callgraph output.
|
||||
|
||||
It expects output generated with
|
||||
|
||||
perf record -g
|
||||
perf script
|
||||
"""
|
||||
|
||||
def __init__(self, infile, symbol):
|
||||
LineParser.__init__(self, infile)
|
||||
self.symbol = symbol
|
||||
|
||||
def readline(self):
|
||||
# Override LineParser.readline to ignore comment lines
|
||||
while True:
|
||||
LineParser.readline(self)
|
||||
if self.eof() or not self.lookahead().startswith('#'):
|
||||
break
|
||||
|
||||
def parse(self):
|
||||
# read lookahead
|
||||
self.readline()
|
||||
|
||||
while not self.eof():
|
||||
self.parse_event()
|
||||
|
||||
asm = lookupAsm(mapFile, self.symbol)
|
||||
|
||||
addresses = samples.keys()
|
||||
addresses.sort()
|
||||
total_samples = 0
|
||||
|
||||
sys.stdout.write('%s:\n' % self.symbol)
|
||||
for address, instr in asm:
|
||||
try:
|
||||
sample = samples.pop(address)
|
||||
except KeyError:
|
||||
sys.stdout.write(6*' ')
|
||||
else:
|
||||
sys.stdout.write('%6u' % (sample))
|
||||
total_samples += sample
|
||||
sys.stdout.write('%6u: %s\n' % (address, instr))
|
||||
print('total:', total_samples)
|
||||
assert len(samples) == 0
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
def parse_event(self):
|
||||
if self.eof():
|
||||
return
|
||||
|
||||
line = self.consume()
|
||||
assert line
|
||||
|
||||
callchain = self.parse_callchain()
|
||||
if not callchain:
|
||||
return
|
||||
|
||||
def parse_callchain(self):
|
||||
callchain = []
|
||||
while self.lookahead():
|
||||
function = self.parse_call(len(callchain) == 0)
|
||||
if function is None:
|
||||
break
|
||||
callchain.append(function)
|
||||
if self.lookahead() == '':
|
||||
self.consume()
|
||||
return callchain
|
||||
|
||||
call_re = re.compile(r'^\s+(?P<address>[0-9a-fA-F]+)\s+(?P<symbol>.*)\s+\((?P<module>[^)]*)\)$')
|
||||
|
||||
def parse_call(self, first):
|
||||
line = self.consume()
|
||||
mo = self.call_re.match(line)
|
||||
assert mo
|
||||
if not mo:
|
||||
return None
|
||||
|
||||
if not first:
|
||||
return None
|
||||
|
||||
function_name = mo.group('symbol')
|
||||
if not function_name:
|
||||
function_name = mo.group('address')
|
||||
|
||||
module = mo.group('module')
|
||||
|
||||
function_id = function_name + ':' + module
|
||||
|
||||
address = mo.group('address')
|
||||
address = int(address, 16)
|
||||
|
||||
if function_name != self.symbol:
|
||||
return None
|
||||
|
||||
start_address = lookupMap(module, function_name)
|
||||
address -= start_address
|
||||
|
||||
#print(function_name, module, address)
|
||||
|
||||
samples[address] = samples.get(address, 0) + 1
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
"""Main program."""
|
||||
|
||||
optparser = optparse.OptionParser(
|
||||
usage="\n\t%prog [options] symbol_name")
|
||||
(options, args) = optparser.parse_args(sys.argv[1:])
|
||||
if len(args) != 1:
|
||||
optparser.error('wrong number of arguments')
|
||||
|
||||
symbol = args[0]
|
||||
|
||||
p = subprocess.Popen(['perf', 'script'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
parser = PerfParser(p.stdout, symbol)
|
||||
parser.parse()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
||||
# vim: set sw=4 et:
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright © 2019-2020 Intel Corporation
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
import asyncio
|
||||
|
||||
import urwid
|
||||
|
||||
from pick.ui import UI, PALETTE
|
||||
|
||||
if __name__ == "__main__":
|
||||
u = UI()
|
||||
asyncio_loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(asyncio_loop)
|
||||
evl = urwid.AsyncioEventLoop(loop=asyncio_loop)
|
||||
loop = urwid.MainLoop(u.render(), PALETTE, event_loop=evl, handle_mouse=False)
|
||||
u.mainloop = loop
|
||||
loop.run()
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
|
||||
this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
|
||||
readonly this_dir
|
||||
|
||||
exec \
|
||||
"$this_dir/python-venv.sh" \
|
||||
"$this_dir/pick/requirements.txt" \
|
||||
"$this_dir/pick-ui.py" "$@"
|
||||
Whitespace-only changes.
@@ -0,0 +1,405 @@
|
||||
# Copyright © 2019-2020 Intel Corporation
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
"""Core data structures and routines for pick."""
|
||||
|
||||
import asyncio
|
||||
import enum
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import typing
|
||||
|
||||
import attr
|
||||
from packaging.version import Version
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from .ui import UI
|
||||
|
||||
import typing_extensions
|
||||
|
||||
class CommitDict(typing_extensions.TypedDict):
|
||||
|
||||
sha: str
|
||||
description: str
|
||||
nominated: bool
|
||||
nomination_type: int
|
||||
resolution: typing.Optional[int]
|
||||
main_sha: typing.Optional[str]
|
||||
because_sha: typing.Optional[str]
|
||||
notes: typing.Optional[str] = attr.ib(None)
|
||||
|
||||
IS_FIX = re.compile(r'^\s*fixes:\s*([a-f0-9]{6,40})', flags=re.MULTILINE | re.IGNORECASE)
|
||||
# FIXME: I dislike the duplication in this regex, but I couldn't get it to work otherwise
|
||||
IS_CC = re.compile(r'^\s*cc:\s*["\']?([0-9]{2}\.[0-9])?["\']?\s*["\']?([0-9]{2}\.[0-9])?["\']?\s*\<?mesa-stable',
|
||||
flags=re.MULTILINE | re.IGNORECASE)
|
||||
IS_REVERT = re.compile(r'This reverts commit ([0-9a-f]{40})')
|
||||
IS_BACKPORT = re.compile(r'^\s*backport-to:\s*(?:(\d{2}\.\d),?\s*(\d{2}\.\d)?|(\*))',
|
||||
flags=re.MULTILINE | re.IGNORECASE)
|
||||
|
||||
# XXX: hack
|
||||
SEM = asyncio.Semaphore(50)
|
||||
|
||||
COMMIT_LOCK = asyncio.Lock()
|
||||
|
||||
git_toplevel = subprocess.check_output(['git', 'rev-parse', '--show-toplevel'],
|
||||
stderr=subprocess.DEVNULL).decode("ascii").strip()
|
||||
pick_status_json = pathlib.Path(git_toplevel) / '.pick_status.json'
|
||||
|
||||
|
||||
class PickUIException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
@enum.unique
|
||||
class NominationType(enum.Enum):
|
||||
|
||||
NONE = 0
|
||||
CC = 1
|
||||
FIXES = 2
|
||||
REVERT = 3
|
||||
BACKPORT = 4
|
||||
|
||||
|
||||
@enum.unique
|
||||
class Resolution(enum.Enum):
|
||||
|
||||
UNRESOLVED = 0
|
||||
MERGED = 1
|
||||
DENOMINATED = 2
|
||||
BACKPORTED = 3
|
||||
NOTNEEDED = 4
|
||||
|
||||
|
||||
async def commit_state(*, amend: bool = False, message: str = 'Update') -> bool:
|
||||
"""Commit the .pick_status.json file."""
|
||||
async with COMMIT_LOCK:
|
||||
p = await asyncio.create_subprocess_exec(
|
||||
'git', 'add', pick_status_json.as_posix(),
|
||||
stdout=asyncio.subprocess.DEVNULL,
|
||||
stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
v = await p.wait()
|
||||
if v != 0:
|
||||
return False
|
||||
|
||||
if amend:
|
||||
cmd = ['--amend', '--no-edit']
|
||||
else:
|
||||
cmd = ['--message', f'.pick_status.json: {message}']
|
||||
p = await asyncio.create_subprocess_exec(
|
||||
'git', 'commit', *cmd,
|
||||
stdout=asyncio.subprocess.DEVNULL,
|
||||
stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
v = await p.wait()
|
||||
if v != 0:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@attr.s(slots=True)
|
||||
class Commit:
|
||||
|
||||
sha: str = attr.ib()
|
||||
description: str = attr.ib()
|
||||
nominated: bool = attr.ib(False)
|
||||
nomination_type: NominationType = attr.ib(NominationType.NONE)
|
||||
resolution: Resolution = attr.ib(Resolution.UNRESOLVED)
|
||||
main_sha: typing.Optional[str] = attr.ib(None)
|
||||
because_sha: typing.Optional[str] = attr.ib(None)
|
||||
notes: typing.Optional[str] = attr.ib(None)
|
||||
|
||||
def to_json(self) -> 'CommitDict':
|
||||
d: typing.Dict[str, typing.Any] = attr.asdict(self)
|
||||
d['nomination_type'] = self.nomination_type.value
|
||||
if self.resolution is not None:
|
||||
d['resolution'] = self.resolution.value
|
||||
return typing.cast('CommitDict', d)
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, data: 'CommitDict') -> 'Commit':
|
||||
c = cls(data['sha'], data['description'], data['nominated'], main_sha=data['main_sha'],
|
||||
because_sha=data['because_sha'], notes=data['notes'])
|
||||
c.nomination_type = NominationType(data['nomination_type'])
|
||||
if data['resolution'] is not None:
|
||||
c.resolution = Resolution(data['resolution'])
|
||||
return c
|
||||
|
||||
def date(self) -> str:
|
||||
# Show commit date, ie. when the commit actually landed
|
||||
# (as opposed to when it was first written)
|
||||
return subprocess.check_output(
|
||||
['git', 'show', '--no-patch', '--format=%cs', self.sha],
|
||||
stderr=subprocess.DEVNULL
|
||||
).decode("ascii").strip()
|
||||
|
||||
async def apply(self, ui: 'UI') -> typing.Tuple[bool, str]:
|
||||
# FIXME: This isn't really enough if we fail to cherry-pick because the
|
||||
# git tree will still be dirty
|
||||
async with COMMIT_LOCK:
|
||||
p = await asyncio.create_subprocess_exec(
|
||||
'git', 'cherry-pick', '-x', self.sha,
|
||||
stdout=asyncio.subprocess.DEVNULL,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
_, err = await p.communicate()
|
||||
|
||||
if p.returncode != 0:
|
||||
return (False, err.decode())
|
||||
|
||||
self.resolution = Resolution.MERGED
|
||||
await ui.feedback(f'{self.sha} ({self.description}) applied successfully')
|
||||
|
||||
# Append the changes to the .pickstatus.json file
|
||||
ui.save()
|
||||
v = await commit_state(amend=True)
|
||||
return (v, '')
|
||||
|
||||
async def abort_cherry(self, ui: 'UI', err: str) -> None:
|
||||
await ui.feedback(f'{self.sha} ({self.description}) failed to apply\n{err}')
|
||||
async with COMMIT_LOCK:
|
||||
p = await asyncio.create_subprocess_exec(
|
||||
'git', 'cherry-pick', '--abort',
|
||||
stdout=asyncio.subprocess.DEVNULL,
|
||||
stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
r = await p.wait()
|
||||
await ui.feedback(f'{"Successfully" if r == 0 else "Failed to"} abort cherry-pick.')
|
||||
|
||||
async def denominate(self, ui: 'UI') -> bool:
|
||||
self.resolution = Resolution.DENOMINATED
|
||||
ui.save()
|
||||
v = await commit_state(message=f'Mark {self.sha} as denominated')
|
||||
assert v
|
||||
await ui.feedback(f'{self.sha} ({self.description}) denominated successfully')
|
||||
return True
|
||||
|
||||
async def backport(self, ui: 'UI') -> bool:
|
||||
self.resolution = Resolution.BACKPORTED
|
||||
ui.save()
|
||||
v = await commit_state(message=f'Mark {self.sha} as backported')
|
||||
assert v
|
||||
await ui.feedback(f'{self.sha} ({self.description}) backported successfully')
|
||||
return True
|
||||
|
||||
async def resolve(self, ui: 'UI') -> None:
|
||||
self.resolution = Resolution.MERGED
|
||||
ui.save()
|
||||
v = await commit_state(amend=True)
|
||||
assert v
|
||||
await ui.feedback(f'{self.sha} ({self.description}) committed successfully')
|
||||
|
||||
async def update_notes(self, ui: 'UI', notes: typing.Optional[str]) -> None:
|
||||
self.notes = notes
|
||||
async with ui.git_lock:
|
||||
ui.save()
|
||||
v = await commit_state(message=f'Updates notes for {self.sha}')
|
||||
assert v
|
||||
await ui.feedback(f'{self.sha} ({self.description}) notes updated successfully')
|
||||
|
||||
|
||||
async def get_new_commits(sha: str) -> typing.List[typing.Tuple[str, str]]:
|
||||
# Try to get the authoritative upstream main
|
||||
p = await asyncio.create_subprocess_exec(
|
||||
'git', 'for-each-ref', '--format=%(upstream)', 'refs/heads/main',
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.DEVNULL)
|
||||
out, _ = await p.communicate()
|
||||
upstream = out.decode().strip()
|
||||
|
||||
p = await asyncio.create_subprocess_exec(
|
||||
'git', 'log', '--pretty=oneline', f'{sha}..{upstream}',
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.DEVNULL)
|
||||
out, _ = await p.communicate()
|
||||
assert p.returncode == 0, f"git log didn't work: {sha}"
|
||||
return list(split_commit_list(out.decode().strip()))
|
||||
|
||||
|
||||
def split_commit_list(commits: str) -> typing.Generator[typing.Tuple[str, str], None, None]:
|
||||
if not commits:
|
||||
return
|
||||
for line in commits.split('\n'):
|
||||
v = tuple(line.split(' ', 1))
|
||||
assert len(v) == 2, 'this is really just for mypy'
|
||||
yield typing.cast(typing.Tuple[str, str], v)
|
||||
|
||||
|
||||
async def is_commit_in_branch(sha: str) -> bool:
|
||||
async with SEM:
|
||||
p = await asyncio.create_subprocess_exec(
|
||||
'git', 'merge-base', '--is-ancestor', sha, 'HEAD',
|
||||
stdout=asyncio.subprocess.DEVNULL,
|
||||
stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
await p.wait()
|
||||
return p.returncode == 0
|
||||
|
||||
|
||||
async def full_sha(sha: str) -> str:
|
||||
async with SEM:
|
||||
p = await asyncio.create_subprocess_exec(
|
||||
'git', 'rev-parse', sha,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
out, _ = await p.communicate()
|
||||
if p.returncode:
|
||||
raise PickUIException(f'Invalid Sha {sha}')
|
||||
return out.decode().strip()
|
||||
|
||||
|
||||
async def resolve_nomination(commit: 'Commit', version: str) -> 'Commit':
|
||||
async with SEM:
|
||||
p = await asyncio.create_subprocess_exec(
|
||||
'git', 'log', '--format=%B', '-1', commit.sha,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
_out, _ = await p.communicate()
|
||||
assert p.returncode == 0, f'git log for {commit.sha} failed'
|
||||
commit_message = _out.decode()
|
||||
|
||||
# We give precedence to fixes and cc tags over revert tags.
|
||||
if fix_for_commit := IS_FIX.search(commit_message):
|
||||
# We set the nomination_type and because_sha here so that we can later
|
||||
# check to see if this fixes another staged commit.
|
||||
try:
|
||||
commit.because_sha = fixed = await full_sha(fix_for_commit.group(1))
|
||||
except PickUIException:
|
||||
pass
|
||||
else:
|
||||
commit.nomination_type = NominationType.FIXES
|
||||
if await is_commit_in_branch(fixed):
|
||||
commit.nominated = True
|
||||
return commit
|
||||
|
||||
if backport_to := IS_BACKPORT.findall(commit_message):
|
||||
for match in backport_to:
|
||||
if any(backport_version == '*' or Version(version) >= Version(backport_version)
|
||||
for backport_version in match if backport_version != ''):
|
||||
commit.nominated = True
|
||||
commit.nomination_type = NominationType.BACKPORT
|
||||
return commit
|
||||
|
||||
if cc_to := IS_CC.search(commit_message):
|
||||
if cc_to.groups() == (None, None) or version in cc_to.groups():
|
||||
commit.nominated = True
|
||||
commit.nomination_type = NominationType.CC
|
||||
return commit
|
||||
|
||||
if revert_of := IS_REVERT.search(commit_message):
|
||||
# See comment for IS_FIX path
|
||||
try:
|
||||
commit.because_sha = reverted = await full_sha(revert_of.group(1))
|
||||
except PickUIException:
|
||||
pass
|
||||
else:
|
||||
commit.nomination_type = NominationType.REVERT
|
||||
if await is_commit_in_branch(reverted):
|
||||
commit.nominated = True
|
||||
return commit
|
||||
|
||||
return commit
|
||||
|
||||
|
||||
async def resolve_fixes(commits: typing.List['Commit'], previous: typing.List['Commit']) -> None:
|
||||
"""Determine if any of the undecided commits fix/revert a staged commit.
|
||||
|
||||
The are still needed if they apply to a commit that is staged for
|
||||
inclusion, but not yet included.
|
||||
|
||||
This must be done in order, because a commit 3 might fix commit 2 which
|
||||
fixes commit 1.
|
||||
"""
|
||||
shas: typing.Set[str] = set(c.sha for c in previous if c.nominated)
|
||||
assert None not in shas, 'None in shas'
|
||||
|
||||
for commit in reversed(commits):
|
||||
if not commit.nominated and commit.nomination_type is NominationType.FIXES:
|
||||
commit.nominated = commit.because_sha in shas
|
||||
|
||||
if commit.nominated:
|
||||
shas.add(commit.sha)
|
||||
|
||||
for commit in commits:
|
||||
if (commit.nomination_type is NominationType.REVERT and
|
||||
commit.because_sha in shas):
|
||||
for oldc in reversed(commits):
|
||||
if oldc.sha == commit.because_sha:
|
||||
# In this case a commit that hasn't yet been applied is
|
||||
# reverted, we don't want to apply that commit at all
|
||||
oldc.nominated = False
|
||||
oldc.resolution = Resolution.DENOMINATED
|
||||
commit.nominated = False
|
||||
commit.resolution = Resolution.DENOMINATED
|
||||
shas.remove(commit.because_sha)
|
||||
break
|
||||
|
||||
|
||||
async def gather_commits(version: str, previous: typing.List['Commit'],
|
||||
new: typing.List[typing.Tuple[str, str]], cb) -> typing.List['Commit']:
|
||||
# We create an array of the final size up front, then we pass that array
|
||||
# to the "inner" co-routine, which is turned into a list of tasks and
|
||||
# collected by asyncio.gather. We do this to allow the tasks to be
|
||||
# asynchronously gathered, but to also ensure that the commits list remains
|
||||
# in order.
|
||||
m_commits: typing.List[typing.Optional['Commit']] = [None] * len(new)
|
||||
tasks = []
|
||||
|
||||
async def inner(commit: 'Commit', version: str,
|
||||
commits: typing.List[typing.Optional['Commit']],
|
||||
index: int, cb) -> None:
|
||||
commits[index] = await resolve_nomination(commit, version)
|
||||
cb()
|
||||
|
||||
for i, (sha, desc) in enumerate(new):
|
||||
tasks.append(asyncio.ensure_future(
|
||||
inner(Commit(sha, desc), version, m_commits, i, cb)))
|
||||
|
||||
await asyncio.gather(*tasks)
|
||||
assert None not in m_commits
|
||||
commits = typing.cast(typing.List[Commit], m_commits)
|
||||
|
||||
await resolve_fixes(commits, previous)
|
||||
|
||||
for commit in commits:
|
||||
if commit.resolution is Resolution.UNRESOLVED and not commit.nominated:
|
||||
commit.resolution = Resolution.NOTNEEDED
|
||||
|
||||
return commits
|
||||
|
||||
|
||||
def load() -> typing.List['Commit']:
|
||||
if not pick_status_json.exists():
|
||||
return []
|
||||
with pick_status_json.open('r') as f:
|
||||
raw = json.load(f)
|
||||
return [Commit.from_json(c) for c in raw]
|
||||
|
||||
|
||||
def save(commits: typing.Iterable['Commit']) -> None:
|
||||
commits = list(commits)
|
||||
with pick_status_json.open('wt') as f:
|
||||
json.dump([c.to_json() for c in commits], f, indent=4)
|
||||
|
||||
asyncio.ensure_future(commit_state(message=f'Update to {commits[0].sha}'))
|
||||
@@ -0,0 +1,600 @@
|
||||
# Copyright © 2019-2020 Intel Corporation
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
"""Tests for pick's core data structures and routines."""
|
||||
|
||||
from unittest import mock
|
||||
import textwrap
|
||||
import typing
|
||||
|
||||
import attr
|
||||
import pytest
|
||||
|
||||
from . import core
|
||||
|
||||
|
||||
class TestCommit:
|
||||
|
||||
@pytest.fixture
|
||||
def unnominated_commit(self) -> 'core.Commit':
|
||||
return core.Commit('abc123', 'sub: A commit', main_sha='45678')
|
||||
|
||||
@pytest.fixture
|
||||
def nominated_commit(self) -> 'core.Commit':
|
||||
return core.Commit('abc123', 'sub: A commit', True,
|
||||
core.NominationType.CC, core.Resolution.UNRESOLVED)
|
||||
|
||||
class TestToJson:
|
||||
|
||||
def test_not_nominated(self, unnominated_commit: 'core.Commit'):
|
||||
c = unnominated_commit
|
||||
v = c.to_json()
|
||||
assert v == {
|
||||
'sha': 'abc123',
|
||||
'description': 'sub: A commit',
|
||||
'nominated': False,
|
||||
'nomination_type': core.NominationType.NONE.value,
|
||||
'resolution': core.Resolution.UNRESOLVED.value,
|
||||
'main_sha': '45678',
|
||||
'because_sha': None,
|
||||
'notes': None,
|
||||
}
|
||||
|
||||
def test_nominated(self, nominated_commit: 'core.Commit'):
|
||||
c = nominated_commit
|
||||
v = c.to_json()
|
||||
assert v == {
|
||||
'sha': 'abc123',
|
||||
'description': 'sub: A commit',
|
||||
'nominated': True,
|
||||
'nomination_type': core.NominationType.CC.value,
|
||||
'resolution': core.Resolution.UNRESOLVED.value,
|
||||
'main_sha': None,
|
||||
'because_sha': None,
|
||||
'notes': None,
|
||||
}
|
||||
|
||||
class TestFromJson:
|
||||
|
||||
def test_not_nominated(self, unnominated_commit: 'core.Commit'):
|
||||
c = unnominated_commit
|
||||
v = c.to_json()
|
||||
c2 = core.Commit.from_json(v)
|
||||
assert c == c2
|
||||
|
||||
def test_nominated(self, nominated_commit: 'core.Commit'):
|
||||
c = nominated_commit
|
||||
v = c.to_json()
|
||||
c2 = core.Commit.from_json(v)
|
||||
assert c == c2
|
||||
|
||||
|
||||
class TestRE:
|
||||
|
||||
"""Tests for the regular expressions used to identify commits."""
|
||||
|
||||
class TestFixes:
|
||||
|
||||
def test_simple(self):
|
||||
message = textwrap.dedent("""\
|
||||
etnaviv: fix vertex buffer state emission for single stream GPUs
|
||||
|
||||
GPUs with a single supported vertex stream must use the single state
|
||||
address to program the stream.
|
||||
|
||||
Fixes: 3d09bb390a39 (etnaviv: GC7000: State changes for HALTI3..5)
|
||||
Signed-off-by: Lucas Stach <[email protected]>
|
||||
Reviewed-by: Jonathan Marek <[email protected]>
|
||||
""")
|
||||
|
||||
fix_for_commit = core.IS_FIX.search(message)
|
||||
assert fix_for_commit is not None
|
||||
assert fix_for_commit.group(1) == '3d09bb390a39'
|
||||
|
||||
class TestCC:
|
||||
|
||||
def test_single_branch(self):
|
||||
"""Tests commit meant for a single branch, ie, 19.1"""
|
||||
message = textwrap.dedent("""\
|
||||
radv: fix DCC fast clear code for intensity formats
|
||||
|
||||
This fixes a rendering issue with DiRT 4 on GFX10. Only GFX10 was
|
||||
affected because intensity formats are different.
|
||||
|
||||
Cc: 19.2 <[email protected]>
|
||||
Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/1923
|
||||
Signed-off-by: Samuel Pitoiset <[email protected]>
|
||||
Reviewed-by: Bas Nieuwenhuizen <[email protected]>
|
||||
""")
|
||||
|
||||
cc_to = core.IS_CC.search(message)
|
||||
assert cc_to is not None
|
||||
assert cc_to.group(1) == '19.2'
|
||||
|
||||
def test_multiple_branches(self):
|
||||
"""Tests commit with more than one branch specified"""
|
||||
message = textwrap.dedent("""\
|
||||
radeonsi: enable zerovram for Rocket League
|
||||
|
||||
Fixes corruption on game startup.
|
||||
Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/1888
|
||||
|
||||
Cc: 19.1 19.2 <[email protected]>
|
||||
Reviewed-by: Pierre-Eric Pelloux-Prayer <[email protected]>
|
||||
""")
|
||||
|
||||
cc_to = core.IS_CC.search(message)
|
||||
assert cc_to is not None
|
||||
assert cc_to.group(1) == '19.1'
|
||||
assert cc_to.group(2) == '19.2'
|
||||
|
||||
def test_no_branch(self):
|
||||
"""Tests commit with no branch specification"""
|
||||
message = textwrap.dedent("""\
|
||||
anv/android: fix images created with external format support
|
||||
|
||||
This fixes a case where user first creates image and then later binds it
|
||||
with memory created from AHW buffer.
|
||||
|
||||
Cc: <[email protected]>
|
||||
Signed-off-by: Tapani Pälli <[email protected]>
|
||||
Reviewed-by: Lionel Landwerlin <[email protected]>
|
||||
""")
|
||||
|
||||
cc_to = core.IS_CC.search(message)
|
||||
assert cc_to is not None
|
||||
|
||||
def test_quotes(self):
|
||||
"""Tests commit with quotes around the versions"""
|
||||
message = textwrap.dedent("""\
|
||||
anv: Always fill out the AUX table even if CCS is disabled
|
||||
|
||||
Cc: "20.0" [email protected]
|
||||
Reviewed-by: Kenneth Graunke <[email protected]>
|
||||
Tested-by: Marge Bot <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/3454>
|
||||
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/3454>
|
||||
""")
|
||||
|
||||
cc_to = core.IS_CC.search(message)
|
||||
assert cc_to is not None
|
||||
assert cc_to.group(1) == '20.0'
|
||||
|
||||
def test_multiple_quotes(self):
|
||||
"""Tests commit with quotes around the versions"""
|
||||
message = textwrap.dedent("""\
|
||||
anv: Always fill out the AUX table even if CCS is disabled
|
||||
|
||||
Cc: "20.0" "20.1" [email protected]
|
||||
Reviewed-by: Kenneth Graunke <[email protected]>
|
||||
Tested-by: Marge Bot <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/3454>
|
||||
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/3454>
|
||||
""")
|
||||
|
||||
cc_to = core.IS_CC.search(message)
|
||||
assert cc_to is not None
|
||||
assert cc_to.group(1) == '20.0'
|
||||
assert cc_to.group(2) == '20.1'
|
||||
|
||||
def test_single_quotes(self):
|
||||
"""Tests commit with quotes around the versions"""
|
||||
message = textwrap.dedent("""\
|
||||
anv: Always fill out the AUX table even if CCS is disabled
|
||||
|
||||
Cc: '20.0' [email protected]
|
||||
Reviewed-by: Kenneth Graunke <[email protected]>
|
||||
Tested-by: Marge Bot <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/3454>
|
||||
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/3454>
|
||||
""")
|
||||
|
||||
cc_to = core.IS_CC.search(message)
|
||||
assert cc_to is not None
|
||||
assert cc_to.group(1) == '20.0'
|
||||
|
||||
def test_multiple_single_quotes(self):
|
||||
"""Tests commit with quotes around the versions"""
|
||||
message = textwrap.dedent("""\
|
||||
anv: Always fill out the AUX table even if CCS is disabled
|
||||
|
||||
Cc: '20.0' '20.1' [email protected]
|
||||
Reviewed-by: Kenneth Graunke <[email protected]>
|
||||
Tested-by: Marge Bot <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/3454>
|
||||
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/3454>
|
||||
""")
|
||||
|
||||
cc_to = core.IS_CC.search(message)
|
||||
assert cc_to is not None
|
||||
assert cc_to.group(1) == '20.0'
|
||||
assert cc_to.group(2) == '20.1'
|
||||
|
||||
class TestRevert:
|
||||
|
||||
def test_simple(self):
|
||||
message = textwrap.dedent("""\
|
||||
Revert "radv: do not emit PKT3_CONTEXT_CONTROL with AMDGPU 3.6.0+"
|
||||
|
||||
This reverts commit 2ca8629fa9b303e24783b76a7b3b0c2513e32fbd.
|
||||
|
||||
This was initially ported from RadeonSI, but in the meantime it has
|
||||
been reverted because it might hang. Be conservative and re-introduce
|
||||
this packet emission.
|
||||
|
||||
Unfortunately this doesn't fix anything known.
|
||||
|
||||
Cc: 19.2 <[email protected]>
|
||||
Signed-off-by: Samuel Pitoiset <[email protected]>
|
||||
Reviewed-by: Bas Nieuwenhuizen <[email protected]>
|
||||
""")
|
||||
|
||||
revert_of = core.IS_REVERT.search(message)
|
||||
assert revert_of is not None
|
||||
assert revert_of.group(1) == '2ca8629fa9b303e24783b76a7b3b0c2513e32fbd'
|
||||
|
||||
class TestBackportTo:
|
||||
|
||||
def test_single_release(self):
|
||||
"""Tests commit meant for a single branch, ie, 19.1"""
|
||||
message = textwrap.dedent("""\
|
||||
radv: fix DCC fast clear code for intensity formats
|
||||
|
||||
This fixes a rendering issue with DiRT 4 on GFX10. Only GFX10 was
|
||||
affected because intensity formats are different.
|
||||
|
||||
Backport-to: 19.2
|
||||
Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/1923
|
||||
Signed-off-by: Samuel Pitoiset <[email protected]>
|
||||
Reviewed-by: Bas Nieuwenhuizen <[email protected]>
|
||||
""")
|
||||
|
||||
backport_to = core.IS_BACKPORT.findall(message)
|
||||
assert backport_to == [('19.2', '', '')]
|
||||
|
||||
def test_multiple_release_space(self):
|
||||
"""Tests commit with more than one branch specified"""
|
||||
message = textwrap.dedent("""\
|
||||
radeonsi: enable zerovram for Rocket League
|
||||
|
||||
Fixes corruption on game startup.
|
||||
Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/1888
|
||||
|
||||
Backport-to: 19.1 19.2
|
||||
Reviewed-by: Pierre-Eric Pelloux-Prayer <[email protected]>
|
||||
""")
|
||||
|
||||
backport_to = core.IS_BACKPORT.findall(message)
|
||||
assert backport_to == [('19.1', '19.2', '')]
|
||||
|
||||
def test_multiple_release_comma(self):
|
||||
"""Tests commit with more than one branch specified"""
|
||||
message = textwrap.dedent("""\
|
||||
radeonsi: enable zerovram for Rocket League
|
||||
|
||||
Fixes corruption on game startup.
|
||||
Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/1888
|
||||
|
||||
Backport-to: 19.1, 19.2
|
||||
Reviewed-by: Pierre-Eric Pelloux-Prayer <[email protected]>
|
||||
""")
|
||||
|
||||
backport_to = core.IS_BACKPORT.findall(message)
|
||||
assert backport_to == [('19.1', '19.2', '')]
|
||||
|
||||
def test_multiple_release_lines(self):
|
||||
"""Tests commit with more than one branch specified in mulitple tags"""
|
||||
message = textwrap.dedent("""\
|
||||
commit title
|
||||
|
||||
Backport-to: 19.0
|
||||
Backport-to: 19.1, 19.2
|
||||
""")
|
||||
|
||||
backport_to = core.IS_BACKPORT.findall(message)
|
||||
assert backport_to == [('19.0', '', ''), ('19.1', '19.2', '')]
|
||||
|
||||
|
||||
class TestResolveNomination:
|
||||
|
||||
@attr.s(slots=True)
|
||||
class FakeSubprocess:
|
||||
|
||||
"""A fake asyncio.subprocess like class for use with mock."""
|
||||
|
||||
out: typing.Optional[bytes] = attr.ib(None)
|
||||
returncode: int = attr.ib(0)
|
||||
|
||||
async def mock(self, *_, **__):
|
||||
"""A dirtly little helper for mocking."""
|
||||
return self
|
||||
|
||||
async def communicate(self) -> typing.Tuple[bytes, bytes]:
|
||||
assert self.out is not None
|
||||
return self.out, b''
|
||||
|
||||
async def wait(self) -> int:
|
||||
return self.returncode
|
||||
|
||||
@staticmethod
|
||||
async def return_true(*_, **__) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
async def return_false(*_, **__) -> bool:
|
||||
return False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fix_is_nominated(self):
|
||||
s = self.FakeSubprocess(b'Fixes: 3d09bb390a39 (etnaviv: GC7000: State changes for HALTI3..5)')
|
||||
c = core.Commit('abcdef1234567890', 'a commit')
|
||||
|
||||
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||
with mock.patch('bin.pick.core.is_commit_in_branch', self.return_true):
|
||||
await core.resolve_nomination(c, '')
|
||||
|
||||
assert c.nominated
|
||||
assert c.nomination_type is core.NominationType.FIXES
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fix_is_not_nominated(self):
|
||||
s = self.FakeSubprocess(b'Fixes: 3d09bb390a39 (etnaviv: GC7000: State changes for HALTI3..5)')
|
||||
c = core.Commit('abcdef1234567890', 'a commit')
|
||||
|
||||
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||
with mock.patch('bin.pick.core.is_commit_in_branch', self.return_false):
|
||||
await core.resolve_nomination(c, '')
|
||||
|
||||
assert not c.nominated
|
||||
assert c.nomination_type is core.NominationType.FIXES
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cc_is_nominated(self):
|
||||
s = self.FakeSubprocess(b'Cc: 16.2 <[email protected]>')
|
||||
c = core.Commit('abcdef1234567890', 'a commit')
|
||||
|
||||
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||
await core.resolve_nomination(c, '16.2')
|
||||
|
||||
assert c.nominated
|
||||
assert c.nomination_type is core.NominationType.CC
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cc_is_nominated2(self):
|
||||
s = self.FakeSubprocess(b'Cc: [email protected]')
|
||||
c = core.Commit('abcdef1234567890', 'a commit')
|
||||
|
||||
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||
await core.resolve_nomination(c, '16.2')
|
||||
|
||||
assert c.nominated
|
||||
assert c.nomination_type is core.NominationType.CC
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cc_is_not_nominated(self):
|
||||
s = self.FakeSubprocess(b'Cc: 16.2 <[email protected]>')
|
||||
c = core.Commit('abcdef1234567890', 'a commit')
|
||||
|
||||
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||
await core.resolve_nomination(c, '16.1')
|
||||
|
||||
assert not c.nominated
|
||||
assert c.nomination_type is core.NominationType.NONE
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backport_is_nominated(self):
|
||||
s = self.FakeSubprocess(b'Backport-to: 16.2')
|
||||
c = core.Commit('abcdef1234567890', 'a commit')
|
||||
|
||||
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||
await core.resolve_nomination(c, '16.2')
|
||||
|
||||
assert c.nominated
|
||||
assert c.nomination_type is core.NominationType.BACKPORT
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backport_all_is_nominated(self):
|
||||
s = self.FakeSubprocess(b'Backport-to: *')
|
||||
c = core.Commit('abcdef1234567890', 'a commit')
|
||||
|
||||
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||
await core.resolve_nomination(c, '0.0')
|
||||
|
||||
assert c.nominated
|
||||
assert c.nomination_type is core.NominationType.BACKPORT
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backport_is_nominated_after(self):
|
||||
s = self.FakeSubprocess(b'Backport-to: 16.2')
|
||||
c = core.Commit('abcdef1234567890', 'a commit')
|
||||
|
||||
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||
await core.resolve_nomination(c, '16.3')
|
||||
|
||||
assert c.nominated
|
||||
assert c.nomination_type is core.NominationType.BACKPORT
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backport_is_not_nominated(self):
|
||||
s = self.FakeSubprocess(b'Backport-to: 16.2')
|
||||
c = core.Commit('abcdef1234567890', 'a commit')
|
||||
|
||||
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||
await core.resolve_nomination(c, '16.1')
|
||||
|
||||
assert not c.nominated
|
||||
assert c.nomination_type is core.NominationType.NONE
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_revert_is_nominated(self):
|
||||
s = self.FakeSubprocess(b'This reverts commit 1234567890123456789012345678901234567890.')
|
||||
c = core.Commit('abcdef1234567890', 'a commit')
|
||||
|
||||
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||
with mock.patch('bin.pick.core.is_commit_in_branch', self.return_true):
|
||||
await core.resolve_nomination(c, '')
|
||||
|
||||
assert c.nominated
|
||||
assert c.nomination_type is core.NominationType.REVERT
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_revert_is_not_nominated(self):
|
||||
s = self.FakeSubprocess(b'This reverts commit 1234567890123456789012345678901234567890.')
|
||||
c = core.Commit('abcdef1234567890', 'a commit')
|
||||
|
||||
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||
with mock.patch('bin.pick.core.is_commit_in_branch', self.return_false):
|
||||
await core.resolve_nomination(c, '')
|
||||
|
||||
assert not c.nominated
|
||||
assert c.nomination_type is core.NominationType.REVERT
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_is_fix_and_backport(self):
|
||||
s = self.FakeSubprocess(
|
||||
b'Fixes: 3d09bb390a39 (etnaviv: GC7000: State changes for HALTI3..5)\n'
|
||||
b'Backport-to: 16.1'
|
||||
)
|
||||
c = core.Commit('abcdef1234567890', 'a commit')
|
||||
|
||||
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||
with mock.patch('bin.pick.core.is_commit_in_branch', self.return_true):
|
||||
await core.resolve_nomination(c, '16.1')
|
||||
|
||||
assert c.nominated
|
||||
assert c.nomination_type is core.NominationType.FIXES
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_is_fix_and_cc(self):
|
||||
s = self.FakeSubprocess(
|
||||
b'Fixes: 3d09bb390a39 (etnaviv: GC7000: State changes for HALTI3..5)\n'
|
||||
b'Cc: 16.1 <[email protected]>'
|
||||
)
|
||||
c = core.Commit('abcdef1234567890', 'a commit')
|
||||
|
||||
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||
with mock.patch('bin.pick.core.is_commit_in_branch', self.return_true):
|
||||
await core.resolve_nomination(c, '16.1')
|
||||
|
||||
assert c.nominated
|
||||
assert c.nomination_type is core.NominationType.FIXES
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_is_fix_and_revert(self):
|
||||
s = self.FakeSubprocess(
|
||||
b'Fixes: 3d09bb390a39 (etnaviv: GC7000: State changes for HALTI3..5)\n'
|
||||
b'This reverts commit 1234567890123456789012345678901234567890.'
|
||||
)
|
||||
c = core.Commit('abcdef1234567890', 'a commit')
|
||||
|
||||
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||
with mock.patch('bin.pick.core.is_commit_in_branch', self.return_true):
|
||||
await core.resolve_nomination(c, '16.1')
|
||||
|
||||
assert c.nominated
|
||||
assert c.nomination_type is core.NominationType.FIXES
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_is_cc_and_revert(self):
|
||||
s = self.FakeSubprocess(
|
||||
b'This reverts commit 1234567890123456789012345678901234567890.\n'
|
||||
b'Cc: 16.1 <[email protected]>'
|
||||
)
|
||||
c = core.Commit('abcdef1234567890', 'a commit')
|
||||
|
||||
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||
with mock.patch('bin.pick.core.is_commit_in_branch', self.return_true):
|
||||
await core.resolve_nomination(c, '16.1')
|
||||
|
||||
assert c.nominated
|
||||
assert c.nomination_type is core.NominationType.CC
|
||||
|
||||
|
||||
class TestResolveFixes:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_in_new(self):
|
||||
"""Because commit abcd is nominated, so f123 should be as well."""
|
||||
c = [
|
||||
core.Commit('f123', 'desc', nomination_type=core.NominationType.FIXES, because_sha='abcd'),
|
||||
core.Commit('abcd', 'desc', True),
|
||||
]
|
||||
await core.resolve_fixes(c, [])
|
||||
assert c[1].nominated
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_not_in_new(self):
|
||||
"""Because commit abcd is not nominated, commit f123 shouldn't be either."""
|
||||
c = [
|
||||
core.Commit('f123', 'desc', nomination_type=core.NominationType.FIXES, because_sha='abcd'),
|
||||
core.Commit('abcd', 'desc'),
|
||||
]
|
||||
await core.resolve_fixes(c, [])
|
||||
assert not c[0].nominated
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_in_previous(self):
|
||||
"""Because commit abcd is nominated, so f123 should be as well."""
|
||||
p = [
|
||||
core.Commit('abcd', 'desc', True),
|
||||
]
|
||||
c = [
|
||||
core.Commit('f123', 'desc', nomination_type=core.NominationType.FIXES, because_sha='abcd'),
|
||||
]
|
||||
await core.resolve_fixes(c, p)
|
||||
assert c[0].nominated
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_not_in_previous(self):
|
||||
"""Because commit abcd is not nominated, commit f123 shouldn't be either."""
|
||||
p = [
|
||||
core.Commit('abcd', 'desc'),
|
||||
]
|
||||
c = [
|
||||
core.Commit('f123', 'desc', nomination_type=core.NominationType.FIXES, because_sha='abcd'),
|
||||
]
|
||||
await core.resolve_fixes(c, p)
|
||||
assert not c[0].nominated
|
||||
|
||||
|
||||
class TestIsCommitInBranch:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no(self):
|
||||
# Hopefully this is never true?
|
||||
value = await core.is_commit_in_branch('ffffffffffffffffffffffffffffff')
|
||||
assert not value
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_yes(self):
|
||||
# This commit is from 2000, it better always be in the branch
|
||||
value = await core.is_commit_in_branch('88f3b89a2cb77766d2009b9868c44e03abe2dbb2')
|
||||
assert value
|
||||
|
||||
|
||||
class TestFullSha:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic(self):
|
||||
# This commit is from 2000, it better always be in the branch
|
||||
value = await core.full_sha('88f3b89a2cb777')
|
||||
assert value
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid(self):
|
||||
# This commit is from 2000, it better always be in the branch
|
||||
with pytest.raises(core.PickUIException):
|
||||
await core.full_sha('fffffffffffffffffffffffffffffffffff')
|
||||
@@ -0,0 +1,3 @@
|
||||
attrs==25.4.0
|
||||
packaging==26.0
|
||||
urwid==3.0.3
|
||||
+322
@@ -0,0 +1,322 @@
|
||||
# Copyright © 2019-2020 Intel Corporation
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
"""Urwid UI for pick script."""
|
||||
|
||||
import asyncio
|
||||
import itertools
|
||||
import textwrap
|
||||
import typing
|
||||
|
||||
import attr
|
||||
import urwid
|
||||
|
||||
from . import core
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
WidgetType = typing.TypeVar('WidgetType', bound=urwid.Widget)
|
||||
|
||||
PALETTE = [
|
||||
('a', 'black', 'light gray'),
|
||||
('b', 'black', 'dark red'),
|
||||
('bg', 'black', 'dark blue'),
|
||||
('reversed', 'standout', ''),
|
||||
]
|
||||
|
||||
|
||||
class RootWidget(urwid.Frame):
|
||||
|
||||
def __init__(self, *args, ui: 'UI', **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.ui = ui
|
||||
|
||||
|
||||
class CommitList(urwid.ListBox):
|
||||
|
||||
def __init__(self, *args, ui: 'UI', **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.ui = ui
|
||||
|
||||
def keypress(self, size: int, key: str) -> typing.Optional[str]:
|
||||
if key == 'q':
|
||||
raise urwid.ExitMainLoop()
|
||||
elif key == 'u':
|
||||
asyncio.ensure_future(self.ui.update())
|
||||
elif key == 'a':
|
||||
self.ui.add()
|
||||
else:
|
||||
return super().keypress(size, key)
|
||||
return None
|
||||
|
||||
|
||||
class CommitWidget(urwid.Text):
|
||||
|
||||
# urwid.Text is normally not interactable, this is required to tell urwid
|
||||
# to use our keypress method
|
||||
_selectable = True
|
||||
|
||||
def __init__(self, ui: 'UI', commit: 'core.Commit'):
|
||||
reason = commit.nomination_type.name.ljust(6)
|
||||
super().__init__(f'{commit.date()} {reason} {commit.sha[:10]} {commit.description}')
|
||||
self.ui = ui
|
||||
self.commit = commit
|
||||
|
||||
async def apply(self) -> None:
|
||||
async with self.ui.git_lock:
|
||||
result, err = await self.commit.apply(self.ui)
|
||||
if not result:
|
||||
self.ui.chp_failed(self, err)
|
||||
else:
|
||||
self.ui.remove_commit(self)
|
||||
|
||||
async def denominate(self) -> None:
|
||||
async with self.ui.git_lock:
|
||||
await self.commit.denominate(self.ui)
|
||||
self.ui.remove_commit(self)
|
||||
|
||||
async def backport(self) -> None:
|
||||
async with self.ui.git_lock:
|
||||
await self.commit.backport(self.ui)
|
||||
self.ui.remove_commit(self)
|
||||
|
||||
def keypress(self, size: int, key: str) -> typing.Optional[str]:
|
||||
if key == 'c':
|
||||
asyncio.ensure_future(self.apply())
|
||||
elif key == 'd':
|
||||
asyncio.ensure_future(self.denominate())
|
||||
elif key == 'b':
|
||||
asyncio.ensure_future(self.backport())
|
||||
else:
|
||||
return key
|
||||
return None
|
||||
|
||||
|
||||
class FocusAwareEdit(urwid.Edit):
|
||||
|
||||
"""An Edit type that signals when it comes into and leaves focus."""
|
||||
|
||||
signals = urwid.Edit.signals + ['focus_changed']
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.__is_focus = False
|
||||
|
||||
def render(self, size: typing.Tuple[int], focus: bool = False) -> urwid.Canvas:
|
||||
if focus != self.__is_focus:
|
||||
self._emit("focus_changed", focus)
|
||||
self.__is_focus = focus
|
||||
return super().render(size, focus)
|
||||
|
||||
|
||||
@attr.s(slots=True)
|
||||
class UI:
|
||||
|
||||
"""Main management object.
|
||||
|
||||
:previous_commits: A list of commits to main since this branch was created
|
||||
:new_commits: Commits added to main since the last time this script was run
|
||||
"""
|
||||
|
||||
commit_list: typing.List['urwid.Button'] = attr.ib(factory=lambda: urwid.SimpleFocusListWalker([]), init=False)
|
||||
feedback_box: typing.List['urwid.Text'] = attr.ib(factory=lambda: urwid.SimpleFocusListWalker([]), init=False)
|
||||
notes: 'FocusAwareEdit' = attr.ib(factory=lambda: FocusAwareEdit('', multiline=True), init=False)
|
||||
header: 'urwid.Text' = attr.ib(factory=lambda: urwid.Text('Mesa Stable Picker', align='center'), init=False)
|
||||
body: 'urwid.Columns' = attr.ib(attr.Factory(lambda s: s._make_body(), True), init=False)
|
||||
footer: 'urwid.Columns' = attr.ib(attr.Factory(lambda s: s._make_footer(), True), init=False)
|
||||
root: RootWidget = attr.ib(attr.Factory(lambda s: s._make_root(), True), init=False)
|
||||
mainloop: urwid.MainLoop = attr.ib(None, init=False)
|
||||
|
||||
previous_commits: typing.List['core.Commit'] = attr.ib(factory=list, init=False)
|
||||
new_commits: typing.List['core.Commit'] = attr.ib(factory=list, init=False)
|
||||
git_lock: asyncio.Lock = attr.ib(factory=asyncio.Lock, init=False)
|
||||
|
||||
def _get_current_commit(self) -> typing.Optional['core.Commit']:
|
||||
entry = self.commit_list.get_focus()[0]
|
||||
return entry.original_widget.commit if entry is not None else None
|
||||
|
||||
def _change_notes_cb(self) -> None:
|
||||
commit = self._get_current_commit()
|
||||
if commit and commit.notes:
|
||||
self.notes.set_edit_text(commit.notes)
|
||||
else:
|
||||
self.notes.set_edit_text('')
|
||||
|
||||
def _change_notes_focus_cb(self, notes: 'FocusAwareEdit', focus: 'bool') -> 'None':
|
||||
# in the case of coming into focus we don't want to do anything
|
||||
if focus:
|
||||
return
|
||||
commit = self._get_current_commit()
|
||||
if commit is None:
|
||||
return
|
||||
text: str = notes.get_edit_text()
|
||||
if text != commit.notes:
|
||||
asyncio.ensure_future(commit.update_notes(self, text))
|
||||
|
||||
def _make_body(self) -> 'urwid.Columns':
|
||||
commits = CommitList(self.commit_list, ui=self)
|
||||
feedback = urwid.ListBox(self.feedback_box)
|
||||
urwid.connect_signal(self.commit_list, 'modified', self._change_notes_cb)
|
||||
notes = urwid.Filler(self.notes)
|
||||
urwid.connect_signal(self.notes, 'focus_changed', self._change_notes_focus_cb)
|
||||
|
||||
return urwid.Columns([urwid.LineBox(commits), urwid.Pile([urwid.LineBox(notes), urwid.LineBox(feedback)])])
|
||||
|
||||
def _make_footer(self) -> 'urwid.Columns':
|
||||
body = [
|
||||
urwid.Text('[U]pdate'),
|
||||
urwid.Text('[Q]uit'),
|
||||
urwid.Text('[C]herry Pick'),
|
||||
urwid.Text('[D]enominate'),
|
||||
urwid.Text('[B]ackport'),
|
||||
urwid.Text('[A]pply additional patch'),
|
||||
]
|
||||
return urwid.Columns(body)
|
||||
|
||||
def _make_root(self) -> 'RootWidget':
|
||||
return RootWidget(self.body, urwid.LineBox(self.header), urwid.LineBox(self.footer), 'body', ui=self)
|
||||
|
||||
def render(self) -> 'WidgetType':
|
||||
asyncio.ensure_future(self.update())
|
||||
return self.root
|
||||
|
||||
def load(self) -> None:
|
||||
self.previous_commits = core.load()
|
||||
|
||||
async def update(self) -> None:
|
||||
self.load()
|
||||
with open('VERSION', 'r') as f:
|
||||
version = '.'.join(f.read().split('.')[:2])
|
||||
if self.previous_commits:
|
||||
sha = self.previous_commits[0].sha
|
||||
else:
|
||||
sha = f'{version}-branchpoint'
|
||||
|
||||
new_commits = await core.get_new_commits(sha)
|
||||
|
||||
if new_commits:
|
||||
pb = urwid.ProgressBar('a', 'b', done=len(new_commits))
|
||||
o = self.mainloop.widget
|
||||
self.mainloop.widget = urwid.Overlay(
|
||||
urwid.Filler(urwid.LineBox(pb)), o, 'center', ('relative', 50), 'middle', ('relative', 50))
|
||||
self.new_commits = await core.gather_commits(
|
||||
version, self.previous_commits, new_commits,
|
||||
lambda: pb.set_completion(pb.current + 1))
|
||||
self.mainloop.widget = o
|
||||
|
||||
for commit in reversed(list(itertools.chain(self.new_commits, self.previous_commits))):
|
||||
if commit.nominated and commit.resolution is core.Resolution.UNRESOLVED:
|
||||
b = urwid.AttrMap(CommitWidget(self, commit), None, focus_map='reversed')
|
||||
self.commit_list.append(b)
|
||||
self.mainloop.draw_screen()
|
||||
self.save()
|
||||
|
||||
async def feedback(self, text: str) -> None:
|
||||
self.feedback_box.append(urwid.AttrMap(urwid.Text(text), None))
|
||||
latest_item_index = len(self.feedback_box) - 1
|
||||
self.feedback_box.set_focus(latest_item_index)
|
||||
|
||||
def remove_commit(self, commit: CommitWidget) -> None:
|
||||
for i, c in enumerate(self.commit_list):
|
||||
if c.base_widget is commit:
|
||||
del self.commit_list[i]
|
||||
break
|
||||
self.mainloop.draw_screen()
|
||||
|
||||
def save(self):
|
||||
core.save(itertools.chain(self.new_commits, self.previous_commits))
|
||||
|
||||
def add(self) -> None:
|
||||
"""Add an additional commit which isn't nominated."""
|
||||
o = self.mainloop.widget
|
||||
|
||||
def reset_cb(_) -> None:
|
||||
self.mainloop.widget = o
|
||||
self.mainloop.draw_screen()
|
||||
|
||||
async def apply_cb(edit: urwid.Edit) -> None:
|
||||
text: str = edit.get_edit_text()
|
||||
|
||||
# In case the text is empty
|
||||
if not text:
|
||||
return
|
||||
|
||||
sha = await core.full_sha(text)
|
||||
for c in reversed(list(itertools.chain(self.new_commits, self.previous_commits))):
|
||||
if c.sha == sha:
|
||||
commit = c
|
||||
break
|
||||
else:
|
||||
raise RuntimeError(f"Couldn't find {sha}")
|
||||
|
||||
await commit.apply(self)
|
||||
self.mainloop.draw_screen()
|
||||
|
||||
q = urwid.Edit("Commit sha\n")
|
||||
ok_btn = urwid.Button('Ok')
|
||||
urwid.connect_signal(ok_btn, 'click', lambda _: asyncio.ensure_future(apply_cb(q)))
|
||||
urwid.connect_signal(ok_btn, 'click', reset_cb)
|
||||
|
||||
can_btn = urwid.Button('Cancel')
|
||||
urwid.connect_signal(can_btn, 'click', reset_cb)
|
||||
|
||||
cols = urwid.Columns([ok_btn, can_btn])
|
||||
pile = urwid.Pile([q, cols])
|
||||
box = urwid.LineBox(pile)
|
||||
|
||||
self.mainloop.widget = urwid.Overlay(
|
||||
urwid.Filler(box), o, 'center', ('relative', 50), 'middle', ('relative', 50)
|
||||
)
|
||||
self.mainloop.draw_screen()
|
||||
|
||||
def chp_failed(self, commit: 'CommitWidget', err: str) -> None:
|
||||
o = self.mainloop.widget
|
||||
|
||||
def reset_cb(_) -> None:
|
||||
self.mainloop.widget = o
|
||||
self.mainloop.draw_screen()
|
||||
|
||||
t = urwid.Text(textwrap.dedent(f"""
|
||||
Failed to apply {commit.commit.sha} {commit.commit.description} with the following error:
|
||||
|
||||
{err}
|
||||
|
||||
You can either cancel, or resolve the conflicts (`git mergetool`), finish the
|
||||
cherry-pick (`git cherry-pick --continue`) and select ok."""))
|
||||
|
||||
can_btn = urwid.Button('Cancel')
|
||||
urwid.connect_signal(can_btn, 'click', reset_cb)
|
||||
urwid.connect_signal(
|
||||
can_btn, 'click', lambda _: asyncio.ensure_future(commit.commit.abort_cherry(self, err)))
|
||||
|
||||
ok_btn = urwid.Button('Ok')
|
||||
urwid.connect_signal(ok_btn, 'click', reset_cb)
|
||||
urwid.connect_signal(
|
||||
ok_btn, 'click', lambda _: asyncio.ensure_future(commit.commit.resolve(self)))
|
||||
urwid.connect_signal(
|
||||
ok_btn, 'click', lambda _: self.remove_commit(commit))
|
||||
|
||||
cols = urwid.Columns([ok_btn, can_btn])
|
||||
pile = urwid.Pile([t, cols])
|
||||
box = urwid.LineBox(pile)
|
||||
|
||||
self.mainloop.widget = urwid.Overlay(
|
||||
urwid.Filler(box), o, 'center', ('relative', 50), 'middle', ('relative', 50)
|
||||
)
|
||||
self.mainloop.draw_screen()
|
||||
Executable
+65
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright © 2019-2020 Intel Corporation
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
"""Update the main page, release notes, and calendar."""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import pathlib
|
||||
import subprocess
|
||||
|
||||
|
||||
def update_calendar(version: str) -> None:
|
||||
p = pathlib.Path('docs') / 'release-calendar.csv'
|
||||
|
||||
with p.open('r') as f:
|
||||
calendar = list(csv.reader(f))
|
||||
|
||||
branch = None
|
||||
for i, line in enumerate(calendar):
|
||||
if line[2] == version:
|
||||
if line[0]:
|
||||
branch = line[0]
|
||||
break
|
||||
if branch is not None:
|
||||
calendar[i + 1][0] = branch
|
||||
del calendar[i]
|
||||
|
||||
with p.open('w') as f:
|
||||
writer = csv.writer(f)
|
||||
writer.writerows(calendar)
|
||||
|
||||
subprocess.run(['git', 'add', p])
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('version', help="The released version.")
|
||||
args = parser.parse_args()
|
||||
|
||||
update_calendar(args.version)
|
||||
|
||||
subprocess.run(['git', 'commit', '-m',
|
||||
f'docs: update calendar for {args.version}'])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,67 @@
|
||||
# Copyright © 2019 Intel Corporation
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from . import post_version
|
||||
|
||||
|
||||
@mock.patch('bin.post_version.subprocess.run', mock.Mock())
|
||||
class TestUpdateCalendar:
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_sideffects(self) -> None:
|
||||
"""Mock out side effects."""
|
||||
with mock.patch('bin.post_version.subprocess.run', mock.Mock()), \
|
||||
mock.patch('bin.post_version.pathlib', mock.MagicMock()):
|
||||
yield
|
||||
|
||||
def test_basic(self):
|
||||
data = [
|
||||
['20.3', '2021-01-13', '20.3.3', 'Dylan Baker', None],
|
||||
[None, '2021-01-27', '20.3.4', 'Dylan Baker', None],
|
||||
]
|
||||
|
||||
m = mock.Mock()
|
||||
with mock.patch('bin.post_version.csv.reader', mock.Mock(return_value=data.copy())), \
|
||||
mock.patch('bin.post_version.csv.writer', mock.Mock(return_value=m)):
|
||||
post_version.update_calendar('20.3.3')
|
||||
|
||||
m.writerows.assert_called_with([data[1]])
|
||||
|
||||
def test_two_releases(self):
|
||||
data = [
|
||||
['20.3', '2021-01-13', '20.3.3', 'Dylan Baker', None],
|
||||
[None, '2021-01-27', '20.3.4', 'Dylan Baker', None],
|
||||
['21.0', '2021-01-13', '21.0.0', 'Dylan Baker', None],
|
||||
[None, '2021-01-13', '21.0.1', 'Dylan Baker', None],
|
||||
]
|
||||
|
||||
m = mock.Mock()
|
||||
with mock.patch('bin.post_version.csv.reader', mock.Mock(return_value=data.copy())), \
|
||||
mock.patch('bin.post_version.csv.writer', mock.Mock(return_value=m)):
|
||||
post_version.update_calendar('20.3.3')
|
||||
|
||||
d = data.copy()
|
||||
del d[0]
|
||||
d[0][0] = '20.3'
|
||||
m.writerows.assert_called_with(d)
|
||||
Executable
+47
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eu
|
||||
|
||||
readonly requirements_file=$1
|
||||
shift
|
||||
|
||||
venv_dir="$(dirname "$requirements_file")"/.venv
|
||||
readonly venv_dir
|
||||
readonly venv_req=$venv_dir/requirements.txt
|
||||
readonly venv_python_version=$venv_dir/python-version.txt
|
||||
|
||||
if [ -d "$venv_dir" ]
|
||||
then
|
||||
if [ ! -r "$venv_python_version" ]
|
||||
then
|
||||
echo "Python environment predates Python version checks."
|
||||
echo "It might be invalid and needs to be regenerated."
|
||||
rm -rf "$venv_dir"
|
||||
elif ! cmp --quiet <(python3 --version) "$venv_python_version"
|
||||
then
|
||||
old=$(cat "$venv_python_version")
|
||||
new=$(python3 --version)
|
||||
echo "Python version has changed ($old -> $new)."
|
||||
echo "Python environment needs to be regenerated."
|
||||
unset old new
|
||||
rm -rf "$venv_dir"
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! [ -r "$venv_dir/bin/activate" ]
|
||||
then
|
||||
echo "Creating Python environment..."
|
||||
python3 -m venv "$venv_dir"
|
||||
python3 --version > "$venv_python_version"
|
||||
fi
|
||||
|
||||
# shellcheck disable=1091
|
||||
source "$venv_dir/bin/activate"
|
||||
|
||||
if ! cmp --quiet "$requirements_file" "$venv_req"
|
||||
then
|
||||
echo "$(realpath --relative-to="$PWD" "$requirements_file") has changed, re-installing..."
|
||||
pip --disable-pip-version-check install --requirement "$requirements_file"
|
||||
cp "$requirements_file" "$venv_req"
|
||||
fi
|
||||
|
||||
python3 "$@"
|
||||
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/python3
|
||||
#
|
||||
# Copyright 2025 Valve Corporation
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import unittest
|
||||
import sys
|
||||
import subprocess
|
||||
import os
|
||||
import shlex
|
||||
from unidecode import unidecode
|
||||
|
||||
def normalize(x):
|
||||
return unidecode(x.lower())
|
||||
|
||||
def name(row):
|
||||
return normalize(row[0])
|
||||
|
||||
def username(row):
|
||||
return normalize(row[2])
|
||||
|
||||
def find_person(x):
|
||||
x = normalize(x)
|
||||
|
||||
filename = 'people.csv'
|
||||
path = os.path.join(os.path.dirname(os.path.realpath(__file__)), filename)
|
||||
with open(path, 'r') as f:
|
||||
people = list(csv.reader(f, skipinitialspace=True))
|
||||
|
||||
# First, try to exactly match username
|
||||
for row in people:
|
||||
if username(row) == x:
|
||||
return row
|
||||
|
||||
# Next, try to exactly match fullname
|
||||
for row in people:
|
||||
if name(row) == x:
|
||||
return row
|
||||
|
||||
# Now we get fuzzy. Try to match a first name.
|
||||
candidates = [r for r in people if name(r).split(' ')[0] == x]
|
||||
if len(candidates) == 1:
|
||||
return candidates[0]
|
||||
|
||||
# Or a last name?
|
||||
candidates = [r for r in people if x in name(r).split(' ')]
|
||||
if len(candidates) == 1:
|
||||
return candidates[0]
|
||||
|
||||
# Well, frick.
|
||||
return None
|
||||
|
||||
# Self-test... is it even worth find a unit test framework for this?
|
||||
TEST_CASES = {
|
||||
'gfxstrand': '[email protected]',
|
||||
'Faith': '[email protected]',
|
||||
'faith': '[email protected]',
|
||||
'alyssa': '[email protected]',
|
||||
'briano': '[email protected]',
|
||||
'schurmann': '[email protected]',
|
||||
'Schürmann': '[email protected]',
|
||||
}
|
||||
|
||||
for test in TEST_CASES:
|
||||
a, b = find_person(test), TEST_CASES[test]
|
||||
if a is None or a[1] != b:
|
||||
print(test, a, b)
|
||||
assert(a is not None and a[1] == b)
|
||||
|
||||
# Now the tool itself
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(
|
||||
prog='rb',
|
||||
description='Add review trailers')
|
||||
parser.add_argument('person', nargs='+', help="Reviewer's username, first name, or full name")
|
||||
parser.add_argument('-a', '--ack', action='store_true', help="Apply an acked-by tag")
|
||||
parser.add_argument('-t', '--test', action='store_true', help="Apply an tested-by tag")
|
||||
parser.add_argument('-d', '--dry-run', action='store_true',
|
||||
help="Print trailer without applying")
|
||||
parser.add_argument('-r', '--rebase', nargs='?',
|
||||
help="Rebase on the specified branch applying tags to all commits")
|
||||
args = parser.parse_args()
|
||||
|
||||
# If we are rebasing, let git reinvoke this script with the rebase related
|
||||
# arguments stripped.
|
||||
if args.rebase is not None:
|
||||
relevant_args = [sys.argv[0]]
|
||||
if args.ack:
|
||||
relevant_args.append("--ack")
|
||||
elif args.test:
|
||||
relevant_args.append("--test")
|
||||
|
||||
relevant_args += args.person
|
||||
|
||||
cmd = f"python3 {' '.join(relevant_args)}"
|
||||
rebase = ['git', 'rebase', '--exec', cmd, args.rebase]
|
||||
|
||||
if args.dry_run:
|
||||
print(' '.join([shlex.quote(s) for s in rebase]))
|
||||
returncode = 0
|
||||
else:
|
||||
returncode = subprocess.run(rebase).returncode
|
||||
|
||||
sys.exit(returncode)
|
||||
|
||||
for p in args.person:
|
||||
person = find_person(p)
|
||||
if person is None:
|
||||
print(f'Could not uniquely identify {p}, skipping')
|
||||
|
||||
trailer = 'Reviewed-by'
|
||||
if args.ack:
|
||||
trailer = 'Acked-by'
|
||||
elif args.test:
|
||||
trailer = 'Tested-by'
|
||||
|
||||
trailer = f'{trailer}: {person[0]} <{person[1]}>'
|
||||
|
||||
if args.dry_run:
|
||||
print(trailer)
|
||||
continue
|
||||
|
||||
diff_index = subprocess.run("git diff-index --quiet --cached HEAD --".split(" "))
|
||||
if diff_index.returncode != 0:
|
||||
print("You have staged changes.")
|
||||
print("Please commit before applying review tags.")
|
||||
sys.exit(1)
|
||||
|
||||
env = os.environ.copy()
|
||||
env['GIT_EDITOR'] = f'git interpret-trailers --trailer "{trailer}" --in-place'
|
||||
subprocess.run(["git", "commit", "--amend"], env=env)
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright © Microsoft Corporation
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a
|
||||
# copy of this software and associated documentation files (the "Software"),
|
||||
# to deal in the Software without restriction, including without limitation
|
||||
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
# and/or sell copies of the Software, and to permit persons to whom the
|
||||
# Software is furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice (including the next
|
||||
# paragraph) shall be included in all copies or substantial portions of the
|
||||
# Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
# IN THE SOFTWARE.
|
||||
|
||||
import argparse
|
||||
|
||||
# Take a log file produced by GALLIUM_REFCNT_LOG, filter it to the objects that
|
||||
# weren't destroyed by the end of the log, and write the results out sorted.
|
||||
# Strips stacks by default to prevent OOM. Could probably be rewritten to walk
|
||||
# the file twice to preserve stacks without OOM, but this was the easy way.
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--input',
|
||||
action='store',
|
||||
required=True,
|
||||
help='path to file containing refcount log')
|
||||
parser.add_argument('--output',
|
||||
action='store',
|
||||
required=True,
|
||||
help='path to trimmed log')
|
||||
parser.add_argument('--filter',
|
||||
help='object type filter')
|
||||
parser.add_argument('--keep-stacks',
|
||||
help='keep stacks, otherwise only headers')
|
||||
args = parser.parse_args()
|
||||
|
||||
objects = {}
|
||||
|
||||
with open(args.input) as in_file:
|
||||
stack = []
|
||||
cur_object = ''
|
||||
|
||||
for line in in_file:
|
||||
if line[0] == '<':
|
||||
|
||||
parts = line.split(' ')
|
||||
prev_object = cur_object
|
||||
cur_object = parts[1]
|
||||
if parts[3].strip() == 'Destroy':
|
||||
if cur_object in objects:
|
||||
del objects[cur_object]
|
||||
else:
|
||||
if parts[3].strip() == 'Create':
|
||||
if (not args.filter) or (args.filter in parts[0]):
|
||||
objects[cur_object] = []
|
||||
if prev_object in objects:
|
||||
objects[prev_object] += stack
|
||||
|
||||
stack = [line]
|
||||
elif args.keep_stacks:
|
||||
stack += line
|
||||
|
||||
with open(args.output, 'wt') as out_file:
|
||||
for stack in objects.values():
|
||||
for stack_line in stack:
|
||||
out_file.write(stack_line)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+31
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import base64
|
||||
import pathlib
|
||||
import requests
|
||||
import subprocess
|
||||
|
||||
def error(msg: str) -> None:
|
||||
print('\033[31m' + msg + '\033[0m')
|
||||
|
||||
if __name__ == '__main__':
|
||||
git_toplevel = subprocess.check_output(['git', 'rev-parse', '--show-toplevel'],
|
||||
stderr=subprocess.DEVNULL).decode("ascii").strip()
|
||||
if not pathlib.Path(git_toplevel).resolve() == pathlib.Path('.').resolve():
|
||||
error('Please run this script from the root folder ({})'.format(git_toplevel))
|
||||
exit(1)
|
||||
|
||||
file = 'include/renderdoc_app.h'
|
||||
url = 'https://raw.githubusercontent.com/baldurk/renderdoc/refs/tags/v1.5/renderdoc/api/app/renderdoc_app.h'
|
||||
|
||||
print('Syncing {}...'.format(file), end=' ', flush=True)
|
||||
req = requests.get(url)
|
||||
|
||||
if not req.ok:
|
||||
error('Failed to retrieve file: {} {}'.format(req.status_code, req.reason))
|
||||
exit(1)
|
||||
|
||||
with open(file, 'wb') as f:
|
||||
f.write(req.content)
|
||||
|
||||
print('Done')
|
||||
@@ -0,0 +1,256 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
|
||||
# This list contains symbols that _might_ be exported for some platforms
|
||||
PLATFORM_SYMBOLS = [
|
||||
'_GLOBAL_OFFSET_TABLE_',
|
||||
'_ITM_deregisterTMCloneTable',
|
||||
'_ITM_registerTMCloneTable',
|
||||
'__bss_end__',
|
||||
'__bss_start',
|
||||
'__bss_start__',
|
||||
'__cxa_allocate_dependent_exception',
|
||||
'__cxa_allocate_exception',
|
||||
'__cxa_begin_catch',
|
||||
'__cxa_call_unexpected',
|
||||
'__cxa_current_exception_type',
|
||||
'__cxa_current_primary_exception',
|
||||
'__cxa_decrement_exception_refcount',
|
||||
'__cxa_deleted_virtual',
|
||||
'__cxa_demangle',
|
||||
'__cxa_end_catch',
|
||||
'__cxa_finalize',
|
||||
'__cxa_free_dependent_exception',
|
||||
'__cxa_free_exception',
|
||||
'__cxa_get_exception_ptr',
|
||||
'__cxa_get_globals',
|
||||
'__cxa_get_globals_fast',
|
||||
'__cxa_guard_abort',
|
||||
'__cxa_guard_acquire',
|
||||
'__cxa_guard_release',
|
||||
'__cxa_increment_exception_refcount',
|
||||
'__cxa_new_handler',
|
||||
'__cxa_pure_virtual',
|
||||
'__cxa_rethrow',
|
||||
'__cxa_rethrow_primary_exception',
|
||||
'__cxa_terminate_handler',
|
||||
'__cxa_throw',
|
||||
'__cxa_uncaught_exception',
|
||||
'__cxa_uncaught_exceptions',
|
||||
'__cxa_unexpected_handler',
|
||||
'__cxa_thread_atexit_impl',
|
||||
'__deregister_frame_info',
|
||||
'__deregister_frame_info_bases',
|
||||
'__dynamic_cast',
|
||||
'__emutls_get_address',
|
||||
'__end__',
|
||||
'__gmon_start__',
|
||||
'__gxx_personality_v0',
|
||||
'__odr_asan._glapi_Context',
|
||||
'__odr_asan._glapi_Dispatch',
|
||||
'__register_frame_info',
|
||||
'__register_frame_info_bases',
|
||||
'_bss_end__',
|
||||
'_edata',
|
||||
'_end',
|
||||
'_fbss',
|
||||
'_fdata',
|
||||
'_fini',
|
||||
'_ftext',
|
||||
'_init',
|
||||
'copy_file_range',
|
||||
'gettid',
|
||||
'pidfd_getpid',
|
||||
'pidfd_spawnp',
|
||||
'posix_spawn_file_actions_addchdir',
|
||||
'posix_spawn_file_actions_addchdir_np',
|
||||
'pthread_mutexattr_destroy',
|
||||
'pthread_mutexattr_init',
|
||||
'pthread_mutexattr_settype',
|
||||
'statx',
|
||||
]
|
||||
|
||||
|
||||
def get_symbols_nm(nm, lib):
|
||||
'''
|
||||
List all the (non platform-specific) symbols exported by the library
|
||||
using `nm`
|
||||
'''
|
||||
symbols = []
|
||||
platform_name = platform.system()
|
||||
output = subprocess.check_output([nm, '-gP', lib],
|
||||
stderr=open(os.devnull, 'w')).decode("ascii")
|
||||
for line in output.splitlines():
|
||||
if line.startswith(' '):
|
||||
continue
|
||||
fields = line.split()
|
||||
if len(fields) >= 2 and fields[1] == 'U':
|
||||
continue
|
||||
symbol_name = fields[0]
|
||||
if platform_name == 'Linux' or platform_name == 'GNU' or platform_name.startswith('GNU/'):
|
||||
if symbol_name.split('@')[0] in PLATFORM_SYMBOLS:
|
||||
continue
|
||||
elif platform_name == 'Darwin':
|
||||
assert symbol_name[0] == '_'
|
||||
symbol_name = symbol_name[1:]
|
||||
symbols.append(symbol_name)
|
||||
return symbols
|
||||
|
||||
|
||||
def get_symbols_dumpbin(dumpbin, lib):
|
||||
'''
|
||||
List all the (non platform-specific) symbols exported by the library
|
||||
using `dumpbin`
|
||||
'''
|
||||
symbols = []
|
||||
output = subprocess.check_output([dumpbin, '/exports', lib],
|
||||
stderr=open(os.devnull, 'w')).decode("ascii")
|
||||
for line in output.splitlines():
|
||||
fields = line.split()
|
||||
# The lines with the symbols are made of at least 4 columns; see details below
|
||||
if len(fields) < 4:
|
||||
continue
|
||||
try:
|
||||
# Making sure the first 3 columns are a dec counter, a hex counter
|
||||
# and a hex address
|
||||
_ = int(fields[0], 10)
|
||||
_ = int(fields[1], 16)
|
||||
_ = int(fields[2], 16)
|
||||
except ValueError:
|
||||
continue
|
||||
symbol_name = fields[3]
|
||||
# De-mangle symbols
|
||||
if symbol_name[0] == '_' and '@' in symbol_name:
|
||||
symbol_name = symbol_name[1:].split('@')[0]
|
||||
symbols.append(symbol_name)
|
||||
return symbols
|
||||
|
||||
def get_symbols_gendef(gendef, lib):
|
||||
'''
|
||||
List all the (non platform-specific) symbols exported by the library
|
||||
using `gendef -`
|
||||
'''
|
||||
symbols = []
|
||||
output = subprocess.check_output([gendef, '-', lib],
|
||||
stderr=open(os.devnull, 'w')).decode("ascii")
|
||||
ordinal_table_found = False
|
||||
for line in output.splitlines():
|
||||
if not ordinal_table_found:
|
||||
if line.strip() == 'EXPORTS':
|
||||
ordinal_table_found = True
|
||||
continue
|
||||
|
||||
symbols.append(line.strip())
|
||||
return symbols
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--symbols-file',
|
||||
action='store',
|
||||
required=True,
|
||||
help='path to file containing symbols')
|
||||
parser.add_argument('--lib',
|
||||
action='store',
|
||||
required=True,
|
||||
help='path to library')
|
||||
parser.add_argument('--nm',
|
||||
action='store',
|
||||
help='path to binary (or name in $PATH)')
|
||||
parser.add_argument('--dumpbin',
|
||||
action='store',
|
||||
help='path to binary (or name in $PATH)')
|
||||
parser.add_argument('--gendef',
|
||||
action='store',
|
||||
help='path to binary (or name in $PATH)')
|
||||
parser.add_argument('--ignore-symbol',
|
||||
action='append',
|
||||
help='do not process this symbol')
|
||||
args = parser.parse_args()
|
||||
|
||||
if platform.system() == 'Windows':
|
||||
if args.dumpbin:
|
||||
lib_symbols = get_symbols_dumpbin(args.dumpbin, args.lib)
|
||||
elif args.gendef:
|
||||
lib_symbols = get_symbols_gendef(args.gendef, args.lib)
|
||||
else:
|
||||
parser.error('--dumpbin is mandatory for msvc, --gendef is mandatory for mingw')
|
||||
else:
|
||||
if not args.nm:
|
||||
parser.error('--nm is mandatory')
|
||||
lib_symbols = get_symbols_nm(args.nm, args.lib)
|
||||
|
||||
mandatory_symbols = []
|
||||
optional_symbols = []
|
||||
with open(args.symbols_file) as symbols_file:
|
||||
qualifier_optional = '(optional)'
|
||||
for line in symbols_file.readlines():
|
||||
|
||||
# Strip comments
|
||||
line = line.split('#')[0]
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
# Line format:
|
||||
# [qualifier] symbol
|
||||
qualifier = None
|
||||
symbol = None
|
||||
|
||||
fields = line.split()
|
||||
if len(fields) == 1:
|
||||
symbol = fields[0]
|
||||
elif len(fields) == 2:
|
||||
qualifier = fields[0]
|
||||
symbol = fields[1]
|
||||
else:
|
||||
print(args.symbols_file + ': invalid format: ' + line)
|
||||
exit(1)
|
||||
|
||||
# The only supported qualifier is 'optional', which means the
|
||||
# symbol doesn't have to be exported by the library
|
||||
if qualifier and not qualifier == qualifier_optional:
|
||||
print(args.symbols_file + ': invalid qualifier: ' + qualifier)
|
||||
exit(1)
|
||||
|
||||
if qualifier == qualifier_optional:
|
||||
optional_symbols.append(symbol)
|
||||
else:
|
||||
mandatory_symbols.append(symbol)
|
||||
|
||||
unknown_symbols = []
|
||||
for symbol in lib_symbols:
|
||||
if symbol in mandatory_symbols:
|
||||
continue
|
||||
if symbol in optional_symbols:
|
||||
continue
|
||||
if args.ignore_symbol and symbol in args.ignore_symbol:
|
||||
continue
|
||||
if symbol[:2] == '_Z':
|
||||
# As ajax found out, the compiler intentionally exports symbols
|
||||
# that we explicitly asked it not to export, and we can't do
|
||||
# anything about it:
|
||||
# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=36022#c4
|
||||
continue
|
||||
unknown_symbols.append(symbol)
|
||||
|
||||
missing_symbols = [
|
||||
sym for sym in mandatory_symbols if sym not in lib_symbols
|
||||
]
|
||||
|
||||
for symbol in unknown_symbols:
|
||||
print(args.lib + ': unknown symbol exported: ' + symbol)
|
||||
|
||||
for symbol in missing_symbols:
|
||||
print(args.lib + ': missing symbol: ' + symbol)
|
||||
|
||||
if unknown_symbols or missing_symbols:
|
||||
exit(1)
|
||||
exit(0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+58
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import pathlib
|
||||
import re
|
||||
|
||||
|
||||
def detect_misleading_indentation(
|
||||
toml_path: str,
|
||||
toml_lines: list[str],
|
||||
) -> bool:
|
||||
issue_detected = False
|
||||
previous_indentation = 0
|
||||
for line_number, line in enumerate(toml_lines, start=1):
|
||||
if match := re.match(r'^(\s*)\S', line):
|
||||
line_indentation = len(match.group(1))
|
||||
if line_indentation < previous_indentation:
|
||||
# Allow de-indenting when starting a new section (`[`) or
|
||||
# terminating a multi-line list (`]`)
|
||||
if not re.match(r'^\s*(\[|\])', line):
|
||||
print(f'{toml_path}:{line_number}: '
|
||||
f'Misleading indentation found')
|
||||
issue_detected = True
|
||||
else:
|
||||
line_indentation = 0
|
||||
previous_indentation = line_indentation
|
||||
|
||||
return issue_detected
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
'toml_files',
|
||||
type=pathlib.Path,
|
||||
nargs=argparse.ZERO_OR_MORE,
|
||||
help='*.toml files to lint (default: src/**/ci/*.toml)',
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.toml_files:
|
||||
args.toml_files = pathlib.Path('src').glob('**/ci/*.toml')
|
||||
|
||||
error = False
|
||||
|
||||
for path in args.toml_files:
|
||||
with path.open('r') as toml_file:
|
||||
toml_lines = toml_file.readlines()
|
||||
if detect_misleading_indentation(path.as_posix(), toml_lines):
|
||||
error = True
|
||||
|
||||
if error:
|
||||
exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+66
@@ -0,0 +1,66 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
if [ ! -e .git ]; then
|
||||
echo must run from top-level directory;
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d platform-hardware-libhardware ]; then
|
||||
git clone --depth 1 https://android.googlesource.com/platform/frameworks/native platform-frameworks-native
|
||||
git clone --depth 1 https://android.googlesource.com/platform/hardware/libhardware platform-hardware-libhardware
|
||||
git clone --depth 1 https://android.googlesource.com/platform/system/core platform-system-core
|
||||
git clone --depth 1 https://android.googlesource.com/platform/system/logging platform-system-logging
|
||||
fi
|
||||
|
||||
dest=include/android_stub
|
||||
|
||||
# Persist the frozen Android N system/window.h for backward compatibility
|
||||
|
||||
cp -av ${dest}/system/window.h platform-system-core/libsystem/include/system
|
||||
|
||||
# Persist the frozen libbacktrace header for backward compatibility, since the
|
||||
# support has been dropped in Android 15.
|
||||
#
|
||||
# TODO: add support for libunwindstack
|
||||
mkdir -p platform-system-unwinding/libbacktrace/include
|
||||
cp -av ${dest}/backtrace platform-system-unwinding/libbacktrace/include/
|
||||
|
||||
rm -rf ${dest}
|
||||
mkdir ${dest}
|
||||
|
||||
|
||||
# These directories contains only the files we need, so copy wholesale
|
||||
|
||||
cp -av \
|
||||
platform-frameworks-native/libs/nativewindow/include/vndk \
|
||||
platform-frameworks-native/libs/nativebase/include/nativebase \
|
||||
platform-system-core/libsync/include/ndk \
|
||||
platform-system-core/libsync/include/sync \
|
||||
platform-system-unwinding/libbacktrace/include/backtrace \
|
||||
${dest}
|
||||
|
||||
|
||||
# We only need a few files from these big directories so just copy those
|
||||
|
||||
mkdir ${dest}/hardware
|
||||
cp -av platform-hardware-libhardware/include_all/hardware/{hardware,gralloc,gralloc1,fb}.h ${dest}/hardware
|
||||
cp -av platform-frameworks-native/vulkan/include/hardware/hwvulkan.h ${dest}/hardware
|
||||
|
||||
mkdir ${dest}/cutils
|
||||
cp -av platform-system-core/libcutils/include/cutils/native_handle.h ${dest}/cutils
|
||||
|
||||
mkdir ${dest}/system
|
||||
cp -av platform-system-core/libsystem/include/system/{graphics*,window.h} ${dest}/system
|
||||
|
||||
# include/android has files from a few different projects
|
||||
|
||||
mkdir ${dest}/android
|
||||
cp -av \
|
||||
platform-frameworks-native/libs/nativewindow/include/android/{data_space,hardware_buffer,native_window}.h \
|
||||
platform-frameworks-native/libs/arect/include/android/rect.h \
|
||||
platform-system-core/libsync/include/android/sync.h \
|
||||
platform-system-logging/liblog/include/android/log.h \
|
||||
${dest}/android
|
||||
|
||||
Reference in new issue
Block a user