mirror of
https://github.com/elwrcl/intel-hard-graphics.git
synced 2026-11-01 11:18:03 +03:00
init
This commit is contained in:
12511 files changed
+6156641
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,4 @@
|
|||||||
|
# Vendored code
|
||||||
|
src/amd/vulkan/radix_sort/*
|
||||||
|
src/gallium/drivers/ethosu/mlw_codec/**/*
|
||||||
|
src/gallium/drivers/ethosu/ethosu_encode_support.h
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
# The following files are opted into `ninja clang-format` and
|
||||||
|
# enforcement in the CI.
|
||||||
|
|
||||||
|
src/gallium/drivers/ethosu/**/*
|
||||||
|
src/gallium/drivers/i915
|
||||||
|
src/gallium/drivers/r300/compiler/*
|
||||||
|
src/gallium/drivers/rocket/**/*
|
||||||
|
src/gallium/targets/teflon/**/*
|
||||||
|
src/gallium/frontends/teflon/**/*
|
||||||
|
src/amd/vulkan/**/*
|
||||||
|
src/amd/compiler/**/*
|
||||||
|
src/egl/**/*
|
||||||
|
src/etnaviv/isa/**/*
|
||||||
|
src/imagination/**/*
|
||||||
|
src/kosmickrisp/**/*
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
# To use this config on you editor, follow the instructions at:
|
||||||
|
# http://editorconfig.org
|
||||||
|
|
||||||
|
root = true
|
||||||
|
|
||||||
|
[*]
|
||||||
|
charset = utf-8
|
||||||
|
insert_final_newline = true
|
||||||
|
tab_width = 8
|
||||||
|
|
||||||
|
[*.{c,h,cpp,hpp,cc,hh,y,yy,cl,glsl}]
|
||||||
|
indent_style = space
|
||||||
|
indent_size = 3
|
||||||
|
max_line_length = 78
|
||||||
|
|
||||||
|
[{Makefile*,*.mk}]
|
||||||
|
indent_style = tab
|
||||||
|
|
||||||
|
[*.py]
|
||||||
|
indent_style = space
|
||||||
|
indent_size = 4
|
||||||
|
|
||||||
|
[*.yml]
|
||||||
|
indent_style = space
|
||||||
|
indent_size = 2
|
||||||
|
|
||||||
|
[*.rst]
|
||||||
|
indent_style = space
|
||||||
|
indent_size = 3
|
||||||
|
|
||||||
|
[*.patch]
|
||||||
|
trim_trailing_whitespace = false
|
||||||
|
|
||||||
|
[{meson.build,meson.options}]
|
||||||
|
indent_style = space
|
||||||
|
indent_size = 2
|
||||||
|
|
||||||
|
[*.ps1]
|
||||||
|
indent_style = space
|
||||||
|
indent_size = 2
|
||||||
|
|
||||||
|
[*.rs]
|
||||||
|
indent_style = space
|
||||||
|
indent_size = 4
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
# Allow using `source` to execute the contents of other files than the one
|
||||||
|
# being checked.
|
||||||
|
external-sources=true
|
||||||
|
|
||||||
|
# Do not print links to wiki pages explaining why this is an issue.
|
||||||
|
wiki-link-count=0
|
||||||
|
|
||||||
|
# All shell scripts are being run using bash, even if they are lacking
|
||||||
|
# a shebang.
|
||||||
|
shell=bash
|
||||||
+182
@@ -0,0 +1,182 @@
|
|||||||
|
# This file contains the GitLab handle of the maintainers/reviewers for
|
||||||
|
# a given file:
|
||||||
|
# https://docs.gitlab.com/user/project/codeowners/
|
||||||
|
#
|
||||||
|
# Consider these as the list of people who want to be involved in MRs
|
||||||
|
# touching these files/folders, and whom you can ask your questions and
|
||||||
|
# tag in issues.
|
||||||
|
#
|
||||||
|
# As of GitLab 14.3, all features surrounding this file are premium-only,
|
||||||
|
# which means this file is only read by humans for now.
|
||||||
|
#
|
||||||
|
# Paths starting with a `/` are relative to the git root, otherwise they
|
||||||
|
# can match any substring of the file's path.
|
||||||
|
# If multiple lines match, only the last one applies; there is no
|
||||||
|
# accumulation.
|
||||||
|
|
||||||
|
|
||||||
|
##################
|
||||||
|
# INFRASTRUCTURE #
|
||||||
|
##################
|
||||||
|
|
||||||
|
# Build system - Meson
|
||||||
|
meson.build @dbaker @eric
|
||||||
|
/meson.options @dbaker @eric
|
||||||
|
/docs/meson.rst @dbaker @eric
|
||||||
|
|
||||||
|
# Build system - Android
|
||||||
|
/android/ @roman.stratiienko
|
||||||
|
/bin/update-android-headers.sh @zzyiwei
|
||||||
|
|
||||||
|
# Compatibility headers
|
||||||
|
/include/c99* @xexaxo
|
||||||
|
/src/c11/ @lygstate
|
||||||
|
|
||||||
|
# Documentation
|
||||||
|
/docs/ @eric @xexaxo
|
||||||
|
|
||||||
|
|
||||||
|
##########
|
||||||
|
# COMMON #
|
||||||
|
##########
|
||||||
|
|
||||||
|
# NIR
|
||||||
|
/src/compiler/nir/ @gfxstrand
|
||||||
|
|
||||||
|
# Vulkan
|
||||||
|
/src/vulkan/ @gfxstrand
|
||||||
|
/include/vulkan/ @gfxstrand
|
||||||
|
|
||||||
|
# Android Gralloc
|
||||||
|
/src/util/u_gralloc/ @roman.stratiienko @zzyiwei
|
||||||
|
|
||||||
|
#############
|
||||||
|
# PLATFORMS #
|
||||||
|
#############
|
||||||
|
|
||||||
|
# EGL
|
||||||
|
/src/egl/ @eric @xexaxo
|
||||||
|
/include/EGL/ @eric @xexaxo
|
||||||
|
|
||||||
|
# EGL - Android support
|
||||||
|
/src/egl/drivers/dri2/platform_android.c @robh @tfiga
|
||||||
|
|
||||||
|
# EGL - Device support
|
||||||
|
/src/egl/drivers/dri2/platform_device.c @xexaxo
|
||||||
|
|
||||||
|
# EGL - Wayland support
|
||||||
|
/src/egl/wayland/ @daniels @eric
|
||||||
|
/src/egl/drivers/dri2/platform_wayland.c @daniels @eric
|
||||||
|
|
||||||
|
# Gallium targets
|
||||||
|
/src/gallium/targets/ @xexaxo
|
||||||
|
|
||||||
|
# GLX
|
||||||
|
/src/glx/ @ajax
|
||||||
|
/include/GL/glx* @ajax
|
||||||
|
|
||||||
|
# GLVND
|
||||||
|
/src/egl/main/eglglvnd.c @kbrenneman
|
||||||
|
/src/egl/main/egldispatchstubs.* @kbrenneman
|
||||||
|
/src/egl/generate/ @kbrenneman
|
||||||
|
/src/glx/*glvnd* @kbrenneman
|
||||||
|
|
||||||
|
# Haiku
|
||||||
|
/src/egl/drivers/haiku/ @kallisti5
|
||||||
|
/src/gallium/frontends/hgl/ @kallisti5
|
||||||
|
/src/gallium/winsys/sw/hgl/ @kallisti5
|
||||||
|
|
||||||
|
# Loader - DRI/classic
|
||||||
|
/src/loader/ @xexaxo
|
||||||
|
|
||||||
|
# Loader - Gallium
|
||||||
|
/src/gallium/auxiliary/pipe-loader/ @xexaxo
|
||||||
|
/src/gallium/auxiliary/target-helpers/ @xexaxo
|
||||||
|
|
||||||
|
# Vulkan WSI - Display
|
||||||
|
/src/vulkan/wsi/wsi_common_display.* @keithp
|
||||||
|
/src/*/vulkan/*_wsi_display.c @keithp
|
||||||
|
|
||||||
|
# Vulkan - Android
|
||||||
|
/include/vulkan/vk_android_native_buffer.h @zzyiwei
|
||||||
|
/src/vulkan/runtime/vk_android.* @zzyiwei
|
||||||
|
|
||||||
|
######
|
||||||
|
# CI #
|
||||||
|
######
|
||||||
|
|
||||||
|
/.gitlab-ci.yml @eric
|
||||||
|
/.gitlab-ci/ @eric
|
||||||
|
gitlab-ci*.yml @eric
|
||||||
|
|
||||||
|
# Broadcom
|
||||||
|
/src/broadcom/ci/ @jasuarez @chema
|
||||||
|
|
||||||
|
# Lima
|
||||||
|
/src/gallium/drivers/lima/ci/ @enunes
|
||||||
|
|
||||||
|
###########
|
||||||
|
# Drivers #
|
||||||
|
###########
|
||||||
|
|
||||||
|
# Asahi
|
||||||
|
/src/asahi/ @alyssa
|
||||||
|
/src/gallium/drivers/asahi/ @alyssa
|
||||||
|
|
||||||
|
# Broadcom
|
||||||
|
/src/broadcom/ @chema @jasuarez @itoral @apinheiro
|
||||||
|
/src/gallium/drivers/v3d/ @chema @jasuarez @itoral @apinheiro
|
||||||
|
/src/gallium/drivers/vc4/ @chema @jasuarez @itoral @apinheiro
|
||||||
|
|
||||||
|
# Freedreno
|
||||||
|
/src/gallium/drivers/freedreno/ @robclark
|
||||||
|
|
||||||
|
# Imagination
|
||||||
|
/include/drm-uapi/pvr_drm.h @aashishc @frankbinns @luigi.santivetti
|
||||||
|
/src/imagination/ @aashishc @frankbinns @luigi.santivetti
|
||||||
|
/src/imagination/rogue/ @simon-perretta-img
|
||||||
|
/src/imagination/pco/ @simon-perretta-img
|
||||||
|
|
||||||
|
# Intel
|
||||||
|
/include/drm-uapi/i915_drm.h @kwg @llandwerlin @idr
|
||||||
|
/include/pci_ids/i*_pci_ids.h @kwg @llandwerlin @idr
|
||||||
|
/src/intel/ @kwg @llandwerlin @idr
|
||||||
|
/src/intel/compiler/jay/ @alyssa @kwg
|
||||||
|
/src/gallium/winsys/iris/ @kwg @llandwerlin @idr
|
||||||
|
/src/gallium/drivers/iris/ @kwg @llandwerlin @idr
|
||||||
|
/src/gallium/drivers/i915/ @anholt
|
||||||
|
|
||||||
|
# KosmicKrisp
|
||||||
|
/src/kosmickrisp @aitor
|
||||||
|
/src/kosmickrisp/compiler @agoldmints
|
||||||
|
|
||||||
|
# Lima
|
||||||
|
/src/gallium/drivers/lima/ @enunes @anarsoul
|
||||||
|
|
||||||
|
# Microsoft
|
||||||
|
/src/microsoft/ @jenatali
|
||||||
|
/src/gallium/drivers/d3d12/ @jenatali
|
||||||
|
|
||||||
|
# Panfrost
|
||||||
|
/src/panfrost/ @bbrezillon @larsivsi @chrpil
|
||||||
|
/src/panfrost/midgard @italove
|
||||||
|
/src/gallium/drivers/panfrost/ @bbrezillon
|
||||||
|
|
||||||
|
# R300
|
||||||
|
/src/gallium/drivers/r300/ @ondracka @gawin
|
||||||
|
|
||||||
|
# Venus
|
||||||
|
/src/virtio/virtio-gpu/venus_hw.h @zzyiwei @olv
|
||||||
|
/src/virtio/venus-protocol/ @zzyiwei @olv
|
||||||
|
/src/virtio/vulkan/ @zzyiwei @olv
|
||||||
|
|
||||||
|
# VirGL - Video
|
||||||
|
/src/gallium/drivers/virgl/virgl_video.* @flynnjiang
|
||||||
|
/src/virtio/virtio-gpu/virgl_video_hw.h @flynnjiang
|
||||||
|
|
||||||
|
# VMware
|
||||||
|
/src/gallium/drivers/svga/ @brianp @charmainel
|
||||||
|
/src/gallium/winsys/svga/ @thomash @drawat
|
||||||
|
|
||||||
|
# Rust util: peripherally uploaded to crates.io
|
||||||
|
src/util/rust @gurchetansingh @aruby @jmacnak @dextero1
|
||||||
@@ -0,0 +1,214 @@
|
|||||||
|
# Mesa 3-D graphics library
|
||||||
|
#
|
||||||
|
# Copyright (C) 2021 GlobalLogic Ukraine
|
||||||
|
# Copyright (C) 2021 Roman Stratiienko ([email protected])
|
||||||
|
#
|
||||||
|
# Permission is hereby granted, free of charge, to any person obtaining a
|
||||||
|
# copy of this software and associated documentation files (the "Software"),
|
||||||
|
# to deal in the Software without restriction, including without limitation
|
||||||
|
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||||
|
# and/or sell copies of the Software, and to permit persons to whom the
|
||||||
|
# Software is furnished to do so, subject to the following conditions:
|
||||||
|
#
|
||||||
|
# The above copyright notice and this permission notice shall be included
|
||||||
|
# in all copies or substantial portions of the Software.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||||
|
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||||
|
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||||
|
# DEALINGS IN THE SOFTWARE.
|
||||||
|
|
||||||
|
ifneq ($(filter true, $(BOARD_MESA3D_USES_MESON_BUILD)),)
|
||||||
|
|
||||||
|
LOCAL_PATH := $(call my-dir)
|
||||||
|
MESA3D_TOP := $(dir $(LOCAL_PATH))
|
||||||
|
|
||||||
|
LIBDRM_VERSION = $(shell cat external/libdrm/meson.build | grep -o "\<version\>\s*:\s*'\w*\.\w*\.\w*'" | grep -o "\w*\.\w*\.\w*" | head -1)
|
||||||
|
|
||||||
|
USE_LLVM_SWIFTSHADER := $(if $(wildcard external/llvm-project), false, \
|
||||||
|
$(if $(wildcard external/swiftshader/third_party/llvm-[0-9]*), true, false))
|
||||||
|
|
||||||
|
LLVM_VERSION_MAJOR = $(shell \
|
||||||
|
if [ -f external/llvm-project/cmake/Modules/LLVMVersion.cmake ]; then \
|
||||||
|
grep 'set.LLVM_VERSION_MAJOR ' external/llvm-project/cmake/Modules/LLVMVersion.cmake | grep -o '[0-9]\+'; \
|
||||||
|
elif [ -f external/llvm-project/llvm/CMakeLists.txt ]; then \
|
||||||
|
grep 'set.LLVM_VERSION_MAJOR ' external/llvm-project/llvm/CMakeLists.txt | grep -o '[0-9]\+'; \
|
||||||
|
else \
|
||||||
|
ss_dir="$$(printf '%s\n' $(wildcard external/swiftshader/third_party/llvm-[0-9]*) | sort -V | tail -n1)"; \
|
||||||
|
if [ -n "$$ss_dir" ]; then \
|
||||||
|
echo "$$ss_dir" | sed -E 's|.*llvm-([0-9]+).*|\1|'; \
|
||||||
|
fi; \
|
||||||
|
fi)
|
||||||
|
|
||||||
|
MESA_VK_LIB_SUFFIX_amd := radeon
|
||||||
|
MESA_VK_LIB_SUFFIX_intel := intel
|
||||||
|
MESA_VK_LIB_SUFFIX_intel_hasvk := intel_hasvk
|
||||||
|
MESA_VK_LIB_SUFFIX_freedreno := freedreno
|
||||||
|
MESA_VK_LIB_SUFFIX_broadcom := broadcom
|
||||||
|
MESA_VK_LIB_SUFFIX_panfrost := panfrost
|
||||||
|
MESA_VK_LIB_SUFFIX_virtio := virtio
|
||||||
|
MESA_VK_LIB_SUFFIX_swrast := lvp
|
||||||
|
|
||||||
|
include $(CLEAR_VARS)
|
||||||
|
|
||||||
|
LOCAL_SHARED_LIBRARIES := libc libdl libdrm libm liblog libcutils libz libc++ libnativewindow libsync libhardware
|
||||||
|
LOCAL_STATIC_LIBRARIES := libexpat libarect libelf
|
||||||
|
LOCAL_HEADER_LIBRARIES := libnativebase_headers hwvulkan_headers
|
||||||
|
MESON_GEN_PKGCONFIGS := log cutils expat hardware libdrm:$(LIBDRM_VERSION) nativewindow sync zlib:1.2.11 libelf
|
||||||
|
LOCAL_CFLAGS += $(BOARD_MESA3D_CFLAGS)
|
||||||
|
|
||||||
|
ifneq ($(filter i915 r300,$(BOARD_MESA3D_GALLIUM_DRIVERS)),)
|
||||||
|
MESON_GEN_LLVM_STUB := true
|
||||||
|
endif
|
||||||
|
|
||||||
|
ifneq ($(filter llvmpipe swrast,$(BOARD_MESA3D_GALLIUM_DRIVERS) $(BOARD_MESA3D_VULKAN_DRIVERS)),)
|
||||||
|
ifeq ($(BOARD_MESA3D_FORCE_SOFTPIPE),)
|
||||||
|
MESON_GEN_LLVM_STUB := true
|
||||||
|
endif
|
||||||
|
endif
|
||||||
|
|
||||||
|
ifneq ($(filter zink,$(BOARD_MESA3D_GALLIUM_DRIVERS)),)
|
||||||
|
LOCAL_SHARED_LIBRARIES += libvulkan
|
||||||
|
MESON_GEN_PKGCONFIGS += vulkan
|
||||||
|
endif
|
||||||
|
|
||||||
|
ifneq ($(filter i915,$(BOARD_MESA3D_GALLIUM_DRIVERS)),)
|
||||||
|
LOCAL_SHARED_LIBRARIES += libdrm_intel
|
||||||
|
MESON_GEN_PKGCONFIGS += libdrm_intel:$(LIBDRM_VERSION)
|
||||||
|
endif
|
||||||
|
|
||||||
|
ifneq ($(filter radeonsi,$(BOARD_MESA3D_GALLIUM_DRIVERS)),)
|
||||||
|
ifneq ($(MESON_GEN_LLVM_STUB),)
|
||||||
|
LOCAL_CFLAGS += -DFORCE_BUILD_AMDGPU # instructs LLVM to declare LLVMInitializeAMDGPU* functions
|
||||||
|
# The flag is required for the Android-x86 LLVM port that follows the AOSP LLVM porting rules
|
||||||
|
# https://osdn.net/projects/android-x86/scm/git/external-llvm-project
|
||||||
|
endif
|
||||||
|
endif
|
||||||
|
|
||||||
|
ifneq ($(filter radeonsi amd,$(BOARD_MESA3D_GALLIUM_DRIVERS) $(BOARD_MESA3D_VULKAN_DRIVERS)),)
|
||||||
|
LOCAL_SHARED_LIBRARIES += libdrm_amdgpu
|
||||||
|
MESON_GEN_PKGCONFIGS += libdrm_amdgpu:$(LIBDRM_VERSION)
|
||||||
|
endif
|
||||||
|
|
||||||
|
ifneq ($(filter radeonsi r300 r600,$(BOARD_MESA3D_GALLIUM_DRIVERS)),)
|
||||||
|
LOCAL_SHARED_LIBRARIES += libdrm_radeon
|
||||||
|
MESON_GEN_PKGCONFIGS += libdrm_radeon:$(LIBDRM_VERSION)
|
||||||
|
endif
|
||||||
|
|
||||||
|
ifneq ($(filter d3d12,$(BOARD_MESA3D_GALLIUM_DRIVERS)),)
|
||||||
|
LOCAL_HEADER_LIBRARIES += DirectX-Headers
|
||||||
|
LOCAL_STATIC_LIBRARIES += DirectX-Guids
|
||||||
|
MESON_GEN_PKGCONFIGS += DirectX-Headers
|
||||||
|
endif
|
||||||
|
|
||||||
|
ifneq ($(MESON_GEN_LLVM_STUB),)
|
||||||
|
MESON_LLVM_VERSION := $(LLVM_VERSION_MAJOR).0.0
|
||||||
|
|
||||||
|
ifeq ($(strip $(USE_LLVM_SWIFTSHADER)),true)
|
||||||
|
LOCAL_C_INCLUDES += $(TOP)/external/swiftshader/third_party/llvm-$(LLVM_VERSION_MAJOR).0/include
|
||||||
|
LOCAL_STATIC_LIBRARIES += libLLVM$(LLVM_VERSION_MAJOR)_swiftshader
|
||||||
|
else
|
||||||
|
LOCAL_SHARED_LIBRARIES += libLLVM$(LLVM_VERSION_MAJOR)
|
||||||
|
endif
|
||||||
|
|
||||||
|
endif
|
||||||
|
|
||||||
|
ifeq ($(shell test $(PLATFORM_SDK_VERSION) -ge 30; echo $$?), 0)
|
||||||
|
LOCAL_SHARED_LIBRARIES += \
|
||||||
|
[email protected] \
|
||||||
|
libgralloctypes \
|
||||||
|
libhidlbase \
|
||||||
|
libutils
|
||||||
|
ifeq ($(shell test $(PLATFORM_SDK_VERSION) -ge 35; echo $$?), 0)
|
||||||
|
LOCAL_SHARED_LIBRARIES += libui
|
||||||
|
LOCAL_STATIC_LIBRARIES += libzstd
|
||||||
|
MESON_GEN_PKGCONFIGS += ui
|
||||||
|
endif
|
||||||
|
MESON_GEN_PKGCONFIGS += android.hardware.graphics.mapper:4.0
|
||||||
|
endif
|
||||||
|
|
||||||
|
__MY_SHARED_LIBRARIES := $(LOCAL_SHARED_LIBRARIES)
|
||||||
|
|
||||||
|
ifeq ($(shell test $(PLATFORM_SDK_VERSION) -ge 30; echo $$?), 0)
|
||||||
|
MESA_LIBGBM_NAME := libgbm_mesa
|
||||||
|
else
|
||||||
|
MESA_LIBGBM_NAME := libgbm
|
||||||
|
endif
|
||||||
|
|
||||||
|
ifeq ($(TARGET_IS_64_BIT),true)
|
||||||
|
LOCAL_MULTILIB := 64
|
||||||
|
else
|
||||||
|
LOCAL_MULTILIB := 32
|
||||||
|
endif
|
||||||
|
include $(LOCAL_PATH)/mesa3d_cross.mk
|
||||||
|
|
||||||
|
ifdef TARGET_2ND_ARCH
|
||||||
|
LOCAL_MULTILIB := 32
|
||||||
|
include $(LOCAL_PATH)/mesa3d_cross.mk
|
||||||
|
endif
|
||||||
|
|
||||||
|
#-------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
# $1: name
|
||||||
|
# $2: subdir
|
||||||
|
# $3: source prebuilt
|
||||||
|
# $4: export headers
|
||||||
|
define mesa3d-lib
|
||||||
|
include $(CLEAR_VARS)
|
||||||
|
LOCAL_MODULE_CLASS := SHARED_LIBRARIES
|
||||||
|
LOCAL_MODULE := $1
|
||||||
|
LOCAL_VENDOR_MODULE := true
|
||||||
|
LOCAL_MODULE_RELATIVE_PATH := $2
|
||||||
|
LOCAL_PREBUILT_MODULE_FILE := $($3)
|
||||||
|
LOCAL_MULTILIB := first
|
||||||
|
LOCAL_CHECK_ELF_FILES := false
|
||||||
|
LOCAL_MODULE_SUFFIX := .so
|
||||||
|
LOCAL_SHARED_LIBRARIES := $(__MY_SHARED_LIBRARIES)
|
||||||
|
LOCAL_EXPORT_C_INCLUDE_DIRS := $4
|
||||||
|
include $(BUILD_PREBUILT)
|
||||||
|
|
||||||
|
ifdef TARGET_2ND_ARCH
|
||||||
|
include $(CLEAR_VARS)
|
||||||
|
LOCAL_MODULE_CLASS := SHARED_LIBRARIES
|
||||||
|
LOCAL_MODULE := $1
|
||||||
|
LOCAL_VENDOR_MODULE := true
|
||||||
|
LOCAL_MODULE_RELATIVE_PATH := $2
|
||||||
|
LOCAL_PREBUILT_MODULE_FILE := $(2ND_$3)
|
||||||
|
LOCAL_MULTILIB := 32
|
||||||
|
LOCAL_CHECK_ELF_FILES := false
|
||||||
|
LOCAL_MODULE_SUFFIX := .so
|
||||||
|
LOCAL_SHARED_LIBRARIES := $(__MY_SHARED_LIBRARIES)
|
||||||
|
LOCAL_EXPORT_C_INCLUDE_DIRS := $4
|
||||||
|
include $(BUILD_PREBUILT)
|
||||||
|
endif
|
||||||
|
endef
|
||||||
|
|
||||||
|
ifneq ($(strip $(BOARD_MESA3D_GALLIUM_DRIVERS)),)
|
||||||
|
# Module 'libgallium_dri', produces '/vendor/lib{64}/libgallium_dri.so'
|
||||||
|
# This module also trigger DRI symlinks creation process
|
||||||
|
$(eval $(call mesa3d-lib,libgallium_dri,,MESA3D_GALLIUM_BIN))
|
||||||
|
|
||||||
|
# Module 'libEGL_mesa', produces '/vendor/lib{64}/egl/libEGL_mesa.so'
|
||||||
|
$(eval $(call mesa3d-lib,libEGL_mesa,egl,MESA3D_LIBEGL_BIN))
|
||||||
|
# Module 'libGLESv1_CM_mesa', produces '/vendor/lib{64}/egl/libGLESv1_CM_mesa.so'
|
||||||
|
$(eval $(call mesa3d-lib,libGLESv1_CM_mesa,egl,MESA3D_LIBGLESV1_BIN))
|
||||||
|
# Module 'libGLESv2_mesa', produces '/vendor/lib{64}/egl/libGLESv2_mesa.so'
|
||||||
|
$(eval $(call mesa3d-lib,libGLESv2_mesa,egl,MESA3D_LIBGLESV2_BIN))
|
||||||
|
endif
|
||||||
|
|
||||||
|
# Modules 'vulkan.{driver_name}', produces '/vendor/lib{64}/hw/vulkan.{driver_name}.so' HAL
|
||||||
|
$(foreach driver,$(BOARD_MESA3D_VULKAN_DRIVERS), \
|
||||||
|
$(eval $(call mesa3d-lib,vulkan.$(MESA_VK_LIB_SUFFIX_$(driver)),hw,MESA3D_VULKAN_$(driver)_BIN)))
|
||||||
|
|
||||||
|
ifneq ($(filter true, $(BOARD_MESA3D_BUILD_LIBGBM)),)
|
||||||
|
# Modules 'libgbm', produces '/vendor/lib{64}/libgbm.so'
|
||||||
|
$(eval $(call mesa3d-lib,$(MESA_LIBGBM_NAME),,MESA3D_LIBGBM_BIN,$(MESA3D_TOP)/src/gbm/main))
|
||||||
|
$(eval $(call mesa3d-lib,dri_gbm,,MESA3D_DRI_GBM_BIN))
|
||||||
|
endif
|
||||||
|
|
||||||
|
#-------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
endif
|
||||||
@@ -0,0 +1,309 @@
|
|||||||
|
# Mesa 3-D graphics library
|
||||||
|
#
|
||||||
|
# Copyright (C) 2021 GlobalLogic Ukraine
|
||||||
|
# Copyright (C) 2021 Roman Stratiienko ([email protected])
|
||||||
|
#
|
||||||
|
# Permission is hereby granted, free of charge, to any person obtaining a
|
||||||
|
# copy of this software and associated documentation files (the "Software"),
|
||||||
|
# to deal in the Software without restriction, including without limitation
|
||||||
|
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||||
|
# and/or sell copies of the Software, and to permit persons to whom the
|
||||||
|
# Software is furnished to do so, subject to the following conditions:
|
||||||
|
#
|
||||||
|
# The above copyright notice and this permission notice shall be included
|
||||||
|
# in all copies or substantial portions of the Software.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||||
|
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||||
|
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||||
|
# DEALINGS IN THE SOFTWARE.
|
||||||
|
|
||||||
|
MY_PATH := $(call my-dir)
|
||||||
|
|
||||||
|
AOSP_ABSOLUTE_PATH := $(realpath .)
|
||||||
|
define relative-to-absolute
|
||||||
|
$(if $(patsubst /%,,$1),$(AOSP_ABSOLUTE_PATH)/$1,$1)
|
||||||
|
endef
|
||||||
|
|
||||||
|
LOCAL_MODULE_CLASS := SHARED_LIBRARIES
|
||||||
|
LOCAL_MODULE := meson.dummy.$(LOCAL_MULTILIB)
|
||||||
|
|
||||||
|
m_dummy := $(local-generated-sources-dir)/dummy.c
|
||||||
|
$(m_dummy):
|
||||||
|
mkdir -p $(dir $@)
|
||||||
|
touch $@
|
||||||
|
|
||||||
|
LOCAL_GENERATED_SOURCES := $(m_dummy)
|
||||||
|
LOCAL_VENDOR_MODULE := true
|
||||||
|
|
||||||
|
# Prepare intermediate variables by AOSP make/core internals
|
||||||
|
include $(BUILD_SHARED_LIBRARY)
|
||||||
|
|
||||||
|
LOCAL_PATH := $(MY_PATH)
|
||||||
|
|
||||||
|
link_deps := \
|
||||||
|
$(built_static_libraries) \
|
||||||
|
$(built_shared_libraries) \
|
||||||
|
$(built_whole_libraries) \
|
||||||
|
$(strip $(all_objects)) \
|
||||||
|
$(my_target_libatomic) \
|
||||||
|
$(my_target_libcrt_builtins) \
|
||||||
|
$(my_target_crtbegin_so_o) \
|
||||||
|
$(my_target_crtend_so_o)
|
||||||
|
|
||||||
|
# Build mesa3d using intermediate variables provided by AOSP make/core internals
|
||||||
|
M_TARGET_PREFIX := $(my_2nd_arch_prefix)
|
||||||
|
|
||||||
|
MESA3D_LIB_DIR := lib$(subst 32,,$(LOCAL_MULTILIB))
|
||||||
|
|
||||||
|
MESON_OUT_DIR := $($(M_TARGET_PREFIX)TARGET_OUT_INTERMEDIATES)/MESON_MESA3D
|
||||||
|
MESON_GEN_DIR := $(MESON_OUT_DIR)_GEN
|
||||||
|
MESON_GEN_FILES_TARGET := $(MESON_GEN_DIR)/.timestamp
|
||||||
|
|
||||||
|
MESA3D_GALLIUM_DIR := $(MESON_OUT_DIR)/install/usr/local/lib
|
||||||
|
$(M_TARGET_PREFIX)MESA3D_GALLIUM_BIN := $(MESON_OUT_DIR)/install/usr/local/lib/libgallium_dri.so
|
||||||
|
$(M_TARGET_PREFIX)MESA3D_LIBEGL_BIN := $(MESON_OUT_DIR)/install/usr/local/lib/libEGL.so
|
||||||
|
$(M_TARGET_PREFIX)MESA3D_LIBGLESV1_BIN := $(MESON_OUT_DIR)/install/usr/local/lib/libGLESv1_CM.so
|
||||||
|
$(M_TARGET_PREFIX)MESA3D_LIBGLESV2_BIN := $(MESON_OUT_DIR)/install/usr/local/lib/libGLESv2.so
|
||||||
|
$(M_TARGET_PREFIX)MESA3D_LIBGBM_BIN := $(MESON_OUT_DIR)/install/usr/local/lib/$(MESA_LIBGBM_NAME).so
|
||||||
|
$(M_TARGET_PREFIX)MESA3D_DRI_GBM_BIN := $(MESON_OUT_DIR)/install/usr/local/lib/gbm/dri_gbm.so
|
||||||
|
|
||||||
|
|
||||||
|
MESA3D_GBM_BINS := \
|
||||||
|
$($(M_TARGET_PREFIX)MESA3D_LIBGBM_BIN) \
|
||||||
|
$($(M_TARGET_PREFIX)MESA3D_DRI_GBM_BIN) \
|
||||||
|
|
||||||
|
MESA3D_GLES_BINS := \
|
||||||
|
$($(M_TARGET_PREFIX)MESA3D_GALLIUM_BIN) \
|
||||||
|
$($(M_TARGET_PREFIX)MESA3D_LIBEGL_BIN) \
|
||||||
|
$($(M_TARGET_PREFIX)MESA3D_LIBGLESV1_BIN) \
|
||||||
|
$($(M_TARGET_PREFIX)MESA3D_LIBGLESV2_BIN) \
|
||||||
|
|
||||||
|
MESON_GEN_NINJA := \
|
||||||
|
cd $(MESON_OUT_DIR) && PATH=/usr/bin:/usr/local/bin:$$PATH meson ./build \
|
||||||
|
--cross-file $(call relative-to-absolute,$(MESON_GEN_DIR))/aosp_cross \
|
||||||
|
--buildtype=release \
|
||||||
|
-Dplatforms=android \
|
||||||
|
-Dplatform-sdk-version=$(PLATFORM_SDK_VERSION) \
|
||||||
|
-Dgallium-drivers=$(subst $(space),$(comma),$(BOARD_MESA3D_GALLIUM_DRIVERS)) \
|
||||||
|
-Dvulkan-drivers=$(subst $(space),$(comma),$(subst radeon,amd,$(BOARD_MESA3D_VULKAN_DRIVERS))) \
|
||||||
|
-Dgbm=enabled \
|
||||||
|
-Dgbm-backends-path=/vendor/$(MESA3D_LIB_DIR) \
|
||||||
|
-Degl=$(if $(BOARD_MESA3D_GALLIUM_DRIVERS),enabled,disabled) \
|
||||||
|
-Dllvm=$(if $(MESON_GEN_LLVM_STUB),enabled,disabled) \
|
||||||
|
-Dcpp_rtti=false \
|
||||||
|
-Dlmsensors=disabled \
|
||||||
|
-Dandroid-libbacktrace=disabled \
|
||||||
|
$(BOARD_MESA3D_MESON_ARGS) \
|
||||||
|
|
||||||
|
MESON_BUILD := PATH=/usr/bin:/bin:/sbin:$$PATH ninja -C $(MESON_OUT_DIR)/build
|
||||||
|
|
||||||
|
$(MESON_GEN_FILES_TARGET): MESON_CPU_FAMILY := $(subst arm64,aarch64,$(TARGET_$(M_TARGET_PREFIX)ARCH))
|
||||||
|
|
||||||
|
define create-pkgconfig
|
||||||
|
echo -e "Name: $2" \
|
||||||
|
"\nDescription: $2" \
|
||||||
|
"\nVersion: $3" > $1/$2.pc
|
||||||
|
|
||||||
|
endef
|
||||||
|
|
||||||
|
# Taken from build/make/core/binary.mk. We need this
|
||||||
|
# to use definitions from build/make/core/definitions.mk
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_GLOBAL_C_INCLUDES := $(my_target_global_c_includes)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_GLOBAL_C_SYSTEM_INCLUDES := $(my_target_global_c_system_includes)
|
||||||
|
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_2ND_ARCH_VAR_PREFIX := $(M_TARGET_PREFIX)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_CC := $(my_cc)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_LINKER := $(my_linker)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_CXX := $(my_cxx)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_CXX_LINK := $(my_cxx_link)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_YACCFLAGS := $(LOCAL_YACCFLAGS)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_ASFLAGS := $(my_asflags)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_CONLYFLAGS := $(my_conlyflags)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_CFLAGS := $(my_cflags)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_CPPFLAGS := $(my_cppflags)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_CFLAGS_NO_OVERRIDE := $(my_cflags_no_override)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_CPPFLAGS_NO_OVERRIDE := $(my_cppflags_no_override)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_RTTI_FLAG := $(LOCAL_RTTI_FLAG)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_DEBUG_CFLAGS := $(debug_cflags)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_C_INCLUDES := $(my_c_includes)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_IMPORTED_INCLUDES := $(imported_includes)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_LDFLAGS := $(my_ldflags)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_LDLIBS := $(my_ldlibs)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_TIDY_CHECKS := $(my_tidy_checks)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_TIDY_FLAGS := $(my_tidy_flags)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_ARFLAGS := $(my_arflags)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_ALL_SHARED_LIBRARIES := $(built_shared_libraries)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_ALL_STATIC_LIBRARIES := $(built_static_libraries)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_ALL_WHOLE_STATIC_LIBRARIES := $(built_whole_libraries)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_ALL_OBJECTS := $(strip $(all_objects))
|
||||||
|
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_ARM_CFLAGS := $(normal_objects_cflags)
|
||||||
|
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_TARGET_GLOBAL_CFLAGS := $(my_target_global_cflags)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_TARGET_GLOBAL_CONLYFLAGS := $(my_target_global_conlyflags)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_TARGET_GLOBAL_CPPFLAGS := $(my_target_global_cppflags)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_TARGET_GLOBAL_LDFLAGS := $(my_target_global_ldflags)
|
||||||
|
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_TARGET_LIBCRT_BUILTINS := $(my_target_libcrt_builtins)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_TARGET_LIBATOMIC := $(my_target_libatomic)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_TARGET_CRTBEGIN_SO_O := $(my_target_crtbegin_so_o)
|
||||||
|
$(MESON_GEN_FILES_TARGET): PRIVATE_TARGET_CRTEND_SO_O := $(my_target_crtend_so_o)
|
||||||
|
##
|
||||||
|
|
||||||
|
define m-lld-flags
|
||||||
|
-Wl,-e,main \
|
||||||
|
-nostdlib -Wl,--gc-sections \
|
||||||
|
$(PRIVATE_TARGET_CRTBEGIN_SO_O) \
|
||||||
|
$(PRIVATE_ALL_OBJECTS) \
|
||||||
|
-Wl,--whole-archive \
|
||||||
|
$(PRIVATE_ALL_WHOLE_STATIC_LIBRARIES) \
|
||||||
|
-Wl,--no-whole-archive \
|
||||||
|
$(if $(PRIVATE_GROUP_STATIC_LIBRARIES),-Wl$(comma)--start-group) \
|
||||||
|
$(PRIVATE_ALL_STATIC_LIBRARIES) \
|
||||||
|
$(if $(PRIVATE_GROUP_STATIC_LIBRARIES),-Wl$(comma)--end-group) \
|
||||||
|
$(if $(filter true,$(NATIVE_COVERAGE)),$(PRIVATE_TARGET_COVERAGE_LIB)) \
|
||||||
|
$(PRIVATE_TARGET_LIBCRT_BUILTINS) \
|
||||||
|
$(PRIVATE_TARGET_LIBATOMIC) \
|
||||||
|
$(PRIVATE_TARGET_GLOBAL_LDFLAGS) \
|
||||||
|
$(PRIVATE_LDFLAGS) \
|
||||||
|
$(PRIVATE_ALL_SHARED_LIBRARIES) \
|
||||||
|
$(PRIVATE_TARGET_CRTEND_SO_O) \
|
||||||
|
$(PRIVATE_LDLIBS)
|
||||||
|
endef
|
||||||
|
|
||||||
|
define m-lld-flags-cleaned
|
||||||
|
$(patsubst -Wl$(comma)--build-id=%,, \
|
||||||
|
$(subst prebuilts/,$(AOSP_ABSOLUTE_PATH)/prebuilts/, \
|
||||||
|
$(subst $(OUT_DIR)/,$(call relative-to-absolute,$(OUT_DIR))/, \
|
||||||
|
$(subst -Wl$(comma)--fatal-warnings,, \
|
||||||
|
$(subst -Wl$(comma)--no-undefined-version,, \
|
||||||
|
$(subst -Wl$(comma)--gc-sections,, \
|
||||||
|
$(patsubst %dummy.o,, \
|
||||||
|
$(m-lld-flags))))))))
|
||||||
|
endef
|
||||||
|
|
||||||
|
define m-cpp-flags
|
||||||
|
$(PRIVATE_TARGET_GLOBAL_CFLAGS) \
|
||||||
|
$(PRIVATE_TARGET_GLOBAL_CPPFLAGS) \
|
||||||
|
$(PRIVATE_ARM_CFLAGS) \
|
||||||
|
$(PRIVATE_RTTI_FLAG) \
|
||||||
|
$(PRIVATE_CFLAGS) \
|
||||||
|
$(PRIVATE_CPPFLAGS) \
|
||||||
|
$(PRIVATE_DEBUG_CFLAGS) \
|
||||||
|
$(PRIVATE_CFLAGS_NO_OVERRIDE) \
|
||||||
|
$(PRIVATE_CPPFLAGS_NO_OVERRIDE)
|
||||||
|
endef
|
||||||
|
|
||||||
|
define m-c-flags
|
||||||
|
$(PRIVATE_TARGET_GLOBAL_CFLAGS) \
|
||||||
|
$(PRIVATE_TARGET_GLOBAL_CONLYFLAGS) \
|
||||||
|
$(PRIVATE_ARM_CFLAGS) \
|
||||||
|
$(PRIVATE_CFLAGS) \
|
||||||
|
$(PRIVATE_CONLYFLAGS) \
|
||||||
|
$(PRIVATE_DEBUG_CFLAGS) \
|
||||||
|
$(PRIVATE_CFLAGS_NO_OVERRIDE)
|
||||||
|
endef
|
||||||
|
|
||||||
|
define filter-c-flags
|
||||||
|
$(filter-out -std=gnu++17 -std=gnu++14 -std=gnu99 -fno-rtti \
|
||||||
|
-enable-trivial-auto-var-init-zero-knowing-it-will-be-removed-from-clang \
|
||||||
|
-ftrivial-auto-var-init=zero,
|
||||||
|
$(patsubst -W%,, $1))
|
||||||
|
endef
|
||||||
|
|
||||||
|
define nospace-includes
|
||||||
|
$(subst $(space)-isystem$(space),$(space)-isystem, \
|
||||||
|
$(subst $(space)-I$(space),$(space)-I, \
|
||||||
|
$(strip $(c-includes))))
|
||||||
|
endef
|
||||||
|
|
||||||
|
# Ensure include paths are always absolute
|
||||||
|
# When OUT_DIR_COMMON_BASE env variable is set the AOSP/KATI will use absolute paths
|
||||||
|
# for headers in intermediate output directories, but relative for all others.
|
||||||
|
define abs-include
|
||||||
|
$(strip \
|
||||||
|
$(if $(patsubst -I%,,$1),\
|
||||||
|
$(if $(patsubst -isystem/%,,$1),\
|
||||||
|
$(subst -isystem,-isystem$(AOSP_ABSOLUTE_PATH)/,$1),\
|
||||||
|
$1\
|
||||||
|
),\
|
||||||
|
$(if $(patsubst -I/%,,$1),\
|
||||||
|
$(subst -I,-I$(AOSP_ABSOLUTE_PATH)/,$1),\
|
||||||
|
$1\
|
||||||
|
)\
|
||||||
|
)
|
||||||
|
)
|
||||||
|
endef
|
||||||
|
|
||||||
|
$(MESON_GEN_FILES_TARGET): PREPROCESS_MESON_CONFIGS:=$(PREPROCESS_MESON_CONFIGS)
|
||||||
|
$(MESON_GEN_FILES_TARGET): MESON_GEN_DIR:=$(MESON_GEN_DIR)
|
||||||
|
$(MESON_GEN_FILES_TARGET): $(sort $(shell find -L $(MESA3D_TOP) -not -path '*/\.*'))
|
||||||
|
mkdir -p $(dir $@)
|
||||||
|
echo -e "[properties]\n" \
|
||||||
|
"c_args = [$(foreach flag,$(call filter-c-flags,$(m-c-flags)),'$(flag)', ) \
|
||||||
|
$(foreach inc,$(nospace-includes),'$(call abs-include,$(inc))', )'']\n" \
|
||||||
|
"cpp_args = [$(foreach flag,$(call filter-c-flags,$(m-cpp-flags)),'$(flag)', ) \
|
||||||
|
$(foreach inc,$(nospace-includes),'$(call abs-include,$(inc))', )'']\n" \
|
||||||
|
"c_link_args = [$(foreach flag, $(m-lld-flags-cleaned),'$(flag)',)'']\n" \
|
||||||
|
"cpp_link_args = [$(foreach flag, $(m-lld-flags-cleaned),'$(flag)',)'']\n" \
|
||||||
|
"needs_exe_wrapper = true\n" \
|
||||||
|
"[binaries]\n" \
|
||||||
|
"ar = '$(AOSP_ABSOLUTE_PATH)/$($($(M_TARGET_PREFIX))TARGET_AR)'\n" \
|
||||||
|
"c = [$(foreach arg,$(PRIVATE_CC),'$(subst prebuilts/,$(AOSP_ABSOLUTE_PATH)/prebuilts/,$(arg))',)'']\n" \
|
||||||
|
"cpp = [$(foreach arg,$(PRIVATE_CXX),'$(subst prebuilts/,$(AOSP_ABSOLUTE_PATH)/prebuilts/,$(arg))',)'']\n" \
|
||||||
|
"c_ld = 'lld'\n" \
|
||||||
|
"cpp_ld = 'lld'\n\n" \
|
||||||
|
"pkgconfig = ['env', 'PKG_CONFIG_LIBDIR=' + '$(call relative-to-absolute,$(MESON_GEN_DIR))', '/usr/bin/pkg-config']\n\n" \
|
||||||
|
"llvm-config = '/dev/null'\n" \
|
||||||
|
"[host_machine]\n" \
|
||||||
|
"system = 'android'\n" \
|
||||||
|
"cpu_family = '$(MESON_CPU_FAMILY)'\n" \
|
||||||
|
"cpu = '$(MESON_CPU_FAMILY)'\n" \
|
||||||
|
"endian = 'little'" > $(dir $@)/aosp_cross
|
||||||
|
|
||||||
|
#
|
||||||
|
$(foreach pkg, $(MESON_GEN_PKGCONFIGS), $(call create-pkgconfig,$(dir $@),$(word 1, $(subst :, ,$(pkg))),$(word 2, $(subst :, ,$(pkg)))))
|
||||||
|
touch $@
|
||||||
|
|
||||||
|
$(MESON_OUT_DIR)/.build.timestamp: MESON_GEN_NINJA:=$(MESON_GEN_NINJA)
|
||||||
|
$(MESON_OUT_DIR)/.build.timestamp: MESON_BUILD:=$(MESON_BUILD)
|
||||||
|
$(MESON_OUT_DIR)/.build.timestamp: $(MESON_GEN_FILES_TARGET) $(link_deps)
|
||||||
|
rm -rf $(dir $@)
|
||||||
|
mkdir -p $(dir $@)
|
||||||
|
mkdir -p $(dir $@)/build
|
||||||
|
# Meson will update timestamps in sources directory, continuously retriggering the build
|
||||||
|
# even if nothing changed. Copy sources into intermediate dir to avoid this effect.
|
||||||
|
cp -r $(MESA3D_TOP)/* $(dir $@)
|
||||||
|
ifneq ($(MESON_GEN_LLVM_STUB),)
|
||||||
|
mkdir -p $(dir $@)/subprojects/llvm/
|
||||||
|
echo -e "project('llvm', 'cpp', version : '$(MESON_LLVM_VERSION)')\n" \
|
||||||
|
"dep_llvm = declare_dependency()\n" \
|
||||||
|
"has_rtti = false\n" > $(dir $@)/subprojects/llvm/meson.build
|
||||||
|
endif
|
||||||
|
$(MESON_GEN_NINJA)
|
||||||
|
$(MESON_BUILD)
|
||||||
|
touch $@
|
||||||
|
|
||||||
|
$(MESON_OUT_DIR)/install/.install.timestamp: MESON_BUILD:=$(MESON_BUILD)
|
||||||
|
$(MESON_OUT_DIR)/install/.install.timestamp: $(MESON_OUT_DIR)/.build.timestamp
|
||||||
|
rm -rf $(dir $@)
|
||||||
|
mkdir -p $(dir $@)
|
||||||
|
DESTDIR=$(call relative-to-absolute,$(dir $@)) $(MESON_BUILD) install
|
||||||
|
touch $@
|
||||||
|
|
||||||
|
$(MESA3D_GBM_BINS) $(MESA3D_GLES_BINS): $(MESON_OUT_DIR)/install/.install.timestamp
|
||||||
|
echo "Build $@"
|
||||||
|
touch $@
|
||||||
|
|
||||||
|
define vulkan_target
|
||||||
|
$(M_TARGET_PREFIX)MESA3D_VULKAN_$1_BIN := $(MESON_OUT_DIR)/install/usr/local/lib/libvulkan_$(MESA_VK_LIB_SUFFIX_$1).so
|
||||||
|
$(MESON_OUT_DIR)/install/usr/local/lib/libvulkan_$(MESA_VK_LIB_SUFFIX_$1).so: $(MESON_OUT_DIR)/install/.install.timestamp
|
||||||
|
touch $(MESON_OUT_DIR)/install/usr/local/lib/libvulkan_$(MESA_VK_LIB_SUFFIX_$1).so
|
||||||
|
|
||||||
|
endef
|
||||||
|
|
||||||
|
$(foreach driver,$(BOARD_MESA3D_VULKAN_DRIVERS), $(eval $(call vulkan_target,$(driver))))
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
[*.sh]
|
||||||
|
indent_style = tab
|
||||||
Whitespace-only changes.
@@ -0,0 +1,2 @@
|
|||||||
|
schema.graphql
|
||||||
|
gitlab_gql.py.cache*
|
||||||
Executable
+202
@@ -0,0 +1,202 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# Copyright © 2023 Collabora Ltd.
|
||||||
|
# Authors:
|
||||||
|
# Helen Koike <[email protected]>
|
||||||
|
#
|
||||||
|
# For the dependencies, see the requirements.txt
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from typing import Dict, List
|
||||||
|
|
||||||
|
import plotly.express as px
|
||||||
|
import plotly.graph_objs as go
|
||||||
|
from gitlab import Gitlab, base
|
||||||
|
from gitlab.v4.objects import ProjectPipeline
|
||||||
|
from gitlab_common import (GITLAB_URL, get_gitlab_pipeline_from_url,
|
||||||
|
get_token_from_default_dir, pretty_duration,
|
||||||
|
read_token)
|
||||||
|
|
||||||
|
|
||||||
|
def calculate_queued_at(job) -> datetime:
|
||||||
|
started_at = job.started_at.replace("Z", "+00:00")
|
||||||
|
return datetime.fromisoformat(started_at) - timedelta(seconds=job.queued_duration)
|
||||||
|
|
||||||
|
|
||||||
|
def calculate_time_difference(time1, time2) -> str:
|
||||||
|
if type(time1) is str:
|
||||||
|
time1 = datetime.fromisoformat(time1.replace("Z", "+00:00"))
|
||||||
|
if type(time2) is str:
|
||||||
|
time2 = datetime.fromisoformat(time2.replace("Z", "+00:00"))
|
||||||
|
|
||||||
|
diff = time2 - time1
|
||||||
|
return pretty_duration(diff.seconds)
|
||||||
|
|
||||||
|
|
||||||
|
def create_task_name(job) -> str:
|
||||||
|
status_color = {"success": "green", "failed": "red"}.get(job.status, "grey")
|
||||||
|
return f"{job.name}\t(<span style='color: {status_color}'>{job.status}</span>,<a href='{job.web_url}'>{job.id}</a>)"
|
||||||
|
|
||||||
|
|
||||||
|
def add_gantt_bar(
|
||||||
|
job: base.RESTObject, tasks: List[Dict[str, str | datetime | timedelta]]
|
||||||
|
) -> None:
|
||||||
|
queued_at = calculate_queued_at(job)
|
||||||
|
task_name = create_task_name(job)
|
||||||
|
|
||||||
|
tasks.append(
|
||||||
|
{
|
||||||
|
"Job": task_name,
|
||||||
|
"Start": job.created_at,
|
||||||
|
"Finish": queued_at,
|
||||||
|
"Duration": calculate_time_difference(job.created_at, queued_at),
|
||||||
|
"Phase": "Waiting dependencies",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
tasks.append(
|
||||||
|
{
|
||||||
|
"Job": task_name,
|
||||||
|
"Start": queued_at,
|
||||||
|
"Finish": job.started_at,
|
||||||
|
"Duration": calculate_time_difference(queued_at, job.started_at),
|
||||||
|
"Phase": "Queued",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
if job.finished_at:
|
||||||
|
tasks.append(
|
||||||
|
{
|
||||||
|
"Job": task_name,
|
||||||
|
"Start": job.started_at,
|
||||||
|
"Finish": job.finished_at,
|
||||||
|
"Duration": calculate_time_difference(job.started_at, job.finished_at),
|
||||||
|
"Phase": "Time spent running",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
current_time = datetime.now(timezone.utc).isoformat()
|
||||||
|
tasks.append(
|
||||||
|
{
|
||||||
|
"Job": task_name,
|
||||||
|
"Start": job.started_at,
|
||||||
|
"Finish": current_time,
|
||||||
|
"Duration": calculate_time_difference(job.started_at, current_time),
|
||||||
|
"Phase": "In-Progress",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def generate_gantt_chart(
|
||||||
|
pipeline: ProjectPipeline, ci_timeout: float = 60
|
||||||
|
) -> go.Figure:
|
||||||
|
if pipeline.yaml_errors:
|
||||||
|
raise ValueError("Pipeline YAML errors detected")
|
||||||
|
|
||||||
|
# Convert the data into a list of dictionaries for plotly
|
||||||
|
tasks: List[Dict[str, str | datetime | timedelta]] = []
|
||||||
|
|
||||||
|
for job in pipeline.jobs.list(all=True, include_retried=True):
|
||||||
|
# we can have queued_duration without started_at when a job is canceled
|
||||||
|
if not job.queued_duration or not job.started_at:
|
||||||
|
continue
|
||||||
|
add_gantt_bar(job, tasks)
|
||||||
|
|
||||||
|
# Make it easier to see retried jobs
|
||||||
|
tasks.sort(key=lambda x: x["Job"])
|
||||||
|
|
||||||
|
title = f"Gantt chart of jobs in pipeline <a href='{pipeline.web_url}'>{pipeline.web_url}</a>."
|
||||||
|
title += (
|
||||||
|
f" Total duration {str(timedelta(seconds=pipeline.duration))}"
|
||||||
|
if pipeline.duration
|
||||||
|
else ""
|
||||||
|
)
|
||||||
|
|
||||||
|
# Create a Gantt chart
|
||||||
|
default_colors = px.colors.qualitative.Plotly
|
||||||
|
fig: go.Figure = px.timeline(
|
||||||
|
tasks,
|
||||||
|
x_start="Start",
|
||||||
|
x_end="Finish",
|
||||||
|
y="Job",
|
||||||
|
color="Phase",
|
||||||
|
title=title,
|
||||||
|
hover_data=["Duration"],
|
||||||
|
color_discrete_map={
|
||||||
|
"In-Progress": default_colors[3], # purple
|
||||||
|
"Waiting dependencies": default_colors[0], # blue
|
||||||
|
"Queued": default_colors[1], # red
|
||||||
|
"Time spent running": default_colors[2], # green
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
# Calculate the height dynamically
|
||||||
|
fig.update_layout(height=len(tasks) * 10, yaxis_tickfont_size=14)
|
||||||
|
|
||||||
|
# Add a deadline line to the chart
|
||||||
|
created_at = datetime.fromisoformat(pipeline.created_at.replace("Z", "+00:00"))
|
||||||
|
timeout_at = created_at + timedelta(minutes=ci_timeout)
|
||||||
|
fig.add_vrect(
|
||||||
|
x0=timeout_at,
|
||||||
|
x1=timeout_at,
|
||||||
|
annotation_text=f"{int(ci_timeout)} min Timeout",
|
||||||
|
fillcolor="gray",
|
||||||
|
line_width=2,
|
||||||
|
line_color="gray",
|
||||||
|
line_dash="dash",
|
||||||
|
annotation_position="top left",
|
||||||
|
annotation_textangle=90,
|
||||||
|
)
|
||||||
|
|
||||||
|
return fig
|
||||||
|
|
||||||
|
|
||||||
|
def main(
|
||||||
|
token: str | None,
|
||||||
|
pipeline_url: str,
|
||||||
|
output: str | None,
|
||||||
|
ci_timeout: float = 60,
|
||||||
|
):
|
||||||
|
if token is None:
|
||||||
|
token = get_token_from_default_dir()
|
||||||
|
|
||||||
|
token = read_token(token)
|
||||||
|
gl = Gitlab(url=GITLAB_URL, private_token=token, retry_transient_errors=True)
|
||||||
|
|
||||||
|
pipeline, _ = get_gitlab_pipeline_from_url(gl, pipeline_url)
|
||||||
|
fig: go.Figure = generate_gantt_chart(pipeline, ci_timeout)
|
||||||
|
if output and "htm" in output:
|
||||||
|
fig.write_html(output)
|
||||||
|
elif output:
|
||||||
|
fig.update_layout(width=1000)
|
||||||
|
fig.write_image(output)
|
||||||
|
else:
|
||||||
|
fig.show()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Generate the Gantt chart from a given pipeline."
|
||||||
|
)
|
||||||
|
parser.add_argument("pipeline_url", type=str, help="URLs to the pipeline.")
|
||||||
|
parser.add_argument(
|
||||||
|
"-o",
|
||||||
|
"--output",
|
||||||
|
type=str,
|
||||||
|
help="Output file name. Use html or image suffixes to choose the format.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--token",
|
||||||
|
metavar="token",
|
||||||
|
help="force GitLab token, otherwise it's read from ~/.config/gitlab-token",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--ci-timeout",
|
||||||
|
metavar="ci_timeout",
|
||||||
|
type=float,
|
||||||
|
default=60,
|
||||||
|
help="Time that marge-bot will wait for ci to finish. Defaults to one hour.",
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
main(args.token, args.pipeline_url, args.output, args.ci_timeout)
|
||||||
Executable
+10
@@ -0,0 +1,10 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
|
||||||
|
readonly this_dir
|
||||||
|
|
||||||
|
exec \
|
||||||
|
"$this_dir/../python-venv.sh" \
|
||||||
|
"$this_dir/requirements.txt" \
|
||||||
|
"$this_dir/ci_gantt_chart.py" "$@"
|
||||||
Executable
+223
@@ -0,0 +1,223 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# Copyright © 2023 Collabora Ltd.
|
||||||
|
# Authors:
|
||||||
|
# Helen Koike <[email protected]>
|
||||||
|
#
|
||||||
|
# For the dependencies, see the requirements.txt
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import logging as log
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import traceback
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from typing import Any, Dict
|
||||||
|
|
||||||
|
import gitlab
|
||||||
|
import pytz
|
||||||
|
from ci_gantt_chart import generate_gantt_chart
|
||||||
|
from gitlab import Gitlab
|
||||||
|
from gitlab.base import RESTObject
|
||||||
|
from gitlab.v4.objects import Project, ProjectPipeline
|
||||||
|
from gitlab_common import (GITLAB_URL, get_gitlab_pipeline_from_url,
|
||||||
|
get_token_from_default_dir, read_token)
|
||||||
|
|
||||||
|
|
||||||
|
class MockGanttExit(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
LAST_MARGE_EVENT_FILE = os.path.expanduser("~/.config/last_marge_event")
|
||||||
|
|
||||||
|
|
||||||
|
def read_last_event_date_from_file() -> str:
|
||||||
|
try:
|
||||||
|
with open(LAST_MARGE_EVENT_FILE, "r") as f:
|
||||||
|
last_event_date = f.read().strip()
|
||||||
|
except FileNotFoundError:
|
||||||
|
# 3 days ago
|
||||||
|
last_event_date = (datetime.now() - timedelta(days=3)).isoformat()
|
||||||
|
return last_event_date
|
||||||
|
|
||||||
|
|
||||||
|
def pretty_time(time_str: str) -> str:
|
||||||
|
"""Pretty print time"""
|
||||||
|
local_timezone = datetime.now().astimezone().tzinfo
|
||||||
|
|
||||||
|
time_d = datetime.fromisoformat(time_str.replace("Z", "+00:00")).astimezone(
|
||||||
|
local_timezone
|
||||||
|
)
|
||||||
|
return f'{time_str} ({time_d.strftime("%d %b %Y %Hh%Mm%Ss")} {local_timezone})'
|
||||||
|
|
||||||
|
|
||||||
|
def compose_message(file_name: str, attachment_url: str) -> str:
|
||||||
|
return f"""
|
||||||
|
[{file_name}]({attachment_url})
|
||||||
|
|
||||||
|
<details>
|
||||||
|
<summary>more info</summary>
|
||||||
|
|
||||||
|
This message was generated by the ci_post_gantt.py script, which is running on a server at Collabora.
|
||||||
|
</details>
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def gitlab_upload_file_get_url(gl: Gitlab, project_id: str, filepath: str) -> str:
|
||||||
|
project: Project = gl.projects.get(project_id)
|
||||||
|
uploaded_file: Dict[str, Any] = project.upload(filepath, filepath=filepath)
|
||||||
|
return uploaded_file["url"]
|
||||||
|
|
||||||
|
|
||||||
|
def gitlab_post_reply_to_note(gl: Gitlab, event: RESTObject, reply_message: str):
|
||||||
|
"""
|
||||||
|
Post a reply to a note in thread based on a GitLab event.
|
||||||
|
|
||||||
|
:param gl: The GitLab connection instance.
|
||||||
|
:param event: The event object containing the note details.
|
||||||
|
:param reply_message: The reply message.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
note_id = event.target_id
|
||||||
|
merge_request_iid = event.note["noteable_iid"]
|
||||||
|
|
||||||
|
project = gl.projects.get(event.project_id)
|
||||||
|
merge_request = project.mergerequests.get(merge_request_iid)
|
||||||
|
|
||||||
|
# Find the discussion to which the note belongs
|
||||||
|
discussions = merge_request.discussions.list(iterator=True)
|
||||||
|
target_discussion = next(
|
||||||
|
(
|
||||||
|
d
|
||||||
|
for d in discussions
|
||||||
|
if any(n["id"] == note_id for n in d.attributes["notes"])
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
|
||||||
|
if target_discussion is None:
|
||||||
|
raise ValueError("Discussion for the note not found.")
|
||||||
|
|
||||||
|
# Add a reply to the discussion
|
||||||
|
reply = target_discussion.notes.create({"body": reply_message})
|
||||||
|
return reply
|
||||||
|
|
||||||
|
except gitlab.exceptions.GitlabError as e:
|
||||||
|
log.error(f"Failed to post a reply to '{event.note['body']}': {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def main(
|
||||||
|
token: str | None,
|
||||||
|
since: str | None,
|
||||||
|
marge_user_id: int = 9716,
|
||||||
|
project_ids: list[int] = [176],
|
||||||
|
ci_timeout: float = 60,
|
||||||
|
):
|
||||||
|
log.basicConfig(level=log.INFO)
|
||||||
|
if token is None:
|
||||||
|
token = get_token_from_default_dir()
|
||||||
|
|
||||||
|
token = read_token(token)
|
||||||
|
gl = Gitlab(url=GITLAB_URL, private_token=token, retry_transient_errors=True)
|
||||||
|
|
||||||
|
user = gl.users.get(marge_user_id)
|
||||||
|
last_event_at = since if since else read_last_event_date_from_file()
|
||||||
|
|
||||||
|
log.info(f"Retrieving Marge messages since {pretty_time(last_event_at)}\n")
|
||||||
|
|
||||||
|
# the "after" only considers the "2023-10-24" part, it doesn't consider the time
|
||||||
|
events = user.events.list(
|
||||||
|
all=True,
|
||||||
|
target_type="note",
|
||||||
|
after=(datetime.now() - timedelta(days=3)).isoformat(),
|
||||||
|
sort="asc",
|
||||||
|
)
|
||||||
|
|
||||||
|
last_event_at_date = datetime.fromisoformat(
|
||||||
|
last_event_at.replace("Z", "+00:00")
|
||||||
|
).replace(tzinfo=pytz.UTC)
|
||||||
|
|
||||||
|
for event in events:
|
||||||
|
if event.project_id not in project_ids:
|
||||||
|
continue
|
||||||
|
created_at_date = datetime.fromisoformat(
|
||||||
|
event.created_at.replace("Z", "+00:00")
|
||||||
|
).replace(tzinfo=pytz.UTC)
|
||||||
|
if created_at_date <= last_event_at_date:
|
||||||
|
continue
|
||||||
|
last_event_at = event.created_at
|
||||||
|
|
||||||
|
escaped_gitlab_url = re.escape(GITLAB_URL)
|
||||||
|
match = re.search(rf"{escaped_gitlab_url}/[^\s<]+", event.note["body"])
|
||||||
|
|
||||||
|
if match:
|
||||||
|
try:
|
||||||
|
log.info(f"Found message: {event.note['body']}")
|
||||||
|
pipeline_url = match.group(0)[:-1]
|
||||||
|
pipeline: ProjectPipeline
|
||||||
|
pipeline, _ = get_gitlab_pipeline_from_url(gl, pipeline_url)
|
||||||
|
log.info("Generating gantt chart...")
|
||||||
|
fig = generate_gantt_chart(pipeline, ci_timeout)
|
||||||
|
file_name = f"{str(pipeline.id)}-Gantt.html"
|
||||||
|
fig.write_html(file_name)
|
||||||
|
log.info("Uploading gantt file...")
|
||||||
|
file_url = gitlab_upload_file_get_url(gl, event.project_id, file_name)
|
||||||
|
log.info("Posting reply ...")
|
||||||
|
message = compose_message(file_name, file_url)
|
||||||
|
gitlab_post_reply_to_note(gl, event, message)
|
||||||
|
except MockGanttExit:
|
||||||
|
pass # Allow tests to exit early without printing a traceback
|
||||||
|
except Exception as e:
|
||||||
|
log.info(f"Failed to generate gantt chart, not posting reply.{e}")
|
||||||
|
traceback.print_exc()
|
||||||
|
|
||||||
|
if not since:
|
||||||
|
log.info(
|
||||||
|
f"Updating last event date to {pretty_time(last_event_at)} on {LAST_MARGE_EVENT_FILE}\n"
|
||||||
|
)
|
||||||
|
with open(LAST_MARGE_EVENT_FILE, "w") as f:
|
||||||
|
f.write(last_event_at)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
parser = argparse.ArgumentParser(description="Monitor rejected pipelines by Marge.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--token",
|
||||||
|
metavar="token",
|
||||||
|
type=str,
|
||||||
|
default=None,
|
||||||
|
help="force GitLab token, otherwise it's read from ~/.config/gitlab-token",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--since",
|
||||||
|
metavar="since",
|
||||||
|
type=str,
|
||||||
|
default=None,
|
||||||
|
help="consider only events after this date (ISO format), otherwise it's read from ~/.config/last_marge_event",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--marge-user-id",
|
||||||
|
metavar="marge_user_id",
|
||||||
|
type=int,
|
||||||
|
default=9716, # default https://gitlab.freedesktop.org/users/marge-bot/activity
|
||||||
|
help="GitLab user ID for marge-bot, defaults to 9716",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--project-id",
|
||||||
|
metavar="project_id",
|
||||||
|
type=int,
|
||||||
|
nargs="+",
|
||||||
|
default=[176], # default is the mesa/mesa project id
|
||||||
|
help="GitLab project id(s) to analyze. Defaults to 176 i.e. mesa/mesa.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--ci-timeout",
|
||||||
|
metavar="ci_timeout",
|
||||||
|
type=float,
|
||||||
|
default=60,
|
||||||
|
help="Time that marge-bot will wait for ci to finish. Defaults to one hour.",
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
main(args.token, args.since, args.marge_user_id, args.project_id, args.ci_timeout)
|
||||||
Executable
+10
@@ -0,0 +1,10 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
|
||||||
|
readonly this_dir
|
||||||
|
|
||||||
|
exec \
|
||||||
|
"$this_dir/../python-venv.sh" \
|
||||||
|
"$this_dir/requirements.txt" \
|
||||||
|
"$this_dir/ci_post_gantt.py" "$@"
|
||||||
Executable
+808
@@ -0,0 +1,808 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# Copyright © 2020 - 2022 Collabora Ltd.
|
||||||
|
# Authors:
|
||||||
|
# Tomeu Vizoso <[email protected]>
|
||||||
|
# David Heidelberg <[email protected]>
|
||||||
|
#
|
||||||
|
# For the dependencies, see the requirements.txt
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
"""
|
||||||
|
Helper script to restrict running only required CI jobs
|
||||||
|
and show the job(s) logs.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from collections import defaultdict, Counter
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from functools import partial
|
||||||
|
from itertools import chain
|
||||||
|
from subprocess import check_output, CalledProcessError
|
||||||
|
from typing import Callable, Dict, TYPE_CHECKING, Iterable, Literal, Optional, Tuple, cast
|
||||||
|
|
||||||
|
import gitlab
|
||||||
|
import gitlab.v4.objects
|
||||||
|
from gitlab_common import (
|
||||||
|
GITLAB_URL,
|
||||||
|
TOKEN_DIR,
|
||||||
|
get_gitlab_pipeline_from_url,
|
||||||
|
get_gitlab_project,
|
||||||
|
get_token_from_default_dir,
|
||||||
|
is_gitlab_job,
|
||||||
|
pretty_duration,
|
||||||
|
read_token,
|
||||||
|
wait_for_pipeline,
|
||||||
|
)
|
||||||
|
from gitlab_gql import GitlabGQL, create_job_needs_dag, filter_dag, print_dag, print_formatted_list
|
||||||
|
from rich.console import Console
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from gitlab_gql import Dag
|
||||||
|
|
||||||
|
REFRESH_WAIT_LOG = 10
|
||||||
|
REFRESH_WAIT_JOBS = 6
|
||||||
|
MAX_ENABLE_JOB_ATTEMPTS = 3
|
||||||
|
|
||||||
|
STATUS_COLORS = defaultdict(lambda: "", {
|
||||||
|
"running": "[blue]",
|
||||||
|
"success": "[green]",
|
||||||
|
"failed": "[red]",
|
||||||
|
"canceled": "[magenta]",
|
||||||
|
"canceling": "[magenta]",
|
||||||
|
})
|
||||||
|
|
||||||
|
COMPLETED_STATUSES = frozenset({"success", "failed"})
|
||||||
|
RUNNING_STATUSES = frozenset({"created", "pending", "running"})
|
||||||
|
|
||||||
|
if is_gitlab_job():
|
||||||
|
console = Console(highlight=False, no_color=False, color_system="truecolor", width=120)
|
||||||
|
else:
|
||||||
|
console = Console(highlight=False)
|
||||||
|
print = console.print
|
||||||
|
|
||||||
|
|
||||||
|
def print_job_status(
|
||||||
|
job: gitlab.v4.objects.ProjectPipelineJob,
|
||||||
|
new_status: bool = False,
|
||||||
|
) -> None:
|
||||||
|
"""It prints a nice, colored job status with a link to the job."""
|
||||||
|
if job.status in {"canceled", "canceling"}:
|
||||||
|
return
|
||||||
|
|
||||||
|
if new_status and job.status == "created":
|
||||||
|
return
|
||||||
|
|
||||||
|
global type_field_pad
|
||||||
|
global name_field_pad
|
||||||
|
jtype = "🞋 job"
|
||||||
|
job_name = job.name
|
||||||
|
type_field_pad = len(jtype) if len(jtype) > type_field_pad else type_field_pad
|
||||||
|
name_field_pad = len(job_name) if len(job_name) > name_field_pad else name_field_pad
|
||||||
|
|
||||||
|
duration = job_duration(job)
|
||||||
|
|
||||||
|
print(
|
||||||
|
f"{STATUS_COLORS[job.status]}"
|
||||||
|
f"{jtype:{type_field_pad}} " # U+1F78B Round target
|
||||||
|
f"{link2print(job.web_url, job.name, name_field_pad)} " +
|
||||||
|
(f" has new status: {job.status}" if new_status else f" {job.status}") +
|
||||||
|
(f" ({pretty_duration(duration)})" if job.started_at else "")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def job_duration(job: gitlab.v4.objects.ProjectPipelineJob) -> float:
|
||||||
|
"""
|
||||||
|
Given a job, report the time lapsed in execution.
|
||||||
|
:param job: Pipeline job
|
||||||
|
:return: Current time in execution
|
||||||
|
"""
|
||||||
|
if job.duration:
|
||||||
|
return job.duration
|
||||||
|
elif job.started_at:
|
||||||
|
# Convert both times to UTC timestamps for consistent comparison
|
||||||
|
current_time = time.time()
|
||||||
|
start_time = job.started_at.timestamp()
|
||||||
|
return current_time - start_time
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def pretty_wait(sec: int) -> None:
|
||||||
|
"""shows progressbar in dots"""
|
||||||
|
if is_gitlab_job():
|
||||||
|
time.sleep(sec)
|
||||||
|
return
|
||||||
|
for val in range(sec, 0, -1):
|
||||||
|
print(f"⏲ {val:2d} seconds", end="\r") # U+23F2 Timer clock
|
||||||
|
time.sleep(1)
|
||||||
|
|
||||||
|
|
||||||
|
def run_target_job(
|
||||||
|
job: gitlab.v4.objects.ProjectPipelineJob,
|
||||||
|
enable_job_fn: Callable,
|
||||||
|
stress: int,
|
||||||
|
execution_times: dict,
|
||||||
|
target_statuses: dict,
|
||||||
|
) -> None:
|
||||||
|
execution_times[job.name][job.id] = (job_duration(job), job.status, job.web_url)
|
||||||
|
if stress and job.status in COMPLETED_STATUSES:
|
||||||
|
if (
|
||||||
|
stress < 0
|
||||||
|
or len(execution_times[job.name]) < stress
|
||||||
|
):
|
||||||
|
enable_job_fn(job=job, action_type="retry")
|
||||||
|
# Wait for the next loop to get the updated job object
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
enable_job_fn(job=job, action_type="target")
|
||||||
|
|
||||||
|
print_job_status(job, job.status not in target_statuses[job.name])
|
||||||
|
target_statuses[job.name] = job.status
|
||||||
|
|
||||||
|
|
||||||
|
def monitor_pipeline(
|
||||||
|
project: gitlab.v4.objects.Project,
|
||||||
|
pipeline: gitlab.v4.objects.ProjectPipeline,
|
||||||
|
job_filter: callable,
|
||||||
|
dependencies: set[str],
|
||||||
|
stress: int,
|
||||||
|
inhibit_single_target_trace: int = False,
|
||||||
|
polling_period: int = REFRESH_WAIT_JOBS,
|
||||||
|
) -> tuple[Optional[int], Optional[int], Dict[str, Dict[int, Tuple[float, str, str]]]]:
|
||||||
|
"""Monitors pipeline and delegate canceling jobs"""
|
||||||
|
statuses: dict[str, str] = defaultdict(str)
|
||||||
|
target_statuses: dict[str, str] = defaultdict(str)
|
||||||
|
execution_times: dict[str, dict[str, tuple[float, str, str]]] = defaultdict(lambda: defaultdict(tuple))
|
||||||
|
target_id: int = -1
|
||||||
|
global type_field_pad
|
||||||
|
type_field_pad = 0
|
||||||
|
global name_field_pad
|
||||||
|
name_field_pad = len(max(dependencies, key=len))+2
|
||||||
|
# In a running pipeline, we can skip following job traces that are in these statuses.
|
||||||
|
skip_follow_statuses: frozenset[str] = (COMPLETED_STATUSES)
|
||||||
|
|
||||||
|
# Pre-populate the stress status counter for already completed target jobs.
|
||||||
|
if stress:
|
||||||
|
# When stress test, it is necessary to collect this information before start.
|
||||||
|
for job in pipeline.jobs.list(all=True, include_retried=True):
|
||||||
|
if job_filter(
|
||||||
|
job_name=job.name,
|
||||||
|
job_stage=job.stage,
|
||||||
|
job_tags=job.tag_list,
|
||||||
|
) and job.status in COMPLETED_STATUSES:
|
||||||
|
execution_times[job.name][job.id] = (job_duration(job), job.status, job.web_url)
|
||||||
|
|
||||||
|
# jobs_waiting is a list of job names that are waiting for status update.
|
||||||
|
# It occurs when a job that we want to run depends on another job that is not yet finished.
|
||||||
|
jobs_waiting = []
|
||||||
|
# Dictionary to track the number of attempts made for each job for a given status
|
||||||
|
enable_attempts: dict[tuple[int, str], int] = {}
|
||||||
|
# FIXME: This function has too many parameters, consider refactoring.
|
||||||
|
enable_job_fn = partial(
|
||||||
|
enable_job,
|
||||||
|
project=project,
|
||||||
|
enable_attempts=enable_attempts,
|
||||||
|
jobs_waiting=jobs_waiting,
|
||||||
|
)
|
||||||
|
while True:
|
||||||
|
deps_failed = []
|
||||||
|
to_cancel = []
|
||||||
|
jobs_waiting.clear()
|
||||||
|
for job in sorted(pipeline.jobs.list(all=True), key=lambda j: j.name):
|
||||||
|
job = cast(gitlab.v4.objects.ProjectPipelineJob, job)
|
||||||
|
if job_filter(
|
||||||
|
job_name=job.name,
|
||||||
|
job_stage=job.stage,
|
||||||
|
job_tags=job.tag_list,
|
||||||
|
):
|
||||||
|
run_target_job(
|
||||||
|
job,
|
||||||
|
enable_job_fn,
|
||||||
|
stress,
|
||||||
|
execution_times,
|
||||||
|
target_statuses
|
||||||
|
)
|
||||||
|
target_id = job.id
|
||||||
|
continue
|
||||||
|
# all other non-target jobs
|
||||||
|
if job.status != statuses[job.name]:
|
||||||
|
print_job_status(job, True)
|
||||||
|
statuses[job.name] = job.status
|
||||||
|
|
||||||
|
# run dependencies and cancel the rest
|
||||||
|
if job.name in dependencies:
|
||||||
|
if not enable_job_fn(job=job, action_type="dep"):
|
||||||
|
# Wait for the next loop to get the updated job object
|
||||||
|
continue
|
||||||
|
if job.status == "failed":
|
||||||
|
deps_failed.append(job.name)
|
||||||
|
else:
|
||||||
|
to_cancel.append(job)
|
||||||
|
|
||||||
|
cancel_jobs(project, to_cancel)
|
||||||
|
|
||||||
|
if stress:
|
||||||
|
enough = True
|
||||||
|
status_counters = {
|
||||||
|
name: Counter(info[1] for info in runs.values())
|
||||||
|
for name, runs in execution_times.items()
|
||||||
|
}
|
||||||
|
for job_name, counter in sorted(status_counters.items()):
|
||||||
|
n_succeed = counter.get("success", 0)
|
||||||
|
n_failed = counter.get("failed", 0)
|
||||||
|
n_total_completed = n_succeed + n_failed
|
||||||
|
n_total_seen = len(execution_times[job_name])
|
||||||
|
print(
|
||||||
|
f"* {job_name:{name_field_pad}} succ: {n_succeed}; "
|
||||||
|
f"fail: {n_failed}; "
|
||||||
|
f"total: {n_total_seen} of {stress}",
|
||||||
|
)
|
||||||
|
if stress < 0 or n_total_completed < stress:
|
||||||
|
enough = False
|
||||||
|
|
||||||
|
if not enough:
|
||||||
|
pretty_wait(polling_period)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if jobs_waiting:
|
||||||
|
print(f"[yellow]Waiting for jobs to update status:")
|
||||||
|
print_formatted_list(jobs_waiting, indentation=8, color="[yellow]")
|
||||||
|
pretty_wait(polling_period)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if (
|
||||||
|
not inhibit_single_target_trace
|
||||||
|
and stress in [0, 1]
|
||||||
|
and len(target_statuses) == 1
|
||||||
|
and RUNNING_STATUSES.intersection(target_statuses.values())
|
||||||
|
):
|
||||||
|
return target_id, None, execution_times
|
||||||
|
|
||||||
|
if (
|
||||||
|
{"failed"}.intersection(target_statuses.values())
|
||||||
|
and not RUNNING_STATUSES.intersection(target_statuses.values())
|
||||||
|
):
|
||||||
|
return None, 1, execution_times
|
||||||
|
|
||||||
|
if (
|
||||||
|
{"skipped"}.intersection(target_statuses.values())
|
||||||
|
and not RUNNING_STATUSES.intersection(target_statuses.values())
|
||||||
|
):
|
||||||
|
print(
|
||||||
|
f"[red]Target in skipped state, aborting. Failed dependencies:{deps_failed}"
|
||||||
|
)
|
||||||
|
return None, 1, execution_times
|
||||||
|
|
||||||
|
if skip_follow_statuses.issuperset(target_statuses.values()):
|
||||||
|
return None, 0, execution_times
|
||||||
|
|
||||||
|
pretty_wait(polling_period)
|
||||||
|
|
||||||
|
|
||||||
|
def enable_job(
|
||||||
|
project: gitlab.v4.objects.Project,
|
||||||
|
job: gitlab.v4.objects.ProjectPipelineJob,
|
||||||
|
enable_attempts: dict[tuple[int, str], int],
|
||||||
|
action_type: Literal["target", "dep", "retry"],
|
||||||
|
jobs_waiting: list[str] = list,
|
||||||
|
) -> bool:
|
||||||
|
"""
|
||||||
|
Enable a job to run.
|
||||||
|
:param project: The GitLab project.
|
||||||
|
:param job: The job to enable.
|
||||||
|
:param enable_attempts: A dictionary to track the number of attempts made for each job for a give status.
|
||||||
|
:param action_type: The type of action to perform.
|
||||||
|
:param jobs_waiting:
|
||||||
|
:return: True if the job was enabled, False otherwise.
|
||||||
|
"""
|
||||||
|
# We want to run this job, but it is not ready to run yet, so let's try again in the next
|
||||||
|
# iteration.
|
||||||
|
if job.status == "created":
|
||||||
|
jobs_waiting.append(job.name)
|
||||||
|
return False
|
||||||
|
|
||||||
|
if (
|
||||||
|
(job.status in COMPLETED_STATUSES and action_type != "retry")
|
||||||
|
or job.status in {"skipped"} | RUNNING_STATUSES
|
||||||
|
):
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Get current attempt number
|
||||||
|
attempt_count = enable_attempts.get((job.id, job.status), 0)
|
||||||
|
# Check if we've exceeded max attempts to avoid infinite loop
|
||||||
|
if attempt_count == MAX_ENABLE_JOB_ATTEMPTS:
|
||||||
|
print(
|
||||||
|
f"[yellow]WARNING: "
|
||||||
|
f"Maximum enabling attempts ({MAX_ENABLE_JOB_ATTEMPTS}) reached for job {job.name} in {job.status} status"
|
||||||
|
f"({link2print(job.web_url, job.id)})."
|
||||||
|
)
|
||||||
|
enable_attempts[(job.id, job.status)] = attempt_count + 1
|
||||||
|
return False
|
||||||
|
elif attempt_count > MAX_ENABLE_JOB_ATTEMPTS:
|
||||||
|
return False
|
||||||
|
|
||||||
|
enable_attempts[(job.id, job.status)] = attempt_count + 1
|
||||||
|
|
||||||
|
pjob = project.jobs.get(job.id, lazy=True)
|
||||||
|
|
||||||
|
if job.status in {"success", "failed", "canceled", "canceling"}:
|
||||||
|
try:
|
||||||
|
pjob.retry()
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error retrying job {job.name}: {e}")
|
||||||
|
return False
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
pjob.play()
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error playing job {job.name}: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
if action_type == "target":
|
||||||
|
jtype = "🞋 target" # U+1F78B Round target
|
||||||
|
elif action_type == "retry":
|
||||||
|
jtype = "↻ retrying" # U+21BB Clockwise open circle arrow
|
||||||
|
else:
|
||||||
|
jtype = "↪ dependency" # U+21AA Left Arrow Curving Right
|
||||||
|
|
||||||
|
global type_field_pad
|
||||||
|
global name_field_pad
|
||||||
|
job_name = job.name
|
||||||
|
type_field_pad = len(jtype) if len(jtype) > type_field_pad else type_field_pad
|
||||||
|
name_field_pad = len(job_name) if len(job_name) > name_field_pad else name_field_pad
|
||||||
|
print(
|
||||||
|
f"[magenta]{jtype:{type_field_pad}} {job.name:{name_field_pad}} manually enabled"
|
||||||
|
)
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def cancel_job(
|
||||||
|
project: gitlab.v4.objects.Project,
|
||||||
|
pipeline_job: gitlab.v4.objects.ProjectPipelineJob
|
||||||
|
) -> Optional[gitlab.v4.objects.ProjectPipelineJob]:
|
||||||
|
"""
|
||||||
|
Cancel GitLab job
|
||||||
|
:param project: project from the pipeline job comes from
|
||||||
|
:param pipeline_job: job made from the pipeline list
|
||||||
|
:return the job object when cancel was called
|
||||||
|
"""
|
||||||
|
if pipeline_job.status not in RUNNING_STATUSES:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
project_job = project.jobs.get(pipeline_job.id, lazy=True)
|
||||||
|
project_job.cancel()
|
||||||
|
except (gitlab.GitlabCancelError, gitlab.GitlabGetError):
|
||||||
|
# If the job failed to cancel, it will be retried in the monitor_pipeline() next iteration
|
||||||
|
return
|
||||||
|
return pipeline_job
|
||||||
|
|
||||||
|
|
||||||
|
def cancel_jobs(
|
||||||
|
project: gitlab.v4.objects.Project,
|
||||||
|
to_cancel: list[gitlab.v4.objects.ProjectPipelineJob]
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Cancel unwanted GitLab jobs
|
||||||
|
:param project: project from where the pipeline comes
|
||||||
|
:param to_cancel: list of jobs to be cancelled
|
||||||
|
"""
|
||||||
|
if not to_cancel:
|
||||||
|
return
|
||||||
|
|
||||||
|
with ThreadPoolExecutor(max_workers=6) as exe:
|
||||||
|
part = partial(cancel_job, project)
|
||||||
|
maybe_cancelled_job = exe.map(part, to_cancel)
|
||||||
|
cancelled_jobs = [f"🗙 {job.name}" for job in maybe_cancelled_job if job] # U+1F5D9 Cancellation X
|
||||||
|
|
||||||
|
# The cancelled jobs are printed without a newline
|
||||||
|
if len(cancelled_jobs):
|
||||||
|
print(f"Cancelled {len(cancelled_jobs)} jobs:")
|
||||||
|
print_formatted_list(cancelled_jobs, indentation=8)
|
||||||
|
|
||||||
|
|
||||||
|
def print_log(
|
||||||
|
project: gitlab.v4.objects.Project,
|
||||||
|
job_id: int
|
||||||
|
) -> None:
|
||||||
|
"""Print job log into output"""
|
||||||
|
printed_lines = 0
|
||||||
|
while True:
|
||||||
|
job = project.jobs.get(job_id)
|
||||||
|
|
||||||
|
# GitLab's REST API doesn't offer pagination for logs, so we have to refetch it all
|
||||||
|
lines = job.trace().decode().splitlines()
|
||||||
|
for line in lines[printed_lines:]:
|
||||||
|
print(line, markup=False)
|
||||||
|
printed_lines = len(lines)
|
||||||
|
|
||||||
|
if job.status in COMPLETED_STATUSES:
|
||||||
|
print(f"[green]Job finished: {job.web_url}")
|
||||||
|
return
|
||||||
|
pretty_wait(REFRESH_WAIT_LOG)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> argparse.Namespace:
|
||||||
|
"""Parse args"""
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Tool to trigger a subset of container jobs "
|
||||||
|
+ "and monitor the progress of a test job",
|
||||||
|
epilog="Example: %(prog)s --rev $(git rev-parse HEAD) "
|
||||||
|
+ '--target ".*traces" ',
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--server",
|
||||||
|
metavar="gitlab-server",
|
||||||
|
type=str,
|
||||||
|
default=GITLAB_URL,
|
||||||
|
help=f"Specify the GitLab server work with (Default: {GITLAB_URL})",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--target",
|
||||||
|
metavar="target-job",
|
||||||
|
help="Target job regex. For multiple targets, pass multiple values, "
|
||||||
|
"eg. `--target foo bar`. Only jobs in the target stage(s) "
|
||||||
|
"supplied, and their dependencies, will be considered.",
|
||||||
|
required=True,
|
||||||
|
nargs=argparse.ONE_OR_MORE,
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--include-stage",
|
||||||
|
metavar="include-stage",
|
||||||
|
help="Job stages to include when searching for target jobs. "
|
||||||
|
"For multiple targets, pass multiple values, eg. "
|
||||||
|
"`--include-stage foo bar`.",
|
||||||
|
default=[".*"],
|
||||||
|
nargs=argparse.ONE_OR_MORE,
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--exclude-stage",
|
||||||
|
metavar="exclude-stage",
|
||||||
|
help="Job stages to exclude when searching for target jobs. "
|
||||||
|
"For multiple targets, pass multiple values, eg. "
|
||||||
|
"`--exclude-stage foo bar`. By default, performance and "
|
||||||
|
"nightly jobs are excluded; pass --exclude-stage '' to "
|
||||||
|
"include them for consideration.",
|
||||||
|
default=["performance", ".*-postmerge", ".*-nightly"],
|
||||||
|
nargs=argparse.ONE_OR_MORE,
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--job-tags",
|
||||||
|
metavar="job-tags",
|
||||||
|
help="Job tags to require when searching for target jobs. If multiple "
|
||||||
|
"values are passed, eg. `--job-tags 'foo.*' 'bar'`, the job will "
|
||||||
|
"need to have a tag matching `foo.*` *and* a tag matching `bar` "
|
||||||
|
"to qualify. Passing `--job-tags '.*'` makes sure the job has "
|
||||||
|
"a tag defined, while not passing `--job-tags` also allows "
|
||||||
|
"untagged jobs.",
|
||||||
|
default=[],
|
||||||
|
nargs=argparse.ONE_OR_MORE,
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--token",
|
||||||
|
metavar="token",
|
||||||
|
type=str,
|
||||||
|
default=get_token_from_default_dir(),
|
||||||
|
help="Use the provided GitLab token (with `api` scope) or token file, "
|
||||||
|
f"otherwise it's read from {TOKEN_DIR / 'gitlab-token'}",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--force-manual", action="store_true",
|
||||||
|
help="Deprecated argument; manual jobs are always force-enabled"
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--stress",
|
||||||
|
metavar="n",
|
||||||
|
type=int,
|
||||||
|
default=0,
|
||||||
|
help="Stresstest job(s). Specify the number of times to rerun the selected jobs, "
|
||||||
|
"or use -1 for indefinite. Defaults to 0. If jobs have already been executed, "
|
||||||
|
"this will ensure the total run count respects the specified number.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--project",
|
||||||
|
metavar="name",
|
||||||
|
type=str,
|
||||||
|
default="mesa",
|
||||||
|
help="GitLab project in the format <user>/<project> or just <project>",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--dry-run",
|
||||||
|
action="store_true",
|
||||||
|
help="Exit after printing target jobs and dependencies",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--no-job-log",
|
||||||
|
action="store_true",
|
||||||
|
help="When there is only one target job, inhibit the job trace output in the console.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--polling-period",
|
||||||
|
type=int,
|
||||||
|
default=REFRESH_WAIT_JOBS,
|
||||||
|
help=f"Specify the waiting seconds between monitor loops. (Default: {REFRESH_WAIT_JOBS})",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
mutex_group1 = parser.add_mutually_exclusive_group()
|
||||||
|
mutex_group1.add_argument(
|
||||||
|
"--rev",
|
||||||
|
metavar="id",
|
||||||
|
type=str,
|
||||||
|
default="HEAD",
|
||||||
|
help="Repository git commit-ish, tag or branch name (default: HEAD)",
|
||||||
|
)
|
||||||
|
mutex_group1.add_argument(
|
||||||
|
"--pipeline-url",
|
||||||
|
metavar="url",
|
||||||
|
type=str,
|
||||||
|
help="URL of the pipeline to use, instead of auto-detecting it.",
|
||||||
|
)
|
||||||
|
mutex_group1.add_argument(
|
||||||
|
"--mr",
|
||||||
|
metavar="id",
|
||||||
|
type=int,
|
||||||
|
help="ID of a merge request; the latest pipeline in that MR will be used.",
|
||||||
|
)
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
# argparse doesn't support groups inside add_mutually_exclusive_group(),
|
||||||
|
# which means we can't just put `--project` and `--rev` in a group together,
|
||||||
|
# we have to do this by heand instead.
|
||||||
|
if args.pipeline_url and args.project != parser.get_default("project"):
|
||||||
|
# weird phrasing but it's the error add_mutually_exclusive_group() gives
|
||||||
|
parser.error("argument --project: not allowed with argument --pipeline-url")
|
||||||
|
|
||||||
|
return args
|
||||||
|
|
||||||
|
|
||||||
|
def print_detected_jobs(
|
||||||
|
target_dep_dag: "Dag",
|
||||||
|
dependency_jobs: Iterable[str],
|
||||||
|
target_jobs: Iterable[str],
|
||||||
|
) -> None:
|
||||||
|
def print_job_set(color: str, kind: str, job_set: Iterable[str]):
|
||||||
|
job_list = list(job_set)
|
||||||
|
print(f"{color}Running {len(job_list)} {kind} jobs:")
|
||||||
|
print_formatted_list(job_list, indentation=8, color=color)
|
||||||
|
|
||||||
|
print("[yellow]Detected target job and its dependencies:")
|
||||||
|
print_dag(target_dep_dag, indentation=8, color="[yellow]")
|
||||||
|
print_job_set("[magenta]", "dependency", dependency_jobs)
|
||||||
|
print_job_set("[blue]", "target", target_jobs)
|
||||||
|
|
||||||
|
|
||||||
|
def find_dependencies(
|
||||||
|
server: str,
|
||||||
|
token: str | None,
|
||||||
|
job_filter: callable,
|
||||||
|
project_path: str,
|
||||||
|
iid: int
|
||||||
|
) -> set[str]:
|
||||||
|
"""
|
||||||
|
Find the dependencies of the target jobs in a GitLab pipeline.
|
||||||
|
|
||||||
|
This function uses the GitLab GraphQL API to fetch the job dependency graph
|
||||||
|
of a pipeline, filters the graph to only include the target jobs and their
|
||||||
|
dependencies, and returns the names of these jobs.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
server (str): The url to the GitLab server.
|
||||||
|
token (str | None): The GitLab API token. If None, the API is accessed without
|
||||||
|
authentication.
|
||||||
|
target_jobs_regex (re.Pattern): A regex pattern to match the names of the target jobs.
|
||||||
|
project_path (str): The path of the GitLab project.
|
||||||
|
iid (int): The internal ID of the pipeline.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
set[str]: A set of the names of the target jobs and their dependencies.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
SystemExit: If no target jobs are found in the pipeline.
|
||||||
|
"""
|
||||||
|
gql_instance = GitlabGQL(
|
||||||
|
url=f"{server}/api/graphql",
|
||||||
|
token=token
|
||||||
|
)
|
||||||
|
dag = create_job_needs_dag(
|
||||||
|
gql_instance, {"projectPath": project_path.path_with_namespace, "iid": iid}
|
||||||
|
)
|
||||||
|
|
||||||
|
target_dep_dag = filter_dag(dag, job_filter)
|
||||||
|
if not target_dep_dag:
|
||||||
|
print("[red]The job(s) were not found in the pipeline.")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
dependency_jobs = set(chain.from_iterable(d["needs"] for d in target_dep_dag.values()))
|
||||||
|
target_jobs = set(target_dep_dag.keys())
|
||||||
|
print_detected_jobs(target_dep_dag, dependency_jobs, target_jobs)
|
||||||
|
return target_jobs.union(dependency_jobs)
|
||||||
|
|
||||||
|
|
||||||
|
def print_monitor_summary(
|
||||||
|
execution_collection: Dict[str, Dict[int, Tuple[float, str, str]]],
|
||||||
|
t_start: float,
|
||||||
|
) -> None:
|
||||||
|
"""Summary of the test execution"""
|
||||||
|
t_end = time.perf_counter()
|
||||||
|
spend_minutes = (t_end - t_start) / 60
|
||||||
|
print(f"⏲ Duration of script execution: {spend_minutes:0.1f} minutes") # U+23F2 Timer clock
|
||||||
|
if len(execution_collection) == 0:
|
||||||
|
return
|
||||||
|
print(f"⏲ Jobs execution times:") # U+23F2 Timer clock
|
||||||
|
job_names = list(execution_collection.keys())
|
||||||
|
job_names.sort()
|
||||||
|
name_field_pad = len(max(job_names, key=len)) + 2
|
||||||
|
for name in job_names:
|
||||||
|
job_executions = execution_collection[name]
|
||||||
|
job_times = ', '.join([__job_duration_record(job_execution)
|
||||||
|
for job_execution in sorted(job_executions.items())])
|
||||||
|
print(f"* {name:{name_field_pad}}: ({len(job_executions)}) {job_times}")
|
||||||
|
|
||||||
|
|
||||||
|
def __job_duration_record(dict_item: tuple) -> str:
|
||||||
|
"""
|
||||||
|
Format each pair of job and its duration.
|
||||||
|
:param job_execution: item of execution_collection[name][idn]: Dict[int, Tuple[float, str, str]]
|
||||||
|
"""
|
||||||
|
job_id = f"{dict_item[0]}" # dictionary key
|
||||||
|
job_duration, job_status, job_url = dict_item[1] # dictionary value, the tuple
|
||||||
|
return (
|
||||||
|
f"{STATUS_COLORS[job_status]}"
|
||||||
|
f"{link2print(job_url, job_id)}: {pretty_duration(job_duration):>8}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def link2print(url: str, text: str, text_pad: int = 0) -> str:
|
||||||
|
text = str(text)
|
||||||
|
text_pad = len(text) if text_pad < 1 else text_pad
|
||||||
|
if console.is_terminal:
|
||||||
|
return f"[link={url}]{text:{text_pad}}[/link]"
|
||||||
|
else:
|
||||||
|
return f"{text:{text_pad}}"
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
try:
|
||||||
|
t_start = time.perf_counter()
|
||||||
|
|
||||||
|
args = parse_args()
|
||||||
|
|
||||||
|
token = read_token(args.token)
|
||||||
|
|
||||||
|
gl = gitlab.Gitlab(url=args.server,
|
||||||
|
private_token=token,
|
||||||
|
retry_transient_errors=True)
|
||||||
|
|
||||||
|
REV: str = args.rev
|
||||||
|
|
||||||
|
if args.pipeline_url:
|
||||||
|
pipe, cur_project = get_gitlab_pipeline_from_url(gl, args.pipeline_url)
|
||||||
|
REV = pipe.sha
|
||||||
|
else:
|
||||||
|
mesa_project = gl.projects.get("mesa/mesa")
|
||||||
|
projects = [mesa_project]
|
||||||
|
if args.mr:
|
||||||
|
REV = mesa_project.mergerequests.get(args.mr).sha
|
||||||
|
else:
|
||||||
|
REV = check_output(['git', 'rev-parse', REV]).decode('ascii').strip()
|
||||||
|
|
||||||
|
if args.rev == 'HEAD':
|
||||||
|
try:
|
||||||
|
branch_name = check_output([
|
||||||
|
'git', 'symbolic-ref', '-q', 'HEAD',
|
||||||
|
]).decode('ascii').strip()
|
||||||
|
except CalledProcessError:
|
||||||
|
branch_name = ""
|
||||||
|
|
||||||
|
# Ignore detached heads
|
||||||
|
if branch_name:
|
||||||
|
tracked_remote = check_output([
|
||||||
|
'git', 'for-each-ref', '--format=%(upstream)',
|
||||||
|
branch_name,
|
||||||
|
]).decode('ascii').strip()
|
||||||
|
|
||||||
|
# Ignore local branches that do not track any remote
|
||||||
|
if tracked_remote:
|
||||||
|
remote_rev = check_output([
|
||||||
|
'git', 'rev-parse', tracked_remote,
|
||||||
|
]).decode('ascii').strip()
|
||||||
|
|
||||||
|
if REV != remote_rev:
|
||||||
|
print(
|
||||||
|
f"Local HEAD commit {REV[:10]} is different than "
|
||||||
|
f"tracked remote HEAD commit {remote_rev[:10]}"
|
||||||
|
)
|
||||||
|
print("Did you forget to `git push` ?")
|
||||||
|
|
||||||
|
projects.append(get_gitlab_project(gl, args.project))
|
||||||
|
(pipe, cur_project) = wait_for_pipeline(projects, REV)
|
||||||
|
|
||||||
|
print(f"Revision: {REV}")
|
||||||
|
print(f"Pipeline: {pipe.web_url}")
|
||||||
|
|
||||||
|
target = '|'.join(args.target)
|
||||||
|
target = target.strip()
|
||||||
|
|
||||||
|
print(f"🞋 target job: [blue]{target}") # U+1F78B Round target
|
||||||
|
|
||||||
|
# Implicitly include `parallel:` jobs
|
||||||
|
target = f'({target})' + r'( \d+/\d+)?'
|
||||||
|
|
||||||
|
target_jobs_regex = re.compile(target)
|
||||||
|
|
||||||
|
include_stage = '|'.join(args.include_stage)
|
||||||
|
include_stage = include_stage.strip()
|
||||||
|
|
||||||
|
print(f"🞋 target from stages: [blue]{include_stage}") # U+1F78B Round target
|
||||||
|
|
||||||
|
include_stage_regex = re.compile(include_stage)
|
||||||
|
|
||||||
|
exclude_stage = '|'.join(args.exclude_stage)
|
||||||
|
exclude_stage = exclude_stage.strip()
|
||||||
|
|
||||||
|
print(f"🞋 target excluding stages: [blue]{exclude_stage}") # U+1F78B Round target
|
||||||
|
|
||||||
|
exclude_stage_regex = re.compile(exclude_stage)
|
||||||
|
|
||||||
|
print(f"🞋 target jobs with tags: [blue]{str(args.job_tags)}") # U+1F78B Round target
|
||||||
|
job_tags_regexes = [re.compile(job_tag) for job_tag in args.job_tags]
|
||||||
|
|
||||||
|
def job_filter(
|
||||||
|
job_name: str,
|
||||||
|
job_stage: str,
|
||||||
|
job_tags: set[str],
|
||||||
|
) -> bool:
|
||||||
|
"""
|
||||||
|
Apply user-specified filters to a job, and return whether the
|
||||||
|
filters allow that job (True) or not (False).
|
||||||
|
"""
|
||||||
|
if not target_jobs_regex.fullmatch(job_name):
|
||||||
|
return False
|
||||||
|
if not include_stage_regex.fullmatch(job_stage):
|
||||||
|
return False
|
||||||
|
if exclude_stage_regex.fullmatch(job_stage):
|
||||||
|
return False
|
||||||
|
if not all(
|
||||||
|
any(job_tags_regex.fullmatch(tag) for tag in job_tags)
|
||||||
|
for job_tags_regex in job_tags_regexes
|
||||||
|
):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
deps = find_dependencies(
|
||||||
|
server=args.server,
|
||||||
|
token=token,
|
||||||
|
job_filter=job_filter,
|
||||||
|
iid=pipe.iid,
|
||||||
|
project_path=cur_project
|
||||||
|
)
|
||||||
|
|
||||||
|
if args.dry_run:
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
target_job_id, ret, exec_t = monitor_pipeline(
|
||||||
|
cur_project,
|
||||||
|
pipe,
|
||||||
|
job_filter,
|
||||||
|
deps,
|
||||||
|
args.stress,
|
||||||
|
args.no_job_log,
|
||||||
|
args.polling_period,
|
||||||
|
)
|
||||||
|
|
||||||
|
if target_job_id:
|
||||||
|
print_log(cur_project, target_job_id)
|
||||||
|
|
||||||
|
print_monitor_summary(exec_t, t_start)
|
||||||
|
|
||||||
|
sys.exit(ret)
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Executable
+10
@@ -0,0 +1,10 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
|
||||||
|
readonly this_dir
|
||||||
|
|
||||||
|
exec \
|
||||||
|
"$this_dir/../python-venv.sh" \
|
||||||
|
"$this_dir/requirements.txt" \
|
||||||
|
"$this_dir/ci_run_n_monitor.py" "$@"
|
||||||
@@ -0,0 +1,353 @@
|
|||||||
|
import argparse
|
||||||
|
import logging
|
||||||
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from structured_logger import StructuredLogger
|
||||||
|
|
||||||
|
|
||||||
|
class CustomLogger:
|
||||||
|
def __init__(self, log_file):
|
||||||
|
self.log_file = log_file
|
||||||
|
self.logger = StructuredLogger(file_name=self.log_file)
|
||||||
|
|
||||||
|
def get_last_dut_job(self):
|
||||||
|
"""
|
||||||
|
Gets the details of the most recent DUT job.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
dict: Details of the most recent DUT job.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ValueError: If no DUT jobs are found in the logger's data.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
job = self.logger.data["dut_jobs"][-1]
|
||||||
|
except KeyError:
|
||||||
|
raise ValueError(
|
||||||
|
"No DUT jobs found. Please create a job via create_dut_job call."
|
||||||
|
)
|
||||||
|
|
||||||
|
return job
|
||||||
|
|
||||||
|
def update(self, **kwargs):
|
||||||
|
"""
|
||||||
|
Updates the log file with provided key-value pairs.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
**kwargs: Key-value pairs to be updated.
|
||||||
|
|
||||||
|
"""
|
||||||
|
with self.logger.edit_context():
|
||||||
|
for key, value in kwargs.items():
|
||||||
|
self.logger.data[key] = value
|
||||||
|
|
||||||
|
def create_dut_job(self, **kwargs):
|
||||||
|
"""
|
||||||
|
Creates a new DUT job with provided key-value pairs.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
**kwargs: Key-value pairs for the new DUT job.
|
||||||
|
|
||||||
|
"""
|
||||||
|
with self.logger.edit_context():
|
||||||
|
if "dut_jobs" not in self.logger.data:
|
||||||
|
self.logger.data["dut_jobs"] = []
|
||||||
|
new_job = {
|
||||||
|
"status": "",
|
||||||
|
"submitter_start_time": datetime.now().isoformat(),
|
||||||
|
"dut_submit_time": "",
|
||||||
|
"dut_start_time": "",
|
||||||
|
"dut_end_time": "",
|
||||||
|
"dut_duration_time": "",
|
||||||
|
"dut_name": "",
|
||||||
|
"dut_state": "pending",
|
||||||
|
"dut_job_phases": [],
|
||||||
|
**kwargs,
|
||||||
|
}
|
||||||
|
self.logger.data["dut_jobs"].append(new_job)
|
||||||
|
|
||||||
|
def update_dut_job(self, key, value):
|
||||||
|
"""
|
||||||
|
Updates the last DUT job with a key-value pair.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
key : The key to be updated.
|
||||||
|
value: The value to be assigned.
|
||||||
|
|
||||||
|
"""
|
||||||
|
with self.logger.edit_context():
|
||||||
|
job = self.get_last_dut_job()
|
||||||
|
job[key] = value
|
||||||
|
|
||||||
|
def update_status_fail(self, reason=""):
|
||||||
|
"""
|
||||||
|
Sets the status of the last DUT job to 'fail' and logs the failure reason.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
reason (str, optional): The reason for the failure. Defaults to "".
|
||||||
|
|
||||||
|
"""
|
||||||
|
with self.logger.edit_context():
|
||||||
|
job = self.get_last_dut_job()
|
||||||
|
job["status"] = "fail"
|
||||||
|
job["dut_job_fail_reason"] = reason
|
||||||
|
|
||||||
|
def create_job_phase(self, phase_name):
|
||||||
|
"""
|
||||||
|
Creates a new job phase for the last DUT job.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
phase_name : The name of the new job phase.
|
||||||
|
|
||||||
|
"""
|
||||||
|
with self.logger.edit_context():
|
||||||
|
job = self.get_last_dut_job()
|
||||||
|
if job["dut_job_phases"] and job["dut_job_phases"][-1]["end_time"] == "":
|
||||||
|
# If the last phase exists and its end time is empty, set the end time
|
||||||
|
timestamp = datetime.now().isoformat()
|
||||||
|
job["dut_job_phases"][-1]["end_time"] = timestamp
|
||||||
|
job["dut_job_phases"][-1]["duration_time"] = self.get_duration_time(job["dut_job_phases"][-1]["start_time"], timestamp)
|
||||||
|
|
||||||
|
# Create a new phase
|
||||||
|
phase_data = {
|
||||||
|
"name": phase_name,
|
||||||
|
"start_time": datetime.now().isoformat(),
|
||||||
|
"end_time": "",
|
||||||
|
"duration_time": "",
|
||||||
|
}
|
||||||
|
job["dut_job_phases"].append(phase_data)
|
||||||
|
|
||||||
|
def check_dut_timings(self, job):
|
||||||
|
"""
|
||||||
|
Check the timing sequence of a job to ensure logical consistency.
|
||||||
|
|
||||||
|
The function verifies that the job's submission time is not earlier than its start time and that
|
||||||
|
the job's end time is not earlier than its start time. If either of these conditions is found to be true,
|
||||||
|
an error is logged for each instance of inconsistency.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
job (dict): A dictionary containing timing information of a job. Expected keys are 'dut_start_time',
|
||||||
|
'dut_submit_time', and 'dut_end_time'.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
None: This function does not return a value; it logs errors if timing inconsistencies are detected.
|
||||||
|
|
||||||
|
The function checks the following:
|
||||||
|
- If 'dut_start_time' and 'dut_submit_time' are both present and correctly sequenced.
|
||||||
|
- If 'dut_start_time' and 'dut_end_time' are both present and correctly sequenced.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Check if the start time and submit time exist
|
||||||
|
if job.get("dut_start_time") and job.get("dut_submit_time"):
|
||||||
|
# If they exist, check if the submission time is before the start time
|
||||||
|
if job["dut_start_time"] < job["dut_submit_time"]:
|
||||||
|
logging.error("Job submission is happening before job start.")
|
||||||
|
|
||||||
|
# Check if the start time and end time exist
|
||||||
|
if job.get("dut_start_time") and job.get("dut_end_time"):
|
||||||
|
# If they exist, check if the end time is after the start time
|
||||||
|
if job["dut_end_time"] < job["dut_start_time"]:
|
||||||
|
logging.error("Job ended before it started.")
|
||||||
|
|
||||||
|
def get_duration_time(self, start_time, end_time):
|
||||||
|
"""
|
||||||
|
Computes duration time, in minutes and seconds.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
start = datetime.fromisoformat(start_time)
|
||||||
|
end = datetime.fromisoformat(end_time)
|
||||||
|
duration = end - start
|
||||||
|
return str(duration)
|
||||||
|
except ValueError:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
# Method to update DUT start, submit and end time
|
||||||
|
def update_dut_time(self, value, custom_time):
|
||||||
|
"""
|
||||||
|
Updates DUT start, submit, and end times.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
value : Specifies which DUT time to update. Options: 'start', 'submit', 'end'.
|
||||||
|
custom_time : Custom time to set. If None, use current time.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ValueError: If an invalid argument is provided for value.
|
||||||
|
|
||||||
|
"""
|
||||||
|
with self.logger.edit_context():
|
||||||
|
job = self.get_last_dut_job()
|
||||||
|
timestamp = custom_time if custom_time else datetime.now().isoformat()
|
||||||
|
if value == "start":
|
||||||
|
job["dut_start_time"] = timestamp
|
||||||
|
job["dut_state"] = "running"
|
||||||
|
elif value == "submit":
|
||||||
|
job["dut_submit_time"] = timestamp
|
||||||
|
job["dut_state"] = "submitted"
|
||||||
|
elif value == "end":
|
||||||
|
job["dut_end_time"] = timestamp
|
||||||
|
job["dut_state"] = "finished"
|
||||||
|
job["dut_duration_time"] = self.get_duration_time(job["dut_start_time"], job["dut_end_time"])
|
||||||
|
else:
|
||||||
|
raise ValueError(
|
||||||
|
"Error: Invalid argument provided for --update-dut-time. Use 'start', 'submit', 'end'."
|
||||||
|
)
|
||||||
|
# check the sanity of the partial structured log
|
||||||
|
self.check_dut_timings(job)
|
||||||
|
|
||||||
|
def close_dut_job(self):
|
||||||
|
"""
|
||||||
|
Closes the most recent DUT (Device Under Test) job in the logger's data.
|
||||||
|
|
||||||
|
The method performs the following operations:
|
||||||
|
1. Validates if there are any DUT jobs in the logger's data.
|
||||||
|
2. If the last phase of the most recent DUT job has an empty end time, it sets the end time to the current time.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ValueError: If no DUT jobs are found in the logger's data.
|
||||||
|
"""
|
||||||
|
with self.logger.edit_context():
|
||||||
|
job = self.get_last_dut_job()
|
||||||
|
# Check if the last phase exists and its end time is empty, then set the end time
|
||||||
|
if job["dut_job_phases"] and job["dut_job_phases"][-1]["end_time"] == "":
|
||||||
|
timestamp = datetime.now().isoformat()
|
||||||
|
job["dut_job_phases"][-1]["end_time"] = timestamp
|
||||||
|
job["dut_job_phases"][-1]["duration_time"] = self.get_duration_time(job["dut_job_phases"][-1]["start_time"], timestamp)
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
"""
|
||||||
|
Closes the most recent DUT (Device Under Test) job in the logger's data.
|
||||||
|
|
||||||
|
The method performs the following operations:
|
||||||
|
1. Determines the combined status of all DUT jobs.
|
||||||
|
2. Sets the submitter's end time to the current time.
|
||||||
|
3. Updates the DUT attempt counter to reflect the total number of DUT jobs.
|
||||||
|
|
||||||
|
"""
|
||||||
|
with self.logger.edit_context():
|
||||||
|
job_status = []
|
||||||
|
for job in self.logger.data["dut_jobs"]:
|
||||||
|
if "status" in job:
|
||||||
|
job_status.append(job["status"])
|
||||||
|
|
||||||
|
if not job_status:
|
||||||
|
job_combined_status = "null"
|
||||||
|
else:
|
||||||
|
# Get job_combined_status
|
||||||
|
if "pass" in job_status:
|
||||||
|
job_combined_status = "pass"
|
||||||
|
else:
|
||||||
|
job_combined_status = "fail"
|
||||||
|
|
||||||
|
self.logger.data["job_combined_status"] = job_combined_status
|
||||||
|
self.logger.data["dut_attempt_counter"] = len(self.logger.data["dut_jobs"])
|
||||||
|
job["submitter_end_time"] = datetime.now().isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def process_args(args):
|
||||||
|
# Function to process key-value pairs and call corresponding logger methods
|
||||||
|
def process_key_value_pairs(args_list, action_func):
|
||||||
|
if not args_list:
|
||||||
|
raise ValueError(
|
||||||
|
f"No key-value pairs provided for {action_func.__name__.replace('_', '-')}"
|
||||||
|
)
|
||||||
|
if len(args_list) % 2 != 0:
|
||||||
|
raise ValueError(
|
||||||
|
f"Incomplete key-value pairs for {action_func.__name__.replace('_', '-')}"
|
||||||
|
)
|
||||||
|
kwargs = dict(zip(args_list[::2], args_list[1::2]))
|
||||||
|
action_func(**kwargs)
|
||||||
|
|
||||||
|
# Create a CustomLogger object with the specified log file path
|
||||||
|
custom_logger = CustomLogger(Path(args.log_file))
|
||||||
|
|
||||||
|
if args.update:
|
||||||
|
process_key_value_pairs(args.update, custom_logger.update)
|
||||||
|
|
||||||
|
if args.create_dut_job:
|
||||||
|
process_key_value_pairs(args.create_dut_job, custom_logger.create_dut_job)
|
||||||
|
|
||||||
|
if args.update_dut_job:
|
||||||
|
key, value = args.update_dut_job
|
||||||
|
custom_logger.update_dut_job(key, value)
|
||||||
|
|
||||||
|
if args.create_job_phase:
|
||||||
|
custom_logger.create_job_phase(args.create_job_phase)
|
||||||
|
|
||||||
|
if args.update_status_fail:
|
||||||
|
custom_logger.update_status_fail(args.update_status_fail)
|
||||||
|
|
||||||
|
if args.update_dut_time:
|
||||||
|
if len(args.update_dut_time) == 2:
|
||||||
|
action, custom_time = args.update_dut_time
|
||||||
|
elif len(args.update_dut_time) == 1:
|
||||||
|
action, custom_time = args.update_dut_time[0], None
|
||||||
|
else:
|
||||||
|
raise ValueError("Invalid number of values for --update-dut-time")
|
||||||
|
|
||||||
|
if action in ["start", "end", "submit"]:
|
||||||
|
custom_logger.update_dut_time(action, custom_time)
|
||||||
|
else:
|
||||||
|
raise ValueError(
|
||||||
|
"Error: Invalid argument provided for --update-dut-time. Use 'start', 'submit', 'end'."
|
||||||
|
)
|
||||||
|
|
||||||
|
if args.close_dut_job:
|
||||||
|
custom_logger.close_dut_job()
|
||||||
|
|
||||||
|
if args.close:
|
||||||
|
custom_logger.close()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="Custom Logger Command Line Tool")
|
||||||
|
parser.add_argument("log_file", help="Path to the log file")
|
||||||
|
parser.add_argument(
|
||||||
|
"--update",
|
||||||
|
nargs=argparse.ZERO_OR_MORE,
|
||||||
|
metavar=("key", "value"),
|
||||||
|
help="Update a key-value pair e.g., --update key1 value1 key2 value2)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--create-dut-job",
|
||||||
|
nargs=argparse.ZERO_OR_MORE,
|
||||||
|
metavar=("key", "value"),
|
||||||
|
help="Create a new DUT job with key-value pairs (e.g., --create-dut-job key1 value1 key2 value2)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--update-dut-job",
|
||||||
|
nargs=argparse.ZERO_OR_MORE,
|
||||||
|
metavar=("key", "value"),
|
||||||
|
help="Update a key-value pair in DUT job",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--create-job-phase",
|
||||||
|
help="Create a new job phase (e.g., --create-job-phase name)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--update-status-fail",
|
||||||
|
help="Update fail as the status and log the failure reason (e.g., --update-status-fail reason)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--update-dut-time",
|
||||||
|
nargs=argparse.ZERO_OR_MORE,
|
||||||
|
metavar=("action", "custom_time"),
|
||||||
|
help="Update DUT start and end time. Provide action ('start', 'submit', 'end') and custom_time (e.g., '2023-01-01T12:00:00')",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--close-dut-job",
|
||||||
|
action="store_true",
|
||||||
|
help="Close the dut job by updating end time of last dut job)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--close",
|
||||||
|
action="store_true",
|
||||||
|
help="Updates combined status, submitter's end time and DUT attempt counter",
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
process_args(args)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Executable
+11
@@ -0,0 +1,11 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
# Helper script to download the schema GraphQL from Gitlab to enable IDEs to
|
||||||
|
# assist the developer to edit gql files
|
||||||
|
|
||||||
|
SOURCE_DIR=$(dirname "$(realpath "$0")")
|
||||||
|
|
||||||
|
(
|
||||||
|
cd $SOURCE_DIR || exit 1
|
||||||
|
gql-cli https://gitlab.freedesktop.org/api/graphql --print-schema > schema.graphql
|
||||||
|
)
|
||||||
@@ -0,0 +1,184 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# Copyright © 2020 - 2022 Collabora Ltd.
|
||||||
|
# Authors:
|
||||||
|
# Tomeu Vizoso <[email protected]>
|
||||||
|
# David Heidelberg <[email protected]>
|
||||||
|
# Guilherme Gallo <[email protected]>
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
|
'''Shared functions between the scripts.'''
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
from functools import cache
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
GITLAB_URL = "https://gitlab.freedesktop.org"
|
||||||
|
TOKEN_DIR = Path(os.environ.get("XDG_CONFIG_HOME", "")
|
||||||
|
if os.environ.get("XDG_CONFIG_HOME", None)
|
||||||
|
else Path.home() / ".config")
|
||||||
|
|
||||||
|
# Known GitLab token prefixes: https://docs.gitlab.com/security/tokens/#token-prefixes
|
||||||
|
TOKEN_PREFIXES: dict[str, str] = {
|
||||||
|
"Personal access token": "glpat-",
|
||||||
|
"OAuth Application Secret": "gloas-",
|
||||||
|
"Deploy token": "gldt-",
|
||||||
|
"Runner authentication token": "glrt-",
|
||||||
|
"CI/CD Job token": "glcbt-",
|
||||||
|
"Trigger token": "glptt-",
|
||||||
|
"Feed token": "glft-",
|
||||||
|
"Incoming mail token": "glimt-",
|
||||||
|
"GitLab Agent for Kubernetes token": "glagent-",
|
||||||
|
"SCIM Tokens": "glsoat-",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@cache
|
||||||
|
def print_once(*args, **kwargs):
|
||||||
|
"""Print without spamming the output"""
|
||||||
|
print(*args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def pretty_duration(seconds: int | float) -> str:
|
||||||
|
"""Pretty print duration"""
|
||||||
|
seconds = int(seconds)
|
||||||
|
hours, rem = divmod(seconds, 3600)
|
||||||
|
minutes, seconds = divmod(rem, 60)
|
||||||
|
if hours:
|
||||||
|
return f"{hours:0.0f}h{minutes:02.0f}m{seconds:02.0f}s"
|
||||||
|
if minutes:
|
||||||
|
return f"{minutes:0.0f}m{seconds:02.0f}s"
|
||||||
|
return f"{seconds:0.0f}s"
|
||||||
|
|
||||||
|
|
||||||
|
def get_gitlab_pipeline_from_url(gl, pipeline_url) -> tuple:
|
||||||
|
"""
|
||||||
|
Extract the project and pipeline object from the url string
|
||||||
|
:param gl: Gitlab object
|
||||||
|
:param pipeline_url: string with a url to a pipeline
|
||||||
|
:return: ProjectPipeline, Project objects
|
||||||
|
"""
|
||||||
|
pattern = rf"^{re.escape(GITLAB_URL)}/(.*)/-/pipelines/([0-9]+)$"
|
||||||
|
match = re.match(pattern, pipeline_url)
|
||||||
|
if not match:
|
||||||
|
raise AssertionError(f"url {pipeline_url} doesn't follow the pattern {pattern}")
|
||||||
|
namespace_with_project, pipeline_id = match.groups()
|
||||||
|
cur_project = gl.projects.get(namespace_with_project)
|
||||||
|
pipe = cur_project.pipelines.get(pipeline_id)
|
||||||
|
return pipe, cur_project
|
||||||
|
|
||||||
|
|
||||||
|
def get_gitlab_project(glab, name: str):
|
||||||
|
"""Finds a specified gitlab project for given user"""
|
||||||
|
if "/" in name:
|
||||||
|
project_path = name
|
||||||
|
else:
|
||||||
|
glab.auth()
|
||||||
|
username = glab.user.username
|
||||||
|
project_path = f"{username}/{name}"
|
||||||
|
return glab.projects.get(project_path)
|
||||||
|
|
||||||
|
|
||||||
|
def get_token_from_default_dir() -> str:
|
||||||
|
"""
|
||||||
|
Retrieves the GitLab token from the default directory.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: The path to the GitLab token file.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
FileNotFoundError: If the token file is not found.
|
||||||
|
"""
|
||||||
|
token_file = TOKEN_DIR / "gitlab-token"
|
||||||
|
try:
|
||||||
|
return str(token_file.resolve())
|
||||||
|
except FileNotFoundError as ex:
|
||||||
|
print(
|
||||||
|
f"Could not find {token_file}, please provide a token file as an argument"
|
||||||
|
)
|
||||||
|
raise ex
|
||||||
|
|
||||||
|
|
||||||
|
def validate_gitlab_token(token: str) -> bool:
|
||||||
|
# Match against recognised token prefixes
|
||||||
|
token_suffix = None
|
||||||
|
for token_type, token_prefix in TOKEN_PREFIXES.items():
|
||||||
|
if token.startswith(token_prefix):
|
||||||
|
logging.info(f"Found probable token type: {token_type}")
|
||||||
|
token_suffix = token[len(token_prefix):]
|
||||||
|
break
|
||||||
|
|
||||||
|
if not token_suffix:
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Basic validation of the token suffix based on:
|
||||||
|
# https://gitlab.com/gitlab-org/gitlab/-/blob/master/gems/gitlab-secret_detection/lib/gitleaks.toml
|
||||||
|
if not re.match(r"(\w+-)?[0-9a-zA-Z_\-]{20,64}", token_suffix):
|
||||||
|
return False
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def get_token_from_arg(token_arg: str | Path | None) -> str | None:
|
||||||
|
if not token_arg:
|
||||||
|
logging.info("No token provided.")
|
||||||
|
return None
|
||||||
|
|
||||||
|
token_path = Path(token_arg)
|
||||||
|
if token_path.is_file():
|
||||||
|
return read_token_from_file(token_path)
|
||||||
|
|
||||||
|
return handle_direct_token(token_path, token_arg)
|
||||||
|
|
||||||
|
|
||||||
|
def read_token_from_file(token_path: Path) -> str:
|
||||||
|
token = token_path.read_text().strip()
|
||||||
|
logging.info(f"Token read from file: {token_path}")
|
||||||
|
return token
|
||||||
|
|
||||||
|
|
||||||
|
def handle_direct_token(token_path: Path, token_arg: str | Path) -> str | None:
|
||||||
|
if token_path == Path(get_token_from_default_dir()):
|
||||||
|
logging.warning(
|
||||||
|
f"The default token file {token_path} was not found. "
|
||||||
|
"Please provide a token file or a token directly via --token arg."
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
logging.info("Token provided directly as an argument.")
|
||||||
|
return str(token_arg)
|
||||||
|
|
||||||
|
|
||||||
|
def read_token(token_arg: str | Path | None) -> str | None:
|
||||||
|
token = get_token_from_arg(token_arg)
|
||||||
|
if token and not validate_gitlab_token(token):
|
||||||
|
logging.warning("The provided token is either an old token or does not seem to "
|
||||||
|
"be a valid token.")
|
||||||
|
logging.warning("Newer tokens are the ones created from a Gitlab 14.5+ instance.")
|
||||||
|
logging.warning("See https://about.gitlab.com/releases/2021/11/22/"
|
||||||
|
"gitlab-14-5-released/"
|
||||||
|
"#new-gitlab-access-token-prefix-and-detection")
|
||||||
|
return token
|
||||||
|
|
||||||
|
|
||||||
|
def wait_for_pipeline(projects, sha: str, timeout=None):
|
||||||
|
"""await until pipeline appears in Gitlab"""
|
||||||
|
project_names = [project.path_with_namespace for project in projects]
|
||||||
|
print(f"⏲ for the pipeline to appear in {project_names}..", end="")
|
||||||
|
start_time = time.time()
|
||||||
|
while True:
|
||||||
|
for project in projects:
|
||||||
|
pipelines = project.pipelines.list(sha=sha)
|
||||||
|
if pipelines:
|
||||||
|
print("", flush=True)
|
||||||
|
return (pipelines[0], project)
|
||||||
|
print("", end=".", flush=True)
|
||||||
|
if timeout and time.time() - start_time > timeout:
|
||||||
|
print(" not found", flush=True)
|
||||||
|
return (None, None)
|
||||||
|
time.sleep(1)
|
||||||
|
|
||||||
|
@cache
|
||||||
|
def is_gitlab_job() -> bool:
|
||||||
|
return os.getenv("CI_JOB_ID") is not None
|
||||||
Executable
+641
@@ -0,0 +1,641 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# For the dependencies, see the requirements.txt
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import traceback
|
||||||
|
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser, Namespace
|
||||||
|
from collections import OrderedDict
|
||||||
|
from copy import deepcopy
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from itertools import accumulate
|
||||||
|
from os import get_terminal_size, getenv
|
||||||
|
from pathlib import Path
|
||||||
|
from subprocess import check_output
|
||||||
|
from textwrap import dedent
|
||||||
|
from typing import Any, Iterable, Optional, TypedDict, Union
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
from filecache import DAY, filecache
|
||||||
|
from gitlab_common import get_token_from_default_dir, is_gitlab_job
|
||||||
|
from gql import Client, gql
|
||||||
|
from gql.transport.requests import RequestsHTTPTransport
|
||||||
|
from graphql import DocumentNode
|
||||||
|
from rich.console import Console
|
||||||
|
|
||||||
|
DEFAULT_TERMINAL_SIZE: int = 80 # columns
|
||||||
|
|
||||||
|
|
||||||
|
class DagNode(TypedDict):
|
||||||
|
needs: set[str]
|
||||||
|
stage: str
|
||||||
|
tags: set[str]
|
||||||
|
# `name` is redundant but is here for retro-compatibility
|
||||||
|
name: str
|
||||||
|
|
||||||
|
|
||||||
|
# see create_job_needs_dag function for more details
|
||||||
|
Dag = dict[str, DagNode]
|
||||||
|
|
||||||
|
|
||||||
|
StageSeq = OrderedDict[str, set[str]]
|
||||||
|
|
||||||
|
if is_gitlab_job():
|
||||||
|
console = Console(highlight=False, no_color=False, color_system="truecolor", width=120)
|
||||||
|
else:
|
||||||
|
console = Console(highlight=False)
|
||||||
|
print = console.print
|
||||||
|
|
||||||
|
|
||||||
|
def get_project_root_dir():
|
||||||
|
root_path = Path(__file__).parent.parent.parent.resolve()
|
||||||
|
gitlab_file = root_path / ".gitlab-ci.yml"
|
||||||
|
assert gitlab_file.exists()
|
||||||
|
|
||||||
|
return root_path
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class GitlabGQL:
|
||||||
|
_transport: Any = field(init=False)
|
||||||
|
client: Client = field(init=False)
|
||||||
|
url: str = "https://gitlab.freedesktop.org/api/graphql"
|
||||||
|
token: Optional[str] = None
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
self._setup_gitlab_gql_client()
|
||||||
|
|
||||||
|
def _setup_gitlab_gql_client(self) -> None:
|
||||||
|
# Select your transport with a defined url endpoint
|
||||||
|
headers = {}
|
||||||
|
if self.token:
|
||||||
|
headers["Authorization"] = f"Bearer {self.token}"
|
||||||
|
self._transport = RequestsHTTPTransport(url=self.url, headers=headers)
|
||||||
|
|
||||||
|
# Create a GraphQL client using the defined transport
|
||||||
|
self.client = Client(transport=self._transport, fetch_schema_from_transport=True)
|
||||||
|
|
||||||
|
def query(
|
||||||
|
self,
|
||||||
|
gql_file: Union[Path, str],
|
||||||
|
params: dict[str, Any] = {},
|
||||||
|
operation_name: Optional[str] = None,
|
||||||
|
paginated_key_loc: Iterable[str] = [],
|
||||||
|
disable_cache: bool = False,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
def run_uncached() -> dict[str, Any]:
|
||||||
|
if paginated_key_loc:
|
||||||
|
return self._sweep_pages(gql_file, params, operation_name, paginated_key_loc)
|
||||||
|
return self._query(gql_file, params, operation_name)
|
||||||
|
|
||||||
|
if disable_cache:
|
||||||
|
return run_uncached()
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Create an auxiliary variable to deliver a cached result and enable catching exceptions
|
||||||
|
# Decorate the query to be cached
|
||||||
|
if paginated_key_loc:
|
||||||
|
result = self._sweep_pages_cached(
|
||||||
|
gql_file, params, operation_name, paginated_key_loc
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
result = self._query_cached(gql_file, params, operation_name)
|
||||||
|
return result # type: ignore
|
||||||
|
except Exception as ex:
|
||||||
|
logging.error(f"Cached query failed with {ex}")
|
||||||
|
# print exception traceback
|
||||||
|
traceback_str = "".join(traceback.format_exception(ex))
|
||||||
|
logging.error(traceback_str)
|
||||||
|
self.invalidate_query_cache()
|
||||||
|
logging.error("Cache invalidated, retrying without cache")
|
||||||
|
return run_uncached()
|
||||||
|
|
||||||
|
def _query(
|
||||||
|
self,
|
||||||
|
gql_file: Union[Path, str],
|
||||||
|
params: dict[str, Any] = {},
|
||||||
|
operation_name: Optional[str] = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
# Provide a GraphQL query
|
||||||
|
source_path: Path = Path(__file__).parent
|
||||||
|
pipeline_query_file: Path = source_path / gql_file
|
||||||
|
|
||||||
|
query: DocumentNode
|
||||||
|
with open(pipeline_query_file, "r") as f:
|
||||||
|
pipeline_query = f.read()
|
||||||
|
query = gql(pipeline_query)
|
||||||
|
|
||||||
|
# Execute the query on the transport
|
||||||
|
return self.client.execute_sync(
|
||||||
|
query, variable_values=params, operation_name=operation_name
|
||||||
|
)
|
||||||
|
|
||||||
|
@filecache(DAY)
|
||||||
|
def _sweep_pages_cached(self, *args, **kwargs):
|
||||||
|
return self._sweep_pages(*args, **kwargs)
|
||||||
|
|
||||||
|
@filecache(DAY)
|
||||||
|
def _query_cached(self, *args, **kwargs):
|
||||||
|
return self._query(*args, **kwargs)
|
||||||
|
|
||||||
|
def _sweep_pages(
|
||||||
|
self, query, params, operation_name=None, paginated_key_loc: Iterable[str] = []
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Retrieve paginated data from a GraphQL API and concatenate the results into a single
|
||||||
|
response.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
query: represents a filepath with the GraphQL query to be executed.
|
||||||
|
params: a dictionary that contains the parameters to be passed to the query. These
|
||||||
|
parameters can be used to filter or modify the results of the query.
|
||||||
|
operation_name: The `operation_name` parameter is an optional parameter that specifies
|
||||||
|
the name of the GraphQL operation to be executed. It is used when making a GraphQL
|
||||||
|
query to specify which operation to execute if there are multiple operations defined
|
||||||
|
in the GraphQL schema. If not provided, the default operation will be executed.
|
||||||
|
paginated_key_loc (Iterable[str]): The `paginated_key_loc` parameter is an iterable of
|
||||||
|
strings that represents the location of the paginated field within the response. It
|
||||||
|
is used to extract the paginated field from the response and append it to the final
|
||||||
|
result. The node has to be a list of objects with a `pageInfo` field that contains
|
||||||
|
at least the `hasNextPage` and `endCursor` fields.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
a dictionary containing the response from the query with the paginated field
|
||||||
|
concatenated.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def fetch_page(cursor: str | None = None) -> dict[str, Any]:
|
||||||
|
if cursor:
|
||||||
|
params["cursor"] = cursor
|
||||||
|
logging.info(
|
||||||
|
f"Found more than 100 elements, paginating. "
|
||||||
|
f"Current cursor at {cursor}"
|
||||||
|
)
|
||||||
|
|
||||||
|
return self._query(query, params, operation_name)
|
||||||
|
|
||||||
|
# Execute the initial query
|
||||||
|
response: dict[str, Any] = fetch_page()
|
||||||
|
|
||||||
|
# Initialize an empty list to store the final result
|
||||||
|
final_partial_field: list[dict[str, Any]] = []
|
||||||
|
|
||||||
|
# Loop until all pages have been retrieved
|
||||||
|
while True:
|
||||||
|
# Get the partial field to be appended to the final result
|
||||||
|
partial_field = response
|
||||||
|
for key in paginated_key_loc:
|
||||||
|
partial_field = partial_field[key]
|
||||||
|
|
||||||
|
# Append the partial field to the final result
|
||||||
|
final_partial_field += partial_field["nodes"]
|
||||||
|
|
||||||
|
# Check if there are more pages to retrieve
|
||||||
|
page_info = partial_field["pageInfo"]
|
||||||
|
if not page_info["hasNextPage"]:
|
||||||
|
break
|
||||||
|
|
||||||
|
# Execute the query with the updated cursor parameter
|
||||||
|
response = fetch_page(page_info["endCursor"])
|
||||||
|
|
||||||
|
# Replace the "nodes" field in the original response with the final result
|
||||||
|
partial_field["nodes"] = final_partial_field
|
||||||
|
return response
|
||||||
|
|
||||||
|
def invalidate_query_cache(self) -> None:
|
||||||
|
logging.warning("Invalidating query cache")
|
||||||
|
try:
|
||||||
|
self._sweep_pages._db.clear()
|
||||||
|
self._query._db.clear()
|
||||||
|
except AttributeError as ex:
|
||||||
|
logging.warning(f"Could not invalidate cache, maybe it was not used in {ex.args}?")
|
||||||
|
|
||||||
|
|
||||||
|
def insert_early_stage_jobs(stage_sequence: StageSeq, jobs_metadata: Dag) -> Dag:
|
||||||
|
pre_processed_dag: dict[str, set[str]] = {}
|
||||||
|
jobs_from_early_stages = list(accumulate(stage_sequence.values(), set.union))
|
||||||
|
for job_name, metadata in jobs_metadata.items():
|
||||||
|
final_needs: set[str] = deepcopy(metadata["needs"])
|
||||||
|
# Pre-process jobs that are not based on needs field
|
||||||
|
# e.g. sanity job in mesa MR pipelines
|
||||||
|
if not final_needs:
|
||||||
|
job_stage: str = jobs_metadata[job_name]["stage"]
|
||||||
|
stage_index: int = list(stage_sequence.keys()).index(job_stage)
|
||||||
|
if stage_index > 0:
|
||||||
|
final_needs |= jobs_from_early_stages[stage_index - 1]
|
||||||
|
pre_processed_dag[job_name] = final_needs
|
||||||
|
|
||||||
|
for job_name, needs in pre_processed_dag.items():
|
||||||
|
jobs_metadata[job_name]["needs"] = needs
|
||||||
|
|
||||||
|
return jobs_metadata
|
||||||
|
|
||||||
|
|
||||||
|
def traverse_dag_needs(jobs_metadata: Dag) -> None:
|
||||||
|
created_jobs = set(jobs_metadata.keys())
|
||||||
|
for job, metadata in jobs_metadata.items():
|
||||||
|
final_needs: set = deepcopy(metadata["needs"]) & created_jobs
|
||||||
|
# Post process jobs that are based on needs field
|
||||||
|
partial = True
|
||||||
|
|
||||||
|
while partial:
|
||||||
|
next_depth: set[str] = {n for dn in final_needs if dn in jobs_metadata for n in jobs_metadata[dn]["needs"]}
|
||||||
|
partial: bool = not final_needs.issuperset(next_depth)
|
||||||
|
final_needs = final_needs.union(next_depth)
|
||||||
|
|
||||||
|
jobs_metadata[job]["needs"] = final_needs
|
||||||
|
|
||||||
|
|
||||||
|
def extract_stages_and_job_needs(
|
||||||
|
pipeline_jobs: dict[str, Any], pipeline_stages: dict[str, Any]
|
||||||
|
) -> tuple[StageSeq, Dag]:
|
||||||
|
jobs_metadata = Dag()
|
||||||
|
# Record the stage sequence to post process deps that are not based on needs
|
||||||
|
# field, for example: sanity job
|
||||||
|
stage_sequence: OrderedDict[str, set[str]] = OrderedDict()
|
||||||
|
for stage in pipeline_stages["nodes"]:
|
||||||
|
stage_sequence[stage["name"]] = set()
|
||||||
|
|
||||||
|
for job in pipeline_jobs["nodes"]:
|
||||||
|
stage_sequence[job["stage"]["name"]].add(job["name"])
|
||||||
|
dag_job: DagNode = {
|
||||||
|
"name": job["name"],
|
||||||
|
"tags": set(job["tags"] or []), # jobs with no tags defined return None here
|
||||||
|
"stage": job["stage"]["name"],
|
||||||
|
"needs": set([j["node"]["name"] for j in job["needs"]["edges"]]),
|
||||||
|
}
|
||||||
|
jobs_metadata[job["name"]] = dag_job
|
||||||
|
|
||||||
|
return stage_sequence, jobs_metadata
|
||||||
|
|
||||||
|
|
||||||
|
def create_job_needs_dag(gl_gql: GitlabGQL, params, disable_cache: bool = True) -> Dag:
|
||||||
|
"""
|
||||||
|
This function creates a Directed Acyclic Graph (DAG) to represent a sequence of jobs, where each
|
||||||
|
job has a set of jobs that it depends on (its "needs") and belongs to a certain "stage".
|
||||||
|
The "name" of the job is used as the key in the dictionary.
|
||||||
|
|
||||||
|
For example, consider the following DAG:
|
||||||
|
|
||||||
|
1. build stage: job1 -> job2 -> job3
|
||||||
|
2. test stage: job2 -> job4
|
||||||
|
|
||||||
|
- The job needs for job3 are: job1, job2
|
||||||
|
- The job needs for job4 are: job2
|
||||||
|
- The job2 needs to wait all jobs from build stage to finish.
|
||||||
|
|
||||||
|
The resulting DAG would look like this:
|
||||||
|
|
||||||
|
dag = {
|
||||||
|
"job1": {"needs": set(), "stage": "build", "name": "job1"},
|
||||||
|
"job2": {"needs": {"job1", "job2", job3"}, "stage": "test", "name": "job2"},
|
||||||
|
"job3": {"needs": {"job1", "job2"}, "stage": "build", "name": "job3"},
|
||||||
|
"job4": {"needs": {"job2"}, "stage": "test", "name": "job4"},
|
||||||
|
}
|
||||||
|
|
||||||
|
To access the job needs, one can do:
|
||||||
|
|
||||||
|
dag["job3"]["needs"]
|
||||||
|
|
||||||
|
This will return the set of jobs that job3 needs: {"job1", "job2"}
|
||||||
|
|
||||||
|
Args:
|
||||||
|
gl_gql (GitlabGQL): The `gl_gql` parameter is an instance of the `GitlabGQL` class, which is
|
||||||
|
used to make GraphQL queries to the GitLab API.
|
||||||
|
params (dict): The `params` parameter is a dictionary that contains the necessary parameters
|
||||||
|
for the GraphQL query. It is used to specify the details of the pipeline for which the
|
||||||
|
job needs DAG is being created.
|
||||||
|
The specific keys and values in the `params` dictionary will depend on
|
||||||
|
the requirements of the GraphQL query being executed
|
||||||
|
disable_cache (bool): The `disable_cache` parameter is a boolean that specifies whether the
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The final DAG (Directed Acyclic Graph) representing the job dependencies sourced from needs
|
||||||
|
or stages rule.
|
||||||
|
"""
|
||||||
|
stages_jobs_gql = gl_gql.query(
|
||||||
|
"pipeline_details.gql",
|
||||||
|
params=params,
|
||||||
|
paginated_key_loc=["project", "pipeline", "jobs"],
|
||||||
|
disable_cache=disable_cache,
|
||||||
|
)
|
||||||
|
pipeline_data = stages_jobs_gql["project"]["pipeline"]
|
||||||
|
if not pipeline_data:
|
||||||
|
raise RuntimeError(f"Could not find any pipelines for {params}")
|
||||||
|
|
||||||
|
stage_sequence, jobs_metadata = extract_stages_and_job_needs(
|
||||||
|
pipeline_data["jobs"], pipeline_data["stages"]
|
||||||
|
)
|
||||||
|
# Fill the DAG with the job needs from stages that don't have any needs but still need to wait
|
||||||
|
# for previous stages
|
||||||
|
final_dag = insert_early_stage_jobs(stage_sequence, jobs_metadata)
|
||||||
|
# Now that each job has its direct needs filled correctly, update the "needs" field for each job
|
||||||
|
# in the DAG by performing a topological traversal
|
||||||
|
traverse_dag_needs(final_dag)
|
||||||
|
|
||||||
|
return final_dag
|
||||||
|
|
||||||
|
|
||||||
|
def filter_dag(dag: Dag, job_filter: callable) -> Dag:
|
||||||
|
return Dag({
|
||||||
|
job: data
|
||||||
|
for job, data in dag.items()
|
||||||
|
if job_filter(
|
||||||
|
job_name=job,
|
||||||
|
job_stage=data["stage"],
|
||||||
|
job_tags=data["tags"],
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def print_dag(dag: Dag, indentation: int = 0, color: str = "") -> None:
|
||||||
|
for job, data in sorted(dag.items()):
|
||||||
|
print(f"{color}{' '*indentation}{job}:")
|
||||||
|
print_formatted_list(list(data['needs']), indentation=indentation+8, color=color)
|
||||||
|
|
||||||
|
|
||||||
|
def print_formatted_list(elements: list[str], indentation: int = 0, color: str = "") -> None:
|
||||||
|
"""
|
||||||
|
When a list of elements is going to be printed, if it is longer than one line, reformat it to be multiple
|
||||||
|
lines with a 'ls' command style.
|
||||||
|
:param elements: list of elements to be printed
|
||||||
|
:param indentation: number of spaces to be injected in front of each line
|
||||||
|
"""
|
||||||
|
if len(elements) == 0:
|
||||||
|
return
|
||||||
|
elements.sort()
|
||||||
|
try:
|
||||||
|
h_size = get_terminal_size().columns if sys.stdin.isatty() else DEFAULT_TERMINAL_SIZE
|
||||||
|
except OSError:
|
||||||
|
h_size = DEFAULT_TERMINAL_SIZE
|
||||||
|
if indentation + sum(len(element) for element in elements) + (len(elements)*2) < h_size: # fits in one line
|
||||||
|
print(f"{color}{' '*indentation}{', '.join([element for element in elements])}")
|
||||||
|
return
|
||||||
|
column_separator_size = 2
|
||||||
|
column_width: int = len(max(elements, key=len)) + column_separator_size
|
||||||
|
n_columns: int = max((h_size - indentation) // column_width, 1)
|
||||||
|
step = (len(elements) // n_columns) + 1
|
||||||
|
rows = [elements[i::step] for i in range(step)]
|
||||||
|
for line in rows:
|
||||||
|
print(' '*indentation, end='')
|
||||||
|
for column in range(len(line)):
|
||||||
|
if line[column] is not None:
|
||||||
|
print(f"{color}{line[column]:<{column_width}}", end='')
|
||||||
|
print()
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_merged_yaml(gl_gql: GitlabGQL, params) -> dict[str, Any]:
|
||||||
|
params["content"] = dedent("""\
|
||||||
|
include:
|
||||||
|
- local: .gitlab-ci.yml
|
||||||
|
""")
|
||||||
|
raw_response = gl_gql.query("job_details.gql", params)
|
||||||
|
ci_config = raw_response["ciConfig"]
|
||||||
|
if merged_yaml := ci_config["mergedYaml"]:
|
||||||
|
return yaml.safe_load(merged_yaml)
|
||||||
|
if "errors" in ci_config:
|
||||||
|
for error in ci_config["errors"]:
|
||||||
|
print(error)
|
||||||
|
|
||||||
|
gl_gql.invalidate_query_cache()
|
||||||
|
raise ValueError(
|
||||||
|
"""
|
||||||
|
Could not fetch any content for merged YAML,
|
||||||
|
please verify if the git SHA exists in remote.
|
||||||
|
Maybe you forgot to `git push`? """
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def recursive_fill(job, relationship_field, target_data, acc_data: dict, merged_yaml):
|
||||||
|
if relatives := job.get(relationship_field):
|
||||||
|
if isinstance(relatives, str):
|
||||||
|
relatives = [relatives]
|
||||||
|
|
||||||
|
for relative in relatives:
|
||||||
|
parent_job = merged_yaml[relative]
|
||||||
|
acc_data = recursive_fill(parent_job, acc_data, merged_yaml) # type: ignore
|
||||||
|
|
||||||
|
acc_data |= job.get(target_data, {})
|
||||||
|
|
||||||
|
return acc_data
|
||||||
|
|
||||||
|
|
||||||
|
def get_variables(job, merged_yaml, project_path, sha) -> dict[str, str]:
|
||||||
|
p = get_project_root_dir() / ".gitlab-ci" / "image-tags.yml"
|
||||||
|
image_tags = yaml.safe_load(p.read_text())
|
||||||
|
|
||||||
|
variables = image_tags["variables"]
|
||||||
|
variables |= merged_yaml["variables"]
|
||||||
|
variables |= job["variables"]
|
||||||
|
variables["CI_PROJECT_PATH"] = project_path
|
||||||
|
variables["CI_PROJECT_NAME"] = project_path.split("/")[1]
|
||||||
|
variables["CI_REGISTRY_IMAGE"] = "registry.freedesktop.org/${CI_PROJECT_PATH}"
|
||||||
|
variables["CI_COMMIT_SHA"] = sha
|
||||||
|
|
||||||
|
while recurse_among_variables_space(variables):
|
||||||
|
pass
|
||||||
|
|
||||||
|
return variables
|
||||||
|
|
||||||
|
|
||||||
|
# Based on: https://stackoverflow.com/a/2158532/1079223
|
||||||
|
def flatten(xs):
|
||||||
|
for x in xs:
|
||||||
|
if isinstance(x, Iterable) and not isinstance(x, (str, bytes)):
|
||||||
|
yield from flatten(x)
|
||||||
|
else:
|
||||||
|
yield x
|
||||||
|
|
||||||
|
|
||||||
|
def get_full_script(job) -> list[str]:
|
||||||
|
script = []
|
||||||
|
for script_part in ("before_script", "script", "after_script"):
|
||||||
|
script.append(f"# {script_part}")
|
||||||
|
lines = flatten(job.get(script_part, []))
|
||||||
|
script.extend(lines)
|
||||||
|
script.append("")
|
||||||
|
|
||||||
|
return script
|
||||||
|
|
||||||
|
|
||||||
|
def recurse_among_variables_space(var_graph) -> bool:
|
||||||
|
updated = False
|
||||||
|
for var, value in var_graph.items():
|
||||||
|
value = str(value)
|
||||||
|
dep_vars = []
|
||||||
|
if match := re.findall(r"(\$[{]?[\w\d_]*[}]?)", value):
|
||||||
|
all_dep_vars = [v.lstrip("${").rstrip("}") for v in match]
|
||||||
|
# print(value, match, all_dep_vars)
|
||||||
|
dep_vars = [v for v in all_dep_vars if v in var_graph]
|
||||||
|
|
||||||
|
for dep_var in dep_vars:
|
||||||
|
dep_value = str(var_graph[dep_var])
|
||||||
|
new_value = var_graph[var]
|
||||||
|
new_value = new_value.replace(f"${{{dep_var}}}", dep_value)
|
||||||
|
new_value = new_value.replace(f"${dep_var}", dep_value)
|
||||||
|
var_graph[var] = new_value
|
||||||
|
updated |= dep_value != new_value
|
||||||
|
|
||||||
|
return updated
|
||||||
|
|
||||||
|
|
||||||
|
def print_job_final_definition(job_name, merged_yaml, project_path, sha):
|
||||||
|
job = merged_yaml[job_name]
|
||||||
|
variables = get_variables(job, merged_yaml, project_path, sha)
|
||||||
|
|
||||||
|
print("# --------- variables ---------------")
|
||||||
|
for var, value in sorted(variables.items()):
|
||||||
|
print(f"export {var}={value!r}")
|
||||||
|
|
||||||
|
# TODO: Recurse into needs to get full script
|
||||||
|
# TODO: maybe create a extra yaml file to avoid too much rework
|
||||||
|
script = get_full_script(job)
|
||||||
|
print()
|
||||||
|
print()
|
||||||
|
print("# --------- full script ---------------")
|
||||||
|
print("\n".join(script))
|
||||||
|
|
||||||
|
if image := variables.get("MESA_IMAGE"):
|
||||||
|
print()
|
||||||
|
print()
|
||||||
|
print("# --------- container image ---------------")
|
||||||
|
print(image)
|
||||||
|
|
||||||
|
|
||||||
|
def from_sha_to_pipeline_iid(gl_gql: GitlabGQL, params) -> str:
|
||||||
|
result = gl_gql.query("pipeline_utils.gql", params)
|
||||||
|
|
||||||
|
return result["project"]["pipelines"]["nodes"][0]["iid"]
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> Namespace:
|
||||||
|
parser = ArgumentParser(
|
||||||
|
formatter_class=ArgumentDefaultsHelpFormatter,
|
||||||
|
description="CLI and library with utility functions to debug jobs via Gitlab GraphQL",
|
||||||
|
epilog=f"""Example:
|
||||||
|
{Path(__file__).name} --print-dag""",
|
||||||
|
)
|
||||||
|
parser.add_argument("-pp", "--project-path", type=str, default="mesa/mesa")
|
||||||
|
parser.add_argument("--sha", "--rev", type=str, default='HEAD')
|
||||||
|
parser.add_argument(
|
||||||
|
"--regex",
|
||||||
|
type=str,
|
||||||
|
required=False,
|
||||||
|
default=".*",
|
||||||
|
help="Regex pattern for the job name to be considered",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--include-stage",
|
||||||
|
type=str,
|
||||||
|
required=False,
|
||||||
|
default=".*",
|
||||||
|
help="Regex pattern for the stage name to be considered",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--exclude-stage",
|
||||||
|
type=str,
|
||||||
|
required=False,
|
||||||
|
default="^$",
|
||||||
|
help="Regex pattern for the stage name to be excluded",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--job-tags",
|
||||||
|
metavar="job-tags",
|
||||||
|
help="Job tags to require when searching for target jobs. If multiple "
|
||||||
|
"values are passed, eg. `--job-tags 'foo.*' 'bar'`, the job will "
|
||||||
|
"need to have a tag matching `foo.*` *and* a tag matching `bar` "
|
||||||
|
"to qualify. Passing `--job-tags '.*'` makes sure the job has "
|
||||||
|
"a tag defined, while not passing `--job-tags` also allows "
|
||||||
|
"untagged jobs.",
|
||||||
|
default=[],
|
||||||
|
nargs='+',
|
||||||
|
)
|
||||||
|
mutex_group_print = parser.add_mutually_exclusive_group()
|
||||||
|
mutex_group_print.add_argument(
|
||||||
|
"--print-dag",
|
||||||
|
action="store_true",
|
||||||
|
help="Print job needs DAG",
|
||||||
|
)
|
||||||
|
mutex_group_print.add_argument(
|
||||||
|
"--print-merged-yaml",
|
||||||
|
action="store_true",
|
||||||
|
help="Print the resulting YAML for the specific SHA",
|
||||||
|
)
|
||||||
|
mutex_group_print.add_argument(
|
||||||
|
"--print-job-manifest",
|
||||||
|
metavar='JOB_NAME',
|
||||||
|
type=str,
|
||||||
|
help="Print the resulting job data"
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--gitlab-token-file",
|
||||||
|
type=str,
|
||||||
|
default=get_token_from_default_dir(),
|
||||||
|
help="force GitLab token, otherwise it's read from $XDG_CONFIG_HOME/gitlab-token",
|
||||||
|
)
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
args.gitlab_token = Path(args.gitlab_token_file).read_text().strip()
|
||||||
|
return args
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
args = parse_args()
|
||||||
|
gl_gql = GitlabGQL(token=args.gitlab_token)
|
||||||
|
|
||||||
|
sha = check_output(['git', 'rev-parse', args.sha]).decode('ascii').strip()
|
||||||
|
|
||||||
|
if args.print_dag:
|
||||||
|
iid = from_sha_to_pipeline_iid(gl_gql, {"projectPath": args.project_path, "sha": sha})
|
||||||
|
dag = create_job_needs_dag(
|
||||||
|
gl_gql, {"projectPath": args.project_path, "iid": iid}, disable_cache=True
|
||||||
|
)
|
||||||
|
|
||||||
|
job_name_regex = re.compile(args.regex)
|
||||||
|
include_stage_regex = re.compile(args.include_stage)
|
||||||
|
exclude_stage_regex = re.compile(args.exclude_stage)
|
||||||
|
job_tags_regexes = [re.compile(job_tag) for job_tag in args.job_tags]
|
||||||
|
|
||||||
|
def job_filter(
|
||||||
|
job_name: str,
|
||||||
|
job_stage: str,
|
||||||
|
job_tags: set[str],
|
||||||
|
) -> bool:
|
||||||
|
"""
|
||||||
|
Apply user-specified filters to a job, and return whether the
|
||||||
|
filters allow that job (True) or not (False).
|
||||||
|
"""
|
||||||
|
if not job_name_regex.fullmatch(job_name):
|
||||||
|
return False
|
||||||
|
if not include_stage_regex.fullmatch(job_stage):
|
||||||
|
return False
|
||||||
|
if exclude_stage_regex.fullmatch(job_stage):
|
||||||
|
return False
|
||||||
|
if not all(
|
||||||
|
any(job_tags_regex.fullmatch(tag) for tag in job_tags)
|
||||||
|
for job_tags_regex in job_tags_regexes
|
||||||
|
):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
dag = filter_dag(dag, job_filter)
|
||||||
|
|
||||||
|
print_dag(dag)
|
||||||
|
|
||||||
|
if args.print_merged_yaml or args.print_job_manifest:
|
||||||
|
merged_yaml = fetch_merged_yaml(
|
||||||
|
gl_gql, {"projectPath": args.project_path, "sha": sha}
|
||||||
|
)
|
||||||
|
|
||||||
|
if args.print_merged_yaml:
|
||||||
|
print(yaml.dump(merged_yaml, indent=2))
|
||||||
|
|
||||||
|
if args.print_job_manifest:
|
||||||
|
print_job_final_definition(
|
||||||
|
args.print_job_manifest, merged_yaml, args.project_path, sha
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Executable
+10
@@ -0,0 +1,10 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
|
||||||
|
readonly this_dir
|
||||||
|
|
||||||
|
exec \
|
||||||
|
"$this_dir/../python-venv.sh" \
|
||||||
|
"$this_dir/requirements.txt" \
|
||||||
|
"$this_dir/gitlab_gql.py" "$@"
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
query getCiConfigData($projectPath: ID!, $sha: String, $content: String!) {
|
||||||
|
ciConfig(projectPath: $projectPath, sha: $sha, content: $content) {
|
||||||
|
errors
|
||||||
|
mergedYaml
|
||||||
|
__typename
|
||||||
|
}
|
||||||
|
}
|
||||||
Executable
+236
@@ -0,0 +1,236 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# Copyright © 2020 - 2023 Collabora Ltd.
|
||||||
|
# Authors:
|
||||||
|
# David Heidelberg <[email protected]>
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
"""
|
||||||
|
Monitors Marge-bot and return number of assigned MRs.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import time
|
||||||
|
import sys
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from dateutil import parser
|
||||||
|
from traceback import print_exc
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
import gitlab
|
||||||
|
from gitlab.base import RESTObjectList
|
||||||
|
from gitlab.exceptions import GitlabAuthenticationError
|
||||||
|
from gitlab.v4.objects import Project, ProjectMergeRequest
|
||||||
|
from gitlab_common import get_token_from_default_dir, read_token, pretty_duration
|
||||||
|
from rich.console import Console
|
||||||
|
|
||||||
|
REFRESH_WAIT = 30
|
||||||
|
MARGE_BOT_USER_ID = 9716
|
||||||
|
ASSIGNED_TO_MARGE = "assigned to @marge-bot"
|
||||||
|
ASSIGNED_AT_PADDING = 19
|
||||||
|
WAITING_PADDING = 8
|
||||||
|
MR_ID_PADDING = 5
|
||||||
|
|
||||||
|
console = Console(highlight=False)
|
||||||
|
print = console.print
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> argparse.Namespace:
|
||||||
|
"""Parse args"""
|
||||||
|
parse = argparse.ArgumentParser(
|
||||||
|
description="Tool to show merge requests assigned to the marge-bot",
|
||||||
|
)
|
||||||
|
parse.add_argument(
|
||||||
|
"--wait", action="store_true", help="wait until CI is free",
|
||||||
|
)
|
||||||
|
parse.add_argument(
|
||||||
|
"--token",
|
||||||
|
metavar="token",
|
||||||
|
default=get_token_from_default_dir(),
|
||||||
|
help="force GitLab token, otherwise it's read from ~/.config/gitlab-token",
|
||||||
|
)
|
||||||
|
return parse.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class MargeMergeRequest:
|
||||||
|
"""Represent a Merge Request assigned to Marge"""
|
||||||
|
id: int = field(init=False)
|
||||||
|
mr: ProjectMergeRequest = field(repr=False)
|
||||||
|
updated_at: datetime | None = field(init=False, repr=False)
|
||||||
|
assigned_at: datetime | None = field(init=False, repr=False)
|
||||||
|
|
||||||
|
def __post_init__(self):
|
||||||
|
self.id = self.mr.iid
|
||||||
|
self.updated_at = parser.parse(self.mr.updated_at)
|
||||||
|
self.assigned_at = self.__find_last_assign_to_marge()
|
||||||
|
|
||||||
|
def __find_last_assign_to_marge(self) -> Optional[datetime]:
|
||||||
|
for note in self.__get_mr_notes_iterator():
|
||||||
|
if note.body.startswith(ASSIGNED_TO_MARGE):
|
||||||
|
return parser.parse(note.created_at)
|
||||||
|
|
||||||
|
def __eq__(self, other: "MargeMergeRequest") -> bool:
|
||||||
|
return self.id == other.id
|
||||||
|
|
||||||
|
@property
|
||||||
|
def time_enqueued(self) -> timedelta:
|
||||||
|
if self.assigned_at is None:
|
||||||
|
raise ValueError("Assign to marge timestamp not defined")
|
||||||
|
return datetime.now(timezone.utc) - self.assigned_at
|
||||||
|
|
||||||
|
@property
|
||||||
|
def web_url(self) -> str:
|
||||||
|
return self.mr.web_url
|
||||||
|
|
||||||
|
@property
|
||||||
|
def title(self) -> str:
|
||||||
|
return self.mr.title
|
||||||
|
|
||||||
|
def __get_mr_notes_iterator(self) -> RESTObjectList:
|
||||||
|
return self.mr.notes.list(
|
||||||
|
iterator=True, order_by="updated_at", sort="desc"
|
||||||
|
) # start with the most recent
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class MargeQueue:
|
||||||
|
"""
|
||||||
|
Collect and sort the merge requests assigned to marge
|
||||||
|
"""
|
||||||
|
elements_sorted: dict[datetime, MargeMergeRequest] = field(
|
||||||
|
init=False, repr=False, default_factory=dict
|
||||||
|
)
|
||||||
|
undetermined: list[MargeMergeRequest] = field(
|
||||||
|
init=False, repr=False, default_factory=list
|
||||||
|
)
|
||||||
|
|
||||||
|
def append(self, mr: MargeMergeRequest) -> None:
|
||||||
|
if mr.assigned_at is not None:
|
||||||
|
self.elements_sorted[mr.assigned_at] = mr
|
||||||
|
else:
|
||||||
|
self.undetermined.append(mr)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def n_merge_requests_enqueued(self) -> int:
|
||||||
|
return len(self.elements_sorted) + len(self.undetermined)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def sorted_queue(self) -> list[MargeMergeRequest]:
|
||||||
|
"""
|
||||||
|
Provide a list of the elements that can be sorted based on the
|
||||||
|
assignment to Marge.
|
||||||
|
"""
|
||||||
|
return list(dict(sorted(self.elements_sorted.items())).values())
|
||||||
|
|
||||||
|
@property
|
||||||
|
def all_assigned(
|
||||||
|
self
|
||||||
|
) -> list[MargeMergeRequest]:
|
||||||
|
"""
|
||||||
|
Provide a single list, but in sorted, of all the elements found
|
||||||
|
assigned to Marge. This include the elements in elements_sorted
|
||||||
|
and the ones undetermined (expected to be empty).
|
||||||
|
"""
|
||||||
|
return self.sorted_queue + self.undetermined
|
||||||
|
|
||||||
|
|
||||||
|
def get_merge_queue(project: Project) -> MargeQueue:
|
||||||
|
queue = MargeQueue()
|
||||||
|
for mr in __get_project_marge_merge_requests(project):
|
||||||
|
marge_merge_request = MargeMergeRequest(mr)
|
||||||
|
queue.append(marge_merge_request)
|
||||||
|
for mr in queue.sorted_queue:
|
||||||
|
assigned_at = f"{mr.assigned_at:%Y-%m-%d %H:%M:%S}"
|
||||||
|
time_enqueued = f"{pretty_duration(mr.time_enqueued.total_seconds())}"
|
||||||
|
print(
|
||||||
|
f"⛭ {assigned_at:<{ASSIGNED_AT_PADDING}} "
|
||||||
|
f"{time_enqueued:>{WAITING_PADDING}} "
|
||||||
|
f"{link2print(mr.web_url, mr.id, MR_ID_PADDING)} {mr.title}"
|
||||||
|
)
|
||||||
|
if queue.undetermined:
|
||||||
|
print(
|
||||||
|
"Unable to determine when they where assigned to Marge:"
|
||||||
|
)
|
||||||
|
for mr in queue.undetermined:
|
||||||
|
updated_at = f"{mr.updated_at:%Y-%m-%d %H:%M:%S}"
|
||||||
|
print(
|
||||||
|
f"⛭ {updated_at:<{ASSIGNED_AT_PADDING}} "
|
||||||
|
f"{" ":{WAITING_PADDING}} "
|
||||||
|
f"{link2print(mr.web_url, mr.id, MR_ID_PADDING)} {mr.title}"
|
||||||
|
)
|
||||||
|
return queue
|
||||||
|
|
||||||
|
|
||||||
|
def __get_gitlab_object(token: str) -> gitlab.Gitlab:
|
||||||
|
return gitlab.Gitlab(url="https://gitlab.freedesktop.org", private_token=token, retry_transient_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
def __get_gitlab_project(gl: gitlab.Gitlab) -> Project:
|
||||||
|
return gl.projects.get("mesa/mesa")
|
||||||
|
|
||||||
|
|
||||||
|
def __get_project_marge_merge_requests(
|
||||||
|
project: Project
|
||||||
|
) -> list[ProjectMergeRequest]:
|
||||||
|
return project.mergerequests.list(
|
||||||
|
assignee_id=MARGE_BOT_USER_ID,
|
||||||
|
scope="all",
|
||||||
|
state="opened",
|
||||||
|
get_all=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def link2print(url: str, text: str, text_pad: int = 0) -> str:
|
||||||
|
text = str(text)
|
||||||
|
text_pad = len(text) if text_pad < 1 else text_pad
|
||||||
|
return f"[link={url}]{text:{text_pad}}[/link]"
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
args = parse_args()
|
||||||
|
token = read_token(args.token)
|
||||||
|
gl = __get_gitlab_object(token)
|
||||||
|
|
||||||
|
project = __get_gitlab_project(gl)
|
||||||
|
|
||||||
|
assigned_at = "Assigned at"
|
||||||
|
waiting = "Waiting"
|
||||||
|
mr = "MR"
|
||||||
|
title = "Title"
|
||||||
|
print(
|
||||||
|
f" {assigned_at:<{ASSIGNED_AT_PADDING}} "
|
||||||
|
f"{waiting:<{WAITING_PADDING}} "
|
||||||
|
f"{mr:<{MR_ID_PADDING}} {title}"
|
||||||
|
)
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
n_mrs = get_merge_queue(project).n_merge_requests_enqueued
|
||||||
|
except GitlabAuthenticationError as autherror:
|
||||||
|
if autherror.response_code == 401: # 401 Unauthorized
|
||||||
|
print(f"Alert! Tool action requires authentication. Use the --token argument")
|
||||||
|
else:
|
||||||
|
print(f"Alert! Something went wrong: {autherror.responce_body}")
|
||||||
|
print_exc()
|
||||||
|
sys.exit(-1)
|
||||||
|
|
||||||
|
print(f"Job waiting: {n_mrs}")
|
||||||
|
|
||||||
|
if n_mrs == 0:
|
||||||
|
sys.exit(0)
|
||||||
|
if not args.wait:
|
||||||
|
sys.exit(min(n_mrs, 127))
|
||||||
|
|
||||||
|
try:
|
||||||
|
time.sleep(REFRESH_WAIT)
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
print(
|
||||||
|
"Sleep interrupted from keyboard, "
|
||||||
|
"doing one last iteration before finish."
|
||||||
|
)
|
||||||
|
args.wait = False
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Executable
+10
@@ -0,0 +1,10 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
|
||||||
|
readonly this_dir
|
||||||
|
|
||||||
|
exec \
|
||||||
|
"$this_dir/../python-venv.sh" \
|
||||||
|
"$this_dir/requirements.txt" \
|
||||||
|
"$this_dir/marge_queue.py" "$@"
|
||||||
Executable
+197
@@ -0,0 +1,197 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# Copyright © 2020 - 2024 Collabora Ltd.
|
||||||
|
# Authors:
|
||||||
|
# David Heidelberg <[email protected]>
|
||||||
|
# Sergi Blanch Torne <[email protected]>
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
"""
|
||||||
|
Compare the two latest scheduled pipelines and provide information
|
||||||
|
about the jobs you're interested in.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import csv
|
||||||
|
import re
|
||||||
|
import requests
|
||||||
|
import io
|
||||||
|
from tabulate import tabulate
|
||||||
|
|
||||||
|
import gitlab
|
||||||
|
from gitlab_common import read_token
|
||||||
|
from rich import print
|
||||||
|
|
||||||
|
|
||||||
|
MARGE_BOT_USER_ID = 9716
|
||||||
|
|
||||||
|
|
||||||
|
def print_failures_csv(id):
|
||||||
|
url = "https://gitlab.freedesktop.org/mesa/mesa"\
|
||||||
|
f"/-/jobs/{id}/artifacts/raw/results/failures.csv"
|
||||||
|
missing: int = 0
|
||||||
|
MAX_MISS: int = 20
|
||||||
|
try:
|
||||||
|
response = requests.get(url)
|
||||||
|
response.raise_for_status()
|
||||||
|
csv_content = io.StringIO(response.text)
|
||||||
|
csv_reader = csv.reader(csv_content)
|
||||||
|
data = list(csv_reader)
|
||||||
|
|
||||||
|
for line in data[:]:
|
||||||
|
if line[1] == "UnexpectedImprovement(Pass)":
|
||||||
|
line[1] = f"[green]{line[1]}[/green]"
|
||||||
|
elif line[1] == "UnexpectedImprovement(Fail)":
|
||||||
|
line[1] = f"[yellow]{line[1]}[/yellow]"
|
||||||
|
elif line[1] == "Crash" or line[1] == "Fail":
|
||||||
|
line[1] = f" [red]{line[1]}[/red]"
|
||||||
|
elif line[1] == "Missing":
|
||||||
|
if missing > MAX_MISS:
|
||||||
|
data.remove(line)
|
||||||
|
continue
|
||||||
|
missing += 1
|
||||||
|
line[1] = f"[yellow]{line[1]}[/yellow]"
|
||||||
|
elif line[1] == "Fail":
|
||||||
|
line[1] = f"[red]{line[1]}[/red]"
|
||||||
|
else:
|
||||||
|
line[1] = f"[white]{line[1]}[/white]"
|
||||||
|
|
||||||
|
if missing > MAX_MISS:
|
||||||
|
data.append(
|
||||||
|
[
|
||||||
|
f"[red]... more than {MAX_MISS} missing tests, "
|
||||||
|
"something crashed?[/red]",
|
||||||
|
"[red]Missing[/red]"
|
||||||
|
]
|
||||||
|
)
|
||||||
|
headers = [f"Test{"":<75}", "Result"]
|
||||||
|
print(tabulate(data, headers, tablefmt="plain"))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def job_failed_before(old_jobs, job):
|
||||||
|
for old_job in old_jobs:
|
||||||
|
if job.name == old_job.name:
|
||||||
|
return old_job
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> None:
|
||||||
|
"""Parse args"""
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Tool to show merge requests assigned to the marge-bot",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--target",
|
||||||
|
metavar="target-job",
|
||||||
|
help="Target job regex. For multiple targets, pass multiple values, "
|
||||||
|
"eg. `--target foo bar`.",
|
||||||
|
required=False,
|
||||||
|
nargs=argparse.ONE_OR_MORE,
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--token",
|
||||||
|
metavar="token",
|
||||||
|
help="force GitLab token, "
|
||||||
|
"otherwise it's read from ~/.config/gitlab-token",
|
||||||
|
)
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
args = parse_args()
|
||||||
|
token = read_token(args.token)
|
||||||
|
gl = gitlab.Gitlab(
|
||||||
|
url="https://gitlab.freedesktop.org",
|
||||||
|
private_token=token,
|
||||||
|
)
|
||||||
|
|
||||||
|
project = gl.projects.get("mesa/mesa")
|
||||||
|
|
||||||
|
print(
|
||||||
|
"[link=https://gitlab.freedesktop.org/mesa/mesa/-/pipelines?"
|
||||||
|
"page=1&scope=all&source=schedule]Scheduled pipelines overview[/link]"
|
||||||
|
)
|
||||||
|
pipelines = project.pipelines.list(
|
||||||
|
source="schedule",
|
||||||
|
ordered_by="created_at",
|
||||||
|
sort="desc",
|
||||||
|
page=1,
|
||||||
|
per_page=2,
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"Old pipeline:"
|
||||||
|
f" {pipelines[1].created_at}"
|
||||||
|
f"\t[link={pipelines[1].web_url}]{pipelines[1].status}[/link]"
|
||||||
|
f"\t{pipelines[1].sha}"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"New pipeline:"
|
||||||
|
f" {pipelines[0].created_at}"
|
||||||
|
f"\t[link={pipelines[0].web_url}]{pipelines[0].status}[/link]"
|
||||||
|
f"\t{pipelines[0].sha}"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"\nWebUI visual compare: "
|
||||||
|
"https://gitlab.freedesktop.org/mesa/mesa/-/compare/"
|
||||||
|
f"{pipelines[1].sha}...{pipelines[0].sha}\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
# regex part
|
||||||
|
if args.target:
|
||||||
|
target = "|".join(args.target)
|
||||||
|
target = target.strip()
|
||||||
|
print(f"🞋 jobs: [blue]{target}[/blue]")
|
||||||
|
|
||||||
|
target = f"({target})" + r"( \d+/\d+)?"
|
||||||
|
else:
|
||||||
|
target = ".*"
|
||||||
|
|
||||||
|
target_jobs_regex: re.Pattern = re.compile(target)
|
||||||
|
|
||||||
|
old_failed_jobs = []
|
||||||
|
for job in pipelines[1].jobs.list(all=True):
|
||||||
|
if (
|
||||||
|
job.status != "failed"
|
||||||
|
or target_jobs_regex
|
||||||
|
and not target_jobs_regex.fullmatch(job.name)
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
old_failed_jobs.append(job)
|
||||||
|
|
||||||
|
job_failed = False
|
||||||
|
for job in pipelines[0].jobs.list(all=True):
|
||||||
|
if (
|
||||||
|
job.status != "failed"
|
||||||
|
or target_jobs_regex
|
||||||
|
and not target_jobs_regex.fullmatch(job.name)
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
|
||||||
|
job_failed = True
|
||||||
|
|
||||||
|
previously_failed_job = job_failed_before(old_failed_jobs, job)
|
||||||
|
if previously_failed_job:
|
||||||
|
print(
|
||||||
|
f"[yellow]"
|
||||||
|
f" :: [link={job.web_url}]{job.name}[/link][/yellow]"
|
||||||
|
f"[magenta]"
|
||||||
|
f" [link={previously_failed_job.web_url}](previous run)[/link]"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
print(
|
||||||
|
f"[red]:: [link={job.web_url}]{job.name}[/link]"
|
||||||
|
)
|
||||||
|
print_failures_csv(job.id)
|
||||||
|
|
||||||
|
if not job_failed:
|
||||||
|
exit(0)
|
||||||
|
|
||||||
|
print("Commits between nightly pipelines:")
|
||||||
|
commit = project.commits.get(pipelines[0].sha)
|
||||||
|
while True:
|
||||||
|
print(
|
||||||
|
f"{commit.id} [link={commit.web_url}]{commit.title}[/link]"
|
||||||
|
)
|
||||||
|
if commit.id == pipelines[1].sha:
|
||||||
|
break
|
||||||
|
commit = project.commits.get(commit.parent_ids[0])
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
|
||||||
|
readonly this_dir
|
||||||
|
|
||||||
|
exec \
|
||||||
|
"$this_dir/../python-venv.sh" \
|
||||||
|
"$this_dir/requirements.txt" \
|
||||||
|
"$this_dir/nightly_compare.py" "$@"
|
||||||
|
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
query jobs($projectPath: ID!, $iid: ID!, $cursor: String) {
|
||||||
|
project(fullPath: $projectPath) {
|
||||||
|
id
|
||||||
|
pipeline(iid: $iid) {
|
||||||
|
id
|
||||||
|
iid
|
||||||
|
complete
|
||||||
|
stages {
|
||||||
|
nodes {
|
||||||
|
name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
jobs(after: $cursor) {
|
||||||
|
pageInfo {
|
||||||
|
hasNextPage
|
||||||
|
endCursor
|
||||||
|
}
|
||||||
|
count
|
||||||
|
nodes {
|
||||||
|
name
|
||||||
|
needs {
|
||||||
|
edges {
|
||||||
|
node {
|
||||||
|
name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
stage {
|
||||||
|
name
|
||||||
|
}
|
||||||
|
tags
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Symlink
+1
@@ -0,0 +1 @@
|
|||||||
|
../../.marge/hooks/pipeline_message.py
|
||||||
Executable
+10
@@ -0,0 +1,10 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
|
||||||
|
readonly this_dir
|
||||||
|
|
||||||
|
exec \
|
||||||
|
"$this_dir/../python-venv.sh" \
|
||||||
|
"$this_dir/requirements.txt" \
|
||||||
|
"$this_dir/pipeline_message.py" "$@"
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
query sha2pipelineIID($projectPath: ID!, $sha: String!) {
|
||||||
|
project(fullPath: $projectPath) {
|
||||||
|
pipelines(last: 1, sha:$sha){
|
||||||
|
nodes {
|
||||||
|
iid
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
PyYAML==6.*
|
||||||
|
filecache==0.81
|
||||||
|
flake8==7.*
|
||||||
|
gql==3.*
|
||||||
|
kaleido==0.2.*
|
||||||
|
pandas==2.*
|
||||||
|
plotly==5.*
|
||||||
|
python-dateutil==2.*
|
||||||
|
python-gitlab==5.*
|
||||||
|
rich==14.1.*
|
||||||
|
ruamel.yaml.clib==0.2.*
|
||||||
|
ruamel.yaml==0.17.*
|
||||||
|
tabulate==0.9.*
|
||||||
|
tomledit==1.*
|
||||||
@@ -0,0 +1,294 @@
|
|||||||
|
"""
|
||||||
|
A structured logging utility supporting multiple data formats such as CSV, JSON,
|
||||||
|
and YAML.
|
||||||
|
|
||||||
|
The main purpose of this script, besides having relevant information available
|
||||||
|
in a condensed and deserialized.
|
||||||
|
|
||||||
|
This script defines a protocol for different file handling strategies and provides
|
||||||
|
implementations for CSV, JSON, and YAML formats. The main class, StructuredLogger,
|
||||||
|
allows for easy interaction with log data, enabling users to load, save, increment,
|
||||||
|
set, and append fields in the log. The script also includes context managers for
|
||||||
|
file locking and editing log data to ensure data integrity and avoid race conditions.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from collections.abc import MutableMapping, MutableSequence
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Protocol
|
||||||
|
|
||||||
|
import fire
|
||||||
|
from filelock import FileLock
|
||||||
|
|
||||||
|
try:
|
||||||
|
import polars as pl
|
||||||
|
|
||||||
|
CSV_LIB_EXCEPTION = None
|
||||||
|
except ImportError as e:
|
||||||
|
CSV_LIB_EXCEPTION: ImportError = e
|
||||||
|
|
||||||
|
try:
|
||||||
|
from ruamel.yaml import YAML
|
||||||
|
|
||||||
|
YAML_LIB_EXCEPTION = None
|
||||||
|
except ImportError as e:
|
||||||
|
YAML_LIB_EXCEPTION: ImportError = e
|
||||||
|
|
||||||
|
|
||||||
|
class ContainerProxy:
|
||||||
|
"""
|
||||||
|
A proxy class that wraps a mutable container object (such as a dictionary or
|
||||||
|
a list) and calls a provided save_callback function whenever the container
|
||||||
|
or its contents
|
||||||
|
are changed.
|
||||||
|
"""
|
||||||
|
def __init__(self, container, save_callback):
|
||||||
|
self.container = container
|
||||||
|
self.save_callback = save_callback
|
||||||
|
|
||||||
|
def __getitem__(self, key):
|
||||||
|
value = self.container[key]
|
||||||
|
if isinstance(value, (MutableMapping, MutableSequence)):
|
||||||
|
return ContainerProxy(value, self.save_callback)
|
||||||
|
return value
|
||||||
|
|
||||||
|
def __setitem__(self, key, value):
|
||||||
|
self.container[key] = value
|
||||||
|
self.save_callback()
|
||||||
|
|
||||||
|
def __delitem__(self, key):
|
||||||
|
del self.container[key]
|
||||||
|
self.save_callback()
|
||||||
|
|
||||||
|
def __getattr__(self, name):
|
||||||
|
attr = getattr(self.container, name)
|
||||||
|
|
||||||
|
if callable(attr):
|
||||||
|
def wrapper(*args, **kwargs):
|
||||||
|
result = attr(*args, **kwargs)
|
||||||
|
self.save_callback()
|
||||||
|
return result
|
||||||
|
|
||||||
|
return wrapper
|
||||||
|
return attr
|
||||||
|
|
||||||
|
def __iter__(self):
|
||||||
|
return iter(self.container)
|
||||||
|
|
||||||
|
def __len__(self):
|
||||||
|
return len(self.container)
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return repr(self.container)
|
||||||
|
|
||||||
|
|
||||||
|
class AutoSaveDict(dict):
|
||||||
|
"""
|
||||||
|
A subclass of the built-in dict class with additional functionality to
|
||||||
|
automatically save changes to the dictionary. It maintains a timestamp of
|
||||||
|
the last modification and automatically wraps nested mutable containers
|
||||||
|
using ContainerProxy.
|
||||||
|
"""
|
||||||
|
timestamp_key = "_timestamp"
|
||||||
|
|
||||||
|
def __init__(self, *args, save_callback, register_timestamp=True, **kwargs):
|
||||||
|
self.save_callback = save_callback
|
||||||
|
self.__register_timestamp = register_timestamp
|
||||||
|
self.__heartbeat()
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
self.__wrap_dictionaries()
|
||||||
|
|
||||||
|
def __heartbeat(self):
|
||||||
|
if self.__register_timestamp:
|
||||||
|
self[AutoSaveDict.timestamp_key] = datetime.now().isoformat()
|
||||||
|
|
||||||
|
def __save(self):
|
||||||
|
self.__heartbeat()
|
||||||
|
self.save_callback()
|
||||||
|
|
||||||
|
def __wrap_dictionaries(self):
|
||||||
|
for key, value in self.items():
|
||||||
|
if isinstance(value, MutableMapping) and not isinstance(
|
||||||
|
value, AutoSaveDict
|
||||||
|
):
|
||||||
|
self[key] = AutoSaveDict(
|
||||||
|
value, save_callback=self.save_callback, register_timestamp=False
|
||||||
|
)
|
||||||
|
|
||||||
|
def __setitem__(self, key, value):
|
||||||
|
if isinstance(value, MutableMapping) and not isinstance(value, AutoSaveDict):
|
||||||
|
value = AutoSaveDict(
|
||||||
|
value, save_callback=self.save_callback, register_timestamp=False
|
||||||
|
)
|
||||||
|
super().__setitem__(key, value)
|
||||||
|
|
||||||
|
if self.__register_timestamp and key == AutoSaveDict.timestamp_key:
|
||||||
|
return
|
||||||
|
self.__save()
|
||||||
|
|
||||||
|
def __getitem__(self, key):
|
||||||
|
value = super().__getitem__(key)
|
||||||
|
if isinstance(value, (MutableMapping, MutableSequence)):
|
||||||
|
return ContainerProxy(value, self.__save)
|
||||||
|
return value
|
||||||
|
|
||||||
|
def __delitem__(self, key):
|
||||||
|
super().__delitem__(key)
|
||||||
|
self.__save()
|
||||||
|
|
||||||
|
def pop(self, *args, **kwargs):
|
||||||
|
result = super().pop(*args, **kwargs)
|
||||||
|
self.__save()
|
||||||
|
return result
|
||||||
|
|
||||||
|
def update(self, *args, **kwargs):
|
||||||
|
super().update(*args, **kwargs)
|
||||||
|
self.__wrap_dictionaries()
|
||||||
|
self.__save()
|
||||||
|
|
||||||
|
|
||||||
|
class StructuredLoggerStrategy(Protocol):
|
||||||
|
def load_data(self, file_path: Path) -> dict:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def save_data(self, file_path: Path, data: dict) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class CSVStrategy:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
if CSV_LIB_EXCEPTION:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Can't parse CSV files. Missing library"
|
||||||
|
) from CSV_LIB_EXCEPTION
|
||||||
|
|
||||||
|
def load_data(self, file_path: Path) -> dict:
|
||||||
|
dicts: list[dict[str, Any]] = pl.read_csv(
|
||||||
|
file_path, try_parse_dates=True
|
||||||
|
).to_dicts()
|
||||||
|
data = {}
|
||||||
|
for d in dicts:
|
||||||
|
for k, v in d.items():
|
||||||
|
if k != AutoSaveDict.timestamp_key and k in data:
|
||||||
|
if isinstance(data[k], list):
|
||||||
|
data[k].append(v)
|
||||||
|
continue
|
||||||
|
data[k] = [data[k], v]
|
||||||
|
else:
|
||||||
|
data[k] = v
|
||||||
|
return data
|
||||||
|
|
||||||
|
def save_data(self, file_path: Path, data: dict) -> None:
|
||||||
|
pl.DataFrame(data).write_csv(file_path)
|
||||||
|
|
||||||
|
|
||||||
|
class JSONStrategy:
|
||||||
|
def load_data(self, file_path: Path) -> dict:
|
||||||
|
return json.loads(file_path.read_text())
|
||||||
|
|
||||||
|
def save_data(self, file_path: Path, data: dict) -> None:
|
||||||
|
with open(file_path, "w") as f:
|
||||||
|
json.dump(data, f, indent=2)
|
||||||
|
|
||||||
|
|
||||||
|
class YAMLStrategy:
|
||||||
|
def __init__(self):
|
||||||
|
if YAML_LIB_EXCEPTION:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Can't parse YAML files. Missing library"
|
||||||
|
) from YAML_LIB_EXCEPTION
|
||||||
|
self.yaml = YAML()
|
||||||
|
self.yaml.indent(sequence=4, offset=2)
|
||||||
|
self.yaml.default_flow_style = False
|
||||||
|
self.yaml.representer.add_representer(AutoSaveDict, self.represent_dict)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def represent_dict(cls, dumper, data):
|
||||||
|
return dumper.represent_mapping("tag:yaml.org,2002:map", data)
|
||||||
|
|
||||||
|
def load_data(self, file_path: Path) -> dict:
|
||||||
|
return self.yaml.load(file_path.read_text())
|
||||||
|
|
||||||
|
def save_data(self, file_path: Path, data: dict) -> None:
|
||||||
|
with open(file_path, "w") as f:
|
||||||
|
self.yaml.dump(data, f)
|
||||||
|
|
||||||
|
|
||||||
|
class StructuredLogger:
|
||||||
|
def __init__(
|
||||||
|
self, file_name: str, strategy: StructuredLoggerStrategy = None, truncate=False
|
||||||
|
):
|
||||||
|
self.file_name: str = file_name
|
||||||
|
self.file_path = Path(self.file_name)
|
||||||
|
self._data: AutoSaveDict = AutoSaveDict(save_callback=self.save_data)
|
||||||
|
|
||||||
|
if strategy is None:
|
||||||
|
self.strategy: StructuredLoggerStrategy = self.guess_strategy_from_file(
|
||||||
|
self.file_path
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.strategy = strategy
|
||||||
|
|
||||||
|
if not self.file_path.exists():
|
||||||
|
Path.mkdir(self.file_path.parent, exist_ok=True)
|
||||||
|
self.save_data()
|
||||||
|
return
|
||||||
|
|
||||||
|
if truncate:
|
||||||
|
with self.get_lock():
|
||||||
|
os.truncate(self.file_path, 0)
|
||||||
|
self.save_data()
|
||||||
|
|
||||||
|
def load_data(self):
|
||||||
|
self._data = self.strategy.load_data(self.file_path)
|
||||||
|
|
||||||
|
def save_data(self):
|
||||||
|
self.strategy.save_data(self.file_path, self._data)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def data(self) -> AutoSaveDict:
|
||||||
|
return self._data
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def get_lock(self):
|
||||||
|
with FileLock(f"{self.file_path}.lock", timeout=10):
|
||||||
|
yield
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def edit_context(self):
|
||||||
|
"""
|
||||||
|
Context manager that ensures proper loading and saving of log data when
|
||||||
|
performing multiple modifications.
|
||||||
|
"""
|
||||||
|
with self.get_lock():
|
||||||
|
try:
|
||||||
|
self.load_data()
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
self.save_data()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def guess_strategy_from_file(file_path: Path) -> StructuredLoggerStrategy:
|
||||||
|
file_extension = file_path.suffix.lower().lstrip(".")
|
||||||
|
return StructuredLogger.get_strategy(file_extension)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_strategy(strategy_name: str) -> StructuredLoggerStrategy:
|
||||||
|
strategies = {
|
||||||
|
"csv": CSVStrategy,
|
||||||
|
"json": JSONStrategy,
|
||||||
|
"yaml": YAMLStrategy,
|
||||||
|
"yml": YAMLStrategy,
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
return strategies[strategy_name]()
|
||||||
|
except KeyError as e:
|
||||||
|
raise ValueError(f"Unknown strategy for: {strategy_name}") from e
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
fire.Fire(StructuredLogger)
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
-r ../requirements.txt
|
||||||
|
aiohttp==3.13.*
|
||||||
|
filelock==3.12.4
|
||||||
|
fire==0.7.0
|
||||||
|
freezegun==1.5.1
|
||||||
|
mock==5.1.0
|
||||||
|
polars==0.19.3
|
||||||
|
pytest==7.4.2
|
||||||
|
pytest-asyncio==0.21.0
|
||||||
|
pytest-cov==3.0.0
|
||||||
@@ -0,0 +1,746 @@
|
|||||||
|
import logging
|
||||||
|
import subprocess
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from custom_logger import CustomLogger
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def tmp_log_file(tmp_path):
|
||||||
|
return tmp_path / "test_log.json"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def custom_logger(tmp_log_file):
|
||||||
|
return CustomLogger(tmp_log_file)
|
||||||
|
|
||||||
|
|
||||||
|
def run_script_with_args(args):
|
||||||
|
import custom_logger
|
||||||
|
|
||||||
|
script_path = custom_logger.__file__
|
||||||
|
return subprocess.run(
|
||||||
|
["python3", str(script_path), *args], capture_output=True, text=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for missing log file
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"key, value", [("dut_attempt_counter", "1"), ("job_combined_status", "pass")]
|
||||||
|
)
|
||||||
|
def test_missing_log_file_argument(key, value):
|
||||||
|
result = run_script_with_args(["--update", "key", "value"])
|
||||||
|
assert result.returncode != 0
|
||||||
|
|
||||||
|
|
||||||
|
# Parametrize test case for valid update arguments
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"key, value", [("dut_attempt_counter", "1"), ("job_combined_status", "pass")]
|
||||||
|
)
|
||||||
|
def test_update_argument_valid(custom_logger, tmp_log_file, key, value):
|
||||||
|
result = run_script_with_args([str(tmp_log_file), "--update", key, value])
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for passing only the key without a value
|
||||||
|
def test_update_argument_key_only(custom_logger, tmp_log_file):
|
||||||
|
key = "dut_attempt_counter"
|
||||||
|
result = run_script_with_args([str(tmp_log_file), "--update", key])
|
||||||
|
assert result.returncode != 0
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for not passing any key-value pair
|
||||||
|
def test_update_argument_no_values(custom_logger, tmp_log_file):
|
||||||
|
result = run_script_with_args([str(tmp_log_file), "--update"])
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
|
||||||
|
# Parametrize test case for valid arguments
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"key, value", [("dut_attempt_counter", "1"), ("job_combined_status", "pass")]
|
||||||
|
)
|
||||||
|
def test_create_argument_valid(custom_logger, tmp_log_file, key, value):
|
||||||
|
result = run_script_with_args([str(tmp_log_file), "--create-dut-job", key, value])
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for passing only the key without a value
|
||||||
|
def test_create_argument_key_only(custom_logger, tmp_log_file):
|
||||||
|
key = "dut_attempt_counter"
|
||||||
|
result = run_script_with_args([str(tmp_log_file), "--create-dut-job", key])
|
||||||
|
assert result.returncode != 0
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for not passing any key-value pair
|
||||||
|
def test_create_argument_no_values(custom_logger, tmp_log_file):
|
||||||
|
result = run_script_with_args([str(tmp_log_file), "--create-dut-job"])
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for updating a DUT job
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"key, value", [("status", "hung"), ("dut_state", "Canceling"), ("dut_name", "asus")]
|
||||||
|
)
|
||||||
|
def test_update_dut_job(custom_logger, tmp_log_file, key, value):
|
||||||
|
result = run_script_with_args([str(tmp_log_file), "--update-dut-job", key, value])
|
||||||
|
assert result.returncode != 0
|
||||||
|
|
||||||
|
result = run_script_with_args([str(tmp_log_file), "--create-dut-job", key, value])
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
result = run_script_with_args([str(tmp_log_file), "--update-dut-job", key, value])
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for updating last DUT job
|
||||||
|
def test_update_dut_multiple_job(custom_logger, tmp_log_file):
|
||||||
|
# Create the first DUT job with the first key
|
||||||
|
result = run_script_with_args(
|
||||||
|
[str(tmp_log_file), "--create-dut-job", "status", "hung"]
|
||||||
|
)
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
# Create the second DUT job with the second key
|
||||||
|
result = run_script_with_args(
|
||||||
|
[str(tmp_log_file), "--create-dut-job", "dut_state", "Canceling"]
|
||||||
|
)
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
result = run_script_with_args(
|
||||||
|
[str(tmp_log_file), "--update-dut-job", "dut_name", "asus"]
|
||||||
|
)
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
|
||||||
|
# Parametrize test case for valid phase arguments
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"phase_name",
|
||||||
|
[("Phase1"), ("Phase2"), ("Phase3")],
|
||||||
|
)
|
||||||
|
def test_create_job_phase_valid(custom_logger, tmp_log_file, phase_name):
|
||||||
|
custom_logger.create_dut_job(status="pass")
|
||||||
|
|
||||||
|
result = run_script_with_args([str(tmp_log_file), "--create-job-phase", phase_name])
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for not passing any arguments for create-job-phase
|
||||||
|
def test_create_job_phase_no_arguments(custom_logger, tmp_log_file):
|
||||||
|
custom_logger.create_dut_job(status="pass")
|
||||||
|
|
||||||
|
result = run_script_with_args([str(tmp_log_file), "--create-job-phase"])
|
||||||
|
assert result.returncode != 0
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for trying to create a phase job without an existing DUT job
|
||||||
|
def test_create_job_phase_no_dut_job(custom_logger, tmp_log_file):
|
||||||
|
phase_name = "Phase1"
|
||||||
|
|
||||||
|
result = run_script_with_args([str(tmp_log_file), "--create-job-phase", phase_name])
|
||||||
|
assert result.returncode != 0
|
||||||
|
|
||||||
|
|
||||||
|
# Combined test cases for valid scenarios
|
||||||
|
def test_valid_scenarios(custom_logger, tmp_log_file):
|
||||||
|
valid_update_args = [("dut_attempt_counter", "1"), ("job_combined_status", "pass")]
|
||||||
|
for key, value in valid_update_args:
|
||||||
|
result = run_script_with_args([str(tmp_log_file), "--update", key, value])
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
valid_create_args = [
|
||||||
|
("status", "hung"),
|
||||||
|
("dut_state", "Canceling"),
|
||||||
|
("dut_name", "asus"),
|
||||||
|
("phase_name", "Bootloader"),
|
||||||
|
]
|
||||||
|
for key, value in valid_create_args:
|
||||||
|
result = run_script_with_args(
|
||||||
|
[str(tmp_log_file), "--create-dut-job", key, value]
|
||||||
|
)
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
result = run_script_with_args(
|
||||||
|
[str(tmp_log_file), "--create-dut-job", "status", "hung"]
|
||||||
|
)
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
result = run_script_with_args(
|
||||||
|
[str(tmp_log_file), "--update-dut-job", "dut_name", "asus"]
|
||||||
|
)
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
result = run_script_with_args(
|
||||||
|
[
|
||||||
|
str(tmp_log_file),
|
||||||
|
"--create-job-phase",
|
||||||
|
"phase_name",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
|
||||||
|
# Parametrize test case for valid update arguments
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"key, value", [("dut_attempt_counter", "1"), ("job_combined_status", "pass")]
|
||||||
|
)
|
||||||
|
def test_update(custom_logger, key, value):
|
||||||
|
custom_logger.update(**{key: value})
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
|
||||||
|
assert key in logger_data
|
||||||
|
assert logger_data[key] == value
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for updating with a key that already exists
|
||||||
|
def test_update_existing_key(custom_logger):
|
||||||
|
key = "status"
|
||||||
|
value = "new_value"
|
||||||
|
custom_logger.logger.data[key] = "old_value"
|
||||||
|
custom_logger.update(**{key: value})
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
|
||||||
|
assert key in logger_data
|
||||||
|
assert logger_data[key] == value
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for updating "dut_jobs"
|
||||||
|
def test_update_dut_jobs(custom_logger):
|
||||||
|
key1 = "status"
|
||||||
|
value1 = "fail"
|
||||||
|
key2 = "state"
|
||||||
|
value2 = "hung"
|
||||||
|
|
||||||
|
custom_logger.create_dut_job(**{key1: value1})
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
|
||||||
|
job1 = logger_data["dut_jobs"][0]
|
||||||
|
assert key1 in job1
|
||||||
|
assert job1[key1] == value1
|
||||||
|
|
||||||
|
custom_logger.update_dut_job(key2, value2)
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
|
||||||
|
job2 = logger_data["dut_jobs"][0]
|
||||||
|
assert key2 in job2
|
||||||
|
assert job2[key2] == value2
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for creating and updating DUT job
|
||||||
|
def test_create_dut_job(custom_logger):
|
||||||
|
key = "status"
|
||||||
|
value1 = "pass"
|
||||||
|
value2 = "fail"
|
||||||
|
value3 = "hung"
|
||||||
|
|
||||||
|
reason = "job_combined_status"
|
||||||
|
result = "Finished"
|
||||||
|
|
||||||
|
custom_logger.update(**{reason: result})
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
|
||||||
|
assert reason in logger_data
|
||||||
|
assert logger_data[reason] == result
|
||||||
|
|
||||||
|
# Create the first DUT job
|
||||||
|
custom_logger.create_dut_job(**{key: value1})
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
|
||||||
|
assert "dut_jobs" in logger_data
|
||||||
|
assert isinstance(logger_data["dut_jobs"], list)
|
||||||
|
assert len(logger_data["dut_jobs"]) == 1
|
||||||
|
assert isinstance(logger_data["dut_jobs"][0], dict)
|
||||||
|
|
||||||
|
# Check the values of the keys in the created first DUT job
|
||||||
|
job1 = logger_data["dut_jobs"][0]
|
||||||
|
assert key in job1
|
||||||
|
assert job1[key] == value1
|
||||||
|
|
||||||
|
# Create the second DUT job
|
||||||
|
custom_logger.create_dut_job(**{key: value2})
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
|
||||||
|
assert "dut_jobs" in logger_data
|
||||||
|
assert isinstance(logger_data["dut_jobs"], list)
|
||||||
|
assert len(logger_data["dut_jobs"]) == 2
|
||||||
|
assert isinstance(logger_data["dut_jobs"][1], dict)
|
||||||
|
|
||||||
|
# Check the values of the keys in the created second DUT job
|
||||||
|
job2 = logger_data["dut_jobs"][1]
|
||||||
|
assert key in job2
|
||||||
|
assert job2[key] == value2
|
||||||
|
|
||||||
|
# Update the second DUT job with value3
|
||||||
|
custom_logger.update_dut_job(key, value3)
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
|
||||||
|
# Check the updated value in the second DUT job
|
||||||
|
job2 = logger_data["dut_jobs"][1]
|
||||||
|
assert key in job2
|
||||||
|
assert job2[key] == value3
|
||||||
|
|
||||||
|
# Find the index of the last DUT job
|
||||||
|
last_job_index = len(logger_data["dut_jobs"]) - 1
|
||||||
|
|
||||||
|
# Update the last DUT job
|
||||||
|
custom_logger.update_dut_job("dut_name", "asus")
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
|
||||||
|
# Check the updated value in the last DUT job
|
||||||
|
job2 = logger_data["dut_jobs"][last_job_index]
|
||||||
|
assert "dut_name" in job2
|
||||||
|
assert job2["dut_name"] == "asus"
|
||||||
|
|
||||||
|
# Check that "dut_name" is not present in other DUT jobs
|
||||||
|
for idx, job in enumerate(logger_data["dut_jobs"]):
|
||||||
|
if idx != last_job_index:
|
||||||
|
assert job.get("dut_name") == ""
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for updating with missing "dut_jobs" key
|
||||||
|
def test_update_dut_job_missing_dut_jobs(custom_logger):
|
||||||
|
key = "status"
|
||||||
|
value = "fail"
|
||||||
|
|
||||||
|
# Attempt to update a DUT job when "dut_jobs" is missing
|
||||||
|
with pytest.raises(ValueError, match="No DUT jobs found."):
|
||||||
|
custom_logger.update_dut_job(key, value)
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for creating a job phase
|
||||||
|
def test_create_job_phase(custom_logger):
|
||||||
|
custom_logger.create_dut_job(status="pass")
|
||||||
|
phase_name = "Phase1"
|
||||||
|
|
||||||
|
custom_logger.create_job_phase(phase_name)
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
|
||||||
|
assert "dut_jobs" in logger_data
|
||||||
|
assert isinstance(logger_data["dut_jobs"], list)
|
||||||
|
assert len(logger_data["dut_jobs"]) == 1
|
||||||
|
|
||||||
|
job = logger_data["dut_jobs"][0]
|
||||||
|
assert "dut_job_phases" in job
|
||||||
|
assert isinstance(job["dut_job_phases"], list)
|
||||||
|
assert len(job["dut_job_phases"]) == 1
|
||||||
|
|
||||||
|
phase = job["dut_job_phases"][0]
|
||||||
|
assert phase["name"] == phase_name
|
||||||
|
try:
|
||||||
|
datetime.fromisoformat(phase["start_time"])
|
||||||
|
assert True
|
||||||
|
except ValueError:
|
||||||
|
assert False
|
||||||
|
assert phase["end_time"] == ""
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for creating multiple phase jobs
|
||||||
|
def test_create_multiple_phase_jobs(custom_logger):
|
||||||
|
custom_logger.create_dut_job(status="pass")
|
||||||
|
|
||||||
|
phase_data = [
|
||||||
|
{
|
||||||
|
"phase_name": "Phase1",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase_name": "Phase2",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"phase_name": "Phase3",
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
for data in phase_data:
|
||||||
|
phase_name = data["phase_name"]
|
||||||
|
|
||||||
|
custom_logger.create_job_phase(phase_name)
|
||||||
|
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
|
||||||
|
assert "dut_jobs" in logger_data
|
||||||
|
assert isinstance(logger_data["dut_jobs"], list)
|
||||||
|
assert len(logger_data["dut_jobs"]) == 1
|
||||||
|
|
||||||
|
job = logger_data["dut_jobs"][0]
|
||||||
|
assert "dut_job_phases" in job
|
||||||
|
assert isinstance(job["dut_job_phases"], list)
|
||||||
|
assert len(job["dut_job_phases"]) == len(phase_data)
|
||||||
|
|
||||||
|
for data in phase_data:
|
||||||
|
phase_name = data["phase_name"]
|
||||||
|
|
||||||
|
phase = job["dut_job_phases"][phase_data.index(data)]
|
||||||
|
|
||||||
|
assert phase["name"] == phase_name
|
||||||
|
try:
|
||||||
|
datetime.fromisoformat(phase["start_time"])
|
||||||
|
assert True
|
||||||
|
except ValueError:
|
||||||
|
assert False
|
||||||
|
|
||||||
|
if phase_data.index(data) != len(phase_data) - 1:
|
||||||
|
try:
|
||||||
|
datetime.fromisoformat(phase["end_time"])
|
||||||
|
assert True
|
||||||
|
except ValueError:
|
||||||
|
assert False
|
||||||
|
|
||||||
|
# Check if the end_time of the last phase is an empty string
|
||||||
|
last_phase = job["dut_job_phases"][-1]
|
||||||
|
assert last_phase["end_time"] == ""
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for creating multiple dut jobs and updating phase job for last dut job
|
||||||
|
def test_create_two_dut_jobs_and_add_phase(custom_logger):
|
||||||
|
# Create the first DUT job
|
||||||
|
custom_logger.create_dut_job(status="pass")
|
||||||
|
|
||||||
|
# Create the second DUT job
|
||||||
|
custom_logger.create_dut_job(status="fail")
|
||||||
|
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
|
||||||
|
assert "dut_jobs" in logger_data
|
||||||
|
assert isinstance(logger_data["dut_jobs"], list)
|
||||||
|
assert len(logger_data["dut_jobs"]) == 2
|
||||||
|
|
||||||
|
first_dut_job = logger_data["dut_jobs"][0]
|
||||||
|
second_dut_job = logger_data["dut_jobs"][1]
|
||||||
|
|
||||||
|
# Add a phase to the second DUT job
|
||||||
|
custom_logger.create_job_phase("Phase1")
|
||||||
|
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
|
||||||
|
assert "dut_jobs" in logger_data
|
||||||
|
assert isinstance(logger_data["dut_jobs"], list)
|
||||||
|
assert len(logger_data["dut_jobs"]) == 2
|
||||||
|
|
||||||
|
first_dut_job = logger_data["dut_jobs"][0]
|
||||||
|
second_dut_job = logger_data["dut_jobs"][1]
|
||||||
|
|
||||||
|
# Check first DUT job does not have a phase
|
||||||
|
assert not first_dut_job.get("dut_job_phases")
|
||||||
|
|
||||||
|
# Check second DUT job has a phase
|
||||||
|
assert second_dut_job.get("dut_job_phases")
|
||||||
|
assert isinstance(second_dut_job["dut_job_phases"], list)
|
||||||
|
assert len(second_dut_job["dut_job_phases"]) == 1
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for updating DUT start time
|
||||||
|
def test_update_dut_start_time(custom_logger):
|
||||||
|
custom_logger.create_dut_job(status="pass")
|
||||||
|
custom_logger.update_dut_time("start", None)
|
||||||
|
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
assert "dut_jobs" in logger_data
|
||||||
|
assert len(logger_data["dut_jobs"]) == 1
|
||||||
|
|
||||||
|
dut_job = logger_data["dut_jobs"][0]
|
||||||
|
assert "dut_start_time" in dut_job
|
||||||
|
assert dut_job["dut_start_time"] != ""
|
||||||
|
|
||||||
|
try:
|
||||||
|
datetime.fromisoformat(dut_job["dut_start_time"])
|
||||||
|
assert True
|
||||||
|
except ValueError:
|
||||||
|
assert False
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for updating DUT submit time
|
||||||
|
def test_update_dut_submit_time(custom_logger):
|
||||||
|
custom_time = "2023-11-09T02:37:06Z"
|
||||||
|
custom_logger.create_dut_job(status="pass")
|
||||||
|
custom_logger.update_dut_time("submit", custom_time)
|
||||||
|
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
assert "dut_jobs" in logger_data
|
||||||
|
assert len(logger_data["dut_jobs"]) == 1
|
||||||
|
|
||||||
|
dut_job = logger_data["dut_jobs"][0]
|
||||||
|
assert "dut_submit_time" in dut_job
|
||||||
|
|
||||||
|
try:
|
||||||
|
datetime.fromisoformat(dut_job["dut_submit_time"])
|
||||||
|
assert True
|
||||||
|
except ValueError:
|
||||||
|
assert False
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for updating DUT end time
|
||||||
|
def test_update_dut_end_time(custom_logger):
|
||||||
|
custom_logger.create_dut_job(status="pass")
|
||||||
|
custom_logger.update_dut_time("end", None)
|
||||||
|
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
assert "dut_jobs" in logger_data
|
||||||
|
assert len(logger_data["dut_jobs"]) == 1
|
||||||
|
|
||||||
|
dut_job = logger_data["dut_jobs"][0]
|
||||||
|
assert "dut_end_time" in dut_job
|
||||||
|
|
||||||
|
try:
|
||||||
|
datetime.fromisoformat(dut_job["dut_end_time"])
|
||||||
|
assert True
|
||||||
|
except ValueError:
|
||||||
|
assert False
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for updating DUT time with invalid value
|
||||||
|
def test_update_dut_time_invalid_value(custom_logger):
|
||||||
|
custom_logger.create_dut_job(status="pass")
|
||||||
|
with pytest.raises(
|
||||||
|
ValueError,
|
||||||
|
match="Error: Invalid argument provided for --update-dut-time. Use 'start', 'submit', 'end'.",
|
||||||
|
):
|
||||||
|
custom_logger.update_dut_time("invalid_value", None)
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for close_dut_job
|
||||||
|
def test_close_dut_job(custom_logger):
|
||||||
|
custom_logger.create_dut_job(status="pass")
|
||||||
|
|
||||||
|
custom_logger.create_job_phase("Phase1")
|
||||||
|
custom_logger.create_job_phase("Phase2")
|
||||||
|
|
||||||
|
custom_logger.close_dut_job()
|
||||||
|
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
assert "dut_jobs" in logger_data
|
||||||
|
assert len(logger_data["dut_jobs"]) == 1
|
||||||
|
|
||||||
|
dut_job = logger_data["dut_jobs"][0]
|
||||||
|
assert "dut_job_phases" in dut_job
|
||||||
|
dut_job_phases = dut_job["dut_job_phases"]
|
||||||
|
|
||||||
|
phase1 = dut_job_phases[0]
|
||||||
|
assert phase1["name"] == "Phase1"
|
||||||
|
|
||||||
|
try:
|
||||||
|
datetime.fromisoformat(phase1["start_time"])
|
||||||
|
assert True
|
||||||
|
except ValueError:
|
||||||
|
assert False
|
||||||
|
|
||||||
|
try:
|
||||||
|
datetime.fromisoformat(phase1["end_time"])
|
||||||
|
assert True
|
||||||
|
except ValueError:
|
||||||
|
assert False
|
||||||
|
|
||||||
|
phase2 = dut_job_phases[1]
|
||||||
|
assert phase2["name"] == "Phase2"
|
||||||
|
|
||||||
|
try:
|
||||||
|
datetime.fromisoformat(phase2["start_time"])
|
||||||
|
assert True
|
||||||
|
except ValueError:
|
||||||
|
assert False
|
||||||
|
|
||||||
|
try:
|
||||||
|
datetime.fromisoformat(phase2["end_time"])
|
||||||
|
assert True
|
||||||
|
except ValueError:
|
||||||
|
assert False
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for close
|
||||||
|
def test_close(custom_logger):
|
||||||
|
custom_logger.create_dut_job(status="pass")
|
||||||
|
|
||||||
|
custom_logger.close()
|
||||||
|
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
assert "dut_jobs" in logger_data
|
||||||
|
assert len(logger_data["dut_jobs"]) == 1
|
||||||
|
assert "dut_attempt_counter" in logger_data
|
||||||
|
assert logger_data["dut_attempt_counter"] == len(logger_data["dut_jobs"])
|
||||||
|
assert "job_combined_status" in logger_data
|
||||||
|
assert logger_data["job_combined_status"] != ""
|
||||||
|
|
||||||
|
dut_job = logger_data["dut_jobs"][0]
|
||||||
|
assert "submitter_end_time" in dut_job
|
||||||
|
try:
|
||||||
|
datetime.fromisoformat(dut_job["submitter_end_time"])
|
||||||
|
assert True
|
||||||
|
except ValueError:
|
||||||
|
assert False
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for updating status to fail with a reason
|
||||||
|
def test_update_status_fail_with_reason(custom_logger):
|
||||||
|
custom_logger.create_dut_job()
|
||||||
|
|
||||||
|
reason = "kernel panic"
|
||||||
|
custom_logger.update_status_fail(reason)
|
||||||
|
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
assert "dut_jobs" in logger_data
|
||||||
|
assert len(logger_data["dut_jobs"]) == 1
|
||||||
|
|
||||||
|
dut_job = logger_data["dut_jobs"][0]
|
||||||
|
assert "status" in dut_job
|
||||||
|
assert dut_job["status"] == "fail"
|
||||||
|
assert "dut_job_fail_reason" in dut_job
|
||||||
|
assert dut_job["dut_job_fail_reason"] == reason
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for updating status to fail without providing a reason
|
||||||
|
def test_update_status_fail_without_reason(custom_logger):
|
||||||
|
custom_logger.create_dut_job()
|
||||||
|
|
||||||
|
custom_logger.update_status_fail()
|
||||||
|
|
||||||
|
# Check if the status is updated and fail reason is empty
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
assert "dut_jobs" in logger_data
|
||||||
|
assert len(logger_data["dut_jobs"]) == 1
|
||||||
|
|
||||||
|
dut_job = logger_data["dut_jobs"][0]
|
||||||
|
assert "status" in dut_job
|
||||||
|
assert dut_job["status"] == "fail"
|
||||||
|
assert "dut_job_fail_reason" in dut_job
|
||||||
|
assert dut_job["dut_job_fail_reason"] == ""
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for check_dut_timings with submission time earlier than start time
|
||||||
|
def test_check_dut_timings_submission_earlier_than_start(custom_logger, caplog):
|
||||||
|
custom_logger.create_dut_job()
|
||||||
|
|
||||||
|
# Set submission time to be earlier than start time
|
||||||
|
custom_logger.update_dut_time("start", "2023-01-01T11:00:00")
|
||||||
|
custom_logger.update_dut_time("submit", "2023-01-01T12:00:00")
|
||||||
|
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
assert "dut_jobs" in logger_data
|
||||||
|
assert len(logger_data["dut_jobs"]) == 1
|
||||||
|
|
||||||
|
job = logger_data["dut_jobs"][0]
|
||||||
|
|
||||||
|
# Call check_dut_timings
|
||||||
|
custom_logger.check_dut_timings(job)
|
||||||
|
|
||||||
|
# Check if an error message is logged
|
||||||
|
assert "Job submission is happening before job start." in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for check_dut_timings with end time earlier than start time
|
||||||
|
def test_check_dut_timings_end_earlier_than_start(custom_logger, caplog):
|
||||||
|
custom_logger.create_dut_job()
|
||||||
|
|
||||||
|
# Set end time to be earlier than start time
|
||||||
|
custom_logger.update_dut_time("end", "2023-01-01T11:00:00")
|
||||||
|
custom_logger.update_dut_time("start", "2023-01-01T12:00:00")
|
||||||
|
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
assert "dut_jobs" in logger_data
|
||||||
|
assert len(logger_data["dut_jobs"]) == 1
|
||||||
|
|
||||||
|
job = logger_data["dut_jobs"][0]
|
||||||
|
|
||||||
|
# Call check_dut_timings
|
||||||
|
custom_logger.check_dut_timings(job)
|
||||||
|
|
||||||
|
# Check if an error message is logged
|
||||||
|
assert "Job ended before it started." in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for check_dut_timings with valid timing sequence
|
||||||
|
def test_check_dut_timings_valid_timing_sequence(custom_logger, caplog):
|
||||||
|
custom_logger.create_dut_job()
|
||||||
|
|
||||||
|
# Set valid timing sequence
|
||||||
|
custom_logger.update_dut_time("submit", "2023-01-01T12:00:00")
|
||||||
|
custom_logger.update_dut_time("start", "2023-01-01T12:30:00")
|
||||||
|
custom_logger.update_dut_time("end", "2023-01-01T13:00:00")
|
||||||
|
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
assert "dut_jobs" in logger_data
|
||||||
|
assert len(logger_data["dut_jobs"]) == 1
|
||||||
|
|
||||||
|
job = logger_data["dut_jobs"][0]
|
||||||
|
|
||||||
|
# Call check_dut_timings
|
||||||
|
custom_logger.check_dut_timings(job)
|
||||||
|
|
||||||
|
# Check that no error messages are logged
|
||||||
|
assert "Job submission is happening before job start." not in caplog.text
|
||||||
|
assert "Job ended before it started." not in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("start_time,end_time,expected", [
|
||||||
|
("2023-01-01T12:00:00", "2023-01-01T12:30:00", "0:30:00"),
|
||||||
|
("invalid", "2023-01-01T12:30:00", ""),
|
||||||
|
("", "", ""),
|
||||||
|
])
|
||||||
|
def test_get_duration_time(custom_logger, start_time, end_time, expected):
|
||||||
|
duration = custom_logger.get_duration_time(start_time, end_time)
|
||||||
|
assert duration == expected
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for DUT job duration tracking
|
||||||
|
def test_dut_job_duration(custom_logger):
|
||||||
|
custom_logger.create_dut_job(status="pass")
|
||||||
|
|
||||||
|
# Set start time
|
||||||
|
custom_logger.update_dut_time("start", "2023-01-01T12:00:00")
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
dut_job = logger_data["dut_jobs"][0]
|
||||||
|
assert dut_job["dut_duration_time"] == ""
|
||||||
|
|
||||||
|
# Set end time
|
||||||
|
custom_logger.update_dut_time("end", "2023-01-01T12:30:00")
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
dut_job = logger_data["dut_jobs"][0]
|
||||||
|
assert dut_job["dut_duration_time"] == "0:30:00"
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for job phase duration tracking
|
||||||
|
def test_job_phase_duration(custom_logger):
|
||||||
|
custom_logger.create_dut_job(status="pass")
|
||||||
|
|
||||||
|
# Create first phase
|
||||||
|
custom_logger.create_job_phase("Phase1")
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
job = logger_data["dut_jobs"][0]
|
||||||
|
phase = job["dut_job_phases"][0]
|
||||||
|
assert phase["duration_time"] == ""
|
||||||
|
|
||||||
|
# Create second phase (which should close the first phase)
|
||||||
|
custom_logger.create_job_phase("Phase2")
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
job = logger_data["dut_jobs"][0]
|
||||||
|
|
||||||
|
# First phase should have duration
|
||||||
|
phase1 = job["dut_job_phases"][0]
|
||||||
|
assert phase1["duration_time"] != ""
|
||||||
|
|
||||||
|
# Verify duration format
|
||||||
|
start = datetime.fromisoformat(phase1["start_time"])
|
||||||
|
end = datetime.fromisoformat(phase1["end_time"])
|
||||||
|
duration = end - start
|
||||||
|
assert phase1["duration_time"] == str(duration)
|
||||||
|
|
||||||
|
# Second phase should have empty duration
|
||||||
|
phase2 = job["dut_job_phases"][1]
|
||||||
|
assert phase2["duration_time"] == ""
|
||||||
|
|
||||||
|
|
||||||
|
# Test case for closing job with phases
|
||||||
|
def test_close_job_with_phases_duration(custom_logger):
|
||||||
|
custom_logger.create_dut_job(status="pass")
|
||||||
|
custom_logger.create_job_phase("Phase1")
|
||||||
|
custom_logger.close_dut_job()
|
||||||
|
|
||||||
|
logger_data = custom_logger.logger.data
|
||||||
|
job = logger_data["dut_jobs"][0]
|
||||||
|
phase = job["dut_job_phases"][0]
|
||||||
|
|
||||||
|
assert phase["duration_time"] != ""
|
||||||
|
|
||||||
|
# Verify duration format
|
||||||
|
start = datetime.fromisoformat(phase["start_time"])
|
||||||
|
end = datetime.fromisoformat(phase["end_time"])
|
||||||
|
duration = end - start
|
||||||
|
assert phase["duration_time"] == str(duration)
|
||||||
@@ -0,0 +1,201 @@
|
|||||||
|
from contextlib import suppress
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from unittest import mock
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
import ci_post_gantt
|
||||||
|
import pytest
|
||||||
|
from ci_gantt_chart import generate_gantt_chart
|
||||||
|
from ci_post_gantt import Gitlab, MockGanttExit
|
||||||
|
|
||||||
|
|
||||||
|
def create_mock_job(
|
||||||
|
name, id, status, created_at, queued_duration, started_at, finished_at=None
|
||||||
|
):
|
||||||
|
mock_job = MagicMock()
|
||||||
|
mock_job.name = name
|
||||||
|
mock_job.status = status
|
||||||
|
mock_job.id = id
|
||||||
|
mock_job.created_at = created_at
|
||||||
|
mock_job.queued_duration = queued_duration
|
||||||
|
mock_job.started_at = started_at
|
||||||
|
mock_job.finished_at = finished_at
|
||||||
|
return mock_job
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def fake_pipeline():
|
||||||
|
current_time = datetime.fromisoformat("2024-12-17 23:54:13.940091+00:00")
|
||||||
|
created_at = current_time - timedelta(minutes=10)
|
||||||
|
|
||||||
|
job1 = create_mock_job(
|
||||||
|
name="job1",
|
||||||
|
id="1",
|
||||||
|
status="success",
|
||||||
|
created_at=created_at.isoformat(),
|
||||||
|
queued_duration=1, # seconds
|
||||||
|
started_at=(created_at + timedelta(seconds=2)).isoformat(),
|
||||||
|
finished_at=(created_at + timedelta(minutes=1)).isoformat(),
|
||||||
|
)
|
||||||
|
|
||||||
|
mock_pipeline = MagicMock()
|
||||||
|
mock_pipeline.web_url = "https://gitlab.freedesktop.org/mesa/mesa/-/pipelines/9999"
|
||||||
|
mock_pipeline.duration = 600 # Total pipeline duration in seconds
|
||||||
|
mock_pipeline.created_at = created_at.isoformat()
|
||||||
|
mock_pipeline.yaml_errors = False
|
||||||
|
mock_pipeline.jobs.list.return_value = [job1]
|
||||||
|
return mock_pipeline
|
||||||
|
|
||||||
|
|
||||||
|
def test_generate_gantt_chart(fake_pipeline):
|
||||||
|
fig = generate_gantt_chart(fake_pipeline)
|
||||||
|
|
||||||
|
fig_dict = fig.to_dict()
|
||||||
|
assert "data" in fig_dict
|
||||||
|
|
||||||
|
# Extract all job names from the "y" axis in the Gantt chart data
|
||||||
|
all_job_names = set()
|
||||||
|
for trace in fig_dict["data"]:
|
||||||
|
if "y" in trace:
|
||||||
|
all_job_names.update(trace["y"])
|
||||||
|
|
||||||
|
assert any(
|
||||||
|
"job1" in job for job in all_job_names
|
||||||
|
), "job1 should be present in the Gantt chart"
|
||||||
|
|
||||||
|
|
||||||
|
def test_ci_timeout(fake_pipeline):
|
||||||
|
fig = generate_gantt_chart(fake_pipeline, ci_timeout=1)
|
||||||
|
|
||||||
|
fig_dict = fig.to_dict()
|
||||||
|
|
||||||
|
timeout_line = None
|
||||||
|
for shape in fig_dict.get("layout", {}).get("shapes", []):
|
||||||
|
if shape.get("line", {}).get("dash") == "dash":
|
||||||
|
timeout_line = shape
|
||||||
|
break
|
||||||
|
|
||||||
|
assert timeout_line is not None, "Timeout line should exist in the Gantt chart"
|
||||||
|
timeout_x = timeout_line["x0"]
|
||||||
|
|
||||||
|
# Check that the timeout line is 1 minute after the pipeline creation time
|
||||||
|
pipeline_created_at = datetime.fromisoformat(fake_pipeline.created_at)
|
||||||
|
expected_timeout = pipeline_created_at + timedelta(minutes=1)
|
||||||
|
assert (
|
||||||
|
timeout_x == expected_timeout
|
||||||
|
), f"Timeout should be at {expected_timeout}, got {timeout_x}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_marge_bot_user_id():
|
||||||
|
with patch("ci_post_gantt.Gitlab") as MockGitlab:
|
||||||
|
mock_gitlab_instance = MagicMock(spec=Gitlab)
|
||||||
|
mock_gitlab_instance.users = MagicMock()
|
||||||
|
MockGitlab.return_value = mock_gitlab_instance
|
||||||
|
|
||||||
|
marge_bot_user_id = 12345
|
||||||
|
ci_post_gantt.main("fake_token", None, marge_bot_user_id)
|
||||||
|
mock_gitlab_instance.users.get.assert_called_once_with(marge_bot_user_id)
|
||||||
|
|
||||||
|
|
||||||
|
def test_project_ids():
|
||||||
|
current_time = datetime.now()
|
||||||
|
project_id_1 = 176
|
||||||
|
event_1 = MagicMock()
|
||||||
|
event_1.project_id = project_id_1
|
||||||
|
event_1.created_at = (current_time - timedelta(days=1)).isoformat()
|
||||||
|
event_1.note = {"body": f"Event for project {project_id_1}"}
|
||||||
|
|
||||||
|
project_id_2 = 166
|
||||||
|
event_2 = MagicMock()
|
||||||
|
event_2.project_id = project_id_2
|
||||||
|
event_2.created_at = (current_time - timedelta(days=2)).isoformat()
|
||||||
|
event_2.note = {"body": f"Event for project {project_id_2}"}
|
||||||
|
|
||||||
|
with patch("ci_post_gantt.Gitlab") as MockGitlab:
|
||||||
|
mock_user = MagicMock()
|
||||||
|
mock_user.events = MagicMock()
|
||||||
|
mock_user.events.list.return_value = [event_1, event_2]
|
||||||
|
|
||||||
|
mock_gitlab_instance = MagicMock(spec=Gitlab)
|
||||||
|
mock_gitlab_instance.users = MagicMock()
|
||||||
|
mock_gitlab_instance.users.get.return_value = mock_user
|
||||||
|
MockGitlab.return_value = mock_gitlab_instance
|
||||||
|
|
||||||
|
last_event_date = (current_time - timedelta(days=3)).isoformat()
|
||||||
|
|
||||||
|
# Test a single project id
|
||||||
|
ci_post_gantt.main("fake_token", last_event_date)
|
||||||
|
marge_bot_single_project_scope = [
|
||||||
|
event.note["body"]
|
||||||
|
for event in mock_user.events.list.return_value
|
||||||
|
if event.project_id == project_id_1
|
||||||
|
]
|
||||||
|
assert f"Event for project {project_id_1}" in marge_bot_single_project_scope
|
||||||
|
assert f"Event for project {project_id_2}" not in marge_bot_single_project_scope
|
||||||
|
|
||||||
|
# Test multiple project ids
|
||||||
|
ci_post_gantt.main(
|
||||||
|
"fake_token", last_event_date, 9716, [project_id_1, project_id_2]
|
||||||
|
)
|
||||||
|
|
||||||
|
marge_bot_multiple_project_scope = [
|
||||||
|
event.note["body"] for event in mock_user.events.list.return_value
|
||||||
|
]
|
||||||
|
assert f"Event for project {project_id_1}" in marge_bot_multiple_project_scope
|
||||||
|
assert f"Event for project {project_id_2}" in marge_bot_multiple_project_scope
|
||||||
|
|
||||||
|
|
||||||
|
def test_add_gantt_after_pipeline_message():
|
||||||
|
current_time = datetime.now()
|
||||||
|
|
||||||
|
plain_url = "https://gitlab.freedesktop.org/mesa/mesa/-/pipelines/12345"
|
||||||
|
plain_message = (
|
||||||
|
f"I couldn't merge this branch: CI failed! See pipeline {plain_url}."
|
||||||
|
)
|
||||||
|
event_plain = MagicMock()
|
||||||
|
event_plain.project_id = 176
|
||||||
|
event_plain.created_at = (current_time - timedelta(days=1)).isoformat()
|
||||||
|
event_plain.note = {"body": plain_message}
|
||||||
|
|
||||||
|
summary_url = "https://gitlab.freedesktop.org/mesa/mesa/-/pipelines/99999"
|
||||||
|
summary_message = (
|
||||||
|
"I couldn't merge this branch: "
|
||||||
|
f"CI failed! See pipeline {summary_url}.<br>There were problems with job:"
|
||||||
|
"[lavapipe](https://gitlab.freedesktop.org/mesa/mesa/-/jobs/68141218)<details><summary> "
|
||||||
|
"3 crashed tests</summary>dEQP-VK.ray_query.builtin.instancecustomindex.frag.aabbs,Crash<br>dEQP"
|
||||||
|
"-VK.ray_query.builtin.objecttoworld.frag.aabbs,Crash<br>dEQP-VK.sparse_resources.shader_intrinsics."
|
||||||
|
"2d_array_sparse_fetch.g16_b16r16_2plane_444_unorm.11_37_3_nontemporal,Crash<br></details>"
|
||||||
|
)
|
||||||
|
event_with_summary = MagicMock()
|
||||||
|
event_with_summary.project_id = 176
|
||||||
|
event_with_summary.created_at = (current_time - timedelta(days=1)).isoformat()
|
||||||
|
event_with_summary.note = {"body": summary_message}
|
||||||
|
|
||||||
|
with patch("ci_post_gantt.Gitlab") as MockGitlab, patch(
|
||||||
|
"ci_post_gantt.get_gitlab_pipeline_from_url", return_value=None
|
||||||
|
) as mock_get_gitlab_pipeline_from_url:
|
||||||
|
|
||||||
|
def safe_mock(*args, **kwargs):
|
||||||
|
with suppress(TypeError):
|
||||||
|
raise MockGanttExit("Exiting for test purposes")
|
||||||
|
|
||||||
|
mock_get_gitlab_pipeline_from_url.side_effect = safe_mock
|
||||||
|
|
||||||
|
mock_user = MagicMock()
|
||||||
|
mock_user.events = MagicMock()
|
||||||
|
mock_user.events.list.return_value = [event_plain, event_with_summary]
|
||||||
|
|
||||||
|
mock_gitlab_instance = MagicMock(spec=Gitlab)
|
||||||
|
mock_gitlab_instance.users = MagicMock()
|
||||||
|
mock_gitlab_instance.users.get.return_value = mock_user
|
||||||
|
MockGitlab.return_value = mock_gitlab_instance
|
||||||
|
|
||||||
|
last_event_date = (current_time - timedelta(days=3)).isoformat()
|
||||||
|
ci_post_gantt.main("fake_token", last_event_date, 12345)
|
||||||
|
mock_get_gitlab_pipeline_from_url.assert_has_calls(
|
||||||
|
[
|
||||||
|
mock.call(mock_gitlab_instance, plain_url),
|
||||||
|
mock.call(mock_gitlab_instance, summary_url),
|
||||||
|
],
|
||||||
|
any_order=True,
|
||||||
|
)
|
||||||
@@ -0,0 +1,176 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# Copyright © 2025 Collabora Ltd.
|
||||||
|
# Authors:
|
||||||
|
# Sergi Blanch Torne <[email protected]>
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import timedelta
|
||||||
|
|
||||||
|
from unittest import TestCase
|
||||||
|
from typing import Iterable
|
||||||
|
|
||||||
|
from marge_queue import MargeMergeRequest, MargeQueue, get_merge_queue
|
||||||
|
|
||||||
|
|
||||||
|
DATA = {
|
||||||
|
37564: {
|
||||||
|
# sample of a Merge Request rejected by marge and later reassigned to marge
|
||||||
|
"updated_at": "2025-09-25T10:59:27.144Z",
|
||||||
|
"title": "etnaviv: Fix util_blitter_save_so_targets(..) call",
|
||||||
|
"notes": [
|
||||||
|
# Those notes must be listed in reverse order
|
||||||
|
# because the list is requested sort="desc"
|
||||||
|
{
|
||||||
|
"body": "assigned to @marge-bot and unassigned @austriancoder",
|
||||||
|
"created_at": "2025-09-25T11:44:01.672Z",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"body": "added 1 commit",
|
||||||
|
"created_at": "2025-09-25T10:43:43.490Z",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"body": "This branch couldn't be merged:",
|
||||||
|
"created_at": "2025-09-25T10:42:27.817Z",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"body": "assigned to @austriancoder and unassigned @marge-bot",
|
||||||
|
"created_at": "2025-09-25T10:42:15.404Z",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"body": "added 3 commits",
|
||||||
|
"created_at": "2025-09-25T10:38:57.332Z",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"body": "assigned to @marge-bot",
|
||||||
|
"created_at": "2025-09-25T10:38:03.993Z",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
37560: {
|
||||||
|
# sample of a Merge Request assigned to marge between the two
|
||||||
|
# assignments of the previous sample. This is meant to show that
|
||||||
|
# this goes first in the queue, and it doesn't confuses the sequence.
|
||||||
|
"updated_at": "2025-09-25T11:38:08.420Z",
|
||||||
|
"title": "tu: limit query pool types logged into RMV",
|
||||||
|
"notes": [
|
||||||
|
{
|
||||||
|
"body": "assigned to @marge-bot",
|
||||||
|
"created_at": "2025-09-25T10:47:40.040Z",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"body": "added 1 commit",
|
||||||
|
"created_at": "2025-09-25T10:46:42.113Z",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"body": "mentioned in issue #13970",
|
||||||
|
"created_at": "2025-09-25T10:45:35.566Z",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"body": "Rb",
|
||||||
|
"created_at": "2025-09-25T08:25:51.527Z",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ProjectMergeRequestNote:
|
||||||
|
body: str
|
||||||
|
created_at: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ProjectMergeRequestNoteManager:
|
||||||
|
"""
|
||||||
|
Mock the gitlab.v4.objects.ProjectMergeRequestNoteManager
|
||||||
|
"""
|
||||||
|
iid: int
|
||||||
|
|
||||||
|
def list(self, *args, **kwargs) -> Iterable:
|
||||||
|
for note in DATA[self.iid]["notes"]:
|
||||||
|
yield ProjectMergeRequestNote(**note)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ProjectMergeRequest:
|
||||||
|
"""
|
||||||
|
Mock the gitlab.v4.objects.ProjectMergeRequest
|
||||||
|
"""
|
||||||
|
iid: int
|
||||||
|
|
||||||
|
@property
|
||||||
|
def updated_at(self) -> str:
|
||||||
|
return DATA[self.iid]["updated_at"]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def web_url(self) -> str:
|
||||||
|
return f"https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/{self.iid}"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def title(self) -> str:
|
||||||
|
return DATA[self.iid]["title"]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def notes(self) -> ProjectMergeRequestNoteManager:
|
||||||
|
return ProjectMergeRequestNoteManager(self.iid)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ProjectMergeRequestManager:
|
||||||
|
"""
|
||||||
|
Mock the gitlab.v4.objects.ProjectMergeRequestManager
|
||||||
|
"""
|
||||||
|
def list(self, *args, **kwargs) -> Iterable:
|
||||||
|
for mr in DATA.keys():
|
||||||
|
yield ProjectMergeRequest(mr)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Project:
|
||||||
|
"""
|
||||||
|
Mock the gitlab.v4.objects.Project
|
||||||
|
"""
|
||||||
|
@property
|
||||||
|
def mergerequests(self) -> ProjectMergeRequestManager:
|
||||||
|
return ProjectMergeRequestManager()
|
||||||
|
|
||||||
|
|
||||||
|
class MargeMergeTestCase(TestCase):
|
||||||
|
def test_queue(self) -> None:
|
||||||
|
queue = MargeQueue()
|
||||||
|
for mr_id in DATA.keys():
|
||||||
|
gl_mr_mock_obj = ProjectMergeRequest(mr_id)
|
||||||
|
marge_mr = MargeMergeRequest(gl_mr_mock_obj)
|
||||||
|
|
||||||
|
# for each element in the data sample, test the MargeMergeRequest construction
|
||||||
|
self.assertEqual(marge_mr.title, gl_mr_mock_obj.title)
|
||||||
|
self.assertEqual(marge_mr.web_url, gl_mr_mock_obj.web_url)
|
||||||
|
self.assertIsInstance(marge_mr.time_enqueued, timedelta)
|
||||||
|
|
||||||
|
queue.append(marge_mr)
|
||||||
|
|
||||||
|
self.assertEqual(queue.n_merge_requests_enqueued, 2)
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_merge_queue(self) -> None:
|
||||||
|
project = Project()
|
||||||
|
queue = get_merge_queue(project)
|
||||||
|
|
||||||
|
self.assertEqual(queue.n_merge_requests_enqueued, 2)
|
||||||
|
self.assertEqual(len(queue.sorted_queue), 2)
|
||||||
|
# They are in reverse order because of the reassignment to marge
|
||||||
|
self.assertEqual(queue.sorted_queue[0].id, list(DATA.keys())[1])
|
||||||
|
self.assertEqual(queue.sorted_queue[1].id, list(DATA.keys())[0])
|
||||||
|
self.assertEqual(len(queue.undetermined), 0)
|
||||||
|
|
||||||
|
for marge_mr in queue.sorted_queue:
|
||||||
|
mr_id = marge_mr.id
|
||||||
|
self.assertEqual(marge_mr.title, DATA[mr_id]["title"])
|
||||||
|
self.assertEqual(
|
||||||
|
marge_mr.web_url,
|
||||||
|
f"https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/{mr_id}"
|
||||||
|
)
|
||||||
|
self.assertIsInstance(marge_mr.time_enqueued, timedelta)
|
||||||
@@ -0,0 +1,309 @@
|
|||||||
|
import logging
|
||||||
|
from unittest.mock import AsyncMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from pipeline_message import (
|
||||||
|
get_failed_test_summary_message,
|
||||||
|
get_problem_jobs,
|
||||||
|
get_trace_failures,
|
||||||
|
main,
|
||||||
|
process_problem_jobs,
|
||||||
|
search_job_log_for_errors,
|
||||||
|
sort_failed_tests_by_status,
|
||||||
|
unexpected_improvements,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_problem_jobs():
|
||||||
|
jobs = [
|
||||||
|
{"stage": "build", "status": "failed"},
|
||||||
|
{"stage": "test", "status": "canceled"},
|
||||||
|
{"stage": "postmerge", "status": "failed"},
|
||||||
|
{"stage": "performance", "status": "failed"},
|
||||||
|
{"stage": "deploy", "status": "failed"},
|
||||||
|
]
|
||||||
|
|
||||||
|
problem_jobs = get_problem_jobs(jobs)
|
||||||
|
|
||||||
|
assert len(problem_jobs) == 3
|
||||||
|
assert problem_jobs[0]["stage"] == "build"
|
||||||
|
assert problem_jobs[1]["stage"] == "test"
|
||||||
|
assert problem_jobs[2]["stage"] == "deploy"
|
||||||
|
|
||||||
|
|
||||||
|
def test_sort_failed_tests_by_status():
|
||||||
|
failures_csv = """\
|
||||||
|
Test1,UnexpectedImprovement
|
||||||
|
Test2,Fail
|
||||||
|
Test3,Crash
|
||||||
|
Test4,Timeout
|
||||||
|
Test5,Fail
|
||||||
|
Test6,UnexpectedImprovement
|
||||||
|
"""
|
||||||
|
sorted_tests = sort_failed_tests_by_status(failures_csv)
|
||||||
|
|
||||||
|
assert len(sorted_tests["unexpected_improvements"]) == 2
|
||||||
|
assert len(sorted_tests["fails"]) == 2
|
||||||
|
assert len(sorted_tests["crashes"]) == 1
|
||||||
|
assert len(sorted_tests["timeouts"]) == 1
|
||||||
|
|
||||||
|
assert sorted_tests["unexpected_improvements"] == [
|
||||||
|
"Test1,UnexpectedImprovement",
|
||||||
|
"Test6,UnexpectedImprovement",
|
||||||
|
]
|
||||||
|
assert sorted_tests["fails"] == ["Test2,Fail", "Test5,Fail"]
|
||||||
|
assert sorted_tests["crashes"] == ["Test3,Crash"]
|
||||||
|
assert sorted_tests["timeouts"] == ["Test4,Timeout"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_failed_test_summary_message():
|
||||||
|
failed_test_array = {
|
||||||
|
"unexpected_improvements": [
|
||||||
|
"test1 UnexpectedImprovement",
|
||||||
|
"test2 UnexpectedImprovement",
|
||||||
|
],
|
||||||
|
"fails": ["test3 Fail", "test4 Fail", "test5 Fail"],
|
||||||
|
"crashes": ["test6 Crash"],
|
||||||
|
"timeouts": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
summary_message = get_failed_test_summary_message(failed_test_array)
|
||||||
|
|
||||||
|
assert "<summary>" in summary_message
|
||||||
|
assert "2 improved tests" in summary_message
|
||||||
|
assert "3 failed tests" in summary_message
|
||||||
|
assert "1 crashed test" in summary_message
|
||||||
|
assert "</summary>" in summary_message
|
||||||
|
|
||||||
|
|
||||||
|
def test_unexpected_improvements():
|
||||||
|
message = "<summary>"
|
||||||
|
failed_test_array = {
|
||||||
|
"unexpected_improvements": ["test_improvement_1", "test_improvement_2"],
|
||||||
|
"fails": [],
|
||||||
|
"crashes": [],
|
||||||
|
"timeouts": [],
|
||||||
|
}
|
||||||
|
result = unexpected_improvements(failed_test_array)
|
||||||
|
assert result == " 2 improved tests", f"Unexpected result: {result}"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@patch("pipeline_message.get_pipeline_status", new_callable=AsyncMock)
|
||||||
|
async def test_gitlab_api_failure(mock_get_pipeline_status):
|
||||||
|
mock_get_pipeline_status.side_effect = Exception("GitLab API not responding")
|
||||||
|
message = await main("1234567")
|
||||||
|
assert message == ""
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_no_message_when_pipeline_not_failed():
|
||||||
|
project_id = "176"
|
||||||
|
pipeline_id = "12345"
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"pipeline_message.get_pipeline_status", new_callable=AsyncMock
|
||||||
|
) as mock_get_pipeline_status:
|
||||||
|
mock_get_pipeline_status.return_value = "success"
|
||||||
|
|
||||||
|
message = await main(pipeline_id, project_id)
|
||||||
|
assert (
|
||||||
|
message == ""
|
||||||
|
), f"Expected no message for successful pipeline, but got: {message}"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_single_problem_job_not_summarized():
|
||||||
|
session = AsyncMock()
|
||||||
|
project_id = "176"
|
||||||
|
problem_jobs = [
|
||||||
|
{
|
||||||
|
"id": 1234,
|
||||||
|
"name": "test-job",
|
||||||
|
"web_url": "http://example.com/job/1234",
|
||||||
|
"status": "canceled",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
mock_response = AsyncMock()
|
||||||
|
mock_response.status = 200
|
||||||
|
mock_response.text.return_value = "" # Empty CSV response for test
|
||||||
|
session.get.return_value = mock_response
|
||||||
|
|
||||||
|
message = await process_problem_jobs(session, project_id, problem_jobs)
|
||||||
|
|
||||||
|
assert "summary" not in message
|
||||||
|
assert "[test-job](http://example.com/job/1234)" in message
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@patch("pipeline_message.get_project_json", new_callable=AsyncMock)
|
||||||
|
@patch("pipeline_message.aiohttp.ClientSession", autospec=True)
|
||||||
|
async def test_get_trace_failures_no_response(
|
||||||
|
mock_client_session_cls, mock_get_project_json, caplog
|
||||||
|
):
|
||||||
|
caplog.set_level(logging.DEBUG)
|
||||||
|
namespace = "mesa"
|
||||||
|
mock_get_project_json.return_value = {"path": namespace}
|
||||||
|
|
||||||
|
mock_get = AsyncMock()
|
||||||
|
mock_get.status = 404
|
||||||
|
|
||||||
|
mock_session_instance = mock_client_session_cls.return_value
|
||||||
|
mock_session_instance.get.return_value = mock_get
|
||||||
|
|
||||||
|
job_id = 12345678
|
||||||
|
job = {"id": job_id}
|
||||||
|
url = await get_trace_failures(mock_session_instance, "176", job)
|
||||||
|
|
||||||
|
assert url == ""
|
||||||
|
|
||||||
|
expected_log_message = f"No response from: https://mesa.pages.freedesktop.org/-/{namespace}/-/jobs/{job_id}/artifacts/results/summary/problems.html"
|
||||||
|
assert any(expected_log_message in record.message for record in caplog.records)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@patch("pipeline_message.get_job_log", new_callable=AsyncMock)
|
||||||
|
async def test_search_job_log_for_errors(mock_get_job_log):
|
||||||
|
session = AsyncMock()
|
||||||
|
project_id = "176"
|
||||||
|
job = {"id": 12345}
|
||||||
|
|
||||||
|
job_log = r"""
|
||||||
|
error_msg: something useful
|
||||||
|
[0m15:41:36.102: GL_KHR_no_error GL_KHR_texture_compression_astc_sliced_3d
|
||||||
|
1 error generated
|
||||||
|
3 errors generated.
|
||||||
|
-- Looking for strerror_r - found
|
||||||
|
-- Looking for strerror_s - not found
|
||||||
|
[49/176] Building CXX object lib/Support/CMakeFiles/LLVMSupport.dir/ErrorHandling.cpp.o
|
||||||
|
[127/2034] Building C object lib/Support/CMakeFiles/LLVMSupport.dir/regerror.c.o
|
||||||
|
-- Performing Test HAS_WERROR_GLOBAL_CTORS
|
||||||
|
-- Performing Test C_SUPPORTS_WERROR_UNGUARDED_AVAILABILITY_NEW - Success
|
||||||
|
-- Performing Test LLVM_LIBSTDCXX_SOFT_ERROR
|
||||||
|
error aborting
|
||||||
|
error_msg : None
|
||||||
|
error_type : Job
|
||||||
|
[0Ksection_end:1734694783:job_data
|
||||||
|
[0K
|
||||||
|
[0m11:39:43.438: [1mFinished executing LAVA job in the attempt #3 [0m
|
||||||
|
[0Ksection_end:1734694783:lava_submit
|
||||||
|
[0K
|
||||||
|
[0;31m[01:54] ERROR: lava_submit: ret code: 1 [0m
|
||||||
|
|
||||||
|
[0;31m[01:54] ERROR: unknown-section: ret code: 1 [0m
|
||||||
|
section_end:1734694783:step_script
|
||||||
|
[0Ksection_start:1734694783:after_script
|
||||||
|
[0K[0K[36;1mRunning after_script[0;m[0;m
|
||||||
|
[32;1mRunning after script...[0;m
|
||||||
|
[32;1m$ curl -L --retry 4 -f --retry-all-errors --retry-delay 60 -s "https://" | tar --warning=no-timestamp --zstd -x[0;m
|
||||||
|
zstd: /*stdin*\: unexpected end of file # noqa: W605
|
||||||
|
tar: Child returned status 1
|
||||||
|
tar: Error is not recoverable: exiting now
|
||||||
|
section_end:1734695025:after_script
|
||||||
|
[0K[0;33mWARNING: after_script failed, but job will continue unaffected: exit code 1[0;m
|
||||||
|
section_start:1734695025:upload_artifacts_on_failure
|
||||||
|
[0K[0K[36;1mUploading artifacts for failed job[0;m[0;m
|
||||||
|
[32;1mUploading artifacts...[0;m
|
||||||
|
results/: found 11 matching artifact files and directories[0;m
|
||||||
|
Uploading artifacts as "archive" to coordinator... 201 Created[0;m id[0;m=68509685 responseStatus[0;m=201 Created token[0;m=glcbt-64
|
||||||
|
[32;1mUploading artifacts...[0;m
|
||||||
|
[0;33mWARNING: results/junit.xml: no matching files. Ensure that the artifact path is relative to the working directory (/builds/mesa/mesa)[0;m
|
||||||
|
[31;1mERROR: No files to upload [0;m
|
||||||
|
section_end:1734695027:upload_artifacts_on_failure
|
||||||
|
[0Ksection_start:1734695027:cleanup_file_variables
|
||||||
|
[0K[0K[36;1mCleaning up project directory and file based variables[0;m[0;m
|
||||||
|
section_end:1734695027:cleanup_file_variables
|
||||||
|
[0K[31;1mERROR: Job failed: exit code 1
|
||||||
|
[0;m
|
||||||
|
[0;m
|
||||||
|
"""
|
||||||
|
|
||||||
|
mock_get_job_log.return_value = job_log
|
||||||
|
|
||||||
|
error_message = await search_job_log_for_errors(session, project_id, job)
|
||||||
|
assert "something useful" in error_message
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@patch("pipeline_message.get_job_log", new_callable=AsyncMock)
|
||||||
|
async def test_search_job_log_for_fatal_errors(mock_get_job_log):
|
||||||
|
session = AsyncMock()
|
||||||
|
project_id = "176"
|
||||||
|
job = {"id": 12345}
|
||||||
|
|
||||||
|
job_log = r"""
|
||||||
|
[0m15:41:36.105: [15:41:31.951] fatal: something fatal
|
||||||
|
Uploading artifacts as "archive" to coordinator... 201 Created[0;m id[0;m=68509685 responseStatus[0;m=201 Created token[0;m=glcbt-64
|
||||||
|
[32;1mUploading artifacts...[0;m
|
||||||
|
[0;33mWARNING: results/junit.xml: no matching files. Ensure that the artifact path is relative to the working directory (/builds/mesa/mesa)[0;m
|
||||||
|
[31;1mERROR: No files to upload [0;m
|
||||||
|
section_end:1734695027:upload_artifacts_on_failure
|
||||||
|
[0Ksection_start:1734695027:cleanup_file_variables
|
||||||
|
[0K[0K[36;1mCleaning up project directory and file based variables[0;m[0;m
|
||||||
|
section_end:1734695027:cleanup_file_variables
|
||||||
|
[0K[31;1mERROR: Job failed: exit code 1
|
||||||
|
[0;m
|
||||||
|
[0;m
|
||||||
|
"""
|
||||||
|
|
||||||
|
mock_get_job_log.return_value = job_log
|
||||||
|
|
||||||
|
error_message = await search_job_log_for_errors(session, project_id, job)
|
||||||
|
assert "something fatal" in error_message
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@patch("pipeline_message.get_job_log", new_callable=AsyncMock)
|
||||||
|
async def test_search_job_log_for_errors_but_find_none(mock_get_job_log):
|
||||||
|
session = AsyncMock()
|
||||||
|
project_id = "176"
|
||||||
|
job = {"id": 12345}
|
||||||
|
|
||||||
|
job_log = r"""
|
||||||
|
[0KRunning with gitlab-runner 17.4.0 (b92ee590)[0;m
|
||||||
|
[0K on fdo-equinix-m3l-30-placeholder_63 XmDXAt7xd, system ID: s_785ae19292ea[0;m
|
||||||
|
section_start:1734736110:prepare_executor
|
||||||
|
[0K[0K[36;1mPreparing the "docker" executor[0;m[0;m
|
||||||
|
[0KUsing Docker executor with image registry.freedesktop.org/mesa/mesa/debian
|
||||||
|
[0KAuthenticating with credentials from job payload (GitLab Registry)[0;m
|
||||||
|
[0KPulling docker image registry.freedesktop.org/mesa/mesa/debian/x86_64_pyuti
|
||||||
|
[0KUsing docker image sha256:ebc7b3fe89be4d390775303adddb33539c235a2663165d78d
|
||||||
|
[0Ksection_start:1734736124:prepare_script
|
||||||
|
[0K[0K[36;1mPreparing environment[0;m[0;m
|
||||||
|
Running on runner-xmdxat7xd-project-23076-concurrent-1 via fdo-equinix-m3l-30...
|
||||||
|
section_end:1734736125:prepare_script
|
||||||
|
[0Ksection_start:1734736125:get_sources
|
||||||
|
[0K[0K[36;1mGetting source from Git repository[0;m[0;m
|
||||||
|
[32;1m$ /host/bin/curl -s -L --cacert /host/ca-certificates.crt --retry 4 -f --retry-delay 60 https://gitlab.
|
||||||
|
Checking if the user of the pipeline is allowed...
|
||||||
|
Checking if the job's project is part of a well-known group...
|
||||||
|
Checking if the job is part of an official MR pipeline...
|
||||||
|
Thank you for contributing to freedesktop.org
|
||||||
|
Running pre-clone script: 'set -o xtrace
|
||||||
|
wget -q -O download-git-cache.sh https://gitlab.freedesktop.org/mesa/mesa/-/raw/0d43b4cba639b809ad0e08a065ce01846e262249/.gitlab-ci/download-git-cache.sh
|
||||||
|
bash download-git-cache.sh
|
||||||
|
rm download-git-cache.sh
|
||||||
|
[31;1m errors
|
||||||
|
[0K[31;1mERROR:
|
||||||
|
[31;1m error
|
||||||
|
[31;1m Here is a blank error:
|
||||||
|
/builds/mesa/mesa/bin/ci/test/test_pipeline_message.py:162: AssertionError
|
||||||
|
Uploading artifacts as "archive" to coordinator... 201 Created[0;m id[0;m=68509685 responseStatus[0;m=201 Created token[0;m=glcbt-64
|
||||||
|
[32;1mUploading artifacts...[0;m
|
||||||
|
[0;33mWARNING: results/junit.xml: no matching files. Ensure that the artifact path is relative to the working directory (/builds/mesa/mesa)[0;m
|
||||||
|
[31;1mERROR: No files to upload [0;m
|
||||||
|
section_end:1734695027:upload_artifacts_on_failure
|
||||||
|
[0Ksection_start:1734695027:cleanup_file_variables
|
||||||
|
[0K[0K[36;1mCleaning up project directory and file based variables[0;m[0;m
|
||||||
|
section_end:1734695027:cleanup_file_variables
|
||||||
|
[0K[31;1mERROR: Job failed: exit code 1
|
||||||
|
[0;m
|
||||||
|
[0;m
|
||||||
|
"""
|
||||||
|
|
||||||
|
mock_get_job_log.return_value = job_log
|
||||||
|
|
||||||
|
error_message = await search_job_log_for_errors(session, project_id, job)
|
||||||
|
assert error_message == "", f"Unexpected error message: {error_message}"
|
||||||
@@ -0,0 +1,182 @@
|
|||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from mock import MagicMock, patch
|
||||||
|
from structured_logger import (
|
||||||
|
AutoSaveDict,
|
||||||
|
CSVStrategy,
|
||||||
|
JSONStrategy,
|
||||||
|
StructuredLogger,
|
||||||
|
YAMLStrategy,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(params=[CSVStrategy, JSONStrategy, YAMLStrategy])
|
||||||
|
def strategy(request):
|
||||||
|
return request.param
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def file_extension(strategy):
|
||||||
|
if strategy == CSVStrategy:
|
||||||
|
return "csv"
|
||||||
|
elif strategy == JSONStrategy:
|
||||||
|
return "json"
|
||||||
|
elif strategy == YAMLStrategy:
|
||||||
|
return "yaml"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def tmp_file(tmp_path):
|
||||||
|
return tmp_path / "test.json"
|
||||||
|
|
||||||
|
|
||||||
|
def test_guess_strategy_from_file(tmp_path, strategy, file_extension):
|
||||||
|
file_name = tmp_path / f"test_guess.{file_extension}"
|
||||||
|
Path(file_name).touch()
|
||||||
|
guessed_strategy = StructuredLogger.guess_strategy_from_file(file_name)
|
||||||
|
assert isinstance(guessed_strategy, strategy)
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_strategy(strategy, file_extension):
|
||||||
|
result = StructuredLogger.get_strategy(file_extension)
|
||||||
|
assert isinstance(result, strategy)
|
||||||
|
|
||||||
|
|
||||||
|
def test_invalid_file_extension(tmp_path):
|
||||||
|
file_name = tmp_path / "test_invalid.xyz"
|
||||||
|
Path(file_name).touch()
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="Unknown strategy for: xyz"):
|
||||||
|
StructuredLogger.guess_strategy_from_file(file_name)
|
||||||
|
|
||||||
|
|
||||||
|
def test_non_existent_file(tmp_path, strategy, file_extension):
|
||||||
|
file_name = tmp_path / f"non_existent.{file_extension}"
|
||||||
|
logger = StructuredLogger(file_name, strategy())
|
||||||
|
|
||||||
|
assert logger.file_path.exists()
|
||||||
|
assert "_timestamp" in logger._data
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def structured_logger_module():
|
||||||
|
with patch.dict("sys.modules", {"polars": None, "ruamel.yaml": None}):
|
||||||
|
import importlib
|
||||||
|
|
||||||
|
import structured_logger
|
||||||
|
|
||||||
|
importlib.reload(structured_logger)
|
||||||
|
yield structured_logger
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_csv_library(tmp_path, structured_logger_module):
|
||||||
|
with pytest.raises(RuntimeError, match="Can't parse CSV files. Missing library"):
|
||||||
|
structured_logger_module.CSVStrategy()
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_yaml_library(tmp_path, structured_logger_module):
|
||||||
|
with pytest.raises(RuntimeError, match="Can't parse YAML files. Missing library"):
|
||||||
|
structured_logger_module.YAMLStrategy()
|
||||||
|
|
||||||
|
|
||||||
|
def test_autosavedict_setitem():
|
||||||
|
save_callback = MagicMock()
|
||||||
|
d = AutoSaveDict(save_callback=save_callback)
|
||||||
|
d["key"] = "value"
|
||||||
|
assert d["key"] == "value"
|
||||||
|
save_callback.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_autosavedict_delitem():
|
||||||
|
save_callback = MagicMock()
|
||||||
|
d = AutoSaveDict({"key": "value"}, save_callback=save_callback)
|
||||||
|
del d["key"]
|
||||||
|
assert "key" not in d
|
||||||
|
save_callback.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_autosavedict_pop():
|
||||||
|
save_callback = MagicMock()
|
||||||
|
d = AutoSaveDict({"key": "value"}, save_callback=save_callback)
|
||||||
|
result = d.pop("key")
|
||||||
|
assert result == "value"
|
||||||
|
assert "key" not in d
|
||||||
|
save_callback.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_autosavedict_update():
|
||||||
|
save_callback = MagicMock()
|
||||||
|
d = AutoSaveDict({"key": "old_value"}, save_callback=save_callback)
|
||||||
|
d.update({"key": "new_value"})
|
||||||
|
assert d["key"] == "new_value"
|
||||||
|
save_callback.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_structured_logger_setitem(tmp_file):
|
||||||
|
logger = StructuredLogger(tmp_file, JSONStrategy())
|
||||||
|
logger.data["field"] = "value"
|
||||||
|
|
||||||
|
with open(tmp_file, "r") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
|
||||||
|
assert data["field"] == "value"
|
||||||
|
|
||||||
|
|
||||||
|
def test_structured_logger_set_recursive(tmp_file):
|
||||||
|
logger = StructuredLogger(tmp_file, JSONStrategy())
|
||||||
|
logger.data["field"] = {"test": True}
|
||||||
|
other = logger.data["field"]
|
||||||
|
other["late"] = True
|
||||||
|
|
||||||
|
with open(tmp_file, "r") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
|
||||||
|
assert data["field"]["test"]
|
||||||
|
assert data["field"]["late"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_structured_logger_set_list(tmp_file):
|
||||||
|
logger = StructuredLogger(tmp_file, JSONStrategy())
|
||||||
|
logger.data["field"] = [True]
|
||||||
|
other = logger.data["field"]
|
||||||
|
other.append(True)
|
||||||
|
|
||||||
|
with open(tmp_file, "r") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
|
||||||
|
assert data["field"][0]
|
||||||
|
assert data["field"][1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_structured_logger_delitem(tmp_file):
|
||||||
|
logger = StructuredLogger(tmp_file, JSONStrategy())
|
||||||
|
logger.data["field"] = "value"
|
||||||
|
del logger.data["field"]
|
||||||
|
|
||||||
|
with open(tmp_file, "r") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
|
||||||
|
assert "field" not in data
|
||||||
|
|
||||||
|
|
||||||
|
def test_structured_logger_pop(tmp_file):
|
||||||
|
logger = StructuredLogger(tmp_file, JSONStrategy())
|
||||||
|
logger.data["field"] = "value"
|
||||||
|
logger.data.pop("field")
|
||||||
|
|
||||||
|
with open(tmp_file, "r") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
|
||||||
|
assert "field" not in data
|
||||||
|
|
||||||
|
|
||||||
|
def test_structured_logger_update(tmp_file):
|
||||||
|
logger = StructuredLogger(tmp_file, JSONStrategy())
|
||||||
|
logger.data.update({"field": "value"})
|
||||||
|
|
||||||
|
with open(tmp_file, "r") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
|
||||||
|
assert data["field"] == "value"
|
||||||
@@ -0,0 +1,391 @@
|
|||||||
|
import sys
|
||||||
|
import subprocess
|
||||||
|
import yaml
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from textwrap import dedent
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from bin.ci.update_tag import (
|
||||||
|
from_component_to_build_tag,
|
||||||
|
filter_components,
|
||||||
|
from_component_to_tag_var,
|
||||||
|
from_script_name_to_component,
|
||||||
|
from_script_name_to_tag_var,
|
||||||
|
load_image_tags_yaml,
|
||||||
|
update_image_tag_in_yaml,
|
||||||
|
run_build_script,
|
||||||
|
main,
|
||||||
|
PATHS,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def temp_image_tags_file(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
Fixture that creates a temporary YAML file (to simulate conditional_tags)
|
||||||
|
and updates the global variable accordingly.
|
||||||
|
"""
|
||||||
|
# Create a temporary test file
|
||||||
|
temp_file = tmp_path / "conditional-build-image-tags.yml"
|
||||||
|
temp_file.write_text(yaml.dump({"variables": {}}))
|
||||||
|
|
||||||
|
# Create a copy of the original PATHS values
|
||||||
|
original_conditional_tags = PATHS.conditional_tags
|
||||||
|
|
||||||
|
# Patch the PATHS instance with our test file
|
||||||
|
PATHS.conditional_tags = temp_file
|
||||||
|
|
||||||
|
# Yield the test file for the test to use
|
||||||
|
yield temp_file
|
||||||
|
|
||||||
|
# Restore the original value after the test
|
||||||
|
PATHS.conditional_tags = original_conditional_tags
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def temp_container_ci_file(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
Fixture that creates a temporary container CI file (to simulate container_ci)
|
||||||
|
and updates the global variable accordingly.
|
||||||
|
"""
|
||||||
|
# Create a temporary test file
|
||||||
|
temp_file = tmp_path / "container-ci.yml"
|
||||||
|
temp_file.touch()
|
||||||
|
|
||||||
|
# Create a copy of the original PATHS values
|
||||||
|
original_container_ci = PATHS.container_ci
|
||||||
|
|
||||||
|
# Patch the PATHS instance with our test file
|
||||||
|
PATHS.container_ci = temp_file
|
||||||
|
|
||||||
|
# Yield the test file for the test to use
|
||||||
|
yield temp_file
|
||||||
|
|
||||||
|
# Restore the original value after the test
|
||||||
|
PATHS.container_ci = original_container_ci
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def mock_container_dir(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
Fixture that creates a dummy container directory and build script.
|
||||||
|
"""
|
||||||
|
# Create a dummy container directory
|
||||||
|
container_dir = tmp_path / "container"
|
||||||
|
container_dir.mkdir()
|
||||||
|
|
||||||
|
# Create a dummy setup-test-env.sh file
|
||||||
|
dummy_setup_path = tmp_path / "setup-test-env.sh"
|
||||||
|
dummy_setup_path.write_text("echo Setup")
|
||||||
|
|
||||||
|
# Save original values
|
||||||
|
original_container_dir = PATHS.container_dir
|
||||||
|
original_setup_test_env = PATHS.setup_test_env
|
||||||
|
|
||||||
|
# Patch the PATHS instance
|
||||||
|
PATHS.container_dir = container_dir
|
||||||
|
PATHS.setup_test_env = dummy_setup_path
|
||||||
|
|
||||||
|
# Yield the directory for the test to use
|
||||||
|
yield container_dir
|
||||||
|
|
||||||
|
# Restore original values
|
||||||
|
PATHS.container_dir = original_container_dir
|
||||||
|
PATHS.setup_test_env = original_setup_test_env
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def mock_build_script(mock_container_dir):
|
||||||
|
"""
|
||||||
|
Fixture that creates a dummy build script in the container directory.
|
||||||
|
"""
|
||||||
|
build_script = mock_container_dir / "build-foo.sh"
|
||||||
|
return build_script
|
||||||
|
|
||||||
|
|
||||||
|
###############################################################################
|
||||||
|
# Tests for argument parsing and helper functions
|
||||||
|
###############################################################################
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"component, expected_tag_var",
|
||||||
|
[
|
||||||
|
("foo", "FOO_TAG"),
|
||||||
|
("my-component", "MY_COMPONENT_TAG"),
|
||||||
|
("foo-bar-baz", "FOO_BAR_BAZ_TAG"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_from_component_to_tag_var(component, expected_tag_var):
|
||||||
|
"""
|
||||||
|
Test that from_component_to_tag_var returns the correct tag variable name.
|
||||||
|
"""
|
||||||
|
assert from_component_to_tag_var(component) == expected_tag_var
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"component, expected_build_tag",
|
||||||
|
[
|
||||||
|
("foo", "CONDITIONAL_BUILD_FOO_TAG"),
|
||||||
|
("my-component", "CONDITIONAL_BUILD_MY_COMPONENT_TAG"),
|
||||||
|
("foo-bar-baz", "CONDITIONAL_BUILD_FOO_BAR_BAZ_TAG"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_from_component_to_build_tag(component, expected_build_tag):
|
||||||
|
"""
|
||||||
|
Test that from_component_to_build_tag returns the correct build tag name.
|
||||||
|
"""
|
||||||
|
assert from_component_to_build_tag(component) == expected_build_tag
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"script_name, expected_component",
|
||||||
|
[
|
||||||
|
("build-foo.sh", "foo"),
|
||||||
|
("build-my-component.sh", "my-component"),
|
||||||
|
("build-foo-bar-baz.sh", "foo-bar-baz"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_from_script_name_to_component(script_name, expected_component):
|
||||||
|
"""
|
||||||
|
Test that from_script_name_to_component returns the correct component name.
|
||||||
|
"""
|
||||||
|
assert from_script_name_to_component(script_name) == expected_component
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"script_name, expected_tag_var",
|
||||||
|
[
|
||||||
|
("build-foo.sh", "FOO_TAG"),
|
||||||
|
("build-my-component.sh", "MY_COMPONENT_TAG"),
|
||||||
|
("build-foo-bar-baz.sh", "FOO_BAR_BAZ_TAG"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_from_script_name_to_tag_var(script_name, expected_tag_var):
|
||||||
|
"""
|
||||||
|
Test that from_script_name_to_tag_var returns the correct tag variable name.
|
||||||
|
"""
|
||||||
|
assert from_script_name_to_tag_var(script_name) == expected_tag_var
|
||||||
|
|
||||||
|
|
||||||
|
def test_filter_components():
|
||||||
|
"""
|
||||||
|
Test that filter_components returns only components that match an include regex
|
||||||
|
and do not match any exclude regex. If includes is empty, an empty list is returned.
|
||||||
|
"""
|
||||||
|
components = ["alpha", "beta", "gamma", "delta"]
|
||||||
|
|
||||||
|
# When includes is empty, should return an empty list.
|
||||||
|
assert filter_components(components, [], []) == []
|
||||||
|
|
||||||
|
# Test includes only.
|
||||||
|
# Components that start with 'a' or 'b'
|
||||||
|
expected = ["alpha", "beta"]
|
||||||
|
result = filter_components(components, ["^[a-b].*$"], [])
|
||||||
|
assert result == expected
|
||||||
|
|
||||||
|
# Test with an exclude that filters out "alpha" (which matches "lph").
|
||||||
|
result = filter_components(components, ["^[a-b].*$"], ["^.*lph.*$"])
|
||||||
|
# "alpha" is removed.
|
||||||
|
assert result == ["beta"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"component, tag",
|
||||||
|
[
|
||||||
|
# Basic case
|
||||||
|
("test", "new_tag"),
|
||||||
|
# Component with hyphen
|
||||||
|
("my-component", "v1.2.3"),
|
||||||
|
# Uppercase component name
|
||||||
|
("UpperCase", "123"),
|
||||||
|
# Numbers in component name
|
||||||
|
("123service", "build-456"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_update_image_tag_in_yaml(component, tag, temp_image_tags_file):
|
||||||
|
"""
|
||||||
|
Test multiple updates with different component names and tags, verifying:
|
||||||
|
1. Correct variable name generation
|
||||||
|
2. Proper tag value storage
|
||||||
|
3. Maintained sorting of variables
|
||||||
|
"""
|
||||||
|
# Initial update
|
||||||
|
update_image_tag_in_yaml(component, tag)
|
||||||
|
|
||||||
|
data = load_image_tags_yaml()
|
||||||
|
expected_var = from_component_to_build_tag(component)
|
||||||
|
assert data["variables"][expected_var] == tag
|
||||||
|
|
||||||
|
# Add second component and verify both exist
|
||||||
|
update_image_tag_in_yaml("another_component", "secondary_tag")
|
||||||
|
updated_data = load_image_tags_yaml()
|
||||||
|
|
||||||
|
assert from_component_to_build_tag("another_component") in updated_data["variables"]
|
||||||
|
assert updated_data["variables"][expected_var] == tag # Original value remains
|
||||||
|
|
||||||
|
# Verify sorting
|
||||||
|
variables = list(data["variables"].keys())
|
||||||
|
assert len(updated_data["variables"]) == len(variables) + 1
|
||||||
|
assert variables == sorted(variables), "Variables are not alphabetically sorted"
|
||||||
|
|
||||||
|
|
||||||
|
def test_if_run_extracts_the_tag_from_stdout(monkeypatch, mock_build_script):
|
||||||
|
"""
|
||||||
|
Test that run_build_script returns the tag (last stdout line) when the build
|
||||||
|
script executes successfully.
|
||||||
|
"""
|
||||||
|
mock_build_script.write_text("#!/bin/bash\necho Build script\necho new_tag")
|
||||||
|
# Create a fake subprocess.CompletedProcess to simulate a successful build.
|
||||||
|
fake_result = subprocess.CompletedProcess(
|
||||||
|
args=[], returncode=0, stdout="line1\nnew_tag", stderr=""
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: fake_result)
|
||||||
|
|
||||||
|
tag = run_build_script("foo", check_only=False)
|
||||||
|
assert tag == "new_tag"
|
||||||
|
|
||||||
|
|
||||||
|
def test_running_real_process_works(monkeypatch, mock_build_script):
|
||||||
|
"""
|
||||||
|
Test that run_build_script returns the tag (last stdout line) when the build
|
||||||
|
script executes successfully.
|
||||||
|
"""
|
||||||
|
mock_build_script.write_text("echo Build script\necho new_tag")
|
||||||
|
tag = run_build_script("foo", check_only=False)
|
||||||
|
assert tag == "new_tag"
|
||||||
|
|
||||||
|
|
||||||
|
###############################################################################
|
||||||
|
# Tests for main() argument features
|
||||||
|
###############################################################################
|
||||||
|
|
||||||
|
|
||||||
|
def setup_build_script_and_container_ci_file(container_dir, container_ci_file, comp):
|
||||||
|
tag_var = from_component_to_build_tag(comp)
|
||||||
|
build_check_var = from_component_to_tag_var(comp)
|
||||||
|
build_script_path = container_dir / f"build-{comp}.sh"
|
||||||
|
build_script_path.write_text(
|
||||||
|
dedent(
|
||||||
|
f"""
|
||||||
|
#!/bin/bash
|
||||||
|
ci_tag_build_time_check {tag_var}
|
||||||
|
echo "new_tag"
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
)
|
||||||
|
container_ci_file.write_text(
|
||||||
|
container_ci_file.read_text()
|
||||||
|
+ dedent(
|
||||||
|
f"""
|
||||||
|
.container-builds-{comp}:
|
||||||
|
variables:
|
||||||
|
{tag_var}: "${{{build_check_var}}}"
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_main_list(monkeypatch, capsys, mock_container_dir, temp_container_ci_file):
|
||||||
|
"""
|
||||||
|
Test that when the --list argument is provided, main() prints all detected
|
||||||
|
components
|
||||||
|
"""
|
||||||
|
# Monkeypatch find_components to return a known list.
|
||||||
|
for comp in ["comp-a", "comp-b"]:
|
||||||
|
setup_build_script_and_container_ci_file(mock_container_dir, temp_container_ci_file, comp)
|
||||||
|
# Set sys.argv to simulate passing --list.
|
||||||
|
monkeypatch.setattr(sys, "argv", ["bin/ci/update_tag.py", "--list"])
|
||||||
|
|
||||||
|
main()
|
||||||
|
|
||||||
|
captured = capsys.readouterr().out
|
||||||
|
assert "Detected components:" in captured
|
||||||
|
assert "comp-a" in captured
|
||||||
|
assert "comp-b" in captured
|
||||||
|
|
||||||
|
|
||||||
|
def test_main_check(monkeypatch, temp_image_tags_file, temp_container_ci_file):
|
||||||
|
"""
|
||||||
|
Test that when --check is provided, main() calls run_build_script in check mode.
|
||||||
|
"""
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"bin.ci.update_tag.run_build_script", return_value="dummy_tag"
|
||||||
|
) as mock_run_build_script:
|
||||||
|
# Simulate command line: --check compX
|
||||||
|
monkeypatch.setattr(sys, "argv", ["bin/ci/update_tag.py", "--check", "comp-x"])
|
||||||
|
|
||||||
|
main()
|
||||||
|
# Verify run_build_script was called with check_only=True.
|
||||||
|
assert mock_run_build_script.call_count == 1
|
||||||
|
assert "comp-x" in mock_run_build_script.call_args.args[0]
|
||||||
|
assert mock_run_build_script.call_args.kwargs["check_only"] is True
|
||||||
|
|
||||||
|
|
||||||
|
EXIT_CODE_SCENARIOS = {
|
||||||
|
"unbound_variable": (
|
||||||
|
"line 2: UNDEFINED_VARIABLE: unbound variable",
|
||||||
|
127,
|
||||||
|
3,
|
||||||
|
"Please set the variable UNDEFINED_VARIABLE",
|
||||||
|
),
|
||||||
|
"build script error": (
|
||||||
|
"",
|
||||||
|
50,
|
||||||
|
50,
|
||||||
|
"",
|
||||||
|
),
|
||||||
|
"tag_mismatch": (
|
||||||
|
"Tag mismatch for foo.",
|
||||||
|
2,
|
||||||
|
2,
|
||||||
|
"Tag mismatch for foo.",
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"stderr_content, script_returncode, expected_exit_code, expected_error_message",
|
||||||
|
EXIT_CODE_SCENARIOS.values(),
|
||||||
|
ids=list(EXIT_CODE_SCENARIOS.keys()),
|
||||||
|
)
|
||||||
|
def test_build_script_error_exit_codes(
|
||||||
|
monkeypatch,
|
||||||
|
mock_build_script,
|
||||||
|
capsys,
|
||||||
|
stderr_content,
|
||||||
|
script_returncode,
|
||||||
|
expected_exit_code,
|
||||||
|
expected_error_message,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Test that build script errors generate appropriate exit codes:
|
||||||
|
1 - Unhandled error in this script
|
||||||
|
2 - Tag mismatch when using --check
|
||||||
|
3 - Unbound variable error in build script
|
||||||
|
x - Build script failed with return code x
|
||||||
|
"""
|
||||||
|
# Create a build script that will fail
|
||||||
|
mock_build_script.write_text("#!/bin/bash\necho 'This will fail'")
|
||||||
|
|
||||||
|
# Create a fake subprocess.CompletedProcess to simulate a failed build
|
||||||
|
fake_result = subprocess.CompletedProcess(
|
||||||
|
args=[], returncode=script_returncode, stdout="", stderr=stderr_content
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(subprocess, "run", lambda *args, **kwargs: fake_result)
|
||||||
|
monkeypatch.setattr(sys, "argv", ["bin/ci/update_tag.py", "--check", "foo"])
|
||||||
|
monkeypatch.setattr("bin.ci.update_tag.get_current_tag_value", lambda *args: "current_tag")
|
||||||
|
|
||||||
|
# Mock sys.exit to capture the exit code instead of exiting the test
|
||||||
|
with pytest.raises(SystemExit) as e:
|
||||||
|
main()
|
||||||
|
|
||||||
|
# Check for expected error message if one is specified
|
||||||
|
if expected_error_message:
|
||||||
|
captured = capsys.readouterr()
|
||||||
|
assert expected_error_message in captured.err
|
||||||
|
|
||||||
|
# Verify correct exit code
|
||||||
|
assert e.value.code == expected_exit_code
|
||||||
Executable
+447
@@ -0,0 +1,447 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import argparse
|
||||||
|
import subprocess
|
||||||
|
import yaml
|
||||||
|
from typing import Optional, Set, Any
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
# Very simple type alias for GitLab YAML data structure
|
||||||
|
# It is composed by a dictionary of job names, each with a dictionary of fields
|
||||||
|
# (e.g., script, stage, rules, etc.)
|
||||||
|
YamlData = dict[str, dict[str, Any]]
|
||||||
|
|
||||||
|
# Dummy environment vars to make build scripts happy
|
||||||
|
# To avoid set -u errors in build scripts
|
||||||
|
DUMMY_ENV_VARS: dict[str, str] = {
|
||||||
|
# setup-test-env.sh
|
||||||
|
"CI_JOB_STARTED_AT": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||||||
|
# build-deqp.sh
|
||||||
|
"DEQP_API": "dummy",
|
||||||
|
"DEQP_TARGET": "dummy",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ProjectPaths:
|
||||||
|
"""Manages all project-related paths"""
|
||||||
|
|
||||||
|
root: Path = field(default_factory=lambda: Path(), init=False)
|
||||||
|
setup_test_env: Path = field(default_factory=lambda: Path(), init=False)
|
||||||
|
conditional_tags: Path = field(default_factory=lambda: Path(), init=False)
|
||||||
|
container_dir: Path = field(default_factory=lambda: Path(), init=False)
|
||||||
|
container_ci: Path = field(default_factory=lambda: Path(), init=False)
|
||||||
|
|
||||||
|
def __post_init__(self):
|
||||||
|
self.root = self.find_root()
|
||||||
|
self.setup_test_env = self.root / ".gitlab-ci" / "setup-test-env.sh"
|
||||||
|
self.conditional_tags = self.root / ".gitlab-ci" / "conditional-build-image-tags.yml"
|
||||||
|
self.container_dir = self.root / ".gitlab-ci" / "container"
|
||||||
|
self.container_ci = self.container_dir / "gitlab-ci-inc.yml"
|
||||||
|
|
||||||
|
def find_root(self) -> Path:
|
||||||
|
"""Find git repository root or fallback to other methods"""
|
||||||
|
try:
|
||||||
|
logging.debug("Getting git repo root using git rev-parse")
|
||||||
|
return Path(
|
||||||
|
subprocess.check_output(
|
||||||
|
["git", "rev-parse", "--show-toplevel"],
|
||||||
|
text=True,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
).strip()
|
||||||
|
)
|
||||||
|
# FileNotFoundError is raised if git is not installed
|
||||||
|
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||||
|
# Fallback to CI_PROJECT_DIR or path-based method
|
||||||
|
if ci_dir := os.environ.get("CI_PROJECT_DIR"):
|
||||||
|
logging.debug(f"Getting git repo root using CI_PROJECT_DIR: {ci_dir}")
|
||||||
|
return Path(ci_dir)
|
||||||
|
|
||||||
|
# Last resort, use the path to this script
|
||||||
|
logging.debug("Getting git repo root using path to this script")
|
||||||
|
return Path(__file__).resolve().parent.parent.parent
|
||||||
|
|
||||||
|
|
||||||
|
PATHS: ProjectPaths = ProjectPaths()
|
||||||
|
|
||||||
|
|
||||||
|
def from_component_to_build_tag(component: str) -> str:
|
||||||
|
# e.g., "angle" -> "CONDITIONAL_BUILD_ANGLE_TAG"
|
||||||
|
return "CONDITIONAL_BUILD_" + re.sub(r"-", "_", component.upper()) + "_TAG"
|
||||||
|
|
||||||
|
|
||||||
|
def from_component_to_tag_var(component: str) -> str:
|
||||||
|
# e.g., "angle" -> "ANGLE_TAG"
|
||||||
|
return re.sub(r"-", "_", component.upper()) + "_TAG"
|
||||||
|
|
||||||
|
|
||||||
|
def from_script_name_to_component(script_name: str) -> str:
|
||||||
|
# e.g., "build-angle.sh" -> "angle"
|
||||||
|
return re.sub(r"^build-([a-z0-9_-]+)\.sh$", r"\1", script_name)
|
||||||
|
|
||||||
|
|
||||||
|
def from_script_name_to_tag_var(script_name: str) -> str:
|
||||||
|
# e.g., "build-angle.sh" -> "ANGLE_TAG"
|
||||||
|
return re.sub(r"^build-([a-z0-9_-]+)\.sh$", r"\1_TAG", script_name).replace("-", "_").upper()
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_setup_env_script() -> Path:
|
||||||
|
"""
|
||||||
|
Sets up dummy environment variables to mimic the script in the CI repo.
|
||||||
|
Returns the path to the setup-test-env.sh script.
|
||||||
|
"""
|
||||||
|
if not PATHS.setup_test_env.exists():
|
||||||
|
logging.error(".gitlab-ci/setup-test-env.sh not found. Exiting.")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
# Dummy environment vars to mimic the script
|
||||||
|
for key, value in DUMMY_ENV_VARS.items():
|
||||||
|
os.environ[key] = value
|
||||||
|
|
||||||
|
os.environ["CI_PROJECT_DIR"] = str(PATHS.root)
|
||||||
|
|
||||||
|
return PATHS.setup_test_env
|
||||||
|
|
||||||
|
|
||||||
|
def validate_build_script(script_filename: str) -> bool:
|
||||||
|
"""
|
||||||
|
Returns True if the build script for the given component uses the structured tag variable.
|
||||||
|
"""
|
||||||
|
build_script = PATHS.container_dir / script_filename
|
||||||
|
with open(build_script, "r", encoding="utf-8") as f:
|
||||||
|
script_content = f.read()
|
||||||
|
|
||||||
|
tag_var = from_script_name_to_tag_var(script_filename)
|
||||||
|
if not re.search(tag_var, script_content, re.IGNORECASE):
|
||||||
|
logging.debug(
|
||||||
|
f"Skipping {build_script} because it doesn't use {tag_var}",
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def load_container_yaml() -> YamlData:
|
||||||
|
if not PATHS.container_ci.is_file():
|
||||||
|
logging.error(f"File not found: {PATHS.container_ci}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
# Ignore !reference and other custom GitLab tags, we just want to know the
|
||||||
|
# job names and fields
|
||||||
|
yaml.SafeLoader.add_multi_constructor("", lambda loader, suffix, node: None)
|
||||||
|
with open(str(PATHS.container_ci), "r", encoding="utf-8") as f:
|
||||||
|
data = yaml.load(f, Loader=yaml.SafeLoader)
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return {"variables": {}}
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def find_candidate_components() -> list[str]:
|
||||||
|
"""
|
||||||
|
1) Reads .gitlab-ci/container/gitlab-ci.yml to find component links:
|
||||||
|
lines matching '.*.container-builds-<component>'
|
||||||
|
2) Looks for matching build-<component>.sh in .gitlab-ci/container/
|
||||||
|
3) Returns a sorted list of components in the intersection of these sets.
|
||||||
|
"""
|
||||||
|
container_yaml = load_container_yaml()
|
||||||
|
# Extract patterns like `container-builds-foo` from job names
|
||||||
|
candidates: Set[str] = set()
|
||||||
|
for job_name in container_yaml:
|
||||||
|
if match := re.search(r"\.container-builds-([a-z0-9_-]+)$", str(job_name)):
|
||||||
|
candidates.add(match.group(1))
|
||||||
|
|
||||||
|
if not candidates:
|
||||||
|
logging.error(
|
||||||
|
f"No viable build components found in {PATHS.container_ci}. "
|
||||||
|
"Please check the file for valid component names. "
|
||||||
|
"They should be named like '.container-builds-<component>'."
|
||||||
|
)
|
||||||
|
return []
|
||||||
|
|
||||||
|
# Find build scripts named build-<component>.sh
|
||||||
|
build_scripts: list[str] = []
|
||||||
|
for path in PATHS.container_dir.glob("build-*.sh"):
|
||||||
|
if validate_build_script(path.name):
|
||||||
|
logging.info(f"Found build script: {path.name}")
|
||||||
|
component = from_script_name_to_component(path.name)
|
||||||
|
build_scripts.append(component)
|
||||||
|
|
||||||
|
# Return sorted intersection of components found in build scripts and candidates
|
||||||
|
return sorted(candidates.intersection(build_scripts))
|
||||||
|
|
||||||
|
|
||||||
|
def filter_components(components: list[str], includes: list[str], excludes: list[str]) -> list[str]:
|
||||||
|
"""
|
||||||
|
Returns components that match at least one `includes` regex and none of the `excludes` regex.
|
||||||
|
If includes is empty, returns an empty list (unless user explicitly does --all or --include).
|
||||||
|
"""
|
||||||
|
if not includes:
|
||||||
|
return []
|
||||||
|
|
||||||
|
filtered = []
|
||||||
|
for comp in components:
|
||||||
|
# Must match at least one "include"
|
||||||
|
if not any(re.fullmatch(inc, comp) for inc in includes):
|
||||||
|
logging.debug(f"Excluding {comp}, no matches in includes.")
|
||||||
|
continue
|
||||||
|
# Must not match any "exclude"
|
||||||
|
if any(re.fullmatch(exc, comp) for exc in excludes):
|
||||||
|
logging.debug(f"Excluding {comp}, matched exclude pattern.")
|
||||||
|
continue
|
||||||
|
filtered.append(comp)
|
||||||
|
return filtered
|
||||||
|
|
||||||
|
|
||||||
|
def run_build_script(component: str, check_only: bool = False) -> Optional[str]:
|
||||||
|
"""
|
||||||
|
Runs .gitlab-ci/container/build-<component>.sh to produce a new tag (last line of stdout).
|
||||||
|
If check_only=True, we skip updates to the YAML (but do the build to see if it passes).
|
||||||
|
Returns the extracted tag (string) on success, or None on failure.
|
||||||
|
"""
|
||||||
|
# 1) Set up environment
|
||||||
|
setup_env_script = prepare_setup_env_script()
|
||||||
|
|
||||||
|
build_script = PATHS.container_dir / f"build-{component}.sh"
|
||||||
|
if not build_script.is_file():
|
||||||
|
logging.error(f"Build script not found for {component}: {build_script}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Tag var should appear in the script, e.g., ANGLE_TAG for 'angle'
|
||||||
|
tag_var = from_component_to_tag_var(component)
|
||||||
|
|
||||||
|
# We set up environment for the child process
|
||||||
|
child_env: dict[str, str] = {}
|
||||||
|
child_env["NEW_TAG_DRY_RUN"] = "1"
|
||||||
|
if check_only:
|
||||||
|
# For checking only
|
||||||
|
child_env.pop("NEW_TAG_DRY_RUN", None)
|
||||||
|
child_env["CI_NOT_BUILDING_ANYTHING"] = "1"
|
||||||
|
if tag_value := get_current_tag_value(component):
|
||||||
|
child_env[tag_var] = tag_value
|
||||||
|
else:
|
||||||
|
logging.error(f"No current tag value for {component}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
logging.debug(
|
||||||
|
f"Running build for {component} with "
|
||||||
|
f"{tag_var}={child_env.get(tag_var)} "
|
||||||
|
f"(check_only={check_only})"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Run the build script
|
||||||
|
result = subprocess.run(
|
||||||
|
["bash", "-c", f"source {setup_env_script} && source {build_script}"],
|
||||||
|
env=os.environ | child_env,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
logging.debug(f"{' '.join(result.args)}")
|
||||||
|
|
||||||
|
# Tag check succeeded
|
||||||
|
if result.returncode == 0:
|
||||||
|
# Tag is assumed to be the last line of stdout
|
||||||
|
lines = result.stdout.strip().splitlines()
|
||||||
|
return lines[-1] if lines else ""
|
||||||
|
|
||||||
|
# Tag check failed, let's dissect the error
|
||||||
|
|
||||||
|
if result.returncode == 2:
|
||||||
|
logging.error(f"Tag mismatch for {component}.")
|
||||||
|
logging.error(result.stdout)
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Check if there's an unbound variable error
|
||||||
|
err_output = result.stderr
|
||||||
|
unbound_match = re.search(r"([A-Z_]+)(?=: unbound variable)", err_output)
|
||||||
|
if unbound_match:
|
||||||
|
var_name = unbound_match.group(1)
|
||||||
|
logging.fatal(f"Please set the variable {var_name} in {build_script}.")
|
||||||
|
sys.exit(3)
|
||||||
|
|
||||||
|
# Unexpected error in the build script, propagate the exit code
|
||||||
|
logging.fatal(f"Build script for {component} failed with return code {result.returncode}")
|
||||||
|
logging.error(result.stdout)
|
||||||
|
sys.exit(result.returncode)
|
||||||
|
|
||||||
|
|
||||||
|
def load_image_tags_yaml() -> YamlData:
|
||||||
|
if not PATHS.conditional_tags.is_file():
|
||||||
|
raise FileNotFoundError(
|
||||||
|
f"Conditional build image tags file not found: {PATHS.conditional_tags}"
|
||||||
|
)
|
||||||
|
|
||||||
|
with open(str(PATHS.conditional_tags), "r", encoding="utf-8") as f:
|
||||||
|
data = yaml.safe_load(f)
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return {"variables": {}}
|
||||||
|
if "variables" not in data:
|
||||||
|
data["variables"] = {}
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def get_current_tag_value(component: str) -> Optional[str]:
|
||||||
|
# If the CI job already set the tag, use it
|
||||||
|
tag_var = from_component_to_tag_var(component)
|
||||||
|
if os.getenv(tag_var):
|
||||||
|
logging.debug(f"Using tag from environment variable {tag_var}")
|
||||||
|
return os.getenv(tag_var)
|
||||||
|
|
||||||
|
# Otherwise, look in the YAML file, which normally occurs when updating tags locally
|
||||||
|
full_tag_var = from_component_to_build_tag(component)
|
||||||
|
data = load_image_tags_yaml()
|
||||||
|
variables = data.get("variables", {})
|
||||||
|
if not isinstance(variables, dict):
|
||||||
|
return None
|
||||||
|
logging.debug(f"Using tag from YAML file {full_tag_var}")
|
||||||
|
return variables.get(full_tag_var)
|
||||||
|
|
||||||
|
|
||||||
|
def update_image_tag_in_yaml(component: str, tag_value: str) -> None:
|
||||||
|
"""
|
||||||
|
Uses PyYAML to edit the YAML file at IMAGE_TAGS_FILE, setting the environment variable
|
||||||
|
for the given component. Maintains sorted keys.
|
||||||
|
"""
|
||||||
|
full_tag_var = from_component_to_build_tag(component)
|
||||||
|
data = load_image_tags_yaml()
|
||||||
|
|
||||||
|
# Ensure we have a variables dictionary
|
||||||
|
if "variables" not in data:
|
||||||
|
data["variables"] = {}
|
||||||
|
elif not isinstance(data["variables"], dict):
|
||||||
|
data["variables"] = {}
|
||||||
|
|
||||||
|
# Update the tag
|
||||||
|
data["variables"][full_tag_var] = tag_value
|
||||||
|
|
||||||
|
# Sort the variables
|
||||||
|
data["variables"] = dict(sorted(data["variables"].items()))
|
||||||
|
|
||||||
|
# Write back to file
|
||||||
|
with open(str(PATHS.conditional_tags), "w", encoding="utf-8") as f:
|
||||||
|
yaml.dump(data, f, sort_keys=False) # Don't sort top-level keys
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args():
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||||
|
description="Manage container image tags for CI builds with regex-based includes/excludes.",
|
||||||
|
epilog="""
|
||||||
|
Exit codes:
|
||||||
|
0 - Success
|
||||||
|
1 - Unhandled error in this script
|
||||||
|
2 - Tag mismatch when using --check
|
||||||
|
3 - Unbound variable error in build script
|
||||||
|
x - Build script failed with return code x
|
||||||
|
""",
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"--include",
|
||||||
|
"-i",
|
||||||
|
action="append",
|
||||||
|
default=[],
|
||||||
|
help="Full match regex pattern for components to include.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--exclude",
|
||||||
|
"-x",
|
||||||
|
action="append",
|
||||||
|
default=[],
|
||||||
|
help="Full match regex pattern for components to exclude.",
|
||||||
|
)
|
||||||
|
parser.add_argument("--all", action="store_true", help="Equivalent to --include '.*'")
|
||||||
|
parser.add_argument(
|
||||||
|
"--check",
|
||||||
|
"-c",
|
||||||
|
action="append",
|
||||||
|
default=[],
|
||||||
|
help="Check matching components instead of updating YAML. "
|
||||||
|
"If any component fails, exit with a non-zero exit code.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--list",
|
||||||
|
"-l",
|
||||||
|
action="store_true",
|
||||||
|
help="List all available components and exit.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"-v",
|
||||||
|
"--verbose",
|
||||||
|
action="count",
|
||||||
|
default=0,
|
||||||
|
help="Increase verbosity level (-v for info, -vv for debug)",
|
||||||
|
)
|
||||||
|
|
||||||
|
if len(sys.argv) == 1:
|
||||||
|
parser.print_help()
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
args = parse_args()
|
||||||
|
|
||||||
|
# Configure logging based on verbosity level
|
||||||
|
if args.verbose == 1:
|
||||||
|
log_level = logging.INFO
|
||||||
|
elif args.verbose == 2:
|
||||||
|
log_level = logging.DEBUG
|
||||||
|
else:
|
||||||
|
log_level = logging.WARNING
|
||||||
|
|
||||||
|
logging.basicConfig(level=log_level, format="%(levelname)s: %(message)s")
|
||||||
|
|
||||||
|
# 1) If checking, just run build scripts in check mode and propagate errors
|
||||||
|
if args.check:
|
||||||
|
tag_mismatch = False
|
||||||
|
for comp in args.check:
|
||||||
|
try:
|
||||||
|
if run_build_script(comp, check_only=True) is None:
|
||||||
|
# The tag is invalid
|
||||||
|
tag_mismatch = True
|
||||||
|
except SystemExit as e:
|
||||||
|
# Let custom exit codes propagate
|
||||||
|
raise e
|
||||||
|
except Exception as e:
|
||||||
|
logging.error(f"Internal error: {e}")
|
||||||
|
sys.exit(1)
|
||||||
|
# If any component failed, exit with code 1
|
||||||
|
if tag_mismatch:
|
||||||
|
sys.exit(2)
|
||||||
|
return
|
||||||
|
|
||||||
|
# Convert --all into a wildcard include
|
||||||
|
if args.all:
|
||||||
|
args.include.append(".*")
|
||||||
|
|
||||||
|
# 2) If --list, just show all discovered components
|
||||||
|
all_components = find_candidate_components()
|
||||||
|
if args.list:
|
||||||
|
print("Detected components:", ", ".join(all_components))
|
||||||
|
return
|
||||||
|
|
||||||
|
# 3) Filter components
|
||||||
|
final_components = filter_components(all_components, args.include, args.exclude)
|
||||||
|
|
||||||
|
if args.verbose:
|
||||||
|
logging.debug(f"Found components: {all_components}")
|
||||||
|
logging.debug(f"Filtered components: {final_components}")
|
||||||
|
|
||||||
|
for comp in final_components:
|
||||||
|
logging.info(f"Updating {comp}...")
|
||||||
|
new_tag = run_build_script(comp, check_only=False)
|
||||||
|
if new_tag is not None:
|
||||||
|
update_image_tag_in_yaml(comp, new_tag)
|
||||||
|
if args.verbose:
|
||||||
|
logging.debug(f"Updated {comp} with tag: {new_tag}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Executable
+217
@@ -0,0 +1,217 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# Copyright © 2022 Collabora Ltd.
|
||||||
|
# Authors:
|
||||||
|
# David Heidelberg <[email protected]>
|
||||||
|
#
|
||||||
|
# For the dependencies, see the requirements.txt
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
"""
|
||||||
|
Helper script to update traces checksums
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import bz2
|
||||||
|
import glob
|
||||||
|
import re
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from ruamel.yaml import YAML
|
||||||
|
from subprocess import check_output
|
||||||
|
from tomledit import Document
|
||||||
|
|
||||||
|
import gitlab
|
||||||
|
from gitlab_common import (get_gitlab_project, read_token, wait_for_pipeline,
|
||||||
|
get_gitlab_pipeline_from_url, TOKEN_DIR, get_token_from_default_dir)
|
||||||
|
from rich import print
|
||||||
|
|
||||||
|
|
||||||
|
DESCRIPTION_FILE = "export PIGLIT_REPLAY_DESCRIPTION_FILE=.*/install/(.*)$"
|
||||||
|
DEVICE_NAME = "(?:declare -x|export) PIGLIT_REPLAY_DEVICE_NAME='?([^']*)'?$"
|
||||||
|
|
||||||
|
def gather_job_results_yml(cur_job):
|
||||||
|
log: list[str] = cur_job.trace().decode().splitlines()
|
||||||
|
filename: str = ''
|
||||||
|
dev_name: str = ''
|
||||||
|
for logline in log:
|
||||||
|
desc_file = re.search(DESCRIPTION_FILE, logline)
|
||||||
|
device_name = re.search(DEVICE_NAME, logline)
|
||||||
|
if desc_file:
|
||||||
|
filename = desc_file.group(1)
|
||||||
|
if device_name:
|
||||||
|
dev_name = device_name.group(1)
|
||||||
|
|
||||||
|
if not filename or not dev_name:
|
||||||
|
print("[red]Couldn't find device name or YML file in the logs!")
|
||||||
|
return
|
||||||
|
|
||||||
|
print(f"👁 Found {dev_name} and file {filename}")
|
||||||
|
|
||||||
|
# find filename in Mesa source
|
||||||
|
traces_file = glob.glob('./**/' + filename, recursive=True)
|
||||||
|
# write into it
|
||||||
|
with open(traces_file[0], 'r', encoding='utf-8') as target_file:
|
||||||
|
yaml = YAML()
|
||||||
|
yaml.compact(seq_seq=False, seq_map=False)
|
||||||
|
yaml.version = 1,2
|
||||||
|
yaml.width = 2048 # do not break the text fields
|
||||||
|
yaml.default_flow_style = None
|
||||||
|
target = yaml.load(target_file)
|
||||||
|
|
||||||
|
# parse artifact
|
||||||
|
results_json_bz2 = cur_job.artifact("results/results.json.bz2")
|
||||||
|
results_json = bz2.decompress(results_json_bz2).decode("utf-8", errors="replace")
|
||||||
|
results = json.loads(results_json)
|
||||||
|
|
||||||
|
for _, value in results["tests"].items():
|
||||||
|
if (
|
||||||
|
not value['images'] or
|
||||||
|
not value['images'][0] or
|
||||||
|
"image_desc" not in value['images'][0]
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
|
||||||
|
trace: str = value['images'][0]['image_desc']
|
||||||
|
checksum: str = value['images'][0]['checksum_render']
|
||||||
|
|
||||||
|
if not checksum:
|
||||||
|
print(f"[red]{dev_name}: {trace}: checksum is missing! Crash?")
|
||||||
|
continue
|
||||||
|
|
||||||
|
if checksum == "error":
|
||||||
|
print(f"[red]{dev_name}: {trace}: crashed")
|
||||||
|
continue
|
||||||
|
|
||||||
|
if target['traces'][trace][dev_name].get('checksum') == checksum:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if "label" in target['traces'][trace][dev_name]:
|
||||||
|
print(
|
||||||
|
f"{dev_name}: {trace}: please verify that label "
|
||||||
|
f"[blue]{target['traces'][trace][dev_name]['label']}[/blue] "
|
||||||
|
"is still valid"
|
||||||
|
)
|
||||||
|
|
||||||
|
print(f"[green]{dev_name}: {trace}: checksum updated")
|
||||||
|
target['traces'][trace][dev_name]['checksum'] = checksum
|
||||||
|
|
||||||
|
with open(traces_file[0], 'w', encoding='utf-8') as target_file:
|
||||||
|
yaml.dump(target, target_file)
|
||||||
|
|
||||||
|
|
||||||
|
def update_toml_checksum(toml_file, trace_path, device, old_checksum, new_checksum):
|
||||||
|
with open(toml_file, 'r', encoding='utf-8') as f:
|
||||||
|
doc = Document.parse(f.read())
|
||||||
|
|
||||||
|
for table in doc["traces"]:
|
||||||
|
if table["path"].value != trace_path:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
table["devices"][device]["checksum"] = new_checksum
|
||||||
|
except KeyError:
|
||||||
|
return False
|
||||||
|
with open(toml_file, 'w', encoding='utf-8') as f:
|
||||||
|
f.write(doc.as_toml())
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def gather_job_results_toml(cur_job) -> None:
|
||||||
|
"""Process checksum-changes.json artifact (gpu-trace-perf replay format)"""
|
||||||
|
toml_files = glob.glob('./**/traces-*.toml', recursive=True)
|
||||||
|
|
||||||
|
changes_json = cur_job.artifact("results/checksum-changes.json")
|
||||||
|
changes = json.loads(changes_json.decode("utf-8"))
|
||||||
|
|
||||||
|
for change in changes:
|
||||||
|
device = change['device']
|
||||||
|
trace = change['trace']
|
||||||
|
old_checksum = change['old_checksum']
|
||||||
|
new_checksum = change['new_checksum']
|
||||||
|
|
||||||
|
if not new_checksum:
|
||||||
|
print(f"[red]{device}: {trace}: no new checksum (crash?)")
|
||||||
|
continue
|
||||||
|
|
||||||
|
updated = False
|
||||||
|
for toml_file in toml_files:
|
||||||
|
if update_toml_checksum(toml_file, trace, device, old_checksum, new_checksum):
|
||||||
|
print(f"[green]{device}: {trace}: checksum updated")
|
||||||
|
updated = True
|
||||||
|
break
|
||||||
|
|
||||||
|
if not updated:
|
||||||
|
print(f"[red]{device}: {trace}: not found in any traces TOML file")
|
||||||
|
|
||||||
|
|
||||||
|
def gather_results(
|
||||||
|
project,
|
||||||
|
pipeline,
|
||||||
|
) -> None:
|
||||||
|
"""Gather results"""
|
||||||
|
|
||||||
|
target_jobs_regex = re.compile(".*-traces(-restricted)?([:].*)?$")
|
||||||
|
|
||||||
|
for job in pipeline.jobs.list(all=True, sort="desc"):
|
||||||
|
if target_jobs_regex.match(job.name) and job.status == "failed":
|
||||||
|
cur_job = project.jobs.get(job.id)
|
||||||
|
# get variables
|
||||||
|
print(f"👁 {job.name}...")
|
||||||
|
try:
|
||||||
|
gather_job_results_toml(cur_job)
|
||||||
|
except Exception:
|
||||||
|
gather_job_results_yml(cur_job)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> None:
|
||||||
|
"""Parse args"""
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Tool to generate patch from checksums ",
|
||||||
|
epilog="Example: update_traces_checksum.py --rev $(git rev-parse HEAD) "
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--rev", metavar="revision", help="repository git revision", default='HEAD'
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--token",
|
||||||
|
metavar="token",
|
||||||
|
type=str,
|
||||||
|
default=get_token_from_default_dir(),
|
||||||
|
help="Use the provided GitLab token or token file, "
|
||||||
|
f"otherwise it's read from {TOKEN_DIR / 'gitlab-token'}",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--pipeline-url",
|
||||||
|
metavar="pipeline_url",
|
||||||
|
help="specify a pipeline url",
|
||||||
|
)
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
try:
|
||||||
|
args = parse_args()
|
||||||
|
|
||||||
|
token = read_token(args.token)
|
||||||
|
|
||||||
|
gl = gitlab.Gitlab(url="https://gitlab.freedesktop.org", private_token=token)
|
||||||
|
|
||||||
|
cur_project = get_gitlab_project(gl, "mesa")
|
||||||
|
|
||||||
|
if args.pipeline_url:
|
||||||
|
pipe, cur_project = get_gitlab_pipeline_from_url(gl, args.pipeline_url)
|
||||||
|
REV = pipe.sha
|
||||||
|
else:
|
||||||
|
if not args.rev:
|
||||||
|
print('error: the following arguments are required: --rev')
|
||||||
|
sys.exit(1)
|
||||||
|
sha = check_output(['git', 'rev-parse', args.rev]).decode('ascii').strip()
|
||||||
|
print(f"Revision: {sha}")
|
||||||
|
(pipe, cur_project) = wait_for_pipeline([cur_project], sha)
|
||||||
|
print(f"Pipeline: {pipe.web_url}")
|
||||||
|
gather_results(cur_project, pipe)
|
||||||
|
|
||||||
|
sys.exit()
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
sys.exit(1)
|
||||||
Executable
+10
@@ -0,0 +1,10 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
|
||||||
|
readonly this_dir
|
||||||
|
|
||||||
|
exec \
|
||||||
|
"$this_dir/../python-venv.sh" \
|
||||||
|
"$this_dir/requirements.txt" \
|
||||||
|
"$this_dir/update_traces_checksum.py" "$@"
|
||||||
Executable
+141
@@ -0,0 +1,141 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
def print_(args: argparse.Namespace, success: bool, message: str) -> None:
|
||||||
|
"""
|
||||||
|
Print function with extra coloring when supported and/or requested,
|
||||||
|
and with a "quiet" switch
|
||||||
|
"""
|
||||||
|
|
||||||
|
COLOR_SUCCESS = '\033[32m'
|
||||||
|
COLOR_FAILURE = '\033[31m'
|
||||||
|
COLOR_RESET = '\033[0m'
|
||||||
|
|
||||||
|
if args.quiet:
|
||||||
|
return
|
||||||
|
|
||||||
|
if args.color == 'auto':
|
||||||
|
use_colors = sys.stdout.isatty()
|
||||||
|
else:
|
||||||
|
use_colors = args.color == 'always'
|
||||||
|
|
||||||
|
s = ''
|
||||||
|
if use_colors:
|
||||||
|
if success:
|
||||||
|
s += COLOR_SUCCESS
|
||||||
|
else:
|
||||||
|
s += COLOR_FAILURE
|
||||||
|
|
||||||
|
s += message
|
||||||
|
|
||||||
|
if use_colors:
|
||||||
|
s += COLOR_RESET
|
||||||
|
|
||||||
|
print(s)
|
||||||
|
|
||||||
|
|
||||||
|
def is_commit_valid(commit: str) -> bool:
|
||||||
|
ret = subprocess.call(['git', 'cat-file', '-e', commit],
|
||||||
|
stdout=subprocess.DEVNULL,
|
||||||
|
stderr=subprocess.DEVNULL)
|
||||||
|
return ret == 0
|
||||||
|
|
||||||
|
|
||||||
|
def branch_has_commit(upstream_branch: str, commit: str) -> bool:
|
||||||
|
"""
|
||||||
|
Returns True if the commit is actually present in the branch
|
||||||
|
"""
|
||||||
|
ret = subprocess.call(['git', 'merge-base', '--is-ancestor',
|
||||||
|
commit, upstream_branch],
|
||||||
|
stdout=subprocess.DEVNULL,
|
||||||
|
stderr=subprocess.DEVNULL)
|
||||||
|
return ret == 0
|
||||||
|
|
||||||
|
|
||||||
|
def branch_has_backport_of_commit(upstream_branch: str, commit: str) -> str:
|
||||||
|
"""
|
||||||
|
Returns the commit hash if the commit has been backported to the branch,
|
||||||
|
or an empty string if is hasn't
|
||||||
|
"""
|
||||||
|
upstream, _ = upstream_branch.split('/', 1)
|
||||||
|
|
||||||
|
out = subprocess.check_output(['git', 'log', '--format=%H',
|
||||||
|
upstream + '..' + upstream_branch,
|
||||||
|
'--grep', 'cherry picked from commit ' + commit],
|
||||||
|
stderr=subprocess.DEVNULL)
|
||||||
|
return out.decode().strip()
|
||||||
|
|
||||||
|
|
||||||
|
def canonicalize_commit(commit: str) -> str:
|
||||||
|
"""
|
||||||
|
Takes a commit-ish and returns a commit sha1 if the commit exists
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Make sure input is valid first
|
||||||
|
if not is_commit_valid(commit):
|
||||||
|
raise argparse.ArgumentTypeError('invalid commit identifier: ' + commit)
|
||||||
|
|
||||||
|
out = subprocess.check_output(['git', 'rev-parse', commit],
|
||||||
|
stderr=subprocess.DEVNULL)
|
||||||
|
return out.decode().strip()
|
||||||
|
|
||||||
|
|
||||||
|
def validate_branch(branch: str) -> str:
|
||||||
|
if '/' not in branch:
|
||||||
|
raise argparse.ArgumentTypeError('must be in the form `remote/branch`')
|
||||||
|
|
||||||
|
out = subprocess.check_output(['git', 'remote', '--verbose'],
|
||||||
|
stderr=subprocess.DEVNULL)
|
||||||
|
remotes = out.decode().splitlines()
|
||||||
|
upstream, _ = branch.split('/', 1)
|
||||||
|
valid_remote = False
|
||||||
|
for line in remotes:
|
||||||
|
if line.startswith(upstream + '\t'):
|
||||||
|
valid_remote = True
|
||||||
|
|
||||||
|
if not valid_remote:
|
||||||
|
raise argparse.ArgumentTypeError('Invalid remote: ' + upstream)
|
||||||
|
|
||||||
|
if not is_commit_valid(branch):
|
||||||
|
raise argparse.ArgumentTypeError('Invalid branch: ' + branch)
|
||||||
|
|
||||||
|
return branch
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
parser = argparse.ArgumentParser(description="""
|
||||||
|
Returns 0 if the commit is present in the branch,
|
||||||
|
1 if it's not,
|
||||||
|
and 2 if it couldn't be determined (eg. invalid commit)
|
||||||
|
""")
|
||||||
|
parser.add_argument('commit',
|
||||||
|
type=canonicalize_commit,
|
||||||
|
help='commit sha1')
|
||||||
|
parser.add_argument('branch',
|
||||||
|
type=validate_branch,
|
||||||
|
help='branch to check, in the form `remote/branch`')
|
||||||
|
parser.add_argument('--quiet',
|
||||||
|
action='store_true',
|
||||||
|
help='suppress all output; exit code can still be used')
|
||||||
|
parser.add_argument('--color',
|
||||||
|
choices=['auto', 'always', 'never'],
|
||||||
|
default='auto',
|
||||||
|
help='colorize output (default: true if stdout is a terminal)')
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if branch_has_commit(args.branch, args.commit):
|
||||||
|
print_(args, True, 'Commit ' + args.commit + ' is in branch ' + args.branch)
|
||||||
|
exit(0)
|
||||||
|
|
||||||
|
backport = branch_has_backport_of_commit(args.branch, args.commit)
|
||||||
|
if backport:
|
||||||
|
print_(args, True,
|
||||||
|
'Commit ' + args.commit + ' was backported to branch ' + args.branch + ' as commit ' + backport)
|
||||||
|
exit(0)
|
||||||
|
|
||||||
|
print_(args, False, 'Commit ' + args.commit + ' is NOT in branch ' + args.branch)
|
||||||
|
exit(1)
|
||||||
@@ -0,0 +1,118 @@
|
|||||||
|
import argparse
|
||||||
|
import pytest # type: ignore
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
from .commit_in_branch import (
|
||||||
|
is_commit_valid,
|
||||||
|
branch_has_commit,
|
||||||
|
branch_has_backport_of_commit,
|
||||||
|
canonicalize_commit,
|
||||||
|
validate_branch,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def get_upstream() -> str:
|
||||||
|
# Let's assume main is bound to the upstream remote and not a fork
|
||||||
|
out = subprocess.check_output(['git', 'for-each-ref',
|
||||||
|
'--format=%(upstream)',
|
||||||
|
'refs/heads/main'],
|
||||||
|
stderr=subprocess.DEVNULL)
|
||||||
|
return out.decode().strip().split('/')[2]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'commit, expected',
|
||||||
|
[
|
||||||
|
('20.1-branchpoint', True),
|
||||||
|
('main', True),
|
||||||
|
('e58a10af640ba58b6001f5c5ad750b782547da76', True),
|
||||||
|
('d043d24654c851f0be57dbbf48274b5373dea42b', True),
|
||||||
|
('dd2bd68fa69124c86cd008b256d06f44fab8e6cd', True),
|
||||||
|
('0000000000000000000000000000000000000000', False),
|
||||||
|
('not-even-a-valid-commit-format', False),
|
||||||
|
])
|
||||||
|
def test_canonicalize_commit(commit: str, expected: bool) -> None:
|
||||||
|
if expected:
|
||||||
|
assert canonicalize_commit(commit)
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
assert canonicalize_commit(commit)
|
||||||
|
except argparse.ArgumentTypeError:
|
||||||
|
return
|
||||||
|
assert False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'commit, expected',
|
||||||
|
[
|
||||||
|
(get_upstream() + '/20.1', True),
|
||||||
|
(get_upstream() + '/staging/20.1', True),
|
||||||
|
(get_upstream() + '/main', True),
|
||||||
|
('20.1', False),
|
||||||
|
('main', False),
|
||||||
|
('e58a10af640ba58b6001f5c5ad750b782547da76', False),
|
||||||
|
('d043d24654c851f0be57dbbf48274b5373dea42b', False),
|
||||||
|
('dd2bd68fa69124c86cd008b256d06f44fab8e6cd', False),
|
||||||
|
('0000000000000000000000000000000000000000', False),
|
||||||
|
('not-even-a-valid-commit-format', False),
|
||||||
|
])
|
||||||
|
def test_validate_branch(commit: str, expected: bool) -> None:
|
||||||
|
if expected:
|
||||||
|
assert validate_branch(commit)
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
assert validate_branch(commit)
|
||||||
|
except argparse.ArgumentTypeError:
|
||||||
|
return
|
||||||
|
assert False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'commit, expected',
|
||||||
|
[
|
||||||
|
('main', True),
|
||||||
|
('20.1-branchpoint', True),
|
||||||
|
('20.1', False),
|
||||||
|
(get_upstream() + '/20.1', True),
|
||||||
|
(get_upstream() + '/staging/20.1', True),
|
||||||
|
('e58a10af640ba58b6001f5c5ad750b782547da76', True),
|
||||||
|
('d043d24654c851f0be57dbbf48274b5373dea42b', True),
|
||||||
|
('dd2bd68fa69124c86cd008b256d06f44fab8e6cd', True),
|
||||||
|
('0000000000000000000000000000000000000000', False),
|
||||||
|
('not-even-a-valid-commit-format', False),
|
||||||
|
])
|
||||||
|
def test_is_commit_valid(commit: str, expected: bool) -> None:
|
||||||
|
assert is_commit_valid(commit) == expected
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'branch, commit, expected',
|
||||||
|
[
|
||||||
|
(get_upstream() + '/20.1', '20.1-branchpoint', True),
|
||||||
|
(get_upstream() + '/20.1', '20.0', False),
|
||||||
|
(get_upstream() + '/20.1', 'main', False),
|
||||||
|
(get_upstream() + '/20.1', 'e58a10af640ba58b6001f5c5ad750b782547da76', True),
|
||||||
|
(get_upstream() + '/20.1', 'd043d24654c851f0be57dbbf48274b5373dea42b', True),
|
||||||
|
(get_upstream() + '/staging/20.1', 'd043d24654c851f0be57dbbf48274b5373dea42b', True),
|
||||||
|
(get_upstream() + '/20.1', 'dd2bd68fa69124c86cd008b256d06f44fab8e6cd', False),
|
||||||
|
(get_upstream() + '/main', 'dd2bd68fa69124c86cd008b256d06f44fab8e6cd', True),
|
||||||
|
(get_upstream() + '/20.0', 'd043d24654c851f0be57dbbf48274b5373dea42b', False),
|
||||||
|
])
|
||||||
|
def test_branch_has_commit(branch: str, commit: str, expected: bool) -> None:
|
||||||
|
assert branch_has_commit(branch, commit) == expected
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'branch, commit, expected',
|
||||||
|
[
|
||||||
|
(get_upstream() + '/20.1', 'dd2bd68fa69124c86cd008b256d06f44fab8e6cd', 'd043d24654c851f0be57dbbf48274b5373dea42b'),
|
||||||
|
(get_upstream() + '/staging/20.1', 'dd2bd68fa69124c86cd008b256d06f44fab8e6cd', 'd043d24654c851f0be57dbbf48274b5373dea42b'),
|
||||||
|
(get_upstream() + '/20.1', '20.1-branchpoint', ''),
|
||||||
|
(get_upstream() + '/20.1', '20.0', ''),
|
||||||
|
(get_upstream() + '/20.1', '20.2', 'abac4859618e02aea00f705b841a7c5c5007ad1a'),
|
||||||
|
(get_upstream() + '/20.1', 'main', ''),
|
||||||
|
(get_upstream() + '/20.1', 'd043d24654c851f0be57dbbf48274b5373dea42b', ''),
|
||||||
|
(get_upstream() + '/20.0', 'dd2bd68fa69124c86cd008b256d06f44fab8e6cd', '8cd4f57381cefe69019a3282d457d5bda3644030'),
|
||||||
|
])
|
||||||
|
def test_branch_has_backport_of_commit(branch: str, commit: str, expected: bool) -> None:
|
||||||
|
assert branch_has_backport_of_commit(branch, commit) == expected
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
#
|
||||||
|
# Copyright 2024 Autodesk, Inc.
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
|
#
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
from bisect import bisect_left, bisect_right
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Instruction:
|
||||||
|
address: int
|
||||||
|
assembly: str
|
||||||
|
samples: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
def mapping_address_key(mapping: tuple[int, int, str]):
|
||||||
|
return mapping[0]
|
||||||
|
|
||||||
|
|
||||||
|
def instruction_address_key(instruction: Instruction):
|
||||||
|
return instruction.address
|
||||||
|
|
||||||
|
|
||||||
|
def parse_mappings(map_file_path: Path):
|
||||||
|
mappings: list[tuple[int, int, str]] = []
|
||||||
|
with open(map_file_path) as map_file:
|
||||||
|
for mapping in map_file:
|
||||||
|
address_hex, size_hex, name = mapping.split(' ')
|
||||||
|
address = int(address_hex, base=16)
|
||||||
|
mappings.append((address, address + int(size_hex, base=16), name.strip()))
|
||||||
|
|
||||||
|
mappings.sort(key=mapping_address_key)
|
||||||
|
return mappings
|
||||||
|
|
||||||
|
|
||||||
|
def parse_traces(trace_file_path: Path):
|
||||||
|
pattern = re.compile(r'((?:[^;]+;)*?[^;]+) (\d+)\n')
|
||||||
|
|
||||||
|
traces: list[tuple[list[str], int]] = []
|
||||||
|
with open(trace_file_path) as trace_file:
|
||||||
|
for trace in trace_file:
|
||||||
|
match = pattern.fullmatch(trace)
|
||||||
|
traces.append((match.group(1).split(';'), int(match.group(2))))
|
||||||
|
|
||||||
|
return traces
|
||||||
|
|
||||||
|
|
||||||
|
def parse_asm(asm_file_path: Path):
|
||||||
|
symbol_pattern = re.compile(r'(\w+) ([0-9a-fA-F]+):\n')
|
||||||
|
instruction_pattern = re.compile(r' *([0-9a-fA-F]+):\t(.*?)\n')
|
||||||
|
|
||||||
|
asm: dict[tuple[int, str], list[Instruction]] = {}
|
||||||
|
with open(asm_file_path) as asm_file:
|
||||||
|
current_instructions = None
|
||||||
|
for line in asm_file:
|
||||||
|
if match := symbol_pattern.fullmatch(line):
|
||||||
|
symbol = (int(match.group(2), base=16), match.group(1))
|
||||||
|
current_instructions = asm[symbol] = []
|
||||||
|
elif match := instruction_pattern.fullmatch(line):
|
||||||
|
current_instructions.append(Instruction(int(match.group(1), base=16), match.group(2)))
|
||||||
|
|
||||||
|
return asm
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description='Map LLVMPipe JIT addresses in FlameGraph style '
|
||||||
|
'collapsed stack traces to their symbol name. Also optionally '
|
||||||
|
'annotate JIT assembly dumps with sample counts.')
|
||||||
|
parser.add_argument('jit_symbol_map', type=Path, help='JIT symbol map from LLVMPipe')
|
||||||
|
parser.add_argument('collapsed_traces', type=Path)
|
||||||
|
parser.add_argument('-a', '--asm', type=Path, nargs='?', const='', metavar='asm_path',
|
||||||
|
help='JIT assembly dump from LLVMPipe. Defaults to "<jit_symbol_map>.asm"')
|
||||||
|
parser.add_argument('-o', '--out', type=Path, metavar='out_path')
|
||||||
|
arguments = parser.parse_args()
|
||||||
|
|
||||||
|
mappings = parse_mappings(arguments.jit_symbol_map)
|
||||||
|
traces = parse_traces(arguments.collapsed_traces)
|
||||||
|
|
||||||
|
asm = {}
|
||||||
|
asm_file_path: Path | None = arguments.asm
|
||||||
|
if asm_file_path:
|
||||||
|
if len(asm_file_path.parts) <= 0:
|
||||||
|
asm_file_path = Path(str(arguments.jit_symbol_map) + '.asm')
|
||||||
|
if asm_file_path.exists():
|
||||||
|
asm = parse_asm(asm_file_path)
|
||||||
|
else:
|
||||||
|
asm = parse_asm(asm_file_path)
|
||||||
|
|
||||||
|
merged_traces: dict[str, int] = {}
|
||||||
|
for stack, count in traces:
|
||||||
|
for i, function in enumerate(stack):
|
||||||
|
if not function.startswith('0x'):
|
||||||
|
continue
|
||||||
|
|
||||||
|
address = int(function, base=16)
|
||||||
|
mapping = mappings[bisect_right(mappings, address, key=mapping_address_key) - 1]
|
||||||
|
if address < mapping[0] or address >= mapping[1]:
|
||||||
|
continue
|
||||||
|
|
||||||
|
stack[i] = f'lp`{mapping[2]}@{mapping[0]:x}'
|
||||||
|
|
||||||
|
symbol = (mapping[0], mapping[2])
|
||||||
|
if symbol in asm:
|
||||||
|
instructions = asm[symbol]
|
||||||
|
instruction_address = address - symbol[0]
|
||||||
|
index = bisect_left(instructions, instruction_address, key=instruction_address_key)
|
||||||
|
if index < len(instructions) and instructions[index].address == instruction_address:
|
||||||
|
instructions[index].samples += count
|
||||||
|
|
||||||
|
stack_key = ';'.join(stack)
|
||||||
|
if stack_key in merged_traces:
|
||||||
|
merged_traces[stack_key] += count
|
||||||
|
else:
|
||||||
|
merged_traces[stack_key] = count
|
||||||
|
|
||||||
|
out_file_path: Path | None = arguments.out
|
||||||
|
if not out_file_path:
|
||||||
|
out_file_path = arguments.collapsed_traces.with_stem(f'{arguments.collapsed_traces.stem}_mapped')
|
||||||
|
with open(out_file_path, 'w') as out:
|
||||||
|
for t, c in merged_traces.items():
|
||||||
|
print(f'{t} {c}', file=out)
|
||||||
|
|
||||||
|
if asm:
|
||||||
|
annotated_asm_file_path = asm_file_path.with_stem(f'{asm_file_path.stem}_annotated')
|
||||||
|
with open(annotated_asm_file_path, 'w') as out:
|
||||||
|
for symbol, instructions in asm.items():
|
||||||
|
print(f'{symbol[1]}: ;{symbol[0]:x}', file=out)
|
||||||
|
for instruction in instructions:
|
||||||
|
print(f'\t{instruction.assembly}', end='', file=out)
|
||||||
|
if instruction.samples:
|
||||||
|
print(f' ;s {instruction.samples}', file=out)
|
||||||
|
else:
|
||||||
|
print(file=out)
|
||||||
|
print(file=out)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
Executable
+253
@@ -0,0 +1,253 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
# Copyright © 2021 Intel Corporation
|
||||||
|
|
||||||
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
# of this software and associated documentation files (the "Software"), to deal
|
||||||
|
# in the Software without restriction, including without limitation the rights
|
||||||
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
# copies of the Software, and to permit persons to whom the Software is
|
||||||
|
# furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
# The above copyright notice and this permission notice shall be included in
|
||||||
|
# all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
# SOFTWARE.
|
||||||
|
|
||||||
|
"""Helper script for manipulating the release calendar."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
import argparse
|
||||||
|
import csv
|
||||||
|
import contextlib
|
||||||
|
import datetime
|
||||||
|
import pathlib
|
||||||
|
import subprocess
|
||||||
|
import typing
|
||||||
|
|
||||||
|
if typing.TYPE_CHECKING:
|
||||||
|
import _csv
|
||||||
|
from typing_extensions import Protocol
|
||||||
|
|
||||||
|
class RCArguments(Protocol):
|
||||||
|
"""Typing information for release-candidate command arguments."""
|
||||||
|
|
||||||
|
manager: str
|
||||||
|
|
||||||
|
class FinalArguments(Protocol):
|
||||||
|
"""Typing information for release command arguments."""
|
||||||
|
|
||||||
|
series: str
|
||||||
|
manager: str
|
||||||
|
zero_released: bool
|
||||||
|
|
||||||
|
class ExtendArguments(Protocol):
|
||||||
|
"""Typing information for extend command arguments."""
|
||||||
|
|
||||||
|
series: str
|
||||||
|
count: int
|
||||||
|
|
||||||
|
|
||||||
|
CalendarRowType = typing.Tuple[typing.Optional[str], str, str, str, typing.Optional[str]]
|
||||||
|
|
||||||
|
|
||||||
|
_ROOT = pathlib.Path(__file__).parent.parent
|
||||||
|
CALENDAR_CSV = _ROOT / 'docs' / 'release-calendar.csv'
|
||||||
|
VERSION = _ROOT / 'VERSION'
|
||||||
|
LAST_RELEASE = 'This is the last planned release of the {}.x series.'
|
||||||
|
OR_FINAL = 'Or {}.0 final.'
|
||||||
|
|
||||||
|
|
||||||
|
def read_calendar() -> typing.List[CalendarRowType]:
|
||||||
|
"""Read the calendar and return a list of it's rows."""
|
||||||
|
with CALENDAR_CSV.open('r') as f:
|
||||||
|
return [typing.cast('CalendarRowType', tuple(r)) for r in csv.reader(f)]
|
||||||
|
|
||||||
|
|
||||||
|
def commit(message: str) -> None:
|
||||||
|
"""Commit the changes the the release-calendar.csv file."""
|
||||||
|
subprocess.run(['git', 'commit', str(CALENDAR_CSV), '--message', message])
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def _calculate_release_start(major: str, minor: str) -> datetime.date:
|
||||||
|
"""Calculate the start of the release for release candidates.
|
||||||
|
|
||||||
|
This is quarterly, on the second wednesday, in January, April, July, and October.
|
||||||
|
"""
|
||||||
|
quarter = datetime.date.fromisoformat(f'20{major}-0{[1, 4, 7, 10][int(minor)]}-01')
|
||||||
|
|
||||||
|
# Wednesday is 3
|
||||||
|
day = quarter.isoweekday()
|
||||||
|
if day > 3:
|
||||||
|
# this will walk back into the previous month, it's much simpler to
|
||||||
|
# duplicate the 14 than handle the calculations for the month and year
|
||||||
|
# changing.
|
||||||
|
return quarter.replace(day=quarter.day - day + 3 + 14)
|
||||||
|
elif day < 3:
|
||||||
|
quarter = quarter.replace(day=quarter.day + 3 - day)
|
||||||
|
return quarter.replace(day=quarter.day + 14)
|
||||||
|
|
||||||
|
|
||||||
|
def release_candidate(args: RCArguments) -> None:
|
||||||
|
"""Add release candidate entries."""
|
||||||
|
with VERSION.open('r') as f:
|
||||||
|
version = f.read().rstrip('-devel')
|
||||||
|
major, minor, _ = version.split('.')
|
||||||
|
date = _calculate_release_start(major, minor)
|
||||||
|
|
||||||
|
data = read_calendar()
|
||||||
|
|
||||||
|
with CALENDAR_CSV.open('w', newline='') as f:
|
||||||
|
writer = csv.writer(f)
|
||||||
|
writer.writerows(data)
|
||||||
|
|
||||||
|
writer.writerow([f'{major}.{minor}', date.isoformat(), f'{major}.{minor}.0-rc1', args.manager])
|
||||||
|
for row in range(2, 4):
|
||||||
|
date = date + datetime.timedelta(days=7)
|
||||||
|
writer.writerow([None, date.isoformat(), f'{major}.{minor}.0-rc{row}', args.manager])
|
||||||
|
date = date + datetime.timedelta(days=7)
|
||||||
|
writer.writerow([None, date.isoformat(), f'{major}.{minor}.0-rc4', args.manager, OR_FINAL.format(f'{major}.{minor}')])
|
||||||
|
|
||||||
|
commit(f'docs: Add calendar entries for {major}.{minor} release candidates.')
|
||||||
|
|
||||||
|
|
||||||
|
def _calculate_next_release_date(next_is_zero: bool) -> datetime.date:
|
||||||
|
"""Calculate the date of the next release.
|
||||||
|
|
||||||
|
If the next is .0, we have the release in seven days, if the next is .1,
|
||||||
|
then it's in 14
|
||||||
|
"""
|
||||||
|
date = datetime.date.today()
|
||||||
|
day = date.isoweekday()
|
||||||
|
if day < 3:
|
||||||
|
delta = 3 - day
|
||||||
|
elif day > 3:
|
||||||
|
# this will walk back into the previous month, it's much simpler to
|
||||||
|
# duplicate the 14 than handle the calculations for the month and year
|
||||||
|
# changing.
|
||||||
|
delta = (3 - day)
|
||||||
|
else:
|
||||||
|
delta = 0
|
||||||
|
delta += 7
|
||||||
|
if not next_is_zero:
|
||||||
|
delta += 7
|
||||||
|
return date + datetime.timedelta(days=delta)
|
||||||
|
|
||||||
|
|
||||||
|
def final_release(args: FinalArguments) -> None:
|
||||||
|
"""Add final release entries."""
|
||||||
|
data = read_calendar()
|
||||||
|
date = _calculate_next_release_date(not args.zero_released)
|
||||||
|
|
||||||
|
with CALENDAR_CSV.open('w', newline='') as f:
|
||||||
|
writer = csv.writer(f)
|
||||||
|
writer.writerows(data)
|
||||||
|
|
||||||
|
base = 1 if args.zero_released else 0
|
||||||
|
|
||||||
|
writer.writerow([args.series, date.isoformat(), f'{args.series}.{base}', args.manager])
|
||||||
|
for row in range(base + 1, 3):
|
||||||
|
date = date + datetime.timedelta(days=14)
|
||||||
|
writer.writerow([None, date.isoformat(), f'{args.series}.{row}', args.manager])
|
||||||
|
date = date + datetime.timedelta(days=14)
|
||||||
|
writer.writerow([None, date.isoformat(), f'{args.series}.3', args.manager, LAST_RELEASE.format(args.series)])
|
||||||
|
|
||||||
|
commit(f'docs: Add calendar entries for {args.series} release.')
|
||||||
|
|
||||||
|
|
||||||
|
def extend(args: ExtendArguments) -> None:
|
||||||
|
"""Extend a release."""
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def write_existing(writer: _csv._writer, current: typing.List[CalendarRowType]) -> typing.Iterator[CalendarRowType]:
|
||||||
|
"""Write the orinal file, yield to insert new entries.
|
||||||
|
|
||||||
|
This is a bit clever, basically what happens it writes out the
|
||||||
|
original csv file until it reaches the start of the release after the
|
||||||
|
one we're appending, then it yields the last row. When control is
|
||||||
|
returned it writes out the rest of the original calendar data.
|
||||||
|
"""
|
||||||
|
last_row: typing.Optional[CalendarRowType] = None
|
||||||
|
in_wanted = False
|
||||||
|
for row in current:
|
||||||
|
if in_wanted and row[0]:
|
||||||
|
in_wanted = False
|
||||||
|
assert last_row is not None
|
||||||
|
yield last_row
|
||||||
|
if row[0] == args.series:
|
||||||
|
in_wanted = True
|
||||||
|
if in_wanted and len(row) >= 5 and row[4] in {LAST_RELEASE.format(args.series), OR_FINAL.format(args.series)}:
|
||||||
|
# If this was the last planned release and we're adding more,
|
||||||
|
# then we need to remove that message and add it elsewhere
|
||||||
|
r = list(row)
|
||||||
|
r[4] = None
|
||||||
|
# Mypy can't figure this out…
|
||||||
|
row = typing.cast('CalendarRowType', tuple(r))
|
||||||
|
last_row = row
|
||||||
|
writer.writerow(row)
|
||||||
|
# If this is the only entry we can hit a case where the contextmanager
|
||||||
|
# hasn't yielded
|
||||||
|
if in_wanted:
|
||||||
|
yield row
|
||||||
|
|
||||||
|
current = read_calendar()
|
||||||
|
|
||||||
|
with CALENDAR_CSV.open('w', newline='') as f:
|
||||||
|
writer = csv.writer(f)
|
||||||
|
with write_existing(writer, current) as row:
|
||||||
|
# Get rid of -rcX as well
|
||||||
|
if '-rc' in row[2]:
|
||||||
|
first_point = int(row[2].split('rc')[-1]) + 1
|
||||||
|
template = '{}.0-rc{}'
|
||||||
|
days = 7
|
||||||
|
else:
|
||||||
|
first_point = int(row[2].split('-')[0].split('.')[-1]) + 1
|
||||||
|
template = '{}.{}'
|
||||||
|
days = 14
|
||||||
|
|
||||||
|
date = datetime.date.fromisoformat(row[1])
|
||||||
|
for i in range(first_point, first_point + args.count):
|
||||||
|
date = date + datetime.timedelta(days=days)
|
||||||
|
r = [None, date.isoformat(), template.format(args.series, i), row[3], None]
|
||||||
|
if i == first_point + args.count - 1:
|
||||||
|
if days == 14:
|
||||||
|
r[4] = LAST_RELEASE.format(args.series)
|
||||||
|
else:
|
||||||
|
r[4] = OR_FINAL.format(args.series)
|
||||||
|
writer.writerow(r)
|
||||||
|
|
||||||
|
commit(f'docs: Extend calendar entries for {args.series} by {args.count} releases.')
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
sub = parser.add_subparsers()
|
||||||
|
|
||||||
|
rc = sub.add_parser('release-candidate', aliases=['rc'], help='Generate calendar entries for a release candidate.')
|
||||||
|
rc.add_argument('manager', help="the name of the person managing the release.")
|
||||||
|
rc.set_defaults(func=release_candidate)
|
||||||
|
|
||||||
|
fr = sub.add_parser('release', help='Generate calendar entries for a final release.')
|
||||||
|
fr.add_argument('manager', help="the name of the person managing the release.")
|
||||||
|
fr.add_argument('series', help='The series to extend, such as "29.3" or "30.0".')
|
||||||
|
fr.add_argument('--zero-released', action='store_true', help='The .0 release was today, the next release is .1')
|
||||||
|
fr.set_defaults(func=final_release)
|
||||||
|
|
||||||
|
ex = sub.add_parser('extend', help='Generate additional entries for a release.')
|
||||||
|
ex.add_argument('series', help='The series to extend, such as "29.3" or "30.0".')
|
||||||
|
ex.add_argument('count', type=int, help='The number of new entries to add.')
|
||||||
|
ex.set_defaults(func=extend)
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
args.func(args)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,319 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
# Copyright © 2021 Intel Corporation
|
||||||
|
|
||||||
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
# of this software and associated documentation files (the "Software"), to deal
|
||||||
|
# in the Software without restriction, including without limitation the rights
|
||||||
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
# copies of the Software, and to permit persons to whom the Software is
|
||||||
|
# furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
# The above copyright notice and this permission notice shall be included in
|
||||||
|
# all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
# SOFTWARE.
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
from unittest import mock
|
||||||
|
import argparse
|
||||||
|
import csv
|
||||||
|
import contextlib
|
||||||
|
import datetime
|
||||||
|
import tempfile
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import typing
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from . import gen_calendar_entries
|
||||||
|
|
||||||
|
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def mock_csv(data: typing.List[gen_calendar_entries.CalendarRowType]) -> typing.Iterator[None]:
|
||||||
|
"""Replace the actual CSV data with our test data."""
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
c = os.path.join(d, 'calendar.csv')
|
||||||
|
with open(c, 'w') as f:
|
||||||
|
writer = csv.writer(f)
|
||||||
|
writer.writerows(data)
|
||||||
|
|
||||||
|
with mock.patch('bin.gen_calendar_entries.CALENDAR_CSV', pathlib.Path(c)):
|
||||||
|
yield
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True, scope='module')
|
||||||
|
def disable_git_commits() -> None:
|
||||||
|
"""Mock out the commit function so no git commits are made during testing."""
|
||||||
|
with mock.patch('bin.gen_calendar_entries.commit', mock.Mock()):
|
||||||
|
yield
|
||||||
|
|
||||||
|
|
||||||
|
class TestReleaseStart:
|
||||||
|
|
||||||
|
def test_first_is_wednesday(self) -> None:
|
||||||
|
d = gen_calendar_entries._calculate_release_start('20', '0')
|
||||||
|
assert d.day == 15
|
||||||
|
assert d.month == 1
|
||||||
|
assert d.year == 2020
|
||||||
|
|
||||||
|
def test_first_is_before_wednesday(self) -> None:
|
||||||
|
d = gen_calendar_entries._calculate_release_start('19', '0')
|
||||||
|
assert d.day == 16
|
||||||
|
assert d.month == 1
|
||||||
|
assert d.year == 2019
|
||||||
|
|
||||||
|
def test_first_is_after_wednesday(self) -> None:
|
||||||
|
d = gen_calendar_entries._calculate_release_start('21', '0')
|
||||||
|
assert d.day == 13
|
||||||
|
assert d.month == 1
|
||||||
|
assert d.year == 2021
|
||||||
|
|
||||||
|
|
||||||
|
class TestNextReleaseDate:
|
||||||
|
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def _patch_date(date: datetime.date) -> typing.Iterator[None]:
|
||||||
|
mdate = mock.Mock()
|
||||||
|
mdate.today = mock.Mock(return_value=date)
|
||||||
|
with mock.patch('bin.gen_calendar_entries.datetime.date', mdate):
|
||||||
|
yield
|
||||||
|
|
||||||
|
class TestIsWeds:
|
||||||
|
|
||||||
|
@pytest.fixture(scope='class', autouse=True)
|
||||||
|
def data(self) -> None:
|
||||||
|
date = datetime.date(2021, 1, 6)
|
||||||
|
with TestNextReleaseDate._patch_date(date):
|
||||||
|
yield
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'is_zero, expected',
|
||||||
|
[
|
||||||
|
(True, 13),
|
||||||
|
(False, 20),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test(self, is_zero: bool, expected: int) -> None:
|
||||||
|
date = gen_calendar_entries._calculate_next_release_date(is_zero)
|
||||||
|
assert date.day == expected
|
||||||
|
|
||||||
|
class TestBeforeWeds:
|
||||||
|
|
||||||
|
@pytest.fixture(scope='class', autouse=True)
|
||||||
|
def data(self) -> None:
|
||||||
|
date = datetime.date(2021, 1, 5)
|
||||||
|
with TestNextReleaseDate._patch_date(date):
|
||||||
|
yield
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'is_zero, expected',
|
||||||
|
[
|
||||||
|
(True, 13),
|
||||||
|
(False, 20),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test(self, is_zero: bool, expected: int) -> None:
|
||||||
|
date = gen_calendar_entries._calculate_next_release_date(is_zero)
|
||||||
|
assert date.day == expected
|
||||||
|
|
||||||
|
class TestAfterWeds:
|
||||||
|
|
||||||
|
@pytest.fixture(scope='class', autouse=True)
|
||||||
|
def data(self) -> None:
|
||||||
|
date = datetime.date(2021, 1, 8)
|
||||||
|
with TestNextReleaseDate._patch_date(date):
|
||||||
|
yield
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'is_zero, expected',
|
||||||
|
[
|
||||||
|
(True, 13),
|
||||||
|
(False, 20),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test(self, is_zero: bool, expected: int) -> None:
|
||||||
|
date = gen_calendar_entries._calculate_next_release_date(is_zero)
|
||||||
|
assert date.day == expected
|
||||||
|
|
||||||
|
|
||||||
|
class TestRC:
|
||||||
|
|
||||||
|
ORIGINAL_DATA = [
|
||||||
|
('20.3', '2021-01-13', '20.3.3', 'Dylan Baker', ''),
|
||||||
|
('', '2021-01-27', '20.3.4', 'Dylan Baker', 'Last planned release of the 20.3.x series'),
|
||||||
|
]
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True, scope='class')
|
||||||
|
def mock_version(self) -> None:
|
||||||
|
"""Keep the version set at a specific value."""
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
v = os.path.join(d, 'version')
|
||||||
|
with open(v, 'w') as f:
|
||||||
|
f.write('21.0.0-devel\n')
|
||||||
|
|
||||||
|
with mock.patch('bin.gen_calendar_entries.VERSION', pathlib.Path(v)):
|
||||||
|
yield
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def csv(self) -> None:
|
||||||
|
"""inject our test data.."""
|
||||||
|
with mock_csv(self.ORIGINAL_DATA):
|
||||||
|
yield
|
||||||
|
|
||||||
|
def test_basic(self) -> None:
|
||||||
|
args: gen_calendar_entries.RCArguments = argparse.Namespace()
|
||||||
|
args.manager = "Dylan Baker"
|
||||||
|
gen_calendar_entries.release_candidate(args)
|
||||||
|
|
||||||
|
expected = self.ORIGINAL_DATA.copy()
|
||||||
|
expected.append(('21.0', '2021-01-13', f'21.0.0-rc1', 'Dylan Baker'))
|
||||||
|
expected.append(( '', '2021-01-20', f'21.0.0-rc2', 'Dylan Baker'))
|
||||||
|
expected.append(( '', '2021-01-27', f'21.0.0-rc3', 'Dylan Baker'))
|
||||||
|
expected.append(( '', '2021-02-03', f'21.0.0-rc4', 'Dylan Baker', 'Or 21.0.0 final.'))
|
||||||
|
|
||||||
|
actual = gen_calendar_entries.read_calendar()
|
||||||
|
|
||||||
|
assert actual == expected
|
||||||
|
|
||||||
|
|
||||||
|
class TestExtend:
|
||||||
|
|
||||||
|
def test_one_release(self) -> None:
|
||||||
|
data = [
|
||||||
|
('20.3', '2021-01-13', '20.3.3', 'Dylan Baker', ''),
|
||||||
|
('', '2021-01-27', '20.3.4', 'Dylan Baker', 'This is the last planned release of the 20.3.x series.'),
|
||||||
|
]
|
||||||
|
|
||||||
|
args: gen_calendar_entries.ExtendArguments = argparse.Namespace()
|
||||||
|
args.series = '20.3'
|
||||||
|
args.count = 2
|
||||||
|
|
||||||
|
with mock_csv(data):
|
||||||
|
gen_calendar_entries.extend(args)
|
||||||
|
actual = gen_calendar_entries.read_calendar()
|
||||||
|
|
||||||
|
expected = [
|
||||||
|
data[0],
|
||||||
|
('', '2021-01-27', '20.3.4', 'Dylan Baker', ''),
|
||||||
|
('', '2021-02-10', '20.3.5', 'Dylan Baker', ''),
|
||||||
|
('', '2021-02-24', '20.3.6', 'Dylan Baker', 'This is the last planned release of the 20.3.x series.'),
|
||||||
|
]
|
||||||
|
|
||||||
|
assert actual == expected
|
||||||
|
def test_one_release(self) -> None:
|
||||||
|
data = [
|
||||||
|
('20.3', '2021-01-13', '20.3.3', 'Dylan Baker', ''),
|
||||||
|
('', '2021-01-27', '20.3.4', 'Dylan Baker', 'This is the last planned release of the 20.3.x series.'),
|
||||||
|
('21.0', '2021-01-13', '21.0.1', 'Dylan Baker', ''),
|
||||||
|
('', '2021-01-27', '21.0.2', 'Dylan Baker', ''),
|
||||||
|
('', '2021-02-10', '21.0.3', 'Dylan Baker', ''),
|
||||||
|
('', '2021-02-24', '21.0.4', 'Dylan Baker', 'This is the last planned release of the 21.0.x series.'),
|
||||||
|
]
|
||||||
|
|
||||||
|
args: gen_calendar_entries.ExtendArguments = argparse.Namespace()
|
||||||
|
args.series = '21.0'
|
||||||
|
args.count = 1
|
||||||
|
|
||||||
|
with mock_csv(data):
|
||||||
|
gen_calendar_entries.extend(args)
|
||||||
|
actual = gen_calendar_entries.read_calendar()
|
||||||
|
|
||||||
|
expected = data.copy()
|
||||||
|
d = list(data[-1])
|
||||||
|
d[-1] = ''
|
||||||
|
expected[-1] = tuple(d)
|
||||||
|
expected.extend([
|
||||||
|
('', '2021-03-10', '21.0.5', 'Dylan Baker', 'This is the last planned release of the 21.0.x series.'),
|
||||||
|
])
|
||||||
|
|
||||||
|
assert actual == expected
|
||||||
|
|
||||||
|
def test_rc(self) -> None:
|
||||||
|
data = [
|
||||||
|
('20.3', '2021-01-13', '20.3.3', 'Dylan Baker', ''),
|
||||||
|
('', '2021-01-27', '20.3.4', 'Dylan Baker', 'This is the last planned release of the 20.3.x series.'),
|
||||||
|
('21.0', '2021-01-13', '21.0.0-rc1', 'Dylan Baker', ''),
|
||||||
|
('', '2021-01-20', '21.0.0-rc2', 'Dylan Baker', gen_calendar_entries.OR_FINAL.format('21.0')),
|
||||||
|
]
|
||||||
|
|
||||||
|
args: gen_calendar_entries.ExtendArguments = argparse.Namespace()
|
||||||
|
args.series = '21.0'
|
||||||
|
args.count = 2
|
||||||
|
|
||||||
|
with mock_csv(data):
|
||||||
|
gen_calendar_entries.extend(args)
|
||||||
|
actual = gen_calendar_entries.read_calendar()
|
||||||
|
|
||||||
|
expected = data.copy()
|
||||||
|
d = list(expected[-1])
|
||||||
|
d[-1] = ''
|
||||||
|
expected[-1] = tuple(d)
|
||||||
|
expected.extend([
|
||||||
|
('', '2021-01-27', '21.0.0-rc3', 'Dylan Baker', ''),
|
||||||
|
('', '2021-02-03', '21.0.0-rc4', 'Dylan Baker', gen_calendar_entries.OR_FINAL.format('21.0')),
|
||||||
|
])
|
||||||
|
|
||||||
|
assert actual == expected
|
||||||
|
|
||||||
|
|
||||||
|
class TestFinal:
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True, scope='class')
|
||||||
|
def _patch_date(self) -> typing.Iterator[None]:
|
||||||
|
mdate = mock.Mock()
|
||||||
|
mdate.today = mock.Mock(return_value=datetime.date(2021, 1, 6))
|
||||||
|
with mock.patch('bin.gen_calendar_entries.datetime.date', mdate):
|
||||||
|
yield
|
||||||
|
|
||||||
|
ORIGINAL_DATA = [
|
||||||
|
('20.3', '2021-01-13', '20.3.3', 'Dylan Baker', ''),
|
||||||
|
('', '2021-01-27', '20.3.4', 'Dylan Baker', 'Last planned release of the 20.3.x series'),
|
||||||
|
]
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def csv(self) -> None:
|
||||||
|
"""inject our test data.."""
|
||||||
|
with mock_csv(self.ORIGINAL_DATA):
|
||||||
|
yield
|
||||||
|
|
||||||
|
def test_zero_released(self) -> None:
|
||||||
|
args: gen_calendar_entries.FinalArguments = argparse.Namespace()
|
||||||
|
args.manager = "Dylan Baker"
|
||||||
|
args.zero_released = True
|
||||||
|
args.series = '21.0'
|
||||||
|
gen_calendar_entries.final_release(args)
|
||||||
|
|
||||||
|
expected = self.ORIGINAL_DATA.copy()
|
||||||
|
expected.append(('21.0', '2021-01-20', f'21.0.1', 'Dylan Baker'))
|
||||||
|
expected.append(( '', '2021-02-03', f'21.0.2', 'Dylan Baker'))
|
||||||
|
expected.append(( '', '2021-02-17', f'21.0.3', 'Dylan Baker', gen_calendar_entries.LAST_RELEASE.format(args.series)))
|
||||||
|
|
||||||
|
actual = gen_calendar_entries.read_calendar()
|
||||||
|
|
||||||
|
assert actual == expected
|
||||||
|
|
||||||
|
def test_zero_not_released(self) -> None:
|
||||||
|
args: gen_calendar_entries.FinalArguments = argparse.Namespace()
|
||||||
|
args.manager = "Dylan Baker"
|
||||||
|
args.zero_released = False
|
||||||
|
args.series = '21.0'
|
||||||
|
gen_calendar_entries.final_release(args)
|
||||||
|
|
||||||
|
expected = self.ORIGINAL_DATA.copy()
|
||||||
|
expected.append(('21.0', '2021-01-13', f'21.0.0', 'Dylan Baker'))
|
||||||
|
expected.append(( '', '2021-01-27', f'21.0.1', 'Dylan Baker'))
|
||||||
|
expected.append(( '', '2021-02-10', f'21.0.2', 'Dylan Baker'))
|
||||||
|
expected.append(( '', '2021-02-24', f'21.0.3', 'Dylan Baker', gen_calendar_entries.LAST_RELEASE.format(args.series)))
|
||||||
|
|
||||||
|
actual = gen_calendar_entries.read_calendar()
|
||||||
|
|
||||||
|
assert actual == expected
|
||||||
Executable
+395
@@ -0,0 +1,395 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# Copyright © 2019-2020 Intel Corporation
|
||||||
|
|
||||||
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
# of this software and associated documentation files (the "Software"), to deal
|
||||||
|
# in the Software without restriction, including without limitation the rights
|
||||||
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
# copies of the Software, and to permit persons to whom the Software is
|
||||||
|
# furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
# The above copyright notice and this permission notice shall be included in
|
||||||
|
# all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
# SOFTWARE.
|
||||||
|
|
||||||
|
"""Generates release notes for a given version of mesa."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import datetime
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import textwrap
|
||||||
|
import typing
|
||||||
|
import urllib.parse
|
||||||
|
|
||||||
|
import aiohttp
|
||||||
|
from mako.template import Template
|
||||||
|
from mako import exceptions
|
||||||
|
|
||||||
|
import docutils.utils
|
||||||
|
import docutils.parsers.rst.states as states
|
||||||
|
|
||||||
|
CURRENT_GL_VERSION = '4.6'
|
||||||
|
CURRENT_CL_VERSION = '3.1'
|
||||||
|
CURRENT_VK_VERSION = '1.4'
|
||||||
|
|
||||||
|
TEMPLATE = Template(textwrap.dedent("""\
|
||||||
|
${header}
|
||||||
|
${header_underline}
|
||||||
|
|
||||||
|
%if not bugfix:
|
||||||
|
Mesa ${this_version} is a new development release. People who are concerned
|
||||||
|
with stability and reliability should stick with a previous release or
|
||||||
|
wait for Mesa ${this_version[:-1]}1.
|
||||||
|
%else:
|
||||||
|
Mesa ${this_version} is a bug fix release which fixes bugs found since the ${previous_version} release.
|
||||||
|
%endif
|
||||||
|
|
||||||
|
Mesa ${this_version} implements the OpenGL ${gl_version} API, but the version reported by
|
||||||
|
glGetString(GL_VERSION) or glGetIntegerv(GL_MAJOR_VERSION) /
|
||||||
|
glGetIntegerv(GL_MINOR_VERSION) depends on the particular driver being used.
|
||||||
|
Some drivers don't support all the features required in OpenGL ${gl_version}. OpenGL
|
||||||
|
${gl_version} is **only** available if requested at context creation.
|
||||||
|
Compatibility contexts may report a lower version depending on each driver.
|
||||||
|
|
||||||
|
Mesa ${this_version} implements the OpenCL ${cl_version} API, but the version reported by
|
||||||
|
the CL_DEVICE_VERSION, CL_DEVICE_NUMERIC_VERSION and CL_DEVICE_OPENCL_C_ALL_VERSIONS
|
||||||
|
clGetDeviceInfo queries depends on the particular driver being used.
|
||||||
|
|
||||||
|
Mesa ${this_version} implements the Vulkan ${vk_version} API, but the version reported by
|
||||||
|
the apiVersion property of the VkPhysicalDeviceProperties struct
|
||||||
|
depends on the particular driver being used.
|
||||||
|
|
||||||
|
SHA checksums
|
||||||
|
-------------
|
||||||
|
|
||||||
|
::
|
||||||
|
|
||||||
|
TBD.
|
||||||
|
|
||||||
|
|
||||||
|
New features
|
||||||
|
------------
|
||||||
|
|
||||||
|
%for f in features:
|
||||||
|
- ${rst_escape(f)}
|
||||||
|
%endfor
|
||||||
|
|
||||||
|
|
||||||
|
Bug fixes
|
||||||
|
---------
|
||||||
|
|
||||||
|
%for b in bugs:
|
||||||
|
- ${rst_escape(b)}
|
||||||
|
%endfor
|
||||||
|
|
||||||
|
|
||||||
|
Changes
|
||||||
|
-------
|
||||||
|
%for c, author_line in changes:
|
||||||
|
%if author_line:
|
||||||
|
|
||||||
|
${rst_escape(c)}
|
||||||
|
|
||||||
|
%else:
|
||||||
|
- ${rst_escape(c)}
|
||||||
|
%endif
|
||||||
|
%endfor
|
||||||
|
"""))
|
||||||
|
|
||||||
|
|
||||||
|
# copied from https://docutils.sourceforge.io/sandbox/xml2rst/xml2rstlib/markup.py
|
||||||
|
class Inliner(states.Inliner):
|
||||||
|
"""
|
||||||
|
Recognizer for inline markup. Derive this from the original inline
|
||||||
|
markup parser for best results.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Copy static attributes from super class
|
||||||
|
vars().update(vars(states.Inliner))
|
||||||
|
|
||||||
|
def quoteInline(self, text):
|
||||||
|
"""
|
||||||
|
`text`: ``str``
|
||||||
|
Return `text` with inline markup quoted.
|
||||||
|
"""
|
||||||
|
# Method inspired by `states.Inliner.parse`
|
||||||
|
self.document = docutils.utils.new_document("<string>")
|
||||||
|
self.document.settings.trim_footnote_reference_space = False
|
||||||
|
self.document.settings.character_level_inline_markup = False
|
||||||
|
self.document.settings.pep_references = False
|
||||||
|
self.document.settings.rfc_references = False
|
||||||
|
|
||||||
|
self.init_customizations(self.document.settings)
|
||||||
|
|
||||||
|
self.reporter = self.document.reporter
|
||||||
|
self.reporter.stream = None
|
||||||
|
self.language = None
|
||||||
|
self.parent = self.document
|
||||||
|
remaining = docutils.utils.escape2null(text)
|
||||||
|
checked = ""
|
||||||
|
processed = []
|
||||||
|
unprocessed = []
|
||||||
|
messages = []
|
||||||
|
while remaining:
|
||||||
|
original = remaining
|
||||||
|
match = self.patterns.initial.search(remaining)
|
||||||
|
if match:
|
||||||
|
groups = match.groupdict()
|
||||||
|
method = self.dispatch[groups['start'] or groups['backquote']
|
||||||
|
or groups['refend'] or groups['fnend']]
|
||||||
|
before, inlines, remaining, sysmessages = method(self, match, 0)
|
||||||
|
checked += before
|
||||||
|
if inlines:
|
||||||
|
assert len(inlines) == 1, "More than one inline found"
|
||||||
|
inline = original[len(before)
|
||||||
|
:len(original) - len(remaining)]
|
||||||
|
rolePfx = re.search("^:" + self.simplename + ":(?=`)",
|
||||||
|
inline)
|
||||||
|
refSfx = re.search("_+$", inline)
|
||||||
|
if rolePfx:
|
||||||
|
# Prefixed roles need to be quoted in the middle
|
||||||
|
checked += (inline[:rolePfx.end()] + "\\"
|
||||||
|
+ inline[rolePfx.end():])
|
||||||
|
elif refSfx and not re.search("^`", inline):
|
||||||
|
# Pure reference markup needs to be quoted at the end
|
||||||
|
checked += (inline[:refSfx.start()] + "\\"
|
||||||
|
+ inline[refSfx.start():])
|
||||||
|
else:
|
||||||
|
# Quote other inlines by prefixing
|
||||||
|
checked += "\\" + inline
|
||||||
|
else:
|
||||||
|
checked += remaining
|
||||||
|
break
|
||||||
|
# Quote all original backslashes
|
||||||
|
checked = re.sub('\x00', "\\\x00", checked)
|
||||||
|
checked = re.sub('@', '\\@', checked)
|
||||||
|
return docutils.utils.unescape(checked, 1)
|
||||||
|
|
||||||
|
inliner = Inliner();
|
||||||
|
|
||||||
|
|
||||||
|
async def gather_commits(version: str) -> str:
|
||||||
|
p = await asyncio.create_subprocess_exec(
|
||||||
|
'git', 'log', '--oneline', f'mesa-{version}..', '-i', '--grep', r'\(Closes\|Fixes\): \(https\|#\).*',
|
||||||
|
stdout=asyncio.subprocess.PIPE)
|
||||||
|
out, _ = await p.communicate()
|
||||||
|
assert p.returncode == 0, f"git log didn't work: {version}"
|
||||||
|
return out.decode().strip()
|
||||||
|
|
||||||
|
|
||||||
|
async def parse_issues(commits: str) -> typing.List[str]:
|
||||||
|
issues: typing.List[str] = []
|
||||||
|
for commit in commits.split('\n'):
|
||||||
|
sha, message = commit.split(maxsplit=1)
|
||||||
|
p = await asyncio.create_subprocess_exec(
|
||||||
|
'git', 'log', '--max-count', '1', r'--format=%b', sha,
|
||||||
|
stdout=asyncio.subprocess.PIPE)
|
||||||
|
_out, _ = await p.communicate()
|
||||||
|
out = _out.decode().split('\n')
|
||||||
|
|
||||||
|
for line in reversed(out):
|
||||||
|
if not line.lower().startswith(('closes:', 'fixes:')):
|
||||||
|
continue
|
||||||
|
bug = line.split(':', 1)[1].strip()
|
||||||
|
if (bug.startswith('https://gitlab.freedesktop.org/mesa/mesa')
|
||||||
|
# Avoid parsing "merge_requests" URL. Note that a valid issue
|
||||||
|
# URL may or may not contain the "/-/" text, so we check if
|
||||||
|
# the word "issues" is contained in URL.
|
||||||
|
and '/issues' in bug):
|
||||||
|
# This means we have a bug in the form "Closes: https://..."
|
||||||
|
issues.append(os.path.basename(urllib.parse.urlparse(bug).path))
|
||||||
|
elif ',' in bug:
|
||||||
|
multiple_bugs = [b.strip().lstrip('#') for b in bug.split(',')]
|
||||||
|
if not all(b.isdigit() for b in multiple_bugs):
|
||||||
|
# this is likely a "Fixes" tag that refers to a commit name
|
||||||
|
continue
|
||||||
|
issues.extend(multiple_bugs)
|
||||||
|
elif bug.startswith('#'):
|
||||||
|
issues.append(bug.lstrip('#'))
|
||||||
|
|
||||||
|
return issues
|
||||||
|
|
||||||
|
|
||||||
|
async def gather_bugs(version: str) -> typing.List[str]:
|
||||||
|
commits = await gather_commits(version)
|
||||||
|
if commits:
|
||||||
|
issues = await parse_issues(commits)
|
||||||
|
else:
|
||||||
|
issues = []
|
||||||
|
|
||||||
|
loop = asyncio.get_event_loop()
|
||||||
|
async with aiohttp.ClientSession(loop=loop) as session:
|
||||||
|
results = await asyncio.gather(*[get_bug(session, i) for i in issues])
|
||||||
|
# Remove duplicates.
|
||||||
|
bugs = sorted(set(results))
|
||||||
|
if not bugs:
|
||||||
|
bugs = ['None']
|
||||||
|
return bugs
|
||||||
|
|
||||||
|
|
||||||
|
async def get_bug(session: aiohttp.ClientSession, bug_id: str) -> str:
|
||||||
|
"""Query gitlab to get the name of the issue that was closed."""
|
||||||
|
# Mesa's gitlab id is 176,
|
||||||
|
url = 'https://gitlab.freedesktop.org/api/v4/projects/176/issues'
|
||||||
|
params = {'iids[]': bug_id}
|
||||||
|
async with session.get(url, params=params) as response:
|
||||||
|
content = await response.json()
|
||||||
|
if not content:
|
||||||
|
# issues marked as "confidential" look like "404" page for
|
||||||
|
# unauthorized users
|
||||||
|
return f'Confidential issue #{bug_id}'
|
||||||
|
else:
|
||||||
|
return content[0]['title']
|
||||||
|
|
||||||
|
|
||||||
|
async def get_shortlog(version: str) -> str:
|
||||||
|
"""Call git shortlog."""
|
||||||
|
p = await asyncio.create_subprocess_exec('git', 'shortlog', f'mesa-{version}..',
|
||||||
|
stdout=asyncio.subprocess.PIPE)
|
||||||
|
out, _ = await p.communicate()
|
||||||
|
assert p.returncode == 0, 'error getting shortlog'
|
||||||
|
assert out is not None, 'just for mypy'
|
||||||
|
return out.decode()
|
||||||
|
|
||||||
|
|
||||||
|
def walk_shortlog(log: str) -> typing.Generator[typing.Tuple[str, bool], None, None]:
|
||||||
|
for l in log.split('\n'):
|
||||||
|
if l.startswith(' '): # this means we have a patch description
|
||||||
|
yield l.lstrip(), False
|
||||||
|
elif l.strip():
|
||||||
|
yield l, True
|
||||||
|
|
||||||
|
|
||||||
|
def calculate_next_version(version: str, is_point: bool) -> str:
|
||||||
|
"""Calculate the version about to be released."""
|
||||||
|
if '-' in version:
|
||||||
|
version = version.split('-')[0]
|
||||||
|
if is_point:
|
||||||
|
base = version.split('.')
|
||||||
|
base[2] = str(int(base[2]) + 1)
|
||||||
|
return '.'.join(base)
|
||||||
|
return version
|
||||||
|
|
||||||
|
|
||||||
|
def calculate_previous_version(version: str, is_point: bool) -> str:
|
||||||
|
"""Calculate the previous version to compare to.
|
||||||
|
|
||||||
|
In the case of -rc to final that version is the previous .0 release,
|
||||||
|
(19.3.0 in the case of 20.0.0, for example). for point releases that is
|
||||||
|
the last point release. This value will be the same as the input value
|
||||||
|
for a point release, but different for a major release.
|
||||||
|
"""
|
||||||
|
if '-' in version:
|
||||||
|
version = version.split('-')[0]
|
||||||
|
if is_point:
|
||||||
|
return version
|
||||||
|
base = version.split('.')
|
||||||
|
if base[1] == '0':
|
||||||
|
base[0] = str(int(base[0]) - 1)
|
||||||
|
base[1] = '3'
|
||||||
|
else:
|
||||||
|
base[1] = str(int(base[1]) - 1)
|
||||||
|
return '.'.join(base)
|
||||||
|
|
||||||
|
|
||||||
|
def get_features(is_point_release: bool) -> typing.Generator[str, None, None]:
|
||||||
|
p = pathlib.Path('docs') / 'relnotes' / 'new_features.txt'
|
||||||
|
if p.exists() and p.stat().st_size > 0:
|
||||||
|
if is_point_release:
|
||||||
|
print("WARNING: new features being introduced in a point release", file=sys.stderr)
|
||||||
|
with p.open('rt') as f:
|
||||||
|
for line in f:
|
||||||
|
yield line.rstrip()
|
||||||
|
p.unlink()
|
||||||
|
subprocess.run(['git', 'add', p])
|
||||||
|
else:
|
||||||
|
yield "None"
|
||||||
|
|
||||||
|
|
||||||
|
def update_release_notes_index(version: str) -> None:
|
||||||
|
relnotes_index_path = pathlib.Path('docs') / 'relnotes.rst'
|
||||||
|
|
||||||
|
with relnotes_index_path.open('r') as f:
|
||||||
|
relnotes = f.readlines()
|
||||||
|
|
||||||
|
new_relnotes = []
|
||||||
|
first_list = True
|
||||||
|
second_list = True
|
||||||
|
for line in relnotes:
|
||||||
|
if first_list and line.startswith('-'):
|
||||||
|
first_list = False
|
||||||
|
new_relnotes.append(f'- :doc:`{version} release notes <relnotes/{version}>`\n')
|
||||||
|
if (not first_list and second_list and
|
||||||
|
re.match(r' \d+.\d+(.\d+)? <relnotes/\d+.\d+(.\d+)?>', line)):
|
||||||
|
second_list = False
|
||||||
|
new_relnotes.append(f' {version} <relnotes/{version}>\n')
|
||||||
|
new_relnotes.append(line)
|
||||||
|
|
||||||
|
with relnotes_index_path.open('w', encoding='utf-8') as f:
|
||||||
|
for line in new_relnotes:
|
||||||
|
f.write(line)
|
||||||
|
|
||||||
|
subprocess.run(['git', 'add', relnotes_index_path])
|
||||||
|
|
||||||
|
|
||||||
|
async def main() -> None:
|
||||||
|
v = pathlib.Path('VERSION')
|
||||||
|
with v.open('rt') as f:
|
||||||
|
raw_version = f.read().strip()
|
||||||
|
is_point_release = '-rc' not in raw_version
|
||||||
|
assert '-devel' not in raw_version, 'Do not run this script on -devel'
|
||||||
|
version = raw_version.split('-')[0]
|
||||||
|
previous_version = calculate_previous_version(version, is_point_release)
|
||||||
|
this_version = calculate_next_version(version, is_point_release)
|
||||||
|
today = datetime.date.today()
|
||||||
|
header = f'Mesa {this_version} Release Notes / {today}'
|
||||||
|
header_underline = '=' * len(header)
|
||||||
|
|
||||||
|
shortlog, bugs = await asyncio.gather(
|
||||||
|
get_shortlog(previous_version),
|
||||||
|
gather_bugs(previous_version),
|
||||||
|
)
|
||||||
|
|
||||||
|
final = pathlib.Path('docs') / 'relnotes' / f'{this_version}.rst'
|
||||||
|
with final.open('wt', encoding='utf-8') as f:
|
||||||
|
try:
|
||||||
|
f.write(TEMPLATE.render(
|
||||||
|
bugfix=is_point_release,
|
||||||
|
bugs=bugs,
|
||||||
|
changes=walk_shortlog(shortlog),
|
||||||
|
features=get_features(is_point_release),
|
||||||
|
gl_version=CURRENT_GL_VERSION,
|
||||||
|
cl_version=CURRENT_CL_VERSION,
|
||||||
|
this_version=this_version,
|
||||||
|
header=header,
|
||||||
|
header_underline=header_underline,
|
||||||
|
previous_version=previous_version,
|
||||||
|
vk_version=CURRENT_VK_VERSION,
|
||||||
|
rst_escape=inliner.quoteInline,
|
||||||
|
))
|
||||||
|
except:
|
||||||
|
print(exceptions.text_error_template().render())
|
||||||
|
return
|
||||||
|
|
||||||
|
subprocess.run(['git', 'add', final])
|
||||||
|
|
||||||
|
update_release_notes_index(this_version)
|
||||||
|
|
||||||
|
subprocess.run(['git', 'commit', '-m',
|
||||||
|
f'docs: add release notes for {this_version}'])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
loop = asyncio.new_event_loop()
|
||||||
|
loop.run_until_complete(main())
|
||||||
@@ -0,0 +1,226 @@
|
|||||||
|
# Copyright © 2019,2021 Intel Corporation
|
||||||
|
|
||||||
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
# of this software and associated documentation files (the "Software"), to deal
|
||||||
|
# in the Software without restriction, including without limitation the rights
|
||||||
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
# copies of the Software, and to permit persons to whom the Software is
|
||||||
|
# furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
# The above copyright notice and this permission notice shall be included in
|
||||||
|
# all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
# SOFTWARE.
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import textwrap
|
||||||
|
import typing
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
if typing.TYPE_CHECKING:
|
||||||
|
import aiohttp
|
||||||
|
|
||||||
|
# AsyncMock is new in 3.8, so if we're using an older version we need the
|
||||||
|
# backported version of mock
|
||||||
|
if sys.version_info >= (3, 8):
|
||||||
|
from unittest import mock
|
||||||
|
else:
|
||||||
|
import mock
|
||||||
|
|
||||||
|
from .gen_release_notes import *
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'current, is_point, expected',
|
||||||
|
[
|
||||||
|
('19.2.0', True, '19.2.1'),
|
||||||
|
('19.3.6', True, '19.3.7'),
|
||||||
|
('20.0.0-rc4', False, '20.0.0'),
|
||||||
|
])
|
||||||
|
def test_next_version(current: str, is_point: bool, expected: str) -> None:
|
||||||
|
assert calculate_next_version(current, is_point) == expected
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'current, is_point, expected',
|
||||||
|
[
|
||||||
|
('19.3.6', True, '19.3.6'),
|
||||||
|
('20.0.0-rc4', False, '19.3.0'),
|
||||||
|
])
|
||||||
|
def test_previous_version(current: str, is_point: bool, expected: str) -> None:
|
||||||
|
assert calculate_previous_version(current, is_point) == expected
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_get_shortlog():
|
||||||
|
# Certainly not perfect, but it's something
|
||||||
|
version = '19.2.0'
|
||||||
|
out = await get_shortlog(version)
|
||||||
|
assert out
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_gather_commits():
|
||||||
|
# Certainly not perfect, but it's something
|
||||||
|
version = '19.2.0'
|
||||||
|
out = await gather_commits(version)
|
||||||
|
assert out
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'content, bugs',
|
||||||
|
[
|
||||||
|
# It is important to have the title on a new line, as
|
||||||
|
# textwrap.dedent won't work otherwise.
|
||||||
|
|
||||||
|
# Test the `Closes: #N` syntax
|
||||||
|
(
|
||||||
|
'''\
|
||||||
|
A commit
|
||||||
|
|
||||||
|
It has a message in it
|
||||||
|
|
||||||
|
Closes: #1
|
||||||
|
''',
|
||||||
|
['1'],
|
||||||
|
),
|
||||||
|
|
||||||
|
# Test the Full url
|
||||||
|
(
|
||||||
|
'''\
|
||||||
|
A commit with no body
|
||||||
|
|
||||||
|
Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/3456
|
||||||
|
''',
|
||||||
|
['3456'],
|
||||||
|
),
|
||||||
|
|
||||||
|
# Test projects that are not mesa
|
||||||
|
(
|
||||||
|
'''\
|
||||||
|
A commit for libdrm
|
||||||
|
|
||||||
|
Closes: https://gitlab.freedesktop.org/mesa/drm/-/3456
|
||||||
|
''',
|
||||||
|
[],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'''\
|
||||||
|
A commit for for something else completely
|
||||||
|
|
||||||
|
Closes: https://github.com/Organization/project/1234
|
||||||
|
''',
|
||||||
|
[],
|
||||||
|
),
|
||||||
|
|
||||||
|
# Test multiple issues on one line
|
||||||
|
(
|
||||||
|
'''\
|
||||||
|
Fix many bugs
|
||||||
|
|
||||||
|
Closes: #1, #2
|
||||||
|
''',
|
||||||
|
['1', '2'],
|
||||||
|
),
|
||||||
|
|
||||||
|
# Test multiple closes
|
||||||
|
(
|
||||||
|
'''\
|
||||||
|
Fix many bugs
|
||||||
|
|
||||||
|
Closes: #1
|
||||||
|
Closes: #2
|
||||||
|
''',
|
||||||
|
['1', '2'],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'''\
|
||||||
|
With long form
|
||||||
|
|
||||||
|
Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/3456
|
||||||
|
Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/3457
|
||||||
|
Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/3458
|
||||||
|
''',
|
||||||
|
['3456', '3457', '3458'],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'''\
|
||||||
|
Without /-/
|
||||||
|
|
||||||
|
Closes: https://gitlab.freedesktop.org/mesa/mesa/issues/36
|
||||||
|
''',
|
||||||
|
['36'],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'''\
|
||||||
|
Ignore merge_requests
|
||||||
|
|
||||||
|
Closes: https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/20241
|
||||||
|
''',
|
||||||
|
[],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'''\
|
||||||
|
Parse "Fixes:" tag too
|
||||||
|
|
||||||
|
Fixes: https://gitlab.freedesktop.org/mesa/mesa/issues/36
|
||||||
|
Fixes: 142565a3bc2
|
||||||
|
Fixes: 142565a3bc2 ("docs: do something very useful")
|
||||||
|
Fixes: 142565a3bc2 ("docs: fix #1234, have a comma")
|
||||||
|
Fixes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/37
|
||||||
|
''',
|
||||||
|
['36', '37'],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
'''\
|
||||||
|
Parse Fixes/Closes in weird cases
|
||||||
|
|
||||||
|
fixes: https://gitlab.freedesktop.org/mesa/mesa/issues/36
|
||||||
|
fiXES: https://gitlab.freedesktop.org/mesa/mesa/issues/37
|
||||||
|
closes: https://gitlab.freedesktop.org/mesa/mesa/issues/38
|
||||||
|
cloSES: https://gitlab.freedesktop.org/mesa/mesa/issues/39
|
||||||
|
''',
|
||||||
|
['36', '37', '38', '39'],
|
||||||
|
),
|
||||||
|
])
|
||||||
|
async def test_parse_issues(content: str, bugs: typing.List[str]) -> None:
|
||||||
|
mock_com = mock.AsyncMock(return_value=(textwrap.dedent(content).encode(), ''))
|
||||||
|
mock_p = mock.Mock()
|
||||||
|
mock_p.communicate = mock_com
|
||||||
|
mock_exec = mock.AsyncMock(return_value=mock_p)
|
||||||
|
|
||||||
|
with mock.patch('bin.gen_release_notes.asyncio.create_subprocess_exec', mock_exec), \
|
||||||
|
mock.patch('bin.gen_release_notes.gather_commits', mock.AsyncMock(return_value='sha\n')):
|
||||||
|
ids = await parse_issues('1234 not used')
|
||||||
|
assert set(ids) == set(bugs)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_rst_escape():
|
||||||
|
out = inliner.quoteInline('foo@bar')
|
||||||
|
assert out == 'foo\@bar'
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_gather_bugs_duplicates():
|
||||||
|
mock_gc = mock.AsyncMock(return_value='something')
|
||||||
|
mock_pi = mock.AsyncMock(return_value=['a', 'b', 'a', 'a', 'c', 'b'])
|
||||||
|
|
||||||
|
async def get_bug(session: 'aiohttp.ClientSession', bug_id: str) -> str:
|
||||||
|
return bug_id
|
||||||
|
|
||||||
|
with mock.patch('bin.gen_release_notes.gather_commits', mock_gc), \
|
||||||
|
mock.patch('bin.gen_release_notes.parse_issues', mock_pi), \
|
||||||
|
mock.patch('bin.gen_release_notes.get_bug', get_bug):
|
||||||
|
bugs = await gather_bugs('')
|
||||||
|
|
||||||
|
assert bugs == ['a', 'b', 'c']
|
||||||
|
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# Copyright © 2021-2021 Yonggang Luo
|
||||||
|
|
||||||
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
# of this software and associated documentation files (the "Software"), to deal
|
||||||
|
# in the Software without restriction, including without limitation the rights
|
||||||
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
# copies of the Software, and to permit persons to whom the Software is
|
||||||
|
# furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
# The above copyright notice and this permission notice shall be included in
|
||||||
|
# all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
# SOFTWARE.
|
||||||
|
|
||||||
|
gen_help = """Generates visual studio module definition file."""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
|
||||||
|
"""
|
||||||
|
For input template definition file
|
||||||
|
For gcc/x64,gcc/arm64,visual studio
|
||||||
|
`wglMakeCurrent@8 @357` => `wglMakeCurrent @357`
|
||||||
|
`DrvCopyContext@12` => `DrvCopyContext`
|
||||||
|
`stw_get_device` => `stw_get_device`
|
||||||
|
For gcc/x86,gcc/arm
|
||||||
|
`wglMakeCurrent@8 @357` => `wglMakeCurrent@8 @357 == wglMakeCurrent`
|
||||||
|
`DrvCopyContext@12` => `DrvCopyContext@12 == DrvCopyContext`
|
||||||
|
`stw_get_device` => `stw_get_device`
|
||||||
|
|
||||||
|
"""
|
||||||
|
def gen_vs_module_def(in_file: str, out_file: str, compiler_abi: str, compiler_id: str, cpu_family: str) -> None:
|
||||||
|
out_file_lines = ['EXPORTS']
|
||||||
|
with open(in_file, 'r', encoding='utf-8') as f:
|
||||||
|
lines = f.readlines()
|
||||||
|
for line in lines:
|
||||||
|
line = line.strip()
|
||||||
|
tokens = line.split(';')
|
||||||
|
if not tokens:
|
||||||
|
continue
|
||||||
|
def_infos = [x for x in tokens[0].split(' ') if len(x) > 0]
|
||||||
|
if not def_infos:
|
||||||
|
if line:
|
||||||
|
out_file_lines.append('\t' + line)
|
||||||
|
else:
|
||||||
|
out_file_lines.append('')
|
||||||
|
continue
|
||||||
|
name_infos = def_infos[0].split('@')
|
||||||
|
if not name_infos:
|
||||||
|
out_file_lines.append('\t;' + line)
|
||||||
|
continue
|
||||||
|
order_info = '' if len(def_infos) <= 1 else def_infos[1]
|
||||||
|
if def_infos[0] != name_infos[0] and \
|
||||||
|
(compiler_abi == 'gcc' and compiler_id != 'clang') and (cpu_family not in {'x86_64', 'aarch64'}):
|
||||||
|
if order_info:
|
||||||
|
out_file_lines.append('\t' + def_infos[0] + ' ' + order_info + ' == ' + name_infos[0])
|
||||||
|
else:
|
||||||
|
out_file_lines.append('\t' + def_infos[0] + ' == ' + name_infos[0])
|
||||||
|
else:
|
||||||
|
if order_info:
|
||||||
|
out_file_lines.append('\t' + name_infos[0] + ' ' + order_info)
|
||||||
|
else:
|
||||||
|
out_file_lines.append('\t' + name_infos[0])
|
||||||
|
with open(out_file, 'wb') as f:
|
||||||
|
out_file_content = '\n'.join(out_file_lines) + '\n'
|
||||||
|
f.write(out_file_content.encode('utf-8'))
|
||||||
|
'''
|
||||||
|
python ./bin/gen_vs_module_defs.py --in_file src/gallium/targets/libgl-gdi/opengl32.def.in --out_file src/gallium/targets/libgl-gdi/opengl32.def --compiler_abi gcc --cpu_family x86_64
|
||||||
|
python ./bin/gen_vs_module_defs.py --in_file src/gallium/targets/libgl-gdi/opengl32.def.in --out_file src/gallium/targets/libgl-gdi/opengl32.mingw.def --compiler_abi gcc --cpu_family x86
|
||||||
|
|
||||||
|
python ./bin/gen_vs_module_defs.py --in_file src/gallium/targets/wgl/gallium_wgl.def.in --out_file src/gallium/targets/wgl/gallium_wgl.def --compiler_abi gcc --cpu_family x86_64
|
||||||
|
python ./bin/gen_vs_module_defs.py --in_file src/gallium/targets/wgl/gallium_wgl.def.in --out_file src/gallium/targets/wgl/gallium_wgl.mingw.def --compiler_abi gcc --cpu_family x86
|
||||||
|
|
||||||
|
python ./bin/gen_vs_module_defs.py --in_file src/egl/main/egl.def.in --out_file src/egl/main/egl.def --compiler_abi gcc --cpu_family x86_64
|
||||||
|
python ./bin/gen_vs_module_defs.py --in_file src/egl/main/egl.def.in --out_file src/egl/main/egl.mingw.def --compiler_abi gcc --cpu_family x86
|
||||||
|
|
||||||
|
python ./bin/gen_vs_module_defs.py --in_file src/gallium/targets/lavapipe/vulkan_lvp.def.in --out_file src/gallium/targets/lavapipe/vulkan_lvp.def --compiler_abi gcc --cpu_family x86_64
|
||||||
|
python ./bin/gen_vs_module_defs.py --in_file src/gallium/targets/lavapipe/vulkan_lvp.def.in --out_file src/gallium/targets/lavapipe/vulkan_lvp.mingw.def --compiler_abi gcc --cpu_family x86
|
||||||
|
|
||||||
|
'''
|
||||||
|
if __name__ == "__main__":
|
||||||
|
parser = argparse.ArgumentParser(description=gen_help)
|
||||||
|
parser.add_argument('--in_file', help='input template module definition file')
|
||||||
|
parser.add_argument('--out_file', help='output module definition file')
|
||||||
|
parser.add_argument('--compiler_abi', help='compiler abi')
|
||||||
|
parser.add_argument('--compiler_id', help='compiler id')
|
||||||
|
parser.add_argument('--cpu_family', help='cpu family')
|
||||||
|
args = parser.parse_args()
|
||||||
|
# print(args)
|
||||||
|
gen_vs_module_def(args.in_file, args.out_file, args.compiler_abi, args.compiler_id, args.cpu_family)
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
"""
|
||||||
|
Generate the contents of the git_sha1.h file.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import os.path
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
def get_git_sha1():
|
||||||
|
"""Try to get the git SHA1 with git rev-parse."""
|
||||||
|
git_dir = os.path.join(os.path.dirname(sys.argv[0]), '..', '.git')
|
||||||
|
try:
|
||||||
|
git_sha1 = subprocess.check_output([
|
||||||
|
'git',
|
||||||
|
'--git-dir=' + git_dir,
|
||||||
|
'rev-parse',
|
||||||
|
'HEAD',
|
||||||
|
], stderr=open(os.devnull, 'w')).decode("ascii")
|
||||||
|
except Exception:
|
||||||
|
# don't print anything if it fails
|
||||||
|
git_sha1 = ''
|
||||||
|
return git_sha1
|
||||||
|
|
||||||
|
|
||||||
|
def write_if_different(contents):
|
||||||
|
"""
|
||||||
|
Avoid touching the output file if it doesn't need modifications
|
||||||
|
Useful to avoid triggering rebuilds when nothing has changed.
|
||||||
|
"""
|
||||||
|
if os.path.isfile(args.output):
|
||||||
|
with open(args.output, 'r') as file:
|
||||||
|
if file.read() == contents:
|
||||||
|
return
|
||||||
|
with open(args.output, 'w') as file:
|
||||||
|
file.write(contents)
|
||||||
|
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument('--output', help='File to write the #define in',
|
||||||
|
required=True)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
git_sha1 = os.environ.get('MESA_GIT_SHA1_OVERRIDE', get_git_sha1())[:10]
|
||||||
|
if git_sha1:
|
||||||
|
write_if_different('#define MESA_GIT_SHA1 " (git-' + git_sha1 + ')"')
|
||||||
|
else:
|
||||||
|
write_if_different('#define MESA_GIT_SHA1 ""')
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# encoding=utf-8
|
||||||
|
# Copyright 2017-2018 Intel Corporation
|
||||||
|
|
||||||
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
# of this software and associated documentation files (the "Software"), to deal
|
||||||
|
# in the Software without restriction, including without limitation the rights
|
||||||
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
# copies of the Software, and to permit persons to whom the Software is
|
||||||
|
# furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
# The above copyright notice and this permission notice shall be included in
|
||||||
|
# all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
# SOFTWARE.
|
||||||
|
|
||||||
|
"""Script to install megadriver symlinks for meson."""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_libdir(libdir):
|
||||||
|
if os.path.isabs(libdir):
|
||||||
|
destdir = os.environ.get('DESTDIR')
|
||||||
|
if destdir:
|
||||||
|
return os.path.join(destdir, libdir[1:])
|
||||||
|
else:
|
||||||
|
return libdir
|
||||||
|
return os.path.join(os.environ['MESON_INSTALL_DESTDIR_PREFIX'], libdir)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument('megadriver')
|
||||||
|
parser.add_argument('libdir')
|
||||||
|
parser.add_argument('drivers', nargs='+')
|
||||||
|
parser.add_argument('--megadriver-libdir')
|
||||||
|
parser.add_argument('--libname-suffix', required=True)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
# Not neccesarily at the end, there might be a version suffix, but let's
|
||||||
|
# make sure that the same suffix is in the megadriver lib name.
|
||||||
|
assert '.' + args.libname_suffix in args.megadriver
|
||||||
|
|
||||||
|
to = resolve_libdir(args.libdir)
|
||||||
|
if args.megadriver_libdir:
|
||||||
|
md_to = resolve_libdir(args.megadriver_libdir)
|
||||||
|
else:
|
||||||
|
md_to = to
|
||||||
|
|
||||||
|
basename = os.path.basename(args.megadriver)
|
||||||
|
master = os.path.join(to, basename)
|
||||||
|
|
||||||
|
if not os.path.exists(to):
|
||||||
|
if os.path.lexists(to):
|
||||||
|
os.unlink(to)
|
||||||
|
os.makedirs(to)
|
||||||
|
|
||||||
|
for driver in args.drivers:
|
||||||
|
abs_driver = os.path.join(to, driver)
|
||||||
|
|
||||||
|
if os.path.lexists(abs_driver):
|
||||||
|
os.unlink(abs_driver)
|
||||||
|
|
||||||
|
symlink = os.path.relpath(os.path.join(md_to, basename), start=to)
|
||||||
|
|
||||||
|
print(f'Installing symlink pointing to {symlink} to {abs_driver}')
|
||||||
|
os.symlink(symlink, abs_driver)
|
||||||
|
|
||||||
|
try:
|
||||||
|
ret = os.getcwd()
|
||||||
|
os.chdir(to)
|
||||||
|
|
||||||
|
name, ext = os.path.splitext(driver)
|
||||||
|
while ext != '.' + args.libname_suffix:
|
||||||
|
if os.path.lexists(name):
|
||||||
|
os.unlink(name)
|
||||||
|
os.symlink(driver, name)
|
||||||
|
name, ext = os.path.splitext(name)
|
||||||
|
finally:
|
||||||
|
os.chdir(ret)
|
||||||
|
|
||||||
|
# Remove meson-created symlinks
|
||||||
|
name, ext = os.path.splitext(master)
|
||||||
|
while ext != '.' + args.libname_suffix:
|
||||||
|
if os.path.lexists(name):
|
||||||
|
os.unlink(name)
|
||||||
|
name, ext = os.path.splitext(name)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
Executable
+286
@@ -0,0 +1,286 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import base64
|
||||||
|
import pathlib
|
||||||
|
import requests
|
||||||
|
import subprocess
|
||||||
|
import typing
|
||||||
|
|
||||||
|
|
||||||
|
def error(msg: str) -> None:
|
||||||
|
print('\033[31m' + msg + '\033[0m')
|
||||||
|
|
||||||
|
|
||||||
|
class Source:
|
||||||
|
def __init__(self, filename: str, url: typing.Optional[str],
|
||||||
|
template: typing.Optional[str] = None, remove:
|
||||||
|
typing.Optional[str] = None):
|
||||||
|
self.file = pathlib.Path(filename)
|
||||||
|
self.url = url
|
||||||
|
self.template = template
|
||||||
|
self.remove = remove
|
||||||
|
|
||||||
|
def sync(self) -> None:
|
||||||
|
if self.url is None:
|
||||||
|
return
|
||||||
|
|
||||||
|
print('Syncing {}...'.format(self.file), end=' ', flush=True)
|
||||||
|
req = requests.get(self.url)
|
||||||
|
|
||||||
|
if not req.ok:
|
||||||
|
error('Failed to retrieve file: {} {}'.format(req.status_code, req.reason))
|
||||||
|
return
|
||||||
|
|
||||||
|
# Gitiles returns base64-encoded strings.
|
||||||
|
# Google has been resisting for years to the idea of allowing plain text: https://github.com/google/gitiles/issues/7
|
||||||
|
if 'format=TEXT' in self.url:
|
||||||
|
content = base64.b64decode(req.content)
|
||||||
|
else:
|
||||||
|
content = req.content
|
||||||
|
|
||||||
|
content = str(content, encoding='utf-8')
|
||||||
|
if self.remove is not None:
|
||||||
|
content = content.replace(self.remove, '')
|
||||||
|
if self.template is not None:
|
||||||
|
content = self.template % content
|
||||||
|
|
||||||
|
with open(self.file, 'w') as f:
|
||||||
|
f.write(content)
|
||||||
|
|
||||||
|
print('Done')
|
||||||
|
|
||||||
|
|
||||||
|
VK_ANDROID_NATIVE_BUFFER_TEMPLATE = """\
|
||||||
|
/*
|
||||||
|
* MESA: buffer_handle_t is defined by all Mesa builds, even if
|
||||||
|
* one is building for a non-Android target. This avoids unnecessary
|
||||||
|
* conditionals in driver code.
|
||||||
|
*
|
||||||
|
* We don't need to define buffer_handle_t locally when (__ANDROID__)
|
||||||
|
* or ANDROID are set. Here's the distinction between the two:
|
||||||
|
*
|
||||||
|
* - AOSP always defines ANDROID, since it just means one is using the
|
||||||
|
* AOSP tree. It means the build environment is Android, roughly.
|
||||||
|
* - __ANDROID__ is defined by the toolchain. This typically means the
|
||||||
|
* build target is Android.
|
||||||
|
*
|
||||||
|
* If the build environment is Android, AOSP can provide common Android
|
||||||
|
* headers, such as <cutils/native_handle.h>. This allows one to build
|
||||||
|
* and test certain aspects of Android window system code, on the host
|
||||||
|
* system rather the build target.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#if defined(__ANDROID__) || defined(ANDROID)
|
||||||
|
|
||||||
|
#include <cutils/native_handle.h>
|
||||||
|
#if ANDROID_API_LEVEL < 28
|
||||||
|
/* buffer_handle_t was defined in the deprecated system/window.h */
|
||||||
|
typedef const native_handle_t *buffer_handle_t;
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#else
|
||||||
|
|
||||||
|
typedef void *buffer_handle_t;
|
||||||
|
|
||||||
|
#endif
|
||||||
|
|
||||||
|
%s\
|
||||||
|
"""
|
||||||
|
|
||||||
|
# a URL of `None` means there is no upstream, because *we* are the upstream
|
||||||
|
SOURCES = [
|
||||||
|
{
|
||||||
|
'api': 'khr',
|
||||||
|
'inc_folder': 'KHR',
|
||||||
|
'sources': [
|
||||||
|
Source('include/KHR/khrplatform.h', 'https://github.com/KhronosGroup/EGL-Registry/raw/main/api/KHR/khrplatform.h'),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
'api': 'egl',
|
||||||
|
'inc_folder': 'EGL',
|
||||||
|
'sources': [
|
||||||
|
Source('src/egl/generate/egl.xml', 'https://github.com/KhronosGroup/EGL-Registry/raw/main/api/egl.xml'),
|
||||||
|
Source('include/EGL/egl.h', 'https://github.com/KhronosGroup/EGL-Registry/raw/main/api/EGL/egl.h'),
|
||||||
|
Source('include/EGL/eglplatform.h', 'https://github.com/KhronosGroup/EGL-Registry/raw/main/api/EGL/eglplatform.h'),
|
||||||
|
Source('include/EGL/eglext.h', 'https://github.com/KhronosGroup/EGL-Registry/raw/main/api/EGL/eglext.h'),
|
||||||
|
Source('include/EGL/eglext_angle.h', 'https://chromium.googlesource.com/angle/angle/+/refs/heads/main/include/EGL/eglext_angle.h?format=TEXT'),
|
||||||
|
Source('include/EGL/eglmesaext.h', None),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
'api': 'gl',
|
||||||
|
'inc_folder': 'GL',
|
||||||
|
'sources': [
|
||||||
|
Source('src/mesa/glapi/glapi/registry/gl.xml', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/xml/gl.xml'),
|
||||||
|
Source('include/GL/glcorearb.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GL/glcorearb.h'),
|
||||||
|
Source('include/GL/glext.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GL/glext.h'),
|
||||||
|
Source('include/GL/glxext.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GL/glxext.h'),
|
||||||
|
Source('include/GL/wglext.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GL/wglext.h'),
|
||||||
|
Source('include/GL/gl.h', None), # FIXME: I don't know what the canonical source is
|
||||||
|
Source('include/GL/glx.h', None), # FIXME: I don't know what the canonical source is
|
||||||
|
Source('include/GL/internal/', None),
|
||||||
|
Source('include/GL/mesa_glinterop.h', None),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
'api': 'gles1',
|
||||||
|
'inc_folder': 'GLES',
|
||||||
|
'sources': [
|
||||||
|
Source('include/GLES/gl.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GLES/gl.h'),
|
||||||
|
Source('include/GLES/glplatform.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GLES/glplatform.h'),
|
||||||
|
Source('include/GLES/glext.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GLES/glext.h'),
|
||||||
|
Source('include/GLES/egl.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GLES/egl.h'),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
'api': 'gles2',
|
||||||
|
'inc_folder': 'GLES2',
|
||||||
|
'sources': [
|
||||||
|
Source('include/GLES2/gl2.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GLES2/gl2.h'),
|
||||||
|
Source('include/GLES2/gl2platform.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GLES2/gl2platform.h'),
|
||||||
|
Source('include/GLES2/gl2ext.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GLES2/gl2ext.h'),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
'api': 'gles3',
|
||||||
|
'inc_folder': 'GLES3',
|
||||||
|
'sources': [
|
||||||
|
Source('include/GLES3/gl3.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GLES3/gl3.h'),
|
||||||
|
Source('include/GLES3/gl31.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GLES3/gl31.h'),
|
||||||
|
Source('include/GLES3/gl32.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GLES3/gl32.h'),
|
||||||
|
Source('include/GLES3/gl3platform.h', 'https://github.com/KhronosGroup/OpenGL-Registry/raw/main/api/GLES3/gl3platform.h'),
|
||||||
|
Source('include/GLES3/gl3ext.h', None), # FIXME: I don't know what the canonical source is
|
||||||
|
],
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
'api': 'opencl',
|
||||||
|
'inc_folder': 'CL',
|
||||||
|
'sources': [
|
||||||
|
Source('include/CL/cl.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl.h'),
|
||||||
|
Source('include/CL/cl_d3d10.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_d3d10.h'),
|
||||||
|
Source('include/CL/cl_d3d11.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_d3d11.h'),
|
||||||
|
Source('include/CL/cl_dx9_media_sharing.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_dx9_media_sharing.h'),
|
||||||
|
Source('include/CL/cl_dx9_media_sharing_intel.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_dx9_media_sharing_intel.h'),
|
||||||
|
Source('include/CL/cl_egl.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_egl.h'),
|
||||||
|
Source('include/CL/cl_ext.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_ext.h'),
|
||||||
|
Source('include/CL/cl_ext_intel.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_ext_intel.h'),
|
||||||
|
Source('include/CL/cl_function_types.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_function_types.h'),
|
||||||
|
Source('include/CL/cl_gl.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_gl.h'),
|
||||||
|
Source('include/CL/cl_gl_ext.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_gl_ext.h'),
|
||||||
|
Source('include/CL/cl_half.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_half.h'),
|
||||||
|
Source('include/CL/cl_icd.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_icd.h'),
|
||||||
|
Source('include/CL/cl_layer.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_layer.h'),
|
||||||
|
Source('include/CL/cl_platform.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_platform.h'),
|
||||||
|
Source('include/CL/cl_va_api_media_sharing_intel.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_va_api_media_sharing_intel.h'),
|
||||||
|
Source('include/CL/cl_version.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/cl_version.h'),
|
||||||
|
Source('include/CL/opencl.h', 'https://github.com/KhronosGroup/OpenCL-Headers/raw/main/CL/opencl.h'),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
'api': 'opencl_intel',
|
||||||
|
'inc_folder': 'CL_intel',
|
||||||
|
'sources': [
|
||||||
|
Source('include/CL_intel/cl_ext_private.h', 'https://github.com/intel/compute-runtime/raw/master/opencl/extensions/public/cl_ext_private.h'),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
'api': 'spirv',
|
||||||
|
'sources': [
|
||||||
|
Source('src/compiler/spirv/NonSemanticShaderDebugInfo100.h', 'https://github.com/KhronosGroup/SPIRV-Headers/raw/main/include/spirv/unified1/NonSemanticShaderDebugInfo100.h'),
|
||||||
|
Source('src/compiler/spirv/spirv.h', 'https://github.com/KhronosGroup/SPIRV-Headers/raw/main/include/spirv/unified1/spirv.h'),
|
||||||
|
Source('src/compiler/spirv/spirv.core.grammar.json', 'https://github.com/KhronosGroup/SPIRV-Headers/raw/main/include/spirv/unified1/spirv.core.grammar.json'),
|
||||||
|
Source('src/compiler/spirv/OpenCL.std.h', 'https://github.com/KhronosGroup/SPIRV-Headers/raw/main/include/spirv/unified1/OpenCL.std.h'),
|
||||||
|
Source('src/compiler/spirv/GLSL.std.450.h', 'https://github.com/KhronosGroup/SPIRV-Headers/raw/main/include/spirv/unified1/GLSL.std.450.h'),
|
||||||
|
Source('src/compiler/spirv/GLSL.ext.AMD.h', 'https://github.com/KhronosGroup/glslang/raw/main/SPIRV/GLSL.ext.AMD.h'), # FIXME: is this the canonical source?
|
||||||
|
],
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
'api': 'vulkan',
|
||||||
|
'inc_folder': 'vulkan',
|
||||||
|
'sources': [
|
||||||
|
Source('src/vulkan/registry/vk.xml', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/registry/vk.xml'),
|
||||||
|
Source('include/vulkan/vulkan.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan.h'),
|
||||||
|
Source('include/vulkan/vulkan_core.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_core.h'),
|
||||||
|
Source('include/vulkan/vulkan_beta.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_beta.h'),
|
||||||
|
Source('include/vulkan/vk_icd.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vk_icd.h'),
|
||||||
|
Source('include/vulkan/vk_layer.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vk_layer.h'),
|
||||||
|
Source('include/vulkan/vk_platform.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vk_platform.h'),
|
||||||
|
Source('include/vulkan/vulkan_android.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_android.h'),
|
||||||
|
Source('include/vulkan/vulkan_directfb.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_directfb.h'),
|
||||||
|
Source('include/vulkan/vulkan_fuchsia.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_fuchsia.h'),
|
||||||
|
Source('include/vulkan/vulkan_ggp.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_ggp.h'),
|
||||||
|
Source('include/vulkan/vulkan_ios.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_ios.h'),
|
||||||
|
Source('include/vulkan/vulkan_macos.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_macos.h'),
|
||||||
|
Source('include/vulkan/vulkan_metal.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_metal.h'),
|
||||||
|
Source('include/vulkan/vulkan_ohos.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_ohos.h'),
|
||||||
|
Source('include/vulkan/vulkan_screen.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_screen.h'),
|
||||||
|
Source('include/vulkan/vulkan_vi.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_vi.h'),
|
||||||
|
Source('include/vulkan/vulkan_wayland.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_wayland.h'),
|
||||||
|
Source('include/vulkan/vulkan_win32.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_win32.h'),
|
||||||
|
Source('include/vulkan/vulkan_xcb.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_xcb.h'),
|
||||||
|
Source('include/vulkan/vulkan_xlib.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_xlib.h'),
|
||||||
|
Source('include/vulkan/vulkan_xlib_xrandr.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vulkan/vulkan_xlib_xrandr.h'),
|
||||||
|
Source('include/vulkan/vk_android_native_buffer.h', 'https://android.googlesource.com/platform/frameworks/native/+/master/vulkan/include/vulkan/vk_android_native_buffer.h?format=TEXT',
|
||||||
|
template=VK_ANDROID_NATIVE_BUFFER_TEMPLATE, remove='#include <cutils/native_handle.h>\n'),
|
||||||
|
Source('include/vk_video/vulkan_video_codec_av1std.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codec_av1std.h'),
|
||||||
|
Source('include/vk_video/vulkan_video_codec_av1std_decode.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codec_av1std_decode.h'),
|
||||||
|
Source('include/vk_video/vulkan_video_codec_av1std_encode.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codec_av1std_encode.h'),
|
||||||
|
Source('include/vk_video/vulkan_video_codec_h264std.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codec_h264std.h'),
|
||||||
|
Source('include/vk_video/vulkan_video_codec_h264std_decode.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codec_h264std_decode.h'),
|
||||||
|
Source('include/vk_video/vulkan_video_codec_h264std_encode.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codec_h264std_encode.h'),
|
||||||
|
Source('include/vk_video/vulkan_video_codec_h265std.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codec_h265std.h'),
|
||||||
|
Source('include/vk_video/vulkan_video_codec_h265std_decode.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codec_h265std_decode.h'),
|
||||||
|
Source('include/vk_video/vulkan_video_codec_h265std_encode.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codec_h265std_encode.h'),
|
||||||
|
Source('include/vk_video/vulkan_video_codec_vp9std.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codec_vp9std.h'),
|
||||||
|
Source('include/vk_video/vulkan_video_codec_vp9std_decode.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codec_vp9std_decode.h'),
|
||||||
|
Source('include/vk_video/vulkan_video_codecs_common.h', 'https://github.com/KhronosGroup/Vulkan-Headers/raw/main/include/vk_video/vulkan_video_codecs_common.h'),
|
||||||
|
Source('include/vulkan/.editorconfig', None),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
git_toplevel = subprocess.check_output(['git', 'rev-parse', '--show-toplevel'],
|
||||||
|
stderr=subprocess.DEVNULL).decode("ascii").strip()
|
||||||
|
if not pathlib.Path(git_toplevel).resolve() == pathlib.Path('.').resolve():
|
||||||
|
error('Please run this script from the root folder ({})'.format(git_toplevel))
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument('apis', nargs='*',
|
||||||
|
# the `[[]]` here is a workaround for python bug 9625
|
||||||
|
# where having `choices` breaks `nargs='*'`:
|
||||||
|
# https://bugs.python.org/issue9625
|
||||||
|
choices=[group['api'] for group in SOURCES] + [[]],
|
||||||
|
help='Only update the APIs specified.')
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
# These APIs all depend on the KHR header
|
||||||
|
depend_on_khr = set(['egl', 'gl', 'gles', 'gles2', 'gles3'])
|
||||||
|
if args.apis and 'khr' not in args.apis and depend_on_khr.intersection(set(args.apis)):
|
||||||
|
args.apis = ['khr'] + args.apis
|
||||||
|
|
||||||
|
for group in SOURCES:
|
||||||
|
if args.apis and group['api'] not in args.apis:
|
||||||
|
continue
|
||||||
|
|
||||||
|
for source in group['sources']:
|
||||||
|
source.sync()
|
||||||
|
|
||||||
|
# Make sure all the API files are handled by this script
|
||||||
|
if 'inc_folder' in group:
|
||||||
|
for file in pathlib.Path('include/' + group['inc_folder']).iterdir():
|
||||||
|
if file not in [source.file for source in group['sources']]:
|
||||||
|
error('{} is unknown, please add it to SOURCES'.format(file))
|
||||||
Executable
+88
@@ -0,0 +1,88 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# Copyright © 2019 Intel Corporation
|
||||||
|
|
||||||
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
# of this software and associated documentation files (the "Software"), to deal
|
||||||
|
# in the Software without restriction, including without limitation the rights
|
||||||
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
# copies of the Software, and to permit persons to whom the Software is
|
||||||
|
# furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
# The above copyright notice and this permission notice shall be included in
|
||||||
|
# all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
# SOFTWARE.
|
||||||
|
|
||||||
|
"""This script reads a meson build directory and gives back the command line it
|
||||||
|
was configured with.
|
||||||
|
|
||||||
|
This only works for meson 0.49.0 and newer.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import ast
|
||||||
|
import configparser
|
||||||
|
import pathlib
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> argparse.Namespace:
|
||||||
|
"""Parse arguments."""
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument(
|
||||||
|
'build_dir',
|
||||||
|
help='Path the meson build directory')
|
||||||
|
args = parser.parse_args()
|
||||||
|
return args
|
||||||
|
|
||||||
|
|
||||||
|
def load_config(path: pathlib.Path) -> configparser.ConfigParser:
|
||||||
|
"""Load config file."""
|
||||||
|
conf = configparser.ConfigParser()
|
||||||
|
with path.open() as f:
|
||||||
|
conf.read_file(f)
|
||||||
|
return conf
|
||||||
|
|
||||||
|
|
||||||
|
def build_cmd(conf: configparser.ConfigParser) -> str:
|
||||||
|
"""Rebuild the command line."""
|
||||||
|
args = []
|
||||||
|
for k, v in conf['options'].items():
|
||||||
|
if ' ' in v:
|
||||||
|
args.append(f'-D{k}="{v}"')
|
||||||
|
else:
|
||||||
|
args.append(f'-D{k}={v}')
|
||||||
|
|
||||||
|
cf = conf['properties'].get('cross_file')
|
||||||
|
if cf:
|
||||||
|
args.append('--cross-file={}'.format(cf))
|
||||||
|
nf = conf['properties'].get('native_file')
|
||||||
|
if nf:
|
||||||
|
# this will be in the form "['str', 'str']", so use ast.literal_eval to
|
||||||
|
# convert it to a list of strings.
|
||||||
|
nf = ast.literal_eval(nf)
|
||||||
|
args.extend(['--native-file={}'.format(f) for f in nf])
|
||||||
|
return ' '.join(args)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
args = parse_args()
|
||||||
|
path = pathlib.Path(args.build_dir, 'meson-private', 'cmd_line.txt')
|
||||||
|
if not path.exists():
|
||||||
|
print('Cannot find the necessary file to rebuild command line. '
|
||||||
|
'Is your meson version >= 0.49.0?', file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
conf = load_config(path)
|
||||||
|
cmd = build_cmd(conf)
|
||||||
|
print(cmd)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
# Copyright © 2017 Eric Engestrom
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
git_sha1_gen_py = files('git_sha1_gen.py')
|
||||||
|
gen_vs_module_defs_py = files('gen_vs_module_defs.py')
|
||||||
|
gen_vs_module_defs_normal_command = [
|
||||||
|
prog_python, gen_vs_module_defs_py,
|
||||||
|
'--in_file', '@INPUT@', '--out_file', '@OUTPUT@',
|
||||||
|
'--compiler_abi', cc.get_argument_syntax(),
|
||||||
|
'--compiler_id', cc.get_id(), '--cpu_family', host_machine.cpu_family()
|
||||||
|
]
|
||||||
|
symbols_check = find_program('symbols-check.py')
|
||||||
|
install_megadrivers_py = find_program('install_megadrivers.py')
|
||||||
|
install_megadrivers = [
|
||||||
|
install_megadrivers_py.full_path(),
|
||||||
|
'--libname-suffix', libname_suffix,
|
||||||
|
]
|
||||||
Executable
+182
@@ -0,0 +1,182 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
# Copyright © 2024 Valve Corporation
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import collections
|
||||||
|
import subprocess
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import textwrap
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
def trim_blank_lines(string, trailing):
|
||||||
|
lines = string.split('\n')
|
||||||
|
string = ''
|
||||||
|
empty_line = True
|
||||||
|
for i in range(len(lines)):
|
||||||
|
line_index = len(lines) - 1 - i if trailing else i
|
||||||
|
if empty_line and lines[line_index].strip() == '':
|
||||||
|
continue
|
||||||
|
|
||||||
|
write_newline = not empty_line if trailing else line_index < len(lines) - 1
|
||||||
|
newline = '\n' if write_newline else ''
|
||||||
|
if trailing:
|
||||||
|
string = lines[line_index] + newline + string
|
||||||
|
else:
|
||||||
|
string += lines[line_index] + newline
|
||||||
|
|
||||||
|
empty_line = False
|
||||||
|
|
||||||
|
return string
|
||||||
|
|
||||||
|
class TestFileChange:
|
||||||
|
def __init__(self, expected, result):
|
||||||
|
self.expected = expected
|
||||||
|
|
||||||
|
# Apply the indentation of the expectation to the result
|
||||||
|
indentation = 1000
|
||||||
|
for expected_line in expected.split('\n'):
|
||||||
|
if match := re.match(r'^(\s*)\S', expected_line):
|
||||||
|
line_indentation = len(match.group(1))
|
||||||
|
if indentation > line_indentation:
|
||||||
|
indentation = line_indentation
|
||||||
|
|
||||||
|
self.result = ''
|
||||||
|
result = result.split('\n')
|
||||||
|
for i in range(len(result)):
|
||||||
|
result_line = result[i]
|
||||||
|
indentation_str = '' if result_line.strip() == '' else ' '*indentation
|
||||||
|
self.result += indentation_str + result_line + ('\n' if i < len(result) - 1 else '')
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument('--build-dir', '-B', required=False)
|
||||||
|
parser.add_argument('--test-filter', '-f', required=False)
|
||||||
|
parser.add_argument('--update-all', '-u', action='store_true')
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
bin_path = 'src/compiler/nir/nir_tests'
|
||||||
|
if args.build_dir:
|
||||||
|
bin_path = args.build_dir + '/' + bin_path
|
||||||
|
|
||||||
|
if not os.path.isfile(bin_path):
|
||||||
|
print(f'{bin_path} \033[91m does not exist!\033[0m')
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
build_args = ['meson', 'compile']
|
||||||
|
if args.build_dir:
|
||||||
|
build_args.append(f'-C{args.build_dir}')
|
||||||
|
subprocess.run(build_args)
|
||||||
|
|
||||||
|
test_args = [bin_path]
|
||||||
|
if args.test_filter:
|
||||||
|
test_args.append(f'--gtest_filter={args.test_filter}')
|
||||||
|
|
||||||
|
env = os.environ.copy()
|
||||||
|
if args.update_all:
|
||||||
|
env['NIR_TEST_DUMP_SHADERS'] = 'true'
|
||||||
|
|
||||||
|
output = subprocess.run(test_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, env=env)
|
||||||
|
|
||||||
|
expected_pattern = re.compile(r'BEGIN EXPECTED\(([\d\w\W/.-_]+)\)')
|
||||||
|
|
||||||
|
expectations = collections.defaultdict(list)
|
||||||
|
|
||||||
|
current_file = None
|
||||||
|
current_result = None
|
||||||
|
current_expected = None
|
||||||
|
inside_result = False
|
||||||
|
inside_expected = False
|
||||||
|
|
||||||
|
# Parse the output of the test binary and gather the changed shaders.
|
||||||
|
for output_line in output.stdout.split('\n'):
|
||||||
|
if output_line.startswith('BEGIN RESULT'):
|
||||||
|
inside_result = True
|
||||||
|
current_result = ''
|
||||||
|
continue
|
||||||
|
|
||||||
|
if output_line.startswith('BEGIN EXPECTED'):
|
||||||
|
match = expected_pattern.match(output_line)
|
||||||
|
current_file = match.group(1).removeprefix('../')
|
||||||
|
inside_expected = True
|
||||||
|
current_expected = ''
|
||||||
|
continue
|
||||||
|
|
||||||
|
if output_line.startswith('END'):
|
||||||
|
if current_result is not None and current_expected is not None:
|
||||||
|
# remove trailing and leading blank lines
|
||||||
|
current_result = trim_blank_lines(current_result, True)
|
||||||
|
current_result = trim_blank_lines(current_result, False)
|
||||||
|
current_expected = trim_blank_lines(current_expected, True)
|
||||||
|
current_expected = trim_blank_lines(current_expected, False)
|
||||||
|
|
||||||
|
expectations[current_file].append(TestFileChange(current_expected, current_result))
|
||||||
|
|
||||||
|
current_result = None
|
||||||
|
current_expected = None
|
||||||
|
|
||||||
|
inside_result = False
|
||||||
|
inside_expected = False
|
||||||
|
continue
|
||||||
|
|
||||||
|
if inside_result:
|
||||||
|
current_result += output_line + '\n'
|
||||||
|
|
||||||
|
if inside_expected:
|
||||||
|
current_expected += output_line + '\n'
|
||||||
|
|
||||||
|
patches = []
|
||||||
|
|
||||||
|
# Generate patches for the changed shaders.
|
||||||
|
for file in expectations:
|
||||||
|
changes = expectations[file]
|
||||||
|
|
||||||
|
updated_test_file = ''
|
||||||
|
|
||||||
|
with open(file) as test_file:
|
||||||
|
updated_test_file = str(test_file.read())
|
||||||
|
|
||||||
|
for change in changes:
|
||||||
|
updated_test_file = updated_test_file.replace(change.expected, change.result)
|
||||||
|
|
||||||
|
# change.expected == change.result can be the case when using NIR_TEST_DUMP_SHADERS.
|
||||||
|
if change.expected in updated_test_file and change.expected != change.result:
|
||||||
|
print(f'Duplicate test case in {file}!')
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
with tempfile.NamedTemporaryFile(delete_on_close=False) as tmp:
|
||||||
|
tmp.write(bytes(updated_test_file, encoding="utf-8"))
|
||||||
|
tmp.close()
|
||||||
|
|
||||||
|
diff = subprocess.run(
|
||||||
|
['git', 'diff', '--no-index', file, tmp.name],
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
|
universal_newlines=True,
|
||||||
|
)
|
||||||
|
patch = diff.stdout.replace(tmp.name, '/' + file)
|
||||||
|
|
||||||
|
print(patch)
|
||||||
|
|
||||||
|
patches.append(patch)
|
||||||
|
|
||||||
|
if len(patches) != 0:
|
||||||
|
sys.stdout.write('\033[96mApply the changes listed above?\033[0m [Y/n]')
|
||||||
|
response = None
|
||||||
|
try:
|
||||||
|
response = input()
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
print()
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
if response in ['', 'y', 'Y']:
|
||||||
|
for patch in patches:
|
||||||
|
apply = subprocess.Popen(
|
||||||
|
['git', 'apply', '--allow-empty'],
|
||||||
|
stdin=subprocess.PIPE,
|
||||||
|
)
|
||||||
|
apply.communicate(input=bytes(patch, encoding="utf-8"))
|
||||||
+140
@@ -0,0 +1,140 @@
|
|||||||
|
Aaron Ruby, [email protected], aruby
|
||||||
|
Adam Jackson, [email protected], ajax
|
||||||
|
Alejandro Piñeiro, [email protected], apinheiro
|
||||||
|
Aleksi Sapon, [email protected], DDoSQc
|
||||||
|
Alyssa Rosenzweig, [email protected], alyssa
|
||||||
|
Andrew Wolfers, [email protected], aswolfers
|
||||||
|
Antonio Ospite, [email protected], ao2
|
||||||
|
Benjamin Cheng, [email protected], bcheng
|
||||||
|
Benjamin ROBIN, [email protected], benjarobin
|
||||||
|
Boris Brezillon, [email protected], bbrezillon
|
||||||
|
Brian Paul, [email protected], brianp
|
||||||
|
Caio Oliveira, [email protected], cmarcelo
|
||||||
|
Caleb Callaway, [email protected], ccallawa
|
||||||
|
Casey Bowman, [email protected], cgbowman
|
||||||
|
Caterina Shablia, [email protected], nanokatze
|
||||||
|
Chia-I Wu, [email protected], olv
|
||||||
|
Christian Gmeiner, [email protected], austriancoder
|
||||||
|
Christopher Michael, [email protected], cpmichael
|
||||||
|
Christoph Pillmayer, [email protected], chrpil
|
||||||
|
Colin Marc, [email protected], colinmarc
|
||||||
|
Connor Abbott, [email protected], cwabbott0
|
||||||
|
Corentin Noël, [email protected], tintou
|
||||||
|
Daniel Schürmann, [email protected], daniel-schuermann
|
||||||
|
Daniel Stone, [email protected], daniels
|
||||||
|
Danylo Piliaiev, [email protected], Danil
|
||||||
|
Dave Airlie, [email protected], airlied
|
||||||
|
David Heidelberg, [email protected], dh
|
||||||
|
David Rosca, [email protected], nowrep
|
||||||
|
David Tobolik, [email protected], dtobolik
|
||||||
|
Deborah Brouwer, [email protected], dbrouwer
|
||||||
|
Detlev Casanova, [email protected], detlev.c
|
||||||
|
Dmitry Baryshkov, [email protected], lumag
|
||||||
|
Dudemanguy, [email protected], Dudemanguy
|
||||||
|
duncan.hopkins, [email protected], duncan.hopkins
|
||||||
|
Dylan Baker, [email protected], dbaker
|
||||||
|
Emma Anholt, [email protected], anholt
|
||||||
|
Emmanuel Gil Peyrot, [email protected], linkmauve
|
||||||
|
Eric Engestrom, [email protected], eric
|
||||||
|
Erico Nunes, [email protected], enunes
|
||||||
|
Eric R. Smith, [email protected], ericsmith
|
||||||
|
Erik Faye-Lund, [email protected], kusma
|
||||||
|
Ernst Persson, [email protected], ernstp
|
||||||
|
Faith Ekstrand, [email protected], gfxstrand
|
||||||
|
Felix DeGrood, [email protected], fjdegroo
|
||||||
|
Feng Jiang, [email protected], flynnjiang
|
||||||
|
Francisco Jerez, [email protected], currojerez
|
||||||
|
Georg Lehmann, [email protected], DadSchoorse
|
||||||
|
Guilherme Gallo, [email protected], gallo
|
||||||
|
Gurchetan Singh, [email protected], gurchetansingh
|
||||||
|
Hans-Kristian Arntzen, [email protected], themaister
|
||||||
|
Hyunjun Ko, [email protected], zzoon
|
||||||
|
Iago Toral Quiroga, [email protected], itoral
|
||||||
|
Ian Romanick, [email protected], idr
|
||||||
|
Icenowy Zheng, [email protected], icenowy
|
||||||
|
Igor Torrente, [email protected], igor.torrente
|
||||||
|
Ivan Avdeev, [email protected], provod
|
||||||
|
Iván Briano, [email protected], ibriano
|
||||||
|
James Hogan, [email protected], jameshogan
|
||||||
|
Janne Grunau, [email protected], jannau
|
||||||
|
Jason Macnak, [email protected], jmacnak
|
||||||
|
Jesse Natalie, [email protected], jenatali
|
||||||
|
Job Noorman, [email protected], jnoorman
|
||||||
|
John Anthony, [email protected], jna
|
||||||
|
Jordan Justen, [email protected], jljusten
|
||||||
|
Jose Maria Casanova Crespo, [email protected], chema
|
||||||
|
José Roberto de Souza, [email protected], zehortigoza
|
||||||
|
Juan A. Suarez Romero, [email protected], jasuarez
|
||||||
|
Julia Zhang, [email protected], Julia
|
||||||
|
Jung-uk Kim, [email protected], jkim
|
||||||
|
Juston Li, [email protected], justonli
|
||||||
|
Karmjit Mahil, [email protected], CreativeCylon
|
||||||
|
Karol Herbst, [email protected], karolherbst
|
||||||
|
Kenneth Graunke, [email protected], kwg
|
||||||
|
Kevin Chuang, [email protected], kaiwenjon
|
||||||
|
Konstantin Seurer, [email protected], KonstantinSeurer
|
||||||
|
Lars-Ivar Hesselberg Simonsen, [email protected], larsivsi
|
||||||
|
lcagustini, [email protected], lcagustini
|
||||||
|
Leonard Göhrs, [email protected], hnez
|
||||||
|
Lionel Landwerlin, [email protected], llandwerlin
|
||||||
|
liuqiang, [email protected], genhoayi
|
||||||
|
Lorenzo Rossi, [email protected], SnowyCoder
|
||||||
|
Lucas De Marchi, [email protected], demarchi
|
||||||
|
Lucas Stach, [email protected], lynxeye
|
||||||
|
Lukas Lipp, [email protected], fknfilewalker
|
||||||
|
Maíra Canal, [email protected], mairacanal
|
||||||
|
Manuel Dun, [email protected], manueldun
|
||||||
|
Marc Herbert, [email protected], marc-hb
|
||||||
|
Marek Olšák, [email protected], mareko
|
||||||
|
Mark Collins, [email protected], PixelyIon
|
||||||
|
Martin Krastev, [email protected], blu
|
||||||
|
Martin Roukala (né Peres), [email protected], mupuf
|
||||||
|
Mary Guillemard, [email protected], marysaka
|
||||||
|
Matthew Brost, [email protected], mbrost
|
||||||
|
Matt Turner, [email protected], mattst88
|
||||||
|
Mauro Rossi, [email protected], issor.oruam
|
||||||
|
Mel Henning, [email protected], mhenning
|
||||||
|
Michael Cheng, [email protected], michaelonchrome
|
||||||
|
Michel Dänzer, [email protected], daenzer
|
||||||
|
Mike Blumenkrantz, [email protected], zmike
|
||||||
|
Mohamed Ahmed, [email protected], mohamexiety
|
||||||
|
Nanley Chery, [email protected], nchery
|
||||||
|
Natalie Vock, [email protected], pixelcluster
|
||||||
|
Olivia Lee, [email protected], olivia
|
||||||
|
Patrick Lerda, [email protected], noblock
|
||||||
|
Paulo Zanoni, [email protected], pzanoni
|
||||||
|
Pavel Ondračka, [email protected], ondracka
|
||||||
|
Peyton Lee, [email protected], peytolee
|
||||||
|
Pierre-Eric Pelloux-Prayer, [email protected], pepp
|
||||||
|
Pohsiang (John) Hsu, [email protected], pohhsu_microsoft
|
||||||
|
Qiang Yu, [email protected], yuq825
|
||||||
|
Rhys Perry, [email protected], pendingchaos
|
||||||
|
Rob Clark, [email protected], robclark
|
||||||
|
Roland Scheidegger, [email protected], sroland
|
||||||
|
Ruijing Dong, [email protected], rdong
|
||||||
|
Ryan Mckeever, [email protected], rmckeever
|
||||||
|
Sagar Ghuge, [email protected], sagarghuge
|
||||||
|
Samuel Pitoiset, [email protected], hakzsam
|
||||||
|
Saroj Kumar, [email protected], sarojk
|
||||||
|
Sathishkumar S, [email protected], sathish
|
||||||
|
Serdar Kocdemir, [email protected], kocdemir
|
||||||
|
Sergi Blanch Torne, [email protected], sergi
|
||||||
|
Sil Vilerino, [email protected], sivileri
|
||||||
|
Simon Ser, [email protected], emersion
|
||||||
|
Sonny Jiang, [email protected], sonny
|
||||||
|
Stéphane Cerveau, [email protected], dabrain34
|
||||||
|
Tapani Pälli, [email protected], tpalli
|
||||||
|
Thomas H.P. Andersen, [email protected], phomes
|
||||||
|
Tim Keller, tjkeller.xyz, tjk
|
||||||
|
Timothy Arceri, [email protected], tarceri
|
||||||
|
Timur Kristóf, [email protected], Venemo
|
||||||
|
Valentine Burley, [email protected], Valentine
|
||||||
|
Vasily Khoruzhick, [email protected], anarsoul
|
||||||
|
Vignesh Raman, [email protected], vigneshraman
|
||||||
|
Vinson Lee, [email protected], vlee
|
||||||
|
Xaver Hugl, [email protected], Zamundaaa
|
||||||
|
Yiwei Zhang, [email protected], zzyiwei
|
||||||
|
Yogesh Mohan Marimuthu, [email protected], yogeshmohan
|
||||||
|
Yonggang Luo, [email protected], lygstate
|
||||||
|
Zan Dobersek, [email protected], zdobersek
|
||||||
|
Zoltán Böszörményi, [email protected], zboszor
|
||||||
|
Executable
+251
@@ -0,0 +1,251 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
#
|
||||||
|
# Copyright 2012 VMware Inc
|
||||||
|
# Copyright 2008-2009 Jose Fonseca
|
||||||
|
#
|
||||||
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
# of this software and associated documentation files (the "Software"), to deal
|
||||||
|
# in the Software without restriction, including without limitation the rights
|
||||||
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
# copies of the Software, and to permit persons to whom the Software is
|
||||||
|
# furnished to do so, subject to the following conditions:
|
||||||
|
#
|
||||||
|
# The above copyright notice and this permission notice shall be included in
|
||||||
|
# all copies or substantial portions of the Software.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||||
|
# THE SOFTWARE.
|
||||||
|
#
|
||||||
|
|
||||||
|
"""Perf annotate for JIT code.
|
||||||
|
|
||||||
|
Linux `perf annotate` does not work with JIT code. This script takes the data
|
||||||
|
produced by `perf script` command, plus the diassemblies outputted by gallivm
|
||||||
|
into /tmp/perf-XXXXX.map.asm and produces output similar to `perf annotate`.
|
||||||
|
|
||||||
|
See docs/llvmpipe.rst for usage instructions.
|
||||||
|
|
||||||
|
The `perf script` output parser was derived from the gprof2dot.py script.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import os.path
|
||||||
|
import re
|
||||||
|
import optparse
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
|
||||||
|
class Parser:
|
||||||
|
"""Parser interface."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def parse(self):
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
|
||||||
|
class LineParser(Parser):
|
||||||
|
"""Base class for parsers that read line-based formats."""
|
||||||
|
|
||||||
|
def __init__(self, file):
|
||||||
|
Parser.__init__(self)
|
||||||
|
self._file = file
|
||||||
|
self.__line = None
|
||||||
|
self.__eof = False
|
||||||
|
self.line_no = 0
|
||||||
|
|
||||||
|
def readline(self):
|
||||||
|
line = self._file.readline()
|
||||||
|
if not line:
|
||||||
|
self.__line = ''
|
||||||
|
self.__eof = True
|
||||||
|
else:
|
||||||
|
self.line_no += 1
|
||||||
|
self.__line = line.rstrip('\r\n')
|
||||||
|
|
||||||
|
def lookahead(self):
|
||||||
|
assert self.__line is not None
|
||||||
|
return self.__line
|
||||||
|
|
||||||
|
def consume(self):
|
||||||
|
assert self.__line is not None
|
||||||
|
line = self.__line
|
||||||
|
self.readline()
|
||||||
|
return line
|
||||||
|
|
||||||
|
def eof(self):
|
||||||
|
assert self.__line is not None
|
||||||
|
return self.__eof
|
||||||
|
|
||||||
|
|
||||||
|
mapFile = None
|
||||||
|
|
||||||
|
def lookupMap(filename, matchSymbol):
|
||||||
|
global mapFile
|
||||||
|
mapFile = filename
|
||||||
|
stream = open(filename, 'rt')
|
||||||
|
for line in stream:
|
||||||
|
start, length, symbol = line.split()
|
||||||
|
|
||||||
|
start = int(start, 16)
|
||||||
|
length = int(length,16)
|
||||||
|
|
||||||
|
if symbol == matchSymbol:
|
||||||
|
return start
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
def lookupAsm(filename, desiredFunction):
|
||||||
|
stream = open(filename + '.asm', 'rt')
|
||||||
|
while not stream.readline().startswith(desiredFunction + ' '):
|
||||||
|
pass
|
||||||
|
|
||||||
|
asm = []
|
||||||
|
line = stream.readline().strip()
|
||||||
|
while line:
|
||||||
|
addr, instr = line.split(':', 1)
|
||||||
|
addr = int(addr)
|
||||||
|
asm.append((addr, instr))
|
||||||
|
line = stream.readline().strip()
|
||||||
|
|
||||||
|
return asm
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
samples = {}
|
||||||
|
|
||||||
|
|
||||||
|
class PerfParser(LineParser):
|
||||||
|
"""Parser for linux perf callgraph output.
|
||||||
|
|
||||||
|
It expects output generated with
|
||||||
|
|
||||||
|
perf record -g
|
||||||
|
perf script
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, infile, symbol):
|
||||||
|
LineParser.__init__(self, infile)
|
||||||
|
self.symbol = symbol
|
||||||
|
|
||||||
|
def readline(self):
|
||||||
|
# Override LineParser.readline to ignore comment lines
|
||||||
|
while True:
|
||||||
|
LineParser.readline(self)
|
||||||
|
if self.eof() or not self.lookahead().startswith('#'):
|
||||||
|
break
|
||||||
|
|
||||||
|
def parse(self):
|
||||||
|
# read lookahead
|
||||||
|
self.readline()
|
||||||
|
|
||||||
|
while not self.eof():
|
||||||
|
self.parse_event()
|
||||||
|
|
||||||
|
asm = lookupAsm(mapFile, self.symbol)
|
||||||
|
|
||||||
|
addresses = samples.keys()
|
||||||
|
addresses.sort()
|
||||||
|
total_samples = 0
|
||||||
|
|
||||||
|
sys.stdout.write('%s:\n' % self.symbol)
|
||||||
|
for address, instr in asm:
|
||||||
|
try:
|
||||||
|
sample = samples.pop(address)
|
||||||
|
except KeyError:
|
||||||
|
sys.stdout.write(6*' ')
|
||||||
|
else:
|
||||||
|
sys.stdout.write('%6u' % (sample))
|
||||||
|
total_samples += sample
|
||||||
|
sys.stdout.write('%6u: %s\n' % (address, instr))
|
||||||
|
print('total:', total_samples)
|
||||||
|
assert len(samples) == 0
|
||||||
|
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
def parse_event(self):
|
||||||
|
if self.eof():
|
||||||
|
return
|
||||||
|
|
||||||
|
line = self.consume()
|
||||||
|
assert line
|
||||||
|
|
||||||
|
callchain = self.parse_callchain()
|
||||||
|
if not callchain:
|
||||||
|
return
|
||||||
|
|
||||||
|
def parse_callchain(self):
|
||||||
|
callchain = []
|
||||||
|
while self.lookahead():
|
||||||
|
function = self.parse_call(len(callchain) == 0)
|
||||||
|
if function is None:
|
||||||
|
break
|
||||||
|
callchain.append(function)
|
||||||
|
if self.lookahead() == '':
|
||||||
|
self.consume()
|
||||||
|
return callchain
|
||||||
|
|
||||||
|
call_re = re.compile(r'^\s+(?P<address>[0-9a-fA-F]+)\s+(?P<symbol>.*)\s+\((?P<module>[^)]*)\)$')
|
||||||
|
|
||||||
|
def parse_call(self, first):
|
||||||
|
line = self.consume()
|
||||||
|
mo = self.call_re.match(line)
|
||||||
|
assert mo
|
||||||
|
if not mo:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if not first:
|
||||||
|
return None
|
||||||
|
|
||||||
|
function_name = mo.group('symbol')
|
||||||
|
if not function_name:
|
||||||
|
function_name = mo.group('address')
|
||||||
|
|
||||||
|
module = mo.group('module')
|
||||||
|
|
||||||
|
function_id = function_name + ':' + module
|
||||||
|
|
||||||
|
address = mo.group('address')
|
||||||
|
address = int(address, 16)
|
||||||
|
|
||||||
|
if function_name != self.symbol:
|
||||||
|
return None
|
||||||
|
|
||||||
|
start_address = lookupMap(module, function_name)
|
||||||
|
address -= start_address
|
||||||
|
|
||||||
|
#print(function_name, module, address)
|
||||||
|
|
||||||
|
samples[address] = samples.get(address, 0) + 1
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
"""Main program."""
|
||||||
|
|
||||||
|
optparser = optparse.OptionParser(
|
||||||
|
usage="\n\t%prog [options] symbol_name")
|
||||||
|
(options, args) = optparser.parse_args(sys.argv[1:])
|
||||||
|
if len(args) != 1:
|
||||||
|
optparser.error('wrong number of arguments')
|
||||||
|
|
||||||
|
symbol = args[0]
|
||||||
|
|
||||||
|
p = subprocess.Popen(['perf', 'script'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
|
parser = PerfParser(p.stdout, symbol)
|
||||||
|
parser.parse()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
|
|
||||||
|
|
||||||
|
# vim: set sw=4 et:
|
||||||
Executable
+35
@@ -0,0 +1,35 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# Copyright © 2019-2020 Intel Corporation
|
||||||
|
|
||||||
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
# of this software and associated documentation files (the "Software"), to deal
|
||||||
|
# in the Software without restriction, including without limitation the rights
|
||||||
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
# copies of the Software, and to permit persons to whom the Software is
|
||||||
|
# furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
# The above copyright notice and this permission notice shall be included in
|
||||||
|
# all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
# SOFTWARE.
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
import urwid
|
||||||
|
|
||||||
|
from pick.ui import UI, PALETTE
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
u = UI()
|
||||||
|
asyncio_loop = asyncio.new_event_loop()
|
||||||
|
asyncio.set_event_loop(asyncio_loop)
|
||||||
|
evl = urwid.AsyncioEventLoop(loop=asyncio_loop)
|
||||||
|
loop = urwid.MainLoop(u.render(), PALETTE, event_loop=evl, handle_mouse=False)
|
||||||
|
u.mainloop = loop
|
||||||
|
loop.run()
|
||||||
Executable
+10
@@ -0,0 +1,10 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
this_dir=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
|
||||||
|
readonly this_dir
|
||||||
|
|
||||||
|
exec \
|
||||||
|
"$this_dir/python-venv.sh" \
|
||||||
|
"$this_dir/pick/requirements.txt" \
|
||||||
|
"$this_dir/pick-ui.py" "$@"
|
||||||
Whitespace-only changes.
@@ -0,0 +1,405 @@
|
|||||||
|
# Copyright © 2019-2020 Intel Corporation
|
||||||
|
|
||||||
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
# of this software and associated documentation files (the "Software"), to deal
|
||||||
|
# in the Software without restriction, including without limitation the rights
|
||||||
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
# copies of the Software, and to permit persons to whom the Software is
|
||||||
|
# furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
# The above copyright notice and this permission notice shall be included in
|
||||||
|
# all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
# SOFTWARE.
|
||||||
|
|
||||||
|
"""Core data structures and routines for pick."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import enum
|
||||||
|
import json
|
||||||
|
import pathlib
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import typing
|
||||||
|
|
||||||
|
import attr
|
||||||
|
from packaging.version import Version
|
||||||
|
|
||||||
|
if typing.TYPE_CHECKING:
|
||||||
|
from .ui import UI
|
||||||
|
|
||||||
|
import typing_extensions
|
||||||
|
|
||||||
|
class CommitDict(typing_extensions.TypedDict):
|
||||||
|
|
||||||
|
sha: str
|
||||||
|
description: str
|
||||||
|
nominated: bool
|
||||||
|
nomination_type: int
|
||||||
|
resolution: typing.Optional[int]
|
||||||
|
main_sha: typing.Optional[str]
|
||||||
|
because_sha: typing.Optional[str]
|
||||||
|
notes: typing.Optional[str] = attr.ib(None)
|
||||||
|
|
||||||
|
IS_FIX = re.compile(r'^\s*fixes:\s*([a-f0-9]{6,40})', flags=re.MULTILINE | re.IGNORECASE)
|
||||||
|
# FIXME: I dislike the duplication in this regex, but I couldn't get it to work otherwise
|
||||||
|
IS_CC = re.compile(r'^\s*cc:\s*["\']?([0-9]{2}\.[0-9])?["\']?\s*["\']?([0-9]{2}\.[0-9])?["\']?\s*\<?mesa-stable',
|
||||||
|
flags=re.MULTILINE | re.IGNORECASE)
|
||||||
|
IS_REVERT = re.compile(r'This reverts commit ([0-9a-f]{40})')
|
||||||
|
IS_BACKPORT = re.compile(r'^\s*backport-to:\s*(?:(\d{2}\.\d),?\s*(\d{2}\.\d)?|(\*))',
|
||||||
|
flags=re.MULTILINE | re.IGNORECASE)
|
||||||
|
|
||||||
|
# XXX: hack
|
||||||
|
SEM = asyncio.Semaphore(50)
|
||||||
|
|
||||||
|
COMMIT_LOCK = asyncio.Lock()
|
||||||
|
|
||||||
|
git_toplevel = subprocess.check_output(['git', 'rev-parse', '--show-toplevel'],
|
||||||
|
stderr=subprocess.DEVNULL).decode("ascii").strip()
|
||||||
|
pick_status_json = pathlib.Path(git_toplevel) / '.pick_status.json'
|
||||||
|
|
||||||
|
|
||||||
|
class PickUIException(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@enum.unique
|
||||||
|
class NominationType(enum.Enum):
|
||||||
|
|
||||||
|
NONE = 0
|
||||||
|
CC = 1
|
||||||
|
FIXES = 2
|
||||||
|
REVERT = 3
|
||||||
|
BACKPORT = 4
|
||||||
|
|
||||||
|
|
||||||
|
@enum.unique
|
||||||
|
class Resolution(enum.Enum):
|
||||||
|
|
||||||
|
UNRESOLVED = 0
|
||||||
|
MERGED = 1
|
||||||
|
DENOMINATED = 2
|
||||||
|
BACKPORTED = 3
|
||||||
|
NOTNEEDED = 4
|
||||||
|
|
||||||
|
|
||||||
|
async def commit_state(*, amend: bool = False, message: str = 'Update') -> bool:
|
||||||
|
"""Commit the .pick_status.json file."""
|
||||||
|
async with COMMIT_LOCK:
|
||||||
|
p = await asyncio.create_subprocess_exec(
|
||||||
|
'git', 'add', pick_status_json.as_posix(),
|
||||||
|
stdout=asyncio.subprocess.DEVNULL,
|
||||||
|
stderr=asyncio.subprocess.DEVNULL,
|
||||||
|
)
|
||||||
|
v = await p.wait()
|
||||||
|
if v != 0:
|
||||||
|
return False
|
||||||
|
|
||||||
|
if amend:
|
||||||
|
cmd = ['--amend', '--no-edit']
|
||||||
|
else:
|
||||||
|
cmd = ['--message', f'.pick_status.json: {message}']
|
||||||
|
p = await asyncio.create_subprocess_exec(
|
||||||
|
'git', 'commit', *cmd,
|
||||||
|
stdout=asyncio.subprocess.DEVNULL,
|
||||||
|
stderr=asyncio.subprocess.DEVNULL,
|
||||||
|
)
|
||||||
|
v = await p.wait()
|
||||||
|
if v != 0:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
@attr.s(slots=True)
|
||||||
|
class Commit:
|
||||||
|
|
||||||
|
sha: str = attr.ib()
|
||||||
|
description: str = attr.ib()
|
||||||
|
nominated: bool = attr.ib(False)
|
||||||
|
nomination_type: NominationType = attr.ib(NominationType.NONE)
|
||||||
|
resolution: Resolution = attr.ib(Resolution.UNRESOLVED)
|
||||||
|
main_sha: typing.Optional[str] = attr.ib(None)
|
||||||
|
because_sha: typing.Optional[str] = attr.ib(None)
|
||||||
|
notes: typing.Optional[str] = attr.ib(None)
|
||||||
|
|
||||||
|
def to_json(self) -> 'CommitDict':
|
||||||
|
d: typing.Dict[str, typing.Any] = attr.asdict(self)
|
||||||
|
d['nomination_type'] = self.nomination_type.value
|
||||||
|
if self.resolution is not None:
|
||||||
|
d['resolution'] = self.resolution.value
|
||||||
|
return typing.cast('CommitDict', d)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_json(cls, data: 'CommitDict') -> 'Commit':
|
||||||
|
c = cls(data['sha'], data['description'], data['nominated'], main_sha=data['main_sha'],
|
||||||
|
because_sha=data['because_sha'], notes=data['notes'])
|
||||||
|
c.nomination_type = NominationType(data['nomination_type'])
|
||||||
|
if data['resolution'] is not None:
|
||||||
|
c.resolution = Resolution(data['resolution'])
|
||||||
|
return c
|
||||||
|
|
||||||
|
def date(self) -> str:
|
||||||
|
# Show commit date, ie. when the commit actually landed
|
||||||
|
# (as opposed to when it was first written)
|
||||||
|
return subprocess.check_output(
|
||||||
|
['git', 'show', '--no-patch', '--format=%cs', self.sha],
|
||||||
|
stderr=subprocess.DEVNULL
|
||||||
|
).decode("ascii").strip()
|
||||||
|
|
||||||
|
async def apply(self, ui: 'UI') -> typing.Tuple[bool, str]:
|
||||||
|
# FIXME: This isn't really enough if we fail to cherry-pick because the
|
||||||
|
# git tree will still be dirty
|
||||||
|
async with COMMIT_LOCK:
|
||||||
|
p = await asyncio.create_subprocess_exec(
|
||||||
|
'git', 'cherry-pick', '-x', self.sha,
|
||||||
|
stdout=asyncio.subprocess.DEVNULL,
|
||||||
|
stderr=asyncio.subprocess.PIPE,
|
||||||
|
)
|
||||||
|
_, err = await p.communicate()
|
||||||
|
|
||||||
|
if p.returncode != 0:
|
||||||
|
return (False, err.decode())
|
||||||
|
|
||||||
|
self.resolution = Resolution.MERGED
|
||||||
|
await ui.feedback(f'{self.sha} ({self.description}) applied successfully')
|
||||||
|
|
||||||
|
# Append the changes to the .pickstatus.json file
|
||||||
|
ui.save()
|
||||||
|
v = await commit_state(amend=True)
|
||||||
|
return (v, '')
|
||||||
|
|
||||||
|
async def abort_cherry(self, ui: 'UI', err: str) -> None:
|
||||||
|
await ui.feedback(f'{self.sha} ({self.description}) failed to apply\n{err}')
|
||||||
|
async with COMMIT_LOCK:
|
||||||
|
p = await asyncio.create_subprocess_exec(
|
||||||
|
'git', 'cherry-pick', '--abort',
|
||||||
|
stdout=asyncio.subprocess.DEVNULL,
|
||||||
|
stderr=asyncio.subprocess.DEVNULL,
|
||||||
|
)
|
||||||
|
r = await p.wait()
|
||||||
|
await ui.feedback(f'{"Successfully" if r == 0 else "Failed to"} abort cherry-pick.')
|
||||||
|
|
||||||
|
async def denominate(self, ui: 'UI') -> bool:
|
||||||
|
self.resolution = Resolution.DENOMINATED
|
||||||
|
ui.save()
|
||||||
|
v = await commit_state(message=f'Mark {self.sha} as denominated')
|
||||||
|
assert v
|
||||||
|
await ui.feedback(f'{self.sha} ({self.description}) denominated successfully')
|
||||||
|
return True
|
||||||
|
|
||||||
|
async def backport(self, ui: 'UI') -> bool:
|
||||||
|
self.resolution = Resolution.BACKPORTED
|
||||||
|
ui.save()
|
||||||
|
v = await commit_state(message=f'Mark {self.sha} as backported')
|
||||||
|
assert v
|
||||||
|
await ui.feedback(f'{self.sha} ({self.description}) backported successfully')
|
||||||
|
return True
|
||||||
|
|
||||||
|
async def resolve(self, ui: 'UI') -> None:
|
||||||
|
self.resolution = Resolution.MERGED
|
||||||
|
ui.save()
|
||||||
|
v = await commit_state(amend=True)
|
||||||
|
assert v
|
||||||
|
await ui.feedback(f'{self.sha} ({self.description}) committed successfully')
|
||||||
|
|
||||||
|
async def update_notes(self, ui: 'UI', notes: typing.Optional[str]) -> None:
|
||||||
|
self.notes = notes
|
||||||
|
async with ui.git_lock:
|
||||||
|
ui.save()
|
||||||
|
v = await commit_state(message=f'Updates notes for {self.sha}')
|
||||||
|
assert v
|
||||||
|
await ui.feedback(f'{self.sha} ({self.description}) notes updated successfully')
|
||||||
|
|
||||||
|
|
||||||
|
async def get_new_commits(sha: str) -> typing.List[typing.Tuple[str, str]]:
|
||||||
|
# Try to get the authoritative upstream main
|
||||||
|
p = await asyncio.create_subprocess_exec(
|
||||||
|
'git', 'for-each-ref', '--format=%(upstream)', 'refs/heads/main',
|
||||||
|
stdout=asyncio.subprocess.PIPE,
|
||||||
|
stderr=asyncio.subprocess.DEVNULL)
|
||||||
|
out, _ = await p.communicate()
|
||||||
|
upstream = out.decode().strip()
|
||||||
|
|
||||||
|
p = await asyncio.create_subprocess_exec(
|
||||||
|
'git', 'log', '--pretty=oneline', f'{sha}..{upstream}',
|
||||||
|
stdout=asyncio.subprocess.PIPE,
|
||||||
|
stderr=asyncio.subprocess.DEVNULL)
|
||||||
|
out, _ = await p.communicate()
|
||||||
|
assert p.returncode == 0, f"git log didn't work: {sha}"
|
||||||
|
return list(split_commit_list(out.decode().strip()))
|
||||||
|
|
||||||
|
|
||||||
|
def split_commit_list(commits: str) -> typing.Generator[typing.Tuple[str, str], None, None]:
|
||||||
|
if not commits:
|
||||||
|
return
|
||||||
|
for line in commits.split('\n'):
|
||||||
|
v = tuple(line.split(' ', 1))
|
||||||
|
assert len(v) == 2, 'this is really just for mypy'
|
||||||
|
yield typing.cast(typing.Tuple[str, str], v)
|
||||||
|
|
||||||
|
|
||||||
|
async def is_commit_in_branch(sha: str) -> bool:
|
||||||
|
async with SEM:
|
||||||
|
p = await asyncio.create_subprocess_exec(
|
||||||
|
'git', 'merge-base', '--is-ancestor', sha, 'HEAD',
|
||||||
|
stdout=asyncio.subprocess.DEVNULL,
|
||||||
|
stderr=asyncio.subprocess.DEVNULL,
|
||||||
|
)
|
||||||
|
await p.wait()
|
||||||
|
return p.returncode == 0
|
||||||
|
|
||||||
|
|
||||||
|
async def full_sha(sha: str) -> str:
|
||||||
|
async with SEM:
|
||||||
|
p = await asyncio.create_subprocess_exec(
|
||||||
|
'git', 'rev-parse', sha,
|
||||||
|
stdout=asyncio.subprocess.PIPE,
|
||||||
|
stderr=asyncio.subprocess.DEVNULL,
|
||||||
|
)
|
||||||
|
out, _ = await p.communicate()
|
||||||
|
if p.returncode:
|
||||||
|
raise PickUIException(f'Invalid Sha {sha}')
|
||||||
|
return out.decode().strip()
|
||||||
|
|
||||||
|
|
||||||
|
async def resolve_nomination(commit: 'Commit', version: str) -> 'Commit':
|
||||||
|
async with SEM:
|
||||||
|
p = await asyncio.create_subprocess_exec(
|
||||||
|
'git', 'log', '--format=%B', '-1', commit.sha,
|
||||||
|
stdout=asyncio.subprocess.PIPE,
|
||||||
|
stderr=asyncio.subprocess.DEVNULL,
|
||||||
|
)
|
||||||
|
_out, _ = await p.communicate()
|
||||||
|
assert p.returncode == 0, f'git log for {commit.sha} failed'
|
||||||
|
commit_message = _out.decode()
|
||||||
|
|
||||||
|
# We give precedence to fixes and cc tags over revert tags.
|
||||||
|
if fix_for_commit := IS_FIX.search(commit_message):
|
||||||
|
# We set the nomination_type and because_sha here so that we can later
|
||||||
|
# check to see if this fixes another staged commit.
|
||||||
|
try:
|
||||||
|
commit.because_sha = fixed = await full_sha(fix_for_commit.group(1))
|
||||||
|
except PickUIException:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
commit.nomination_type = NominationType.FIXES
|
||||||
|
if await is_commit_in_branch(fixed):
|
||||||
|
commit.nominated = True
|
||||||
|
return commit
|
||||||
|
|
||||||
|
if backport_to := IS_BACKPORT.findall(commit_message):
|
||||||
|
for match in backport_to:
|
||||||
|
if any(backport_version == '*' or Version(version) >= Version(backport_version)
|
||||||
|
for backport_version in match if backport_version != ''):
|
||||||
|
commit.nominated = True
|
||||||
|
commit.nomination_type = NominationType.BACKPORT
|
||||||
|
return commit
|
||||||
|
|
||||||
|
if cc_to := IS_CC.search(commit_message):
|
||||||
|
if cc_to.groups() == (None, None) or version in cc_to.groups():
|
||||||
|
commit.nominated = True
|
||||||
|
commit.nomination_type = NominationType.CC
|
||||||
|
return commit
|
||||||
|
|
||||||
|
if revert_of := IS_REVERT.search(commit_message):
|
||||||
|
# See comment for IS_FIX path
|
||||||
|
try:
|
||||||
|
commit.because_sha = reverted = await full_sha(revert_of.group(1))
|
||||||
|
except PickUIException:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
commit.nomination_type = NominationType.REVERT
|
||||||
|
if await is_commit_in_branch(reverted):
|
||||||
|
commit.nominated = True
|
||||||
|
return commit
|
||||||
|
|
||||||
|
return commit
|
||||||
|
|
||||||
|
|
||||||
|
async def resolve_fixes(commits: typing.List['Commit'], previous: typing.List['Commit']) -> None:
|
||||||
|
"""Determine if any of the undecided commits fix/revert a staged commit.
|
||||||
|
|
||||||
|
The are still needed if they apply to a commit that is staged for
|
||||||
|
inclusion, but not yet included.
|
||||||
|
|
||||||
|
This must be done in order, because a commit 3 might fix commit 2 which
|
||||||
|
fixes commit 1.
|
||||||
|
"""
|
||||||
|
shas: typing.Set[str] = set(c.sha for c in previous if c.nominated)
|
||||||
|
assert None not in shas, 'None in shas'
|
||||||
|
|
||||||
|
for commit in reversed(commits):
|
||||||
|
if not commit.nominated and commit.nomination_type is NominationType.FIXES:
|
||||||
|
commit.nominated = commit.because_sha in shas
|
||||||
|
|
||||||
|
if commit.nominated:
|
||||||
|
shas.add(commit.sha)
|
||||||
|
|
||||||
|
for commit in commits:
|
||||||
|
if (commit.nomination_type is NominationType.REVERT and
|
||||||
|
commit.because_sha in shas):
|
||||||
|
for oldc in reversed(commits):
|
||||||
|
if oldc.sha == commit.because_sha:
|
||||||
|
# In this case a commit that hasn't yet been applied is
|
||||||
|
# reverted, we don't want to apply that commit at all
|
||||||
|
oldc.nominated = False
|
||||||
|
oldc.resolution = Resolution.DENOMINATED
|
||||||
|
commit.nominated = False
|
||||||
|
commit.resolution = Resolution.DENOMINATED
|
||||||
|
shas.remove(commit.because_sha)
|
||||||
|
break
|
||||||
|
|
||||||
|
|
||||||
|
async def gather_commits(version: str, previous: typing.List['Commit'],
|
||||||
|
new: typing.List[typing.Tuple[str, str]], cb) -> typing.List['Commit']:
|
||||||
|
# We create an array of the final size up front, then we pass that array
|
||||||
|
# to the "inner" co-routine, which is turned into a list of tasks and
|
||||||
|
# collected by asyncio.gather. We do this to allow the tasks to be
|
||||||
|
# asynchronously gathered, but to also ensure that the commits list remains
|
||||||
|
# in order.
|
||||||
|
m_commits: typing.List[typing.Optional['Commit']] = [None] * len(new)
|
||||||
|
tasks = []
|
||||||
|
|
||||||
|
async def inner(commit: 'Commit', version: str,
|
||||||
|
commits: typing.List[typing.Optional['Commit']],
|
||||||
|
index: int, cb) -> None:
|
||||||
|
commits[index] = await resolve_nomination(commit, version)
|
||||||
|
cb()
|
||||||
|
|
||||||
|
for i, (sha, desc) in enumerate(new):
|
||||||
|
tasks.append(asyncio.ensure_future(
|
||||||
|
inner(Commit(sha, desc), version, m_commits, i, cb)))
|
||||||
|
|
||||||
|
await asyncio.gather(*tasks)
|
||||||
|
assert None not in m_commits
|
||||||
|
commits = typing.cast(typing.List[Commit], m_commits)
|
||||||
|
|
||||||
|
await resolve_fixes(commits, previous)
|
||||||
|
|
||||||
|
for commit in commits:
|
||||||
|
if commit.resolution is Resolution.UNRESOLVED and not commit.nominated:
|
||||||
|
commit.resolution = Resolution.NOTNEEDED
|
||||||
|
|
||||||
|
return commits
|
||||||
|
|
||||||
|
|
||||||
|
def load() -> typing.List['Commit']:
|
||||||
|
if not pick_status_json.exists():
|
||||||
|
return []
|
||||||
|
with pick_status_json.open('r') as f:
|
||||||
|
raw = json.load(f)
|
||||||
|
return [Commit.from_json(c) for c in raw]
|
||||||
|
|
||||||
|
|
||||||
|
def save(commits: typing.Iterable['Commit']) -> None:
|
||||||
|
commits = list(commits)
|
||||||
|
with pick_status_json.open('wt') as f:
|
||||||
|
json.dump([c.to_json() for c in commits], f, indent=4)
|
||||||
|
|
||||||
|
asyncio.ensure_future(commit_state(message=f'Update to {commits[0].sha}'))
|
||||||
@@ -0,0 +1,600 @@
|
|||||||
|
# Copyright © 2019-2020 Intel Corporation
|
||||||
|
|
||||||
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
# of this software and associated documentation files (the "Software"), to deal
|
||||||
|
# in the Software without restriction, including without limitation the rights
|
||||||
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
# copies of the Software, and to permit persons to whom the Software is
|
||||||
|
# furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
# The above copyright notice and this permission notice shall be included in
|
||||||
|
# all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
# SOFTWARE.
|
||||||
|
|
||||||
|
"""Tests for pick's core data structures and routines."""
|
||||||
|
|
||||||
|
from unittest import mock
|
||||||
|
import textwrap
|
||||||
|
import typing
|
||||||
|
|
||||||
|
import attr
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from . import core
|
||||||
|
|
||||||
|
|
||||||
|
class TestCommit:
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def unnominated_commit(self) -> 'core.Commit':
|
||||||
|
return core.Commit('abc123', 'sub: A commit', main_sha='45678')
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def nominated_commit(self) -> 'core.Commit':
|
||||||
|
return core.Commit('abc123', 'sub: A commit', True,
|
||||||
|
core.NominationType.CC, core.Resolution.UNRESOLVED)
|
||||||
|
|
||||||
|
class TestToJson:
|
||||||
|
|
||||||
|
def test_not_nominated(self, unnominated_commit: 'core.Commit'):
|
||||||
|
c = unnominated_commit
|
||||||
|
v = c.to_json()
|
||||||
|
assert v == {
|
||||||
|
'sha': 'abc123',
|
||||||
|
'description': 'sub: A commit',
|
||||||
|
'nominated': False,
|
||||||
|
'nomination_type': core.NominationType.NONE.value,
|
||||||
|
'resolution': core.Resolution.UNRESOLVED.value,
|
||||||
|
'main_sha': '45678',
|
||||||
|
'because_sha': None,
|
||||||
|
'notes': None,
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_nominated(self, nominated_commit: 'core.Commit'):
|
||||||
|
c = nominated_commit
|
||||||
|
v = c.to_json()
|
||||||
|
assert v == {
|
||||||
|
'sha': 'abc123',
|
||||||
|
'description': 'sub: A commit',
|
||||||
|
'nominated': True,
|
||||||
|
'nomination_type': core.NominationType.CC.value,
|
||||||
|
'resolution': core.Resolution.UNRESOLVED.value,
|
||||||
|
'main_sha': None,
|
||||||
|
'because_sha': None,
|
||||||
|
'notes': None,
|
||||||
|
}
|
||||||
|
|
||||||
|
class TestFromJson:
|
||||||
|
|
||||||
|
def test_not_nominated(self, unnominated_commit: 'core.Commit'):
|
||||||
|
c = unnominated_commit
|
||||||
|
v = c.to_json()
|
||||||
|
c2 = core.Commit.from_json(v)
|
||||||
|
assert c == c2
|
||||||
|
|
||||||
|
def test_nominated(self, nominated_commit: 'core.Commit'):
|
||||||
|
c = nominated_commit
|
||||||
|
v = c.to_json()
|
||||||
|
c2 = core.Commit.from_json(v)
|
||||||
|
assert c == c2
|
||||||
|
|
||||||
|
|
||||||
|
class TestRE:
|
||||||
|
|
||||||
|
"""Tests for the regular expressions used to identify commits."""
|
||||||
|
|
||||||
|
class TestFixes:
|
||||||
|
|
||||||
|
def test_simple(self):
|
||||||
|
message = textwrap.dedent("""\
|
||||||
|
etnaviv: fix vertex buffer state emission for single stream GPUs
|
||||||
|
|
||||||
|
GPUs with a single supported vertex stream must use the single state
|
||||||
|
address to program the stream.
|
||||||
|
|
||||||
|
Fixes: 3d09bb390a39 (etnaviv: GC7000: State changes for HALTI3..5)
|
||||||
|
Signed-off-by: Lucas Stach <[email protected]>
|
||||||
|
Reviewed-by: Jonathan Marek <[email protected]>
|
||||||
|
""")
|
||||||
|
|
||||||
|
fix_for_commit = core.IS_FIX.search(message)
|
||||||
|
assert fix_for_commit is not None
|
||||||
|
assert fix_for_commit.group(1) == '3d09bb390a39'
|
||||||
|
|
||||||
|
class TestCC:
|
||||||
|
|
||||||
|
def test_single_branch(self):
|
||||||
|
"""Tests commit meant for a single branch, ie, 19.1"""
|
||||||
|
message = textwrap.dedent("""\
|
||||||
|
radv: fix DCC fast clear code for intensity formats
|
||||||
|
|
||||||
|
This fixes a rendering issue with DiRT 4 on GFX10. Only GFX10 was
|
||||||
|
affected because intensity formats are different.
|
||||||
|
|
||||||
|
Cc: 19.2 <[email protected]>
|
||||||
|
Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/1923
|
||||||
|
Signed-off-by: Samuel Pitoiset <[email protected]>
|
||||||
|
Reviewed-by: Bas Nieuwenhuizen <[email protected]>
|
||||||
|
""")
|
||||||
|
|
||||||
|
cc_to = core.IS_CC.search(message)
|
||||||
|
assert cc_to is not None
|
||||||
|
assert cc_to.group(1) == '19.2'
|
||||||
|
|
||||||
|
def test_multiple_branches(self):
|
||||||
|
"""Tests commit with more than one branch specified"""
|
||||||
|
message = textwrap.dedent("""\
|
||||||
|
radeonsi: enable zerovram for Rocket League
|
||||||
|
|
||||||
|
Fixes corruption on game startup.
|
||||||
|
Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/1888
|
||||||
|
|
||||||
|
Cc: 19.1 19.2 <[email protected]>
|
||||||
|
Reviewed-by: Pierre-Eric Pelloux-Prayer <[email protected]>
|
||||||
|
""")
|
||||||
|
|
||||||
|
cc_to = core.IS_CC.search(message)
|
||||||
|
assert cc_to is not None
|
||||||
|
assert cc_to.group(1) == '19.1'
|
||||||
|
assert cc_to.group(2) == '19.2'
|
||||||
|
|
||||||
|
def test_no_branch(self):
|
||||||
|
"""Tests commit with no branch specification"""
|
||||||
|
message = textwrap.dedent("""\
|
||||||
|
anv/android: fix images created with external format support
|
||||||
|
|
||||||
|
This fixes a case where user first creates image and then later binds it
|
||||||
|
with memory created from AHW buffer.
|
||||||
|
|
||||||
|
Cc: <[email protected]>
|
||||||
|
Signed-off-by: Tapani Pälli <[email protected]>
|
||||||
|
Reviewed-by: Lionel Landwerlin <[email protected]>
|
||||||
|
""")
|
||||||
|
|
||||||
|
cc_to = core.IS_CC.search(message)
|
||||||
|
assert cc_to is not None
|
||||||
|
|
||||||
|
def test_quotes(self):
|
||||||
|
"""Tests commit with quotes around the versions"""
|
||||||
|
message = textwrap.dedent("""\
|
||||||
|
anv: Always fill out the AUX table even if CCS is disabled
|
||||||
|
|
||||||
|
Cc: "20.0" [email protected]
|
||||||
|
Reviewed-by: Kenneth Graunke <[email protected]>
|
||||||
|
Tested-by: Marge Bot <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/3454>
|
||||||
|
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/3454>
|
||||||
|
""")
|
||||||
|
|
||||||
|
cc_to = core.IS_CC.search(message)
|
||||||
|
assert cc_to is not None
|
||||||
|
assert cc_to.group(1) == '20.0'
|
||||||
|
|
||||||
|
def test_multiple_quotes(self):
|
||||||
|
"""Tests commit with quotes around the versions"""
|
||||||
|
message = textwrap.dedent("""\
|
||||||
|
anv: Always fill out the AUX table even if CCS is disabled
|
||||||
|
|
||||||
|
Cc: "20.0" "20.1" [email protected]
|
||||||
|
Reviewed-by: Kenneth Graunke <[email protected]>
|
||||||
|
Tested-by: Marge Bot <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/3454>
|
||||||
|
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/3454>
|
||||||
|
""")
|
||||||
|
|
||||||
|
cc_to = core.IS_CC.search(message)
|
||||||
|
assert cc_to is not None
|
||||||
|
assert cc_to.group(1) == '20.0'
|
||||||
|
assert cc_to.group(2) == '20.1'
|
||||||
|
|
||||||
|
def test_single_quotes(self):
|
||||||
|
"""Tests commit with quotes around the versions"""
|
||||||
|
message = textwrap.dedent("""\
|
||||||
|
anv: Always fill out the AUX table even if CCS is disabled
|
||||||
|
|
||||||
|
Cc: '20.0' [email protected]
|
||||||
|
Reviewed-by: Kenneth Graunke <[email protected]>
|
||||||
|
Tested-by: Marge Bot <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/3454>
|
||||||
|
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/3454>
|
||||||
|
""")
|
||||||
|
|
||||||
|
cc_to = core.IS_CC.search(message)
|
||||||
|
assert cc_to is not None
|
||||||
|
assert cc_to.group(1) == '20.0'
|
||||||
|
|
||||||
|
def test_multiple_single_quotes(self):
|
||||||
|
"""Tests commit with quotes around the versions"""
|
||||||
|
message = textwrap.dedent("""\
|
||||||
|
anv: Always fill out the AUX table even if CCS is disabled
|
||||||
|
|
||||||
|
Cc: '20.0' '20.1' [email protected]
|
||||||
|
Reviewed-by: Kenneth Graunke <[email protected]>
|
||||||
|
Tested-by: Marge Bot <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/3454>
|
||||||
|
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/3454>
|
||||||
|
""")
|
||||||
|
|
||||||
|
cc_to = core.IS_CC.search(message)
|
||||||
|
assert cc_to is not None
|
||||||
|
assert cc_to.group(1) == '20.0'
|
||||||
|
assert cc_to.group(2) == '20.1'
|
||||||
|
|
||||||
|
class TestRevert:
|
||||||
|
|
||||||
|
def test_simple(self):
|
||||||
|
message = textwrap.dedent("""\
|
||||||
|
Revert "radv: do not emit PKT3_CONTEXT_CONTROL with AMDGPU 3.6.0+"
|
||||||
|
|
||||||
|
This reverts commit 2ca8629fa9b303e24783b76a7b3b0c2513e32fbd.
|
||||||
|
|
||||||
|
This was initially ported from RadeonSI, but in the meantime it has
|
||||||
|
been reverted because it might hang. Be conservative and re-introduce
|
||||||
|
this packet emission.
|
||||||
|
|
||||||
|
Unfortunately this doesn't fix anything known.
|
||||||
|
|
||||||
|
Cc: 19.2 <[email protected]>
|
||||||
|
Signed-off-by: Samuel Pitoiset <[email protected]>
|
||||||
|
Reviewed-by: Bas Nieuwenhuizen <[email protected]>
|
||||||
|
""")
|
||||||
|
|
||||||
|
revert_of = core.IS_REVERT.search(message)
|
||||||
|
assert revert_of is not None
|
||||||
|
assert revert_of.group(1) == '2ca8629fa9b303e24783b76a7b3b0c2513e32fbd'
|
||||||
|
|
||||||
|
class TestBackportTo:
|
||||||
|
|
||||||
|
def test_single_release(self):
|
||||||
|
"""Tests commit meant for a single branch, ie, 19.1"""
|
||||||
|
message = textwrap.dedent("""\
|
||||||
|
radv: fix DCC fast clear code for intensity formats
|
||||||
|
|
||||||
|
This fixes a rendering issue with DiRT 4 on GFX10. Only GFX10 was
|
||||||
|
affected because intensity formats are different.
|
||||||
|
|
||||||
|
Backport-to: 19.2
|
||||||
|
Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/1923
|
||||||
|
Signed-off-by: Samuel Pitoiset <[email protected]>
|
||||||
|
Reviewed-by: Bas Nieuwenhuizen <[email protected]>
|
||||||
|
""")
|
||||||
|
|
||||||
|
backport_to = core.IS_BACKPORT.findall(message)
|
||||||
|
assert backport_to == [('19.2', '', '')]
|
||||||
|
|
||||||
|
def test_multiple_release_space(self):
|
||||||
|
"""Tests commit with more than one branch specified"""
|
||||||
|
message = textwrap.dedent("""\
|
||||||
|
radeonsi: enable zerovram for Rocket League
|
||||||
|
|
||||||
|
Fixes corruption on game startup.
|
||||||
|
Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/1888
|
||||||
|
|
||||||
|
Backport-to: 19.1 19.2
|
||||||
|
Reviewed-by: Pierre-Eric Pelloux-Prayer <[email protected]>
|
||||||
|
""")
|
||||||
|
|
||||||
|
backport_to = core.IS_BACKPORT.findall(message)
|
||||||
|
assert backport_to == [('19.1', '19.2', '')]
|
||||||
|
|
||||||
|
def test_multiple_release_comma(self):
|
||||||
|
"""Tests commit with more than one branch specified"""
|
||||||
|
message = textwrap.dedent("""\
|
||||||
|
radeonsi: enable zerovram for Rocket League
|
||||||
|
|
||||||
|
Fixes corruption on game startup.
|
||||||
|
Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/1888
|
||||||
|
|
||||||
|
Backport-to: 19.1, 19.2
|
||||||
|
Reviewed-by: Pierre-Eric Pelloux-Prayer <[email protected]>
|
||||||
|
""")
|
||||||
|
|
||||||
|
backport_to = core.IS_BACKPORT.findall(message)
|
||||||
|
assert backport_to == [('19.1', '19.2', '')]
|
||||||
|
|
||||||
|
def test_multiple_release_lines(self):
|
||||||
|
"""Tests commit with more than one branch specified in mulitple tags"""
|
||||||
|
message = textwrap.dedent("""\
|
||||||
|
commit title
|
||||||
|
|
||||||
|
Backport-to: 19.0
|
||||||
|
Backport-to: 19.1, 19.2
|
||||||
|
""")
|
||||||
|
|
||||||
|
backport_to = core.IS_BACKPORT.findall(message)
|
||||||
|
assert backport_to == [('19.0', '', ''), ('19.1', '19.2', '')]
|
||||||
|
|
||||||
|
|
||||||
|
class TestResolveNomination:
|
||||||
|
|
||||||
|
@attr.s(slots=True)
|
||||||
|
class FakeSubprocess:
|
||||||
|
|
||||||
|
"""A fake asyncio.subprocess like class for use with mock."""
|
||||||
|
|
||||||
|
out: typing.Optional[bytes] = attr.ib(None)
|
||||||
|
returncode: int = attr.ib(0)
|
||||||
|
|
||||||
|
async def mock(self, *_, **__):
|
||||||
|
"""A dirtly little helper for mocking."""
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def communicate(self) -> typing.Tuple[bytes, bytes]:
|
||||||
|
assert self.out is not None
|
||||||
|
return self.out, b''
|
||||||
|
|
||||||
|
async def wait(self) -> int:
|
||||||
|
return self.returncode
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def return_true(*_, **__) -> bool:
|
||||||
|
return True
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def return_false(*_, **__) -> bool:
|
||||||
|
return False
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_fix_is_nominated(self):
|
||||||
|
s = self.FakeSubprocess(b'Fixes: 3d09bb390a39 (etnaviv: GC7000: State changes for HALTI3..5)')
|
||||||
|
c = core.Commit('abcdef1234567890', 'a commit')
|
||||||
|
|
||||||
|
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||||
|
with mock.patch('bin.pick.core.is_commit_in_branch', self.return_true):
|
||||||
|
await core.resolve_nomination(c, '')
|
||||||
|
|
||||||
|
assert c.nominated
|
||||||
|
assert c.nomination_type is core.NominationType.FIXES
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_fix_is_not_nominated(self):
|
||||||
|
s = self.FakeSubprocess(b'Fixes: 3d09bb390a39 (etnaviv: GC7000: State changes for HALTI3..5)')
|
||||||
|
c = core.Commit('abcdef1234567890', 'a commit')
|
||||||
|
|
||||||
|
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||||
|
with mock.patch('bin.pick.core.is_commit_in_branch', self.return_false):
|
||||||
|
await core.resolve_nomination(c, '')
|
||||||
|
|
||||||
|
assert not c.nominated
|
||||||
|
assert c.nomination_type is core.NominationType.FIXES
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cc_is_nominated(self):
|
||||||
|
s = self.FakeSubprocess(b'Cc: 16.2 <[email protected]>')
|
||||||
|
c = core.Commit('abcdef1234567890', 'a commit')
|
||||||
|
|
||||||
|
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||||
|
await core.resolve_nomination(c, '16.2')
|
||||||
|
|
||||||
|
assert c.nominated
|
||||||
|
assert c.nomination_type is core.NominationType.CC
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cc_is_nominated2(self):
|
||||||
|
s = self.FakeSubprocess(b'Cc: [email protected]')
|
||||||
|
c = core.Commit('abcdef1234567890', 'a commit')
|
||||||
|
|
||||||
|
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||||
|
await core.resolve_nomination(c, '16.2')
|
||||||
|
|
||||||
|
assert c.nominated
|
||||||
|
assert c.nomination_type is core.NominationType.CC
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cc_is_not_nominated(self):
|
||||||
|
s = self.FakeSubprocess(b'Cc: 16.2 <[email protected]>')
|
||||||
|
c = core.Commit('abcdef1234567890', 'a commit')
|
||||||
|
|
||||||
|
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||||
|
await core.resolve_nomination(c, '16.1')
|
||||||
|
|
||||||
|
assert not c.nominated
|
||||||
|
assert c.nomination_type is core.NominationType.NONE
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_backport_is_nominated(self):
|
||||||
|
s = self.FakeSubprocess(b'Backport-to: 16.2')
|
||||||
|
c = core.Commit('abcdef1234567890', 'a commit')
|
||||||
|
|
||||||
|
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||||
|
await core.resolve_nomination(c, '16.2')
|
||||||
|
|
||||||
|
assert c.nominated
|
||||||
|
assert c.nomination_type is core.NominationType.BACKPORT
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_backport_all_is_nominated(self):
|
||||||
|
s = self.FakeSubprocess(b'Backport-to: *')
|
||||||
|
c = core.Commit('abcdef1234567890', 'a commit')
|
||||||
|
|
||||||
|
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||||
|
await core.resolve_nomination(c, '0.0')
|
||||||
|
|
||||||
|
assert c.nominated
|
||||||
|
assert c.nomination_type is core.NominationType.BACKPORT
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_backport_is_nominated_after(self):
|
||||||
|
s = self.FakeSubprocess(b'Backport-to: 16.2')
|
||||||
|
c = core.Commit('abcdef1234567890', 'a commit')
|
||||||
|
|
||||||
|
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||||
|
await core.resolve_nomination(c, '16.3')
|
||||||
|
|
||||||
|
assert c.nominated
|
||||||
|
assert c.nomination_type is core.NominationType.BACKPORT
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_backport_is_not_nominated(self):
|
||||||
|
s = self.FakeSubprocess(b'Backport-to: 16.2')
|
||||||
|
c = core.Commit('abcdef1234567890', 'a commit')
|
||||||
|
|
||||||
|
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||||
|
await core.resolve_nomination(c, '16.1')
|
||||||
|
|
||||||
|
assert not c.nominated
|
||||||
|
assert c.nomination_type is core.NominationType.NONE
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_revert_is_nominated(self):
|
||||||
|
s = self.FakeSubprocess(b'This reverts commit 1234567890123456789012345678901234567890.')
|
||||||
|
c = core.Commit('abcdef1234567890', 'a commit')
|
||||||
|
|
||||||
|
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||||
|
with mock.patch('bin.pick.core.is_commit_in_branch', self.return_true):
|
||||||
|
await core.resolve_nomination(c, '')
|
||||||
|
|
||||||
|
assert c.nominated
|
||||||
|
assert c.nomination_type is core.NominationType.REVERT
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_revert_is_not_nominated(self):
|
||||||
|
s = self.FakeSubprocess(b'This reverts commit 1234567890123456789012345678901234567890.')
|
||||||
|
c = core.Commit('abcdef1234567890', 'a commit')
|
||||||
|
|
||||||
|
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||||
|
with mock.patch('bin.pick.core.is_commit_in_branch', self.return_false):
|
||||||
|
await core.resolve_nomination(c, '')
|
||||||
|
|
||||||
|
assert not c.nominated
|
||||||
|
assert c.nomination_type is core.NominationType.REVERT
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_is_fix_and_backport(self):
|
||||||
|
s = self.FakeSubprocess(
|
||||||
|
b'Fixes: 3d09bb390a39 (etnaviv: GC7000: State changes for HALTI3..5)\n'
|
||||||
|
b'Backport-to: 16.1'
|
||||||
|
)
|
||||||
|
c = core.Commit('abcdef1234567890', 'a commit')
|
||||||
|
|
||||||
|
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||||
|
with mock.patch('bin.pick.core.is_commit_in_branch', self.return_true):
|
||||||
|
await core.resolve_nomination(c, '16.1')
|
||||||
|
|
||||||
|
assert c.nominated
|
||||||
|
assert c.nomination_type is core.NominationType.FIXES
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_is_fix_and_cc(self):
|
||||||
|
s = self.FakeSubprocess(
|
||||||
|
b'Fixes: 3d09bb390a39 (etnaviv: GC7000: State changes for HALTI3..5)\n'
|
||||||
|
b'Cc: 16.1 <[email protected]>'
|
||||||
|
)
|
||||||
|
c = core.Commit('abcdef1234567890', 'a commit')
|
||||||
|
|
||||||
|
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||||
|
with mock.patch('bin.pick.core.is_commit_in_branch', self.return_true):
|
||||||
|
await core.resolve_nomination(c, '16.1')
|
||||||
|
|
||||||
|
assert c.nominated
|
||||||
|
assert c.nomination_type is core.NominationType.FIXES
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_is_fix_and_revert(self):
|
||||||
|
s = self.FakeSubprocess(
|
||||||
|
b'Fixes: 3d09bb390a39 (etnaviv: GC7000: State changes for HALTI3..5)\n'
|
||||||
|
b'This reverts commit 1234567890123456789012345678901234567890.'
|
||||||
|
)
|
||||||
|
c = core.Commit('abcdef1234567890', 'a commit')
|
||||||
|
|
||||||
|
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||||
|
with mock.patch('bin.pick.core.is_commit_in_branch', self.return_true):
|
||||||
|
await core.resolve_nomination(c, '16.1')
|
||||||
|
|
||||||
|
assert c.nominated
|
||||||
|
assert c.nomination_type is core.NominationType.FIXES
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_is_cc_and_revert(self):
|
||||||
|
s = self.FakeSubprocess(
|
||||||
|
b'This reverts commit 1234567890123456789012345678901234567890.\n'
|
||||||
|
b'Cc: 16.1 <[email protected]>'
|
||||||
|
)
|
||||||
|
c = core.Commit('abcdef1234567890', 'a commit')
|
||||||
|
|
||||||
|
with mock.patch('bin.pick.core.asyncio.create_subprocess_exec', s.mock):
|
||||||
|
with mock.patch('bin.pick.core.is_commit_in_branch', self.return_true):
|
||||||
|
await core.resolve_nomination(c, '16.1')
|
||||||
|
|
||||||
|
assert c.nominated
|
||||||
|
assert c.nomination_type is core.NominationType.CC
|
||||||
|
|
||||||
|
|
||||||
|
class TestResolveFixes:
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_in_new(self):
|
||||||
|
"""Because commit abcd is nominated, so f123 should be as well."""
|
||||||
|
c = [
|
||||||
|
core.Commit('f123', 'desc', nomination_type=core.NominationType.FIXES, because_sha='abcd'),
|
||||||
|
core.Commit('abcd', 'desc', True),
|
||||||
|
]
|
||||||
|
await core.resolve_fixes(c, [])
|
||||||
|
assert c[1].nominated
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_not_in_new(self):
|
||||||
|
"""Because commit abcd is not nominated, commit f123 shouldn't be either."""
|
||||||
|
c = [
|
||||||
|
core.Commit('f123', 'desc', nomination_type=core.NominationType.FIXES, because_sha='abcd'),
|
||||||
|
core.Commit('abcd', 'desc'),
|
||||||
|
]
|
||||||
|
await core.resolve_fixes(c, [])
|
||||||
|
assert not c[0].nominated
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_in_previous(self):
|
||||||
|
"""Because commit abcd is nominated, so f123 should be as well."""
|
||||||
|
p = [
|
||||||
|
core.Commit('abcd', 'desc', True),
|
||||||
|
]
|
||||||
|
c = [
|
||||||
|
core.Commit('f123', 'desc', nomination_type=core.NominationType.FIXES, because_sha='abcd'),
|
||||||
|
]
|
||||||
|
await core.resolve_fixes(c, p)
|
||||||
|
assert c[0].nominated
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_not_in_previous(self):
|
||||||
|
"""Because commit abcd is not nominated, commit f123 shouldn't be either."""
|
||||||
|
p = [
|
||||||
|
core.Commit('abcd', 'desc'),
|
||||||
|
]
|
||||||
|
c = [
|
||||||
|
core.Commit('f123', 'desc', nomination_type=core.NominationType.FIXES, because_sha='abcd'),
|
||||||
|
]
|
||||||
|
await core.resolve_fixes(c, p)
|
||||||
|
assert not c[0].nominated
|
||||||
|
|
||||||
|
|
||||||
|
class TestIsCommitInBranch:
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_no(self):
|
||||||
|
# Hopefully this is never true?
|
||||||
|
value = await core.is_commit_in_branch('ffffffffffffffffffffffffffffff')
|
||||||
|
assert not value
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_yes(self):
|
||||||
|
# This commit is from 2000, it better always be in the branch
|
||||||
|
value = await core.is_commit_in_branch('88f3b89a2cb77766d2009b9868c44e03abe2dbb2')
|
||||||
|
assert value
|
||||||
|
|
||||||
|
|
||||||
|
class TestFullSha:
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_basic(self):
|
||||||
|
# This commit is from 2000, it better always be in the branch
|
||||||
|
value = await core.full_sha('88f3b89a2cb777')
|
||||||
|
assert value
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_invalid(self):
|
||||||
|
# This commit is from 2000, it better always be in the branch
|
||||||
|
with pytest.raises(core.PickUIException):
|
||||||
|
await core.full_sha('fffffffffffffffffffffffffffffffffff')
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
attrs==25.4.0
|
||||||
|
packaging==26.0
|
||||||
|
urwid==3.0.3
|
||||||
+322
@@ -0,0 +1,322 @@
|
|||||||
|
# Copyright © 2019-2020 Intel Corporation
|
||||||
|
|
||||||
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
# of this software and associated documentation files (the "Software"), to deal
|
||||||
|
# in the Software without restriction, including without limitation the rights
|
||||||
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
# copies of the Software, and to permit persons to whom the Software is
|
||||||
|
# furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
# The above copyright notice and this permission notice shall be included in
|
||||||
|
# all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
# SOFTWARE.
|
||||||
|
|
||||||
|
"""Urwid UI for pick script."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import itertools
|
||||||
|
import textwrap
|
||||||
|
import typing
|
||||||
|
|
||||||
|
import attr
|
||||||
|
import urwid
|
||||||
|
|
||||||
|
from . import core
|
||||||
|
|
||||||
|
if typing.TYPE_CHECKING:
|
||||||
|
WidgetType = typing.TypeVar('WidgetType', bound=urwid.Widget)
|
||||||
|
|
||||||
|
PALETTE = [
|
||||||
|
('a', 'black', 'light gray'),
|
||||||
|
('b', 'black', 'dark red'),
|
||||||
|
('bg', 'black', 'dark blue'),
|
||||||
|
('reversed', 'standout', ''),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class RootWidget(urwid.Frame):
|
||||||
|
|
||||||
|
def __init__(self, *args, ui: 'UI', **kwargs):
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
self.ui = ui
|
||||||
|
|
||||||
|
|
||||||
|
class CommitList(urwid.ListBox):
|
||||||
|
|
||||||
|
def __init__(self, *args, ui: 'UI', **kwargs):
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
self.ui = ui
|
||||||
|
|
||||||
|
def keypress(self, size: int, key: str) -> typing.Optional[str]:
|
||||||
|
if key == 'q':
|
||||||
|
raise urwid.ExitMainLoop()
|
||||||
|
elif key == 'u':
|
||||||
|
asyncio.ensure_future(self.ui.update())
|
||||||
|
elif key == 'a':
|
||||||
|
self.ui.add()
|
||||||
|
else:
|
||||||
|
return super().keypress(size, key)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class CommitWidget(urwid.Text):
|
||||||
|
|
||||||
|
# urwid.Text is normally not interactable, this is required to tell urwid
|
||||||
|
# to use our keypress method
|
||||||
|
_selectable = True
|
||||||
|
|
||||||
|
def __init__(self, ui: 'UI', commit: 'core.Commit'):
|
||||||
|
reason = commit.nomination_type.name.ljust(6)
|
||||||
|
super().__init__(f'{commit.date()} {reason} {commit.sha[:10]} {commit.description}')
|
||||||
|
self.ui = ui
|
||||||
|
self.commit = commit
|
||||||
|
|
||||||
|
async def apply(self) -> None:
|
||||||
|
async with self.ui.git_lock:
|
||||||
|
result, err = await self.commit.apply(self.ui)
|
||||||
|
if not result:
|
||||||
|
self.ui.chp_failed(self, err)
|
||||||
|
else:
|
||||||
|
self.ui.remove_commit(self)
|
||||||
|
|
||||||
|
async def denominate(self) -> None:
|
||||||
|
async with self.ui.git_lock:
|
||||||
|
await self.commit.denominate(self.ui)
|
||||||
|
self.ui.remove_commit(self)
|
||||||
|
|
||||||
|
async def backport(self) -> None:
|
||||||
|
async with self.ui.git_lock:
|
||||||
|
await self.commit.backport(self.ui)
|
||||||
|
self.ui.remove_commit(self)
|
||||||
|
|
||||||
|
def keypress(self, size: int, key: str) -> typing.Optional[str]:
|
||||||
|
if key == 'c':
|
||||||
|
asyncio.ensure_future(self.apply())
|
||||||
|
elif key == 'd':
|
||||||
|
asyncio.ensure_future(self.denominate())
|
||||||
|
elif key == 'b':
|
||||||
|
asyncio.ensure_future(self.backport())
|
||||||
|
else:
|
||||||
|
return key
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class FocusAwareEdit(urwid.Edit):
|
||||||
|
|
||||||
|
"""An Edit type that signals when it comes into and leaves focus."""
|
||||||
|
|
||||||
|
signals = urwid.Edit.signals + ['focus_changed']
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
self.__is_focus = False
|
||||||
|
|
||||||
|
def render(self, size: typing.Tuple[int], focus: bool = False) -> urwid.Canvas:
|
||||||
|
if focus != self.__is_focus:
|
||||||
|
self._emit("focus_changed", focus)
|
||||||
|
self.__is_focus = focus
|
||||||
|
return super().render(size, focus)
|
||||||
|
|
||||||
|
|
||||||
|
@attr.s(slots=True)
|
||||||
|
class UI:
|
||||||
|
|
||||||
|
"""Main management object.
|
||||||
|
|
||||||
|
:previous_commits: A list of commits to main since this branch was created
|
||||||
|
:new_commits: Commits added to main since the last time this script was run
|
||||||
|
"""
|
||||||
|
|
||||||
|
commit_list: typing.List['urwid.Button'] = attr.ib(factory=lambda: urwid.SimpleFocusListWalker([]), init=False)
|
||||||
|
feedback_box: typing.List['urwid.Text'] = attr.ib(factory=lambda: urwid.SimpleFocusListWalker([]), init=False)
|
||||||
|
notes: 'FocusAwareEdit' = attr.ib(factory=lambda: FocusAwareEdit('', multiline=True), init=False)
|
||||||
|
header: 'urwid.Text' = attr.ib(factory=lambda: urwid.Text('Mesa Stable Picker', align='center'), init=False)
|
||||||
|
body: 'urwid.Columns' = attr.ib(attr.Factory(lambda s: s._make_body(), True), init=False)
|
||||||
|
footer: 'urwid.Columns' = attr.ib(attr.Factory(lambda s: s._make_footer(), True), init=False)
|
||||||
|
root: RootWidget = attr.ib(attr.Factory(lambda s: s._make_root(), True), init=False)
|
||||||
|
mainloop: urwid.MainLoop = attr.ib(None, init=False)
|
||||||
|
|
||||||
|
previous_commits: typing.List['core.Commit'] = attr.ib(factory=list, init=False)
|
||||||
|
new_commits: typing.List['core.Commit'] = attr.ib(factory=list, init=False)
|
||||||
|
git_lock: asyncio.Lock = attr.ib(factory=asyncio.Lock, init=False)
|
||||||
|
|
||||||
|
def _get_current_commit(self) -> typing.Optional['core.Commit']:
|
||||||
|
entry = self.commit_list.get_focus()[0]
|
||||||
|
return entry.original_widget.commit if entry is not None else None
|
||||||
|
|
||||||
|
def _change_notes_cb(self) -> None:
|
||||||
|
commit = self._get_current_commit()
|
||||||
|
if commit and commit.notes:
|
||||||
|
self.notes.set_edit_text(commit.notes)
|
||||||
|
else:
|
||||||
|
self.notes.set_edit_text('')
|
||||||
|
|
||||||
|
def _change_notes_focus_cb(self, notes: 'FocusAwareEdit', focus: 'bool') -> 'None':
|
||||||
|
# in the case of coming into focus we don't want to do anything
|
||||||
|
if focus:
|
||||||
|
return
|
||||||
|
commit = self._get_current_commit()
|
||||||
|
if commit is None:
|
||||||
|
return
|
||||||
|
text: str = notes.get_edit_text()
|
||||||
|
if text != commit.notes:
|
||||||
|
asyncio.ensure_future(commit.update_notes(self, text))
|
||||||
|
|
||||||
|
def _make_body(self) -> 'urwid.Columns':
|
||||||
|
commits = CommitList(self.commit_list, ui=self)
|
||||||
|
feedback = urwid.ListBox(self.feedback_box)
|
||||||
|
urwid.connect_signal(self.commit_list, 'modified', self._change_notes_cb)
|
||||||
|
notes = urwid.Filler(self.notes)
|
||||||
|
urwid.connect_signal(self.notes, 'focus_changed', self._change_notes_focus_cb)
|
||||||
|
|
||||||
|
return urwid.Columns([urwid.LineBox(commits), urwid.Pile([urwid.LineBox(notes), urwid.LineBox(feedback)])])
|
||||||
|
|
||||||
|
def _make_footer(self) -> 'urwid.Columns':
|
||||||
|
body = [
|
||||||
|
urwid.Text('[U]pdate'),
|
||||||
|
urwid.Text('[Q]uit'),
|
||||||
|
urwid.Text('[C]herry Pick'),
|
||||||
|
urwid.Text('[D]enominate'),
|
||||||
|
urwid.Text('[B]ackport'),
|
||||||
|
urwid.Text('[A]pply additional patch'),
|
||||||
|
]
|
||||||
|
return urwid.Columns(body)
|
||||||
|
|
||||||
|
def _make_root(self) -> 'RootWidget':
|
||||||
|
return RootWidget(self.body, urwid.LineBox(self.header), urwid.LineBox(self.footer), 'body', ui=self)
|
||||||
|
|
||||||
|
def render(self) -> 'WidgetType':
|
||||||
|
asyncio.ensure_future(self.update())
|
||||||
|
return self.root
|
||||||
|
|
||||||
|
def load(self) -> None:
|
||||||
|
self.previous_commits = core.load()
|
||||||
|
|
||||||
|
async def update(self) -> None:
|
||||||
|
self.load()
|
||||||
|
with open('VERSION', 'r') as f:
|
||||||
|
version = '.'.join(f.read().split('.')[:2])
|
||||||
|
if self.previous_commits:
|
||||||
|
sha = self.previous_commits[0].sha
|
||||||
|
else:
|
||||||
|
sha = f'{version}-branchpoint'
|
||||||
|
|
||||||
|
new_commits = await core.get_new_commits(sha)
|
||||||
|
|
||||||
|
if new_commits:
|
||||||
|
pb = urwid.ProgressBar('a', 'b', done=len(new_commits))
|
||||||
|
o = self.mainloop.widget
|
||||||
|
self.mainloop.widget = urwid.Overlay(
|
||||||
|
urwid.Filler(urwid.LineBox(pb)), o, 'center', ('relative', 50), 'middle', ('relative', 50))
|
||||||
|
self.new_commits = await core.gather_commits(
|
||||||
|
version, self.previous_commits, new_commits,
|
||||||
|
lambda: pb.set_completion(pb.current + 1))
|
||||||
|
self.mainloop.widget = o
|
||||||
|
|
||||||
|
for commit in reversed(list(itertools.chain(self.new_commits, self.previous_commits))):
|
||||||
|
if commit.nominated and commit.resolution is core.Resolution.UNRESOLVED:
|
||||||
|
b = urwid.AttrMap(CommitWidget(self, commit), None, focus_map='reversed')
|
||||||
|
self.commit_list.append(b)
|
||||||
|
self.mainloop.draw_screen()
|
||||||
|
self.save()
|
||||||
|
|
||||||
|
async def feedback(self, text: str) -> None:
|
||||||
|
self.feedback_box.append(urwid.AttrMap(urwid.Text(text), None))
|
||||||
|
latest_item_index = len(self.feedback_box) - 1
|
||||||
|
self.feedback_box.set_focus(latest_item_index)
|
||||||
|
|
||||||
|
def remove_commit(self, commit: CommitWidget) -> None:
|
||||||
|
for i, c in enumerate(self.commit_list):
|
||||||
|
if c.base_widget is commit:
|
||||||
|
del self.commit_list[i]
|
||||||
|
break
|
||||||
|
self.mainloop.draw_screen()
|
||||||
|
|
||||||
|
def save(self):
|
||||||
|
core.save(itertools.chain(self.new_commits, self.previous_commits))
|
||||||
|
|
||||||
|
def add(self) -> None:
|
||||||
|
"""Add an additional commit which isn't nominated."""
|
||||||
|
o = self.mainloop.widget
|
||||||
|
|
||||||
|
def reset_cb(_) -> None:
|
||||||
|
self.mainloop.widget = o
|
||||||
|
self.mainloop.draw_screen()
|
||||||
|
|
||||||
|
async def apply_cb(edit: urwid.Edit) -> None:
|
||||||
|
text: str = edit.get_edit_text()
|
||||||
|
|
||||||
|
# In case the text is empty
|
||||||
|
if not text:
|
||||||
|
return
|
||||||
|
|
||||||
|
sha = await core.full_sha(text)
|
||||||
|
for c in reversed(list(itertools.chain(self.new_commits, self.previous_commits))):
|
||||||
|
if c.sha == sha:
|
||||||
|
commit = c
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
raise RuntimeError(f"Couldn't find {sha}")
|
||||||
|
|
||||||
|
await commit.apply(self)
|
||||||
|
self.mainloop.draw_screen()
|
||||||
|
|
||||||
|
q = urwid.Edit("Commit sha\n")
|
||||||
|
ok_btn = urwid.Button('Ok')
|
||||||
|
urwid.connect_signal(ok_btn, 'click', lambda _: asyncio.ensure_future(apply_cb(q)))
|
||||||
|
urwid.connect_signal(ok_btn, 'click', reset_cb)
|
||||||
|
|
||||||
|
can_btn = urwid.Button('Cancel')
|
||||||
|
urwid.connect_signal(can_btn, 'click', reset_cb)
|
||||||
|
|
||||||
|
cols = urwid.Columns([ok_btn, can_btn])
|
||||||
|
pile = urwid.Pile([q, cols])
|
||||||
|
box = urwid.LineBox(pile)
|
||||||
|
|
||||||
|
self.mainloop.widget = urwid.Overlay(
|
||||||
|
urwid.Filler(box), o, 'center', ('relative', 50), 'middle', ('relative', 50)
|
||||||
|
)
|
||||||
|
self.mainloop.draw_screen()
|
||||||
|
|
||||||
|
def chp_failed(self, commit: 'CommitWidget', err: str) -> None:
|
||||||
|
o = self.mainloop.widget
|
||||||
|
|
||||||
|
def reset_cb(_) -> None:
|
||||||
|
self.mainloop.widget = o
|
||||||
|
self.mainloop.draw_screen()
|
||||||
|
|
||||||
|
t = urwid.Text(textwrap.dedent(f"""
|
||||||
|
Failed to apply {commit.commit.sha} {commit.commit.description} with the following error:
|
||||||
|
|
||||||
|
{err}
|
||||||
|
|
||||||
|
You can either cancel, or resolve the conflicts (`git mergetool`), finish the
|
||||||
|
cherry-pick (`git cherry-pick --continue`) and select ok."""))
|
||||||
|
|
||||||
|
can_btn = urwid.Button('Cancel')
|
||||||
|
urwid.connect_signal(can_btn, 'click', reset_cb)
|
||||||
|
urwid.connect_signal(
|
||||||
|
can_btn, 'click', lambda _: asyncio.ensure_future(commit.commit.abort_cherry(self, err)))
|
||||||
|
|
||||||
|
ok_btn = urwid.Button('Ok')
|
||||||
|
urwid.connect_signal(ok_btn, 'click', reset_cb)
|
||||||
|
urwid.connect_signal(
|
||||||
|
ok_btn, 'click', lambda _: asyncio.ensure_future(commit.commit.resolve(self)))
|
||||||
|
urwid.connect_signal(
|
||||||
|
ok_btn, 'click', lambda _: self.remove_commit(commit))
|
||||||
|
|
||||||
|
cols = urwid.Columns([ok_btn, can_btn])
|
||||||
|
pile = urwid.Pile([t, cols])
|
||||||
|
box = urwid.LineBox(pile)
|
||||||
|
|
||||||
|
self.mainloop.widget = urwid.Overlay(
|
||||||
|
urwid.Filler(box), o, 'center', ('relative', 50), 'middle', ('relative', 50)
|
||||||
|
)
|
||||||
|
self.mainloop.draw_screen()
|
||||||
Executable
+65
@@ -0,0 +1,65 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# Copyright © 2019-2020 Intel Corporation
|
||||||
|
|
||||||
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
# of this software and associated documentation files (the "Software"), to deal
|
||||||
|
# in the Software without restriction, including without limitation the rights
|
||||||
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
# copies of the Software, and to permit persons to whom the Software is
|
||||||
|
# furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
# The above copyright notice and this permission notice shall be included in
|
||||||
|
# all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
# SOFTWARE.
|
||||||
|
|
||||||
|
"""Update the main page, release notes, and calendar."""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import csv
|
||||||
|
import pathlib
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
|
||||||
|
def update_calendar(version: str) -> None:
|
||||||
|
p = pathlib.Path('docs') / 'release-calendar.csv'
|
||||||
|
|
||||||
|
with p.open('r') as f:
|
||||||
|
calendar = list(csv.reader(f))
|
||||||
|
|
||||||
|
branch = None
|
||||||
|
for i, line in enumerate(calendar):
|
||||||
|
if line[2] == version:
|
||||||
|
if line[0]:
|
||||||
|
branch = line[0]
|
||||||
|
break
|
||||||
|
if branch is not None:
|
||||||
|
calendar[i + 1][0] = branch
|
||||||
|
del calendar[i]
|
||||||
|
|
||||||
|
with p.open('w') as f:
|
||||||
|
writer = csv.writer(f)
|
||||||
|
writer.writerows(calendar)
|
||||||
|
|
||||||
|
subprocess.run(['git', 'add', p])
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument('version', help="The released version.")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
update_calendar(args.version)
|
||||||
|
|
||||||
|
subprocess.run(['git', 'commit', '-m',
|
||||||
|
f'docs: update calendar for {args.version}'])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
# Copyright © 2019 Intel Corporation
|
||||||
|
|
||||||
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
# of this software and associated documentation files (the "Software"), to deal
|
||||||
|
# in the Software without restriction, including without limitation the rights
|
||||||
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
# copies of the Software, and to permit persons to whom the Software is
|
||||||
|
# furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
# The above copyright notice and this permission notice shall be included in
|
||||||
|
# all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
# SOFTWARE.
|
||||||
|
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from . import post_version
|
||||||
|
|
||||||
|
|
||||||
|
@mock.patch('bin.post_version.subprocess.run', mock.Mock())
|
||||||
|
class TestUpdateCalendar:
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def mock_sideffects(self) -> None:
|
||||||
|
"""Mock out side effects."""
|
||||||
|
with mock.patch('bin.post_version.subprocess.run', mock.Mock()), \
|
||||||
|
mock.patch('bin.post_version.pathlib', mock.MagicMock()):
|
||||||
|
yield
|
||||||
|
|
||||||
|
def test_basic(self):
|
||||||
|
data = [
|
||||||
|
['20.3', '2021-01-13', '20.3.3', 'Dylan Baker', None],
|
||||||
|
[None, '2021-01-27', '20.3.4', 'Dylan Baker', None],
|
||||||
|
]
|
||||||
|
|
||||||
|
m = mock.Mock()
|
||||||
|
with mock.patch('bin.post_version.csv.reader', mock.Mock(return_value=data.copy())), \
|
||||||
|
mock.patch('bin.post_version.csv.writer', mock.Mock(return_value=m)):
|
||||||
|
post_version.update_calendar('20.3.3')
|
||||||
|
|
||||||
|
m.writerows.assert_called_with([data[1]])
|
||||||
|
|
||||||
|
def test_two_releases(self):
|
||||||
|
data = [
|
||||||
|
['20.3', '2021-01-13', '20.3.3', 'Dylan Baker', None],
|
||||||
|
[None, '2021-01-27', '20.3.4', 'Dylan Baker', None],
|
||||||
|
['21.0', '2021-01-13', '21.0.0', 'Dylan Baker', None],
|
||||||
|
[None, '2021-01-13', '21.0.1', 'Dylan Baker', None],
|
||||||
|
]
|
||||||
|
|
||||||
|
m = mock.Mock()
|
||||||
|
with mock.patch('bin.post_version.csv.reader', mock.Mock(return_value=data.copy())), \
|
||||||
|
mock.patch('bin.post_version.csv.writer', mock.Mock(return_value=m)):
|
||||||
|
post_version.update_calendar('20.3.3')
|
||||||
|
|
||||||
|
d = data.copy()
|
||||||
|
del d[0]
|
||||||
|
d[0][0] = '20.3'
|
||||||
|
m.writerows.assert_called_with(d)
|
||||||
Executable
+47
@@ -0,0 +1,47 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
readonly requirements_file=$1
|
||||||
|
shift
|
||||||
|
|
||||||
|
venv_dir="$(dirname "$requirements_file")"/.venv
|
||||||
|
readonly venv_dir
|
||||||
|
readonly venv_req=$venv_dir/requirements.txt
|
||||||
|
readonly venv_python_version=$venv_dir/python-version.txt
|
||||||
|
|
||||||
|
if [ -d "$venv_dir" ]
|
||||||
|
then
|
||||||
|
if [ ! -r "$venv_python_version" ]
|
||||||
|
then
|
||||||
|
echo "Python environment predates Python version checks."
|
||||||
|
echo "It might be invalid and needs to be regenerated."
|
||||||
|
rm -rf "$venv_dir"
|
||||||
|
elif ! cmp --quiet <(python3 --version) "$venv_python_version"
|
||||||
|
then
|
||||||
|
old=$(cat "$venv_python_version")
|
||||||
|
new=$(python3 --version)
|
||||||
|
echo "Python version has changed ($old -> $new)."
|
||||||
|
echo "Python environment needs to be regenerated."
|
||||||
|
unset old new
|
||||||
|
rm -rf "$venv_dir"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! [ -r "$venv_dir/bin/activate" ]
|
||||||
|
then
|
||||||
|
echo "Creating Python environment..."
|
||||||
|
python3 -m venv "$venv_dir"
|
||||||
|
python3 --version > "$venv_python_version"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# shellcheck disable=1091
|
||||||
|
source "$venv_dir/bin/activate"
|
||||||
|
|
||||||
|
if ! cmp --quiet "$requirements_file" "$venv_req"
|
||||||
|
then
|
||||||
|
echo "$(realpath --relative-to="$PWD" "$requirements_file") has changed, re-installing..."
|
||||||
|
pip --disable-pip-version-check install --requirement "$requirements_file"
|
||||||
|
cp "$requirements_file" "$venv_req"
|
||||||
|
fi
|
||||||
|
|
||||||
|
python3 "$@"
|
||||||
@@ -0,0 +1,133 @@
|
|||||||
|
#!/usr/bin/python3
|
||||||
|
#
|
||||||
|
# Copyright 2025 Valve Corporation
|
||||||
|
# SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import csv
|
||||||
|
import unittest
|
||||||
|
import sys
|
||||||
|
import subprocess
|
||||||
|
import os
|
||||||
|
import shlex
|
||||||
|
from unidecode import unidecode
|
||||||
|
|
||||||
|
def normalize(x):
|
||||||
|
return unidecode(x.lower())
|
||||||
|
|
||||||
|
def name(row):
|
||||||
|
return normalize(row[0])
|
||||||
|
|
||||||
|
def username(row):
|
||||||
|
return normalize(row[2])
|
||||||
|
|
||||||
|
def find_person(x):
|
||||||
|
x = normalize(x)
|
||||||
|
|
||||||
|
filename = 'people.csv'
|
||||||
|
path = os.path.join(os.path.dirname(os.path.realpath(__file__)), filename)
|
||||||
|
with open(path, 'r') as f:
|
||||||
|
people = list(csv.reader(f, skipinitialspace=True))
|
||||||
|
|
||||||
|
# First, try to exactly match username
|
||||||
|
for row in people:
|
||||||
|
if username(row) == x:
|
||||||
|
return row
|
||||||
|
|
||||||
|
# Next, try to exactly match fullname
|
||||||
|
for row in people:
|
||||||
|
if name(row) == x:
|
||||||
|
return row
|
||||||
|
|
||||||
|
# Now we get fuzzy. Try to match a first name.
|
||||||
|
candidates = [r for r in people if name(r).split(' ')[0] == x]
|
||||||
|
if len(candidates) == 1:
|
||||||
|
return candidates[0]
|
||||||
|
|
||||||
|
# Or a last name?
|
||||||
|
candidates = [r for r in people if x in name(r).split(' ')]
|
||||||
|
if len(candidates) == 1:
|
||||||
|
return candidates[0]
|
||||||
|
|
||||||
|
# Well, frick.
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Self-test... is it even worth find a unit test framework for this?
|
||||||
|
TEST_CASES = {
|
||||||
|
'gfxstrand': '[email protected]',
|
||||||
|
'Faith': '[email protected]',
|
||||||
|
'faith': '[email protected]',
|
||||||
|
'alyssa': '[email protected]',
|
||||||
|
'briano': '[email protected]',
|
||||||
|
'schurmann': '[email protected]',
|
||||||
|
'Schürmann': '[email protected]',
|
||||||
|
}
|
||||||
|
|
||||||
|
for test in TEST_CASES:
|
||||||
|
a, b = find_person(test), TEST_CASES[test]
|
||||||
|
if a is None or a[1] != b:
|
||||||
|
print(test, a, b)
|
||||||
|
assert(a is not None and a[1] == b)
|
||||||
|
|
||||||
|
# Now the tool itself
|
||||||
|
if __name__ == "__main__":
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
prog='rb',
|
||||||
|
description='Add review trailers')
|
||||||
|
parser.add_argument('person', nargs='+', help="Reviewer's username, first name, or full name")
|
||||||
|
parser.add_argument('-a', '--ack', action='store_true', help="Apply an acked-by tag")
|
||||||
|
parser.add_argument('-t', '--test', action='store_true', help="Apply an tested-by tag")
|
||||||
|
parser.add_argument('-d', '--dry-run', action='store_true',
|
||||||
|
help="Print trailer without applying")
|
||||||
|
parser.add_argument('-r', '--rebase', nargs='?',
|
||||||
|
help="Rebase on the specified branch applying tags to all commits")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
# If we are rebasing, let git reinvoke this script with the rebase related
|
||||||
|
# arguments stripped.
|
||||||
|
if args.rebase is not None:
|
||||||
|
relevant_args = [sys.argv[0]]
|
||||||
|
if args.ack:
|
||||||
|
relevant_args.append("--ack")
|
||||||
|
elif args.test:
|
||||||
|
relevant_args.append("--test")
|
||||||
|
|
||||||
|
relevant_args += args.person
|
||||||
|
|
||||||
|
cmd = f"python3 {' '.join(relevant_args)}"
|
||||||
|
rebase = ['git', 'rebase', '--exec', cmd, args.rebase]
|
||||||
|
|
||||||
|
if args.dry_run:
|
||||||
|
print(' '.join([shlex.quote(s) for s in rebase]))
|
||||||
|
returncode = 0
|
||||||
|
else:
|
||||||
|
returncode = subprocess.run(rebase).returncode
|
||||||
|
|
||||||
|
sys.exit(returncode)
|
||||||
|
|
||||||
|
for p in args.person:
|
||||||
|
person = find_person(p)
|
||||||
|
if person is None:
|
||||||
|
print(f'Could not uniquely identify {p}, skipping')
|
||||||
|
|
||||||
|
trailer = 'Reviewed-by'
|
||||||
|
if args.ack:
|
||||||
|
trailer = 'Acked-by'
|
||||||
|
elif args.test:
|
||||||
|
trailer = 'Tested-by'
|
||||||
|
|
||||||
|
trailer = f'{trailer}: {person[0]} <{person[1]}>'
|
||||||
|
|
||||||
|
if args.dry_run:
|
||||||
|
print(trailer)
|
||||||
|
continue
|
||||||
|
|
||||||
|
diff_index = subprocess.run("git diff-index --quiet --cached HEAD --".split(" "))
|
||||||
|
if diff_index.returncode != 0:
|
||||||
|
print("You have staged changes.")
|
||||||
|
print("Please commit before applying review tags.")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
env = os.environ.copy()
|
||||||
|
env['GIT_EDITOR'] = f'git interpret-trailers --trailer "{trailer}" --in-place'
|
||||||
|
subprocess.run(["git", "commit", "--amend"], env=env)
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# Copyright © Microsoft Corporation
|
||||||
|
#
|
||||||
|
# Permission is hereby granted, free of charge, to any person obtaining a
|
||||||
|
# copy of this software and associated documentation files (the "Software"),
|
||||||
|
# to deal in the Software without restriction, including without limitation
|
||||||
|
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||||
|
# and/or sell copies of the Software, and to permit persons to whom the
|
||||||
|
# Software is furnished to do so, subject to the following conditions:
|
||||||
|
#
|
||||||
|
# The above copyright notice and this permission notice (including the next
|
||||||
|
# paragraph) shall be included in all copies or substantial portions of the
|
||||||
|
# Software.
|
||||||
|
#
|
||||||
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||||
|
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||||
|
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||||
|
# IN THE SOFTWARE.
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
|
||||||
|
# Take a log file produced by GALLIUM_REFCNT_LOG, filter it to the objects that
|
||||||
|
# weren't destroyed by the end of the log, and write the results out sorted.
|
||||||
|
# Strips stacks by default to prevent OOM. Could probably be rewritten to walk
|
||||||
|
# the file twice to preserve stacks without OOM, but this was the easy way.
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument('--input',
|
||||||
|
action='store',
|
||||||
|
required=True,
|
||||||
|
help='path to file containing refcount log')
|
||||||
|
parser.add_argument('--output',
|
||||||
|
action='store',
|
||||||
|
required=True,
|
||||||
|
help='path to trimmed log')
|
||||||
|
parser.add_argument('--filter',
|
||||||
|
help='object type filter')
|
||||||
|
parser.add_argument('--keep-stacks',
|
||||||
|
help='keep stacks, otherwise only headers')
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
objects = {}
|
||||||
|
|
||||||
|
with open(args.input) as in_file:
|
||||||
|
stack = []
|
||||||
|
cur_object = ''
|
||||||
|
|
||||||
|
for line in in_file:
|
||||||
|
if line[0] == '<':
|
||||||
|
|
||||||
|
parts = line.split(' ')
|
||||||
|
prev_object = cur_object
|
||||||
|
cur_object = parts[1]
|
||||||
|
if parts[3].strip() == 'Destroy':
|
||||||
|
if cur_object in objects:
|
||||||
|
del objects[cur_object]
|
||||||
|
else:
|
||||||
|
if parts[3].strip() == 'Create':
|
||||||
|
if (not args.filter) or (args.filter in parts[0]):
|
||||||
|
objects[cur_object] = []
|
||||||
|
if prev_object in objects:
|
||||||
|
objects[prev_object] += stack
|
||||||
|
|
||||||
|
stack = [line]
|
||||||
|
elif args.keep_stacks:
|
||||||
|
stack += line
|
||||||
|
|
||||||
|
with open(args.output, 'wt') as out_file:
|
||||||
|
for stack in objects.values():
|
||||||
|
for stack_line in stack:
|
||||||
|
out_file.write(stack_line)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
Executable
+31
@@ -0,0 +1,31 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import pathlib
|
||||||
|
import requests
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
def error(msg: str) -> None:
|
||||||
|
print('\033[31m' + msg + '\033[0m')
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
git_toplevel = subprocess.check_output(['git', 'rev-parse', '--show-toplevel'],
|
||||||
|
stderr=subprocess.DEVNULL).decode("ascii").strip()
|
||||||
|
if not pathlib.Path(git_toplevel).resolve() == pathlib.Path('.').resolve():
|
||||||
|
error('Please run this script from the root folder ({})'.format(git_toplevel))
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
file = 'include/renderdoc_app.h'
|
||||||
|
url = 'https://raw.githubusercontent.com/baldurk/renderdoc/refs/tags/v1.5/renderdoc/api/app/renderdoc_app.h'
|
||||||
|
|
||||||
|
print('Syncing {}...'.format(file), end=' ', flush=True)
|
||||||
|
req = requests.get(url)
|
||||||
|
|
||||||
|
if not req.ok:
|
||||||
|
error('Failed to retrieve file: {} {}'.format(req.status_code, req.reason))
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
with open(file, 'wb') as f:
|
||||||
|
f.write(req.content)
|
||||||
|
|
||||||
|
print('Done')
|
||||||
@@ -0,0 +1,256 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import platform
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
# This list contains symbols that _might_ be exported for some platforms
|
||||||
|
PLATFORM_SYMBOLS = [
|
||||||
|
'_GLOBAL_OFFSET_TABLE_',
|
||||||
|
'_ITM_deregisterTMCloneTable',
|
||||||
|
'_ITM_registerTMCloneTable',
|
||||||
|
'__bss_end__',
|
||||||
|
'__bss_start',
|
||||||
|
'__bss_start__',
|
||||||
|
'__cxa_allocate_dependent_exception',
|
||||||
|
'__cxa_allocate_exception',
|
||||||
|
'__cxa_begin_catch',
|
||||||
|
'__cxa_call_unexpected',
|
||||||
|
'__cxa_current_exception_type',
|
||||||
|
'__cxa_current_primary_exception',
|
||||||
|
'__cxa_decrement_exception_refcount',
|
||||||
|
'__cxa_deleted_virtual',
|
||||||
|
'__cxa_demangle',
|
||||||
|
'__cxa_end_catch',
|
||||||
|
'__cxa_finalize',
|
||||||
|
'__cxa_free_dependent_exception',
|
||||||
|
'__cxa_free_exception',
|
||||||
|
'__cxa_get_exception_ptr',
|
||||||
|
'__cxa_get_globals',
|
||||||
|
'__cxa_get_globals_fast',
|
||||||
|
'__cxa_guard_abort',
|
||||||
|
'__cxa_guard_acquire',
|
||||||
|
'__cxa_guard_release',
|
||||||
|
'__cxa_increment_exception_refcount',
|
||||||
|
'__cxa_new_handler',
|
||||||
|
'__cxa_pure_virtual',
|
||||||
|
'__cxa_rethrow',
|
||||||
|
'__cxa_rethrow_primary_exception',
|
||||||
|
'__cxa_terminate_handler',
|
||||||
|
'__cxa_throw',
|
||||||
|
'__cxa_uncaught_exception',
|
||||||
|
'__cxa_uncaught_exceptions',
|
||||||
|
'__cxa_unexpected_handler',
|
||||||
|
'__cxa_thread_atexit_impl',
|
||||||
|
'__deregister_frame_info',
|
||||||
|
'__deregister_frame_info_bases',
|
||||||
|
'__dynamic_cast',
|
||||||
|
'__emutls_get_address',
|
||||||
|
'__end__',
|
||||||
|
'__gmon_start__',
|
||||||
|
'__gxx_personality_v0',
|
||||||
|
'__odr_asan._glapi_Context',
|
||||||
|
'__odr_asan._glapi_Dispatch',
|
||||||
|
'__register_frame_info',
|
||||||
|
'__register_frame_info_bases',
|
||||||
|
'_bss_end__',
|
||||||
|
'_edata',
|
||||||
|
'_end',
|
||||||
|
'_fbss',
|
||||||
|
'_fdata',
|
||||||
|
'_fini',
|
||||||
|
'_ftext',
|
||||||
|
'_init',
|
||||||
|
'copy_file_range',
|
||||||
|
'gettid',
|
||||||
|
'pidfd_getpid',
|
||||||
|
'pidfd_spawnp',
|
||||||
|
'posix_spawn_file_actions_addchdir',
|
||||||
|
'posix_spawn_file_actions_addchdir_np',
|
||||||
|
'pthread_mutexattr_destroy',
|
||||||
|
'pthread_mutexattr_init',
|
||||||
|
'pthread_mutexattr_settype',
|
||||||
|
'statx',
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def get_symbols_nm(nm, lib):
|
||||||
|
'''
|
||||||
|
List all the (non platform-specific) symbols exported by the library
|
||||||
|
using `nm`
|
||||||
|
'''
|
||||||
|
symbols = []
|
||||||
|
platform_name = platform.system()
|
||||||
|
output = subprocess.check_output([nm, '-gP', lib],
|
||||||
|
stderr=open(os.devnull, 'w')).decode("ascii")
|
||||||
|
for line in output.splitlines():
|
||||||
|
if line.startswith(' '):
|
||||||
|
continue
|
||||||
|
fields = line.split()
|
||||||
|
if len(fields) >= 2 and fields[1] == 'U':
|
||||||
|
continue
|
||||||
|
symbol_name = fields[0]
|
||||||
|
if platform_name == 'Linux' or platform_name == 'GNU' or platform_name.startswith('GNU/'):
|
||||||
|
if symbol_name.split('@')[0] in PLATFORM_SYMBOLS:
|
||||||
|
continue
|
||||||
|
elif platform_name == 'Darwin':
|
||||||
|
assert symbol_name[0] == '_'
|
||||||
|
symbol_name = symbol_name[1:]
|
||||||
|
symbols.append(symbol_name)
|
||||||
|
return symbols
|
||||||
|
|
||||||
|
|
||||||
|
def get_symbols_dumpbin(dumpbin, lib):
|
||||||
|
'''
|
||||||
|
List all the (non platform-specific) symbols exported by the library
|
||||||
|
using `dumpbin`
|
||||||
|
'''
|
||||||
|
symbols = []
|
||||||
|
output = subprocess.check_output([dumpbin, '/exports', lib],
|
||||||
|
stderr=open(os.devnull, 'w')).decode("ascii")
|
||||||
|
for line in output.splitlines():
|
||||||
|
fields = line.split()
|
||||||
|
# The lines with the symbols are made of at least 4 columns; see details below
|
||||||
|
if len(fields) < 4:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
# Making sure the first 3 columns are a dec counter, a hex counter
|
||||||
|
# and a hex address
|
||||||
|
_ = int(fields[0], 10)
|
||||||
|
_ = int(fields[1], 16)
|
||||||
|
_ = int(fields[2], 16)
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
symbol_name = fields[3]
|
||||||
|
# De-mangle symbols
|
||||||
|
if symbol_name[0] == '_' and '@' in symbol_name:
|
||||||
|
symbol_name = symbol_name[1:].split('@')[0]
|
||||||
|
symbols.append(symbol_name)
|
||||||
|
return symbols
|
||||||
|
|
||||||
|
def get_symbols_gendef(gendef, lib):
|
||||||
|
'''
|
||||||
|
List all the (non platform-specific) symbols exported by the library
|
||||||
|
using `gendef -`
|
||||||
|
'''
|
||||||
|
symbols = []
|
||||||
|
output = subprocess.check_output([gendef, '-', lib],
|
||||||
|
stderr=open(os.devnull, 'w')).decode("ascii")
|
||||||
|
ordinal_table_found = False
|
||||||
|
for line in output.splitlines():
|
||||||
|
if not ordinal_table_found:
|
||||||
|
if line.strip() == 'EXPORTS':
|
||||||
|
ordinal_table_found = True
|
||||||
|
continue
|
||||||
|
|
||||||
|
symbols.append(line.strip())
|
||||||
|
return symbols
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument('--symbols-file',
|
||||||
|
action='store',
|
||||||
|
required=True,
|
||||||
|
help='path to file containing symbols')
|
||||||
|
parser.add_argument('--lib',
|
||||||
|
action='store',
|
||||||
|
required=True,
|
||||||
|
help='path to library')
|
||||||
|
parser.add_argument('--nm',
|
||||||
|
action='store',
|
||||||
|
help='path to binary (or name in $PATH)')
|
||||||
|
parser.add_argument('--dumpbin',
|
||||||
|
action='store',
|
||||||
|
help='path to binary (or name in $PATH)')
|
||||||
|
parser.add_argument('--gendef',
|
||||||
|
action='store',
|
||||||
|
help='path to binary (or name in $PATH)')
|
||||||
|
parser.add_argument('--ignore-symbol',
|
||||||
|
action='append',
|
||||||
|
help='do not process this symbol')
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if platform.system() == 'Windows':
|
||||||
|
if args.dumpbin:
|
||||||
|
lib_symbols = get_symbols_dumpbin(args.dumpbin, args.lib)
|
||||||
|
elif args.gendef:
|
||||||
|
lib_symbols = get_symbols_gendef(args.gendef, args.lib)
|
||||||
|
else:
|
||||||
|
parser.error('--dumpbin is mandatory for msvc, --gendef is mandatory for mingw')
|
||||||
|
else:
|
||||||
|
if not args.nm:
|
||||||
|
parser.error('--nm is mandatory')
|
||||||
|
lib_symbols = get_symbols_nm(args.nm, args.lib)
|
||||||
|
|
||||||
|
mandatory_symbols = []
|
||||||
|
optional_symbols = []
|
||||||
|
with open(args.symbols_file) as symbols_file:
|
||||||
|
qualifier_optional = '(optional)'
|
||||||
|
for line in symbols_file.readlines():
|
||||||
|
|
||||||
|
# Strip comments
|
||||||
|
line = line.split('#')[0]
|
||||||
|
line = line.strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Line format:
|
||||||
|
# [qualifier] symbol
|
||||||
|
qualifier = None
|
||||||
|
symbol = None
|
||||||
|
|
||||||
|
fields = line.split()
|
||||||
|
if len(fields) == 1:
|
||||||
|
symbol = fields[0]
|
||||||
|
elif len(fields) == 2:
|
||||||
|
qualifier = fields[0]
|
||||||
|
symbol = fields[1]
|
||||||
|
else:
|
||||||
|
print(args.symbols_file + ': invalid format: ' + line)
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
# The only supported qualifier is 'optional', which means the
|
||||||
|
# symbol doesn't have to be exported by the library
|
||||||
|
if qualifier and not qualifier == qualifier_optional:
|
||||||
|
print(args.symbols_file + ': invalid qualifier: ' + qualifier)
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
if qualifier == qualifier_optional:
|
||||||
|
optional_symbols.append(symbol)
|
||||||
|
else:
|
||||||
|
mandatory_symbols.append(symbol)
|
||||||
|
|
||||||
|
unknown_symbols = []
|
||||||
|
for symbol in lib_symbols:
|
||||||
|
if symbol in mandatory_symbols:
|
||||||
|
continue
|
||||||
|
if symbol in optional_symbols:
|
||||||
|
continue
|
||||||
|
if args.ignore_symbol and symbol in args.ignore_symbol:
|
||||||
|
continue
|
||||||
|
if symbol[:2] == '_Z':
|
||||||
|
# As ajax found out, the compiler intentionally exports symbols
|
||||||
|
# that we explicitly asked it not to export, and we can't do
|
||||||
|
# anything about it:
|
||||||
|
# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=36022#c4
|
||||||
|
continue
|
||||||
|
unknown_symbols.append(symbol)
|
||||||
|
|
||||||
|
missing_symbols = [
|
||||||
|
sym for sym in mandatory_symbols if sym not in lib_symbols
|
||||||
|
]
|
||||||
|
|
||||||
|
for symbol in unknown_symbols:
|
||||||
|
print(args.lib + ': unknown symbol exported: ' + symbol)
|
||||||
|
|
||||||
|
for symbol in missing_symbols:
|
||||||
|
print(args.lib + ': missing symbol: ' + symbol)
|
||||||
|
|
||||||
|
if unknown_symbols or missing_symbols:
|
||||||
|
exit(1)
|
||||||
|
exit(0)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
Executable
+58
@@ -0,0 +1,58 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import pathlib
|
||||||
|
import re
|
||||||
|
|
||||||
|
|
||||||
|
def detect_misleading_indentation(
|
||||||
|
toml_path: str,
|
||||||
|
toml_lines: list[str],
|
||||||
|
) -> bool:
|
||||||
|
issue_detected = False
|
||||||
|
previous_indentation = 0
|
||||||
|
for line_number, line in enumerate(toml_lines, start=1):
|
||||||
|
if match := re.match(r'^(\s*)\S', line):
|
||||||
|
line_indentation = len(match.group(1))
|
||||||
|
if line_indentation < previous_indentation:
|
||||||
|
# Allow de-indenting when starting a new section (`[`) or
|
||||||
|
# terminating a multi-line list (`]`)
|
||||||
|
if not re.match(r'^\s*(\[|\])', line):
|
||||||
|
print(f'{toml_path}:{line_number}: '
|
||||||
|
f'Misleading indentation found')
|
||||||
|
issue_detected = True
|
||||||
|
else:
|
||||||
|
line_indentation = 0
|
||||||
|
previous_indentation = line_indentation
|
||||||
|
|
||||||
|
return issue_detected
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument(
|
||||||
|
'toml_files',
|
||||||
|
type=pathlib.Path,
|
||||||
|
nargs=argparse.ZERO_OR_MORE,
|
||||||
|
help='*.toml files to lint (default: src/**/ci/*.toml)',
|
||||||
|
)
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if not args.toml_files:
|
||||||
|
args.toml_files = pathlib.Path('src').glob('**/ci/*.toml')
|
||||||
|
|
||||||
|
error = False
|
||||||
|
|
||||||
|
for path in args.toml_files:
|
||||||
|
with path.open('r') as toml_file:
|
||||||
|
toml_lines = toml_file.readlines()
|
||||||
|
if detect_misleading_indentation(path.as_posix(), toml_lines):
|
||||||
|
error = True
|
||||||
|
|
||||||
|
if error:
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
Executable
+66
@@ -0,0 +1,66 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
if [ ! -e .git ]; then
|
||||||
|
echo must run from top-level directory;
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -d platform-hardware-libhardware ]; then
|
||||||
|
git clone --depth 1 https://android.googlesource.com/platform/frameworks/native platform-frameworks-native
|
||||||
|
git clone --depth 1 https://android.googlesource.com/platform/hardware/libhardware platform-hardware-libhardware
|
||||||
|
git clone --depth 1 https://android.googlesource.com/platform/system/core platform-system-core
|
||||||
|
git clone --depth 1 https://android.googlesource.com/platform/system/logging platform-system-logging
|
||||||
|
fi
|
||||||
|
|
||||||
|
dest=include/android_stub
|
||||||
|
|
||||||
|
# Persist the frozen Android N system/window.h for backward compatibility
|
||||||
|
|
||||||
|
cp -av ${dest}/system/window.h platform-system-core/libsystem/include/system
|
||||||
|
|
||||||
|
# Persist the frozen libbacktrace header for backward compatibility, since the
|
||||||
|
# support has been dropped in Android 15.
|
||||||
|
#
|
||||||
|
# TODO: add support for libunwindstack
|
||||||
|
mkdir -p platform-system-unwinding/libbacktrace/include
|
||||||
|
cp -av ${dest}/backtrace platform-system-unwinding/libbacktrace/include/
|
||||||
|
|
||||||
|
rm -rf ${dest}
|
||||||
|
mkdir ${dest}
|
||||||
|
|
||||||
|
|
||||||
|
# These directories contains only the files we need, so copy wholesale
|
||||||
|
|
||||||
|
cp -av \
|
||||||
|
platform-frameworks-native/libs/nativewindow/include/vndk \
|
||||||
|
platform-frameworks-native/libs/nativebase/include/nativebase \
|
||||||
|
platform-system-core/libsync/include/ndk \
|
||||||
|
platform-system-core/libsync/include/sync \
|
||||||
|
platform-system-unwinding/libbacktrace/include/backtrace \
|
||||||
|
${dest}
|
||||||
|
|
||||||
|
|
||||||
|
# We only need a few files from these big directories so just copy those
|
||||||
|
|
||||||
|
mkdir ${dest}/hardware
|
||||||
|
cp -av platform-hardware-libhardware/include_all/hardware/{hardware,gralloc,gralloc1,fb}.h ${dest}/hardware
|
||||||
|
cp -av platform-frameworks-native/vulkan/include/hardware/hwvulkan.h ${dest}/hardware
|
||||||
|
|
||||||
|
mkdir ${dest}/cutils
|
||||||
|
cp -av platform-system-core/libcutils/include/cutils/native_handle.h ${dest}/cutils
|
||||||
|
|
||||||
|
mkdir ${dest}/system
|
||||||
|
cp -av platform-system-core/libsystem/include/system/{graphics*,window.h} ${dest}/system
|
||||||
|
|
||||||
|
# include/android has files from a few different projects
|
||||||
|
|
||||||
|
mkdir ${dest}/android
|
||||||
|
cp -av \
|
||||||
|
platform-frameworks-native/libs/nativewindow/include/android/{data_space,hardware_buffer,native_window}.h \
|
||||||
|
platform-frameworks-native/libs/arect/include/android/rect.h \
|
||||||
|
platform-system-core/libsync/include/android/sync.h \
|
||||||
|
platform-system-logging/liblog/include/android/log.h \
|
||||||
|
${dest}/android
|
||||||
|
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
{
|
||||||
|
radeon_drm_winsys_create;
|
||||||
|
};
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
VERSION_1 {
|
||||||
|
global:
|
||||||
|
main;
|
||||||
|
local:
|
||||||
|
*;
|
||||||
|
};
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
# For performance reasons we don't use a lock here and reading
|
||||||
|
# a stale value is of no consequence
|
||||||
|
fun:util_queue_fence_is_signalled
|
||||||
|
# We also have to blacklist this function, because otherwise tsan will
|
||||||
|
# still report the unlocked read above
|
||||||
|
fun:util_queue_fence_signal
|
||||||
|
|
||||||
|
# lavapipe:
|
||||||
|
# Same as above for perf reasons the fence signal value is is read without
|
||||||
|
# lock
|
||||||
|
fun:lp_fence_signalled
|
||||||
|
fun:lp_fence_signal
|
||||||
|
|
||||||
|
# gallium/tc
|
||||||
|
# Keeping track of tc->last_completed is an optimization and it is of no
|
||||||
|
# consequence to read a stale value there, so surpress the warning about the
|
||||||
|
# race condition
|
||||||
|
fun:tc_batch_execute
|
||||||
|
|
||||||
|
# This is a debug feature and ATM it is simpler to surpress the race warning
|
||||||
|
fun:tc_set_driver_thread
|
||||||
|
|
||||||
|
# vulkan/runtime
|
||||||
|
# Even with the data race the returned value is always the same
|
||||||
|
fun:get_max_abs_timeout_ns
|
||||||
|
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
# Keep this in sync with the `rustc.version()` check in meson.build
|
||||||
|
msrv = "1.85.0"
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
Known issues in the ARB_color_buffer_float implementation:
|
||||||
|
- Rendering to multiple render targets, some fixed-point, some floating-point, with FIXED_ONLY fragment clamping and polygon smooth enabled may write incorrect values to the fixed point buffers (depends on spec interpretation)
|
||||||
|
- For fragment programs with ARB_fog_* options, colors are clamped before fog application regardless of the fragment clamping setting (this depends on spec interpretation)
|
||||||
|
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
The software may implement third party technologies (e.g. third party
|
||||||
|
libraries) that are not licensed to you by AMD and for which you may need
|
||||||
|
to obtain licenses from other parties. Unless explicitly stated otherwise,
|
||||||
|
these third party technologies are not licensed hereunder. Such third
|
||||||
|
party technologies include, but are not limited, to H.264, H.265, HEVC, MPEG-2,
|
||||||
|
MPEG-4, AVC, and VC-1.
|
||||||
|
|
||||||
|
For MPEG-2 Encoding Products ANY USE OF THIS PRODUCT IN ANY MANNER OTHER
|
||||||
|
THAN PERSONAL USE THAT COMPLIES WITH THE MPEG-2 STANDARD FOR ENCODING VIDEO
|
||||||
|
INFORMATION FOR PACKAGED MEDIA IS EXPRESSLY PROHIBITED WITHOUT A LICENSE
|
||||||
|
UNDER APPLICABLE PATENTS IN THE MPEG-2 PATENT PORTFOLIO, WHICH LICENSES IS
|
||||||
|
AVAILABLE FROM MPEG LA, LLC, 6312 S. Fiddlers Green Circle, Suite 400E,
|
||||||
|
Greenwood Village, Colorado 80111 U.S.A.
|
||||||
|
|
||||||
|
WARRANTY DISCLAIMER: THE SOFTWARE IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY
|
||||||
|
KIND. AMD DISCLAIMS ALL WARRANTIES, EXPRESS, IMPLIED, OR STATUTORY, INCLUDING
|
||||||
|
BUT NOT LIMITED TO THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
|
||||||
|
PARTICULAR PURPOSE, TITLE, NON-INFRINGEMENT, THAT THE SOFTWARE WILL RUN
|
||||||
|
UNINTERRUPTED OR ERROR-FREE OR WARRANTIES ARISING FROM CUSTOM OF TRADE OR
|
||||||
|
COURSE OF USAGE. THE ENTIRE RISK ASSOCIATED WITH THE USE OF THE SOFTWARE IS
|
||||||
|
ASSUMED BY YOU. Some jurisdictions do not allow the exclusion of implied
|
||||||
|
warranties, so the above exclusion may not apply to You.
|
||||||
|
|
||||||
|
LIMITATION OF LIABILITY AND INDEMNIFICATION: AMD AND ITS LICENSORS WILL NOT,
|
||||||
|
UNDER ANY CIRCUMSTANCES BE LIABLE FOR ANY PUNITIVE, DIRECT, INCIDENTAL,
|
||||||
|
INDIRECT, SPECIAL OR CONSEQUENTIAL DAMAGES ARISING FROM USE OF THE SOFTWARE OR
|
||||||
|
THIS AGREEMENT EVEN IF AMD AND ITS LICENSORS HAVE BEEN ADVISED OF THE
|
||||||
|
POSSIBILITY OF SUCH DAMAGES. In no event shall AMD's total liability to You
|
||||||
|
for all damages, losses, and causes of action (whether in contract, tort
|
||||||
|
(including negligence) or otherwise) exceed the amount of $100 USD. You agree
|
||||||
|
to defend, indemnify and hold harmless AMD and its licensors, and any of their
|
||||||
|
directors, officers, employees, affiliates or agents from and against any and
|
||||||
|
all loss, damage, liability and other expenses (including reasonable
|
||||||
|
attorneys' fees), resulting from Your use of the Software or violation of the
|
||||||
|
terms and conditions of this Agreement.
|
||||||
|
|
||||||
|
U.S. GOVERNMENT RESTRICTED RIGHTS: The Software is provided with "RESTRICTED
|
||||||
|
RIGHTS." Use, duplication, or disclosure by the Government is subject to the
|
||||||
|
restrictions as set forth in FAR 52.227-14 and DFAR252.227-7013, et seq., or
|
||||||
|
its successor. Use of the Software by the Government constitutes
|
||||||
|
acknowledgement of AMD's proprietary rights in them.
|
||||||
|
|
||||||
|
EXPORT RESTRICTIONS: The Software may be subject to export restrictions as
|
||||||
|
stated in the Software License Agreement.
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
The software may implement third party technologies (e.g. third party
|
||||||
|
libraries) that are not licensed to you by AMD and for which you may need
|
||||||
|
to obtain licenses from other parties. Unless explicitly stated otherwise,
|
||||||
|
these third party technologies are not licensed hereunder. Such third
|
||||||
|
party technologies include, but are not limited, to H.264, MPEG-2, MPEG-4,
|
||||||
|
AVC, and VC-1.
|
||||||
|
|
||||||
|
For MPEG-2 Intermediate Products: ANY USE OF THIS PRODUCT IN ANY MANNER OTHER
|
||||||
|
THAN PERSONAL USE THAT COMPLIES WITH THE MPEG-2 STANDARD IS EXPRESSLY
|
||||||
|
PROHIBITED WITHOUT A LICENSE UNDER APPLICABLE PATENTS IN THE MPEG-2 PATENT
|
||||||
|
PORTFOLIO, WHICH LICENSES IS AVAILABLE FROM MPEG LA, LLC, 6312 S. Fiddlers
|
||||||
|
Green Circle, Suite 400E, Greenwood Village, Colorado 80111 U.S.A.
|
||||||
|
|
||||||
|
WARRANTY DISCLAIMER: THE SOFTWARE IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY
|
||||||
|
KIND. AMD DISCLAIMS ALL WARRANTIES, EXPRESS, IMPLIED, OR STATUTORY, INCLUDING
|
||||||
|
BUT NOT LIMITED TO THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
|
||||||
|
PARTICULAR PURPOSE, TITLE, NON-INFRINGEMENT, THAT THE SOFTWARE WILL RUN
|
||||||
|
UNINTERRUPTED OR ERROR-FREE OR WARRANTIES ARISING FROM CUSTOM OF TRADE OR
|
||||||
|
COURSE OF USAGE. THE ENTIRE RISK ASSOCIATED WITH THE USE OF THE SOFTWARE IS
|
||||||
|
ASSUMED BY YOU. Some jurisdictions do not allow the exclusion of implied
|
||||||
|
warranties, so the above exclusion may not apply to You.
|
||||||
|
|
||||||
|
LIMITATION OF LIABILITY AND INDEMNIFICATION: AMD AND ITS LICENSORS WILL NOT,
|
||||||
|
UNDER ANY CIRCUMSTANCES BE LIABLE FOR ANY PUNITIVE, DIRECT, INCIDENTAL,
|
||||||
|
INDIRECT, SPECIAL OR CONSEQUENTIAL DAMAGES ARISING FROM USE OF THE SOFTWARE OR
|
||||||
|
THIS AGREEMENT EVEN IF AMD AND ITS LICENSORS HAVE BEEN ADVISED OF THE
|
||||||
|
POSSIBILITY OF SUCH DAMAGES. In no event shall AMD's total liability to You
|
||||||
|
for all damages, losses, and causes of action (whether in contract, tort
|
||||||
|
(including negligence) or otherwise) exceed the amount of $100 USD. You agree
|
||||||
|
to defend, indemnify and hold harmless AMD and its licensors, and any of their
|
||||||
|
directors, officers, employees, affiliates or agents from and against any and
|
||||||
|
all loss, damage, liability and other expenses (including reasonable
|
||||||
|
attorneys' fees), resulting from Your use of the Software or violation of the
|
||||||
|
terms and conditions of this Agreement.
|
||||||
|
|
||||||
|
U.S. GOVERNMENT RESTRICTED RIGHTS: The Software is provided with "RESTRICTED
|
||||||
|
RIGHTS." Use, duplication, or disclosure by the Government is subject to the
|
||||||
|
restrictions as set forth in FAR 52.227-14 and DFAR252.227-7013, et seq., or
|
||||||
|
its successor. Use of the Software by the Government constitutes
|
||||||
|
acknowledgement of AMD's proprietary rights in them.
|
||||||
|
|
||||||
|
EXPORT RESTRICTIONS: The Software may be subject to export restrictions as
|
||||||
|
stated in the Software License Agreement.
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
/webmaster.html https://www.mesa3d.org/website/
|
||||||
|
/developers.html https://www.mesa3d.org/developers/
|
||||||
|
/thanks.html https://gitlab.freedesktop.org/mesa/mesa/-/blob/amber/docs/thanks.rst
|
||||||
|
|
||||||
|
/drivers/vmware-guest.html /drivers/svga3d.html 301
|
||||||
|
/gallium/drivers/freedreno.html /drivers/freedreno.html 301
|
||||||
|
/gallium/drivers/freedreno/ir3-notes.html /drivers/freedreno/ir3-notes.html 301
|
||||||
|
/gallium/drivers/llvmpipe.html /drivers/llvmpipe.html 301
|
||||||
|
/gallium/drivers/zink.html /drivers/zink.html 301
|
||||||
|
/llvmpipe.html /drivers/llvmpipe.html 301
|
||||||
|
/versions.html /relnotes.html 301
|
||||||
|
/vmware-guest.html /drivers/vmware-guest.html 301
|
||||||
|
/shading.html /glsl.html
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
|
||||||
|
Mesa 3.1 release notes
|
||||||
|
|
||||||
|
PLEASE READ!!!!
|
||||||
|
|
||||||
|
|
||||||
|
New copyright
|
||||||
|
-------------
|
||||||
|
|
||||||
|
Mesa 3.1 will be distributed under an XFree86-style copyright instead
|
||||||
|
of the GNU LGPL.
|
||||||
|
|
||||||
|
|
||||||
|
New directories
|
||||||
|
---------------
|
||||||
|
|
||||||
|
All documentation files are now in the docs/ directory.
|
||||||
|
All shell scripts are now in the bin/ directory.
|
||||||
|
|
||||||
|
|
||||||
|
New library names
|
||||||
|
-----------------
|
||||||
|
|
||||||
|
Formerly, the main Mesa library was named libMesaGL.so (or libMesaGL.a)
|
||||||
|
and the GLU library was named libMesaGLU.so (or libMesaGLU.a).
|
||||||
|
|
||||||
|
Now, the main library is named libGL.so (or libGL.a) and the GLU library
|
||||||
|
is named libGLU.so (or libGLU.a).
|
||||||
|
|
||||||
|
The change allows Mesa to be more easily substituted for OpenGL.
|
||||||
|
Specifically, the linker/loader on some Unix-like systems won't
|
||||||
|
allow libMesaGL.so to be used instead of libGL.so if the application
|
||||||
|
was linked with the former.
|
||||||
|
|
||||||
|
Warning: if you have another OpenGL implementation installed on your
|
||||||
|
system (i.e. you have another OpenGL libGL.so) you'll have to be
|
||||||
|
carefull about which library (OpenGL or Mesa) you link against. Be
|
||||||
|
aware of -L linker flags and the value of the LD_LIBRARY_PATH environment
|
||||||
|
variable.
|
||||||
|
|
||||||
|
|
||||||
|
New library versioning
|
||||||
|
----------------------
|
||||||
|
|
||||||
|
Previously, the Mesa GL library was named libMesaGL.so.3.0
|
||||||
|
To better support Linux/OpenGL standards, the Mesa GL library is now
|
||||||
|
named libGL.so.1.2.030100 This indicates version 1.2 of the OpenGL spec
|
||||||
|
and Mesa implementation 3.1.0
|
||||||
|
|
||||||
|
In the long term this will allow better interoperability with other
|
||||||
|
OpenGL implementations, especially on Linux. In the short term,
|
||||||
|
OpenGL apps may have to be relinked to use the new library naming.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
New makefiles
|
||||||
|
-------------
|
||||||
|
|
||||||
|
The old Makefiles found in the various directories have been renamed
|
||||||
|
to Makefile.X11 in order to prevent filename collisions with autoconfig-
|
||||||
|
generated Makefiles.
|
||||||
|
|
||||||
|
The top-level Makefile simply includes Makefile.X11
|
||||||
|
If your top-level Makefile get's overwritten/destroyed you can restore
|
||||||
|
it by copying Makefile.X11 to Makefile
|
||||||
|
|
||||||
|
|
||||||
|
New extensions
|
||||||
|
--------------
|
||||||
|
|
||||||
|
GL_EXT_stencil_wrap
|
||||||
|
Implements two new stencil operations: GL_INCR_WRAP_EXT and
|
||||||
|
GL_DECR_WRAP_EXT which allow stencil increment and decrement
|
||||||
|
without clamping.
|
||||||
|
|
||||||
|
GL_INGR_blend_func_separate
|
||||||
|
Allows specification of blend factors for RGB and Alpha independently.
|
||||||
|
(INGR = Intergraph)
|
||||||
|
|
||||||
|
GL_ARB_multitexture
|
||||||
|
Multiple simultaneous textures. (ARB = Architecture Review Board)
|
||||||
|
|
||||||
|
GL_NV_texgen_reflection
|
||||||
|
nVidia texgen extension for better reflection mapping.
|
||||||
|
|
||||||
|
GL_PGI_misc_hints
|
||||||
|
Assorted transformation hints.
|
||||||
|
|
||||||
|
GL_EXT_compiled_vertex_array
|
||||||
|
Compiled vertex arrays.
|
||||||
|
|
||||||
|
GL_EXT_clip_volume_hint
|
||||||
|
Allows one to disable clip volume (frustum) testing.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Extensions removed
|
||||||
|
------------------
|
||||||
|
|
||||||
|
GL_EXT_multitexture - obsolete in favor of GL_ARB_multitexture
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Config file
|
||||||
|
-----------
|
||||||
|
|
||||||
|
By default, /etc/mesa.conf will be read when Mesa starts. This
|
||||||
|
file controls default hints, enable/disable of extensions, and
|
||||||
|
more. See the CONFIG file for documentation.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Optimizations
|
||||||
|
-------------
|
||||||
|
|
||||||
|
Keith Whitwell has contributed significant optimizations to Mesa's
|
||||||
|
vertex transformation code. Basically, the whole transformation
|
||||||
|
stage of Mesa has been rewritten.
|
||||||
|
|
||||||
|
It's impossible to give a speedup factor. You'll just have to
|
||||||
|
try your app and see how it performs.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Device Driver changes
|
||||||
|
---------------------
|
||||||
|
|
||||||
|
A bunch of new device driver functions have been added. See src/dd.h
|
||||||
|
Keith Harrison contributed many of them. I've been planning on adding
|
||||||
|
a bunch of functions like these to make writing hardware drivers easier.
|
||||||
|
More such function will probably be added in the near future.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Miscellaneous
|
||||||
|
-------------
|
||||||
|
|
||||||
|
util/glstate.c has some handy functions for debugging. Basically, it
|
||||||
|
offers a simple function for printing GL state variables. It's not
|
||||||
|
finished yet. There's a LOT more GLenum records to be added (see the
|
||||||
|
code). Anyone want to help?
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
|
||||||
|
Mesa 3.2 release notes
|
||||||
|
|
||||||
|
PLEASE READ!!!!
|
||||||
|
|
||||||
|
|
||||||
|
Mesa 3.2 is a stabilization of the Mesa 3.1 release. No new features
|
||||||
|
have been added. For a list of bug fixes please read the VERSIONS file.
|
||||||
|
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
|
||||||
|
Mesa 3.2.1 release notes
|
||||||
|
|
||||||
|
PLEASE READ!!!!
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
The Mesa 3.2.1 release mainly just fixes bugs since the 3.2 release.
|
||||||
|
See the VERSIONS file for the exact list.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
GLU Polygon Tessellator
|
||||||
|
-----------------------
|
||||||
|
|
||||||
|
The GLU tessellator has been reverted back to the version included
|
||||||
|
with Mesa 3.0 since it's more stable. The Mesa 3.1/3.2 tessellator
|
||||||
|
implemented the GLU 1.3 specification but suffered from a number of
|
||||||
|
bugs.
|
||||||
|
|
||||||
|
Mesa implements GLU 1.1.
|
||||||
|
|
||||||
|
Ideally, people should use the GLU 1.3 library included in SGI's
|
||||||
|
OpenGL Sample Implementation (SI) available from
|
||||||
|
http://oss.sgi.com/projects/ogl-sample/
|
||||||
|
People are working to make easy-to-install Linux RPMs of the
|
||||||
|
GLU library.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
@@ -0,0 +1,270 @@
|
|||||||
|
|
||||||
|
Mesa 3.3 release notes
|
||||||
|
|
||||||
|
July 21, 2000
|
||||||
|
|
||||||
|
PLEASE READ!!!!
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Introduction
|
||||||
|
------------
|
||||||
|
|
||||||
|
Mesa uses an even/odd version number scheme like the Linux kernel.
|
||||||
|
Odd numbered versions (such as 3.3) designate new developmental releases.
|
||||||
|
Even numbered versions (such as 3.2.1) designate stable releases.
|
||||||
|
|
||||||
|
Mesa 3.3 has a undergone many internal changes since version 3.2
|
||||||
|
and features a lot of new extensions. 3.3 is expected to be pretty
|
||||||
|
stable, but perhaps not as stable as 3.2 which has been used by
|
||||||
|
thousands of users over the past months.
|
||||||
|
|
||||||
|
Everyone is encouraged to try Mesa 3.3. Bugs should be reported to
|
||||||
|
the Mesa bug database on www.sourceforge.net.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Header file / GLenum changes
|
||||||
|
----------------------------
|
||||||
|
|
||||||
|
The gl.h and glu.h headers now use #defines to define all GL_* tokens
|
||||||
|
instead of C-language enums. This change improves Mesa/OpenGL
|
||||||
|
interoperability.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
New API dispatch code
|
||||||
|
---------------------
|
||||||
|
|
||||||
|
The core Mesa gl* functions are now implemented with a new dispatch
|
||||||
|
(jump table) which will allow simultaneous direct/indirect rendering.
|
||||||
|
|
||||||
|
The code is found in the glapi*.[ch] files.
|
||||||
|
|
||||||
|
Of interest: the actual "glFooBar" functions are generated with
|
||||||
|
templatized code defined in glapitemp.h and included by glapi.c
|
||||||
|
The glapitemp.h template should be reusable for all sorts of OpenGL
|
||||||
|
projects.
|
||||||
|
|
||||||
|
The new dispatch code has also optimized with x86 assembly code.
|
||||||
|
This optimization eliminates copying the function arguments during
|
||||||
|
dispatch.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
New thread support
|
||||||
|
------------------
|
||||||
|
|
||||||
|
Thread support in Mesa has been rewritten. The glthread.[ch] files
|
||||||
|
replace mthreads.[ch]. Thread safety is always enabled (on platforms
|
||||||
|
which support threads, that is). There is virtually no performance
|
||||||
|
penalty for typical single-thread applications. See the glapi.c
|
||||||
|
file for details.
|
||||||
|
|
||||||
|
The Xlib driver (XMesa) is now thread-safe as well. Be sure to
|
||||||
|
call XInitThreads() in your app first. See the xdemos/glthreads.c
|
||||||
|
demo for an example.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Make configuration changes
|
||||||
|
--------------------------
|
||||||
|
|
||||||
|
If you use the old-style (non GNU automake) method to build Mesa note
|
||||||
|
that several of the configuration names have changed:
|
||||||
|
|
||||||
|
Old name New name
|
||||||
|
------------- ----------------
|
||||||
|
linux-elf linux
|
||||||
|
linux linux-static
|
||||||
|
linux-386-elf linux-386
|
||||||
|
linux-386 linux-386-static
|
||||||
|
etc.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
New extensions
|
||||||
|
--------------
|
||||||
|
|
||||||
|
GL_ARB_transpose_matrix
|
||||||
|
Adds glLoadTransposeMatrixARB() and glMultTransposeMatrixARB()
|
||||||
|
functions.
|
||||||
|
|
||||||
|
GL_ARB_texture_cube_map
|
||||||
|
For cube-based reflection mapping.
|
||||||
|
|
||||||
|
GL_EXT_texture_add_env
|
||||||
|
Adds GL_ADD texture environment mode.
|
||||||
|
See http://www.berkelium.com/OpenGL/EXT/texture_env_add.txt
|
||||||
|
|
||||||
|
GL_EXT_texture_lod_bias
|
||||||
|
Allows mipmapped texture blurring and sharpening.
|
||||||
|
|
||||||
|
GLX_EXT_visual_rating extension
|
||||||
|
This extension has no effect in stand-alone Mesa (used for DRI).
|
||||||
|
|
||||||
|
GL_HP_occlusion_test
|
||||||
|
Used for bounding box occlusion testing (see demos/occlude.c).
|
||||||
|
|
||||||
|
GL_SGIX_pixel_texture / GL_SGIS_pixel_texture
|
||||||
|
Lets glDraw/CopyPixels draw a texture coordinate image.
|
||||||
|
|
||||||
|
GL_SGI_color_matrix
|
||||||
|
Adds a color matrix and another set of scale and bias parameters
|
||||||
|
to the glDraw/CopyPixels paths.
|
||||||
|
|
||||||
|
GL_SGI_color_table
|
||||||
|
Adds additional color tables to the glDraw/Read/CopyPixels paths.
|
||||||
|
|
||||||
|
GL_EXT_histogram
|
||||||
|
Compute histograms for glDraw/Read/CopyPixels.
|
||||||
|
|
||||||
|
GL_EXT_blend_func_separate
|
||||||
|
This is the same as GL_INGR_blend_func_separate.
|
||||||
|
|
||||||
|
GL_ARB_texture_cube_mapping
|
||||||
|
6-face cube mapping, nicer than sphere mapping
|
||||||
|
|
||||||
|
GL_EXT_texture_env_combine
|
||||||
|
For advanced texture environment effects.
|
||||||
|
|
||||||
|
|
||||||
|
Documentation for all these functions can be found at
|
||||||
|
http://oss.sgi.com/projects/ogl-sample/registry/
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
GLX_SGI_make_current_read functionality
|
||||||
|
---------------------------------------
|
||||||
|
|
||||||
|
The functionality of this extension is needed for GLX 1.3 (and required
|
||||||
|
for the Linux/OpenGL standards base).
|
||||||
|
|
||||||
|
Implementing this function required a **DEVICE DRIVER CHANGE**.
|
||||||
|
The old SetBuffer() function has been replaced by SetReadBuffer() and
|
||||||
|
SetDrawBuffer(). All device drivers will have to be updated because
|
||||||
|
of this change.
|
||||||
|
|
||||||
|
The new function, glXMakeContextCurrent(), in GLX 1.3 now works in Mesa.
|
||||||
|
The xdemos/wincopy.c program demonstrates it.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Image-related code changes
|
||||||
|
--------------------------
|
||||||
|
|
||||||
|
The imaging path code used by glDrawPixels, glTexImage[123]D,
|
||||||
|
glTexSubImage[123], etc has been rewritten. It's now faster,
|
||||||
|
uses less memory and has several bug fixes. This work was
|
||||||
|
actually started in Mesa 3.1 with the glTexImage paths but has now
|
||||||
|
been carried over to glDrawPixels as well.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Device driver interface changes
|
||||||
|
-------------------------------
|
||||||
|
|
||||||
|
Added new functions for hardware stencil buffer support:
|
||||||
|
WriteStencilSpan
|
||||||
|
ReadStencilSpan
|
||||||
|
WriteStencilPixels
|
||||||
|
ReadStencilPixels
|
||||||
|
|
||||||
|
|
||||||
|
Removed old depth buffer functions:
|
||||||
|
AllocDepthBuffer
|
||||||
|
DepthTestSpan
|
||||||
|
DepthTestPixels
|
||||||
|
ReadDepthSpanFloat
|
||||||
|
ReadDepthSpanInt
|
||||||
|
|
||||||
|
|
||||||
|
Added new depth buffer functions:
|
||||||
|
WriteDepthSpan
|
||||||
|
ReadDepthSpan
|
||||||
|
WriteDepthPixels
|
||||||
|
ReadDepthPixels
|
||||||
|
|
||||||
|
These functions always read/write 32-bit GLuints. This will allow
|
||||||
|
drivers to have anywhere from 0 to 32-bit Z buffers without
|
||||||
|
recompiling for 16 vs 32 bits as was previously needed.
|
||||||
|
|
||||||
|
|
||||||
|
New texture image functions
|
||||||
|
The entire interface for texture image specification has been updated.
|
||||||
|
With the new functions, it's optional for Mesa to keep an internal copy
|
||||||
|
of all textures. Texture download should be a lot faster when the extra
|
||||||
|
copy isn't made.
|
||||||
|
|
||||||
|
Misc changes
|
||||||
|
TexEnv now takes a target argument
|
||||||
|
Removed UseGlobalTexturePalette (use Enable function instead)
|
||||||
|
|
||||||
|
|
||||||
|
Also added
|
||||||
|
ReadPixels
|
||||||
|
CopyPixels
|
||||||
|
|
||||||
|
|
||||||
|
The SetBufffer function has been replaced by SetDrawBuffer and
|
||||||
|
SetReadBuffer functions. This lets core Mesa independently
|
||||||
|
specify which buffer is to be used for reading and which for
|
||||||
|
drawing.
|
||||||
|
|
||||||
|
The Clear function's mask parameter has changed. Instead of
|
||||||
|
mask being the flags specified by the user to glClear, the
|
||||||
|
mask is now a bitmask of the DD_*_BIT flags in dd.h. Now
|
||||||
|
multiple color buffers can be specified for clearing (ala
|
||||||
|
glDrawBuffers). The driver's Clear function must also
|
||||||
|
check the glColorMask glIndexMask, and glStencilMask settings
|
||||||
|
and do the right thing. See the X/Mesa, OS/Mesa, or FX/Mesa
|
||||||
|
drivers for examples.
|
||||||
|
|
||||||
|
|
||||||
|
The depth buffer changes shouldn't be hard to make for existing
|
||||||
|
drivers. In fact, it should simply the code. Be careful with
|
||||||
|
the depthBits value passed to gl_create_context(). 1 is a bad
|
||||||
|
value! It should normally be 0, 16, 24, or 32.
|
||||||
|
|
||||||
|
|
||||||
|
gl_create_framebuffer() takes new arguments which explicitly tell
|
||||||
|
core Mesa which ancillary buffers (depth, stencil, accum, alpha)
|
||||||
|
should be implemented in software. Mesa hardware drivers should
|
||||||
|
carefully set these flags depending on which buffers are in the
|
||||||
|
graphics card.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Internal constants
|
||||||
|
------------------
|
||||||
|
|
||||||
|
Point and line size range and granularity limits are now stored
|
||||||
|
in the gl_constants struct, which is the Const member of GLcontext.
|
||||||
|
The limits are initialized from values in config.h but may be
|
||||||
|
overridden by device drivers to reflect the limits of that driver's
|
||||||
|
hardware.
|
||||||
|
|
||||||
|
Also added constants for NumAuxBuffers and SubPixelBits.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
OpenGL Conformance
|
||||||
|
------------------
|
||||||
|
|
||||||
|
Mesa now passes all the OpenGL 1.1 conformance tests, except for
|
||||||
|
antialiased lines. AA lines fail on some, but not all, the tests.
|
||||||
|
In order to fix the remaining failures, a new AA line algorithm will
|
||||||
|
be needed (which computes coverage values for end-point fragments).
|
||||||
|
This will be done for Mesa 3.5/3.6.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
OpenGL 1.2 GL_ARB_imaging subset
|
||||||
|
--------------------------------
|
||||||
|
|
||||||
|
Mesa 3.3 implements all the features of GL_ARB_imaging except for
|
||||||
|
image convolution. This will (hopefully) be done for Mesa 3.5/3.6.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
|
||||||
|
Mesa 3.4 release notes
|
||||||
|
|
||||||
|
November 3, 2000
|
||||||
|
|
||||||
|
PLEASE READ!!!!
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Introduction
|
||||||
|
------------
|
||||||
|
|
||||||
|
Mesa uses an even/odd version number scheme like the Linux kernel.
|
||||||
|
Odd numbered versions (such as 3.3) designate new developmental releases.
|
||||||
|
Even numbered versions (such as 3.4) designate stable releases.
|
||||||
|
|
||||||
|
Mesa 3.4 simply fixes bugs found in the Mesa 3.3 release. For details,
|
||||||
|
see the VERSIONS file.
|
||||||
|
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
|
||||||
|
Mesa 3.4.1 release notes
|
||||||
|
|
||||||
|
February 9, 2001
|
||||||
|
|
||||||
|
PLEASE READ!!!!
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Introduction
|
||||||
|
------------
|
||||||
|
|
||||||
|
Mesa uses an even/odd version number scheme like the Linux kernel.
|
||||||
|
Odd numbered versions (such as 3.3) designate new developmental releases.
|
||||||
|
Even numbered versions (such as 3.4) designate stable releases.
|
||||||
|
|
||||||
|
Mesa 3.4.1 is a maintenance release that simply fixes bugs found since
|
||||||
|
the Mesa 3.4 release. For details, see the VERSIONS file.
|
||||||
|
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
|
||||||
|
Mesa 3.4.2 release notes
|
||||||
|
|
||||||
|
May 17, 2001
|
||||||
|
|
||||||
|
PLEASE READ!!!!
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Introduction
|
||||||
|
------------
|
||||||
|
|
||||||
|
Mesa uses an even/odd version number scheme like the Linux kernel.
|
||||||
|
Odd numbered versions (such as 3.3) designate new developmental releases.
|
||||||
|
Even numbered versions (such as 3.4) designate stable releases.
|
||||||
|
|
||||||
|
Mesa 3.4.2 is a maintenance release that simply fixes bugs found since
|
||||||
|
the Mesa 3.4.1 release. For details, see the VERSIONS file.
|
||||||
|
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
@@ -0,0 +1,227 @@
|
|||||||
|
|
||||||
|
Mesa 3.5 release notes
|
||||||
|
|
||||||
|
June 21, 2001
|
||||||
|
|
||||||
|
PLEASE READ!!!!
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Introduction
|
||||||
|
------------
|
||||||
|
|
||||||
|
Mesa uses an even/odd version number scheme like the Linux kernel.
|
||||||
|
Odd numbered versions (such as 3.5) designate new developmental releases.
|
||||||
|
Even numbered versions (such as 3.4) designate stable releases.
|
||||||
|
|
||||||
|
The biggest change in Mesa 3.5 is a complete overhaul of the source
|
||||||
|
code in order to make it more modular. This was driven by the DRI
|
||||||
|
hardware drivers. It simplifies the DRI drivers and opens the door
|
||||||
|
to hardware transform/clip/lighting (TCL). Keith Whitwell can take
|
||||||
|
the credit for that.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Driver Support
|
||||||
|
--------------
|
||||||
|
|
||||||
|
The device driver interface in Mesa 3.5 has changed a lot since Mesa 3.4
|
||||||
|
Not all of the older Mesa drivers have been updated. Here's the status:
|
||||||
|
|
||||||
|
Driver Status
|
||||||
|
---------------------- -----------
|
||||||
|
XMesa (Xlib) updated
|
||||||
|
OSMesa (off-screen) updated
|
||||||
|
FX (3dfx Voodoo1/2) updated
|
||||||
|
SVGA updated
|
||||||
|
GGI not updated
|
||||||
|
Windows/Win32 not updated
|
||||||
|
DOS/DJGPP not updated
|
||||||
|
BeOS not updated
|
||||||
|
Allegro not updated
|
||||||
|
D3D not updated
|
||||||
|
DOS not updated
|
||||||
|
|
||||||
|
We're looking for volunteers to update the remaining drivers. Please
|
||||||
|
post to the Mesa3d-dev mailing list if you can help.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
GLU 1.3
|
||||||
|
-------
|
||||||
|
|
||||||
|
Mesa 3.5 includes the SGI Sample Implementation (SI) GLU library.
|
||||||
|
This version of GLU supports the GLU 1.3 specification. The old
|
||||||
|
Mesa GLU library implemented the 1.1 specification. The SI GLU
|
||||||
|
library should work much better.
|
||||||
|
|
||||||
|
You'll need a C++ compiler to compile the SI GLU library. This may
|
||||||
|
be a problem on some systems.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
New Extensions
|
||||||
|
--------------
|
||||||
|
|
||||||
|
GL_EXT_convolution
|
||||||
|
Adds image convolution to glRead/Copy/DrawPixels/TexImage.
|
||||||
|
|
||||||
|
GL_ARB_imaging
|
||||||
|
This is the optional imaging subset of OpenGL 1.2.
|
||||||
|
It's the GL_EXT_convolution, GL_HP_convolution_border_modes,
|
||||||
|
GL_EXT_histogram, GL_EXT_color_table, GL_EXT_color_subtable
|
||||||
|
GL_EXT_blend_color, GL_EXT_blend_minmax, GL_EXT_blend_subtract
|
||||||
|
and GL_SGI_color_matrix extensions all rolled together.
|
||||||
|
This is supported in all software renderers but not in all
|
||||||
|
hardware drivers (3dfx for example).
|
||||||
|
|
||||||
|
GL_ARB_texture_compression
|
||||||
|
This is supported in Mesa but only used by the 3dfx DRI drivers
|
||||||
|
for Voodoo4 and later.
|
||||||
|
|
||||||
|
GL_ARB_texture_env_add
|
||||||
|
This is identical to GL_EXT_texture_env_add.
|
||||||
|
|
||||||
|
GL_NV_blend_square
|
||||||
|
Adds extra blend source and dest factors which allow squaring
|
||||||
|
of color values.
|
||||||
|
|
||||||
|
GL_EXT_fog_coord
|
||||||
|
Allows specification of a per-vertex fog coordinate instead of
|
||||||
|
having fog always computed from the eye distance.
|
||||||
|
|
||||||
|
GL_EXT_secondary_color
|
||||||
|
Allows specifying the secondary (specular) color for each vertex
|
||||||
|
instead of getting it only from lighting in GL_SEPARATE_SPECULAR_COLOR
|
||||||
|
mode.
|
||||||
|
|
||||||
|
GL_ARB_texture_env_combine
|
||||||
|
Basically the same as GL_EXT_texture_env_combine
|
||||||
|
|
||||||
|
GL_ARB_texture_env_add extension
|
||||||
|
Texture addition mode.
|
||||||
|
|
||||||
|
GL_ARB_texture_env_dot3 extension
|
||||||
|
Dot product texture environment.
|
||||||
|
|
||||||
|
GL_ARB_texture_border_clamp
|
||||||
|
Adds GL_CLAMP_TO_BORDER_ARB texture wrap mode
|
||||||
|
|
||||||
|
GL_SGIX_depth_texture, GL_SGIX_shadow and GL_SGIX_shadow_ambient
|
||||||
|
Implements a shadow casting algorithm based on depth map textures
|
||||||
|
|
||||||
|
GL_SGIS_generate_mipmap
|
||||||
|
Automatically generate lower mipmap images whenever the base mipmap
|
||||||
|
image is changed with glTexImage, glCopyTexImage, etc.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
libOSMesa.so
|
||||||
|
------------
|
||||||
|
|
||||||
|
libOSMesa.so is a new library which contains the OSMesa interface for
|
||||||
|
off-screen rendering. Apps which need the OSMesa interface should link
|
||||||
|
with both -lOSMesa and -lGL. This change was made so that stand-alone
|
||||||
|
Mesa works the same way as XFree86/DRI's libGL.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Device Driver Changes / Core Mesa Changes
|
||||||
|
-----------------------------------------
|
||||||
|
|
||||||
|
The ctx->Driver.LogicOp() function has been removed. It used to
|
||||||
|
be called during state update in order to determine if the driver
|
||||||
|
could do glLogicOp() operations, and if not, set the SWLogicOpEnabled
|
||||||
|
flag. Drivers should instead examine the LogicOp state themselves
|
||||||
|
and choose specialized point, line, and triangle functions appropriately,
|
||||||
|
or fall back to software rendering. The Xlib driver was the only driver
|
||||||
|
to use this function. And since the Xlib driver no longer draws
|
||||||
|
points, lines or triangles using Xlib, the LogicOp function isn't needed.
|
||||||
|
|
||||||
|
The ctx->Driver.Dither() function has been removed. Drivers should
|
||||||
|
detect dither enable/disable via ctx->Driver.Enable() instead.
|
||||||
|
|
||||||
|
The ctx->Driver.IndexMask() and ctx->Driver.ColorMask() functions
|
||||||
|
are now just called from glIndexMask and glColorMask like the other
|
||||||
|
GL state-changing functions. They are no longer called from inside
|
||||||
|
gl_update_state(). Also, they now return void. The change was made
|
||||||
|
mostly for sake of uniformity.
|
||||||
|
|
||||||
|
The NEW_DRVSTATE[0123] flags have been removed. They weren't being used
|
||||||
|
and are obsolete w.r.t. the way state updates are done in DRI drivers.
|
||||||
|
|
||||||
|
|
||||||
|
Removed obsolete gl_create_visual() and gl_destroy_visual().
|
||||||
|
|
||||||
|
Renamed functions (new namespace):
|
||||||
|
|
||||||
|
old new
|
||||||
|
gl_create_framebuffer _mesa_create_framebuffer
|
||||||
|
gl_destroy_framebuffer _mesa_destroy_framebuffer
|
||||||
|
gl_create_context _mesa_create_context
|
||||||
|
gl_destroy_context _mesa_destroy_context
|
||||||
|
gl_context_initialize _mesa_context_initialize
|
||||||
|
gl_copy_context _mesa_copy_context
|
||||||
|
gl_make_current _mesa_make_current
|
||||||
|
gl_make_current2 _mesa_make_current2
|
||||||
|
gl_get_current_context _mesa_get_current_context
|
||||||
|
gl_flush_vb _mesa_flush_vb
|
||||||
|
gl_warning _mesa_warning
|
||||||
|
gl_compile_error _mesa_compile_error
|
||||||
|
|
||||||
|
|
||||||
|
All the drivers have been updated, but not all of them have been
|
||||||
|
tested since I can't test some platforms (DOS, Windows, Allegro, etc).
|
||||||
|
|
||||||
|
|
||||||
|
X/Mesa Driver
|
||||||
|
-------------
|
||||||
|
|
||||||
|
The source files for the X/Mesa driver in src/X have been renamed.
|
||||||
|
The xmesa[1234].c files are gone. The new files are xm_api.c,
|
||||||
|
xm_dd.c, xm_line.c, xm_span.c and xm_tri.c.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Multitexture
|
||||||
|
------------
|
||||||
|
|
||||||
|
Eight texture units are now supported by default.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
OpenGL SI related changes
|
||||||
|
-------------------------
|
||||||
|
|
||||||
|
In an effort to make Mesa's internal interfaces more like the OpenGL
|
||||||
|
SI interfaces, a number of changes have been made:
|
||||||
|
|
||||||
|
1. Importing the SI's glcore.h file which defines a number of
|
||||||
|
interface structures like __GLimports and __GLexports.
|
||||||
|
|
||||||
|
2. Renamed "struct gl_context" to "struct __GLcontextRec".
|
||||||
|
|
||||||
|
3. Added __glCoreCreateContext() and __glCoreNopDispatch() functions.
|
||||||
|
|
||||||
|
4. The GLcontext member Visual is no longer a pointer.
|
||||||
|
|
||||||
|
5. New file: imports.c to setup default import functions for Mesa.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
16-bit color channels
|
||||||
|
---------------------
|
||||||
|
|
||||||
|
There's experimental support for 16-bit color channels (64-bit pixels)
|
||||||
|
in Mesa 3.5. Only the OSMesa interface can be used for 16-bit rendering.
|
||||||
|
Type "make linux-osmesa16" in the top-level directory to build the
|
||||||
|
special libOSMesa16.so library.
|
||||||
|
|
||||||
|
This hasn't been tested very thoroughly yet so please file bug reports
|
||||||
|
if you have trouble.
|
||||||
|
|
||||||
|
In the future I hope to implement support for 32-bit, floating point
|
||||||
|
color channels.
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
|
||||||
|
Mesa 4.0 release notes
|
||||||
|
|
||||||
|
October 18, 2001
|
||||||
|
|
||||||
|
PLEASE READ!!!!
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Introduction
|
||||||
|
------------
|
||||||
|
|
||||||
|
Mesa uses an even/odd version number scheme like the Linux kernel.
|
||||||
|
Odd numbered versions (such as 3.3) designate new developmental releases.
|
||||||
|
Even numbered versions (such as 3.4) designate stable releases.
|
||||||
|
|
||||||
|
Mesa version 4.0 signifies two things:
|
||||||
|
|
||||||
|
1. A stabilization of the 3.5 development release
|
||||||
|
2. Implementation of the OpenGL 1.3 specification
|
||||||
|
|
||||||
|
|
||||||
|
Note that the Mesa major version number is incremented with the OpenGL
|
||||||
|
minor version number:
|
||||||
|
|
||||||
|
Mesa 1.x == OpenGL 1.0
|
||||||
|
Mesa 2.x == OpenGL 1.1
|
||||||
|
Mesa 3.x == OpenGL 1.2
|
||||||
|
Mesa 4.x == OpenGL 1.3
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
New Features
|
||||||
|
------------
|
||||||
|
|
||||||
|
Mesa 3.5 already had all the new features of OpenGL 1.3, implemented as
|
||||||
|
extensions. These extensions were simply promoted to standard features:
|
||||||
|
|
||||||
|
GL_ARB_multisample
|
||||||
|
GL_ARB_multitexture
|
||||||
|
GL_ARB_texture_border_clamp
|
||||||
|
GL_ARB_texture_compression
|
||||||
|
GL_ARB_texture_cube_map
|
||||||
|
GL_ARB_texture_env_add
|
||||||
|
GL_ARB_texture_env_combine
|
||||||
|
GL_ARB_texture_env_dot3
|
||||||
|
GL_ARB_transpose_matrix
|
||||||
|
|
||||||
|
In Mesa 4.0 the functions defined by these extensions are now available
|
||||||
|
without the "ARB" suffix. For example, glLoadTransposeMatrixf() is now
|
||||||
|
a standard API function. The new functions in OpenGL 1.3 and Mesa 4.0 are:
|
||||||
|
|
||||||
|
glActiveTexture
|
||||||
|
glClientActiveTexture
|
||||||
|
glCompressedTexImage1D
|
||||||
|
glCompressedTexImage2D
|
||||||
|
glCompressedTexImage3D
|
||||||
|
glCompressedTexSubImage1D
|
||||||
|
glCompressedTexSubImage2D
|
||||||
|
glCompressedTexSubImage3D
|
||||||
|
glGetCompressedTexImage
|
||||||
|
glLoadTransposeMatrixd
|
||||||
|
glLoadTransposeMatrixf
|
||||||
|
glMultiTexCoord1d
|
||||||
|
glMultiTexCoord1dv
|
||||||
|
glMultiTexCoord1f
|
||||||
|
glMultiTexCoord1fv
|
||||||
|
glMultiTexCoord1i
|
||||||
|
glMultiTexCoord1iv
|
||||||
|
glMultiTexCoord1s
|
||||||
|
glMultiTexCoord1sv
|
||||||
|
glMultiTexCoord2d
|
||||||
|
glMultiTexCoord2dv
|
||||||
|
glMultiTexCoord2f
|
||||||
|
glMultiTexCoord2fv
|
||||||
|
glMultiTexCoord2i
|
||||||
|
glMultiTexCoord2iv
|
||||||
|
glMultiTexCoord2s
|
||||||
|
glMultiTexCoord2sv
|
||||||
|
glMultiTexCoord3d
|
||||||
|
glMultiTexCoord3dv
|
||||||
|
glMultiTexCoord3f
|
||||||
|
glMultiTexCoord3fv
|
||||||
|
glMultiTexCoord3i
|
||||||
|
glMultiTexCoord3iv
|
||||||
|
glMultiTexCoord3s
|
||||||
|
glMultiTexCoord3sv
|
||||||
|
glMultiTexCoord4d
|
||||||
|
glMultiTexCoord4dv
|
||||||
|
glMultiTexCoord4f
|
||||||
|
glMultiTexCoord4fv
|
||||||
|
glMultiTexCoord4i
|
||||||
|
glMultiTexCoord4iv
|
||||||
|
glMultiTexCoord4s
|
||||||
|
glMultiTexCoord4sv
|
||||||
|
glMultTransposeMatrixd
|
||||||
|
glMultTransposeMatrixf
|
||||||
|
glSampleCoverage
|
||||||
|
glSamplePass
|
||||||
|
|
||||||
|
|
||||||
|
GLX 1.4 is the companion to OpenGL 1.3. The only new features in GLX 1.4
|
||||||
|
are support for multisampling and the GLX_ARB_get_proc_address extension.
|
||||||
|
glXGetProcAddress() is the only new function in GLX 1.4.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Multisample and Texture Compression
|
||||||
|
-----------------------------------
|
||||||
|
|
||||||
|
The OpenGL 1.3 specification allows the multisample and texture compression
|
||||||
|
features to essentially be no-ops. For example, if you query for multisample
|
||||||
|
support you'll find none, but the API functions work.
|
||||||
|
|
||||||
|
Similarly, texture compression is not implemented by any of the software
|
||||||
|
drivers but you can specify a generic compressed texture format (like
|
||||||
|
GL_COMPRESSED_RGBA) to glTexImage2D and it'll be accepted.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Device Drivers
|
||||||
|
--------------
|
||||||
|
|
||||||
|
Mesa advertises itself as either OpenGL 1.2 or OpenGL 1.3 depending on the
|
||||||
|
device driver. If the driver enables all the ARB extensions which are part
|
||||||
|
of OpenGL 1.3 then glGetString(GL_VERSION) will return "1.3". Otherwise,
|
||||||
|
it'll return "1.2".
|
||||||
|
|
||||||
|
A number of Mesa's software drivers haven't been actively maintained for
|
||||||
|
some time. We rely on volunteers to maintain many of the drivers.
|
||||||
|
Here's the current status of all included drivers:
|
||||||
|
|
||||||
|
|
||||||
|
Driver Status
|
||||||
|
---------------------- ---------------------
|
||||||
|
XMesa (Xlib) implements OpenGL 1.3
|
||||||
|
OSMesa (off-screen) implements OpenGL 1.3
|
||||||
|
FX (3dfx Voodoo1/2) implements OpenGL 1.3
|
||||||
|
SVGA implements OpenGL 1.3
|
||||||
|
Wind River UGL implements OpenGL 1.3
|
||||||
|
Windows/Win32 implements OpenGL 1.3
|
||||||
|
GGI needs updating
|
||||||
|
DOS/DJGPP needs updating
|
||||||
|
BeOS needs updating
|
||||||
|
Allegro needs updating
|
||||||
|
D3D needs updating
|
||||||
|
DOS needs updating
|
||||||
|
|
||||||
|
Special thanks go to Karl Schultz for updating the Windows driver.
|
||||||
|
|
||||||
|
The XFree86/DRI drivers have not yet been updated to use Mesa 4.0 as of
|
||||||
|
September 2001, but that should happen eventually.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Other Changes
|
||||||
|
-------------
|
||||||
|
|
||||||
|
See the VERSIONS file for more details about bug fixes, etc. in Mesa 4.0.
|
||||||
|
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
|
||||||
|
Mesa 4.0.1 release notes
|
||||||
|
|
||||||
|
December 17, 2001
|
||||||
|
|
||||||
|
PLEASE READ!!!!
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Introduction
|
||||||
|
------------
|
||||||
|
|
||||||
|
Mesa uses an even/odd version number scheme like the Linux kernel.
|
||||||
|
Odd numbered versions (such as 3.3) designate new developmental releases.
|
||||||
|
Even numbered versions (such as 3.4) designate stable releases.
|
||||||
|
|
||||||
|
Mesa 4.0.1 only contains bug fixes since version 4.0.
|
||||||
|
|
||||||
|
See the docs/VERSIONS file for the list of bug fixes.
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
|
||||||
|
Mesa 4.0.2 release notes
|
||||||
|
|
||||||
|
March 25, 2002
|
||||||
|
|
||||||
|
PLEASE READ!!!!
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Introduction
|
||||||
|
------------
|
||||||
|
|
||||||
|
Mesa uses an even/odd version number scheme like the Linux kernel.
|
||||||
|
Odd numbered versions (such as 3.3) designate new developmental releases.
|
||||||
|
Even numbered versions (such as 3.4) designate stable releases.
|
||||||
|
|
||||||
|
Mesa 4.0.2 only contains bug fixes and a new DOS driver since version 4.0.1.
|
||||||
|
|
||||||
|
See the docs/VERSIONS file for the list of bug fixes.
|
||||||
|
|
||||||
|
|
||||||
|
Device Drivers
|
||||||
|
--------------
|
||||||
|
|
||||||
|
Mesa advertises itself as either OpenGL 1.2 or OpenGL 1.3 depending on the
|
||||||
|
device driver. If the driver enables all the ARB extensions which are part
|
||||||
|
of OpenGL 1.3 then glGetString(GL_VERSION) will return "1.3". Otherwise,
|
||||||
|
it'll return "1.2".
|
||||||
|
|
||||||
|
A number of Mesa's software drivers haven't been actively maintained for
|
||||||
|
some time. We rely on volunteers to maintain many of the drivers.
|
||||||
|
Here's the current status of all included drivers:
|
||||||
|
|
||||||
|
Driver Status
|
||||||
|
---------------------- ---------------------
|
||||||
|
XMesa (Xlib) implements OpenGL 1.3
|
||||||
|
OSMesa (off-screen) implements OpenGL 1.3
|
||||||
|
FX (3dfx Voodoo1/2) implements OpenGL 1.3
|
||||||
|
SVGA implements OpenGL 1.3
|
||||||
|
Wind River UGL implements OpenGL 1.3
|
||||||
|
Windows/Win32 implements OpenGL 1.3
|
||||||
|
DOS/DJGPP implements OpenGL 1.3 (new in Mesa 4.0.2)
|
||||||
|
GGI needs updating
|
||||||
|
BeOS needs updating
|
||||||
|
Allegro needs updating
|
||||||
|
D3D needs updating
|
||||||
|
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
|
||||||
|
Mesa 4.0.3 release notes
|
||||||
|
|
||||||
|
June 25, 2002
|
||||||
|
|
||||||
|
PLEASE READ!!!!
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Introduction
|
||||||
|
------------
|
||||||
|
|
||||||
|
Mesa uses an even/odd version number scheme like the Linux kernel.
|
||||||
|
Odd numbered versions (such as 3.3) designate new developmental releases.
|
||||||
|
Even numbered versions (such as 3.4) designate stable releases.
|
||||||
|
|
||||||
|
Mesa 4.0.3 basically just contains bug fixes version 4.0.2.
|
||||||
|
|
||||||
|
See the docs/VERSIONS file for the list of bug fixes.
|
||||||
|
|
||||||
|
The GGI driver has been updated, thanks to Filip Spacek.
|
||||||
|
|
||||||
|
|
||||||
|
Device Drivers
|
||||||
|
--------------
|
||||||
|
|
||||||
|
Mesa advertises itself as either OpenGL 1.2 or OpenGL 1.3 depending on the
|
||||||
|
device driver. If the driver enables all the ARB extensions which are part
|
||||||
|
of OpenGL 1.3 then glGetString(GL_VERSION) will return "1.3". Otherwise,
|
||||||
|
it'll return "1.2".
|
||||||
|
|
||||||
|
A number of Mesa's software drivers haven't been actively maintained for
|
||||||
|
some time. We rely on volunteers to maintain many of the drivers.
|
||||||
|
Here's the current status of all included drivers:
|
||||||
|
|
||||||
|
Driver Status
|
||||||
|
---------------------- ---------------------
|
||||||
|
XMesa (Xlib) implements OpenGL 1.3
|
||||||
|
OSMesa (off-screen) implements OpenGL 1.3
|
||||||
|
FX (3dfx Voodoo1/2) implements OpenGL 1.3
|
||||||
|
SVGA implements OpenGL 1.3
|
||||||
|
Wind River UGL implements OpenGL 1.3
|
||||||
|
Windows/Win32 implements OpenGL 1.3
|
||||||
|
DOS/DJGPP implements OpenGL 1.3 (new in Mesa 4.0.2)
|
||||||
|
GGI implements OpenGL 1.3
|
||||||
|
BeOS needs updating
|
||||||
|
Allegro needs updating
|
||||||
|
D3D needs updating
|
||||||
|
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
@@ -0,0 +1,307 @@
|
|||||||
|
|
||||||
|
Mesa 4.1 release notes
|
||||||
|
|
||||||
|
October 29, 2002
|
||||||
|
|
||||||
|
PLEASE READ!!!!
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Introduction
|
||||||
|
------------
|
||||||
|
|
||||||
|
Mesa uses an even/odd version number scheme like the Linux kernel.
|
||||||
|
Even numbered versions (such as 4.0) designate stable releases.
|
||||||
|
Odd numbered versions (such as 4.1) designate new developmental releases.
|
||||||
|
|
||||||
|
|
||||||
|
New Features in Mesa 4.1
|
||||||
|
------------------------
|
||||||
|
|
||||||
|
New extensions. Docs at http://oss.sgi.com/projects/ogl-sample/registry/
|
||||||
|
|
||||||
|
GL_NV_vertex_program
|
||||||
|
|
||||||
|
NVIDIA's vertex programming extension
|
||||||
|
|
||||||
|
GL_NV_vertex_program1_1
|
||||||
|
|
||||||
|
A few features built on top of GL_NV_vertex_program
|
||||||
|
|
||||||
|
GL_ARB_window_pos
|
||||||
|
|
||||||
|
This is the ARB-approved version of GL_MESA_window_pos
|
||||||
|
|
||||||
|
GL_ARB_depth_texture
|
||||||
|
|
||||||
|
This is the ARB-approved version of GL_SGIX_depth_texture.
|
||||||
|
It allows depth (Z buffer) data to be stored in textures.
|
||||||
|
This is used by GL_ARB_shadow
|
||||||
|
|
||||||
|
GL_ARB_shadow
|
||||||
|
|
||||||
|
Shadow mapping with depth textures.
|
||||||
|
This is the ARB-approved version of GL_SGIX_shadow.
|
||||||
|
|
||||||
|
GL_ARB_shadow_ambient
|
||||||
|
|
||||||
|
Allows one to specify the luminance of shadowed pixels.
|
||||||
|
This is the ARB-approved version of GL_SGIX_shadow_ambient.
|
||||||
|
|
||||||
|
GL_EXT_shadow_funcs
|
||||||
|
|
||||||
|
Extends the set of GL_ARB_shadow texture comparision functions to
|
||||||
|
include all eight of standard OpenGL dept-test functions.
|
||||||
|
|
||||||
|
GL_ARB_point_parameters
|
||||||
|
|
||||||
|
This is basically the same as GL_EXT_point_parameters.
|
||||||
|
|
||||||
|
GL_ARB_texture_env_crossbar
|
||||||
|
|
||||||
|
Allows any texture combine stage to reference any texture source unit.
|
||||||
|
|
||||||
|
GL_NV_point_sprite
|
||||||
|
|
||||||
|
For rendering points as textured quads. Useful for particle effects.
|
||||||
|
|
||||||
|
GL_NV_texture_rectangle (new in 4.0.4 actually)
|
||||||
|
|
||||||
|
Allows one to use textures with sizes that are not powers of two.
|
||||||
|
Note that mipmapping and several texture wrap modes are not allowed.
|
||||||
|
|
||||||
|
GL_EXT_multi_draw_arrays
|
||||||
|
|
||||||
|
Allows arrays of vertex arrays to be rendered with one call.
|
||||||
|
|
||||||
|
GL_EXT_stencil_two_side
|
||||||
|
|
||||||
|
Separate stencil modes for front and back-facing polygons.
|
||||||
|
|
||||||
|
GLX_SGIX_fbconfig & GLX_SGIX_pbuffer
|
||||||
|
|
||||||
|
Off-screen rendering support.
|
||||||
|
|
||||||
|
GL_ATI_texture_mirror_once
|
||||||
|
|
||||||
|
Adds two new texture wrap modes: GL_MIRROR_CLAMP_ATI and
|
||||||
|
GL_MIRROR_CLAMP_TO_EDGE_ATI.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Device Driver Status
|
||||||
|
--------------------
|
||||||
|
|
||||||
|
A number of Mesa's software drivers haven't been actively maintained for
|
||||||
|
some time. We rely on volunteers to maintain many of these drivers.
|
||||||
|
Here's the current status of all included drivers:
|
||||||
|
|
||||||
|
|
||||||
|
Driver Status
|
||||||
|
---------------------- ---------------------
|
||||||
|
XMesa (Xlib) implements OpenGL 1.3
|
||||||
|
OSMesa (off-screen) implements OpenGL 1.3
|
||||||
|
FX (3dfx Voodoo1/2) implements OpenGL 1.3
|
||||||
|
SVGA implements OpenGL 1.3
|
||||||
|
Wind River UGL implements OpenGL 1.3
|
||||||
|
Windows/Win32 implements OpenGL 1.3
|
||||||
|
DOS/DJGPP implements OpenGL 1.3
|
||||||
|
GGI implements OpenGL 1.3
|
||||||
|
BeOS needs updating (underway)
|
||||||
|
Allegro needs updating
|
||||||
|
D3D needs updating
|
||||||
|
DOS needs updating
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
New features in GLUT
|
||||||
|
--------------------
|
||||||
|
|
||||||
|
1. Frames per second printing
|
||||||
|
|
||||||
|
GLUT now looks for an environment variable called "GLUT_FPS". If it's
|
||||||
|
set, GLUT will print out a frames/second statistic to stderr when
|
||||||
|
glutSwapBuffers() is called. By default, frames/second is computed
|
||||||
|
and displayed once every 5 seconds. You can specify a different
|
||||||
|
interval (in milliseconds) when you set the env var. For example
|
||||||
|
'export GLUT_FPS=1000' or 'setenv GLUT_FPS 1000' will set the interval
|
||||||
|
to one second.
|
||||||
|
|
||||||
|
NOTE: the demo or application must call the glutInit() function for
|
||||||
|
this to work. Otherwise, the env var will be ignored.
|
||||||
|
|
||||||
|
Finally, this feature may not be reliable in multi-window programs.
|
||||||
|
|
||||||
|
|
||||||
|
2. glutGetProcAddress() function
|
||||||
|
|
||||||
|
The new function:
|
||||||
|
|
||||||
|
void *glutGetProcAddress(const char *procName)
|
||||||
|
|
||||||
|
is a wrapper for glXGetProcAddressARB() and wglGetProcAddress(). It
|
||||||
|
lets you dynamically get the address of an OpenGL function at runtime.
|
||||||
|
The GLUT_API_VERSION has been bumped to 5, but I haven't bumped the
|
||||||
|
GLUT version number from 3.7 since that's probably Mark Kilgard's role.
|
||||||
|
|
||||||
|
This function should probably also be able to return the address of
|
||||||
|
GLUT functions themselves, but it doesn't do that yet.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
XXX Things To Do Yet XXXX
|
||||||
|
-------------------------
|
||||||
|
|
||||||
|
isosurf with vertex program exhibits some missing triangles (probably
|
||||||
|
when recycling the vertex buffer for long prims).
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Porting Info
|
||||||
|
------------
|
||||||
|
|
||||||
|
If you're porting a DRI or other driver from Mesa 4.0.x to Mesa 4.1 here
|
||||||
|
are some things to change:
|
||||||
|
|
||||||
|
1. ctx->Texture._ReallyEnabled is obsolete.
|
||||||
|
|
||||||
|
Since there are now 5 texture targets (1D, 2D, 3D, cube and rect) that
|
||||||
|
left room for only 6 units (6*5 < 32) in this field.
|
||||||
|
This field is being replaced by ctx->Texture._EnabledUnits which has one
|
||||||
|
bit per texture unit. If the bit k of _EnabledUnits is set, that means
|
||||||
|
ctx->Texture.Unit[k]._ReallyEnabled is non-zero. You'll have to look at
|
||||||
|
ctx->Texture.Unit[k]._ReallyEnabled to learn if the 1D, 2D, 3D, cube or
|
||||||
|
rect texture is enabled for unit k.
|
||||||
|
|
||||||
|
This also means that the constants TEXTURE1_*, TEXTURE2_*, etc are
|
||||||
|
obsolete.
|
||||||
|
|
||||||
|
The tokens TEXTURE0_* have been replaced as well (since there's no
|
||||||
|
significance to the "0" part:
|
||||||
|
|
||||||
|
old token new token
|
||||||
|
TEXTURE0_1D TEXTURE_1D_BIT
|
||||||
|
TEXTURE0_2D TEXTURE_2D_BIT
|
||||||
|
TEXTURE0_3D TEXTURE_3D_BIT
|
||||||
|
TEXTURE0_CUBE TEXTURE_CUBE_BIT
|
||||||
|
<none> TEXTURE_RECT_BIT
|
||||||
|
|
||||||
|
These tokens are only used for the ctx->Texture.Unit[i].Enabled and
|
||||||
|
ctx->Texture.Unit[i]._ReallyEnabled fields. Exactly 0 or 1 bits will
|
||||||
|
be set in _ReallyEnabled at any time!
|
||||||
|
|
||||||
|
Q: "What's the purpose of Unit[i].Enabled vs Unit[i]._ReallyEnabled?"
|
||||||
|
A: The user can enable GL_TEXTURE_1D, GL_TEXTURE_2D, etc for any
|
||||||
|
texure unit all at once (an unusual thing to do).
|
||||||
|
OpenGL defines priorities that basically say GL_TEXTURE_2D has
|
||||||
|
higher priority than GL_TEXTURE_1D, etc. Also, just because a
|
||||||
|
texture target is enabled by the user doesn't mean we'll actually
|
||||||
|
use that texture! If a texture object is incomplete (missing mip-
|
||||||
|
map levels, etc) it's as if texturing is disabled for that target.
|
||||||
|
The _ReallyEnabled field will have a bit set ONLY if the texture
|
||||||
|
target is enabled and complete. This spares the driver writer from
|
||||||
|
examining a _lot_ of GL state to determine which texture target is
|
||||||
|
to be used.
|
||||||
|
|
||||||
|
|
||||||
|
2. Tnl tokens changes
|
||||||
|
|
||||||
|
During the implementation of GL_NV_vertex_program some of the vertex
|
||||||
|
buffer code was changed. Specifically, the VERT_* bits defined in
|
||||||
|
tnl/t_context.h have been renamed to better match the conventions of
|
||||||
|
GL_NV_vertex_program. The old names are still present but obsolete.
|
||||||
|
Drivers should use the newer names.
|
||||||
|
|
||||||
|
For example: VERT_RGBA is now VERT_BIT_COLOR0 and
|
||||||
|
VERT_SPEC_RGB is now VERT_BIT_COLOR1.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
3. Read/Draw Buffer changes
|
||||||
|
|
||||||
|
The business of setting the current read/draw buffers in Mesa 4.0.x
|
||||||
|
was complicated. It's much simpler now in Mesa 4.1.
|
||||||
|
|
||||||
|
Here are the changes:
|
||||||
|
|
||||||
|
- Renamed ctx->Color.DrawDestMask to ctx->Color._DrawDestMask
|
||||||
|
- Removed ctx->Color.DriverDrawBuffer
|
||||||
|
- Removed ctx->Pixel.DriverReadBuffer
|
||||||
|
- Removed ctx->Color.MultiDrawBuffer
|
||||||
|
- Removed ctx->Driver.SetDrawBuffer()
|
||||||
|
- Removed swrast->Driver.SetReadBuffer().
|
||||||
|
- Added ctx->Color._DrawDestMask - a bitmask of FRONT/BACK_LEFT/RIGHT_BIT
|
||||||
|
values to indicate the current draw buffers.
|
||||||
|
- Added ctx->Pixel._ReadSrcMask to indicate the source for pixel reading.
|
||||||
|
The value is _one_ of the FRONT/BACK_LEFT/RIGHT_BIT values.
|
||||||
|
- Added ctx->Driver.DrawBuffer() and ctx->Driver.ReadBuffer().
|
||||||
|
These functions exactly correspond to glDrawBuffer and glReadBuffer calls.
|
||||||
|
Many drivers will set ctx->Driver.DrawBuffer = _swrast_DrawBuffer and
|
||||||
|
leave ctx->Draw.ReadBuffer NULL.
|
||||||
|
DRI drivers should implement their own function for ctx->Driver.DrawBuffer
|
||||||
|
and use it to set the current hardware drawing buffer. You'll probably
|
||||||
|
also want to check for GL_FRONT_AND_BACK mode and fall back to software.
|
||||||
|
Call _swrast_DrawBuffer() too, to update the swrast state.
|
||||||
|
- Added swrast->Driver.SetBuffer().
|
||||||
|
This function should be implemented by all device drivers that use swrast.
|
||||||
|
Mesa will call it to specify the buffer to use for span reading AND
|
||||||
|
writing and point/line/triangle rendering.
|
||||||
|
There should be no confusion between current read or draw buffer anymore.
|
||||||
|
- Added swrast->CurrentBuffer to indicate which color buffer to read/draw.
|
||||||
|
Will be FRONT_LEFT_BIT, BACK_LEFT_BIT, FRONT_RIGHT_BIT or BACK_RIGHT_BIT.
|
||||||
|
This value is usually passed to swrast->Driver.SetBuffer().
|
||||||
|
|
||||||
|
|
||||||
|
4. _mesa_create_context() changes. This function now takes a pointer to
|
||||||
|
a __GLimports object. The __GLimports structure contains function
|
||||||
|
pointers to system functions like fprintf(), malloc(), etc.
|
||||||
|
The _mesa_init_default_imports() function can be used to initialize
|
||||||
|
a __GLimports object. Most device drivers (like the DRI drivers)
|
||||||
|
should use this.
|
||||||
|
|
||||||
|
|
||||||
|
5. In tnl's struct vertex_buffer, the field "ProjectedClipCoords"
|
||||||
|
has been replaced by "NdcPtr" to better match the OpenGL spec's
|
||||||
|
terminology.
|
||||||
|
|
||||||
|
|
||||||
|
6. Since GL_EXT_stencil_two_side has been implemented, many of the
|
||||||
|
ctx->Stencil fields are now 2-element arrays. For example,
|
||||||
|
"GLenum Ref" is now "GLenum Ref[2]" The [0] elements are the front-face
|
||||||
|
values and the [1] elements are the back-face values.
|
||||||
|
ctx->Stencil.ActiveFace is 0 or 1 to indicate the current face for
|
||||||
|
the glStencilOp/Func/Mask() functions.
|
||||||
|
ctx->Stencil.TestTwoSide controls whether or not 1 or 2-sided stenciling
|
||||||
|
is enabled.
|
||||||
|
|
||||||
|
|
||||||
|
7. Removed ctx->Polygon._OffsetAny. Removed ctx->Polygon.OffsetMRD.
|
||||||
|
|
||||||
|
|
||||||
|
8. GLfloat / GLchan changes:
|
||||||
|
|
||||||
|
- Changed ctx->Driver.ClearColor() to take GLfloat[4] instead of GLchan[4].
|
||||||
|
ctx->Color.ClearColor is now GLfloat[4] too.
|
||||||
|
- Changed ctx->Driver.AlphaRef() to take GLfloat instead of GLchan.
|
||||||
|
- ctx->Color.AlphaRef is now GLfloat.
|
||||||
|
- texObj->BorderColor is now GLfloat[4]. texObj->_BorderChan is GLchan[4].
|
||||||
|
|
||||||
|
This is part of an effort to remove all GLchan types from core Mesa so
|
||||||
|
that someday we can support 8, 16 and 32-bit color channels dynamically
|
||||||
|
at runtime, instead of at compile-time.
|
||||||
|
|
||||||
|
|
||||||
|
9. GLboolean ctx->Tranform.ClipEnabled[MAX_CLIP_PLANES] has been replaced
|
||||||
|
by GLuint ctx->Transform.ClipPlanesEnabled. The later is a bitfield.
|
||||||
|
|
||||||
|
|
||||||
|
10. There's a new matrix_stack type in mtypes.h used for the Modelview,
|
||||||
|
Projection, Color and Texcoord matrix stacks.
|
||||||
|
|
||||||
|
|
||||||
|
11. The ctx->Current.* fields have changed a lot. Now, there's a
|
||||||
|
ctx->Current.Attrib[] array for all vertex attributes which matches
|
||||||
|
the NV vertex program conventions.
|
||||||
|
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
@@ -0,0 +1,84 @@
|
|||||||
|
|
||||||
|
Mesa 5.0 release notes
|
||||||
|
|
||||||
|
November 13, 2002
|
||||||
|
|
||||||
|
PLEASE READ!!!!
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Introduction
|
||||||
|
------------
|
||||||
|
|
||||||
|
Mesa uses an even/odd version number scheme like the Linux kernel.
|
||||||
|
Even-numbered versions (such as 5.0) designate stable releases.
|
||||||
|
Odd-numbered versions (such as 4.1) designate new developmental releases.
|
||||||
|
|
||||||
|
Mesa 5.0 is basically just a stabilization of Mesa 4.1. To see a list of
|
||||||
|
bug fixes, etc. see the VERSIONS file.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
New Features in Mesa 5.0
|
||||||
|
------------------------
|
||||||
|
|
||||||
|
Mesa 5.0 supports OpenGL 1.4. Note Mesa's versioning convention:
|
||||||
|
|
||||||
|
OpenGL Version Mesa Version
|
||||||
|
------------------------------
|
||||||
|
1.0 1.x
|
||||||
|
1.1 2.x
|
||||||
|
1.2 3.x
|
||||||
|
1.3 4.x
|
||||||
|
1.4 5.x
|
||||||
|
|
||||||
|
OpenGL 1.4 (and Mesa 5.0) incorporates the following OpenGL extensions as
|
||||||
|
standard features:
|
||||||
|
|
||||||
|
GL_ARB_depth_texture
|
||||||
|
GL_ARB_shadow
|
||||||
|
GL_ARB_texture_env_crossbar
|
||||||
|
GL_ARB_texture_mirror_repeat
|
||||||
|
GL_ARB_window_pos
|
||||||
|
GL_EXT_blend_color
|
||||||
|
GL_EXT_blend_func_separate
|
||||||
|
GL_EXT_blend_logic_op
|
||||||
|
GL_EXT_blend_minmax
|
||||||
|
GL_EXT_blend_subtract
|
||||||
|
GL_EXT_fog_coord
|
||||||
|
GL_EXT_multi_draw_arrays
|
||||||
|
GL_EXT_point_parameters
|
||||||
|
GL_EXT_secondary_color
|
||||||
|
GL_EXT_stencil_wrap
|
||||||
|
GL_SGIS_generate_mipmap
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Device Driver Status
|
||||||
|
--------------------
|
||||||
|
|
||||||
|
A number of Mesa's software drivers haven't been actively maintained for
|
||||||
|
some time. We rely on volunteers to maintain many of these drivers.
|
||||||
|
Here's the current status of all included drivers:
|
||||||
|
|
||||||
|
|
||||||
|
Driver Status
|
||||||
|
---------------------- ---------------------
|
||||||
|
XMesa (Xlib) implements OpenGL 1.4
|
||||||
|
OSMesa (off-screen) implements OpenGL 1.4
|
||||||
|
FX (3dfx Voodoo1/2) implements OpenGL 1.3
|
||||||
|
SVGA implements OpenGL 1.3
|
||||||
|
Wind River UGL implements OpenGL 1.3
|
||||||
|
Windows/Win32 implements OpenGL 1.4
|
||||||
|
DOS/DJGPP implements OpenGL 1.3
|
||||||
|
GGI implements OpenGL 1.3
|
||||||
|
DOS implements OpenGL 1.4
|
||||||
|
BeOS needs updating (underway)
|
||||||
|
Allegro needs updating
|
||||||
|
D3D needs updating
|
||||||
|
|
||||||
|
Note: supporting OpenGL 1.4 (vs. 1.3 or 1.2) usually only requires that the
|
||||||
|
driver call the _mesa_enable_1_4_extensions() function.
|
||||||
|
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
|
||||||
|
Mesa 5.0.1 release notes
|
||||||
|
|
||||||
|
March 30, 2003
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Introduction
|
||||||
|
------------
|
||||||
|
|
||||||
|
Mesa uses an even/odd version number scheme like the Linux kernel.
|
||||||
|
Even-numbered versions (such as 5.0.x) designate stable releases.
|
||||||
|
Odd-numbered versions (such as 4.1.x) designate new developmental releases.
|
||||||
|
|
||||||
|
Mesa 5.0.1 just fixes bugs found since the 5.0 release. See the VERSIONS
|
||||||
|
file for details.
|
||||||
|
|
||||||
|
|
||||||
|
Device Driver Status
|
||||||
|
--------------------
|
||||||
|
|
||||||
|
A number of Mesa's software drivers haven't been actively maintained for
|
||||||
|
some time. We rely on volunteers to maintain many of these drivers.
|
||||||
|
Here's the current status of all included drivers:
|
||||||
|
|
||||||
|
|
||||||
|
Driver Status
|
||||||
|
---------------------- ---------------------
|
||||||
|
XMesa (Xlib) implements OpenGL 1.4
|
||||||
|
OSMesa (off-screen) implements OpenGL 1.4
|
||||||
|
FX (3dfx Voodoo1/2) implements OpenGL 1.3
|
||||||
|
SVGA implements OpenGL 1.3
|
||||||
|
Wind River UGL implements OpenGL 1.3
|
||||||
|
Windows/Win32 implements OpenGL 1.4
|
||||||
|
DJGPP implements OpenGL 1.4
|
||||||
|
GGI implements OpenGL 1.3
|
||||||
|
BeOS implements OpenGL 1.4
|
||||||
|
Allegro needs updating
|
||||||
|
D3D needs updating
|
||||||
|
|
||||||
|
Note: supporting OpenGL 1.4 (vs. 1.3 or 1.2) usually only requires that the
|
||||||
|
driver call the _mesa_enable_1_4_extensions() function.
|
||||||
|
|
||||||
|
|
||||||
|
----------------------------------------------------------------------
|
||||||
Loaded 100 of 12511 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user