mirror of
https://github.com/elwrcl/intel-hard-graphics.git
synced 2026-11-01 11:18:03 +03:00
308 lines
12 KiB
Rust
308 lines
12 KiB
Rust
// Copyright © 2022 Collabora, Ltd.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
use crate::{
|
|
api::{GetDebugFlags, DEBUG},
|
|
ir::*,
|
|
};
|
|
|
|
use std::cmp::max;
|
|
|
|
struct LowerCopySwap {
|
|
slm_start: u32,
|
|
slm_size: u32,
|
|
}
|
|
|
|
impl LowerCopySwap {
|
|
fn new(slm_size: u32) -> Self {
|
|
Self {
|
|
slm_start: slm_size,
|
|
slm_size: slm_size,
|
|
}
|
|
}
|
|
|
|
fn lower_copy(&mut self, b: &mut impl Builder, copy: OpCopy) {
|
|
let dst_reg = copy.dst.as_reg().unwrap();
|
|
assert!(dst_reg.comps() == 1);
|
|
assert!(copy.src.is_unmodified());
|
|
assert!(copy.src.is_uniform() || !dst_reg.is_uniform());
|
|
|
|
match dst_reg.file() {
|
|
RegFile::GPR | RegFile::UGPR => match copy.src.src_ref {
|
|
SrcRef::Zero | SrcRef::Imm32(_) => {
|
|
b.push_op(OpMov {
|
|
dst: copy.dst,
|
|
src: copy.src,
|
|
quad_lanes: 0xf,
|
|
});
|
|
}
|
|
SrcRef::CBuf(_) => match dst_reg.file() {
|
|
RegFile::GPR => {
|
|
if b.sm() >= 100 {
|
|
b.push_op(OpLdc {
|
|
dst: copy.dst,
|
|
cb: copy.src,
|
|
offset: 0.into(),
|
|
mode: LdcMode::Indexed,
|
|
mem_type: MemType::B32,
|
|
});
|
|
} else {
|
|
b.push_op(OpMov {
|
|
dst: copy.dst,
|
|
src: copy.src,
|
|
quad_lanes: 0xf,
|
|
});
|
|
}
|
|
}
|
|
RegFile::UGPR => {
|
|
b.push_op(OpLdc {
|
|
dst: copy.dst,
|
|
cb: copy.src,
|
|
offset: 0.into(),
|
|
mode: LdcMode::Indexed,
|
|
mem_type: MemType::B32,
|
|
});
|
|
}
|
|
_ => panic!("Invalid cbuf destination"),
|
|
},
|
|
SrcRef::True | SrcRef::False => {
|
|
panic!("Cannot copy to GPR");
|
|
}
|
|
SrcRef::Reg(src_reg) => match src_reg.file() {
|
|
RegFile::GPR | RegFile::UGPR => {
|
|
b.push_op(OpMov {
|
|
dst: copy.dst,
|
|
src: copy.src,
|
|
quad_lanes: 0xf,
|
|
});
|
|
}
|
|
RegFile::Bar => {
|
|
b.push_op(OpBMov {
|
|
dst: copy.dst,
|
|
src: copy.src,
|
|
clear: false,
|
|
});
|
|
}
|
|
RegFile::Mem => {
|
|
let access = MemAccess {
|
|
mem_type: MemType::B32,
|
|
space: MemSpace::Local,
|
|
order: MemOrder::Strong(MemScope::CTA),
|
|
eviction_priority: MemEvictionPriority::Normal,
|
|
};
|
|
let addr = self.slm_start + src_reg.base_idx() * 4;
|
|
self.slm_size = max(self.slm_size, addr + 4);
|
|
b.push_op(OpLd {
|
|
dst: copy.dst,
|
|
addr: Src::ZERO,
|
|
uniform_addr: Src::ZERO,
|
|
pred: true.into(),
|
|
offset: addr.try_into().unwrap(),
|
|
stride: OffsetStride::X1,
|
|
access: access,
|
|
});
|
|
}
|
|
_ => panic!("Cannot copy to GPR"),
|
|
},
|
|
SrcRef::SSA(_) => panic!("Should be run after RA"),
|
|
},
|
|
RegFile::Pred | RegFile::UPred => match copy.src.src_ref {
|
|
SrcRef::Zero | SrcRef::Imm32(_) | SrcRef::CBuf(_) => {
|
|
panic!("Cannot copy to Pred");
|
|
}
|
|
SrcRef::True => {
|
|
b.lop2_to(
|
|
copy.dst,
|
|
LogicOp2::PassB,
|
|
Src::new_imm_bool(true),
|
|
Src::new_imm_bool(true),
|
|
);
|
|
}
|
|
SrcRef::False => {
|
|
b.lop2_to(
|
|
copy.dst,
|
|
LogicOp2::PassB,
|
|
Src::new_imm_bool(true),
|
|
Src::new_imm_bool(false),
|
|
);
|
|
}
|
|
SrcRef::Reg(src_reg) => match src_reg.file() {
|
|
RegFile::Pred => {
|
|
b.lop2_to(
|
|
copy.dst,
|
|
LogicOp2::PassB,
|
|
Src::new_imm_bool(true),
|
|
copy.src,
|
|
);
|
|
}
|
|
RegFile::UPred => {
|
|
// PLOP3 supports a UPred in src[2]
|
|
b.push_op(OpPLop3 {
|
|
dsts: [copy.dst, Dst::None],
|
|
srcs: [true.into(), true.into(), copy.src],
|
|
ops: [
|
|
LogicOp3::new_lut(&|_, _, z| z),
|
|
LogicOp3::new_const(false),
|
|
],
|
|
});
|
|
}
|
|
_ => panic!("Cannot copy to Pred"),
|
|
},
|
|
SrcRef::SSA(_) => panic!("Should be run after RA"),
|
|
},
|
|
RegFile::Bar => match copy.src.src_ref {
|
|
SrcRef::Reg(src_reg) => match src_reg.file() {
|
|
RegFile::GPR | RegFile::UGPR => {
|
|
b.push_op(OpBMov {
|
|
dst: copy.dst,
|
|
src: copy.src,
|
|
clear: false,
|
|
});
|
|
}
|
|
_ => panic!("Cannot copy to Bar"),
|
|
},
|
|
_ => panic!("Cannot copy to Bar"),
|
|
},
|
|
RegFile::Mem => match copy.src.src_ref {
|
|
SrcRef::Reg(src_reg) => match src_reg.file() {
|
|
RegFile::GPR => {
|
|
let access = MemAccess {
|
|
mem_type: MemType::B32,
|
|
space: MemSpace::Local,
|
|
order: MemOrder::Strong(MemScope::CTA),
|
|
eviction_priority: MemEvictionPriority::Normal,
|
|
};
|
|
let addr = self.slm_start + dst_reg.base_idx() * 4;
|
|
self.slm_size = max(self.slm_size, addr + 4);
|
|
b.push_op(OpSt {
|
|
addr: Src::ZERO,
|
|
uniform_addr: Src::ZERO,
|
|
data: copy.src,
|
|
offset: addr.try_into().unwrap(),
|
|
stride: OffsetStride::X1,
|
|
access: access,
|
|
});
|
|
}
|
|
_ => panic!("Cannot copy to Mem"),
|
|
},
|
|
_ => panic!("Cannot copy to Mem"),
|
|
},
|
|
_ => panic!("Unhandled register file"),
|
|
}
|
|
}
|
|
|
|
fn lower_r2ur(&mut self, b: &mut impl Builder, r2ur: OpR2UR) {
|
|
assert!(r2ur.src.is_unmodified());
|
|
if r2ur.src.is_uniform() {
|
|
let copy = OpCopy {
|
|
dst: r2ur.dst,
|
|
src: r2ur.src,
|
|
};
|
|
self.lower_copy(b, copy);
|
|
} else {
|
|
let src_file = r2ur.src.src_ref.as_reg().unwrap().file();
|
|
let dst_file = r2ur.dst.as_reg().unwrap().file();
|
|
match src_file {
|
|
RegFile::GPR => {
|
|
assert!(dst_file == RegFile::UGPR);
|
|
b.push_op(r2ur);
|
|
}
|
|
RegFile::Pred => {
|
|
assert!(dst_file == RegFile::UPred);
|
|
// It doesn't matter what channel we take
|
|
b.push_op(OpVote {
|
|
op: VoteOp::Any,
|
|
ballot: Dst::None,
|
|
vote: r2ur.dst,
|
|
pred: r2ur.src,
|
|
});
|
|
}
|
|
_ => panic!("No matching uniform register file"),
|
|
}
|
|
}
|
|
}
|
|
|
|
fn lower_swap(&mut self, b: &mut impl Builder, swap: OpSwap) {
|
|
let x = *swap.dsts[0].as_reg().unwrap();
|
|
let y = *swap.dsts[1].as_reg().unwrap();
|
|
|
|
assert!(x.file() == y.file());
|
|
assert!(x.file() != RegFile::Mem);
|
|
assert!(x.comps() == 1 && y.comps() == 1);
|
|
assert!(swap.srcs[0].is_unmodified());
|
|
assert!(*swap.srcs[0].src_ref.as_reg().unwrap() == y);
|
|
assert!(swap.srcs[1].is_unmodified());
|
|
assert!(*swap.srcs[1].src_ref.as_reg().unwrap() == x);
|
|
|
|
if x == y {
|
|
// Nothing to do
|
|
} else if x.is_predicate() && b.sm() >= 70 {
|
|
b.push_op(OpPLop3 {
|
|
dsts: [x.into(), y.into()],
|
|
srcs: [x.into(), y.into(), Src::new_imm_bool(true)],
|
|
ops: [
|
|
LogicOp3::new_lut(&|_, y, _| y),
|
|
LogicOp3::new_lut(&|x, _, _| x),
|
|
],
|
|
});
|
|
} else {
|
|
b.lop2_to(x.into(), LogicOp2::Xor, x.into(), y.into());
|
|
b.lop2_to(y.into(), LogicOp2::Xor, x.into(), y.into());
|
|
b.lop2_to(x.into(), LogicOp2::Xor, x.into(), y.into());
|
|
}
|
|
}
|
|
|
|
fn run(&mut self, s: &mut Shader) {
|
|
let sm = s.sm;
|
|
s.map_instrs(|instr: Instr, _| -> MappedInstrs {
|
|
match instr.op {
|
|
Op::R2UR(r2ur) => {
|
|
debug_assert!(instr.pred.is_true());
|
|
let mut b = InstrBuilder::new(sm);
|
|
if DEBUG.annotate() {
|
|
b.push_instr(Instr::new(OpAnnotate {
|
|
annotation: "r2ur lowered by lower_copy_swap"
|
|
.into(),
|
|
}));
|
|
}
|
|
self.lower_r2ur(&mut b, *r2ur);
|
|
b.into_mapped_instrs()
|
|
}
|
|
Op::Copy(copy) => {
|
|
debug_assert!(instr.pred.is_true());
|
|
let mut b = InstrBuilder::new(sm);
|
|
if DEBUG.annotate() {
|
|
b.push_instr(Instr::new(OpAnnotate {
|
|
annotation: "copy lowered by lower_copy_swap"
|
|
.into(),
|
|
}));
|
|
}
|
|
self.lower_copy(&mut b, *copy);
|
|
b.into_mapped_instrs()
|
|
}
|
|
Op::Swap(swap) => {
|
|
debug_assert!(instr.pred.is_true());
|
|
let mut b = InstrBuilder::new(sm);
|
|
if DEBUG.annotate() {
|
|
b.push_instr(Instr::new(OpAnnotate {
|
|
annotation: "swap lowered by lower_copy_swap"
|
|
.into(),
|
|
}));
|
|
}
|
|
self.lower_swap(&mut b, *swap);
|
|
b.into_mapped_instrs()
|
|
}
|
|
_ => [instr].into(),
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
impl Shader<'_> {
|
|
pub fn lower_copy_swap(&mut self) {
|
|
let mut pass = LowerCopySwap::new(self.info.slm_size);
|
|
pass.run(self);
|
|
self.info.slm_size = pass.slm_size;
|
|
}
|
|
}
|