mirror of
https://github.com/elwrcl/intel-hard-graphics.git
synced 2026-11-01 11:18:03 +03:00
1391 lines
48 KiB
Rust
1391 lines
48 KiB
Rust
// Copyright © 2025 Red Hat.
|
|
// SPDX-License-Identifier: MIT
|
|
#![allow(non_camel_case_types)]
|
|
|
|
use crate::ir::*;
|
|
|
|
// This contains the register scheduling information provided by NVIDIA. This
|
|
// file is for Turing only.
|
|
|
|
// Coupled instructions are ones with fixed latencies, they need delays but not
|
|
// scoreboards. Decoupled instructions are ones with variable latencies, need
|
|
// scoreboards but not delays. There are also redirected instructions which
|
|
// depending on the SM, can be coupled or decoupled so both delays and
|
|
// scoreboards needs to be provided.
|
|
|
|
#[allow(dead_code)]
|
|
#[derive(Debug)]
|
|
enum RegLatencySM75 {
|
|
CoupledDisp64,
|
|
CoupledDisp,
|
|
CoupledAlu,
|
|
CoupledFMA,
|
|
IMADLo,
|
|
IMADWideAB, // readers only
|
|
IMADWideLower,
|
|
IMADWideUpper,
|
|
RedirectedFP64,
|
|
RedirectedFP16,
|
|
RedirectedHMMA_884_F16(usize),
|
|
RedirectedHMMA_884_F32(usize),
|
|
RedirectedHMMA_1688,
|
|
RedirectedHMMA_16816,
|
|
IMMA(usize),
|
|
Decoupled,
|
|
DecoupledOther, //reads only
|
|
BMov,
|
|
GuardPredicate,
|
|
}
|
|
|
|
pub const fn pred(has_pred: bool, a: u32, b: u32) -> u32 {
|
|
if has_pred {
|
|
a + b
|
|
} else {
|
|
b
|
|
}
|
|
}
|
|
|
|
impl RegLatencySM75 {
|
|
fn op_category(op: &Op, reader: bool, op_reg_idx: usize) -> RegLatencySM75 {
|
|
use RegLatencySM75::*;
|
|
match op {
|
|
// this will need updating if imad grows support for input predicates
|
|
Op::IMad(_) | Op::IMul(_) => IMADLo,
|
|
Op::IMad64(_) => {
|
|
if reader {
|
|
match op_reg_idx {
|
|
0 | 1 => IMADWideAB,
|
|
2 => IMADWideLower, // vs upper C operand - work it out
|
|
_ => {
|
|
panic!("Illegal field in imadwide")
|
|
}
|
|
}
|
|
} else {
|
|
IMADWideUpper // as above this needs more work
|
|
}
|
|
}
|
|
|
|
Op::PopC(_) => Decoupled,
|
|
Op::IAdd3(_) | Op::IAdd3X(_) => CoupledAlu,
|
|
|
|
Op::BMsk(_) => CoupledAlu,
|
|
Op::Sgxt(_) => CoupledAlu,
|
|
Op::Lop3(_) => CoupledAlu,
|
|
Op::Flo(_) => Decoupled,
|
|
Op::ISetP(_) => CoupledAlu,
|
|
Op::IAbs(_) => CoupledAlu,
|
|
Op::Lea(_) => CoupledAlu,
|
|
Op::LeaX(_) => CoupledAlu,
|
|
Op::IMnMx(_) => CoupledAlu,
|
|
Op::I2I(_) => CoupledAlu,
|
|
// I2IP => CoupledAlu
|
|
Op::Shf(_) => CoupledAlu,
|
|
|
|
Op::FFma(_) => CoupledFMA,
|
|
Op::FAdd(_) => CoupledFMA,
|
|
Op::FMul(_) => CoupledFMA,
|
|
Op::FMnMx(_) => CoupledAlu,
|
|
Op::FSwzAdd(_) => CoupledFMA,
|
|
Op::FSet(_) => CoupledAlu,
|
|
// FSel => CoupledAlu,
|
|
Op::FSetP(_) => CoupledAlu,
|
|
// FChk => Decoupled,
|
|
Op::DAdd(_) | Op::DFma(_) | Op::DMul(_) | Op::DSetP(_) => {
|
|
RedirectedFP64
|
|
}
|
|
|
|
Op::DMnMx(_) => RedirectedFP64, // not in docs
|
|
|
|
Op::HAdd2(_)
|
|
| Op::HFma2(_)
|
|
| Op::HMul2(_)
|
|
| Op::HSet2(_)
|
|
| Op::HSetP2(_) => RedirectedFP16,
|
|
|
|
Op::HMnMx2(_) => RedirectedFP16, // not in docs
|
|
// let in for documentation purposes
|
|
Op::Hmma(h) => match (h.mat_size, h.dst_type) {
|
|
(HmmaSize::M16N8K8, _) => RedirectedHMMA_1688,
|
|
(HmmaSize::M16N8K16, _) => RedirectedHMMA_16816,
|
|
_ => panic!("Illegal HMMA in reg category {}", h),
|
|
},
|
|
Op::Ipa(_) => Decoupled,
|
|
Op::MuFu(_) => Decoupled,
|
|
|
|
// Conversion functions all decoupled
|
|
Op::F2F(_) => Decoupled,
|
|
Op::F2FP(_) => CoupledAlu, // undocumented, copy Ampere
|
|
Op::F2I(_) => Decoupled,
|
|
Op::I2F(_) => Decoupled,
|
|
Op::FRnd(_) => Decoupled,
|
|
Op::AL2P(_) => Decoupled,
|
|
|
|
Op::Mov(_) => CoupledAlu,
|
|
Op::Movm(_) => Decoupled,
|
|
Op::Sel(_) => CoupledAlu,
|
|
Op::BRev(_) => Decoupled,
|
|
// P2R => CoupledAlu,
|
|
// R2P => CoupledAlu,
|
|
Op::PLop3(_) => CoupledAlu,
|
|
Op::Prmt(_) => CoupledAlu,
|
|
Op::Nop(_) => CoupledDisp,
|
|
Op::Vote(_) => CoupledDisp,
|
|
Op::Match(_) => Decoupled,
|
|
Op::S2R(_) => Decoupled,
|
|
// S2UR => Decoupled,
|
|
Op::R2UR(_) => {
|
|
if reader {
|
|
Decoupled
|
|
} else {
|
|
panic!("Illegal R2UR");
|
|
}
|
|
}
|
|
Op::CS2R(cs2r) => {
|
|
if cs2r.dst.comps() == 2 {
|
|
CoupledDisp64
|
|
} else {
|
|
CoupledAlu
|
|
}
|
|
}
|
|
// B2R => Decoupled,
|
|
// LEPC => CoupledDisp64
|
|
Op::BMov(bmov) => match bmov.dst {
|
|
Dst::Reg(reg) => {
|
|
if reg.is_gpr() {
|
|
BMov
|
|
} else {
|
|
Decoupled
|
|
}
|
|
}
|
|
_ => Decoupled,
|
|
},
|
|
// RPCMOV.32 => CoupledAlu,
|
|
// RPCMOV.64 => CoupledDisp64
|
|
// PMTRIG => CoupledDisp64
|
|
// CSMTEST => CoupledAlu,
|
|
Op::Bar(_) => Decoupled,
|
|
Op::Imma(_) => IMMA(op_reg_idx),
|
|
Op::IDp4(_) => CoupledFMA,
|
|
Op::BClear(_) => Decoupled,
|
|
Op::Bra(_) => Decoupled,
|
|
Op::BSSy(_) => Decoupled,
|
|
Op::Kill(_) => Decoupled,
|
|
Op::Exit(_) => Decoupled,
|
|
Op::BSync(_) => Decoupled,
|
|
Op::Tex(_) => Decoupled,
|
|
Op::Tld(_) => Decoupled,
|
|
Op::Tld4(_) => Decoupled,
|
|
Op::Tmml(_) => Decoupled,
|
|
Op::Txd(_) => Decoupled,
|
|
Op::Txq(_) => Decoupled,
|
|
Op::Ldc(_) => Decoupled,
|
|
Op::ALd(_) => Decoupled,
|
|
Op::ASt(_) => Decoupled,
|
|
Op::Out(_) => Decoupled,
|
|
Op::OutFinal(_) => Decoupled,
|
|
Op::Ld(_) => Decoupled,
|
|
Op::St(_) => Decoupled,
|
|
Op::Atom(_) => Decoupled,
|
|
//CCtl.i,c are coupled
|
|
Op::CCtl(_) => DecoupledOther,
|
|
Op::MemBar(_) => Decoupled,
|
|
Op::SuLd(_) => Decoupled,
|
|
Op::SuSt(_) => Decoupled,
|
|
Op::SuAtom(_) => Decoupled,
|
|
Op::PixLd(_) => Decoupled,
|
|
Op::Isberd(_) => Decoupled,
|
|
Op::Isbewr(_) => Decoupled,
|
|
Op::LdTram(_) => Decoupled,
|
|
Op::Shfl(_) => Decoupled,
|
|
Op::Ldsm(_) => Decoupled,
|
|
x => {
|
|
panic!("Illegal instuction in reg category {}", x);
|
|
}
|
|
}
|
|
}
|
|
|
|
pub fn read_after_write(
|
|
writer: RegLatencySM75,
|
|
reader: RegLatencySM75,
|
|
) -> u32 {
|
|
use RegLatencySM75::*;
|
|
match writer {
|
|
IMADWideAB | DecoupledOther => {
|
|
panic!("Illegal IMADWideAB for writer");
|
|
}
|
|
_ => {}
|
|
}
|
|
|
|
match reader {
|
|
CoupledDisp64 | CoupledDisp | CoupledAlu => match writer {
|
|
CoupledDisp64 => 6,
|
|
CoupledAlu | CoupledDisp => 4,
|
|
CoupledFMA | IMADLo => 5,
|
|
IMADWideLower => 3,
|
|
IMADWideUpper => 5,
|
|
RedirectedFP64 => 9,
|
|
RedirectedFP16 => 8,
|
|
RedirectedHMMA_884_F16(_) => 13,
|
|
RedirectedHMMA_884_F32(_) => 10,
|
|
RedirectedHMMA_1688 => 14,
|
|
RedirectedHMMA_16816 => 22,
|
|
IMMA(_) => 10,
|
|
_ => 1,
|
|
},
|
|
CoupledFMA | IMADLo => match writer {
|
|
CoupledDisp64 => 6,
|
|
CoupledAlu | CoupledDisp => 5,
|
|
CoupledFMA | IMADLo => 4,
|
|
IMADWideLower => 2,
|
|
IMADWideUpper => 4,
|
|
RedirectedFP64 => 9,
|
|
RedirectedFP16 => 8,
|
|
RedirectedHMMA_884_F16(_) => 13,
|
|
RedirectedHMMA_884_F32(_) => 10,
|
|
RedirectedHMMA_1688 => 14,
|
|
RedirectedHMMA_16816 => 22,
|
|
IMMA(_) => 10,
|
|
_ => 1,
|
|
},
|
|
IMADWideAB => match writer {
|
|
CoupledDisp64 => 6,
|
|
CoupledAlu | CoupledDisp => 5,
|
|
CoupledFMA | IMADLo => 4,
|
|
IMADWideLower => 4,
|
|
IMADWideUpper => 6,
|
|
RedirectedFP64 => 9,
|
|
RedirectedFP16 => 8,
|
|
RedirectedHMMA_884_F16(_) => 13,
|
|
RedirectedHMMA_884_F32(_) => 10,
|
|
RedirectedHMMA_1688 => 14,
|
|
RedirectedHMMA_16816 => 22,
|
|
IMMA(_) => 10,
|
|
_ => 1,
|
|
},
|
|
IMADWideLower | IMADWideUpper => match reader {
|
|
IMADWideLower => match writer {
|
|
CoupledDisp64 => 6,
|
|
CoupledAlu | CoupledDisp => 5,
|
|
CoupledFMA | IMADLo => 4,
|
|
IMADWideLower => 2,
|
|
IMADWideUpper => 2,
|
|
RedirectedFP64 => 9,
|
|
RedirectedFP16 => 8,
|
|
RedirectedHMMA_884_F16(_) => 13,
|
|
RedirectedHMMA_884_F32(_) => 10,
|
|
RedirectedHMMA_1688 => 14,
|
|
RedirectedHMMA_16816 => 22,
|
|
IMMA(_) => 10,
|
|
_ => 1,
|
|
},
|
|
IMADWideUpper => match writer {
|
|
CoupledDisp64 => 4,
|
|
CoupledAlu | CoupledDisp => 3,
|
|
CoupledFMA | IMADLo => 2,
|
|
IMADWideLower => 2,
|
|
IMADWideUpper => 2,
|
|
RedirectedFP64 => 7,
|
|
RedirectedFP16 => 6,
|
|
RedirectedHMMA_884_F16(_) => 11,
|
|
RedirectedHMMA_884_F32(_) => 8,
|
|
RedirectedHMMA_1688 => 12,
|
|
RedirectedHMMA_16816 => 20,
|
|
IMMA(_) => 8,
|
|
_ => 1,
|
|
},
|
|
_ => {
|
|
panic!("Illegal IMAD field");
|
|
}
|
|
},
|
|
RedirectedFP64 => match writer {
|
|
CoupledDisp64 => 6,
|
|
CoupledAlu | CoupledDisp => 6,
|
|
CoupledFMA | IMADLo => 6,
|
|
IMADWideLower => 6,
|
|
IMADWideUpper => 6,
|
|
RedirectedFP64 => 8,
|
|
RedirectedFP16 => 8,
|
|
RedirectedHMMA_884_F16(_) => 13,
|
|
RedirectedHMMA_884_F32(_) => 10,
|
|
RedirectedHMMA_1688 => 14,
|
|
RedirectedHMMA_16816 => 22,
|
|
IMMA(_) => 10,
|
|
_ => 1,
|
|
},
|
|
RedirectedFP16 => match writer {
|
|
CoupledDisp64 => 6,
|
|
CoupledAlu | CoupledDisp => 6,
|
|
CoupledFMA | IMADLo => 6,
|
|
IMADWideLower => 6,
|
|
IMADWideUpper => 6,
|
|
RedirectedFP64 => 9,
|
|
RedirectedFP16 => 6,
|
|
RedirectedHMMA_884_F16(_) => 13,
|
|
RedirectedHMMA_884_F32(_) => 10,
|
|
RedirectedHMMA_1688 => 14,
|
|
RedirectedHMMA_16816 => 22,
|
|
IMMA(_) => 10,
|
|
_ => 1,
|
|
},
|
|
RedirectedHMMA_884_F16(read_idx) => match writer {
|
|
CoupledDisp64 => 6,
|
|
CoupledAlu | CoupledDisp => 6,
|
|
CoupledFMA | IMADLo => 6,
|
|
IMADWideLower => 6,
|
|
IMADWideUpper => 6,
|
|
RedirectedFP64 => 9,
|
|
RedirectedFP16 => 8,
|
|
RedirectedHMMA_884_F16(_) if read_idx == 2 => 4,
|
|
RedirectedHMMA_884_F16(_) => 13,
|
|
RedirectedHMMA_884_F32(_) => 10,
|
|
RedirectedHMMA_1688 => 14,
|
|
RedirectedHMMA_16816 => 22,
|
|
IMMA(_) => 10,
|
|
_ => 1,
|
|
},
|
|
RedirectedHMMA_884_F32(read_idx) => match writer {
|
|
CoupledDisp64 => 6,
|
|
CoupledAlu | CoupledDisp => 6,
|
|
CoupledFMA | IMADLo => 6,
|
|
IMADWideLower => 6,
|
|
IMADWideUpper => 6,
|
|
RedirectedFP64 => 9,
|
|
RedirectedFP16 => 8,
|
|
RedirectedHMMA_884_F16(_) => 13,
|
|
RedirectedHMMA_884_F32(_) if read_idx == 2 => 4,
|
|
RedirectedHMMA_884_F32(_) => 10,
|
|
RedirectedHMMA_1688 => 14,
|
|
RedirectedHMMA_16816 => 22,
|
|
IMMA(_) => 10,
|
|
_ => 1,
|
|
},
|
|
RedirectedHMMA_1688 | RedirectedHMMA_16816 | Decoupled => {
|
|
match writer {
|
|
CoupledDisp64 => 6,
|
|
CoupledAlu | CoupledDisp => 6,
|
|
CoupledFMA | IMADLo => 6,
|
|
IMADWideLower => 6,
|
|
IMADWideUpper => 6,
|
|
RedirectedFP64 => 9,
|
|
RedirectedFP16 => 8,
|
|
RedirectedHMMA_884_F16(_) => 13,
|
|
RedirectedHMMA_884_F32(_) => 10,
|
|
RedirectedHMMA_1688 => 14,
|
|
RedirectedHMMA_16816 => 22,
|
|
IMMA(_) => 10,
|
|
_ => 1,
|
|
}
|
|
}
|
|
IMMA(read_idx) => match writer {
|
|
CoupledDisp64 => 8,
|
|
CoupledAlu | CoupledDisp => 8,
|
|
CoupledFMA | IMADLo => 8,
|
|
IMADWideLower => 8,
|
|
IMADWideUpper => 8,
|
|
RedirectedFP64 => 9,
|
|
RedirectedFP16 => 8,
|
|
RedirectedHMMA_884_F16(_) => 13,
|
|
RedirectedHMMA_884_F32(_) => 10,
|
|
RedirectedHMMA_1688 => 14,
|
|
RedirectedHMMA_16816 => 22,
|
|
IMMA(_) if read_idx == 2 => 4,
|
|
IMMA(_) => 10,
|
|
_ => 1,
|
|
},
|
|
DecoupledOther => match writer {
|
|
CoupledDisp64 => 8,
|
|
CoupledAlu | CoupledDisp => 8,
|
|
CoupledFMA | IMADLo => 8,
|
|
IMADWideLower => 8,
|
|
IMADWideUpper => 8,
|
|
RedirectedFP64 => 9,
|
|
RedirectedFP16 => 8,
|
|
RedirectedHMMA_884_F16(_) => 13,
|
|
RedirectedHMMA_884_F32(_) => 10,
|
|
RedirectedHMMA_1688 => 14,
|
|
RedirectedHMMA_16816 => 22,
|
|
IMMA(_) => 10,
|
|
_ => 1,
|
|
},
|
|
BMov | GuardPredicate => {
|
|
panic!("Not a RAW category")
|
|
}
|
|
}
|
|
}
|
|
|
|
fn write_after_write(
|
|
writer1: RegLatencySM75,
|
|
writer2: RegLatencySM75,
|
|
has_pred: bool,
|
|
) -> u32 {
|
|
use RegLatencySM75::*;
|
|
match writer1 {
|
|
IMADWideAB | DecoupledOther => {
|
|
panic!("Illegal reg latency for writer");
|
|
}
|
|
_ => {}
|
|
}
|
|
match writer2 {
|
|
CoupledDisp64 => match writer1 {
|
|
CoupledDisp64 | CoupledDisp | CoupledAlu | CoupledFMA
|
|
| IMADLo | IMADWideLower | IMADWideUpper => 1,
|
|
RedirectedFP64 => 4,
|
|
RedirectedFP16 => 3,
|
|
RedirectedHMMA_884_F16(_) => 8,
|
|
RedirectedHMMA_884_F32(_) => pred(has_pred, 2, 2),
|
|
RedirectedHMMA_1688 => 9,
|
|
RedirectedHMMA_16816 => 17,
|
|
IMMA(_) => 5,
|
|
_ => 1,
|
|
},
|
|
CoupledDisp | CoupledAlu => match writer1 {
|
|
CoupledDisp64 => 2,
|
|
CoupledDisp | CoupledAlu | CoupledFMA | IMADLo
|
|
| IMADWideLower | IMADWideUpper => 1,
|
|
RedirectedFP64 => pred(has_pred, 4, 1),
|
|
RedirectedFP16 => pred(has_pred, 3, 1),
|
|
RedirectedHMMA_884_F16(_) => pred(has_pred, 8, 1),
|
|
RedirectedHMMA_884_F32(_) => pred(has_pred, 5, 1),
|
|
RedirectedHMMA_1688 => pred(has_pred, 9, 1),
|
|
RedirectedHMMA_16816 => pred(has_pred, 17, 1),
|
|
IMMA(_) => pred(has_pred, 5, 1),
|
|
_ => 1,
|
|
},
|
|
CoupledFMA | IMADLo => match writer1 {
|
|
CoupledDisp64 => 2,
|
|
CoupledDisp | CoupledAlu | CoupledFMA | IMADLo
|
|
| IMADWideLower => 1,
|
|
IMADWideUpper => pred(has_pred, 1, 1),
|
|
RedirectedFP64 => pred(has_pred, 4, 1),
|
|
RedirectedFP16 => pred(has_pred, 3, 1),
|
|
RedirectedHMMA_884_F16(_) => pred(has_pred, 8, 1),
|
|
RedirectedHMMA_884_F32(_) => pred(has_pred, 5, 1),
|
|
RedirectedHMMA_1688 => pred(has_pred, 9, 1),
|
|
RedirectedHMMA_16816 => pred(has_pred, 17, 1),
|
|
IMMA(_) => pred(has_pred, 5, 1),
|
|
_ => 1,
|
|
},
|
|
IMADWideLower => match writer1 {
|
|
CoupledDisp64 => pred(has_pred, 2, 2),
|
|
CoupledDisp | CoupledAlu => pred(has_pred, 2, 1),
|
|
CoupledFMA | IMADLo => pred(has_pred, 1, 1),
|
|
IMADWideLower => 1,
|
|
IMADWideUpper => 1,
|
|
RedirectedFP64 => pred(has_pred, 4, 3),
|
|
RedirectedFP16 => pred(has_pred, 3, 3),
|
|
RedirectedHMMA_884_F16(_) => pred(has_pred, 8, 3),
|
|
RedirectedHMMA_884_F32(_) => pred(has_pred, 5, 3),
|
|
RedirectedHMMA_1688 => pred(has_pred, 9, 3),
|
|
RedirectedHMMA_16816 => pred(has_pred, 17, 3),
|
|
IMMA(_) => pred(has_pred, 5, 3),
|
|
_ => 1,
|
|
},
|
|
IMADWideUpper => match writer1 {
|
|
CoupledDisp64 => pred(has_pred, 1, 1),
|
|
CoupledDisp | CoupledAlu | CoupledFMA | IMADLo
|
|
| IMADWideLower | IMADWideUpper => 1,
|
|
RedirectedFP64 => pred(has_pred, 4, 1),
|
|
RedirectedFP16 => pred(has_pred, 3, 1),
|
|
RedirectedHMMA_884_F16(_) => pred(has_pred, 8, 1),
|
|
RedirectedHMMA_884_F32(_) => pred(has_pred, 5, 1),
|
|
RedirectedHMMA_1688 => pred(has_pred, 9, 1),
|
|
RedirectedHMMA_16816 => pred(has_pred, 17, 1),
|
|
IMMA(_) => pred(has_pred, 5, 1),
|
|
_ => 1,
|
|
},
|
|
RedirectedFP64 => match writer1 {
|
|
CoupledDisp64 | CoupledDisp | CoupledAlu | CoupledFMA
|
|
| IMADLo | IMADWideLower | IMADWideUpper => 2,
|
|
RedirectedFP64 => 1,
|
|
RedirectedFP16 => 2,
|
|
RedirectedHMMA_884_F16(_) => 5,
|
|
RedirectedHMMA_884_F32(_) => 2,
|
|
RedirectedHMMA_1688 => 6,
|
|
RedirectedHMMA_16816 => 14,
|
|
IMMA(_) => 2,
|
|
_ => 1,
|
|
},
|
|
RedirectedFP16 => match writer1 {
|
|
CoupledDisp64 | CoupledDisp | CoupledAlu | CoupledFMA
|
|
| IMADLo | IMADWideLower | IMADWideUpper => 2,
|
|
RedirectedFP64 => pred(has_pred, 1, 1),
|
|
RedirectedFP16 => 1,
|
|
RedirectedHMMA_884_F16(_) => pred(has_pred, 6, 1),
|
|
RedirectedHMMA_884_F32(_) => pred(has_pred, 3, 1),
|
|
RedirectedHMMA_1688 => pred(has_pred, 7, 1),
|
|
RedirectedHMMA_16816 => pred(has_pred, 15, 1),
|
|
IMMA(_) => pred(has_pred, 3, 1),
|
|
_ => 1,
|
|
},
|
|
RedirectedHMMA_884_F16(_) => match writer1 {
|
|
CoupledDisp64 | CoupledDisp | CoupledAlu | CoupledFMA
|
|
| IMADLo | IMADWideLower | IMADWideUpper => 2,
|
|
RedirectedFP64 => pred(has_pred, 3, 2),
|
|
RedirectedFP16 => pred(has_pred, 2, 2),
|
|
RedirectedHMMA_884_F16(_) => 1,
|
|
RedirectedHMMA_884_F32(_) => pred(has_pred, 2, 4),
|
|
RedirectedHMMA_1688 => pred(has_pred, 6, 4),
|
|
RedirectedHMMA_16816 => pred(has_pred, 16, 2),
|
|
IMMA(_) => pred(has_pred, 2, 4),
|
|
_ => 1,
|
|
},
|
|
RedirectedHMMA_884_F32(_) => match writer1 {
|
|
CoupledDisp64 | CoupledDisp | CoupledAlu | CoupledFMA
|
|
| IMADLo | IMADWideLower | IMADWideUpper => 2,
|
|
RedirectedFP64 => pred(has_pred, 3, 2),
|
|
RedirectedFP16 => pred(has_pred, 2, 2),
|
|
RedirectedHMMA_884_F16(_) => pred(has_pred, 4, 5),
|
|
RedirectedHMMA_884_F32(_) => 1,
|
|
RedirectedHMMA_1688 => pred(has_pred, 6, 4),
|
|
RedirectedHMMA_16816 => pred(has_pred, 16, 2),
|
|
IMMA(_) => pred(has_pred, 2, 4),
|
|
_ => 1,
|
|
},
|
|
RedirectedHMMA_1688 => match writer1 {
|
|
CoupledDisp64 | CoupledDisp | CoupledAlu | CoupledFMA
|
|
| IMADLo | IMADWideLower | IMADWideUpper | RedirectedFP64
|
|
| RedirectedFP16 => 2,
|
|
RedirectedHMMA_884_F16(_) => 4,
|
|
RedirectedHMMA_884_F32(_) => 2,
|
|
RedirectedHMMA_1688 => 1,
|
|
RedirectedHMMA_16816 => 16,
|
|
IMMA(_) => 2,
|
|
_ => 1,
|
|
},
|
|
RedirectedHMMA_16816 => match writer1 {
|
|
CoupledDisp64 | CoupledDisp | CoupledAlu | CoupledFMA
|
|
| IMADLo | IMADWideLower | IMADWideUpper | RedirectedFP64
|
|
| RedirectedFP16 => 2,
|
|
RedirectedHMMA_884_F16(_) => 4,
|
|
RedirectedHMMA_884_F32(_) => 2,
|
|
RedirectedHMMA_1688 => 6,
|
|
RedirectedHMMA_16816 => 1,
|
|
IMMA(_) => 2,
|
|
_ => 1,
|
|
},
|
|
IMMA(_) => match writer1 {
|
|
CoupledDisp64 | CoupledDisp | CoupledAlu | CoupledFMA
|
|
| IMADLo | IMADWideLower | IMADWideUpper => {
|
|
pred(has_pred, 2, 2)
|
|
}
|
|
RedirectedFP64 => pred(has_pred, 2, 3),
|
|
RedirectedFP16 => pred(has_pred, 2, 2),
|
|
RedirectedHMMA_884_F16(_) => pred(has_pred, 2, 7),
|
|
RedirectedHMMA_884_F32(_) => pred(has_pred, 2, 4),
|
|
RedirectedHMMA_1688 => pred(has_pred, 6, 4),
|
|
RedirectedHMMA_16816 => pred(has_pred, 14, 4),
|
|
IMMA(_) => 1,
|
|
_ => 1,
|
|
},
|
|
Decoupled => match writer1 {
|
|
CoupledDisp64
|
|
| CoupledDisp
|
|
| CoupledAlu
|
|
| CoupledFMA
|
|
| IMADLo
|
|
| IMADWideLower
|
|
| IMADWideUpper
|
|
| RedirectedFP64
|
|
| RedirectedFP16
|
|
| RedirectedHMMA_884_F16(_)
|
|
| RedirectedHMMA_884_F32(_)
|
|
| RedirectedHMMA_1688 => 6,
|
|
RedirectedHMMA_16816 => 14,
|
|
IMMA(_) => 2,
|
|
_ => 1,
|
|
},
|
|
BMov => {
|
|
// BMOV Writing to RF?
|
|
match writer1 {
|
|
CoupledDisp64
|
|
| CoupledDisp
|
|
| CoupledAlu
|
|
| CoupledFMA
|
|
| IMADLo
|
|
| IMADWideLower
|
|
| IMADWideUpper
|
|
| RedirectedFP64
|
|
| RedirectedFP16
|
|
| RedirectedHMMA_884_F16(_)
|
|
| RedirectedHMMA_884_F32(_)
|
|
| RedirectedHMMA_1688 => 9,
|
|
RedirectedHMMA_16816 => 14,
|
|
IMMA(_) => 9,
|
|
_ => 1,
|
|
}
|
|
}
|
|
IMADWideAB | DecoupledOther | GuardPredicate => {
|
|
panic!("Not a WAW category")
|
|
}
|
|
}
|
|
}
|
|
|
|
fn write_after_read(reader: RegLatencySM75, writer: RegLatencySM75) -> u32 {
|
|
use RegLatencySM75::*;
|
|
match writer {
|
|
CoupledDisp64 | CoupledDisp | CoupledAlu | CoupledFMA | IMADLo
|
|
| IMADWideLower | IMADWideUpper => match reader {
|
|
RedirectedHMMA_1688 => 5,
|
|
RedirectedHMMA_16816 => 13,
|
|
_ => 1,
|
|
},
|
|
RedirectedFP64 => match reader {
|
|
RedirectedFP64 => 1,
|
|
RedirectedHMMA_1688 => 6,
|
|
RedirectedHMMA_16816 => 14,
|
|
Decoupled => 1,
|
|
_ => 2,
|
|
},
|
|
RedirectedFP16 => match reader {
|
|
RedirectedFP16 => 1,
|
|
RedirectedHMMA_1688 => 6,
|
|
RedirectedHMMA_16816 => 14,
|
|
Decoupled => 1,
|
|
_ => 2,
|
|
},
|
|
RedirectedHMMA_884_F16(_) => match reader {
|
|
RedirectedHMMA_884_F16(_) => 1,
|
|
RedirectedHMMA_1688 => 6,
|
|
RedirectedHMMA_16816 => 14,
|
|
Decoupled => 1,
|
|
_ => 2,
|
|
},
|
|
RedirectedHMMA_884_F32(_) => match reader {
|
|
RedirectedHMMA_884_F32(_) => 1,
|
|
RedirectedHMMA_1688 => 6,
|
|
RedirectedHMMA_16816 => 14,
|
|
Decoupled => 1,
|
|
_ => 2,
|
|
},
|
|
RedirectedHMMA_1688 => match reader {
|
|
RedirectedHMMA_1688 => 1,
|
|
RedirectedHMMA_16816 => 14,
|
|
Decoupled => 1,
|
|
_ => 2,
|
|
},
|
|
RedirectedHMMA_16816 => match reader {
|
|
RedirectedHMMA_1688 => 6,
|
|
RedirectedHMMA_16816 => 1,
|
|
Decoupled => 1,
|
|
_ => 2,
|
|
},
|
|
IMMA(_) => match reader {
|
|
RedirectedHMMA_1688 => 6,
|
|
RedirectedHMMA_16816 => 14,
|
|
IMMA(_) => 1,
|
|
Decoupled => 1,
|
|
_ => 2,
|
|
},
|
|
Decoupled => match reader {
|
|
RedirectedHMMA_1688 => 2,
|
|
RedirectedHMMA_16816 => 14,
|
|
Decoupled => 1,
|
|
_ => 2,
|
|
},
|
|
BMov => match reader {
|
|
RedirectedHMMA_1688 => 9,
|
|
RedirectedHMMA_16816 => 14,
|
|
Decoupled => 1,
|
|
_ => 9,
|
|
},
|
|
IMADWideAB | DecoupledOther | GuardPredicate => {
|
|
panic!("Illegal in WAR");
|
|
}
|
|
}
|
|
}
|
|
|
|
fn pred_read_after_write(
|
|
writer: RegLatencySM75,
|
|
reader: RegLatencySM75,
|
|
) -> u32 {
|
|
use RegLatencySM75::*;
|
|
match reader {
|
|
CoupledDisp => match writer {
|
|
CoupledDisp | CoupledAlu | CoupledFMA | IMADLo
|
|
| IMADWideUpper | IMADWideLower => 12,
|
|
RedirectedFP64 => 15,
|
|
RedirectedFP16 => 14,
|
|
Decoupled => 1,
|
|
_ => {
|
|
panic!("Illegal RAW in Predicate");
|
|
}
|
|
},
|
|
CoupledAlu => match writer {
|
|
CoupledDisp | CoupledAlu => 4,
|
|
CoupledFMA | IMADLo | IMADWideUpper | IMADWideLower => 5,
|
|
RedirectedFP64 => 9,
|
|
RedirectedFP16 => 8,
|
|
Decoupled => 1,
|
|
_ => {
|
|
panic!("Illegal RAW in Predicate");
|
|
}
|
|
},
|
|
CoupledFMA | IMADLo => match writer {
|
|
CoupledDisp | CoupledAlu => 5,
|
|
CoupledFMA | IMADLo | IMADWideUpper | IMADWideLower => 4,
|
|
RedirectedFP64 => 9,
|
|
RedirectedFP16 => 8,
|
|
Decoupled => 1,
|
|
_ => {
|
|
panic!("Illegal RAW in Predicate");
|
|
}
|
|
},
|
|
IMADWideUpper | IMADWideLower => match writer {
|
|
CoupledDisp | CoupledAlu => 5,
|
|
CoupledFMA | IMADLo => 4,
|
|
IMADWideUpper | IMADWideLower => 2,
|
|
RedirectedFP64 => 9,
|
|
RedirectedFP16 => 8,
|
|
Decoupled => 1,
|
|
_ => {
|
|
panic!("Illegal RAW in Predicate");
|
|
}
|
|
},
|
|
RedirectedFP64 => match writer {
|
|
CoupledDisp | CoupledAlu | CoupledFMA | IMADLo
|
|
| IMADWideUpper | IMADWideLower => 12,
|
|
RedirectedFP64 => 8,
|
|
RedirectedFP16 => 14,
|
|
Decoupled => 1,
|
|
_ => {
|
|
panic!("Illegal RAW in Predicate");
|
|
}
|
|
},
|
|
RedirectedFP16 => match writer {
|
|
CoupledDisp | CoupledAlu | CoupledFMA | IMADLo
|
|
| IMADWideUpper | IMADWideLower => 12,
|
|
RedirectedFP64 => 15,
|
|
RedirectedFP16 => 6,
|
|
Decoupled => 1,
|
|
_ => {
|
|
panic!("Illegal RAW in Predicate");
|
|
}
|
|
},
|
|
Decoupled | GuardPredicate => match writer {
|
|
CoupledDisp | CoupledAlu | CoupledFMA | IMADLo
|
|
| IMADWideUpper | IMADWideLower => 12,
|
|
RedirectedFP64 => 15,
|
|
RedirectedFP16 => 14,
|
|
Decoupled => 1,
|
|
_ => {
|
|
panic!("Illegal RAW in Predicate");
|
|
}
|
|
},
|
|
_ => {
|
|
panic!("Illegal reader in reg predicate");
|
|
}
|
|
}
|
|
}
|
|
|
|
fn pred_write_after_write(
|
|
writer1: RegLatencySM75,
|
|
writer2: RegLatencySM75,
|
|
has_pred: bool,
|
|
) -> u32 {
|
|
use RegLatencySM75::*;
|
|
match writer2 {
|
|
CoupledDisp | CoupledAlu | CoupledFMA | IMADLo => match writer1 {
|
|
CoupledDisp | CoupledAlu | CoupledFMA | IMADLo
|
|
| IMADWideUpper | IMADWideLower => 1,
|
|
RedirectedFP64 => pred(has_pred, 4, 1),
|
|
RedirectedFP16 => pred(has_pred, 3, 1),
|
|
Decoupled => 1,
|
|
_ => {
|
|
panic!("Illegal RAW in Predicate");
|
|
}
|
|
},
|
|
IMADWideUpper | IMADWideLower => match writer1 {
|
|
CoupledDisp | CoupledAlu => pred(has_pred, 1, 2),
|
|
CoupledFMA | IMADLo => pred(has_pred, 1, 1),
|
|
IMADWideUpper | IMADWideLower => 1,
|
|
RedirectedFP64 => pred(has_pred, 4, 3),
|
|
RedirectedFP16 => pred(has_pred, 3, 3),
|
|
Decoupled => 1,
|
|
_ => {
|
|
panic!("Illegal RAW in Predicate");
|
|
}
|
|
},
|
|
RedirectedFP64 => match writer1 {
|
|
CoupledDisp | CoupledAlu | CoupledFMA | IMADLo
|
|
| IMADWideUpper | IMADWideLower => pred(has_pred, 2, 2),
|
|
RedirectedFP64 => 1,
|
|
RedirectedFP16 => pred(has_pred, 2, 4),
|
|
Decoupled => 1,
|
|
_ => {
|
|
panic!("Illegal RAW in Predicate");
|
|
}
|
|
},
|
|
RedirectedFP16 => match writer1 {
|
|
CoupledDisp | CoupledAlu | CoupledFMA | IMADLo
|
|
| IMADWideUpper | IMADWideLower => pred(has_pred, 2, 4),
|
|
RedirectedFP64 => pred(has_pred, 2, 7),
|
|
RedirectedFP16 => 1,
|
|
Decoupled => 1,
|
|
_ => {
|
|
panic!("Illegal RAW in Predicate");
|
|
}
|
|
},
|
|
Decoupled => match writer1 {
|
|
CoupledDisp | CoupledAlu | CoupledFMA | IMADLo
|
|
| IMADWideUpper | IMADWideLower | RedirectedFP64
|
|
| RedirectedFP16 => 2,
|
|
Decoupled => 1,
|
|
_ => {
|
|
panic!("Illegal RAW in Predicate");
|
|
}
|
|
},
|
|
_ => {
|
|
panic!("Illegal WAR category in Predicates");
|
|
}
|
|
}
|
|
}
|
|
|
|
fn pred_write_after_read(
|
|
reader: RegLatencySM75,
|
|
writer: RegLatencySM75,
|
|
) -> u32 {
|
|
use RegLatencySM75::*;
|
|
match writer {
|
|
CoupledDisp | CoupledAlu | CoupledFMA | IMADLo | IMADWideUpper
|
|
| IMADWideLower => 1,
|
|
RedirectedFP64 => match reader {
|
|
CoupledAlu | CoupledFMA | IMADLo | IMADWideUpper
|
|
| IMADWideLower | RedirectedFP16 => 2,
|
|
_ => 1,
|
|
},
|
|
RedirectedFP16 => match reader {
|
|
CoupledAlu | CoupledFMA | IMADLo | IMADWideUpper
|
|
| IMADWideLower | RedirectedFP64 => 2,
|
|
_ => 1,
|
|
},
|
|
Decoupled => match reader {
|
|
CoupledAlu | CoupledFMA | IMADLo | IMADWideUpper
|
|
| IMADWideLower | RedirectedFP16 | RedirectedFP64 => 2,
|
|
_ => 1,
|
|
},
|
|
_ => {
|
|
panic!("Illegal WAR category in Predicates");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug)]
|
|
enum URegLatencySM75 {
|
|
Udp,
|
|
VectorCoupled,
|
|
VectorDecoupled,
|
|
Uldc,
|
|
Umov,
|
|
VectorCoupledBindless,
|
|
VectorDecoupledBindless,
|
|
VoteU,
|
|
GuardPredicate,
|
|
R2UR,
|
|
}
|
|
|
|
impl URegLatencySM75 {
|
|
fn op_category(
|
|
op: &Op,
|
|
reader: bool,
|
|
op_reg_idx: usize,
|
|
) -> URegLatencySM75 {
|
|
use URegLatencySM75::*;
|
|
// is this using a bindless cbuf as a src register.
|
|
// this decides between the category types for readers.
|
|
let bindless =
|
|
reader && op.srcs_as_slice()[op_reg_idx].is_bindless_cbuf();
|
|
|
|
let vcoupled = if bindless {
|
|
VectorCoupledBindless
|
|
} else {
|
|
VectorCoupled
|
|
};
|
|
let vdecoupled = if bindless {
|
|
VectorDecoupledBindless
|
|
} else {
|
|
VectorDecoupled
|
|
};
|
|
|
|
// if this is a reader from a ureg, it could be a U* instruction or a regular instruction.
|
|
let uniform_op = op.is_uniform();
|
|
|
|
let vcoupled = if uniform_op { Udp } else { vcoupled };
|
|
let vdecoupled = if uniform_op { Udp } else { vdecoupled };
|
|
|
|
match op {
|
|
Op::BMsk(_) => vcoupled,
|
|
Op::BRev(_) => vcoupled,
|
|
// uclea?
|
|
Op::Flo(_) => vdecoupled,
|
|
Op::IAdd3(_) | Op::IAdd3X(_) => vcoupled,
|
|
Op::IAbs(_) => vcoupled,
|
|
Op::IDp4(_) => vcoupled,
|
|
Op::IMnMx(_) => vcoupled,
|
|
Op::IMad(_) => vcoupled,
|
|
|
|
Op::IMad64(_) => vcoupled,
|
|
Op::ISetP(_) => vcoupled,
|
|
Op::ALd(_)
|
|
| Op::ASt(_)
|
|
| Op::Ld(_)
|
|
| Op::Ldsm(_)
|
|
| Op::St(_)
|
|
| Op::Atom(_) => vdecoupled,
|
|
Op::SuLd(_) | Op::SuSt(_) | Op::SuAtom(_) => vdecoupled,
|
|
Op::Ldc(_) => {
|
|
if uniform_op {
|
|
Uldc
|
|
} else {
|
|
vdecoupled
|
|
}
|
|
}
|
|
Op::Lea(_) => vcoupled,
|
|
Op::LeaX(_) => vcoupled,
|
|
Op::Lop2(_) | Op::Lop3(_) => vcoupled,
|
|
|
|
Op::MuFu(_) => vdecoupled,
|
|
Op::Mov(_) => {
|
|
if uniform_op {
|
|
Umov
|
|
} else {
|
|
vcoupled
|
|
}
|
|
}
|
|
|
|
// mov32i => URegLatency::Uldc,
|
|
// p2ur => Udp,
|
|
Op::PLop3(_) => vcoupled,
|
|
Op::PopC(_) => vdecoupled,
|
|
Op::Prmt(_) => vcoupled,
|
|
Op::PSetP(_) => vcoupled,
|
|
// UR2UP
|
|
Op::Sel(_) => vcoupled,
|
|
Op::Sgxt(_) => vcoupled,
|
|
Op::Shf(_) => vcoupled,
|
|
Op::Shfl(_) => vdecoupled,
|
|
|
|
Op::I2F(_) => vdecoupled,
|
|
Op::F2I(_) => vdecoupled,
|
|
Op::F2F(_) => vdecoupled,
|
|
Op::R2UR(_) => {
|
|
if !reader {
|
|
R2UR
|
|
} else {
|
|
panic!("Illegal R2UR in ureg");
|
|
}
|
|
}
|
|
Op::S2R(_) => {
|
|
if !reader {
|
|
R2UR
|
|
} else {
|
|
panic!("Illegal S2UR in ureg");
|
|
}
|
|
}
|
|
Op::Vote(_) => VoteU,
|
|
|
|
Op::FRnd(_) => vdecoupled,
|
|
Op::FAdd(_)
|
|
| Op::FMul(_)
|
|
| Op::FFma(_)
|
|
| Op::FSet(_)
|
|
| Op::FSetP(_)
|
|
| Op::FMnMx(_)
|
|
| Op::HAdd2(_)
|
|
| Op::HMul2(_)
|
|
| Op::HSet2(_)
|
|
| Op::HFma2(_)
|
|
| Op::HSetP2(_) => vcoupled,
|
|
Op::DMul(_) | Op::DFma(_) | Op::DAdd(_) | Op::DSetP(_) => {
|
|
vdecoupled
|
|
}
|
|
_ => {
|
|
panic!("Illegal instuction in ureg category {}", op);
|
|
}
|
|
}
|
|
}
|
|
|
|
fn read_after_write(
|
|
writer: URegLatencySM75,
|
|
reader: URegLatencySM75,
|
|
) -> u32 {
|
|
use URegLatencySM75::*;
|
|
match reader {
|
|
Udp => match writer {
|
|
Udp => 4,
|
|
R2UR => 2,
|
|
Uldc | VoteU | Umov => 2,
|
|
_ => {
|
|
panic!("Illegal writer in raw ureg latency {:?}", writer)
|
|
}
|
|
},
|
|
VectorCoupled => match writer {
|
|
Udp => 6,
|
|
R2UR => 2,
|
|
Uldc | VoteU | Umov => 2,
|
|
_ => {
|
|
panic!("Illegal writer in raw ureg latency {:?}", writer)
|
|
}
|
|
},
|
|
VectorDecoupled => match writer {
|
|
Udp => 9,
|
|
R2UR => 2,
|
|
Uldc | VoteU | Umov => 2,
|
|
_ => {
|
|
panic!("Illegal writer in raw ureg latency {:?}", writer)
|
|
}
|
|
},
|
|
Uldc | VectorCoupledBindless | VectorDecoupledBindless => {
|
|
match writer {
|
|
Udp => 12,
|
|
R2UR => 2,
|
|
Uldc | VoteU | Umov => 5,
|
|
_ => {
|
|
panic!(
|
|
"Illegal writer in raw ureg latency {:?}",
|
|
writer
|
|
)
|
|
}
|
|
}
|
|
}
|
|
Umov => match writer {
|
|
Udp => 7,
|
|
R2UR => 2,
|
|
Uldc | VoteU | Umov => 2,
|
|
_ => {
|
|
panic!("Illegal writer in raw ureg latency")
|
|
}
|
|
},
|
|
_ => {
|
|
panic!("Illegal read in ureg raw latency")
|
|
}
|
|
}
|
|
}
|
|
|
|
fn write_after_write(
|
|
writer1: URegLatencySM75,
|
|
writer2: URegLatencySM75,
|
|
has_pred: bool,
|
|
) -> u32 {
|
|
use URegLatencySM75::*;
|
|
match writer2 {
|
|
Udp => match writer1 {
|
|
Udp => 1,
|
|
R2UR => 2,
|
|
Uldc | VoteU | Umov => 1,
|
|
_ => {
|
|
panic!("Illegal writer in ureg waw latency")
|
|
}
|
|
},
|
|
R2UR => match writer1 {
|
|
Udp => pred(has_pred, 4, 6),
|
|
R2UR => 2,
|
|
Uldc | VoteU | Umov => 4,
|
|
_ => {
|
|
panic!("Illegal writer in ureg waw latency")
|
|
}
|
|
},
|
|
Uldc | VoteU | Umov => match writer1 {
|
|
Udp => 7,
|
|
R2UR => 2,
|
|
Uldc | VoteU | Umov => 1,
|
|
_ => {
|
|
panic!("Illegal writer in ureg waw latency")
|
|
}
|
|
},
|
|
_ => {
|
|
panic!("Illegal writer in ureg waw latency")
|
|
}
|
|
}
|
|
}
|
|
|
|
fn write_after_read(
|
|
reader: URegLatencySM75,
|
|
writer: URegLatencySM75,
|
|
) -> u32 {
|
|
use URegLatencySM75::*;
|
|
match writer {
|
|
Udp => 1,
|
|
R2UR => 1,
|
|
Uldc | VoteU | Umov => match reader {
|
|
Udp => 3,
|
|
_ => 1,
|
|
},
|
|
_ => {
|
|
panic!("Illegal writer in ureg war latency")
|
|
}
|
|
}
|
|
}
|
|
|
|
fn pred_read_after_write(
|
|
writer: URegLatencySM75,
|
|
reader: URegLatencySM75,
|
|
) -> u32 {
|
|
use URegLatencySM75::*;
|
|
match reader {
|
|
Udp => match writer {
|
|
Udp => 4,
|
|
VoteU => 1,
|
|
_ => {
|
|
panic!("Illegal writer in upred raw latency")
|
|
}
|
|
},
|
|
VectorCoupled => match writer {
|
|
Udp => 6,
|
|
VoteU => 1,
|
|
_ => {
|
|
panic!("Illegal writer in upred raw latency")
|
|
}
|
|
},
|
|
GuardPredicate => match writer {
|
|
Udp => 11,
|
|
VoteU => 5,
|
|
_ => {
|
|
panic!("Illegal writer in upred raw latency")
|
|
}
|
|
},
|
|
_ => {
|
|
panic!("Illegal reader in upred raw latency")
|
|
}
|
|
}
|
|
}
|
|
|
|
fn pred_write_after_write(
|
|
writer1: URegLatencySM75,
|
|
writer2: URegLatencySM75,
|
|
) -> u32 {
|
|
use URegLatencySM75::*;
|
|
match writer2 {
|
|
Udp => 1,
|
|
VoteU => match writer1 {
|
|
Udp => 7,
|
|
VoteU => 1,
|
|
_ => {
|
|
panic!("Illegal writer1 in upred raw latency")
|
|
}
|
|
},
|
|
_ => {
|
|
panic!("Illegal writer2 in upred raw latency")
|
|
}
|
|
}
|
|
}
|
|
|
|
fn pred_write_after_read(
|
|
reader: URegLatencySM75,
|
|
writer: URegLatencySM75,
|
|
) -> u32 {
|
|
use URegLatencySM75::*;
|
|
match writer {
|
|
Udp => 1,
|
|
VoteU => match reader {
|
|
Udp => 2,
|
|
_ => 1,
|
|
},
|
|
_ => {
|
|
panic!("Illegal writer2 in upred raw latency")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
pub struct SM75Latency {}
|
|
|
|
impl SM75Latency {
|
|
pub fn needs_scoreboards(op: &Op) -> bool {
|
|
if op.is_uniform() {
|
|
matches!(
|
|
URegLatencySM75::op_category(op, false, 0),
|
|
URegLatencySM75::R2UR
|
|
)
|
|
} else {
|
|
match RegLatencySM75::op_category(op, false, 0) {
|
|
RegLatencySM75::RedirectedFP64 |
|
|
// We don't think fp16 needs scoreboarding on any known hw
|
|
// Put this back if we figure out it does.
|
|
//RegLatencySM75::RedirectedFP16 |
|
|
RegLatencySM75::RedirectedHMMA_884_F16(_) |
|
|
RegLatencySM75::RedirectedHMMA_884_F32(_) |
|
|
RegLatencySM75::RedirectedHMMA_1688 |
|
|
RegLatencySM75::RedirectedHMMA_16816 |
|
|
RegLatencySM75::IMMA(_) |
|
|
RegLatencySM75::Decoupled => true,
|
|
_ => false
|
|
}
|
|
}
|
|
}
|
|
|
|
/// if read is None pick the worst case raw latency
|
|
pub fn raw(
|
|
write: &Op,
|
|
dst_idx: usize,
|
|
read: Option<&Op>,
|
|
src_idx: usize,
|
|
) -> u32 {
|
|
let Some(dst_file) = write.dsts_as_slice()[dst_idx].file() else {
|
|
return 0;
|
|
};
|
|
|
|
match dst_file {
|
|
RegFile::GPR => {
|
|
let write_latency =
|
|
RegLatencySM75::op_category(write, false, dst_idx);
|
|
let read_latency = match read {
|
|
Some(op) => RegLatencySM75::op_category(op, true, src_idx),
|
|
None => RegLatencySM75::RedirectedFP64,
|
|
};
|
|
|
|
RegLatencySM75::read_after_write(write_latency, read_latency)
|
|
}
|
|
RegFile::UGPR => {
|
|
let write_latency =
|
|
URegLatencySM75::op_category(write, false, dst_idx);
|
|
let read_latency = match read {
|
|
Some(op) => URegLatencySM75::op_category(op, true, src_idx),
|
|
None => URegLatencySM75::Uldc,
|
|
};
|
|
|
|
URegLatencySM75::read_after_write(write_latency, read_latency)
|
|
}
|
|
RegFile::Pred => {
|
|
let write_latency =
|
|
RegLatencySM75::op_category(write, false, dst_idx);
|
|
let read_latency = match read {
|
|
Some(op) => RegLatencySM75::op_category(op, true, src_idx),
|
|
None => RegLatencySM75::GuardPredicate,
|
|
};
|
|
|
|
RegLatencySM75::pred_read_after_write(
|
|
write_latency,
|
|
read_latency,
|
|
)
|
|
}
|
|
RegFile::UPred => {
|
|
let write_latency =
|
|
URegLatencySM75::op_category(write, false, dst_idx);
|
|
let read_latency = match read {
|
|
Some(op) => URegLatencySM75::op_category(op, true, src_idx),
|
|
None => URegLatencySM75::GuardPredicate,
|
|
};
|
|
|
|
URegLatencySM75::pred_read_after_write(
|
|
write_latency,
|
|
read_latency,
|
|
)
|
|
}
|
|
RegFile::Bar => 0, // Barriers have a HW scoreboard
|
|
_ => panic!("Not a register"),
|
|
}
|
|
}
|
|
|
|
pub fn war(read: &Op, src_idx: usize, write: &Op, dst_idx: usize) -> u32 {
|
|
let Some(dst_file) = write.dsts_as_slice()[dst_idx].file() else {
|
|
return 0;
|
|
};
|
|
|
|
match dst_file {
|
|
RegFile::GPR => {
|
|
let write_latency =
|
|
RegLatencySM75::op_category(write, false, dst_idx);
|
|
let read_latency =
|
|
RegLatencySM75::op_category(read, true, src_idx);
|
|
|
|
RegLatencySM75::write_after_read(read_latency, write_latency)
|
|
}
|
|
RegFile::UGPR => {
|
|
let write_latency =
|
|
URegLatencySM75::op_category(write, false, dst_idx);
|
|
let read_latency =
|
|
URegLatencySM75::op_category(read, true, src_idx);
|
|
|
|
URegLatencySM75::write_after_read(read_latency, write_latency)
|
|
}
|
|
RegFile::Pred => {
|
|
let write_latency =
|
|
RegLatencySM75::op_category(write, false, dst_idx);
|
|
let read_latency =
|
|
RegLatencySM75::op_category(read, true, src_idx);
|
|
|
|
RegLatencySM75::pred_write_after_read(
|
|
read_latency,
|
|
write_latency,
|
|
)
|
|
}
|
|
RegFile::UPred => {
|
|
let write_latency =
|
|
URegLatencySM75::op_category(write, false, dst_idx);
|
|
let read_latency =
|
|
URegLatencySM75::op_category(read, true, src_idx);
|
|
|
|
URegLatencySM75::pred_write_after_read(
|
|
read_latency,
|
|
write_latency,
|
|
)
|
|
}
|
|
_ => panic!("Not a register"),
|
|
}
|
|
}
|
|
|
|
pub fn waw(
|
|
a: &Op,
|
|
a_dst_idx: usize,
|
|
b: &Op,
|
|
b_dst_idx: usize,
|
|
a_op_pred: bool,
|
|
) -> u32 {
|
|
let Some(dst_file) = a.dsts_as_slice()[a_dst_idx].file() else {
|
|
return 0;
|
|
};
|
|
|
|
match dst_file {
|
|
RegFile::GPR => {
|
|
let write1_latency =
|
|
RegLatencySM75::op_category(a, false, a_dst_idx);
|
|
let write2_latency =
|
|
RegLatencySM75::op_category(b, false, b_dst_idx);
|
|
|
|
RegLatencySM75::write_after_write(
|
|
write1_latency,
|
|
write2_latency,
|
|
a_op_pred,
|
|
)
|
|
}
|
|
RegFile::UGPR => {
|
|
let write1_latency =
|
|
URegLatencySM75::op_category(a, false, a_dst_idx);
|
|
let write2_latency =
|
|
URegLatencySM75::op_category(b, false, b_dst_idx);
|
|
|
|
URegLatencySM75::write_after_write(
|
|
write1_latency,
|
|
write2_latency,
|
|
a_op_pred,
|
|
)
|
|
}
|
|
RegFile::Pred => {
|
|
let write1_latency =
|
|
RegLatencySM75::op_category(a, false, a_dst_idx);
|
|
let write2_latency =
|
|
RegLatencySM75::op_category(b, false, b_dst_idx);
|
|
|
|
RegLatencySM75::pred_write_after_write(
|
|
write1_latency,
|
|
write2_latency,
|
|
a_op_pred,
|
|
)
|
|
}
|
|
RegFile::UPred => {
|
|
let write1_latency =
|
|
URegLatencySM75::op_category(a, false, a_dst_idx);
|
|
let write2_latency =
|
|
URegLatencySM75::op_category(b, false, b_dst_idx);
|
|
|
|
URegLatencySM75::pred_write_after_write(
|
|
write1_latency,
|
|
write2_latency,
|
|
)
|
|
}
|
|
_ => panic!("Not a register"),
|
|
}
|
|
}
|
|
}
|