[clang] 5370255 - [RISCV] Merge Masked and unMasked RVV manual codegen
Piyou Chen via cfe-commits
cfe-commits at lists.llvm.org
Wed Dec 21 20:36:46 PST 2022
Author: Piyou Chen
Date: 2022-12-21T20:28:12-08:00
New Revision: 5370255ff18ac101a73685b77615148142ee7552
URL: https://github.com/llvm/llvm-project/commit/5370255ff18ac101a73685b77615148142ee7552
DIFF: https://github.com/llvm/llvm-project/commit/5370255ff18ac101a73685b77615148142ee7552.diff
LOG: [RISCV] Merge Masked and unMasked RVV manual codegen
RVV intrinsic function will generate riscv_vector_builtin_cg.inc for CGBuiltin.cpp to produce the corresponding RVV intrinsic LLVM IR.
In this stage, riscv_vector.td will describe the bunch of manual codegen C++ code to tell CGBuiltin how to handle these instructions.
In this patch, we merge the masked RVV manual codegen and unmasked RVV manual codegen to reduce the number of manual codegen, and make more policy addition easier in the future.
This is a clean-up job that will not affect the RVV intrinsic functionality.
Reviewed By: kito-cheng
Differential Revision: https://reviews.llvm.org/D140361
Added:
Modified:
clang/include/clang/Basic/riscv_vector.td
clang/lib/CodeGen/CGBuiltin.cpp
clang/utils/TableGen/RISCVVEmitter.cpp
Removed:
################################################################################
diff --git a/clang/include/clang/Basic/riscv_vector.td b/clang/include/clang/Basic/riscv_vector.td
index 90e18df0403e0..7dbd331b2c5ec 100644
--- a/clang/include/clang/Basic/riscv_vector.td
+++ b/clang/include/clang/Basic/riscv_vector.td
@@ -217,7 +217,6 @@ class RVVBuiltin<string suffix, string prototype, string type_range,
// Manual code in clang codegen riscv_vector_builtin_cg.inc
code ManualCodegen = [{}];
- code MaskedManualCodegen = [{}];
// When emit the automatic clang codegen, it describes what types we have to use
// to obtain the specific LLVM intrinsic. -1 means the return type, otherwise,
@@ -627,31 +626,18 @@ multiclass RVVVLEFFBuiltin<list<string> types> {
UnMaskedPolicyScheme = HasPassthruOperand,
ManualCodegen = [{
{
- if (DefaultPolicy == TAIL_AGNOSTIC)
- Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
- IntrinsicTypes = {ResultType, Ops[3]->getType()};
- Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo());
- Value *NewVL = Ops[2];
- Ops.erase(Ops.begin() + 2);
- llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
- llvm::Value *LoadValue = Builder.CreateCall(F, Ops, "");
- llvm::Value *V = Builder.CreateExtractValue(LoadValue, {0});
- // Store new_vl.
- clang::CharUnits Align =
- CGM.getNaturalPointeeTypeAlignment(E->getArg(1)->getType());
- llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {1});
- Builder.CreateStore(Val, Address(NewVL, Val->getType(), Align));
- return V;
- }
- }],
- MaskedManualCodegen = [{
- {
- // Move mask to right before vl.
- std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
- if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC)
- Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
- Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
- IntrinsicTypes = {ResultType, Ops[4]->getType()};
+ if (IsMasked) {
+ // Move mask to right before vl.
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
+ if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC)
+ Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
+ Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
+ IntrinsicTypes = {ResultType, Ops[4]->getType()};
+ } else {
+ if (DefaultPolicy == TAIL_AGNOSTIC)
+ Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
+ IntrinsicTypes = {ResultType, Ops[3]->getType()};
+ }
Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo());
Value *NewVL = Ops[2];
Ops.erase(Ops.begin() + 2);
@@ -659,8 +645,11 @@ multiclass RVVVLEFFBuiltin<list<string> types> {
llvm::Value *LoadValue = Builder.CreateCall(F, Ops, "");
llvm::Value *V = Builder.CreateExtractValue(LoadValue, {0});
// Store new_vl.
- clang::CharUnits Align = CGM.getNaturalPointeeTypeAlignment(
- E->getArg(E->getNumArgs()-2)->getType());
+ clang::CharUnits Align;
+ if (IsMasked)
+ Align = CGM.getNaturalPointeeTypeAlignment(E->getArg(E->getNumArgs()-2)->getType());
+ else
+ Align = CGM.getNaturalPointeeTypeAlignment(E->getArg(1)->getType());
llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {1});
Builder.CreateStore(Val, Address(NewVL, Val->getType(), Align));
return V;
@@ -720,16 +709,18 @@ multiclass RVVIndexedLoad<string op> {
let HasMaskedOffOperand = false,
MaskedPolicyScheme = NonePolicy,
ManualCodegen = [{
- // Builtin: (ptr, value, vl). Intrinsic: (value, ptr, vl)
- std::swap(Ops[0], Ops[1]);
- Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo());
- IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType()};
- }],
- MaskedManualCodegen= [{
- // Builtin: (mask, ptr, value, vl). Intrinsic: (value, ptr, mask, vl)
- std::swap(Ops[0], Ops[2]);
+ if (IsMasked) {
+ // Builtin: (mask, ptr, value, vl). Intrinsic: (value, ptr, mask, vl)
+ std::swap(Ops[0], Ops[2]);
+ } else {
+ // Builtin: (ptr, value, vl). Intrinsic: (value, ptr, vl)
+ std::swap(Ops[0], Ops[1]);
+ }
Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo());
- IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()};
+ if (IsMasked)
+ IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()};
+ else
+ IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType()};
}] in {
class RVVVSEMaskBuiltin : RVVBuiltin<"m", "0PUem", "c"> {
let Name = "vsm_v";
@@ -757,16 +748,18 @@ multiclass RVVVSSEBuiltin<list<string> types> {
HasMaskedOffOperand = false,
MaskedPolicyScheme = NonePolicy,
ManualCodegen = [{
- // Builtin: (ptr, stride, value, vl). Intrinsic: (value, ptr, stride, vl)
- std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3);
- Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo());
- IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()};
- }],
- MaskedManualCodegen= [{
- // Builtin: (mask, ptr, stride, value, vl). Intrinsic: (value, ptr, stride, mask, vl)
- std::swap(Ops[0], Ops[3]);
+ if (IsMasked) {
+ // Builtin: (mask, ptr, stride, value, vl). Intrinsic: (value, ptr, stride, mask, vl)
+ std::swap(Ops[0], Ops[3]);
+ } else {
+ // Builtin: (ptr, stride, value, vl). Intrinsic: (value, ptr, stride, vl)
+ std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3);
+ }
Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo());
- IntrinsicTypes = {Ops[0]->getType(), Ops[4]->getType()};
+ if (IsMasked)
+ IntrinsicTypes = {Ops[0]->getType(), Ops[4]->getType()};
+ else
+ IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()};
}] in {
foreach type = types in {
def : RVVBuiltin<"v", "0Petv", type>;
@@ -781,16 +774,18 @@ multiclass RVVIndexedStore<string op> {
let HasMaskedOffOperand = false,
MaskedPolicyScheme = NonePolicy,
ManualCodegen = [{
- // Builtin: (ptr, index, value, vl). Intrinsic: (value, ptr, index, vl)
- std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3);
- Ops[1] = Builder.CreateBitCast(Ops[1],Ops[0]->getType()->getPointerTo());
- IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[3]->getType()};
- }],
- MaskedManualCodegen= [{
- // Builtin: (mask, ptr, index, value, vl). Intrinsic: (value, ptr, index, mask, vl)
- std::swap(Ops[0], Ops[3]);
+ if (IsMasked) {
+ // Builtin: (mask, ptr, index, value, vl). Intrinsic: (value, ptr, index, mask, vl)
+ std::swap(Ops[0], Ops[3]);
+ } else {
+ // Builtin: (ptr, index, value, vl). Intrinsic: (value, ptr, index, vl)
+ std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3);
+ }
Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo());
- IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[4]->getType()};
+ if (IsMasked)
+ IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[4]->getType()};
+ else
+ IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[3]->getType()};
}] in {
foreach type = TypeList in {
foreach eew_list = EEWList[0-2] in {
@@ -846,57 +841,43 @@ multiclass RVVUnitStridedSegLoad<string op> {
ManualCodegen = [{
{
ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType());
- // TA builtin: (val0 address, val1 address, ..., ptr, vl)
- // TU builtin: (val0 address, ..., passthru0, ..., ptr, vl)
IntrinsicTypes = {ResultType, Ops.back()->getType()};
- // intrinsic: (passthru0, passthru1, ..., ptr, vl)
SmallVector<llvm::Value*, 10> Operands;
- if (DefaultPolicy == TAIL_AGNOSTIC) {
- for (unsigned I = 0; I < NF; ++I)
- Operands.push_back(llvm::PoisonValue::get(ResultType));
- Operands.push_back(Ops[NF]);
- Operands.push_back(Ops[NF + 1]);
- } else {
- for (unsigned I = 0; I < NF; ++I)
- Operands.push_back(Ops[NF + I]);
- Operands.push_back(Ops[2 * NF]);
- Operands.push_back(Ops[2 * NF + 1]);
- }
- llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
- llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
- clang::CharUnits Align =
- CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType());
- llvm::Value *V;
- for (unsigned I = 0; I < NF; ++I) {
- llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {I});
- V = Builder.CreateStore(Val, Address(Ops[I], Val->getType(), Align));
- }
- return V;
- }
- }],
- MaskedManualCodegen = [{
- {
- // TAMA builtin: (val0 address, ..., mask, ptr, vl)
- // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, vl)
- // intrinsic: (maskedoff0, ..., ptr, mask, vl)
- ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType());
- IntrinsicTypes = {ResultType, Ops.back()->getType()};
- SmallVector<llvm::Value*, 12> Operands;
- if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) {
- for (unsigned I = 0; I < NF; ++I)
- Operands.push_back(llvm::PoisonValue::get(ResultType));
- Operands.push_back(Ops[NF + 1]);
- Operands.push_back(Ops[NF]);
- Operands.push_back(Ops[NF + 2]);
+ if (IsMasked) {
+ // TAMA builtin: (val0 address, ..., mask, ptr, vl)
+ // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, vl)
+ // intrinsic: (maskedoff0, ..., ptr, mask, vl)
+ if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) {
+ for (unsigned I = 0; I < NF; ++I)
+ Operands.push_back(llvm::PoisonValue::get(ResultType));
+ Operands.push_back(Ops[NF + 1]);
+ Operands.push_back(Ops[NF]);
+ Operands.push_back(Ops[NF + 2]);
+ } else {
+ for (unsigned I = 0; I < NF; ++I)
+ Operands.push_back(Ops[NF + I + 1]);
+ Operands.push_back(Ops[2 * NF + 1]);
+ Operands.push_back(Ops[NF]);
+ Operands.push_back(Ops[2 * NF + 2]);
+ }
+ Operands.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
+ assert(Operands.size() == NF + 4);
} else {
- for (unsigned I = 0; I < NF; ++I)
- Operands.push_back(Ops[NF + I + 1]);
- Operands.push_back(Ops[2 * NF + 1]);
- Operands.push_back(Ops[NF]);
- Operands.push_back(Ops[2 * NF + 2]);
+ // TA builtin: (val0 address, val1 address, ..., ptr, vl)
+ // TU builtin: (val0 address, ..., passthru0, ..., ptr, vl)
+ // intrinsic: (passthru0, passthru1, ..., ptr, vl)
+ if (DefaultPolicy == TAIL_AGNOSTIC) {
+ for (unsigned I = 0; I < NF; ++I)
+ Operands.push_back(llvm::PoisonValue::get(ResultType));
+ Operands.push_back(Ops[NF]);
+ Operands.push_back(Ops[NF + 1]);
+ } else {
+ for (unsigned I = 0; I < NF; ++I)
+ Operands.push_back(Ops[NF + I]);
+ Operands.push_back(Ops[2 * NF]);
+ Operands.push_back(Ops[2 * NF + 1]);
+ }
}
- Operands.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
- assert(Operands.size() == NF + 4);
llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
clang::CharUnits Align =
@@ -908,7 +889,7 @@ multiclass RVVUnitStridedSegLoad<string op> {
}
return V;
}
- }] in {
+ }] in {
defvar PV = PVString<nf, /*signed=*/true>.S;
defvar PUV = PVString<nf, /*signed=*/false>.S;
def : RVVBuiltin<"v", "0" # PV # "PCe", type>;
@@ -936,65 +917,50 @@ multiclass RVVUnitStridedSegLoadFF<string op> {
NF = nf,
ManualCodegen = [{
{
- // TA builtin: (val0 address, val1 address, ..., ptr, new_vl, vl)
- // TU builtin: (val0 address, ..., passthru0, ..., ptr, new_vl, vl)
ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType());
IntrinsicTypes = {ResultType, Ops.back()->getType()};
- // intrinsic: (passthru0, passthru1, ..., ptr, vl)
SmallVector<llvm::Value*, 12> Operands;
Value *NewVL;
- if (DefaultPolicy == TAIL_AGNOSTIC) {
- for (unsigned I = 0; I < NF; ++I)
- Operands.push_back(llvm::PoisonValue::get(ResultType));
- Operands.push_back(Ops[NF]);
- Operands.push_back(Ops[NF + 2]);
- NewVL = Ops[NF + 1];
- } else {
- for (unsigned I = 0; I < NF; ++I)
- Operands.push_back(Ops[NF + I]);
- Operands.push_back(Ops[2 * NF]);
- Operands.push_back(Ops[2 * NF + 2]);
- NewVL = Ops[2 * NF + 1];
- }
- llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
- llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
- clang::CharUnits Align =
- CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType());
- for (unsigned I = 0; I < NF; ++I) {
- llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {I});
- Builder.CreateStore(Val, Address(Ops[I], Val->getType(), Align));
- }
- // Store new_vl.
- llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {NF});
- return Builder.CreateStore(Val, Address(NewVL, Val->getType(), Align));
- }
- }],
- MaskedManualCodegen = [{
- {
- // TAMA builtin: (val0 address, ..., mask, ptr, new_vl, vl)
- // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, new_vl, vl)
- // intrinsic: (maskedoff0, ..., ptr, mask, vl)
- ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType());
- IntrinsicTypes = {ResultType, Ops.back()->getType()};
- SmallVector<llvm::Value*, 12> Operands;
- Value *NewVL;
- if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) {
- for (unsigned I = 0; I < NF; ++I)
- Operands.push_back(llvm::PoisonValue::get(ResultType));
- Operands.push_back(Ops[NF + 1]);
- Operands.push_back(Ops[NF]);
- Operands.push_back(Ops[NF + 3]);
- NewVL = Ops[NF + 2];
+
+ if (IsMasked) {
+ // TAMA builtin: (val0 address, ..., mask, ptr, new_vl, vl)
+ // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, new_vl, vl)
+ // intrinsic: (maskedoff0, ..., ptr, mask, vl)
+ if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) {
+ for (unsigned I = 0; I < NF; ++I)
+ Operands.push_back(llvm::PoisonValue::get(ResultType));
+ Operands.push_back(Ops[NF + 1]);
+ Operands.push_back(Ops[NF]);
+ Operands.push_back(Ops[NF + 3]);
+ NewVL = Ops[NF + 2];
+ } else {
+ for (unsigned I = 0; I < NF; ++I)
+ Operands.push_back(Ops[NF + I + 1]);
+ Operands.push_back(Ops[2 * NF + 1]);
+ Operands.push_back(Ops[NF]);
+ Operands.push_back(Ops[2 * NF + 3]);
+ NewVL = Ops[2 * NF + 2];
+ }
+ Operands.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
+ assert(Operands.size() == NF + 4);
} else {
- for (unsigned I = 0; I < NF; ++I)
- Operands.push_back(Ops[NF + I + 1]);
- Operands.push_back(Ops[2 * NF + 1]);
- Operands.push_back(Ops[NF]);
- Operands.push_back(Ops[2 * NF + 3]);
- NewVL = Ops[2 * NF + 2];
+ // TA builtin: (val0 address, val1 address, ..., ptr, new_vl, vl)
+ // TU builtin: (val0 address, ..., passthru0, ..., ptr, new_vl, vl)
+ // intrinsic: (passthru0, passthru1, ..., ptr, vl)
+ if (DefaultPolicy == TAIL_AGNOSTIC) {
+ for (unsigned I = 0; I < NF; ++I)
+ Operands.push_back(llvm::PoisonValue::get(ResultType));
+ Operands.push_back(Ops[NF]);
+ Operands.push_back(Ops[NF + 2]);
+ NewVL = Ops[NF + 1];
+ } else {
+ for (unsigned I = 0; I < NF; ++I)
+ Operands.push_back(Ops[NF + I]);
+ Operands.push_back(Ops[2 * NF]);
+ Operands.push_back(Ops[2 * NF + 2]);
+ NewVL = Ops[2 * NF + 1];
+ }
}
- Operands.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
- assert(Operands.size() == NF + 4);
llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
clang::CharUnits Align =
@@ -1007,7 +973,7 @@ multiclass RVVUnitStridedSegLoadFF<string op> {
llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {NF});
return Builder.CreateStore(Val, Address(NewVL, Val->getType(), Align));
}
- }] in {
+ }] in {
defvar PV = PVString<nf, /*signed=*/true>.S;
defvar PUV = PVString<nf, /*signed=*/false>.S;
def : RVVBuiltin<"v", "0" # PV # "PCe" # "Pz", type>;
@@ -1035,62 +1001,49 @@ multiclass RVVStridedSegLoad<string op> {
NF = nf,
ManualCodegen = [{
{
- // TA builtin: (val0 address, val1 address, ..., ptr, stride, vl)
- // TU builtin: (val0 address, ..., passthru0, ..., ptr, stride, vl)
ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType());
IntrinsicTypes = {ResultType, Ops.back()->getType()};
- // intrinsic: (passthru0, passthru1, ..., ptr, stride, vl)
SmallVector<llvm::Value*, 12> Operands;
- if (DefaultPolicy == TAIL_AGNOSTIC) {
- for (unsigned I = 0; I < NF; ++I)
- Operands.push_back(llvm::PoisonValue::get(ResultType));
- Operands.push_back(Ops[NF]);
- Operands.push_back(Ops[NF + 1]);
- Operands.push_back(Ops[NF + 2]);
- } else {
- for (unsigned I = 0; I < NF; ++I)
- Operands.push_back(Ops[NF + I]);
- Operands.push_back(Ops[2 * NF]);
- Operands.push_back(Ops[2 * NF + 1]);
- Operands.push_back(Ops[2 * NF + 2]);
- }
- llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
- llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
- clang::CharUnits Align =
- CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType());
- llvm::Value *V;
- for (unsigned I = 0; I < NF; ++I) {
- llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {I});
- V = Builder.CreateStore(Val, Address(Ops[I], Val->getType(), Align));
- }
- return V;
- }
- }],
- MaskedManualCodegen = [{
- {
- //TAMA builtin: (val0 address, ..., mask, ptr, stride, vl)
- // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, stride, vl)
- // intrinsic: (maskedoff0, ..., ptr, stride, mask, vl)
- ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType());
- IntrinsicTypes = {ResultType, Ops.back()->getType()};
- SmallVector<llvm::Value*, 12> Operands;
- if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) {
- for (unsigned I = 0; I < NF; ++I)
- Operands.push_back(llvm::PoisonValue::get(ResultType));
- Operands.push_back(Ops[NF + 1]);
- Operands.push_back(Ops[NF + 2]);
- Operands.push_back(Ops[NF]);
- Operands.push_back(Ops[NF + 3]);
+
+ if (IsMasked) {
+ // TAMA builtin: (val0 address, ..., mask, ptr, stride, vl)
+ // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, stride, vl)
+ // intrinsic: (maskedoff0, ..., ptr, stride, mask, vl)
+ if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) {
+ for (unsigned I = 0; I < NF; ++I)
+ Operands.push_back(llvm::PoisonValue::get(ResultType));
+ Operands.push_back(Ops[NF + 1]);
+ Operands.push_back(Ops[NF + 2]);
+ Operands.push_back(Ops[NF]);
+ Operands.push_back(Ops[NF + 3]);
+ } else {
+ for (unsigned I = 0; I < NF; ++I)
+ Operands.push_back(Ops[NF + I + 1]);
+ Operands.push_back(Ops[2 * NF + 1]);
+ Operands.push_back(Ops[2 * NF + 2]);
+ Operands.push_back(Ops[NF]);
+ Operands.push_back(Ops[2 * NF + 3]);
+ }
+ Operands.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
+ assert(Operands.size() == NF + 5);
} else {
- for (unsigned I = 0; I < NF; ++I)
- Operands.push_back(Ops[NF + I + 1]);
- Operands.push_back(Ops[2 * NF + 1]);
- Operands.push_back(Ops[2 * NF + 2]);
- Operands.push_back(Ops[NF]);
- Operands.push_back(Ops[2 * NF + 3]);
+ // TA builtin: (val0 address, val1 address, ..., ptr, stride, vl)
+ // TU builtin: (val0 address, ..., passthru0, ..., ptr, stride, vl)
+ // intrinsic: (passthru0, passthru1, ..., ptr, stride, vl)
+ if (DefaultPolicy == TAIL_AGNOSTIC) {
+ for (unsigned I = 0; I < NF; ++I)
+ Operands.push_back(llvm::PoisonValue::get(ResultType));
+ Operands.push_back(Ops[NF]);
+ Operands.push_back(Ops[NF + 1]);
+ Operands.push_back(Ops[NF + 2]);
+ } else {
+ for (unsigned I = 0; I < NF; ++I)
+ Operands.push_back(Ops[NF + I]);
+ Operands.push_back(Ops[2 * NF]);
+ Operands.push_back(Ops[2 * NF + 1]);
+ Operands.push_back(Ops[2 * NF + 2]);
+ }
}
- Operands.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
- assert(Operands.size() == NF + 5);
llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
clang::CharUnits Align =
@@ -1102,7 +1055,7 @@ multiclass RVVStridedSegLoad<string op> {
}
return V;
}
- }] in {
+ }] in {
defvar PV = PVString<nf, /*signed=*/true>.S;
defvar PUV = PVString<nf, /*signed=*/false>.S;
def : RVVBuiltin<"v", "0" # PV # "PCe" # "t", type>;
@@ -1126,64 +1079,51 @@ multiclass RVVIndexedSegLoad<string op> {
NF = nf,
ManualCodegen = [{
{
- // TA builtin: (val0 address, val1 address, ..., ptr, index, vl)
- // TU builtin: (val0 address, ..., passthru0, ..., ptr, index, vl)
ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType());
- // intrinsic: (passthru0, passthru1, ..., ptr, index, vl)
SmallVector<llvm::Value*, 12> Operands;
- if (DefaultPolicy == TAIL_AGNOSTIC) {
- for (unsigned I = 0; I < NF; ++I)
- Operands.push_back(llvm::PoisonValue::get(ResultType));
- Operands.push_back(Ops[NF]);
- Operands.push_back(Ops[NF + 1]);
- Operands.push_back(Ops[NF + 2]);
- IntrinsicTypes = {ResultType, Ops[NF + 1]->getType(), Ops.back()->getType()};
- } else {
- for (unsigned I = 0; I < NF; ++I)
- Operands.push_back(Ops[NF + I]);
- Operands.push_back(Ops[2 * NF]);
- Operands.push_back(Ops[2 * NF + 1]);
- Operands.push_back(Ops[2 * NF + 2]);
- IntrinsicTypes = {ResultType, Ops[2 * NF + 1]->getType(), Ops.back()->getType()};
- }
- llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
- llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
- clang::CharUnits Align =
- CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType());
- llvm::Value *V;
- for (unsigned I = 0; I < NF; ++I) {
- llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {I});
- V = Builder.CreateStore(Val, Address(Ops[I], Val->getType(), Align));
- }
- return V;
- }
- }],
- MaskedManualCodegen = [{
- {
- // TAMA builtin: (val0 address, ..., mask, ptr, index, vl)
- // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, index, vl)
- ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType());
- // intrinsic: (maskedoff0, ..., ptr, index, mask, vl)
- SmallVector<llvm::Value*, 12> Operands;
- if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) {
- for (unsigned I = 0; I < NF; ++I)
- Operands.push_back(llvm::PoisonValue::get(ResultType));
- Operands.push_back(Ops[NF + 1]);
- Operands.push_back(Ops[NF + 2]);
- Operands.push_back(Ops[NF]);
- Operands.push_back(Ops[NF + 3]);
- IntrinsicTypes = {ResultType, Ops[NF + 2]->getType(), Ops.back()->getType()};
+ if (IsMasked) {
+ // TAMA builtin: (val0 address, ..., mask, ptr, index, vl)
+ // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, index, vl)
+ // intrinsic: (maskedoff0, ..., ptr, index, mask, vl)
+ if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) {
+ for (unsigned I = 0; I < NF; ++I)
+ Operands.push_back(llvm::PoisonValue::get(ResultType));
+ Operands.push_back(Ops[NF + 1]);
+ Operands.push_back(Ops[NF + 2]);
+ Operands.push_back(Ops[NF]);
+ Operands.push_back(Ops[NF + 3]);
+ IntrinsicTypes = {ResultType, Ops[NF + 2]->getType(), Ops.back()->getType()};
+ } else {
+ for (unsigned I = 0; I < NF; ++I)
+ Operands.push_back(Ops[NF + I + 1]);
+ Operands.push_back(Ops[2 * NF + 1]);
+ Operands.push_back(Ops[2 * NF + 2]);
+ Operands.push_back(Ops[NF]);
+ Operands.push_back(Ops[2 * NF + 3]);
+ IntrinsicTypes = {ResultType, Ops[2 * NF + 2]->getType(), Ops.back()->getType()};
+ }
+ Operands.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
+ assert(Operands.size() == NF + 5);
} else {
- for (unsigned I = 0; I < NF; ++I)
- Operands.push_back(Ops[NF + I + 1]);
- Operands.push_back(Ops[2 * NF + 1]);
- Operands.push_back(Ops[2 * NF + 2]);
- Operands.push_back(Ops[NF]);
- Operands.push_back(Ops[2 * NF + 3]);
- IntrinsicTypes = {ResultType, Ops[2 * NF + 2]->getType(), Ops.back()->getType()};
+ // TA builtin: (val0 address, val1 address, ..., ptr, index, vl)
+ // TU builtin: (val0 address, ..., passthru0, ..., ptr, index, vl)
+ // intrinsic: (passthru0, passthru1, ..., ptr, index, vl)
+ if (DefaultPolicy == TAIL_AGNOSTIC) {
+ for (unsigned I = 0; I < NF; ++I)
+ Operands.push_back(llvm::PoisonValue::get(ResultType));
+ Operands.push_back(Ops[NF]);
+ Operands.push_back(Ops[NF + 1]);
+ Operands.push_back(Ops[NF + 2]);
+ IntrinsicTypes = {ResultType, Ops[NF + 1]->getType(), Ops.back()->getType()};
+ } else {
+ for (unsigned I = 0; I < NF; ++I)
+ Operands.push_back(Ops[NF + I]);
+ Operands.push_back(Ops[2 * NF]);
+ Operands.push_back(Ops[2 * NF + 1]);
+ Operands.push_back(Ops[2 * NF + 2]);
+ IntrinsicTypes = {ResultType, Ops[2 * NF + 1]->getType(), Ops.back()->getType()};
+ }
}
- Operands.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
- assert(Operands.size() == NF + 5);
llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
clang::CharUnits Align =
@@ -1195,7 +1135,7 @@ multiclass RVVIndexedSegLoad<string op> {
}
return V;
}
- }] in {
+ }] in {
defvar PV = PVString<nf, /*signed=*/true>.S;
defvar PUV = PVString<nf, /*signed=*/false>.S;
def : RVVBuiltin<"v", "0" # PV # "PCe" # eew_type # "Uv", type>;
@@ -1236,21 +1176,20 @@ multiclass RVVUnitStridedSegStore<string op> {
MaskedPolicyScheme = NonePolicy,
ManualCodegen = [{
{
- // Builtin: (ptr, val0, val1, ..., vl)
- // Intrinsic: (val0, val1, ..., ptr, vl)
- std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
- IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 1]->getType()};
- assert(Ops.size() == NF + 2);
- }
- }],
- MaskedManualCodegen = [{
- {
- // Builtin: (mask, ptr, val0, val1, ..., vl)
- // Intrinsic: (val0, val1, ..., ptr, mask, vl)
- std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1);
- std::swap(Ops[NF], Ops[NF + 1]);
- IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 2]->getType()};
- assert(Ops.size() == NF + 3);
+ if (IsMasked) {
+ // Builtin: (mask, ptr, val0, val1, ..., vl)
+ // Intrinsic: (val0, val1, ..., ptr, mask, vl)
+ std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1);
+ std::swap(Ops[NF], Ops[NF + 1]);
+ IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 2]->getType()};
+ assert(Ops.size() == NF + 3);
+ } else {
+ // Builtin: (ptr, val0, val1, ..., vl)
+ // Intrinsic: (val0, val1, ..., ptr, vl)
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
+ IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 1]->getType()};
+ assert(Ops.size() == NF + 2);
+ }
}
}] in {
defvar V = VString<nf, /*signed=*/true>.S;
@@ -1282,21 +1221,19 @@ multiclass RVVStridedSegStore<string op> {
MaskedPolicyScheme = NonePolicy,
ManualCodegen = [{
{
- // Builtin: (ptr, stride, val0, val1, ..., vl).
- // Intrinsic: (val0, val1, ..., ptr, stride, vl)
- std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1);
- IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 1]->getType()};
- assert(Ops.size() == NF + 3);
- }
- }],
- MaskedManualCodegen = [{
- {
- // Builtin: (mask, ptr, stride, val0, val1, ..., vl).
- // Intrinsic: (val0, val1, ..., ptr, stride, mask, vl)
- std::rotate(Ops.begin(), Ops.begin() + 3, Ops.end() - 1);
- std::rotate(Ops.begin() + NF, Ops.begin() + NF + 1, Ops.begin() + NF + 3);
+ if (IsMasked) {
+ // Builtin: (mask, ptr, stride, val0, val1, ..., vl).
+ // Intrinsic: (val0, val1, ..., ptr, stride, mask, vl)
+ std::rotate(Ops.begin(), Ops.begin() + 3, Ops.end() - 1);
+ std::rotate(Ops.begin() + NF, Ops.begin() + NF + 1, Ops.begin() + NF + 3);
+ assert(Ops.size() == NF + 4);
+ } else {
+ // Builtin: (ptr, stride, val0, val1, ..., vl).
+ // Intrinsic: (val0, val1, ..., ptr, stride, vl)
+ std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1);
+ assert(Ops.size() == NF + 3);
+ }
IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 1]->getType()};
- assert(Ops.size() == NF + 4);
}
}] in {
defvar V = VString<nf, /*signed=*/true>.S;
@@ -1324,23 +1261,22 @@ multiclass RVVIndexedSegStore<string op> {
MaskedPolicyScheme = NonePolicy,
ManualCodegen = [{
{
- // Builtin: (ptr, index, val0, val1, ..., vl)
- // Intrinsic: (val0, val1, ..., ptr, index, vl)
- std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1);
- IntrinsicTypes = {Ops[0]->getType(),
- Ops[NF + 1]->getType(), Ops[NF + 2]->getType()};
- assert(Ops.size() == NF + 3);
- }
- }],
- MaskedManualCodegen = [{
- {
- // Builtin: (mask, ptr, index, val0, val1, ..., vl)
- // Intrinsic: (val0, val1, ..., ptr, index, mask, vl)
- std::rotate(Ops.begin(), Ops.begin() + 3, Ops.end() - 1);
- std::rotate(Ops.begin() + NF, Ops.begin() + NF + 1, Ops.begin() + NF + 3);
- IntrinsicTypes = {Ops[0]->getType(),
- Ops[NF + 1]->getType(), Ops[NF + 3]->getType()};
- assert(Ops.size() == NF + 4);
+ if (IsMasked) {
+ // Builtin: (mask, ptr, index, val0, val1, ..., vl)
+ // Intrinsic: (val0, val1, ..., ptr, index, mask, vl)
+ std::rotate(Ops.begin(), Ops.begin() + 3, Ops.end() - 1);
+ std::rotate(Ops.begin() + NF, Ops.begin() + NF + 1, Ops.begin() + NF + 3);
+ IntrinsicTypes = {Ops[0]->getType(),
+ Ops[NF + 1]->getType(), Ops[NF + 3]->getType()};
+ assert(Ops.size() == NF + 4);
+ } else {
+ // Builtin: (ptr, index, val0, val1, ..., vl)
+ // Intrinsic: (val0, val1, ..., ptr, index, vl)
+ std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1);
+ IntrinsicTypes = {Ops[0]->getType(),
+ Ops[NF + 1]->getType(), Ops[NF + 2]->getType()};
+ assert(Ops.size() == NF + 3);
+ }
}
}] in {
defvar V = VString<nf, /*signed=*/true>.S;
@@ -1362,25 +1298,25 @@ multiclass RVVPseudoUnaryBuiltin<string IR, string type_range> {
UnMaskedPolicyScheme = HasPassthruOperand,
ManualCodegen = [{
{
- if (DefaultPolicy == TAIL_AGNOSTIC)
- Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
- auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType();
- Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy));
- // passthru, op1, op2, vl
- IntrinsicTypes = {ResultType, ElemTy, Ops[3]->getType()};
- break;
- }
- }],
- MaskedManualCodegen = [{
- {
- std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
- if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC)
- Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
+ if (IsMasked) {
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
+ if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC)
+ Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
+ } else {
+ if (DefaultPolicy == TAIL_AGNOSTIC)
+ Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
+ }
auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType();
Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy));
- Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
- // maskedoff, op1, op2, mask, vl, policy
- IntrinsicTypes = {ResultType, ElemTy, Ops[4]->getType()};
+
+ if (IsMasked) {
+ Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
+ // maskedoff, op1, op2, mask, vl, policy
+ IntrinsicTypes = {ResultType, ElemTy, Ops[4]->getType()};
+ } else {
+ // passthru, op1, op2, vl
+ IntrinsicTypes = {ResultType, ElemTy, Ops[3]->getType()};
+ }
break;
}
}] in {
@@ -1395,31 +1331,29 @@ multiclass RVVPseudoVNotBuiltin<string IR, string type_range> {
UnMaskedPolicyScheme = HasPassthruOperand,
ManualCodegen = [{
{
- if (DefaultPolicy == TAIL_AGNOSTIC)
- Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
- auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType();
- Ops.insert(Ops.begin() + 2,
- llvm::Constant::getAllOnesValue(ElemTy));
- // passthru, op1, op2, vl
- IntrinsicTypes = {ResultType,
- ElemTy,
- Ops[3]->getType()};
- break;
- }
- }],
- MaskedManualCodegen = [{
- {
- std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
- if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC)
- Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
+ if (IsMasked) {
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
+ if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC)
+ Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
+ } else {
+ if (DefaultPolicy == TAIL_AGNOSTIC)
+ Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
+ }
auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType();
Ops.insert(Ops.begin() + 2,
llvm::Constant::getAllOnesValue(ElemTy));
- Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
- // maskedoff, op1, po2, mask, vl, policy
- IntrinsicTypes = {ResultType,
- ElemTy,
- Ops[4]->getType()};
+ if (IsMasked) {
+ Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
+ // maskedoff, op1, po2, mask, vl, policy
+ IntrinsicTypes = {ResultType,
+ ElemTy,
+ Ops[4]->getType()};
+ } else {
+ // passthru, op1, op2, vl
+ IntrinsicTypes = {ResultType,
+ ElemTy,
+ Ops[3]->getType()};
+ }
break;
}
}] in {
@@ -1452,26 +1386,25 @@ multiclass RVVPseudoVFUnaryBuiltin<string IR, string type_range> {
UnMaskedPolicyScheme = HasPassthruOperand,
ManualCodegen = [{
{
- if (DefaultPolicy == TAIL_AGNOSTIC)
- Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
- // op1, po2, vl
- IntrinsicTypes = {ResultType,
- Ops[1]->getType(), Ops[2]->getType()};
- Ops.insert(Ops.begin() + 2, Ops[1]);
- break;
- }
- }],
- MaskedManualCodegen = [{
- {
- std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
- if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC)
- Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
- Ops.insert(Ops.begin() + 2, Ops[1]);
- Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
- // maskedoff, op1, op2, mask, vl
- IntrinsicTypes = {ResultType,
- Ops[2]->getType(),
- Ops.back()->getType()};
+ if (IsMasked) {
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
+ if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC)
+ Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
+ Ops.insert(Ops.begin() + 2, Ops[1]);
+ Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
+ // maskedoff, op1, op2, mask, vl
+ IntrinsicTypes = {ResultType,
+ Ops[2]->getType(),
+ Ops.back()->getType()};
+ } else {
+ if (DefaultPolicy == TAIL_AGNOSTIC)
+ Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
+ // op1, po2, vl
+ IntrinsicTypes = {ResultType,
+ Ops[1]->getType(), Ops[2]->getType()};
+ Ops.insert(Ops.begin() + 2, Ops[1]);
+ break;
+ }
break;
}
}] in {
@@ -1488,31 +1421,30 @@ multiclass RVVPseudoVWCVTBuiltin<string IR, string MName, string type_range,
UnMaskedPolicyScheme = HasPassthruOperand,
ManualCodegen = [{
{
- if (DefaultPolicy == TAIL_AGNOSTIC)
- Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
- auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType();
- Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy));
- // passtru, op1, op2, vl
- IntrinsicTypes = {ResultType,
- Ops[1]->getType(),
- ElemTy,
- Ops[3]->getType()};
- break;
- }
- }],
- MaskedManualCodegen = [{
- {
- std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
- if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC)
- Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
+ if (IsMasked) {
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
+ if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC)
+ Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
+ } else {
+ if (DefaultPolicy == TAIL_AGNOSTIC)
+ Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
+ }
auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType();
Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy));
- Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
- // maskedoff, op1, op2, mask, vl, policy
- IntrinsicTypes = {ResultType,
- Ops[1]->getType(),
- ElemTy,
- Ops[4]->getType()};
+ if (IsMasked) {
+ Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
+ // maskedoff, op1, op2, mask, vl, policy
+ IntrinsicTypes = {ResultType,
+ Ops[1]->getType(),
+ ElemTy,
+ Ops[4]->getType()};
+ } else {
+ // passtru, op1, op2, vl
+ IntrinsicTypes = {ResultType,
+ Ops[1]->getType(),
+ ElemTy,
+ Ops[3]->getType()};
+ }
break;
}
}] in {
@@ -1531,29 +1463,29 @@ multiclass RVVPseudoVNCVTBuiltin<string IR, string MName, string type_range,
UnMaskedPolicyScheme = HasPassthruOperand,
ManualCodegen = [{
{
- if (DefaultPolicy == TAIL_AGNOSTIC)
- Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
- Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(Ops.back()->getType()));
- // passthru, op1, xlen, vl
- IntrinsicTypes = {ResultType,
- Ops[1]->getType(),
- Ops[3]->getType(),
- Ops[3]->getType()};
- break;
- }
- }],
- MaskedManualCodegen = [{
- {
- std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
- if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC)
- Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
- Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(Ops.back()->getType()));
- Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
- // maskedoff, op1, xlen, mask, vl
- IntrinsicTypes = {ResultType,
- Ops[1]->getType(),
- Ops[4]->getType(),
- Ops[4]->getType()};
+ if (IsMasked) {
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
+ if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC)
+ Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
+ } else {
+ if (DefaultPolicy == TAIL_AGNOSTIC)
+ Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
+ }
+ Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(Ops.back()->getType()));
+ if (IsMasked) {
+ Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy));
+ // maskedoff, op1, xlen, mask, vl
+ IntrinsicTypes = {ResultType,
+ Ops[1]->getType(),
+ Ops[4]->getType(),
+ Ops[4]->getType()};
+ } else {
+ // passthru, op1, xlen, vl
+ IntrinsicTypes = {ResultType,
+ Ops[1]->getType(),
+ Ops[3]->getType(),
+ Ops[3]->getType()};
+ }
break;
}
}] in {
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index ef0802e4f6e3e..fc7b73d9ee6e5 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -19442,6 +19442,7 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID,
constexpr unsigned TAIL_AGNOSTIC = 1;
constexpr unsigned TAIL_AGNOSTIC_MASK_AGNOSTIC = 3;
int DefaultPolicy = TAIL_UNDISTURBED;
+ bool IsMasked = false;
// Required for overloaded intrinsics.
llvm::SmallVector<llvm::Type *, 2> IntrinsicTypes;
diff --git a/clang/utils/TableGen/RISCVVEmitter.cpp b/clang/utils/TableGen/RISCVVEmitter.cpp
index ecf3b35e4a47d..ddf2c5c52393e 100644
--- a/clang/utils/TableGen/RISCVVEmitter.cpp
+++ b/clang/utils/TableGen/RISCVVEmitter.cpp
@@ -171,6 +171,10 @@ void emitCodeGenSwitchBody(const RVVIntrinsic *RVVI, raw_ostream &OS) {
if (RVVI->hasManualCodegen()) {
OS << " DefaultPolicy = " << RVVI->getDefaultPolicyBits() << ";\n";
+ if (RVVI->isMasked())
+ OS << "IsMasked = true;\n";
+ else
+ OS << "IsMasked = false;\n";
OS << RVVI->getManualCodegen();
OS << "break;\n";
return;
@@ -517,7 +521,6 @@ void RVVEmitter::createRVVIntrinsics(
bool SupportOverloading = R->getValueAsBit("SupportOverloading");
bool HasBuiltinAlias = R->getValueAsBit("HasBuiltinAlias");
StringRef ManualCodegen = R->getValueAsString("ManualCodegen");
- StringRef MaskedManualCodegen = R->getValueAsString("MaskedManualCodegen");
std::vector<int64_t> IntrinsicTypes =
R->getValueAsListOfInts("IntrinsicTypes");
std::vector<StringRef> RequiredFeatures =
@@ -598,7 +601,7 @@ void RVVEmitter::createRVVIntrinsics(
Out.push_back(std::make_unique<RVVIntrinsic>(
Name, SuffixStr, OverloadedName, OverloadedSuffixStr, MaskedIRName,
/*IsMasked=*/true, HasMaskedOffOperand, HasVL, MaskedPolicyScheme,
- SupportOverloading, HasBuiltinAlias, MaskedManualCodegen,
+ SupportOverloading, HasBuiltinAlias, ManualCodegen,
*MaskTypes, IntrinsicTypes, RequiredFeatures, NF,
Policy(), IsPrototypeDefaultTU));
if (MaskedPolicyScheme == PolicyScheme::SchemeNone)
@@ -614,7 +617,7 @@ void RVVEmitter::createRVVIntrinsics(
Name, SuffixStr, OverloadedName, OverloadedSuffixStr,
MaskedIRName, /*IsMasked=*/true, HasMaskedOffOperand, HasVL,
MaskedPolicyScheme, SupportOverloading, HasBuiltinAlias,
- MaskedManualCodegen, *PolicyTypes, IntrinsicTypes,
+ ManualCodegen, *PolicyTypes, IntrinsicTypes,
RequiredFeatures, NF, P, IsPrototypeDefaultTU));
}
} // End for Log2LMULList
More information about the cfe-commits
mailing list