[clang] 83b3304 - [AArch64] Implement __arm_rsr128/__arm_wsr128
Archibald Elliott via cfe-commits
cfe-commits at lists.llvm.org
Tue Dec 6 04:13:55 PST 2022
Author: Archibald Elliott
Date: 2022-12-06T11:39:05Z
New Revision: 83b3304dd2a3bee8b6b4f76282adb734a5d1c8ff
URL: https://github.com/llvm/llvm-project/commit/83b3304dd2a3bee8b6b4f76282adb734a5d1c8ff
DIFF: https://github.com/llvm/llvm-project/commit/83b3304dd2a3bee8b6b4f76282adb734a5d1c8ff.diff
LOG: [AArch64] Implement __arm_rsr128/__arm_wsr128
This only contains the SelectionDAG implementation. GlobalISel to
follow.
The broad approach is:
- Introduce new builtins for 128-bit wide instructions.
- Lower these to @llvm.read_register.i128/@llvm.write_register.i128
- Introduce target-specific ISD nodes which have legal operands (two
i64s rather than an i128). These are named AArch64::{MRRS, MSRR} to
match the instructions they are for. These are a little complex as
they need to match the "shape" of what they're replacing or the
legaliser complains.
- Select these using the existing tryReadRegister/tryWriteRegister to
share the MDString parsing code, and introduce additional code to
ensure these are selected into the right MRRS/MSRR instructions. What
makes this hard is ensuring that the two i64s end up in an XSeqPair
register pair, because SelectionDAG doesn't care that much about
register classes if it can avoid doing so.
The main change to existing code is the reorganisation of
tryReadRegister and tryWriteRegister to try to keep the string parsing
code separate from the instruction creating code.
This also includes the changes to clang to define and use the ACLE
feature macro named `__ARM_FEATURE_SYSREG128`.
Contributors:
Sam Elliott
Lucas Prates
Differential Revision: https://reviews.llvm.org/D139086
Added:
llvm/test/CodeGen/AArch64/aarch64-sysreg128.ll
Modified:
clang/include/clang/Basic/BuiltinsAArch64.def
clang/lib/CodeGen/CGBuiltin.cpp
clang/lib/Headers/arm_acle.h
clang/test/CodeGen/arm_acle.c
clang/test/Preprocessor/aarch64-target-features.c
llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/lib/Target/AArch64/AArch64ISelLowering.h
Removed:
################################################################################
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.def b/clang/include/clang/Basic/BuiltinsAArch64.def
index bc8ab4eade91a..c9b3272e74bc8 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.def
+++ b/clang/include/clang/Basic/BuiltinsAArch64.def
@@ -82,9 +82,11 @@ BUILTIN(__builtin_arm_prefetch, "vvC*UiUiUiUi", "nc")
// System Registers
BUILTIN(__builtin_arm_rsr, "UicC*", "nc")
BUILTIN(__builtin_arm_rsr64, "WUicC*", "nc")
+BUILTIN(__builtin_arm_rsr128, "LLLUicC*", "nc")
BUILTIN(__builtin_arm_rsrp, "v*cC*", "nc")
BUILTIN(__builtin_arm_wsr, "vcC*Ui", "nc")
BUILTIN(__builtin_arm_wsr64, "vcC*WUi", "nc")
+BUILTIN(__builtin_arm_wsr128, "vcC*LLLUi", "nc")
BUILTIN(__builtin_arm_wsrp, "vcC*vC*", "nc")
// MSVC
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 2b717208313f2..0138111758657 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -7605,9 +7605,10 @@ static Value *EmitSpecialRegisterBuiltin(CodeGenFunction &CGF,
llvm::Type *ValueType,
SpecialRegisterAccessKind AccessKind,
StringRef SysReg = "") {
- // write and register intrinsics only support 32 and 64 bit operations.
- assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64))
- && "Unsupported size for register.");
+ // write and register intrinsics only support 32, 64 and 128 bit operations.
+ assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64) ||
+ RegisterType->isIntegerTy(128)) &&
+ "Unsupported size for register.");
CodeGen::CGBuilderTy &Builder = CGF.Builder;
CodeGen::CodeGenModule &CGM = CGF.CGM;
@@ -10173,32 +10174,43 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
BuiltinID == clang::AArch64::BI__builtin_arm_wsr ||
BuiltinID == clang::AArch64::BI__builtin_arm_wsr64 ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_wsr128 ||
BuiltinID == clang::AArch64::BI__builtin_arm_wsrp) {
SpecialRegisterAccessKind AccessKind = Write;
if (BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
BuiltinID == clang::AArch64::BI__builtin_arm_rsr64 ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
BuiltinID == clang::AArch64::BI__builtin_arm_rsrp)
AccessKind = VolatileRead;
bool IsPointerBuiltin = BuiltinID == clang::AArch64::BI__builtin_arm_rsrp ||
BuiltinID == clang::AArch64::BI__builtin_arm_wsrp;
- bool Is64Bit = BuiltinID != clang::AArch64::BI__builtin_arm_rsr &&
- BuiltinID != clang::AArch64::BI__builtin_arm_wsr;
+ bool Is32Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_wsr;
+
+ bool Is128Bit = BuiltinID == clang::AArch64::BI__builtin_arm_rsr128 ||
+ BuiltinID == clang::AArch64::BI__builtin_arm_wsr128;
llvm::Type *ValueType;
llvm::Type *RegisterType = Int64Ty;
- if (IsPointerBuiltin) {
+ if (Is32Bit) {
+ ValueType = Int32Ty;
+ } else if (Is128Bit) {
+ llvm::Type *Int128Ty =
+ llvm::IntegerType::getInt128Ty(CGM.getLLVMContext());
+ ValueType = Int128Ty;
+ RegisterType = Int128Ty;
+ } else if (IsPointerBuiltin) {
ValueType = VoidPtrTy;
- } else if (Is64Bit) {
- ValueType = Int64Ty;
} else {
- ValueType = Int32Ty;
- }
+ ValueType = Int64Ty;
+ };
return EmitSpecialRegisterBuiltin(*this, E, RegisterType, ValueType,
AccessKind);
diff --git a/clang/lib/Headers/arm_acle.h b/clang/lib/Headers/arm_acle.h
index b30010274392c..469f24eadaf33 100644
--- a/clang/lib/Headers/arm_acle.h
+++ b/clang/lib/Headers/arm_acle.h
@@ -712,11 +712,17 @@ __arm_st64bv0(void *__addr, data512_t __value) {
/* 10.1 Special register intrinsics */
#define __arm_rsr(sysreg) __builtin_arm_rsr(sysreg)
#define __arm_rsr64(sysreg) __builtin_arm_rsr64(sysreg)
+#if __ARM_FEATURE_SYSREG128
+#define __arm_rsr128(sysreg) __builtin_arm_rsr128(sysreg)
+#endif
#define __arm_rsrp(sysreg) __builtin_arm_rsrp(sysreg)
#define __arm_rsrf(sysreg) __builtin_bit_cast(float, __arm_rsr(sysreg))
#define __arm_rsrf64(sysreg) __builtin_bit_cast(double, __arm_rsr64(sysreg))
#define __arm_wsr(sysreg, v) __builtin_arm_wsr(sysreg, v)
#define __arm_wsr64(sysreg, v) __builtin_arm_wsr64(sysreg, v)
+#if __ARM_FEATURE_SYSREG128
+#define __arm_wsr128(sysreg, v) __builtin_arm_wsr128(sysreg, v)
+#endif
#define __arm_wsrp(sysreg, v) __builtin_arm_wsrp(sysreg, v)
#define __arm_wsrf(sysreg, v) __arm_wsr(sysreg, __builtin_bit_cast(uint32_t, v))
#define __arm_wsrf64(sysreg, v) __arm_wsr64(sysreg, __builtin_bit_cast(uint64_t, v))
diff --git a/clang/test/CodeGen/arm_acle.c b/clang/test/CodeGen/arm_acle.c
index d85f60e196119..3697d297f89f1 100644
--- a/clang/test/CodeGen/arm_acle.c
+++ b/clang/test/CodeGen/arm_acle.c
@@ -3,6 +3,8 @@
// RUN: %clang_cc1 -ffreestanding -Wno-error=implicit-function-declaration -triple aarch64-none-eabi -target-feature +neon -target-feature +crc -target-feature +crypto -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s -check-prefixes=ARM,AArch64
// RUN: %clang_cc1 -ffreestanding -triple aarch64-none-eabi -target-feature +v8.3a -target-feature +crc -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s -check-prefixes=ARM,AArch64,AArch6483
// RUN: %clang_cc1 -ffreestanding -triple aarch64-none-eabi -target-feature +v8.5a -target-feature +crc -target-feature +rand -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s -check-prefixes=ARM,AArch64,AArch6483,AArch6485
+// RUN: %clang_cc1 -ffreestanding -triple aarch64-none-eabi -target-feature +v9.4a -target-feature +crc -target-feature +rand -target-feature +d128 -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s -check-prefixes=ARM,AArch64,AArch6483,AArch6485,AArch6494D128
+
#include <arm_acle.h>
@@ -1479,6 +1481,17 @@ uint64_t test_rsr64() {
#endif
}
+#ifdef __ARM_FEATURE_SYSREG128
+// AArch6494D128-LABEL: @test_rsr128(
+// AArch6494D128-NEXT: entry:
+// AArch6494D128-NEXT: [[TMP0:%.*]] = call i128 @llvm.read_volatile_register.i128(metadata [[META8]])
+// AArch6494D128-NEXT: ret i128 [[TMP0]]
+//
+__uint128_t test_rsr128() {
+ return __arm_rsr128("1:2:3:4:5");
+}
+#endif
+
// AArch32-LABEL: @test_rsrp(
// AArch32-NEXT: entry:
// AArch32-NEXT: [[TMP0:%.*]] = call i32 @llvm.read_volatile_register.i32(metadata [[META11:![0-9]+]])
@@ -1532,6 +1545,18 @@ void test_wsr64(uint64_t v) {
#endif
}
+#ifdef __ARM_FEATURE_SYSREG128
+// AArch6494D128-LABEL: @test_wsr128(
+// AArch6494D128-NEXT: entry:
+// AArch6494D128-NEXT: call void @llvm.write_register.i128(metadata [[META8]], i128 [[V:%.*]])
+// AArch6494D128-NEXT: ret void
+//
+void test_wsr128(__uint128_t v) {
+ __arm_wsr128("1:2:3:4:5", v);
+
+}
+#endif
+
// AArch32-LABEL: @test_wsrp(
// AArch32-NEXT: entry:
// AArch32-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[V:%.*]] to i32
diff --git a/clang/test/Preprocessor/aarch64-target-features.c b/clang/test/Preprocessor/aarch64-target-features.c
index e0004ab9fdb6c..40ea003a85848 100644
--- a/clang/test/Preprocessor/aarch64-target-features.c
+++ b/clang/test/Preprocessor/aarch64-target-features.c
@@ -539,6 +539,13 @@
// CHECK-MOPS: __ARM_FEATURE_MOPS 1
// CHECK-NOMOPS-NOT: __ARM_FEATURE_MOPS 1
+// ================== Check Armv8.9-A/Armv9.4-A 128-bit System Registers (FEAT_SYSREG128)
+// RUN: %clang -target aarch64-arm-none-eabi -march=armv8.9-a -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-NOSYS128 %s
+// RUN: %clang -target aarch64-arm-none-eabi -march=armv9.4-a -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-NOSYS128 %s
+// RUN: %clang -target aarch64-arm-none-eabi -march=armv9.4-a+d128 -x c -E -dM %s -o - | FileCheck --check-prefix=CHECK-SYS128 %s
+// CHECK-SYS128: __ARM_FEATURE_SYSREG128 1
+// CHECK-NOSYS128-NOT: __ARM_FEATURE_SYSREG128 1
+
// ================== Check default macros for Armv8.1-A and later
// RUN: %clang -target aarch64-arm-none-eabi -march=armv8.1-a -x c -E -dM %s -o - | FileCheck --check-prefixes=CHECK-V81-OR-LATER,CHECK-BEFORE-V83,CHECK-BEFORE-V85 %s
// RUN: %clang -target aarch64-arm-none-eabi -march=armv8.2-a -x c -E -dM %s -o - | FileCheck --check-prefixes=CHECK-V81-OR-LATER,CHECK-BEFORE-V83,CHECK-BEFORE-V85 %s
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 87fd2e2fb407a..5ed6293c4c1ab 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -14,6 +14,7 @@
#include "AArch64TargetMachine.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "llvm/ADT/APSInt.h"
+#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/SelectionDAGISel.h"
#include "llvm/IR/Function.h" // To access function attributes.
#include "llvm/IR/GlobalValue.h"
@@ -3412,41 +3413,56 @@ bool AArch64DAGToDAGISel::tryReadRegister(SDNode *N) {
const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
SDLoc DL(N);
- int Reg = getIntOperandFromRegisterString(RegString->getString());
- if (Reg != -1) {
- ReplaceNode(N, CurDAG->getMachineNode(
- AArch64::MRS, DL, N->getSimpleValueType(0), MVT::Other,
- CurDAG->getTargetConstant(Reg, DL, MVT::i32),
- N->getOperand(0)));
- return true;
- }
-
- // Use the sysreg mapper to map the remaining possible strings to the
- // value for the register to be used for the instruction operand.
- auto TheReg = AArch64SysReg::lookupSysRegByName(RegString->getString());
- if (TheReg && TheReg->Readable &&
- TheReg->haveFeatures(Subtarget->getFeatureBits()))
- Reg = TheReg->Encoding;
- else
- Reg = AArch64SysReg::parseGenericRegister(RegString->getString());
-
- if (Reg != -1) {
- ReplaceNode(N, CurDAG->getMachineNode(
- AArch64::MRS, DL, N->getSimpleValueType(0), MVT::Other,
- CurDAG->getTargetConstant(Reg, DL, MVT::i32),
- N->getOperand(0)));
- return true;
- }
+ bool ReadIs128Bit = N->getOpcode() == AArch64ISD::MRRS;
+
+ unsigned Opcode64Bit = AArch64::MRS;
+ int Imm = getIntOperandFromRegisterString(RegString->getString());
+ if (Imm == -1) {
+ // No match, Use the sysreg mapper to map the remaining possible strings to
+ // the value for the register to be used for the instruction operand.
+ const auto *TheReg =
+ AArch64SysReg::lookupSysRegByName(RegString->getString());
+ if (TheReg && TheReg->Readable &&
+ TheReg->haveFeatures(Subtarget->getFeatureBits()))
+ Imm = TheReg->Encoding;
+ else
+ Imm = AArch64SysReg::parseGenericRegister(RegString->getString());
- if (RegString->getString() == "pc") {
- ReplaceNode(N, CurDAG->getMachineNode(
- AArch64::ADR, DL, N->getSimpleValueType(0), MVT::Other,
- CurDAG->getTargetConstant(0, DL, MVT::i32),
- N->getOperand(0)));
- return true;
+ if (Imm == -1) {
+ // Still no match, see if this is "pc" or give up.
+ if (!ReadIs128Bit && RegString->getString() == "pc") {
+ Opcode64Bit = AArch64::ADR;
+ Imm = 0;
+ } else {
+ return false;
+ }
+ }
}
- return false;
+ SDValue InChain = N->getOperand(0);
+ SDValue SysRegImm = CurDAG->getTargetConstant(Imm, DL, MVT::i32);
+ if (!ReadIs128Bit) {
+ CurDAG->SelectNodeTo(N, Opcode64Bit, MVT::i64, MVT::Other /* Chain */,
+ {SysRegImm, InChain});
+ } else {
+ SDNode *MRRS = CurDAG->getMachineNode(
+ AArch64::MRRS, DL,
+ {MVT::Untyped /* XSeqPair */, MVT::Other /* Chain */},
+ {SysRegImm, InChain});
+
+ // Sysregs are not endian. The even register always contains the low half
+ // of the register.
+ SDValue Lo = CurDAG->getTargetExtractSubreg(AArch64::sube64, DL, MVT::i64,
+ SDValue(MRRS, 0));
+ SDValue Hi = CurDAG->getTargetExtractSubreg(AArch64::subo64, DL, MVT::i64,
+ SDValue(MRRS, 0));
+ SDValue OutChain = SDValue(MRRS, 1);
+
+ ReplaceUses(SDValue(N, 0), Lo);
+ ReplaceUses(SDValue(N, 1), Hi);
+ ReplaceUses(SDValue(N, 2), OutChain);
+ };
+ return true;
}
// Lower the write_register intrinsic to an MSR instruction node if the special
@@ -3458,60 +3474,75 @@ bool AArch64DAGToDAGISel::tryWriteRegister(SDNode *N) {
const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
SDLoc DL(N);
- int Reg = getIntOperandFromRegisterString(RegString->getString());
- if (Reg != -1) {
- ReplaceNode(
- N, CurDAG->getMachineNode(AArch64::MSR, DL, MVT::Other,
- CurDAG->getTargetConstant(Reg, DL, MVT::i32),
- N->getOperand(2), N->getOperand(0)));
- return true;
+ bool WriteIs128Bit = N->getOpcode() == AArch64ISD::MSRR;
+
+ if (!WriteIs128Bit) {
+ // Check if the register was one of those allowed as the pstatefield value
+ // in the MSR (immediate) instruction. To accept the values allowed in the
+ // pstatefield for the MSR (immediate) instruction, we also require that an
+ // immediate value has been provided as an argument, we know that this is
+ // the case as it has been ensured by semantic checking.
+ auto PMapper = AArch64PState::lookupPStateByName(RegString->getString());
+ if (PMapper) {
+ assert(isa<ConstantSDNode>(N->getOperand(2)) &&
+ "Expected a constant integer expression.");
+ unsigned Reg = PMapper->Encoding;
+ uint64_t Immed = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
+ unsigned State;
+ if (Reg == AArch64PState::PAN || Reg == AArch64PState::UAO ||
+ Reg == AArch64PState::SSBS) {
+ assert(Immed < 2 && "Bad imm");
+ State = AArch64::MSRpstateImm1;
+ } else {
+ assert(Immed < 16 && "Bad imm");
+ State = AArch64::MSRpstateImm4;
+ }
+ CurDAG->SelectNodeTo(
+ N, State, MVT::Other, CurDAG->getTargetConstant(Reg, DL, MVT::i32),
+ CurDAG->getTargetConstant(Immed, DL, MVT::i16), N->getOperand(0));
+ return true;
+ }
}
- // Check if the register was one of those allowed as the pstatefield value in
- // the MSR (immediate) instruction. To accept the values allowed in the
- // pstatefield for the MSR (immediate) instruction, we also require that an
- // immediate value has been provided as an argument, we know that this is
- // the case as it has been ensured by semantic checking.
- auto PMapper = AArch64PState::lookupPStateByName(RegString->getString());
- if (PMapper) {
- assert (isa<ConstantSDNode>(N->getOperand(2))
- && "Expected a constant integer expression.");
- unsigned Reg = PMapper->Encoding;
- uint64_t Immed = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
- unsigned State;
- if (Reg == AArch64PState::PAN || Reg == AArch64PState::UAO || Reg == AArch64PState::SSBS) {
- assert(Immed < 2 && "Bad imm");
- State = AArch64::MSRpstateImm1;
- } else {
- assert(Immed < 16 && "Bad imm");
- State = AArch64::MSRpstateImm4;
- }
- ReplaceNode(N, CurDAG->getMachineNode(
- State, DL, MVT::Other,
- CurDAG->getTargetConstant(Reg, DL, MVT::i32),
- CurDAG->getTargetConstant(Immed, DL, MVT::i16),
- N->getOperand(0)));
- return true;
+ int Imm = getIntOperandFromRegisterString(RegString->getString());
+ if (Imm == -1) {
+ // Use the sysreg mapper to attempt to map the remaining possible strings
+ // to the value for the register to be used for the MSR (register)
+ // instruction operand.
+ auto TheReg = AArch64SysReg::lookupSysRegByName(RegString->getString());
+ if (TheReg && TheReg->Writeable &&
+ TheReg->haveFeatures(Subtarget->getFeatureBits()))
+ Imm = TheReg->Encoding;
+ else
+ Imm = AArch64SysReg::parseGenericRegister(RegString->getString());
+
+ if (Imm == -1)
+ return false;
}
- // Use the sysreg mapper to attempt to map the remaining possible strings
- // to the value for the register to be used for the MSR (register)
- // instruction operand.
- auto TheReg = AArch64SysReg::lookupSysRegByName(RegString->getString());
- if (TheReg && TheReg->Writeable &&
- TheReg->haveFeatures(Subtarget->getFeatureBits()))
- Reg = TheReg->Encoding;
- else
- Reg = AArch64SysReg::parseGenericRegister(RegString->getString());
- if (Reg != -1) {
- ReplaceNode(N, CurDAG->getMachineNode(
- AArch64::MSR, DL, MVT::Other,
- CurDAG->getTargetConstant(Reg, DL, MVT::i32),
- N->getOperand(2), N->getOperand(0)));
- return true;
+ SDValue InChain = N->getOperand(0);
+ if (!WriteIs128Bit) {
+ CurDAG->SelectNodeTo(N, AArch64::MSR, MVT::Other,
+ CurDAG->getTargetConstant(Imm, DL, MVT::i32),
+ N->getOperand(2), InChain);
+ } else {
+ // No endian swap. The lower half always goes into the even subreg, and the
+ // higher half always into the odd supreg.
+ SDNode *Pair = CurDAG->getMachineNode(
+ TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped /* XSeqPair */,
+ {CurDAG->getTargetConstant(AArch64::XSeqPairsClassRegClass.getID(), DL,
+ MVT::i32),
+ N->getOperand(2),
+ CurDAG->getTargetConstant(AArch64::sube64, DL, MVT::i32),
+ N->getOperand(3),
+ CurDAG->getTargetConstant(AArch64::subo64, DL, MVT::i32)});
+
+ CurDAG->SelectNodeTo(N, AArch64::MSRR, MVT::Other,
+ CurDAG->getTargetConstant(Imm, DL, MVT::i32),
+ SDValue(Pair, 0), InChain);
}
- return false;
+ return true;
}
/// We've got special pseudo-instructions for these
@@ -3870,11 +3901,13 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
break;
case ISD::READ_REGISTER:
+ case AArch64ISD::MRRS:
if (tryReadRegister(Node))
return;
break;
case ISD::WRITE_REGISTER:
+ case AArch64ISD::MSRR:
if (tryWriteRegister(Node))
return;
break;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index da3c702525a75..3410271946f27 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1627,6 +1627,11 @@ void AArch64TargetLowering::addTypeForNEON(MVT VT) {
setIndexedStoreAction(im, VT, Legal);
}
}
+
+ if (Subtarget->hasD128()) {
+ setOperationAction(ISD::READ_REGISTER, MVT::i128, Custom);
+ setOperationAction(ISD::WRITE_REGISTER, MVT::i128, Custom);
+ }
}
bool AArch64TargetLowering::shouldExpandGetActiveLaneMask(EVT ResVT,
@@ -2491,6 +2496,8 @@ const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
MAKE_CASE(AArch64ISD::MOPS_MEMCOPY)
MAKE_CASE(AArch64ISD::MOPS_MEMMOVE)
MAKE_CASE(AArch64ISD::CALL_BTI)
+ MAKE_CASE(AArch64ISD::MRRS)
+ MAKE_CASE(AArch64ISD::MSRR)
}
#undef MAKE_CASE
return nullptr;
@@ -5943,6 +5950,26 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
return DAG.getNode(Op.getOpcode(), DL, {Op.getValueType(), MVT::Other},
{Ext.getValue(1), Ext.getValue(0)});
}
+ case ISD::WRITE_REGISTER: {
+ assert(Op.getOperand(2).getValueType() == MVT::i128 &&
+ "WRITE_REGISTER custom lowering is only for 128-bit sysregs");
+ SDLoc DL(Op);
+
+ SDValue Chain = Op.getOperand(0);
+ SDValue SysRegName = Op.getOperand(1);
+ SDValue Pair = Op.getOperand(2);
+
+ SDValue PairLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, Pair,
+ DAG.getConstant(0, DL, MVT::i32));
+ SDValue PairHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, Pair,
+ DAG.getConstant(1, DL, MVT::i32));
+
+ // chain = MSRR(chain, sysregname, lo, hi)
+ SDValue Result = DAG.getNode(AArch64ISD::MSRR, DL, MVT::Other, Chain,
+ SysRegName, PairLo, PairHi);
+
+ return Result;
+ }
}
}
@@ -21734,6 +21761,26 @@ void AArch64TargetLowering::ReplaceNodeResults(
}
}
}
+ case ISD::READ_REGISTER: {
+ SDLoc DL(N);
+ EVT VT = N->getValueType(0);
+ assert(VT == MVT::i128 &&
+ "READ_REGISTER custom lowering is only for 128-bit sysregs");
+ SDValue Chain = N->getOperand(0);
+ SDValue SysRegName = N->getOperand(1);
+
+ SDValue Result = DAG.getNode(
+ AArch64ISD::MRRS, DL, DAG.getVTList({MVT::i64, MVT::i64, MVT::Other}),
+ Chain, SysRegName);
+
+ // Sysregs are not endian. Result.getValue(0) always contains the lower half
+ // of the 128-bit System Register value.
+ SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i128,
+ Result.getValue(0), Result.getValue(1));
+ Results.push_back(Pair);
+ Results.push_back(Result.getValue(2)); // Chain
+ return;
+ }
}
}
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 54d287915e5fd..112e88535aae0 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -430,6 +430,12 @@ enum NodeType : unsigned {
// the caller
ASSERT_ZEXT_BOOL,
+ // 128-bit system register accesses
+ // lo64, hi64, chain = MRRS(chain, sysregname)
+ MRRS,
+ // chain = MSRR(chain, sysregname, lo64, hi64)
+ MSRR,
+
// Strict (exception-raising) floating point comparison
STRICT_FCMP = ISD::FIRST_TARGET_STRICTFP_OPCODE,
STRICT_FCMPE,
diff --git a/llvm/test/CodeGen/AArch64/aarch64-sysreg128.ll b/llvm/test/CodeGen/AArch64/aarch64-sysreg128.ll
new file mode 100644
index 0000000000000..7f20b5e5ee4df
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64-sysreg128.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple aarch64 -mattr=+d128 | FileCheck %s --check-prefixes=CHECK-LE
+; RUN: llc < %s -mtriple aarch64_be -mattr=+d128 | FileCheck %s --check-prefixes=CHECK-BE
+
+define i128 @test_rsr128() #0 {
+; CHECK-LE-LABEL: test_rsr128:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: mrrs x0, x1, S1_2_C3_C4_5
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_rsr128:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: mrrs x2, x3, S1_2_C3_C4_5
+; CHECK-BE-NEXT: mov x0, x3
+; CHECK-BE-NEXT: mov x1, x2
+; CHECK-BE-NEXT: ret
+entry:
+ %0 = call i128 @llvm.read_volatile_register.i128(metadata !1)
+ ret i128 %0
+}
+
+declare i128 @llvm.read_volatile_register.i128(metadata) #1
+
+define void @test_wsr128(i128 noundef %v) #0 {
+; CHECK-LE-LABEL: test_wsr128:
+; CHECK-LE: // %bb.0: // %entry
+; CHECK-LE-NEXT: // kill: def $x1 killed $x1 killed $x0_x1 def $x0_x1
+; CHECK-LE-NEXT: // kill: def $x0 killed $x0 killed $x0_x1 def $x0_x1
+; CHECK-LE-NEXT: msrr S1_2_C3_C4_5, x0, x1
+; CHECK-LE-NEXT: ret
+;
+; CHECK-BE-LABEL: test_wsr128:
+; CHECK-BE: // %bb.0: // %entry
+; CHECK-BE-NEXT: mov x2, x1
+; CHECK-BE-NEXT: mov x3, x0
+; CHECK-BE-NEXT: msrr S1_2_C3_C4_5, x2, x3
+; CHECK-BE-NEXT: ret
+entry:
+ call void @llvm.write_register.i128(metadata !1, i128 %v)
+ ret void
+}
+
+declare void @llvm.write_register.i128(metadata, i128) #1
+
+attributes #0 = { noinline nounwind }
+attributes #1 = { nounwind }
+
+!1 = !{!"1:2:3:4:5"}
More information about the cfe-commits
mailing list