[llvm] [AArch64][GISel] Add legalizer support for @llvm.umul.with.overflow.i128 (PR #170101)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Dec 1 03:08:53 PST 2025
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: None (ayank227)
<details>
<summary>Changes</summary>
This matches the similar pattern how SelectionDAG handles this. I have also added split_store_128 to optimize i128 G_STORE(G_MERGE_VALUES(x, y)) into two i64 G_STOREs.
---
Patch is 47.92 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/170101.diff
15 Files Affected:
- (modified) llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h (+1)
- (modified) llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp (+94)
- (modified) llvm/lib/Target/AArch64/AArch64Combine.td (+10-1)
- (modified) llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp (+63)
- (modified) llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic-128.ll (+15-45)
- (modified) llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll (-13)
- (modified) llvm/test/CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll (+11-31)
- (modified) llvm/test/CodeGen/AArch64/aarch64-fold-lslfast.ll (+11-15)
- (modified) llvm/test/CodeGen/AArch64/arm64-sli-sri-opt.ll (+5-13)
- (modified) llvm/test/CodeGen/AArch64/dup.ll (+30-33)
- (modified) llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll (+39-63)
- (modified) llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll (+39-63)
- (modified) llvm/test/CodeGen/AArch64/i128_with_overflow.ll (+82-37)
- (modified) llvm/test/CodeGen/AArch64/insertextract.ll (+19-23)
- (modified) llvm/test/CodeGen/AArch64/store.ll (+2-5)
``````````diff
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index a458cbd94ccb1..5cbbf719c5504 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -426,6 +426,7 @@ class LegalizerHelper {
LLVM_ABI LegalizeResult narrowScalarAddSub(MachineInstr &MI, unsigned TypeIdx,
LLT NarrowTy);
LLVM_ABI LegalizeResult narrowScalarMul(MachineInstr &MI, LLT Ty);
+ LLVM_ABI LegalizeResult narrowScalarMULO(MachineInstr &MI, LLT Ty);
LLVM_ABI LegalizeResult narrowScalarFPTOI(MachineInstr &MI, unsigned TypeIdx,
LLT Ty);
LLVM_ABI LegalizeResult narrowScalarExtract(MachineInstr &MI,
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 120c38ab8404c..8c682923d47a9 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -1669,6 +1669,8 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
case TargetOpcode::G_MUL:
case TargetOpcode::G_UMULH:
return narrowScalarMul(MI, NarrowTy);
+ case TargetOpcode::G_UMULO:
+ return narrowScalarMULO(MI, NarrowTy);
case TargetOpcode::G_EXTRACT:
return narrowScalarExtract(MI, TypeIdx, NarrowTy);
case TargetOpcode::G_INSERT:
@@ -7193,6 +7195,98 @@ LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) {
return Legalized;
}
+// Narrow unsigned multiplication with overflow (G_UMULO).
+LegalizerHelper::LegalizeResult
+LegalizerHelper::narrowScalarMULO(MachineInstr &MI, LLT NarrowTy) {
+ auto [DstReg, OverflowReg, Src1, Src2] = MI.getFirst4Regs();
+
+ LLT Ty = MRI.getType(DstReg);
+ if (Ty.isVector())
+ return UnableToLegalize;
+
+ unsigned Size = Ty.getSizeInBits();
+ unsigned NarrowSize = NarrowTy.getSizeInBits();
+ if (Size % NarrowSize != 0)
+ return UnableToLegalize;
+
+ unsigned NumParts = Size / NarrowSize;
+ if (NumParts != 2)
+ return UnableToLegalize; // Only handle i128→i64 narrowing
+
+ // Split inputs into high/low parts
+ SmallVector<Register, 2> Src1Parts, Src2Parts;
+ extractParts(Src1, NarrowTy, NumParts, Src1Parts, MIRBuilder, MRI);
+ extractParts(Src2, NarrowTy, NumParts, Src2Parts, MIRBuilder, MRI);
+
+ Register LHSLo = Src1Parts[0];
+ Register LHSHi = Src1Parts[1];
+ Register RHSLo = Src2Parts[0];
+ Register RHSHi = Src2Parts[1];
+
+ // Check if both high parts are non-zero → guaranteed overflow
+ Register Zero = MIRBuilder.buildConstant(NarrowTy, 0).getReg(0);
+ Register LHSHiNZ =
+ MIRBuilder.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), LHSHi, Zero)
+ .getReg(0);
+ Register RHSHiNZ =
+ MIRBuilder.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), RHSHi, Zero)
+ .getReg(0);
+ Register BothHiNonZero =
+ MIRBuilder.buildAnd(LLT::scalar(1), LHSHiNZ, RHSHiNZ).getReg(0);
+
+ // Cross multiply LHSHi × RHSLo with overflow (use MUL+UMULH directly)
+ Register Mid1 = MIRBuilder.buildMul(NarrowTy, LHSHi, RHSLo).getReg(0);
+ Register Mid1Hi = MIRBuilder.buildUMulH(NarrowTy, LHSHi, RHSLo).getReg(0);
+ Register Ovf1 =
+ MIRBuilder.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Mid1Hi, Zero)
+ .getReg(0);
+
+ // Cross multiply LHSLo × RHSHi with overflow (use MUL+UMULH directly)
+ Register Mid2 = MIRBuilder.buildMul(NarrowTy, LHSLo, RHSHi).getReg(0);
+ Register Mid2Hi = MIRBuilder.buildUMulH(NarrowTy, LHSLo, RHSHi).getReg(0);
+ Register Ovf2 =
+ MIRBuilder.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Mid2Hi, Zero)
+ .getReg(0);
+
+ // Add the cross products (HighSum = Mid1 + Mid2)
+ Register HighSum = MIRBuilder.buildAdd(NarrowTy, Mid1, Mid2).getReg(0);
+
+ // Multiply low parts to get full 128-bit result (using ZEXT pattern)
+ LLT WideTy = LLT::scalar(Size);
+ Register LHSLoExt = MIRBuilder.buildZExt(WideTy, LHSLo).getReg(0);
+ Register RHSLoExt = MIRBuilder.buildZExt(WideTy, RHSLo).getReg(0);
+ Register FullMul = MIRBuilder.buildMul(WideTy, LHSLoExt, RHSLoExt).getReg(0);
+
+ SmallVector<Register, 2> LowMulParts;
+ extractParts(FullMul, NarrowTy, NumParts, LowMulParts, MIRBuilder, MRI);
+ Register ResLo = LowMulParts[0];
+ Register ResHi = LowMulParts[1];
+
+ // Add HighSum to ResHi with overflow detection
+ auto AddHighSum =
+ MIRBuilder.buildUAddo(NarrowTy, LLT::scalar(1), ResHi, HighSum);
+ Register FinalHi = AddHighSum.getReg(0);
+ Register Ovf3 = AddHighSum.getReg(1);
+
+ // Combine all overflow flags
+ // overflow = BothHiNonZero || Ovf1 || Ovf2 || Ovf3
+ Register Ovf12 = MIRBuilder.buildOr(LLT::scalar(1), Ovf1, Ovf2).getReg(0);
+ Register Ovf123 = MIRBuilder.buildOr(LLT::scalar(1), Ovf12, Ovf3).getReg(0);
+ Register FinalOvf =
+ MIRBuilder.buildOr(LLT::scalar(1), BothHiNonZero, Ovf123).getReg(0);
+
+ // Build final result
+ // Emit G_MERGE_VALUES for the result
+ SmallVector<Register, 2> ResultParts = {ResLo, FinalHi};
+ MIRBuilder.buildMergeLikeInstr(DstReg, ResultParts);
+
+ // Normalize overflow to s1 type
+ MIRBuilder.buildCopy(OverflowReg, FinalOvf);
+
+ MI.eraseFromParent();
+ return Legalized;
+}
+
LegalizerHelper::LegalizeResult
LegalizerHelper::narrowScalarFPTOI(MachineInstr &MI, unsigned TypeIdx,
LLT NarrowTy) {
diff --git a/llvm/lib/Target/AArch64/AArch64Combine.td b/llvm/lib/Target/AArch64/AArch64Combine.td
index 278314792bfb9..32955f3a6f952 100644
--- a/llvm/lib/Target/AArch64/AArch64Combine.td
+++ b/llvm/lib/Target/AArch64/AArch64Combine.td
@@ -280,6 +280,14 @@ def form_truncstore : GICombineRule<
(apply [{ applyFormTruncstore(*${root}, MRI, B, Observer, ${matchinfo}); }])
>;
+def split_store_128_matchdata : GIDefMatchData<"std::pair<Register, Register>">;
+def split_store_128 : GICombineRule<
+ (defs root:$root, split_store_128_matchdata:$matchinfo),
+ (match (G_STORE $src, $addr):$root,
+ [{ return matchSplitStore128(*${root}, MRI, ${matchinfo}); }]),
+ (apply [{ applySplitStore128(*${root}, MRI, B, Observer, ${matchinfo}); }])
+>;
+
def fold_merge_to_zext : GICombineRule<
(defs root:$d),
(match (wip_match_opcode G_MERGE_VALUES):$d,
@@ -339,7 +347,8 @@ def AArch64PostLegalizerLowering
: GICombiner<"AArch64PostLegalizerLoweringImpl",
[shuffle_vector_lowering, vashr_vlshr_imm,
icmp_lowering, build_vector_lowering,
- lower_vector_fcmp, form_truncstore, fconstant_to_constant,
+ lower_vector_fcmp, form_truncstore, split_store_128,
+ fconstant_to_constant,
vector_sext_inreg_to_shift,
unmerge_ext_to_unmerge, lower_mulv2s64,
vector_unmerge_lowering, insertelt_nonconst,
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
index 4fba593b3d0fb..7152558580763 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
@@ -1119,6 +1119,69 @@ void applyFormTruncstore(MachineInstr &MI, MachineRegisterInfo &MRI,
Observer.changedInstr(MI);
}
+/// Optimize i128 stores by splitting into two i64 stores for STP pairing
+bool matchSplitStore128(MachineInstr &MI, MachineRegisterInfo &MRI,
+ std::pair<Register, Register> &Parts) {
+ assert(MI.getOpcode() == TargetOpcode::G_STORE);
+ GStore &Store = cast<GStore>(MI);
+
+ Register ValueReg = Store.getValueReg();
+ LLT ValueTy = MRI.getType(ValueReg);
+
+ // Only handle scalar types
+ if (!ValueTy.isScalar())
+ return false;
+
+ if (ValueTy.getSizeInBits() != 128)
+ return false;
+
+ // Check if the value comes from G_MERGE_VALUES
+ MachineInstr *DefMI = MRI.getVRegDef(ValueReg);
+ if (!DefMI || DefMI->getOpcode() != TargetOpcode::G_MERGE_VALUES)
+ return false;
+
+ // Get the two i64 parts
+ if (DefMI->getNumOperands() != 3) // Dst + 2 sources
+ return false;
+
+ Register Part0 = DefMI->getOperand(1).getReg();
+ Register Part1 = DefMI->getOperand(2).getReg();
+
+ if (MRI.getType(Part0) != LLT::scalar(64) ||
+ MRI.getType(Part1) != LLT::scalar(64))
+ return false;
+
+ Parts = {Part0, Part1};
+ return true;
+}
+
+void applySplitStore128(MachineInstr &MI, MachineRegisterInfo &MRI,
+ MachineIRBuilder &B, GISelChangeObserver &Observer,
+ std::pair<Register, Register> &Parts) {
+ assert(MI.getOpcode() == TargetOpcode::G_STORE);
+ GStore &Store = cast<GStore>(MI);
+
+ B.setInstrAndDebugLoc(MI);
+
+ Register PtrReg = Store.getPointerReg();
+ MachineMemOperand &MMO = Store.getMMO();
+
+ // Create two i64 stores
+ // Store low part at [ptr]
+ B.buildStore(Parts.first, PtrReg, MMO.getPointerInfo(), MMO.getAlign(),
+ MMO.getFlags());
+
+ // Calculate offset for high part: ptr + 8
+ auto Offset = B.buildConstant(LLT::scalar(64), 8);
+ auto PtrHi = B.buildPtrAdd(MRI.getType(PtrReg), PtrReg, Offset);
+
+ // Store high part at [ptr + 8]
+ B.buildStore(Parts.second, PtrHi, MMO.getPointerInfo().getWithOffset(8),
+ commonAlignment(MMO.getAlign(), 8), MMO.getFlags());
+
+ MI.eraseFromParent();
+}
+
// Lower vector G_SEXT_INREG back to shifts for selection. We allowed them to
// form in the first place for combine opportunities, so any remaining ones
// at this stage need be lowered back.
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic-128.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic-128.ll
index be51210882eaa..ac18249950e9e 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic-128.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-atomic-128.ll
@@ -27,9 +27,7 @@ define void @val_compare_and_swap(ptr %p, i128 %oldval, i128 %newval) {
; CHECK-LLSC-O1-NEXT: stxp w10, x4, x5, [x0]
; CHECK-LLSC-O1-NEXT: cbnz w10, .LBB0_1
; CHECK-LLSC-O1-NEXT: .LBB0_4:
-; CHECK-LLSC-O1-NEXT: mov v0.d[0], x8
-; CHECK-LLSC-O1-NEXT: mov v0.d[1], x9
-; CHECK-LLSC-O1-NEXT: str q0, [x0]
+; CHECK-LLSC-O1-NEXT: stp x8, x9, [x0]
; CHECK-LLSC-O1-NEXT: ret
;
; CHECK-OUTLINE-LLSC-O1-LABEL: val_compare_and_swap:
@@ -45,9 +43,7 @@ define void @val_compare_and_swap(ptr %p, i128 %oldval, i128 %newval) {
; CHECK-OUTLINE-LLSC-O1-NEXT: mov x3, x5
; CHECK-OUTLINE-LLSC-O1-NEXT: mov x4, x19
; CHECK-OUTLINE-LLSC-O1-NEXT: bl __aarch64_cas16_acq
-; CHECK-OUTLINE-LLSC-O1-NEXT: mov v0.d[0], x0
-; CHECK-OUTLINE-LLSC-O1-NEXT: mov v0.d[1], x1
-; CHECK-OUTLINE-LLSC-O1-NEXT: str q0, [x19]
+; CHECK-OUTLINE-LLSC-O1-NEXT: stp x0, x1, [x19]
; CHECK-OUTLINE-LLSC-O1-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload
; CHECK-OUTLINE-LLSC-O1-NEXT: ret
;
@@ -58,9 +54,7 @@ define void @val_compare_and_swap(ptr %p, i128 %oldval, i128 %newval) {
; CHECK-CAS-O1-NEXT: // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3
; CHECK-CAS-O1-NEXT: // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5
; CHECK-CAS-O1-NEXT: caspa x2, x3, x4, x5, [x0]
-; CHECK-CAS-O1-NEXT: mov v0.d[0], x2
-; CHECK-CAS-O1-NEXT: mov v0.d[1], x3
-; CHECK-CAS-O1-NEXT: str q0, [x0]
+; CHECK-CAS-O1-NEXT: stp x2, x3, [x0]
; CHECK-CAS-O1-NEXT: ret
;
; CHECK-LLSC-O0-LABEL: val_compare_and_swap:
@@ -154,9 +148,7 @@ define void @val_compare_and_swap_monotonic_seqcst(ptr %p, i128 %oldval, i128 %n
; CHECK-LLSC-O1-NEXT: stlxp w10, x4, x5, [x0]
; CHECK-LLSC-O1-NEXT: cbnz w10, .LBB1_1
; CHECK-LLSC-O1-NEXT: .LBB1_4:
-; CHECK-LLSC-O1-NEXT: mov v0.d[0], x8
-; CHECK-LLSC-O1-NEXT: mov v0.d[1], x9
-; CHECK-LLSC-O1-NEXT: str q0, [x0]
+; CHECK-LLSC-O1-NEXT: stp x8, x9, [x0]
; CHECK-LLSC-O1-NEXT: ret
;
; CHECK-OUTLINE-LLSC-O1-LABEL: val_compare_and_swap_monotonic_seqcst:
@@ -172,9 +164,7 @@ define void @val_compare_and_swap_monotonic_seqcst(ptr %p, i128 %oldval, i128 %n
; CHECK-OUTLINE-LLSC-O1-NEXT: mov x3, x5
; CHECK-OUTLINE-LLSC-O1-NEXT: mov x4, x19
; CHECK-OUTLINE-LLSC-O1-NEXT: bl __aarch64_cas16_acq_rel
-; CHECK-OUTLINE-LLSC-O1-NEXT: mov v0.d[0], x0
-; CHECK-OUTLINE-LLSC-O1-NEXT: mov v0.d[1], x1
-; CHECK-OUTLINE-LLSC-O1-NEXT: str q0, [x19]
+; CHECK-OUTLINE-LLSC-O1-NEXT: stp x0, x1, [x19]
; CHECK-OUTLINE-LLSC-O1-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload
; CHECK-OUTLINE-LLSC-O1-NEXT: ret
;
@@ -185,9 +175,7 @@ define void @val_compare_and_swap_monotonic_seqcst(ptr %p, i128 %oldval, i128 %n
; CHECK-CAS-O1-NEXT: // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3
; CHECK-CAS-O1-NEXT: // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5
; CHECK-CAS-O1-NEXT: caspal x2, x3, x4, x5, [x0]
-; CHECK-CAS-O1-NEXT: mov v0.d[0], x2
-; CHECK-CAS-O1-NEXT: mov v0.d[1], x3
-; CHECK-CAS-O1-NEXT: str q0, [x0]
+; CHECK-CAS-O1-NEXT: stp x2, x3, [x0]
; CHECK-CAS-O1-NEXT: ret
;
; CHECK-LLSC-O0-LABEL: val_compare_and_swap_monotonic_seqcst:
@@ -281,9 +269,7 @@ define void @val_compare_and_swap_release_acquire(ptr %p, i128 %oldval, i128 %ne
; CHECK-LLSC-O1-NEXT: stlxp w10, x4, x5, [x0]
; CHECK-LLSC-O1-NEXT: cbnz w10, .LBB2_1
; CHECK-LLSC-O1-NEXT: .LBB2_4:
-; CHECK-LLSC-O1-NEXT: mov v0.d[0], x8
-; CHECK-LLSC-O1-NEXT: mov v0.d[1], x9
-; CHECK-LLSC-O1-NEXT: str q0, [x0]
+; CHECK-LLSC-O1-NEXT: stp x8, x9, [x0]
; CHECK-LLSC-O1-NEXT: ret
;
; CHECK-OUTLINE-LLSC-O1-LABEL: val_compare_and_swap_release_acquire:
@@ -299,9 +285,7 @@ define void @val_compare_and_swap_release_acquire(ptr %p, i128 %oldval, i128 %ne
; CHECK-OUTLINE-LLSC-O1-NEXT: mov x3, x5
; CHECK-OUTLINE-LLSC-O1-NEXT: mov x4, x19
; CHECK-OUTLINE-LLSC-O1-NEXT: bl __aarch64_cas16_acq_rel
-; CHECK-OUTLINE-LLSC-O1-NEXT: mov v0.d[0], x0
-; CHECK-OUTLINE-LLSC-O1-NEXT: mov v0.d[1], x1
-; CHECK-OUTLINE-LLSC-O1-NEXT: str q0, [x19]
+; CHECK-OUTLINE-LLSC-O1-NEXT: stp x0, x1, [x19]
; CHECK-OUTLINE-LLSC-O1-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload
; CHECK-OUTLINE-LLSC-O1-NEXT: ret
;
@@ -312,9 +296,7 @@ define void @val_compare_and_swap_release_acquire(ptr %p, i128 %oldval, i128 %ne
; CHECK-CAS-O1-NEXT: // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3
; CHECK-CAS-O1-NEXT: // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5
; CHECK-CAS-O1-NEXT: caspal x2, x3, x4, x5, [x0]
-; CHECK-CAS-O1-NEXT: mov v0.d[0], x2
-; CHECK-CAS-O1-NEXT: mov v0.d[1], x3
-; CHECK-CAS-O1-NEXT: str q0, [x0]
+; CHECK-CAS-O1-NEXT: stp x2, x3, [x0]
; CHECK-CAS-O1-NEXT: ret
;
; CHECK-LLSC-O0-LABEL: val_compare_and_swap_release_acquire:
@@ -408,9 +390,7 @@ define void @val_compare_and_swap_monotonic(ptr %p, i128 %oldval, i128 %newval)
; CHECK-LLSC-O1-NEXT: stlxp w10, x4, x5, [x0]
; CHECK-LLSC-O1-NEXT: cbnz w10, .LBB3_1
; CHECK-LLSC-O1-NEXT: .LBB3_4:
-; CHECK-LLSC-O1-NEXT: mov v0.d[0], x8
-; CHECK-LLSC-O1-NEXT: mov v0.d[1], x9
-; CHECK-LLSC-O1-NEXT: str q0, [x0]
+; CHECK-LLSC-O1-NEXT: stp x8, x9, [x0]
; CHECK-LLSC-O1-NEXT: ret
;
; CHECK-OUTLINE-LLSC-O1-LABEL: val_compare_and_swap_monotonic:
@@ -426,9 +406,7 @@ define void @val_compare_and_swap_monotonic(ptr %p, i128 %oldval, i128 %newval)
; CHECK-OUTLINE-LLSC-O1-NEXT: mov x3, x5
; CHECK-OUTLINE-LLSC-O1-NEXT: mov x4, x19
; CHECK-OUTLINE-LLSC-O1-NEXT: bl __aarch64_cas16_acq_rel
-; CHECK-OUTLINE-LLSC-O1-NEXT: mov v0.d[0], x0
-; CHECK-OUTLINE-LLSC-O1-NEXT: mov v0.d[1], x1
-; CHECK-OUTLINE-LLSC-O1-NEXT: str q0, [x19]
+; CHECK-OUTLINE-LLSC-O1-NEXT: stp x0, x1, [x19]
; CHECK-OUTLINE-LLSC-O1-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload
; CHECK-OUTLINE-LLSC-O1-NEXT: ret
;
@@ -439,9 +417,7 @@ define void @val_compare_and_swap_monotonic(ptr %p, i128 %oldval, i128 %newval)
; CHECK-CAS-O1-NEXT: // kill: def $x3 killed $x3 killed $x2_x3 def $x2_x3
; CHECK-CAS-O1-NEXT: // kill: def $x5 killed $x5 killed $x4_x5 def $x4_x5
; CHECK-CAS-O1-NEXT: caspal x2, x3, x4, x5, [x0]
-; CHECK-CAS-O1-NEXT: mov v0.d[0], x2
-; CHECK-CAS-O1-NEXT: mov v0.d[1], x3
-; CHECK-CAS-O1-NEXT: str q0, [x0]
+; CHECK-CAS-O1-NEXT: stp x2, x3, [x0]
; CHECK-CAS-O1-NEXT: ret
;
; CHECK-LLSC-O0-LABEL: val_compare_and_swap_monotonic:
@@ -525,9 +501,7 @@ define void @atomic_load_relaxed(i64, i64, ptr %p, ptr %p2) {
; CHECK-LLSC-O1-NEXT: stxp w10, x9, x8, [x2]
; CHECK-LLSC-O1-NEXT: cbnz w10, .LBB4_1
; CHECK-LLSC-O1-NEXT: // %bb.2: // %atomicrmw.end
-; CHECK-LLSC-O1-NEXT: mov v0.d[0], x9
-; CHECK-LLSC-O1-NEXT: mov v0.d[1], x8
-; CHECK-LLSC-O1-NEXT: str q0, [x3]
+; CHECK-LLSC-O1-NEXT: stp x9, x8, [x3]
; CHECK-LLSC-O1-NEXT: ret
;
; CHECK-OUTLINE-LLSC-O1-LABEL: atomic_load_relaxed:
@@ -538,9 +512,7 @@ define void @atomic_load_relaxed(i64, i64, ptr %p, ptr %p2) {
; CHECK-OUTLINE-LLSC-O1-NEXT: stxp w10, x9, x8, [x2]
; CHECK-OUTLINE-LLSC-O1-NEXT: cbnz w10, .LBB4_1
; CHECK-OUTLINE-LLSC-O1-NEXT: // %bb.2: // %atomicrmw.end
-; CHECK-OUTLINE-LLSC-O1-NEXT: mov v0.d[0], x9
-; CHECK-OUTLINE-LLSC-O1-NEXT: mov v0.d[1], x8
-; CHECK-OUTLINE-LLSC-O1-NEXT: str q0, [x3]
+; CHECK-OUTLINE-LLSC-O1-NEXT: stp x9, x8, [x3]
; CHECK-OUTLINE-LLSC-O1-NEXT: ret
;
; CHECK-CAS-O1-LABEL: atomic_load_relaxed:
@@ -548,9 +520,7 @@ define void @atomic_load_relaxed(i64, i64, ptr %p, ptr %p2) {
; CHECK-CAS-O1-NEXT: mov x0, xzr
; CHECK-CAS-O1-NEXT: mov x1, xzr
; CHECK-CAS-O1-NEXT: casp x0, x1, x0, x1, [x2]
-; CHECK-CAS-O1-NEXT: mov v0.d[0], x0
-; CHECK-CAS-O1-NEXT: mov v0.d[1], x1
-; CHECK-CAS-O1-NEXT: str q0, [x3]
+; CHECK-CAS-O1-NEXT: stp x0, x1, [x3]
; CHECK-CAS-O1-NEXT: ret
;
; CHECK-LLSC-O0-LABEL: atomic_load_relaxed:
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
index f8cd868a4c755..94469cf262e3e 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
@@ -101,19 +101,6 @@ entry:
ret void
}
-; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to legalize instruction: %4:_(s128), %5:_(s1) = G_UMULO %0:_, %6:_ (in function: umul_s128)
-; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for umul_s128
-; FALLBACK-WITH-REPORT-OUT-LABEL: umul_s128
-declare {i128, i1} @llvm.umul.with.overflow.i128(i128, i128) nounwind readnone
-define zeroext i1 @umul_s128(i128 %v1, ptr %res) {
-entry:
- %t = call {i128, i1} @llvm.umul.with.overflow.i128(i128 %v1, i128 2)
- %val = extractvalue {i128, i1} %t, 0
- %obit = extractvalue {i128, i1} %t, 1
- store i128 %val, ptr %res
- ret i1 %obit
-}
-
; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to translate instruction: {{.*}}llvm.experimental.gc.statepoint{{.*}} (in function: gc_intr)
; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for gc_intr
; FALLBACK-WITH-REPORT-OUT-LABEL: gc_intr
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll b/llvm/test/CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll
index fe7e24c2d8ba5..a4dcd0155a449 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/v8.4-atomic-128.ll
@@ -5,56 +5,42 @@ define void @test_atomic_load(ptr %addr) {
; CHECK-LABEL: test_atomic_load:
; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
-; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
-; CHECK: mov v[[Q]].d[1], [[HI]]
-; CHECK: str q[[Q]], [x0]
+; CHECK: stp [[LO]], [[HI]], [x0]
%res.0 = load atomic i128, ptr %addr monotonic, align 16
store i128 %res.0, ptr %addr
; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
-; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
-; CHECK: mov v[[Q]].d[1], [[HI]]
-; CHECK: str q[[Q]], [x0]
+; CHECK: stp [[LO]], [[HI]], [x0]
%res.1 = load atomic i128, ptr %addr unordered, align 16
store i128 %res.1, ptr %addr
; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
-; CHECK: dmb ish
-; CHECK: mov v[[Q:[0-9]+]].d[0], [[LO]]
-; CHECK: mov v[[Q]].d[1], [[HI]]
-; CHECK: str q[[Q]], [x0]
+; CHECK: dmb ishld
+; CHECK: stp [[LO]], [[HI]], [x0]
%res.2 = load atomic i128, ptr %addr acquire, align 16
store i128 %res.2, ptr %addr
; CHECK: ldp [[LO:x[0-9]+]], [[HI:x[0-9]+]], [x0]
; CHECK: dmb ish
-; C...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/170101
More information about the llvm-commits
mailing list