[llvm] [GlobalISel] Support fixed vectors in constant-fold combines. (PR #220876)
Vikash Gupta via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 28 23:14:36 PDT 2026
https://github.com/vg0204 updated https://github.com/llvm/llvm-project/pull/220876
>From c6314aaac59905a5e2b4e2260e6b6b590653cd0b Mon Sep 17 00:00:00 2001
From: vg0204 <Vikash.Gupta at amd.com>
Date: Thu, 3 Sep 2026 16:26:58 +0530
Subject: [PATCH 1/7] [GlobalISel] Support fixed vectors in constant-fold
combines.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This patch Extend the GlobalISel constant-folding combines to
element-wise fold **fixed vectors**, in addition to scalars.
Previously these rules only fired when the operands were scalar
constants; a `G_BUILD_VECTOR` of constants was left un-folded.
This covers all five families:
- Integer/pointer binops (`constant_fold_binop`)
- FP binops (`constant_fold_fp_binop`)
- FMA (`constant_fold_fma`)
- Integer casts — `G_ZEXT`/`G_SEXT`/`G_ANYEXT` (`constant_fold_cast_op`)
- FP unary ops — `G_FNEG`, `G_FABS`, `G_FSQRT`, `G_FLOG2`, `G_FPEXT`,
`G_FPTRUNC`, `G_FCEIL`, `G_FFLOOR`,
`G_INTRINSIC_TRUNC/ROUND/ROUNDEVEN`,
`G_FRINT`, `G_FNEARBYINT` (`constant_fold_unary_fp_op_rule`)
---
.../llvm/CodeGen/GlobalISel/CombinerHelper.h | 16 +-
.../CodeGen/GlobalISel/MachineIRBuilder.h | 5 +
llvm/include/llvm/CodeGen/GlobalISel/Utils.h | 14 ++
.../include/llvm/Target/GlobalISel/Combine.td | 23 +-
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 215 ++++++++++--------
.../CodeGen/GlobalISel/MachineIRBuilder.cpp | 11 +
llvm/lib/CodeGen/GlobalISel/Utils.cpp | 106 +++++++++
.../GlobalISel/combine-logic-of-compare.mir | 5 +-
.../AArch64/GlobalISel/combine-overflow.mir | 5 +-
.../AArch64/GlobalISel/combine-udiv.ll | 13 +-
.../postlegalizer-combiner-constant-fold.mir | 108 +++++++++
...relegalizer-combiner-constant-fold-fma.mir | 58 +++++
llvm/test/CodeGen/AArch64/arm64-vabs.ll | 66 +++---
llvm/test/CodeGen/AArch64/combine-sdiv.ll | 174 +++++++-------
.../CodeGen/AArch64/extract-vector-elt.ll | 6 -
llvm/test/CodeGen/AArch64/fcvt_combine.ll | 38 ++--
llvm/test/CodeGen/AArch64/hadd-combine.ll | 56 ++---
.../half-precision-signof-no-assert.ll | 12 +-
.../AArch64/neon-compare-instructions.ll | 35 +--
.../GlobalISel/combine-or-redundant.mir | 9 +-
.../GlobalISel/combine-redundant-and.mir | 7 +-
...mbine-shl-from-extend-narrow.postlegal.mir | 6 +-
...ombine-shl-from-extend-narrow.prelegal.mir | 12 +-
.../GlobalISel/postlegalizercombiner-ashr.mir | 9 +-
.../GlobalISel/postlegalizercombiner-lshr.mir | 9 +-
.../GlobalISel/postlegalizercombiner-shl.mir | 9 +-
26 files changed, 643 insertions(+), 384 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-constant-fold-fma.mir
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index 6b80367d786a5..cc4e77d02cabb 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -445,10 +445,6 @@ class CombinerHelper {
LLVM_ABI bool matchCombineUnmergeZExtToZExt(MachineInstr &MI) const;
LLVM_ABI void applyCombineUnmergeZExtToZExt(MachineInstr &MI) const;
- /// Transform fp_instr(cst) to constant result of the fp operation.
- LLVM_ABI void applyCombineConstantFoldFpUnary(MachineInstr &MI,
- const ConstantFP *Cst) const;
-
/// Constant fold a unary integer op (G_CTLZ, G_CTTZ, G_CTPOP and their
/// _ZERO_POISON variants, G_ABS, G_BSWAP, G_BITREVERSE) when the operand is
/// a scalar constant or a G_BUILD_VECTOR of constants.
@@ -777,19 +773,23 @@ class CombinerHelper {
/// Do constant folding when opportunities are exposed after MIR building.
LLVM_ABI bool matchConstantFoldCastOp(MachineInstr &MI,
- APInt &MatchInfo) const;
+ BuildFnTy &MatchInfo) const;
/// Do constant folding when opportunities are exposed after MIR building.
LLVM_ABI bool matchConstantFoldBinOp(MachineInstr &MI,
- APInt &MatchInfo) const;
+ BuildFnTy &MatchInfo) const;
/// Do constant FP folding when opportunities are exposed after MIR building.
LLVM_ABI bool matchConstantFoldFPBinOp(MachineInstr &MI,
- ConstantFP *&MatchInfo) const;
+ BuildFnTy &MatchInfo) const;
+
+ /// Constant fold a unary FP operation when the source is constant.
+ LLVM_ABI bool matchConstantFoldFPUnary(MachineInstr &MI,
+ BuildFnTy &MatchInfo) const;
/// Constant fold G_FMA/G_FMAD.
LLVM_ABI bool matchConstantFoldFMA(MachineInstr &MI,
- ConstantFP *&MatchInfo) const;
+ BuildFnTy &MatchInfo) const;
/// \returns true if it is possible to narrow the width of a scalar binop
/// feeding a G_AND instruction \p MI.
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index 0816186507ca1..3126be704837c 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -1191,6 +1191,11 @@ class LLVM_ABI MachineIRBuilder {
MachineInstrBuilder buildBuildVectorConstant(const DstOp &Res,
ArrayRef<APInt> Ops);
+ /// Build and insert \p Res = G_BUILD_VECTOR \p Op0, ... where each OpN is
+ /// built with G_FCONSTANT.
+ MachineInstrBuilder buildBuildVectorFConstant(const DstOp &Res,
+ ArrayRef<APFloat> Ops);
+
/// Build and insert \p Res = G_BUILD_VECTOR with \p Src replicated to fill
/// the number of elements
MachineInstrBuilder buildSplatBuildVector(const DstOp &Res, const SrcOp &Src);
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
index 7b339211cc3ca..ef23467ba4337 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -321,6 +321,12 @@ LLVM_ABI SmallVector<APInt>
ConstantFoldVectorBinop(unsigned Opcode, const Register Op1, const Register Op2,
const MachineRegisterInfo &MRI);
+/// Tries to constant fold a vector floating-point binop with sources \p Op1 and
+/// \p Op2. Returns an empty vector on failure.
+LLVM_ABI SmallVector<APFloat>
+ConstantFoldVectorFPBinop(unsigned Opcode, const Register Op1,
+ const Register Op2, const MachineRegisterInfo &MRI);
+
LLVM_ABI std::optional<APInt>
ConstantFoldCastOp(unsigned Opcode, LLT DstTy, const Register Op0,
const MachineRegisterInfo &MRI);
@@ -341,6 +347,14 @@ LLVM_ABI SmallVector<APInt>
ConstantFoldUnaryIntOp(unsigned Opcode, LLT DstTy, Register Src,
const MachineRegisterInfo &MRI);
+/// Tries to constant fold a unary floating-point operation (G_FNEG, G_FABS,
+/// G_FSQRT, G_FLOG2, G_FPEXT, G_FPTRUNC, G_FCEIL, G_FFLOOR, G_INTRINSIC_TRUNC,
+/// G_INTRINSIC_ROUND, G_INTRINSIC_ROUNDEVEN, G_FRINT, G_FNEARBYINT) on \p Src.
+/// If \p Src is a vector then it tries to do an element-wise constant fold.
+LLVM_ABI SmallVector<APFloat>
+ConstantFoldUnaryFPOp(unsigned Opcode, LLT DstTy, Register Src,
+ const MachineRegisterInfo &MRI);
+
LLVM_ABI std::optional<SmallVector<APInt>>
ConstantFoldICmp(unsigned Pred, const Register Op1, const Register Op2,
unsigned DstScalarSizeInBits, unsigned ExtOp,
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index 3735e95ee8f68..76ffcb4fbf565 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -864,9 +864,10 @@ def simplify_add_to_sub : GICombineRule <
// Fold fp_op(cst) to the constant result of the floating point operation.
class constant_fold_unary_fp_op_rule<Instruction opcode> : GICombineRule <
- (defs root:$dst),
- (match (opcode $dst, $src0):$root, (G_FCONSTANT $src0, $cst)),
- (apply [{ Helper.applyCombineConstantFoldFpUnary(*${root}, ${cst}.getFPImm()); }])
+ (defs root:$dst, build_fn_matchinfo:$matchinfo),
+ (match (opcode $dst, $src0):$root,
+ [{ return Helper.matchConstantFoldFPUnary(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])
>;
def constant_fold_fneg : constant_fold_unary_fp_op_rule<G_FNEG>;
@@ -1638,12 +1639,12 @@ def reassocs : GICombineGroup<[reassoc_ptradd, reassoc_comm_binops]>;
// Constant fold operations.
def constant_fold_binop : GICombineRule<
- (defs root:$d, apint_matchinfo:$matchinfo),
+ (defs root:$d, build_fn_matchinfo:$matchinfo),
(match (wip_match_opcode G_ADD, G_PTR_ADD, G_AND, G_ASHR, G_LSHR, G_MUL, G_OR,
G_SHL, G_SUB, G_XOR, G_UDIV, G_SDIV, G_UREM, G_SREM,
G_SMIN, G_SMAX, G_UMIN, G_UMAX):$d,
[{ return Helper.matchConstantFoldBinOp(*${d}, ${matchinfo}); }]),
- (apply [{ Helper.replaceInstWithConstant(*${d}, ${matchinfo}); }])>;
+ (apply [{ Helper.applyBuildFn(*${d}, ${matchinfo}); }])>;
def constant_fold_fp_binop_frags : GICombinePatFrag<
(outs root:$dst), (ins),
@@ -1654,10 +1655,10 @@ def constant_fold_fp_binop_frags : GICombinePatFrag<
(pattern (op $dst, $src0, $src1)))>;
def constant_fold_fp_binop : GICombineRule<
- (defs root:$d, constantfp_matchinfo:$matchinfo),
+ (defs root:$d, build_fn_matchinfo:$matchinfo),
(match (constant_fold_fp_binop_frags $d):$mi,
[{ return Helper.matchConstantFoldFPBinOp(*${mi}, ${matchinfo}); }]),
- (apply [{ Helper.replaceInstWithFConstant(*${mi}, ${matchinfo}); }])>;
+ (apply [{ Helper.applyBuildFn(*${mi}, ${matchinfo}); }])>;
def constant_fold_fma_frags : GICombinePatFrag<
@@ -1665,20 +1666,20 @@ def constant_fold_fma_frags : GICombinePatFrag<
!foreach(op, [G_FMAD, G_FMA], (pattern (op $dst, $src0, $src1, $src2)))>;
def constant_fold_fma : GICombineRule<
- (defs root:$dst, constantfp_matchinfo:$matchinfo),
+ (defs root:$dst, build_fn_matchinfo:$matchinfo),
(match (constant_fold_fma_frags $dst):$d,
[{ return Helper.matchConstantFoldFMA(*${d}, ${matchinfo}); }]),
- (apply [{ Helper.replaceInstWithFConstant(*${d}, ${matchinfo}); }])>;
+ (apply [{ Helper.applyBuildFn(*${d}, ${matchinfo}); }])>;
def constant_fold_cast_op_frags : GICombinePatFrag<
(outs root:$dst), (ins),
!foreach(op, [G_ZEXT, G_SEXT, G_ANYEXT], (pattern (op $dst, $src)))>;
def constant_fold_cast_op : GICombineRule<
- (defs root:$dst, apint_matchinfo:$matchinfo),
+ (defs root:$dst, build_fn_matchinfo:$matchinfo),
(match (constant_fold_cast_op_frags $dst):$d,
[{ return Helper.matchConstantFoldCastOp(*${d}, ${matchinfo}); }]),
- (apply [{ Helper.replaceInstWithConstant(*${d}, ${matchinfo}); }])>;
+ (apply [{ Helper.applyBuildFn(*${d}, ${matchinfo}); }])>;
def unary_int_op_frags : GICombinePatFrag<
(outs root:$dst), (ins),
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 47145f2cffa17..a6d965746d989 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -1753,80 +1753,6 @@ bool CombinerHelper::tryCombineMemCpyFamily(MachineInstr &MI,
LegalizerHelper::LegalizeResult::Legalized;
}
-static APFloat constantFoldFpUnary(const MachineInstr &MI,
- const MachineRegisterInfo &MRI,
- const APFloat &Val) {
- APFloat Result(Val);
- switch (MI.getOpcode()) {
- default:
- llvm_unreachable("Unexpected opcode!");
- case TargetOpcode::G_FNEG: {
- Result.changeSign();
- return Result;
- }
- case TargetOpcode::G_FABS: {
- Result.clearSign();
- return Result;
- }
- case TargetOpcode::G_FCEIL:
- Result.roundToIntegral(APFloat::rmTowardPositive);
- return Result;
- case TargetOpcode::G_FFLOOR:
- Result.roundToIntegral(APFloat::rmTowardNegative);
- return Result;
- case TargetOpcode::G_INTRINSIC_TRUNC:
- Result.roundToIntegral(APFloat::rmTowardZero);
- return Result;
- case TargetOpcode::G_INTRINSIC_ROUND:
- Result.roundToIntegral(APFloat::rmNearestTiesToAway);
- return Result;
- case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
- Result.roundToIntegral(APFloat::rmNearestTiesToEven);
- return Result;
- case TargetOpcode::G_FRINT:
- case TargetOpcode::G_FNEARBYINT:
- // Use default rounding mode (round to nearest, ties to even)
- Result.roundToIntegral(APFloat::rmNearestTiesToEven);
- return Result;
- case TargetOpcode::G_FPEXT:
- case TargetOpcode::G_FPTRUNC: {
- bool Unused;
- LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
- Result.convert(getFltSemanticForLLT(DstTy), APFloat::rmNearestTiesToEven,
- &Unused);
- return Result;
- }
- case TargetOpcode::G_FSQRT: {
- bool Unused;
- Result.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
- &Unused);
- Result = APFloat(sqrt(Result.convertToDouble()));
- break;
- }
- case TargetOpcode::G_FLOG2: {
- bool Unused;
- Result.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
- &Unused);
- Result = APFloat(log2(Result.convertToDouble()));
- break;
- }
- }
- // Convert `APFloat` to appropriate IEEE type depending on `DstTy`. Otherwise,
- // `buildFConstant` will assert on size mismatch. Only `G_FSQRT`, and
- // `G_FLOG2` reach here.
- bool Unused;
- Result.convert(Val.getSemantics(), APFloat::rmNearestTiesToEven, &Unused);
- return Result;
-}
-
-void CombinerHelper::applyCombineConstantFoldFpUnary(
- MachineInstr &MI, const ConstantFP *Cst) const {
- APFloat Folded = constantFoldFpUnary(MI, MRI, Cst->getValue());
- const ConstantFP *NewCst = ConstantFP::get(Builder.getContext(), Folded);
- Builder.buildFConstant(MI.getOperand(0), *NewCst);
- MI.eraseFromParent();
-}
-
bool CombinerHelper::matchPtrAddImmedChain(MachineInstr &MI,
PtrAddChain &MatchInfo) const {
// We're trying to match the following pattern:
@@ -5335,12 +5261,33 @@ bool CombinerHelper::matchReassocCommBinOp(MachineInstr &MI,
}
bool CombinerHelper::matchConstantFoldCastOp(MachineInstr &MI,
- APInt &MatchInfo) const {
- LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
+ BuildFnTy &MatchInfo) const {
+ Register Dst = MI.getOperand(0).getReg();
Register SrcOp = MI.getOperand(1).getReg();
+ LLT DstTy = MRI.getType(Dst);
+ unsigned Opc = MI.getOpcode();
+
+ if (DstTy.isVector()) {
+ auto *BV = getOpcodeDef<GBuildVector>(SrcOp, MRI);
+ if (!BV)
+ return false;
+ SmallVector<APInt> Csts;
+ for (unsigned I = 0, E = BV->getNumSources(); I != E; ++I) {
+ auto MaybeCst = ConstantFoldCastOp(Opc, DstTy, BV->getSourceReg(I), MRI);
+ if (!MaybeCst)
+ return false;
+ Csts.push_back(*MaybeCst);
+ }
+ MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
+ B.buildBuildVectorConstant(Dst, Csts);
+ };
+ return true;
+ }
- if (auto MaybeCst = ConstantFoldCastOp(MI.getOpcode(), DstTy, SrcOp, MRI)) {
- MatchInfo = *MaybeCst;
+ if (auto MaybeCst = ConstantFoldCastOp(Opc, DstTy, SrcOp, MRI)) {
+ MatchInfo = [Dst, Cst = *MaybeCst](MachineIRBuilder &B) {
+ B.buildConstant(Dst, Cst);
+ };
return true;
}
@@ -5365,50 +5312,128 @@ bool CombinerHelper::matchConstantFoldUnaryIntOp(MachineInstr &MI,
}
bool CombinerHelper::matchConstantFoldBinOp(MachineInstr &MI,
- APInt &MatchInfo) const {
+ BuildFnTy &MatchInfo) const {
+ Register Dst = MI.getOperand(0).getReg();
Register Op1 = MI.getOperand(1).getReg();
Register Op2 = MI.getOperand(2).getReg();
+ LLT DstTy = MRI.getType(Dst);
+
+ if (DstTy.isVector()) {
+ // Pointer-element vectors (e.g. <n x ptr> from a vector G_PTR_ADD) have no
+ // G_BUILD_VECTOR of G_CONSTANT representation.
+ if (DstTy.getElementType().isPointer())
+ return false;
+ SmallVector<APInt> Csts =
+ ConstantFoldVectorBinop(MI.getOpcode(), Op1, Op2, MRI);
+ if (Csts.empty())
+ return false;
+ MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
+ B.buildBuildVectorConstant(Dst, Csts);
+ };
+ return true;
+ }
+
auto MaybeCst = ConstantFoldBinOp(MI.getOpcode(), Op1, Op2, MRI);
if (!MaybeCst)
return false;
- MatchInfo = *MaybeCst;
+ MatchInfo = [Dst, Cst = *MaybeCst](MachineIRBuilder &B) {
+ B.buildConstant(Dst, Cst);
+ };
return true;
}
bool CombinerHelper::matchConstantFoldFPBinOp(MachineInstr &MI,
- ConstantFP *&MatchInfo) const {
+ BuildFnTy &MatchInfo) const {
+ Register Dst = MI.getOperand(0).getReg();
Register Op1 = MI.getOperand(1).getReg();
Register Op2 = MI.getOperand(2).getReg();
+ LLT DstTy = MRI.getType(Dst);
+
+ if (DstTy.isVector()) {
+ SmallVector<APFloat> Csts =
+ ConstantFoldVectorFPBinop(MI.getOpcode(), Op1, Op2, MRI);
+ if (Csts.empty())
+ return false;
+ MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
+ B.buildBuildVectorFConstant(Dst, Csts);
+ };
+ return true;
+ }
+
auto MaybeCst = ConstantFoldFPBinOp(MI.getOpcode(), Op1, Op2, MRI);
if (!MaybeCst)
return false;
- MatchInfo =
- ConstantFP::get(MI.getMF()->getFunction().getContext(), *MaybeCst);
+ MatchInfo = [Dst, Cst = *MaybeCst](MachineIRBuilder &B) {
+ B.buildFConstant(Dst, Cst);
+ };
+ return true;
+}
+
+bool CombinerHelper::matchConstantFoldFPUnary(MachineInstr &MI,
+ BuildFnTy &MatchInfo) const {
+ Register Dst = MI.getOperand(0).getReg();
+ SmallVector<APFloat> Csts = ConstantFoldUnaryFPOp(
+ MI.getOpcode(), MRI.getType(Dst), MI.getOperand(1).getReg(), MRI);
+ if (Csts.empty())
+ return false;
+ MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
+ if (Csts.size() == 1)
+ B.buildFConstant(Dst, Csts[0]);
+ else
+ B.buildBuildVectorFConstant(Dst, Csts);
+ };
return true;
}
bool CombinerHelper::matchConstantFoldFMA(MachineInstr &MI,
- ConstantFP *&MatchInfo) const {
+ BuildFnTy &MatchInfo) const {
assert(MI.getOpcode() == TargetOpcode::G_FMA ||
MI.getOpcode() == TargetOpcode::G_FMAD);
+ Register Dst = MI.getOperand(0).getReg();
auto [_, Op1, Op2, Op3] = MI.getFirst4Regs();
+ LLT DstTy = MRI.getType(Dst);
- const ConstantFP *Op3Cst = getConstantFPVRegVal(Op3, MRI);
- if (!Op3Cst)
- return false;
+ auto FoldFMA = [](const APFloat &A, const APFloat &B,
+ const APFloat &C) -> APFloat {
+ APFloat Res(A);
+ Res.fusedMultiplyAdd(B, C, APFloat::rmNearestTiesToEven);
+ return Res;
+ };
- const ConstantFP *Op2Cst = getConstantFPVRegVal(Op2, MRI);
- if (!Op2Cst)
- return false;
+ if (DstTy.isVector()) {
+ auto *BV1 = getOpcodeDef<GBuildVector>(Op1, MRI);
+ auto *BV2 = getOpcodeDef<GBuildVector>(Op2, MRI);
+ auto *BV3 = getOpcodeDef<GBuildVector>(Op3, MRI);
+ if (!BV1 || !BV2 || !BV3)
+ return false;
+ unsigned NumElts = BV1->getNumSources();
+ if (BV2->getNumSources() != NumElts || BV3->getNumSources() != NumElts)
+ return false;
+ SmallVector<APFloat> Csts;
+ for (unsigned I = 0; I != NumElts; ++I) {
+ const ConstantFP *C1 = getConstantFPVRegVal(BV1->getSourceReg(I), MRI);
+ const ConstantFP *C2 = getConstantFPVRegVal(BV2->getSourceReg(I), MRI);
+ const ConstantFP *C3 = getConstantFPVRegVal(BV3->getSourceReg(I), MRI);
+ if (!C1 || !C2 || !C3)
+ return false;
+ Csts.push_back(
+ FoldFMA(C1->getValueAPF(), C2->getValueAPF(), C3->getValueAPF()));
+ }
+ MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
+ B.buildBuildVectorFConstant(Dst, Csts);
+ };
+ return true;
+ }
+ const ConstantFP *Op3Cst = getConstantFPVRegVal(Op3, MRI);
+ const ConstantFP *Op2Cst = getConstantFPVRegVal(Op2, MRI);
const ConstantFP *Op1Cst = getConstantFPVRegVal(Op1, MRI);
- if (!Op1Cst)
+ if (!Op1Cst || !Op2Cst || !Op3Cst)
return false;
- APFloat Op1F = Op1Cst->getValueAPF();
- Op1F.fusedMultiplyAdd(Op2Cst->getValueAPF(), Op3Cst->getValueAPF(),
- APFloat::rmNearestTiesToEven);
- MatchInfo = ConstantFP::get(MI.getMF()->getFunction().getContext(), Op1F);
+ APFloat Res = FoldFMA(Op1Cst->getValueAPF(), Op2Cst->getValueAPF(),
+ Op3Cst->getValueAPF());
+ MatchInfo = [Dst, Res](MachineIRBuilder &B) { B.buildFConstant(Dst, Res); };
return true;
}
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index 8237710380946..398e4343de801 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -778,6 +778,17 @@ MachineIRBuilder::buildBuildVectorConstant(const DstOp &Res,
return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec);
}
+MachineInstrBuilder
+MachineIRBuilder::buildBuildVectorFConstant(const DstOp &Res,
+ ArrayRef<APFloat> Ops) {
+ SmallVector<SrcOp> TmpVec;
+ TmpVec.reserve(Ops.size());
+ LLT EltTy = Res.getLLTTy(*getMRI()).getElementType();
+ for (const auto &Op : Ops)
+ TmpVec.push_back(buildFConstant(EltTy, Op));
+ return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec);
+}
+
MachineInstrBuilder MachineIRBuilder::buildSplatBuildVector(const DstOp &Res,
const SrcOp &Src) {
SmallVector<SrcOp, 8> TmpVec(Res.getLLTTy(*getMRI()).getNumElements(), Src);
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index d0cc4a823d7f5..011f23f171089 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -37,6 +37,7 @@
#include "llvm/Support/UndefPoison.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/Utils/SizeOpts.h"
+#include <cmath>
#include <limits>
#include <numeric>
#include <optional>
@@ -828,6 +829,32 @@ llvm::ConstantFoldVectorBinop(unsigned Opcode, const Register Op1,
return FoldedElements;
}
+SmallVector<APFloat>
+llvm::ConstantFoldVectorFPBinop(unsigned Opcode, const Register Op1,
+ const Register Op2,
+ const MachineRegisterInfo &MRI) {
+ auto *SrcVec2 = getBuildVectorLikeDef(Op2, MRI);
+ if (!SrcVec2)
+ return SmallVector<APFloat>();
+
+ auto *SrcVec1 = getBuildVectorLikeDef(Op1, MRI);
+ if (!SrcVec1)
+ return SmallVector<APFloat>();
+
+ if (SrcVec1->getNumSources() != SrcVec2->getNumSources())
+ return SmallVector<APFloat>();
+
+ SmallVector<APFloat> FoldedElements;
+ for (unsigned Idx = 0, E = SrcVec1->getNumSources(); Idx < E; ++Idx) {
+ auto MaybeCst = ConstantFoldFPBinOp(Opcode, SrcVec1->getSourceReg(Idx),
+ SrcVec2->getSourceReg(Idx), MRI);
+ if (!MaybeCst)
+ return SmallVector<APFloat>();
+ FoldedElements.push_back(*MaybeCst);
+ }
+ return FoldedElements;
+}
+
Align llvm::inferAlignFromPtrInfo(MachineFunction &MF,
const MachinePointerInfo &MPO) {
auto PSV = dyn_cast_if_present<const PseudoSourceValue *>(MPO.V);
@@ -979,6 +1006,85 @@ llvm::ConstantFoldUnaryIntOp(unsigned Opcode, LLT DstTy, Register Src,
return {};
}
+SmallVector<APFloat>
+llvm::ConstantFoldUnaryFPOp(unsigned Opcode, LLT DstTy, Register Src,
+ const MachineRegisterInfo &MRI) {
+ LLT DstEltTy = DstTy.getScalarType();
+ auto Fold = [Opcode, DstEltTy](const APFloat &V) -> APFloat {
+ APFloat Result(V);
+ bool Unused;
+ switch (Opcode) {
+ case TargetOpcode::G_FNEG:
+ Result.changeSign();
+ return Result;
+ case TargetOpcode::G_FABS:
+ Result.clearSign();
+ return Result;
+ case TargetOpcode::G_FCEIL:
+ Result.roundToIntegral(APFloat::rmTowardPositive);
+ return Result;
+ case TargetOpcode::G_FFLOOR:
+ Result.roundToIntegral(APFloat::rmTowardNegative);
+ return Result;
+ case TargetOpcode::G_INTRINSIC_TRUNC:
+ Result.roundToIntegral(APFloat::rmTowardZero);
+ return Result;
+ case TargetOpcode::G_INTRINSIC_ROUND:
+ Result.roundToIntegral(APFloat::rmNearestTiesToAway);
+ return Result;
+ case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
+ case TargetOpcode::G_FRINT:
+ case TargetOpcode::G_FNEARBYINT:
+ Result.roundToIntegral(APFloat::rmNearestTiesToEven);
+ return Result;
+ case TargetOpcode::G_FPEXT:
+ case TargetOpcode::G_FPTRUNC:
+ Result.convert(getFltSemanticForLLT(DstEltTy),
+ APFloat::rmNearestTiesToEven, &Unused);
+ return Result;
+ case TargetOpcode::G_FSQRT:
+ Result.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
+ &Unused);
+ Result = APFloat(std::sqrt(Result.convertToDouble()));
+ break;
+ case TargetOpcode::G_FLOG2:
+ Result.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
+ &Unused);
+ Result = APFloat(std::log2(Result.convertToDouble()));
+ break;
+ default:
+ llvm_unreachable("unexpected opcode in ConstantFoldUnaryFPOp");
+ }
+ // Only G_FSQRT and G_FLOG2 reach here; convert the double result back to
+ // the source (== destination) semantics.
+ Result.convert(V.getSemantics(), APFloat::rmNearestTiesToEven, &Unused);
+ return Result;
+ };
+
+ auto tryFoldScalar = [&](Register R) -> std::optional<APFloat> {
+ if (const ConstantFP *Cst = getConstantFPVRegVal(R, MRI))
+ return Fold(Cst->getValueAPF());
+ return std::nullopt;
+ };
+ if (MRI.getType(Src).isVector()) {
+ auto *BV = getOpcodeDef<GBuildVector>(Src, MRI);
+ if (!BV)
+ return {};
+ SmallVector<APFloat> Folded;
+ for (unsigned SrcIdx = 0; SrcIdx < BV->getNumSources(); ++SrcIdx) {
+ if (auto MaybeFold = tryFoldScalar(BV->getSourceReg(SrcIdx))) {
+ Folded.emplace_back(std::move(*MaybeFold));
+ continue;
+ }
+ return {};
+ }
+ return Folded;
+ }
+ if (auto MaybeCst = tryFoldScalar(Src))
+ return {std::move(*MaybeCst)};
+ return {};
+}
+
std::optional<SmallVector<APInt>>
llvm::ConstantFoldICmp(unsigned Pred, const Register Op1, const Register Op2,
unsigned DstScalarSizeInBits, unsigned ExtOp,
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-logic-of-compare.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-logic-of-compare.mir
index 7bb166e0312c2..92d1a81098e1d 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-logic-of-compare.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-logic-of-compare.mir
@@ -386,9 +386,8 @@ body: |
; CHECK-LABEL: name: test_fcmp_and_fcmp_with_vectors
; CHECK: liveins: $x0, $x1
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(i1) = G_CONSTANT i1 false
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x i1>) = G_BUILD_VECTOR [[C]](i1), [[C]](i1)
- ; CHECK-NEXT: %zext:_(<2 x i64>) = G_ZEXT [[BUILD_VECTOR]](<2 x i1>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: %zext:_(<2 x i64>) = G_BUILD_VECTOR [[C]](i64), [[C]](i64)
; CHECK-NEXT: $q0 = COPY %zext(<2 x i64>)
%0:_(f64) = COPY $x0
%1:_(f64) = COPY $x1
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-overflow.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-overflow.mir
index df2cdb15ef899..9b98cd45da0de 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-overflow.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-overflow.mir
@@ -157,9 +157,8 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $w0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $w1
; CHECK-NEXT: %bv0:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY]](i32), [[COPY1]](i32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(i1) = G_CONSTANT i1 false
- ; CHECK-NEXT: %o:_(<4 x i1>) = G_BUILD_VECTOR [[C]](i1), [[C]](i1), [[C]](i1), [[C]](i1)
- ; CHECK-NEXT: %o_wide:_(<4 x i32>) = G_ZEXT %o(<4 x i1>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: %o_wide:_(<4 x i32>) = G_BUILD_VECTOR [[C]](i32), [[C]](i32), [[C]](i32), [[C]](i32)
; CHECK-NEXT: $q0 = COPY %bv0(<4 x i32>)
; CHECK-NEXT: $q1 = COPY %o_wide(<4 x i32>)
; CHECK-NEXT: RET_ReallyLR implicit $w0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll b/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll
index 461a7ef67e9e0..66fcd1d0c02e0 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll
@@ -210,24 +210,23 @@ define <8 x i16> @pr38477(<8 x i16> %a0) {
; GISEL-LABEL: pr38477:
; GISEL: // %bb.0:
; GISEL-NEXT: adrp x8, .LCPI5_3
+; GISEL-NEXT: adrp x9, .LCPI5_0
; GISEL-NEXT: ldr q1, [x8, :lo12:.LCPI5_3]
; GISEL-NEXT: adrp x8, .LCPI5_2
; GISEL-NEXT: ldr q3, [x8, :lo12:.LCPI5_2]
-; GISEL-NEXT: adrp x8, .LCPI5_0
+; GISEL-NEXT: adrp x8, .LCPI5_1
; GISEL-NEXT: umull2 v2.4s, v0.8h, v1.8h
; GISEL-NEXT: umull v1.4s, v0.4h, v1.4h
; GISEL-NEXT: uzp2 v1.8h, v1.8h, v2.8h
; GISEL-NEXT: sub v2.8h, v0.8h, v1.8h
; GISEL-NEXT: umull2 v4.4s, v2.8h, v3.8h
; GISEL-NEXT: umull v2.4s, v2.4h, v3.4h
-; GISEL-NEXT: ldr d3, [x8, :lo12:.LCPI5_0]
-; GISEL-NEXT: adrp x8, .LCPI5_1
-; GISEL-NEXT: ushll v3.8h, v3.8b, #0
+; GISEL-NEXT: ldr q3, [x8, :lo12:.LCPI5_1]
; GISEL-NEXT: uzp2 v2.8h, v2.8h, v4.8h
-; GISEL-NEXT: ldr q4, [x8, :lo12:.LCPI5_1]
-; GISEL-NEXT: shl v3.8h, v3.8h, #15
+; GISEL-NEXT: ldr q4, [x9, :lo12:.LCPI5_0]
; GISEL-NEXT: add v1.8h, v2.8h, v1.8h
-; GISEL-NEXT: neg v2.8h, v4.8h
+; GISEL-NEXT: neg v2.8h, v3.8h
+; GISEL-NEXT: shl v3.8h, v4.8h, #15
; GISEL-NEXT: ushl v1.8h, v1.8h, v2.8h
; GISEL-NEXT: cmlt v2.8h, v3.8h, #0
; GISEL-NEXT: bif v0.16b, v1.16b, v2.16b
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir
index 9840360c41838..66b1b9eb6caea 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir
@@ -408,3 +408,111 @@ body: |
RET_ReallyLR implicit $s0
...
+---
+name: add_vector
+legalized: true
+liveins:
+ - { reg: '$q0' }
+body: |
+ bb.1.entry:
+ liveins: $q0
+
+ ; Element-wise fold of an integer vector binop: <40, 10> + <2, 2> = <42, 12>.
+ ; CHECK-LABEL: name: add_vector
+ ; CHECK: liveins: $q0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 42
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 12
+ ; CHECK-NEXT: %res:_(<2 x i64>) = G_BUILD_VECTOR [[C]](i64), [[C1]](i64)
+ ; CHECK-NEXT: $q0 = COPY %res(<2 x i64>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %a0:_(i64) = G_CONSTANT i64 40
+ %a1:_(i64) = G_CONSTANT i64 10
+ %b:_(i64) = G_CONSTANT i64 2
+ %av:_(<2 x i64>) = G_BUILD_VECTOR %a0(i64), %a1(i64)
+ %bv:_(<2 x i64>) = G_BUILD_VECTOR %b(i64), %b(i64)
+ %res:_(<2 x i64>) = G_ADD %av, %bv
+ $q0 = COPY %res(<2 x i64>)
+ RET_ReallyLR implicit $q0
+
+...
+---
+name: fadd_vector
+legalized: true
+liveins:
+ - { reg: '$q0' }
+body: |
+ bb.1.entry:
+ liveins: $q0
+
+ ; Element-wise fold of an FP vector binop: <40.0, 1.0> + <2.0, 2.0>.
+ ; CHECK-LABEL: name: fadd_vector
+ ; CHECK: liveins: $q0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double 4.200000e+01
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(f64) = G_FCONSTANT double 3.000000e+00
+ ; CHECK-NEXT: %res:_(<2 x f64>) = G_BUILD_VECTOR [[C]](f64), [[C1]](f64)
+ ; CHECK-NEXT: $q0 = COPY %res(<2 x f64>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %a0:_(f64) = G_FCONSTANT double 40.0
+ %a1:_(f64) = G_FCONSTANT double 1.0
+ %b:_(f64) = G_FCONSTANT double 2.0
+ %av:_(<2 x f64>) = G_BUILD_VECTOR %a0(f64), %a1(f64)
+ %bv:_(<2 x f64>) = G_BUILD_VECTOR %b(f64), %b(f64)
+ %res:_(<2 x f64>) = G_FADD %av, %bv
+ $q0 = COPY %res(<2 x f64>)
+ RET_ReallyLR implicit $q0
+
+...
+---
+name: zext_vector
+legalized: true
+liveins:
+ - { reg: '$q0' }
+body: |
+ bb.1.entry:
+ liveins: $q0
+
+ ; Element-wise fold of a vector cast: zext(<7, 8> : <2 x i32>) to <2 x i64>.
+ ; CHECK-LABEL: name: zext_vector
+ ; CHECK: liveins: $q0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 7
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
+ ; CHECK-NEXT: %res:_(<2 x i64>) = G_BUILD_VECTOR [[C]](i64), [[C1]](i64)
+ ; CHECK-NEXT: $q0 = COPY %res(<2 x i64>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %a0:_(i32) = G_CONSTANT i32 7
+ %a1:_(i32) = G_CONSTANT i32 8
+ %src:_(<2 x i32>) = G_BUILD_VECTOR %a0(i32), %a1(i32)
+ %res:_(<2 x i64>) = G_ZEXT %src(<2 x i32>)
+ $q0 = COPY %res(<2 x i64>)
+ RET_ReallyLR implicit $q0
+
+...
+---
+name: fneg_vector
+legalized: true
+liveins:
+ - { reg: '$q0' }
+body: |
+ bb.1.entry:
+ liveins: $q0
+
+ ; Element-wise fold of a unary FP op: fneg(<3.0, -4.0>) = <-3.0, 4.0>.
+ ; CHECK-LABEL: name: fneg_vector
+ ; CHECK: liveins: $q0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double -3.000000e+00
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(f64) = G_FCONSTANT double 4.000000e+00
+ ; CHECK-NEXT: %res:_(<2 x f64>) = G_BUILD_VECTOR [[C]](f64), [[C1]](f64)
+ ; CHECK-NEXT: $q0 = COPY %res(<2 x f64>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %a0:_(f64) = G_FCONSTANT double 3.0
+ %a1:_(f64) = G_FCONSTANT double -4.0
+ %src:_(<2 x f64>) = G_BUILD_VECTOR %a0(f64), %a1(f64)
+ %res:_(<2 x f64>) = G_FNEG %src(<2 x f64>)
+ $q0 = COPY %res(<2 x f64>)
+ RET_ReallyLR implicit $q0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-constant-fold-fma.mir b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-constant-fold-fma.mir
new file mode 100644
index 0000000000000..83ed5c03ecd39
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-constant-fold-fma.mir
@@ -0,0 +1,58 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64 -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name: fma_scalar
+legalized: true
+liveins:
+ - { reg: '$d0' }
+body: |
+ bb.1.entry:
+ liveins: $d0
+
+ ; fma(3.0, 4.0, 5.0) = 17.0
+ ; CHECK-LABEL: name: fma_scalar
+ ; CHECK: liveins: $d0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %res:_(f64) = G_FCONSTANT double 1.700000e+01
+ ; CHECK-NEXT: $d0 = COPY %res(f64)
+ ; CHECK-NEXT: RET_ReallyLR implicit $d0
+ %a:_(f64) = G_FCONSTANT double 3.0
+ %b:_(f64) = G_FCONSTANT double 4.0
+ %c:_(f64) = G_FCONSTANT double 5.0
+ %res:_(f64) = G_FMA %a, %b, %c
+ $d0 = COPY %res(f64)
+ RET_ReallyLR implicit $d0
+
+...
+---
+name: fma_vector
+legalized: true
+liveins:
+ - { reg: '$q0' }
+body: |
+ bb.1.entry:
+ liveins: $q0
+
+ ; Element-wise fma: fma(<3.0, 2.0>, <4.0, 4.0>, <5.0, 1.0>) = <17.0, 9.0>.
+ ; CHECK-LABEL: name: fma_vector
+ ; CHECK: liveins: $q0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double 1.700000e+01
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(f64) = G_FCONSTANT double 9.000000e+00
+ ; CHECK-NEXT: %res:_(<2 x f64>) = G_BUILD_VECTOR [[C]](f64), [[C1]](f64)
+ ; CHECK-NEXT: $q0 = COPY %res(<2 x f64>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %a0:_(f64) = G_FCONSTANT double 3.0
+ %a1:_(f64) = G_FCONSTANT double 2.0
+ %b:_(f64) = G_FCONSTANT double 4.0
+ %c0:_(f64) = G_FCONSTANT double 5.0
+ %c1:_(f64) = G_FCONSTANT double 1.0
+ %av:_(<2 x f64>) = G_BUILD_VECTOR %a0(f64), %a1(f64)
+ %bv:_(<2 x f64>) = G_BUILD_VECTOR %b(f64), %b(f64)
+ %cv:_(<2 x f64>) = G_BUILD_VECTOR %c0(f64), %c1(f64)
+ %res:_(<2 x f64>) = G_FMA %av, %bv, %cv
+ $q0 = COPY %res(<2 x f64>)
+ RET_ReallyLR implicit $q0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/arm64-vabs.ll b/llvm/test/CodeGen/AArch64/arm64-vabs.ll
index d175247cca3d9..bf69869b192b4 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vabs.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vabs.ll
@@ -2032,25 +2032,26 @@ define <16 x i16> @uabd16b_i16_const_select(<16 x i8> %a) {
;
; CHECK-GI-LABEL: uabd16b_i16_const_select:
; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI106_3
+; CHECK-GI-NEXT: ushll.8h v2, v0, #0
+; CHECK-GI-NEXT: ushll2.8h v4, v0, #0
+; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI106_3]
; CHECK-GI-NEXT: adrp x8, .LCPI106_1
-; CHECK-GI-NEXT: mov d3, v0[1]
-; CHECK-GI-NEXT: ushll.8h v4, v0, #0
-; CHECK-GI-NEXT: ldr d1, [x8, :lo12:.LCPI106_1]
+; CHECK-GI-NEXT: ldr q3, [x8, :lo12:.LCPI106_1]
; CHECK-GI-NEXT: adrp x8, .LCPI106_0
-; CHECK-GI-NEXT: ushll2.8h v6, v0, #0
-; CHECK-GI-NEXT: ldr d2, [x8, :lo12:.LCPI106_0]
-; CHECK-GI-NEXT: ushll.8h v5, v1, #0
-; CHECK-GI-NEXT: usubl.8h v16, v0, v1
-; CHECK-GI-NEXT: usubl.8h v1, v1, v0
-; CHECK-GI-NEXT: ushll.8h v7, v2, #0
-; CHECK-GI-NEXT: usubl.8h v2, v3, v2
-; CHECK-GI-NEXT: cmhi.8h v3, v5, v4
-; CHECK-GI-NEXT: cmhi.8h v4, v7, v6
-; CHECK-GI-NEXT: usubw2.8h v5, v7, v0
-; CHECK-GI-NEXT: mov.16b v0, v3
-; CHECK-GI-NEXT: bsl.16b v0, v1, v16
+; CHECK-GI-NEXT: ldr q5, [x8, :lo12:.LCPI106_0]
+; CHECK-GI-NEXT: adrp x8, .LCPI106_2
+; CHECK-GI-NEXT: cmhi.8h v2, v1, v2
+; CHECK-GI-NEXT: ldr q6, [x8, :lo12:.LCPI106_2]
+; CHECK-GI-NEXT: uaddw.8h v3, v3, v0
+; CHECK-GI-NEXT: usubw.8h v1, v1, v0
+; CHECK-GI-NEXT: uaddw2.8h v5, v5, v0
+; CHECK-GI-NEXT: cmhi.8h v4, v6, v4
+; CHECK-GI-NEXT: usubw2.8h v6, v6, v0
+; CHECK-GI-NEXT: mov.16b v0, v2
+; CHECK-GI-NEXT: bsl.16b v0, v1, v3
; CHECK-GI-NEXT: mov.16b v1, v4
-; CHECK-GI-NEXT: bsl.16b v1, v5, v2
+; CHECK-GI-NEXT: bsl.16b v1, v6, v5
; CHECK-GI-NEXT: ret
%aext = zext <16 x i8> %a to <16 x i16>
%bext = zext <16 x i8> <i8 39, i8 42, i8 51, i8 51, i8 0, i8 0, i8 54, i8 57, i8 66, i8 69, i8 75, i8 69, i8 75, i8 81, i8 255, i8 99> to <16 x i16>
@@ -2074,25 +2075,26 @@ define <16 x i16> @sabd16b_i16_const_select(<16 x i8> %a) {
;
; CHECK-GI-LABEL: sabd16b_i16_const_select:
; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI107_3
+; CHECK-GI-NEXT: sshll.8h v2, v0, #0
+; CHECK-GI-NEXT: sshll2.8h v4, v0, #0
+; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI107_3]
; CHECK-GI-NEXT: adrp x8, .LCPI107_1
-; CHECK-GI-NEXT: mov d3, v0[1]
-; CHECK-GI-NEXT: sshll.8h v4, v0, #0
-; CHECK-GI-NEXT: ldr d1, [x8, :lo12:.LCPI107_1]
+; CHECK-GI-NEXT: ldr q3, [x8, :lo12:.LCPI107_1]
; CHECK-GI-NEXT: adrp x8, .LCPI107_0
-; CHECK-GI-NEXT: sshll2.8h v6, v0, #0
-; CHECK-GI-NEXT: ldr d2, [x8, :lo12:.LCPI107_0]
-; CHECK-GI-NEXT: sshll.8h v5, v1, #0
-; CHECK-GI-NEXT: ssubl.8h v16, v0, v1
-; CHECK-GI-NEXT: ssubl.8h v1, v1, v0
-; CHECK-GI-NEXT: sshll.8h v7, v2, #0
-; CHECK-GI-NEXT: ssubl.8h v2, v3, v2
-; CHECK-GI-NEXT: cmgt.8h v3, v5, v4
-; CHECK-GI-NEXT: cmgt.8h v4, v7, v6
-; CHECK-GI-NEXT: ssubw2.8h v5, v7, v0
-; CHECK-GI-NEXT: mov.16b v0, v3
-; CHECK-GI-NEXT: bsl.16b v0, v1, v16
+; CHECK-GI-NEXT: ldr q5, [x8, :lo12:.LCPI107_0]
+; CHECK-GI-NEXT: adrp x8, .LCPI107_2
+; CHECK-GI-NEXT: cmgt.8h v2, v1, v2
+; CHECK-GI-NEXT: ldr q6, [x8, :lo12:.LCPI107_2]
+; CHECK-GI-NEXT: saddw.8h v3, v3, v0
+; CHECK-GI-NEXT: ssubw.8h v1, v1, v0
+; CHECK-GI-NEXT: saddw2.8h v5, v5, v0
+; CHECK-GI-NEXT: cmgt.8h v4, v6, v4
+; CHECK-GI-NEXT: ssubw2.8h v6, v6, v0
+; CHECK-GI-NEXT: mov.16b v0, v2
+; CHECK-GI-NEXT: bsl.16b v0, v1, v3
; CHECK-GI-NEXT: mov.16b v1, v4
-; CHECK-GI-NEXT: bsl.16b v1, v5, v2
+; CHECK-GI-NEXT: bsl.16b v1, v6, v5
; CHECK-GI-NEXT: ret
%aext = sext <16 x i8> %a to <16 x i16>
%bext = sext <16 x i8> <i8 -39, i8 42, i8 -51, i8 51, i8 0, i8 0, i8 -54, i8 57, i8 -66, i8 69, i8 75, i8 69, i8 75, i8 81, i8 -128, i8 99> to <16 x i16>
diff --git a/llvm/test/CodeGen/AArch64/combine-sdiv.ll b/llvm/test/CodeGen/AArch64/combine-sdiv.ll
index b160333edb487..ac33cf38cae1c 100644
--- a/llvm/test/CodeGen/AArch64/combine-sdiv.ll
+++ b/llvm/test/CodeGen/AArch64/combine-sdiv.ll
@@ -260,19 +260,18 @@ define <8 x i16> @combine_vec_sdiv_by_pow2b_v8i16(<8 x i16> %x) {
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: adrp x8, .LCPI15_1
; CHECK-GI-NEXT: cmlt v2.8h, v0.8h, #0
+; CHECK-GI-NEXT: adrp x9, .LCPI15_0
; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI15_1]
-; CHECK-GI-NEXT: adrp x8, .LCPI15_0
-; CHECK-GI-NEXT: ldr d3, [x8, :lo12:.LCPI15_0]
; CHECK-GI-NEXT: adrp x8, .LCPI15_2
+; CHECK-GI-NEXT: ldr q3, [x9, :lo12:.LCPI15_0]
; CHECK-GI-NEXT: neg v1.8h, v1.8h
+; CHECK-GI-NEXT: shl v3.8h, v3.8h, #15
; CHECK-GI-NEXT: ushl v1.8h, v2.8h, v1.8h
-; CHECK-GI-NEXT: ushll v2.8h, v3.8b, #0
-; CHECK-GI-NEXT: ldr q3, [x8, :lo12:.LCPI15_2]
-; CHECK-GI-NEXT: neg v3.8h, v3.8h
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI15_2]
+; CHECK-GI-NEXT: neg v2.8h, v2.8h
; CHECK-GI-NEXT: add v1.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT: shl v2.8h, v2.8h, #15
-; CHECK-GI-NEXT: sshl v1.8h, v1.8h, v3.8h
-; CHECK-GI-NEXT: cmlt v2.8h, v2.8h, #0
+; CHECK-GI-NEXT: sshl v1.8h, v1.8h, v2.8h
+; CHECK-GI-NEXT: cmlt v2.8h, v3.8h, #0
; CHECK-GI-NEXT: bif v0.16b, v1.16b, v2.16b
; CHECK-GI-NEXT: ret
%1 = sdiv <8 x i16> %x, <i16 1, i16 4, i16 2, i16 16, i16 8, i16 32, i16 64, i16 2>
@@ -306,15 +305,14 @@ define <16 x i16> @combine_vec_sdiv_by_pow2b_v16i16(<16 x i16> %x) {
; CHECK-GI-NEXT: cmlt v3.8h, v0.8h, #0
; CHECK-GI-NEXT: cmlt v4.8h, v1.8h, #0
; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI16_1]
-; CHECK-GI-NEXT: adrp x8, .LCPI16_0
-; CHECK-GI-NEXT: ldr d5, [x8, :lo12:.LCPI16_0]
; CHECK-GI-NEXT: adrp x8, .LCPI16_2
+; CHECK-GI-NEXT: adrp x9, .LCPI16_0
+; CHECK-GI-NEXT: ldr q5, [x9, :lo12:.LCPI16_0]
; CHECK-GI-NEXT: neg v2.8h, v2.8h
-; CHECK-GI-NEXT: ushll v5.8h, v5.8b, #0
+; CHECK-GI-NEXT: shl v5.8h, v5.8h, #15
; CHECK-GI-NEXT: ushl v3.8h, v3.8h, v2.8h
; CHECK-GI-NEXT: ushl v2.8h, v4.8h, v2.8h
; CHECK-GI-NEXT: ldr q4, [x8, :lo12:.LCPI16_2]
-; CHECK-GI-NEXT: shl v5.8h, v5.8h, #15
; CHECK-GI-NEXT: neg v4.8h, v4.8h
; CHECK-GI-NEXT: add v3.8h, v0.8h, v3.8h
; CHECK-GI-NEXT: add v2.8h, v1.8h, v2.8h
@@ -365,33 +363,32 @@ define <32 x i16> @combine_vec_sdiv_by_pow2b_v32i16(<32 x i16> %x) {
; CHECK-GI-NEXT: cmlt v5.8h, v0.8h, #0
; CHECK-GI-NEXT: cmlt v6.8h, v1.8h, #0
; CHECK-GI-NEXT: ldr q4, [x8, :lo12:.LCPI17_1]
-; CHECK-GI-NEXT: adrp x8, .LCPI17_0
; CHECK-GI-NEXT: cmlt v7.8h, v2.8h, #0
; CHECK-GI-NEXT: cmlt v16.8h, v3.8h, #0
-; CHECK-GI-NEXT: ldr d17, [x8, :lo12:.LCPI17_0]
; CHECK-GI-NEXT: adrp x8, .LCPI17_2
; CHECK-GI-NEXT: neg v4.8h, v4.8h
-; CHECK-GI-NEXT: ushll v17.8h, v17.8b, #0
+; CHECK-GI-NEXT: ldr q17, [x8, :lo12:.LCPI17_2]
+; CHECK-GI-NEXT: adrp x8, .LCPI17_0
+; CHECK-GI-NEXT: neg v17.8h, v17.8h
; CHECK-GI-NEXT: ushl v5.8h, v5.8h, v4.8h
; CHECK-GI-NEXT: ushl v6.8h, v6.8h, v4.8h
; CHECK-GI-NEXT: ushl v7.8h, v7.8h, v4.8h
; CHECK-GI-NEXT: ushl v4.8h, v16.8h, v4.8h
-; CHECK-GI-NEXT: ldr q16, [x8, :lo12:.LCPI17_2]
-; CHECK-GI-NEXT: shl v17.8h, v17.8h, #15
-; CHECK-GI-NEXT: neg v16.8h, v16.8h
+; CHECK-GI-NEXT: ldr q16, [x8, :lo12:.LCPI17_0]
+; CHECK-GI-NEXT: shl v16.8h, v16.8h, #15
; CHECK-GI-NEXT: add v5.8h, v0.8h, v5.8h
; CHECK-GI-NEXT: add v6.8h, v1.8h, v6.8h
; CHECK-GI-NEXT: add v7.8h, v2.8h, v7.8h
; CHECK-GI-NEXT: add v4.8h, v3.8h, v4.8h
-; CHECK-GI-NEXT: cmlt v17.8h, v17.8h, #0
-; CHECK-GI-NEXT: sshl v5.8h, v5.8h, v16.8h
-; CHECK-GI-NEXT: sshl v6.8h, v6.8h, v16.8h
-; CHECK-GI-NEXT: sshl v7.8h, v7.8h, v16.8h
-; CHECK-GI-NEXT: sshl v4.8h, v4.8h, v16.8h
-; CHECK-GI-NEXT: bif v0.16b, v5.16b, v17.16b
-; CHECK-GI-NEXT: bif v1.16b, v6.16b, v17.16b
-; CHECK-GI-NEXT: bif v2.16b, v7.16b, v17.16b
-; CHECK-GI-NEXT: bif v3.16b, v4.16b, v17.16b
+; CHECK-GI-NEXT: cmlt v16.8h, v16.8h, #0
+; CHECK-GI-NEXT: sshl v5.8h, v5.8h, v17.8h
+; CHECK-GI-NEXT: sshl v6.8h, v6.8h, v17.8h
+; CHECK-GI-NEXT: sshl v7.8h, v7.8h, v17.8h
+; CHECK-GI-NEXT: sshl v4.8h, v4.8h, v17.8h
+; CHECK-GI-NEXT: bif v0.16b, v5.16b, v16.16b
+; CHECK-GI-NEXT: bif v1.16b, v6.16b, v16.16b
+; CHECK-GI-NEXT: bif v2.16b, v7.16b, v16.16b
+; CHECK-GI-NEXT: bif v3.16b, v4.16b, v16.16b
; CHECK-GI-NEXT: ret
%1 = sdiv <32 x i16> %x, <i16 1, i16 4, i16 2, i16 16, i16 8, i16 32, i16 64, i16 2, i16 1, i16 4, i16 2, i16 16, i16 8, i16 32, i16 64, i16 2, i16 1, i16 4, i16 2, i16 16, i16 8, i16 32, i16 64, i16 2, i16 1, i16 4, i16 2, i16 16, i16 8, i16 32, i16 64, i16 2>
ret <32 x i16> %1
@@ -463,22 +460,25 @@ define <8 x i32> @combine_vec_sdiv_by_pow2b_v8i32(<8 x i32> %x) {
; CHECK-GI-NEXT: cmlt v4.4s, v1.4s, #0
; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI19_1]
; CHECK-GI-NEXT: adrp x8, .LCPI19_0
-; CHECK-GI-NEXT: ldr d5, [x8, :lo12:.LCPI19_0]
+; CHECK-GI-NEXT: ldr q5, [x8, :lo12:.LCPI19_0]
; CHECK-GI-NEXT: adrp x8, .LCPI19_2
; CHECK-GI-NEXT: neg v2.4s, v2.4s
-; CHECK-GI-NEXT: ushll v5.4s, v5.4h, #0
+; CHECK-GI-NEXT: ldr q6, [x8, :lo12:.LCPI19_2]
+; CHECK-GI-NEXT: neg v6.4s, v6.4s
; CHECK-GI-NEXT: ushl v3.4s, v3.4s, v2.4s
; CHECK-GI-NEXT: ushl v2.4s, v4.4s, v2.4s
-; CHECK-GI-NEXT: ldr q4, [x8, :lo12:.LCPI19_2]
-; CHECK-GI-NEXT: shl v5.4s, v5.4s, #31
-; CHECK-GI-NEXT: neg v4.4s, v4.4s
+; CHECK-GI-NEXT: ushll v4.4s, v5.4h, #0
+; CHECK-GI-NEXT: ushll2 v5.4s, v5.8h, #0
; CHECK-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-GI-NEXT: shl v4.4s, v4.4s, #31
; CHECK-GI-NEXT: add v2.4s, v1.4s, v2.4s
-; CHECK-GI-NEXT: sshl v3.4s, v3.4s, v4.4s
-; CHECK-GI-NEXT: sshl v2.4s, v2.4s, v4.4s
-; CHECK-GI-NEXT: cmlt v4.4s, v5.4s, #0
+; CHECK-GI-NEXT: shl v5.4s, v5.4s, #31
+; CHECK-GI-NEXT: sshl v3.4s, v3.4s, v6.4s
+; CHECK-GI-NEXT: sshl v2.4s, v2.4s, v6.4s
+; CHECK-GI-NEXT: cmlt v4.4s, v4.4s, #0
+; CHECK-GI-NEXT: cmlt v5.4s, v5.4s, #0
; CHECK-GI-NEXT: bif v0.16b, v3.16b, v4.16b
-; CHECK-GI-NEXT: bif v1.16b, v2.16b, v4.16b
+; CHECK-GI-NEXT: bif v1.16b, v2.16b, v5.16b
; CHECK-GI-NEXT: ret
%1 = sdiv <8 x i32> %x, <i32 1, i32 4, i32 8, i32 16, i32 1, i32 4, i32 8, i32 16>
ret <8 x i32> %1
@@ -522,32 +522,35 @@ define <16 x i32> @combine_vec_sdiv_by_pow2b_v16i32(<16 x i32> %x) {
; CHECK-GI-NEXT: cmlt v6.4s, v1.4s, #0
; CHECK-GI-NEXT: ldr q4, [x8, :lo12:.LCPI20_1]
; CHECK-GI-NEXT: adrp x8, .LCPI20_0
-; CHECK-GI-NEXT: cmlt v7.4s, v2.4s, #0
-; CHECK-GI-NEXT: cmlt v16.4s, v3.4s, #0
-; CHECK-GI-NEXT: ldr d17, [x8, :lo12:.LCPI20_0]
+; CHECK-GI-NEXT: cmlt v16.4s, v2.4s, #0
+; CHECK-GI-NEXT: cmlt v17.4s, v3.4s, #0
+; CHECK-GI-NEXT: ldr q7, [x8, :lo12:.LCPI20_0]
; CHECK-GI-NEXT: adrp x8, .LCPI20_2
; CHECK-GI-NEXT: neg v4.4s, v4.4s
-; CHECK-GI-NEXT: ushll v17.4s, v17.4h, #0
+; CHECK-GI-NEXT: ldr q18, [x8, :lo12:.LCPI20_2]
+; CHECK-GI-NEXT: ushll v19.4s, v7.4h, #0
+; CHECK-GI-NEXT: ushll2 v7.4s, v7.8h, #0
; CHECK-GI-NEXT: ushl v5.4s, v5.4s, v4.4s
; CHECK-GI-NEXT: ushl v6.4s, v6.4s, v4.4s
-; CHECK-GI-NEXT: ushl v7.4s, v7.4s, v4.4s
-; CHECK-GI-NEXT: ushl v4.4s, v16.4s, v4.4s
-; CHECK-GI-NEXT: ldr q16, [x8, :lo12:.LCPI20_2]
-; CHECK-GI-NEXT: shl v17.4s, v17.4s, #31
-; CHECK-GI-NEXT: neg v16.4s, v16.4s
+; CHECK-GI-NEXT: ushl v16.4s, v16.4s, v4.4s
+; CHECK-GI-NEXT: ushl v4.4s, v17.4s, v4.4s
+; CHECK-GI-NEXT: neg v17.4s, v18.4s
+; CHECK-GI-NEXT: shl v18.4s, v19.4s, #31
+; CHECK-GI-NEXT: shl v7.4s, v7.4s, #31
; CHECK-GI-NEXT: add v5.4s, v0.4s, v5.4s
; CHECK-GI-NEXT: add v6.4s, v1.4s, v6.4s
-; CHECK-GI-NEXT: add v7.4s, v2.4s, v7.4s
+; CHECK-GI-NEXT: add v16.4s, v2.4s, v16.4s
; CHECK-GI-NEXT: add v4.4s, v3.4s, v4.4s
-; CHECK-GI-NEXT: cmlt v17.4s, v17.4s, #0
-; CHECK-GI-NEXT: sshl v5.4s, v5.4s, v16.4s
-; CHECK-GI-NEXT: sshl v6.4s, v6.4s, v16.4s
-; CHECK-GI-NEXT: sshl v7.4s, v7.4s, v16.4s
-; CHECK-GI-NEXT: sshl v4.4s, v4.4s, v16.4s
-; CHECK-GI-NEXT: bif v0.16b, v5.16b, v17.16b
-; CHECK-GI-NEXT: bif v1.16b, v6.16b, v17.16b
-; CHECK-GI-NEXT: bif v2.16b, v7.16b, v17.16b
-; CHECK-GI-NEXT: bif v3.16b, v4.16b, v17.16b
+; CHECK-GI-NEXT: cmlt v18.4s, v18.4s, #0
+; CHECK-GI-NEXT: cmlt v7.4s, v7.4s, #0
+; CHECK-GI-NEXT: sshl v5.4s, v5.4s, v17.4s
+; CHECK-GI-NEXT: sshl v6.4s, v6.4s, v17.4s
+; CHECK-GI-NEXT: sshl v16.4s, v16.4s, v17.4s
+; CHECK-GI-NEXT: sshl v4.4s, v4.4s, v17.4s
+; CHECK-GI-NEXT: bif v0.16b, v5.16b, v18.16b
+; CHECK-GI-NEXT: bif v1.16b, v6.16b, v7.16b
+; CHECK-GI-NEXT: bif v2.16b, v16.16b, v18.16b
+; CHECK-GI-NEXT: bif v3.16b, v4.16b, v7.16b
; CHECK-GI-NEXT: ret
%1 = sdiv <16 x i32> %x, <i32 1, i32 4, i32 8, i32 16, i32 1, i32 4, i32 8, i32 16, i32 1, i32 4, i32 8, i32 16, i32 1, i32 4, i32 8, i32 16>
ret <16 x i32> %1
@@ -687,45 +690,49 @@ define <8 x i64> @combine_vec_sdiv_by_pow2b_v8i64(<8 x i64> %x) {
; CHECK-GI-NEXT: cmlt v7.2d, v0.2d, #0
; CHECK-GI-NEXT: ldr q4, [x8, :lo12:.LCPI23_2]
; CHECK-GI-NEXT: adrp x8, .LCPI23_1
-; CHECK-GI-NEXT: ldr d5, [x9, :lo12:.LCPI23_0]
-; CHECK-GI-NEXT: ldr q6, [x8, :lo12:.LCPI23_1]
+; CHECK-GI-NEXT: ldr q6, [x9, :lo12:.LCPI23_0]
+; CHECK-GI-NEXT: ldr q5, [x8, :lo12:.LCPI23_1]
; CHECK-GI-NEXT: cmlt v16.2d, v1.2d, #0
; CHECK-GI-NEXT: cmlt v17.2d, v2.2d, #0
; CHECK-GI-NEXT: neg v4.2d, v4.2d
-; CHECK-GI-NEXT: ushll v5.4s, v5.4h, #0
; CHECK-GI-NEXT: cmlt v19.2d, v3.2d, #0
-; CHECK-GI-NEXT: neg v6.2d, v6.2d
+; CHECK-GI-NEXT: ushll v20.4s, v6.4h, #0
+; CHECK-GI-NEXT: neg v5.2d, v5.2d
+; CHECK-GI-NEXT: ushll2 v6.4s, v6.8h, #0
; CHECK-GI-NEXT: adrp x8, .LCPI23_4
; CHECK-GI-NEXT: ldr q18, [x8, :lo12:.LCPI23_4]
; CHECK-GI-NEXT: adrp x8, .LCPI23_3
; CHECK-GI-NEXT: ushl v7.2d, v7.2d, v4.2d
-; CHECK-GI-NEXT: ushll v20.2d, v5.2s, #0
; CHECK-GI-NEXT: ushl v4.2d, v17.2d, v4.2d
-; CHECK-GI-NEXT: ushl v16.2d, v16.2d, v6.2d
-; CHECK-GI-NEXT: ushll v17.2d, v5.2s, #0
-; CHECK-GI-NEXT: ushl v6.2d, v19.2d, v6.2d
-; CHECK-GI-NEXT: ushll2 v5.2d, v5.4s, #0
-; CHECK-GI-NEXT: ldr q19, [x8, :lo12:.LCPI23_3]
-; CHECK-GI-NEXT: neg v18.2d, v18.2d
+; CHECK-GI-NEXT: ushll v17.2d, v20.2s, #0
+; CHECK-GI-NEXT: ushl v16.2d, v16.2d, v5.2d
+; CHECK-GI-NEXT: ushl v5.2d, v19.2d, v5.2d
+; CHECK-GI-NEXT: ushll2 v19.2d, v20.4s, #0
+; CHECK-GI-NEXT: ushll v20.2d, v6.2s, #0
+; CHECK-GI-NEXT: ushll2 v6.2d, v6.4s, #0
+; CHECK-GI-NEXT: ldr q21, [x8, :lo12:.LCPI23_3]
; CHECK-GI-NEXT: add v7.2d, v0.2d, v7.2d
-; CHECK-GI-NEXT: shl v20.2d, v20.2d, #63
; CHECK-GI-NEXT: add v4.2d, v2.2d, v4.2d
-; CHECK-GI-NEXT: add v1.2d, v1.2d, v16.2d
-; CHECK-GI-NEXT: shl v16.2d, v17.2d, #63
-; CHECK-GI-NEXT: neg v17.2d, v19.2d
-; CHECK-GI-NEXT: add v3.2d, v3.2d, v6.2d
-; CHECK-GI-NEXT: shl v5.2d, v5.2d, #63
-; CHECK-GI-NEXT: sshl v6.2d, v7.2d, v18.2d
-; CHECK-GI-NEXT: cmlt v7.2d, v20.2d, #0
+; CHECK-GI-NEXT: neg v18.2d, v18.2d
+; CHECK-GI-NEXT: add v16.2d, v1.2d, v16.2d
+; CHECK-GI-NEXT: add v5.2d, v3.2d, v5.2d
+; CHECK-GI-NEXT: neg v21.2d, v21.2d
+; CHECK-GI-NEXT: shl v17.2d, v17.2d, #63
+; CHECK-GI-NEXT: shl v19.2d, v19.2d, #63
+; CHECK-GI-NEXT: shl v20.2d, v20.2d, #63
+; CHECK-GI-NEXT: shl v6.2d, v6.2d, #63
+; CHECK-GI-NEXT: sshl v7.2d, v7.2d, v18.2d
; CHECK-GI-NEXT: sshl v4.2d, v4.2d, v18.2d
-; CHECK-GI-NEXT: cmlt v16.2d, v16.2d, #0
-; CHECK-GI-NEXT: sshl v1.2d, v1.2d, v17.2d
-; CHECK-GI-NEXT: sshl v3.2d, v3.2d, v17.2d
-; CHECK-GI-NEXT: cmlt v5.2d, v5.2d, #0
-; CHECK-GI-NEXT: bif v0.16b, v6.16b, v7.16b
-; CHECK-GI-NEXT: bif v2.16b, v4.16b, v16.16b
-; CHECK-GI-NEXT: bic v1.16b, v1.16b, v5.16b
-; CHECK-GI-NEXT: bic v3.16b, v3.16b, v5.16b
+; CHECK-GI-NEXT: sshl v16.2d, v16.2d, v21.2d
+; CHECK-GI-NEXT: sshl v5.2d, v5.2d, v21.2d
+; CHECK-GI-NEXT: cmlt v17.2d, v17.2d, #0
+; CHECK-GI-NEXT: cmlt v18.2d, v19.2d, #0
+; CHECK-GI-NEXT: cmlt v19.2d, v20.2d, #0
+; CHECK-GI-NEXT: cmlt v6.2d, v6.2d, #0
+; CHECK-GI-NEXT: bif v0.16b, v7.16b, v17.16b
+; CHECK-GI-NEXT: bif v1.16b, v16.16b, v18.16b
+; CHECK-GI-NEXT: bif v2.16b, v4.16b, v19.16b
+; CHECK-GI-NEXT: bif v3.16b, v5.16b, v6.16b
; CHECK-GI-NEXT: ret
%1 = sdiv <8 x i64> %x, <i64 1, i64 4, i64 8, i64 16, i64 1, i64 4, i64 8, i64 16>
ret <8 x i64> %1
@@ -1156,8 +1163,7 @@ define <8 x i16> @combine_vec_sdiv_nonuniform7(<8 x i16> %x) {
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: adrp x8, .LCPI34_0
; CHECK-GI-NEXT: neg v2.8h, v0.8h
-; CHECK-GI-NEXT: ldr d1, [x8, :lo12:.LCPI34_0]
-; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI34_0]
; CHECK-GI-NEXT: shl v1.8h, v1.8h, #15
; CHECK-GI-NEXT: cmlt v1.8h, v1.8h, #0
; CHECK-GI-NEXT: bit v0.16b, v2.16b, v1.16b
diff --git a/llvm/test/CodeGen/AArch64/extract-vector-elt.ll b/llvm/test/CodeGen/AArch64/extract-vector-elt.ll
index 3ab413b50e17d..515c5e45ad965 100644
--- a/llvm/test/CodeGen/AArch64/extract-vector-elt.ll
+++ b/llvm/test/CodeGen/AArch64/extract-vector-elt.ll
@@ -694,9 +694,6 @@ define i32 @extract_v4i32_abs_const(<4 x float> %a, i32 %c) {
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: adrp x8, .LCPI25_0
; CHECK-GI-NEXT: ldr q0, [x8, :lo12:.LCPI25_0]
-; CHECK-GI-NEXT: frintp v0.4s, v0.4s
-; CHECK-GI-NEXT: frintm v0.4s, v0.4s
-; CHECK-GI-NEXT: fabs v0.4s, v0.4s
; CHECK-GI-NEXT: fcvtzs v0.4s, v0.4s
; CHECK-GI-NEXT: abs v0.4s, v0.4s
; CHECK-GI-NEXT: mov s0, v0.s[1]
@@ -736,9 +733,6 @@ define i32 @extract_v4i32_abs_half_const(<4 x float> %a, i32 %c) {
; CHECK-GI-NEXT: ldr q0, [x8, :lo12:.LCPI26_0]
; CHECK-GI-NEXT: mov w8, w0
; CHECK-GI-NEXT: and x8, x8, #0x3
-; CHECK-GI-NEXT: frintp v0.4s, v0.4s
-; CHECK-GI-NEXT: frintm v0.4s, v0.4s
-; CHECK-GI-NEXT: fabs v0.4s, v0.4s
; CHECK-GI-NEXT: fcvtzs v0.4s, v0.4s
; CHECK-GI-NEXT: abs v0.4s, v0.4s
; CHECK-GI-NEXT: str q0, [sp]
diff --git a/llvm/test/CodeGen/AArch64/fcvt_combine.ll b/llvm/test/CodeGen/AArch64/fcvt_combine.ll
index 70220cbaa99aa..3a5654ca2ba82 100644
--- a/llvm/test/CodeGen/AArch64/fcvt_combine.ll
+++ b/llvm/test/CodeGen/AArch64/fcvt_combine.ll
@@ -356,12 +356,11 @@ define <8 x i16> @test_v8f16(<8 x half> %in) {
;
; CHECK-NO16-GI-LABEL: test_v8f16:
; CHECK-NO16-GI: // %bb.0:
-; CHECK-NO16-GI-NEXT: movi v1.4h, #68, lsl #8
; CHECK-NO16-GI-NEXT: fcvtl v2.4s, v0.4h
; CHECK-NO16-GI-NEXT: fcvtl2 v0.4s, v0.8h
-; CHECK-NO16-GI-NEXT: fcvtl v1.4s, v1.4h
-; CHECK-NO16-GI-NEXT: fmul v2.4s, v2.4s, v1.4s
-; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-NO16-GI-NEXT: fmov s1, #4.00000000
+; CHECK-NO16-GI-NEXT: fmul v2.4s, v2.4s, v1.s[0]
+; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.s[0]
; CHECK-NO16-GI-NEXT: fcvtn v1.4h, v2.4s
; CHECK-NO16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-NO16-GI-NEXT: fcvtl v1.4s, v1.4h
@@ -394,10 +393,9 @@ define <4 x i16> @test_v4f16(<4 x half> %in) {
;
; CHECK-NO16-GI-LABEL: test_v4f16:
; CHECK-NO16-GI: // %bb.0:
-; CHECK-NO16-GI-NEXT: movi v1.4h, #68, lsl #8
; CHECK-NO16-GI-NEXT: fcvtl v0.4s, v0.4h
-; CHECK-NO16-GI-NEXT: fcvtl v1.4s, v1.4h
-; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-NO16-GI-NEXT: fmov s1, #4.00000000
+; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.s[0]
; CHECK-NO16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-NO16-GI-NEXT: fcvtl v0.4s, v0.4h
; CHECK-NO16-GI-NEXT: fcvtzu v0.4s, v0.4s
@@ -429,10 +427,9 @@ define <4 x i32> @test_v4f16_i32(<4 x half> %in) {
;
; CHECK-NO16-GI-LABEL: test_v4f16_i32:
; CHECK-NO16-GI: // %bb.0:
-; CHECK-NO16-GI-NEXT: movi v1.4h, #68, lsl #8
; CHECK-NO16-GI-NEXT: fcvtl v0.4s, v0.4h
-; CHECK-NO16-GI-NEXT: fcvtl v1.4s, v1.4h
-; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-NO16-GI-NEXT: fmov s1, #4.00000000
+; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.s[0]
; CHECK-NO16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-NO16-GI-NEXT: fcvtl v0.4s, v0.4h
; CHECK-NO16-GI-NEXT: fcvtzs v0.4s, v0.4s
@@ -470,10 +467,10 @@ define <8 x i16> @test_v8f16_nan(<8 x half> %a) {
;
; CHECK-NO16-GI-LABEL: test_v8f16_nan:
; CHECK-NO16-GI: // %bb.0: // %entry
-; CHECK-NO16-GI-NEXT: movi v1.4h, #126, lsl #8
+; CHECK-NO16-GI-NEXT: mvni v1.4s, #63, msl #16
; CHECK-NO16-GI-NEXT: fcvtl v2.4s, v0.4h
; CHECK-NO16-GI-NEXT: fcvtl2 v0.4s, v0.8h
-; CHECK-NO16-GI-NEXT: fcvtl v1.4s, v1.4h
+; CHECK-NO16-GI-NEXT: fneg v1.4s, v1.4s
; CHECK-NO16-GI-NEXT: fmul v2.4s, v2.4s, v1.4s
; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
; CHECK-NO16-GI-NEXT: fcvtn v1.4h, v2.4s
@@ -946,12 +943,11 @@ define <8 x i16> @test_v8f16_sat(<8 x half> %in) {
;
; CHECK-NO16-GI-LABEL: test_v8f16_sat:
; CHECK-NO16-GI: // %bb.0:
-; CHECK-NO16-GI-NEXT: movi v1.4h, #68, lsl #8
; CHECK-NO16-GI-NEXT: fcvtl v2.4s, v0.4h
+; CHECK-NO16-GI-NEXT: fmov s1, #4.00000000
; CHECK-NO16-GI-NEXT: fcvtl2 v0.4s, v0.8h
-; CHECK-NO16-GI-NEXT: fcvtl v1.4s, v1.4h
-; CHECK-NO16-GI-NEXT: fmul v2.4s, v2.4s, v1.4s
-; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-NO16-GI-NEXT: fmul v2.4s, v2.4s, v1.s[0]
+; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.s[0]
; CHECK-NO16-GI-NEXT: fcvtn v1.4h, v2.4s
; CHECK-NO16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-NO16-GI-NEXT: fcvtl v1.4s, v1.4h
@@ -985,10 +981,9 @@ define <4 x i16> @test_v4f16_sat(<4 x half> %in) {
;
; CHECK-NO16-GI-LABEL: test_v4f16_sat:
; CHECK-NO16-GI: // %bb.0:
-; CHECK-NO16-GI-NEXT: movi v1.4h, #68, lsl #8
; CHECK-NO16-GI-NEXT: fcvtl v0.4s, v0.4h
-; CHECK-NO16-GI-NEXT: fcvtl v1.4s, v1.4h
-; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-NO16-GI-NEXT: fmov s1, #4.00000000
+; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.s[0]
; CHECK-NO16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-NO16-GI-NEXT: fcvtl v0.4s, v0.4h
; CHECK-NO16-GI-NEXT: fcvtzu v0.4s, v0.4s
@@ -1020,10 +1015,9 @@ define <4 x i32> @test_v4f16_i32_sat(<4 x half> %in) {
;
; CHECK-NO16-GI-LABEL: test_v4f16_i32_sat:
; CHECK-NO16-GI: // %bb.0:
-; CHECK-NO16-GI-NEXT: movi v1.4h, #68, lsl #8
; CHECK-NO16-GI-NEXT: fcvtl v0.4s, v0.4h
-; CHECK-NO16-GI-NEXT: fcvtl v1.4s, v1.4h
-; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-NO16-GI-NEXT: fmov s1, #4.00000000
+; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.s[0]
; CHECK-NO16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-NO16-GI-NEXT: fcvtl v0.4s, v0.4h
; CHECK-NO16-GI-NEXT: fcvtzs v0.4s, v0.4s
diff --git a/llvm/test/CodeGen/AArch64/hadd-combine.ll b/llvm/test/CodeGen/AArch64/hadd-combine.ll
index 450069cd27428..6dbe5b2c3ecf7 100644
--- a/llvm/test/CodeGen/AArch64/hadd-combine.ll
+++ b/llvm/test/CodeGen/AArch64/hadd-combine.ll
@@ -95,19 +95,10 @@ define <8 x i16> @haddu_const_both() {
}
define <8 x i16> @haddu_const_bothhigh() {
-; CHECK-SD-LABEL: haddu_const_bothhigh:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: mvni v0.8h, #1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: haddu_const_bothhigh:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi d0, #0xffffffffffffffff
-; CHECK-GI-NEXT: mvni v1.4h, #1
-; CHECK-GI-NEXT: uaddl v1.4s, v1.4h, v0.4h
-; CHECK-GI-NEXT: shrn v0.4h, v1.4s, #1
-; CHECK-GI-NEXT: shrn2 v0.8h, v1.4s, #1
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: haddu_const_bothhigh:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mvni v0.8h, #1
+; CHECK-NEXT: ret
%ext1 = zext <8 x i16> <i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534> to <8 x i32>
%ext2 = zext <8 x i16> <i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535> to <8 x i32>
%add = add <8 x i32> %ext1, %ext2
@@ -341,9 +332,7 @@ define <8 x i16> @hadds_const_bothhigh() {
; CHECK-GI-LABEL: hadds_const_bothhigh:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: adrp x8, .LCPI19_0
-; CHECK-GI-NEXT: mvni v0.8h, #128, lsl #8
-; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI19_0]
-; CHECK-GI-NEXT: shadd v0.8h, v1.8h, v0.8h
+; CHECK-GI-NEXT: ldr q0, [x8, :lo12:.LCPI19_0]
; CHECK-GI-NEXT: ret
%ext1 = sext <8 x i16> <i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766> to <8 x i32>
%ext2 = sext <8 x i16> <i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767> to <8 x i32>
@@ -615,21 +604,10 @@ define <8 x i16> @rhaddu_const_both() {
}
define <8 x i16> @rhaddu_const_bothhigh() {
-; CHECK-SD-LABEL: rhaddu_const_bothhigh:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: movi v0.2d, #0xffffffffffffffff
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: rhaddu_const_bothhigh:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi d0, #0xffffffffffffffff
-; CHECK-GI-NEXT: mvni v1.4h, #1
-; CHECK-GI-NEXT: movi v2.4s, #1
-; CHECK-GI-NEXT: uaddl v0.4s, v1.4h, v0.4h
-; CHECK-GI-NEXT: add v1.4s, v0.4s, v2.4s
-; CHECK-GI-NEXT: shrn v0.4h, v1.4s, #1
-; CHECK-GI-NEXT: shrn2 v0.8h, v1.4s, #1
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: rhaddu_const_bothhigh:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v0.2d, #0xffffffffffffffff
+; CHECK-NEXT: ret
%ext1 = zext <8 x i16> <i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534> to <8 x i32>
%ext2 = zext <8 x i16> <i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535> to <8 x i32>
%add1 = add <8 x i32> %ext1, %ext2
@@ -863,18 +841,10 @@ define <8 x i16> @rhadds_const_both() {
}
define <8 x i16> @rhadds_const_bothhigh() {
-; CHECK-SD-LABEL: rhadds_const_bothhigh:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: mvni v0.8h, #128, lsl #8
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: rhadds_const_bothhigh:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: adrp x8, .LCPI49_0
-; CHECK-GI-NEXT: mvni v0.8h, #128, lsl #8
-; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI49_0]
-; CHECK-GI-NEXT: srhadd v0.8h, v1.8h, v0.8h
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: rhadds_const_bothhigh:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mvni v0.8h, #128, lsl #8
+; CHECK-NEXT: ret
%ext1 = sext <8 x i16> <i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766> to <8 x i32>
%ext2 = sext <8 x i16> <i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767> to <8 x i32>
%add1 = add <8 x i32> %ext1, %ext2
diff --git a/llvm/test/CodeGen/AArch64/half-precision-signof-no-assert.ll b/llvm/test/CodeGen/AArch64/half-precision-signof-no-assert.ll
index b5831e3236622..d538004d5cd9a 100644
--- a/llvm/test/CodeGen/AArch64/half-precision-signof-no-assert.ll
+++ b/llvm/test/CodeGen/AArch64/half-precision-signof-no-assert.ll
@@ -34,15 +34,13 @@ define ptr @fn(ptr %in, ptr %out) {
;
; CHECK-GI-LABEL: fn:
; CHECK-GI: // %bb.0: // %fn
-; CHECK-GI-NEXT: movi v0.2d, #0000000000000000
; CHECK-GI-NEXT: ldr d1, [x0]
+; CHECK-GI-NEXT: movi v0.4s, #1
; CHECK-GI-NEXT: fcvtl v1.4s, v1.4h
-; CHECK-GI-NEXT: fcvtl v0.4s, v0.4h
-; CHECK-GI-NEXT: fcmgt v2.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT: fcmgt v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: movi v1.4s, #1
-; CHECK-GI-NEXT: orr v0.16b, v0.16b, v2.16b
-; CHECK-GI-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT: fcmgt v2.4s, v1.4s, #0.0
+; CHECK-GI-NEXT: fcmlt v1.4s, v1.4s, #0.0
+; CHECK-GI-NEXT: orr v1.16b, v1.16b, v2.16b
+; CHECK-GI-NEXT: and v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT: mvni v1.4h, #128, lsl #8
; CHECK-GI-NEXT: ucvtf v0.4s, v0.4s
; CHECK-GI-NEXT: fcvtn v0.4h, v0.4s
diff --git a/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll b/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll
index 017f7ea0de891..ada83c54680a7 100644
--- a/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll
@@ -2528,9 +2528,7 @@ define <2 x i32> @fcmal2xfloat(<2 x float> %A, <2 x float> %B) {
;
; CHECK-GI-LABEL: fcmal2xfloat:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi v0.2s, #1
-; CHECK-GI-NEXT: shl v0.2s, v0.2s, #31
-; CHECK-GI-NEXT: cmlt v0.2s, v0.2s, #0
+; CHECK-GI-NEXT: movi d0, #0xffffffffffffffff
; CHECK-GI-NEXT: ret
%tmp3 = fcmp true <2 x float> %A, %B
%tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
@@ -2538,34 +2536,19 @@ define <2 x i32> @fcmal2xfloat(<2 x float> %A, <2 x float> %B) {
}
define <4 x i32> @fcmal4xfloat(<4 x float> %A, <4 x float> %B) {
-; CHECK-SD-LABEL: fcmal4xfloat:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: movi v0.2d, #0xffffffffffffffff
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: fcmal4xfloat:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi v0.4s, #1
-; CHECK-GI-NEXT: shl v0.4s, v0.4s, #31
-; CHECK-GI-NEXT: cmlt v0.4s, v0.4s, #0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: fcmal4xfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v0.2d, #0xffffffffffffffff
+; CHECK-NEXT: ret
%tmp3 = fcmp true <4 x float> %A, %B
%tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
ret <4 x i32> %tmp4
}
define <2 x i64> @fcmal2xdouble(<2 x double> %A, <2 x double> %B) {
-; CHECK-SD-LABEL: fcmal2xdouble:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: movi v0.2d, #0xffffffffffffffff
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: fcmal2xdouble:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: adrp x8, .LCPI221_0
-; CHECK-GI-NEXT: ldr q0, [x8, :lo12:.LCPI221_0]
-; CHECK-GI-NEXT: shl v0.2d, v0.2d, #63
-; CHECK-GI-NEXT: cmlt v0.2d, v0.2d, #0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: fcmal2xdouble:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v0.2d, #0xffffffffffffffff
+; CHECK-NEXT: ret
%tmp3 = fcmp true <2 x double> %A, %B
%tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
ret <2 x i64> %tmp4
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-or-redundant.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-or-redundant.mir
index e75a811c0bee6..6e3d2f7639887 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-or-redundant.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-or-redundant.mir
@@ -201,12 +201,9 @@ tracksRegLiveness: true
body: |
bb.0:
; CHECK-LABEL: name: vector_dont_combine_const_too_wide
- ; CHECK: %fifteen:_(s16) = G_CONSTANT i16 15
- ; CHECK-NEXT: %too_wide:_(s16) = G_CONSTANT i16 257
- ; CHECK-NEXT: %mask:_(s16) = G_CONSTANT i16 255
- ; CHECK-NEXT: %c1:_(<2 x s16>) = G_BUILD_VECTOR %fifteen(s16), %too_wide(s16)
- ; CHECK-NEXT: %c2:_(<2 x s16>) = G_BUILD_VECTOR %mask(s16), %mask(s16)
- ; CHECK-NEXT: %and:_(<2 x s16>) = G_OR %c1, %c2
+ ; CHECK: %mask:_(s16) = G_CONSTANT i16 255
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 511
+ ; CHECK-NEXT: %and:_(<2 x s16>) = G_BUILD_VECTOR %mask(s16), [[C]](s16)
; CHECK-NEXT: $vgpr0 = COPY %and(<2 x s16>)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%fifteen:_(s16) = G_CONSTANT i16 15
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-redundant-and.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-redundant-and.mir
index 70bcd3d95c662..a476ca669cc00 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-redundant-and.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-redundant-and.mir
@@ -215,11 +215,8 @@ body: |
bb.0:
; CHECK-LABEL: name: vector_dont_combine_const_too_wide
; CHECK: %fifteen:_(s16) = G_CONSTANT i16 15
- ; CHECK-NEXT: %too_wide:_(s16) = G_CONSTANT i16 257
- ; CHECK-NEXT: %mask:_(s16) = G_CONSTANT i16 255
- ; CHECK-NEXT: %c1:_(<2 x s16>) = G_BUILD_VECTOR %fifteen(s16), %too_wide(s16)
- ; CHECK-NEXT: %c2:_(<2 x s16>) = G_BUILD_VECTOR %mask(s16), %mask(s16)
- ; CHECK-NEXT: %and:_(<2 x s16>) = G_AND %c1, %c2
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 1
+ ; CHECK-NEXT: %and:_(<2 x s16>) = G_BUILD_VECTOR %fifteen(s16), [[C]](s16)
; CHECK-NEXT: $vgpr0 = COPY %and(<2 x s16>)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%fifteen:_(s16) = G_CONSTANT i16 15
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir
index 5a337b251bab6..5ac9d99af28ba 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir
@@ -375,14 +375,16 @@ body: |
; GFX6: liveins: $vgpr0
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: %shl:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: %shl:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C1]](s32)
; GFX6-NEXT: $vgpr0_vgpr1 = COPY %shl(<2 x s32>)
;
; GFX9-LABEL: name: do_not_shl_v2s32_zero_by_16_from_zext_v2s16
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: %shl:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32)
+ ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX9-NEXT: %shl:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C1]](s32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY %shl(<2 x s32>)
%zero:_(s16) = G_CONSTANT i16 0
%zerovector:_(<2 x s16>) = G_BUILD_VECTOR %zero, %zero:_(s16)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.prelegal.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.prelegal.mir
index 25feacdca01ce..628ac494b1304 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.prelegal.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.prelegal.mir
@@ -246,16 +246,16 @@ body: |
; GFX6-LABEL: name: do_not_shl_v2s32_zero_by_16_from_zext_v2s16
; GFX6: liveins: $vgpr0, $vgpr1
; GFX6-NEXT: {{ $}}
- ; GFX6-NEXT: %6:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: %shl:_(<2 x s32>) = G_BUILD_VECTOR %6(s32), %6(s32)
- ; GFX6-NEXT: $vgpr0_vgpr1 = COPY %shl(<2 x s32>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: %extend:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32)
+ ; GFX6-NEXT: $vgpr0_vgpr1 = COPY %extend(<2 x s32>)
;
; GFX9-LABEL: name: do_not_shl_v2s32_zero_by_16_from_zext_v2s16
; GFX9: liveins: $vgpr0, $vgpr1
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: %6:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: %shl:_(<2 x s32>) = G_BUILD_VECTOR %6(s32), %6(s32)
- ; GFX9-NEXT: $vgpr0_vgpr1 = COPY %shl(<2 x s32>)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX9-NEXT: %extend:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32)
+ ; GFX9-NEXT: $vgpr0_vgpr1 = COPY %extend(<2 x s32>)
%zero:_(s16) = G_CONSTANT i16 0
%zerovector:_(<2 x s16>) = G_BUILD_VECTOR %zero, %zero:_(s16)
%shiftamt:_(s16) = G_CONSTANT i16 16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-ashr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-ashr.mir
index 95009fb1b686e..14b5a8833d9dc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-ashr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-ashr.mir
@@ -59,12 +59,9 @@ tracksRegLiveness: true
body: |
bb.0:
; CHECK-LABEL: name: constant_fold_ashr_v2s16_v2s16
- ; CHECK: %val0:_(s16) = G_CONSTANT i16 1234
- ; CHECK-NEXT: %val1:_(s16) = G_CONSTANT i16 -5678
- ; CHECK-NEXT: %val:_(<2 x s16>) = G_BUILD_VECTOR %val0(s16), %val1(s16)
- ; CHECK-NEXT: %shift_amt_elt:_(s16) = G_CONSTANT i16 8
- ; CHECK-NEXT: %shift_amt:_(<2 x s16>) = G_BUILD_VECTOR %shift_amt_elt(s16), %shift_amt_elt(s16)
- ; CHECK-NEXT: %shift:_(<2 x s16>) = G_ASHR %val, %shift_amt(<2 x s16>)
+ ; CHECK: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 4
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -23
+ ; CHECK-NEXT: %shift:_(<2 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C1]](s16)
; CHECK-NEXT: $vgpr0 = COPY %shift(<2 x s16>)
%val0:_(s16) = G_CONSTANT i16 1234
%val1:_(s16) = G_CONSTANT i16 -5678
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-lshr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-lshr.mir
index 09844b944eea2..645c0333ffe09 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-lshr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-lshr.mir
@@ -59,12 +59,9 @@ tracksRegLiveness: true
body: |
bb.0:
; CHECK-LABEL: name: constant_fold_lshr_v2s16_v2s16
- ; CHECK: %val0:_(s16) = G_CONSTANT i16 1234
- ; CHECK-NEXT: %val1:_(s16) = G_CONSTANT i16 -5678
- ; CHECK-NEXT: %val:_(<2 x s16>) = G_BUILD_VECTOR %val0(s16), %val1(s16)
- ; CHECK-NEXT: %shift_amt_elt:_(s16) = G_CONSTANT i16 8
- ; CHECK-NEXT: %shift_amt:_(<2 x s16>) = G_BUILD_VECTOR %shift_amt_elt(s16), %shift_amt_elt(s16)
- ; CHECK-NEXT: %shift:_(<2 x s16>) = G_LSHR %val, %shift_amt(<2 x s16>)
+ ; CHECK: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 4
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 233
+ ; CHECK-NEXT: %shift:_(<2 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C1]](s16)
; CHECK-NEXT: $vgpr0 = COPY %shift(<2 x s16>)
%val0:_(s16) = G_CONSTANT i16 1234
%val1:_(s16) = G_CONSTANT i16 -5678
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-shl.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-shl.mir
index ae2c63942dbff..a90a5025bc48d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-shl.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-shl.mir
@@ -59,12 +59,9 @@ tracksRegLiveness: true
body: |
bb.0:
; CHECK-LABEL: name: constant_fold_shl_v2s16_v2s16
- ; CHECK: %val0:_(s16) = G_CONSTANT i16 1234
- ; CHECK-NEXT: %val1:_(s16) = G_CONSTANT i16 5678
- ; CHECK-NEXT: %val:_(<2 x s16>) = G_BUILD_VECTOR %val0(s16), %val1(s16)
- ; CHECK-NEXT: %shift_amt_elt:_(s16) = G_CONSTANT i16 8
- ; CHECK-NEXT: %shift_amt:_(<2 x s16>) = G_BUILD_VECTOR %shift_amt_elt(s16), %shift_amt_elt(s16)
- ; CHECK-NEXT: %shift:_(<2 x s16>) = G_SHL %val, %shift_amt(<2 x s16>)
+ ; CHECK: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 -11776
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 11776
+ ; CHECK-NEXT: %shift:_(<2 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C1]](s16)
; CHECK-NEXT: $vgpr0 = COPY %shift(<2 x s16>)
%val0:_(s16) = G_CONSTANT i16 1234
%val1:_(s16) = G_CONSTANT i16 5678
>From 45bd4893d500b9693c0cd3b5a349b4651226e065 Mon Sep 17 00:00:00 2001
From: vg0204 <Vikash.Gupta at amd.com>
Date: Mon, 7 Sep 2026 13:07:03 +0530
Subject: [PATCH 2/7] Added ConstantFoldFP for host-libm and addressed reviewed
changes.
---
llvm/include/llvm/Analysis/ConstantFolding.h | 8 +++
llvm/lib/Analysis/ConstantFolding.cpp | 46 ++++++------
llvm/lib/CodeGen/GlobalISel/Utils.cpp | 31 ++++----
.../combine-constant-fold-fp-rounding.mir | 28 ++++++++
.../postlegalizer-combiner-constant-fold.mir | 72 +++++++++++--------
...relegalizer-combiner-constant-fold-fma.mir | 18 +----
6 files changed, 123 insertions(+), 80 deletions(-)
diff --git a/llvm/include/llvm/Analysis/ConstantFolding.h b/llvm/include/llvm/Analysis/ConstantFolding.h
index ddbeecf0b43f0..e751fafcd7981 100644
--- a/llvm/include/llvm/Analysis/ConstantFolding.h
+++ b/llvm/include/llvm/Analysis/ConstantFolding.h
@@ -19,6 +19,7 @@
#ifndef LLVM_ANALYSIS_CONSTANTFOLDING_H
#define LLVM_ANALYSIS_CONSTANTFOLDING_H
+#include "llvm/ADT/APFloat.h"
#include "llvm/Support/Compiler.h"
#include <stdint.h>
@@ -108,6 +109,13 @@ ConstantFoldFPInstOperands(unsigned Opcode, Constant *LHS, Constant *RHS,
const DataLayout &DL, const Instruction *I,
bool AllowNonDeterministic = true);
+/// Evaluate \p NativeFP on \p V using the host libm, then convert the result to
+/// \p Ty. Returns nullptr if the host call raised a non-inexact floating-point
+/// exception, or if \p DenormMode is invalid or dynamic.
+LLVM_ABI Constant *
+ConstantFoldFP(double (*NativeFP)(double), const APFloat &V, Type *Ty,
+ DenormalMode DenormMode = DenormalMode::getIEEE());
+
/// Attempt to flush float point constant according to denormal mode set in the
/// instruction's parent function attributes. If so, return a zero with the
/// correct sign, otherwise return the original constant. Inputs and outputs to
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index aca2556cd48eb..ae384104f293f 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -2283,29 +2283,6 @@ static APFloat FlushWithDenormKind(const APFloat &V,
}
}
-Constant *ConstantFoldFP(double (*NativeFP)(double), const APFloat &V, Type *Ty,
- DenormalMode DenormMode = DenormalMode::getIEEE()) {
- if (!DenormMode.isValid() ||
- DenormMode.Input == DenormalMode::DenormalModeKind::Dynamic ||
- DenormMode.Output == DenormalMode::DenormalModeKind::Dynamic)
- return nullptr;
-
- llvm_fenv_clearexcept();
- auto Input = FlushWithDenormKind(V, DenormMode.Input);
- double Result = NativeFP(Input.convertToDouble());
- if (llvm_fenv_testexcept()) {
- llvm_fenv_clearexcept();
- return nullptr;
- }
-
- Constant *Output = GetConstantFoldFPValue(Result, Ty);
- if (DenormMode.Output == DenormalMode::DenormalModeKind::IEEE)
- return Output;
- const auto *CFP = static_cast<ConstantFP *>(Output);
- const auto Res = FlushWithDenormKind(CFP->getValueAPF(), DenormMode.Output);
- return ConstantFP::get(Ty->getContext(), Res);
-}
-
#if defined(HAS_IEE754_FLOAT128) && defined(HAS_LOGF128)
Constant *ConstantFoldFP128(float128 (*NativeFP)(float128), const APFloat &V,
Type *Ty) {
@@ -4718,6 +4695,29 @@ ConstantFoldStructCall(StringRef Name, Intrinsic::ID IntrinsicID,
} // end anonymous namespace
+Constant *llvm::ConstantFoldFP(double (*NativeFP)(double), const APFloat &V,
+ Type *Ty, DenormalMode DenormMode) {
+ if (!DenormMode.isValid() ||
+ DenormMode.Input == DenormalMode::DenormalModeKind::Dynamic ||
+ DenormMode.Output == DenormalMode::DenormalModeKind::Dynamic)
+ return nullptr;
+
+ llvm_fenv_clearexcept();
+ auto Input = FlushWithDenormKind(V, DenormMode.Input);
+ double Result = NativeFP(Input.convertToDouble());
+ if (llvm_fenv_testexcept()) {
+ llvm_fenv_clearexcept();
+ return nullptr;
+ }
+
+ Constant *Output = GetConstantFoldFPValue(Result, Ty);
+ if (DenormMode.Output == DenormalMode::DenormalModeKind::IEEE)
+ return Output;
+ const auto *CFP = static_cast<ConstantFP *>(Output);
+ const auto Res = FlushWithDenormKind(CFP->getValueAPF(), DenormMode.Output);
+ return ConstantFP::get(Ty->getContext(), Res);
+}
+
Constant *llvm::ConstantFoldIntrinsic(Intrinsic::ID ID,
ArrayRef<Constant *> Ops, Type *Ty,
const DataLayout &DL, Function *CxtF) {
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 011f23f171089..485b1d2d524f1 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -12,6 +12,7 @@
#include "llvm/CodeGen/GlobalISel/Utils.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
+#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/CodeGenCommonISel.h"
#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
@@ -34,6 +35,8 @@
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/IR/Constants.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Type.h"
#include "llvm/Support/UndefPoison.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/Utils/SizeOpts.h"
@@ -1010,7 +1013,8 @@ SmallVector<APFloat>
llvm::ConstantFoldUnaryFPOp(unsigned Opcode, LLT DstTy, Register Src,
const MachineRegisterInfo &MRI) {
LLT DstEltTy = DstTy.getScalarType();
- auto Fold = [Opcode, DstEltTy](const APFloat &V) -> APFloat {
+ auto Fold = [Opcode, DstEltTy,
+ &MRI](const APFloat &V) -> std::optional<APFloat> {
APFloat Result(V);
bool Unused;
switch (Opcode) {
@@ -1043,22 +1047,21 @@ llvm::ConstantFoldUnaryFPOp(unsigned Opcode, LLT DstTy, Register Src,
APFloat::rmNearestTiesToEven, &Unused);
return Result;
case TargetOpcode::G_FSQRT:
- Result.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
- &Unused);
- Result = APFloat(std::sqrt(Result.convertToDouble()));
- break;
- case TargetOpcode::G_FLOG2:
- Result.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
- &Unused);
- Result = APFloat(std::log2(Result.convertToDouble()));
- break;
+ case TargetOpcode::G_FLOG2: {
+ Type *Ty =
+ Type::getFloatingPointTy(MRI.getMF().getFunction().getContext(),
+ getFltSemanticForLLT(DstEltTy));
+ Constant *C = ConstantFoldFP(Opcode == TargetOpcode::G_FSQRT
+ ? static_cast<double (*)(double)>(sqrt)
+ : static_cast<double (*)(double)>(log2),
+ V, Ty);
+ if (!C)
+ return std::nullopt;
+ return cast<ConstantFP>(C)->getValueAPF();
+ }
default:
llvm_unreachable("unexpected opcode in ConstantFoldUnaryFPOp");
}
- // Only G_FSQRT and G_FLOG2 reach here; convert the double result back to
- // the source (== destination) semantics.
- Result.convert(V.getSemantics(), APFloat::rmNearestTiesToEven, &Unused);
- return Result;
};
auto tryFoldScalar = [&](Register R) -> std::optional<APFloat> {
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-constant-fold-fp-rounding.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-constant-fold-fp-rounding.mir
index e483f86097c43..b44a37b326621 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-constant-fold-fp-rounding.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-constant-fold-fp-rounding.mir
@@ -165,3 +165,31 @@ body: |
RET_ReallyLR implicit $d0
...
+---
+name: fsqrt
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: fsqrt
+ ; CHECK: %res:_(f64) = G_FCONSTANT double 2.000000e+00
+ ; CHECK-NEXT: $d0 = COPY %res(f64)
+ ; CHECK-NEXT: RET_ReallyLR implicit $d0
+ %a:_(f64) = G_FCONSTANT double 4.0
+ %res:_(f64) = G_FSQRT %a
+ $d0 = COPY %res(f64)
+ RET_ReallyLR implicit $d0
+
+...
+---
+name: flog2
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: flog2
+ ; CHECK: %res:_(f64) = G_FCONSTANT double 3.000000e+00
+ ; CHECK-NEXT: $d0 = COPY %res(f64)
+ ; CHECK-NEXT: RET_ReallyLR implicit $d0
+ %a:_(f64) = G_FCONSTANT double 8.0
+ %res:_(f64) = G_FLOG2 %a
+ $d0 = COPY %res(f64)
+ RET_ReallyLR implicit $d0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir
index 66b1b9eb6caea..9fbab67265f34 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir
@@ -411,17 +411,11 @@ body: |
---
name: add_vector
legalized: true
-liveins:
- - { reg: '$q0' }
body: |
bb.1.entry:
- liveins: $q0
-
; Element-wise fold of an integer vector binop: <40, 10> + <2, 2> = <42, 12>.
; CHECK-LABEL: name: add_vector
- ; CHECK: liveins: $q0
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 42
+ ; CHECK: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 42
; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 12
; CHECK-NEXT: %res:_(<2 x i64>) = G_BUILD_VECTOR [[C]](i64), [[C1]](i64)
; CHECK-NEXT: $q0 = COPY %res(<2 x i64>)
@@ -439,17 +433,11 @@ body: |
---
name: fadd_vector
legalized: true
-liveins:
- - { reg: '$q0' }
body: |
bb.1.entry:
- liveins: $q0
-
; Element-wise fold of an FP vector binop: <40.0, 1.0> + <2.0, 2.0>.
; CHECK-LABEL: name: fadd_vector
- ; CHECK: liveins: $q0
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double 4.200000e+01
+ ; CHECK: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double 4.200000e+01
; CHECK-NEXT: [[C1:%[0-9]+]]:_(f64) = G_FCONSTANT double 3.000000e+00
; CHECK-NEXT: %res:_(<2 x f64>) = G_BUILD_VECTOR [[C]](f64), [[C1]](f64)
; CHECK-NEXT: $q0 = COPY %res(<2 x f64>)
@@ -467,17 +455,11 @@ body: |
---
name: zext_vector
legalized: true
-liveins:
- - { reg: '$q0' }
body: |
bb.1.entry:
- liveins: $q0
-
; Element-wise fold of a vector cast: zext(<7, 8> : <2 x i32>) to <2 x i64>.
; CHECK-LABEL: name: zext_vector
- ; CHECK: liveins: $q0
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 7
+ ; CHECK: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 7
; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; CHECK-NEXT: %res:_(<2 x i64>) = G_BUILD_VECTOR [[C]](i64), [[C1]](i64)
; CHECK-NEXT: $q0 = COPY %res(<2 x i64>)
@@ -493,17 +475,11 @@ body: |
---
name: fneg_vector
legalized: true
-liveins:
- - { reg: '$q0' }
body: |
bb.1.entry:
- liveins: $q0
-
; Element-wise fold of a unary FP op: fneg(<3.0, -4.0>) = <-3.0, 4.0>.
; CHECK-LABEL: name: fneg_vector
- ; CHECK: liveins: $q0
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double -3.000000e+00
+ ; CHECK: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double -3.000000e+00
; CHECK-NEXT: [[C1:%[0-9]+]]:_(f64) = G_FCONSTANT double 4.000000e+00
; CHECK-NEXT: %res:_(<2 x f64>) = G_BUILD_VECTOR [[C]](f64), [[C1]](f64)
; CHECK-NEXT: $q0 = COPY %res(<2 x f64>)
@@ -516,3 +492,43 @@ body: |
RET_ReallyLR implicit $q0
...
+---
+name: fsqrt_vector
+legalized: true
+body: |
+ bb.1.entry:
+ ; Element-wise fold of a host-libm unary FP op: fsqrt(<4.0, 9.0>) = <2.0, 3.0>.
+ ; CHECK-LABEL: name: fsqrt_vector
+ ; CHECK: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double 2.000000e+00
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(f64) = G_FCONSTANT double 3.000000e+00
+ ; CHECK-NEXT: %res:_(<2 x f64>) = G_BUILD_VECTOR [[C]](f64), [[C1]](f64)
+ ; CHECK-NEXT: $q0 = COPY %res(<2 x f64>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %a0:_(f64) = G_FCONSTANT double 4.0
+ %a1:_(f64) = G_FCONSTANT double 9.0
+ %src:_(<2 x f64>) = G_BUILD_VECTOR %a0(f64), %a1(f64)
+ %res:_(<2 x f64>) = G_FSQRT %src(<2 x f64>)
+ $q0 = COPY %res(<2 x f64>)
+ RET_ReallyLR implicit $q0
+
+...
+---
+name: flog2_vector
+legalized: true
+body: |
+ bb.1.entry:
+ ; Element-wise fold of a host-libm unary FP op: flog2(<8.0, 4.0>) = <3.0, 2.0>.
+ ; CHECK-LABEL: name: flog2_vector
+ ; CHECK: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double 3.000000e+00
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(f64) = G_FCONSTANT double 2.000000e+00
+ ; CHECK-NEXT: %res:_(<2 x f64>) = G_BUILD_VECTOR [[C]](f64), [[C1]](f64)
+ ; CHECK-NEXT: $q0 = COPY %res(<2 x f64>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %a0:_(f64) = G_FCONSTANT double 8.0
+ %a1:_(f64) = G_FCONSTANT double 4.0
+ %src:_(<2 x f64>) = G_BUILD_VECTOR %a0(f64), %a1(f64)
+ %res:_(<2 x f64>) = G_FLOG2 %src(<2 x f64>)
+ $q0 = COPY %res(<2 x f64>)
+ RET_ReallyLR implicit $q0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-constant-fold-fma.mir b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-constant-fold-fma.mir
index 83ed5c03ecd39..e6bfb8597a059 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-constant-fold-fma.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-constant-fold-fma.mir
@@ -1,20 +1,14 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple aarch64 -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple aarch64 -run-pass=aarch64-prelegalizer-combiner %s -o - | FileCheck %s
---
name: fma_scalar
legalized: true
-liveins:
- - { reg: '$d0' }
body: |
bb.1.entry:
- liveins: $d0
-
; fma(3.0, 4.0, 5.0) = 17.0
; CHECK-LABEL: name: fma_scalar
- ; CHECK: liveins: $d0
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: %res:_(f64) = G_FCONSTANT double 1.700000e+01
+ ; CHECK: %res:_(f64) = G_FCONSTANT double 1.700000e+01
; CHECK-NEXT: $d0 = COPY %res(f64)
; CHECK-NEXT: RET_ReallyLR implicit $d0
%a:_(f64) = G_FCONSTANT double 3.0
@@ -28,17 +22,11 @@ body: |
---
name: fma_vector
legalized: true
-liveins:
- - { reg: '$q0' }
body: |
bb.1.entry:
- liveins: $q0
-
; Element-wise fma: fma(<3.0, 2.0>, <4.0, 4.0>, <5.0, 1.0>) = <17.0, 9.0>.
; CHECK-LABEL: name: fma_vector
- ; CHECK: liveins: $q0
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double 1.700000e+01
+ ; CHECK: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double 1.700000e+01
; CHECK-NEXT: [[C1:%[0-9]+]]:_(f64) = G_FCONSTANT double 9.000000e+00
; CHECK-NEXT: %res:_(<2 x f64>) = G_BUILD_VECTOR [[C]](f64), [[C1]](f64)
; CHECK-NEXT: $q0 = COPY %res(<2 x f64>)
>From 8bbf9a91bc80c7a107ce31a2da9cbbcfcd9acbcd Mon Sep 17 00:00:00 2001
From: vg0204 <Vikash.Gupta at amd.com>
Date: Wed, 9 Sep 2026 12:21:07 +0530
Subject: [PATCH 3/7] Addresse the reviewed changes.
---
llvm/lib/CodeGen/GlobalISel/Utils.cpp | 7 +++----
.../GlobalISel/prelegalizercombiner-commute-shift.mir | 5 +++--
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 485b1d2d524f1..ca09ee19166c3 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -1051,10 +1051,9 @@ llvm::ConstantFoldUnaryFPOp(unsigned Opcode, LLT DstTy, Register Src,
Type *Ty =
Type::getFloatingPointTy(MRI.getMF().getFunction().getContext(),
getFltSemanticForLLT(DstEltTy));
- Constant *C = ConstantFoldFP(Opcode == TargetOpcode::G_FSQRT
- ? static_cast<double (*)(double)>(sqrt)
- : static_cast<double (*)(double)>(log2),
- V, Ty);
+ Constant *C = Opcode == TargetOpcode::G_FSQRT
+ ? ConstantFoldFP(sqrt, V, Ty)
+ : ConstantFoldFP(log2, V, Ty);
if (!C)
return std::nullopt;
return cast<ConstantFP>(C)->getValueAPF();
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-commute-shift.mir b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-commute-shift.mir
index 7bc1503e9da32..8f78c1dacaf2a 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-commute-shift.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-commute-shift.mir
@@ -108,8 +108,9 @@ body: |
; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 2
; CHECK-NEXT: %veccst2:_(<4 x i32>) = G_BUILD_VECTOR [[C]](i32), [[C]](i32), [[C]](i32), [[C]](i32)
; CHECK-NEXT: [[SHL:%[0-9]+]]:_(<4 x i32>) = G_SHL %xvec, %veccst2(<4 x i32>)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(<4 x i32>) = G_SHL %veccst2, %veccst2(<4 x i32>)
- ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(<4 x i32>) = G_ADD [[SHL]], [[SHL1]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[C1]](i32), [[C1]](i32), [[C1]](i32), [[C1]](i32)
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(<4 x i32>) = G_ADD [[SHL]], [[BUILD_VECTOR]]
; CHECK-NEXT: G_STORE [[ADD]](<4 x i32>), [[COPY]](p0) :: (store (<4 x i32>))
; CHECK-NEXT: RET_ReallyLR
%0:_(p0) = COPY $x0
>From 85bede8d9ced8ac2c654a4ee1df294fc8b56e406 Mon Sep 17 00:00:00 2001
From: vg0204 <Vikash.Gupta at amd.com>
Date: Wed, 9 Sep 2026 14:17:39 +0530
Subject: [PATCH 4/7] Extend the same BuildFnTy-based scalar/vector branching
used by matchConstantFoldBinOp/matchConstantFoldUnaryIntOp to three more
matchinfo-driven combines, so they no longer rely on replaceInstWithConstant.
They are as follows :
- matchICmpToTrueFalseKnownBits
- matchShiftsTooBig
- matchCastOfInteger
---
.../llvm/CodeGen/GlobalISel/CombinerHelper.h | 7 +-
.../include/llvm/Target/GlobalISel/Combine.td | 19 ++----
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 64 ++++++++++++++-----
.../GlobalISel/CombinerHelperCasts.cpp | 10 ++-
4 files changed, 65 insertions(+), 35 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index cc4e77d02cabb..edabb5c0bab64 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -724,7 +724,7 @@ class CombinerHelper {
/// \returns true if a G_ICMP instruction \p MI can be replaced with a true
/// or false constant based off of KnownBits information.
LLVM_ABI bool matchICmpToTrueFalseKnownBits(MachineInstr &MI,
- int64_t &MatchInfo) const;
+ BuildFnTy &MatchInfo) const;
/// \returns true if a G_ICMP \p MI can be replaced with its LHS based off of
/// KnownBits information.
@@ -971,8 +971,7 @@ class CombinerHelper {
/// Match shifts greater or equal to the range (the bitwidth of the result
/// datatype, or the effective bitwidth of the source value).
- LLVM_ABI bool matchShiftsTooBig(MachineInstr &MI,
- std::optional<int64_t> &MatchInfo) const;
+ LLVM_ABI bool matchShiftsTooBig(MachineInstr &MI, BuildFnTy &MatchInfo) const;
/// Match constant LHS ops that should be commuted.
LLVM_ABI bool matchCommuteConstantToRHS(MachineInstr &MI) const;
@@ -1018,7 +1017,7 @@ class CombinerHelper {
BuildFnTy &MatchInfo) const;
LLVM_ABI bool matchCastOfInteger(const MachineInstr &CastMI,
- APInt &MatchInfo) const;
+ BuildFnTy &MatchInfo) const;
/// Combine addos.
LLVM_ABI bool matchAddOverflow(MachineInstr &MI, BuildFnTy &MatchInfo) const;
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index 76ffcb4fbf565..c0db51f02ca9c 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -355,18 +355,11 @@ def shift_const_op : GICombinePatFrag<
!foreach(op,
[G_SHL, G_ASHR, G_LSHR],
(pattern (op $dst, $shifted, $amt)))>;
-def shift_result_matchdata : GIDefMatchData<"std::optional<int64_t>">;
def shifts_too_big : GICombineRule<
- (defs root:$root, shift_result_matchdata:$matchinfo),
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
(match (shift_const_op $root):$mi,
[{ return Helper.matchShiftsTooBig(*${mi}, ${matchinfo}); }]),
- (apply [{
- if (${matchinfo}) {
- Helper.replaceInstWithConstant(*${mi}, *${matchinfo});
- } else {
- Helper.replaceInstWithUndef(*${mi});
- }
- }])>;
+ (apply [{ Helper.applyBuildFn(*${mi}, ${matchinfo}); }])>;
// Fold shift (shift base x), y -> shift base, (x+y), if shifts are same
def shift_immed_matchdata : GIDefMatchData<"RegisterImmPair">;
@@ -1422,10 +1415,10 @@ def rotate_out_of_range : GICombineRule<
>;
def icmp_to_true_false_known_bits : GICombineRule<
- (defs root:$d, int64_matchinfo:$matchinfo),
+ (defs root:$d, build_fn_matchinfo:$matchinfo),
(match (G_ICMP $dst, $pred, $src1, $src2):$d,
[{ return Helper.matchICmpToTrueFalseKnownBits(*${d}, ${matchinfo}); }]),
- (apply [{ Helper.replaceInstWithConstant(*${d}, ${matchinfo}); }])>;
+ (apply [{ Helper.applyBuildFn(*${d}, ${matchinfo}); }])>;
def icmp_to_lhs_known_bits : GICombineRule<
(defs root:$root, build_fn_matchinfo:$info),
@@ -2631,11 +2624,11 @@ def narrow_binop_xor : narrow_binop_opcode<G_XOR>;
// Cast of integer.
class integer_of_opcode<Instruction castOpcode> : GICombineRule <
- (defs root:$root, apint_matchinfo:$matchinfo),
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_CONSTANT $int, $imm),
(castOpcode $root, $int):$Cast,
[{ return Helper.matchCastOfInteger(*${Cast}, ${matchinfo}); }]),
- (apply [{ Helper.replaceInstWithConstant(*${Cast}, ${matchinfo}); }])>;
+ (apply [{ Helper.applyBuildFn(*${Cast}, ${matchinfo}); }])>;
def integer_of_truncate : integer_of_opcode<G_TRUNC>;
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index a6d965746d989..2843319fea97f 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -4653,7 +4653,7 @@ void CombinerHelper::applyRotateOutOfRange(MachineInstr &MI) const {
}
bool CombinerHelper::matchICmpToTrueFalseKnownBits(MachineInstr &MI,
- int64_t &MatchInfo) const {
+ BuildFnTy &MatchInfo) const {
assert(MI.getOpcode() == TargetOpcode::G_ICMP);
auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
@@ -4686,13 +4686,25 @@ bool CombinerHelper::matchICmpToTrueFalseKnownBits(MachineInstr &MI,
if (!KnownVal)
return false;
- MatchInfo =
- *KnownVal
- ? getICmpTrueVal(getTargetLowering(),
- /*IsVector = */
- MRI.getType(MI.getOperand(0).getReg()).isVector(),
- /* IsFP = */ false)
- : 0;
+
+ Register Dst = MI.getOperand(0).getReg();
+ LLT DstTy = MRI.getType(Dst);
+ int64_t Val = *KnownVal ? getICmpTrueVal(getTargetLowering(),
+ /*IsVector = */ DstTy.isVector(),
+ /* IsFP = */ false)
+ : 0;
+
+ if (DstTy.isVector()) {
+ SmallVector<APInt> Csts(DstTy.getNumElements(),
+ APInt(DstTy.getScalarSizeInBits(), Val,
+ /*isSigned=*/true));
+ MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
+ B.buildBuildVectorConstant(Dst, Csts);
+ };
+ return true;
+ }
+
+ MatchInfo = [Dst, Val](MachineIRBuilder &B) { B.buildConstant(Dst, Val); };
return true;
}
@@ -7391,24 +7403,46 @@ getMinUselessShift(KnownBits ValueKB, unsigned Opcode,
return ValueKB.getBitWidth() - SignificantBits;
}
-bool CombinerHelper::matchShiftsTooBig(
- MachineInstr &MI, std::optional<int64_t> &MatchInfo) const {
+bool CombinerHelper::matchShiftsTooBig(MachineInstr &MI,
+ BuildFnTy &MatchInfo) const {
+ Register Dst = MI.getOperand(0).getReg();
Register ShiftVal = MI.getOperand(1).getReg();
Register ShiftReg = MI.getOperand(2).getReg();
- LLT ResTy = MRI.getType(MI.getOperand(0).getReg());
+ LLT ResTy = MRI.getType(Dst);
+ std::optional<int64_t> Result;
auto IsShiftTooBig = [&](const Constant *C) {
auto *CI = dyn_cast<ConstantInt>(C);
if (!CI)
return false;
if (CI->uge(ResTy.getScalarSizeInBits())) {
- MatchInfo = std::nullopt;
+ Result = std::nullopt;
return true;
}
- auto OptMaxUsefulShift = getMinUselessShift(VT->getKnownBits(ShiftVal),
- MI.getOpcode(), MatchInfo);
+ auto OptMaxUsefulShift =
+ getMinUselessShift(VT->getKnownBits(ShiftVal), MI.getOpcode(), Result);
return OptMaxUsefulShift && CI->uge(*OptMaxUsefulShift);
};
- return matchUnaryPredicate(MRI, ShiftReg, IsShiftTooBig);
+ if (!matchUnaryPredicate(MRI, ShiftReg, IsShiftTooBig))
+ return false;
+
+ if (!Result) {
+ MatchInfo = [Dst](MachineIRBuilder &B) { B.buildUndef(Dst); };
+ return true;
+ }
+
+ int64_t Val = *Result;
+ if (ResTy.isVector()) {
+ SmallVector<APInt> Csts(ResTy.getNumElements(),
+ APInt(ResTy.getScalarSizeInBits(), Val,
+ /*isSigned=*/true));
+ MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
+ B.buildBuildVectorConstant(Dst, Csts);
+ };
+ return true;
+ }
+
+ MatchInfo = [Dst, Val](MachineIRBuilder &B) { B.buildConstant(Dst, Val); };
+ return true;
}
bool CombinerHelper::matchCommuteConstantToRHS(MachineInstr &MI) const {
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelperCasts.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelperCasts.cpp
index a310cb8174f43..43c48303a4dea 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelperCasts.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelperCasts.cpp
@@ -362,19 +362,23 @@ bool CombinerHelper::matchNarrowBinop(const MachineInstr &TruncMI,
}
bool CombinerHelper::matchCastOfInteger(const MachineInstr &CastMI,
- APInt &MatchInfo) const {
+ BuildFnTy &MatchInfo) const {
const GExtOrTruncOp *Cast = cast<GExtOrTruncOp>(&CastMI);
APInt Input = getIConstantFromReg(Cast->getSrcReg(), MRI);
- LLT DstTy = MRI.getType(Cast->getReg(0));
+ Register Dst = Cast->getReg(0);
+ LLT DstTy = MRI.getType(Dst);
if (!isConstantLegalOrBeforeLegalizer(DstTy))
return false;
switch (Cast->getOpcode()) {
case TargetOpcode::G_TRUNC: {
- MatchInfo = Input.trunc(DstTy.getScalarSizeInBits());
+ APInt Result = Input.trunc(DstTy.getScalarSizeInBits());
+ MatchInfo = [Dst, Result](MachineIRBuilder &B) {
+ B.buildConstant(Dst, Result);
+ };
return true;
}
default:
>From f938329a205de56f149668d78ca959d936c6d431 Mon Sep 17 00:00:00 2001
From: vg0204 <Vikash.Gupta at amd.com>
Date: Tue, 15 Sep 2026 14:27:01 +0530
Subject: [PATCH 5/7] Addressed the reviewed changes.
---
llvm/lib/CodeGen/GlobalISel/Utils.cpp | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index ca09ee19166c3..c935272e12ed7 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -838,21 +838,18 @@ llvm::ConstantFoldVectorFPBinop(unsigned Opcode, const Register Op1,
const MachineRegisterInfo &MRI) {
auto *SrcVec2 = getBuildVectorLikeDef(Op2, MRI);
if (!SrcVec2)
- return SmallVector<APFloat>();
+ return {};
auto *SrcVec1 = getBuildVectorLikeDef(Op1, MRI);
if (!SrcVec1)
- return SmallVector<APFloat>();
-
- if (SrcVec1->getNumSources() != SrcVec2->getNumSources())
- return SmallVector<APFloat>();
+ return {};
SmallVector<APFloat> FoldedElements;
for (unsigned Idx = 0, E = SrcVec1->getNumSources(); Idx < E; ++Idx) {
auto MaybeCst = ConstantFoldFPBinOp(Opcode, SrcVec1->getSourceReg(Idx),
SrcVec2->getSourceReg(Idx), MRI);
if (!MaybeCst)
- return SmallVector<APFloat>();
+ return {};
FoldedElements.push_back(*MaybeCst);
}
return FoldedElements;
>From e9f2c81e709f8afb72b1e8b5d01747e10b1d7bf7 Mon Sep 17 00:00:00 2001
From: vg0204 <Vikash.Gupta at amd.com>
Date: Fri, 18 Sep 2026 17:46:31 +0530
Subject: [PATCH 6/7] Addressed review: restrict constant folding to fixed
vectors
---
.../llvm/CodeGen/GlobalISel/CombinerHelper.h | 8 ++-
.../include/llvm/Target/GlobalISel/Combine.td | 4 +-
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 69 ++++++++-----------
.../postlegalizer-combiner-constant-fold.mir | 52 ++++++++++++++
...mbine-shl-from-extend-narrow.postlegal.mir | 10 ++-
5 files changed, 91 insertions(+), 52 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index edabb5c0bab64..02554eb588a7c 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -169,8 +169,10 @@ class CombinerHelper {
/// perform a FewerElements action on the target.
LLVM_ABI bool isLegalOrHasFewerElements(const LegalityQuery &Query) const;
- /// \return true if the combine is running prior to legalization, or if \p Ty
- /// is a legal integer constant type on the target.
+ /// \return true if the combine is running prior to legalization, or if a
+ /// scalar \p Ty G_CONSTANT is legal. For fixed vectors, also requires
+ /// G_BUILD_VECTOR of those scalar constants to be legal. Scalable vectors are
+ /// not constant folded and return false.
LLVM_ABI bool isConstantLegalOrBeforeLegalizer(const LLT Ty) const;
/// MachineRegisterInfo::replaceRegWith() and inform the observer of the changes
@@ -724,7 +726,7 @@ class CombinerHelper {
/// \returns true if a G_ICMP instruction \p MI can be replaced with a true
/// or false constant based off of KnownBits information.
LLVM_ABI bool matchICmpToTrueFalseKnownBits(MachineInstr &MI,
- BuildFnTy &MatchInfo) const;
+ int64_t &MatchInfo) const;
/// \returns true if a G_ICMP \p MI can be replaced with its LHS based off of
/// KnownBits information.
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index c0db51f02ca9c..ee629558c82c2 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -1415,10 +1415,10 @@ def rotate_out_of_range : GICombineRule<
>;
def icmp_to_true_false_known_bits : GICombineRule<
- (defs root:$d, build_fn_matchinfo:$matchinfo),
+ (defs root:$d, int64_matchinfo:$matchinfo),
(match (G_ICMP $dst, $pred, $src1, $src2):$d,
[{ return Helper.matchICmpToTrueFalseKnownBits(*${d}, ${matchinfo}); }]),
- (apply [{ Helper.applyBuildFn(*${d}, ${matchinfo}); }])>;
+ (apply [{ Helper.replaceInstWithConstant(*${d}, ${matchinfo}); }])>;
def icmp_to_lhs_known_bits : GICombineRule<
(defs root:$root, build_fn_matchinfo:$info),
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 2843319fea97f..554514b3d5570 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -178,9 +178,11 @@ bool CombinerHelper::isLegalOrHasFewerElements(
bool CombinerHelper::isConstantLegalOrBeforeLegalizer(const LLT Ty) const {
if (!Ty.isVector())
return isLegalOrBeforeLegalizer({TargetOpcode::G_CONSTANT, {Ty}});
- // Vector constants are represented as a G_BUILD_VECTOR of scalar G_CONSTANTs.
if (isPreLegalize())
return true;
+ // Scalable vectors cannot be constant folded here.
+ if (!Ty.isFixedVector())
+ return false;
LLT EltTy = Ty.getElementType();
return isLegal({TargetOpcode::G_BUILD_VECTOR, {Ty, EltTy}}) &&
isLegal({TargetOpcode::G_CONSTANT, {EltTy}});
@@ -4653,7 +4655,7 @@ void CombinerHelper::applyRotateOutOfRange(MachineInstr &MI) const {
}
bool CombinerHelper::matchICmpToTrueFalseKnownBits(MachineInstr &MI,
- BuildFnTy &MatchInfo) const {
+ int64_t &MatchInfo) const {
assert(MI.getOpcode() == TargetOpcode::G_ICMP);
auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
@@ -4686,25 +4688,13 @@ bool CombinerHelper::matchICmpToTrueFalseKnownBits(MachineInstr &MI,
if (!KnownVal)
return false;
-
- Register Dst = MI.getOperand(0).getReg();
- LLT DstTy = MRI.getType(Dst);
- int64_t Val = *KnownVal ? getICmpTrueVal(getTargetLowering(),
- /*IsVector = */ DstTy.isVector(),
- /* IsFP = */ false)
- : 0;
-
- if (DstTy.isVector()) {
- SmallVector<APInt> Csts(DstTy.getNumElements(),
- APInt(DstTy.getScalarSizeInBits(), Val,
- /*isSigned=*/true));
- MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
- B.buildBuildVectorConstant(Dst, Csts);
- };
- return true;
- }
-
- MatchInfo = [Dst, Val](MachineIRBuilder &B) { B.buildConstant(Dst, Val); };
+ MatchInfo =
+ *KnownVal
+ ? getICmpTrueVal(getTargetLowering(),
+ /*IsVector = */
+ MRI.getType(MI.getOperand(0).getReg()).isVector(),
+ /* IsFP = */ false)
+ : 0;
return true;
}
@@ -5279,7 +5269,10 @@ bool CombinerHelper::matchConstantFoldCastOp(MachineInstr &MI,
LLT DstTy = MRI.getType(Dst);
unsigned Opc = MI.getOpcode();
- if (DstTy.isVector()) {
+ if (!isConstantLegalOrBeforeLegalizer(DstTy))
+ return false;
+
+ if (DstTy.isFixedVector()) {
auto *BV = getOpcodeDef<GBuildVector>(SrcOp, MRI);
if (!BV)
return false;
@@ -5309,7 +5302,9 @@ bool CombinerHelper::matchConstantFoldCastOp(MachineInstr &MI,
bool CombinerHelper::matchConstantFoldUnaryIntOp(MachineInstr &MI,
BuildFnTy &MatchInfo) const {
Register Dst = MI.getOperand(0).getReg();
- auto Csts = ConstantFoldUnaryIntOp(MI.getOpcode(), MRI.getType(Dst),
+ LLT DstTy = MRI.getType(Dst);
+
+ auto Csts = ConstantFoldUnaryIntOp(MI.getOpcode(), DstTy,
MI.getOperand(1).getReg(), MRI);
if (Csts.empty())
return false;
@@ -5330,11 +5325,12 @@ bool CombinerHelper::matchConstantFoldBinOp(MachineInstr &MI,
Register Op2 = MI.getOperand(2).getReg();
LLT DstTy = MRI.getType(Dst);
- if (DstTy.isVector()) {
- // Pointer-element vectors (e.g. <n x ptr> from a vector G_PTR_ADD) have no
- // G_BUILD_VECTOR of G_CONSTANT representation.
- if (DstTy.getElementType().isPointer())
- return false;
+ // Pointer-element vectors have no G_BUILD_VECTOR of G_CONSTANT
+ // representation.
+ if (DstTy.isVector() && DstTy.getElementType().isPointer())
+ return false;
+
+ if (DstTy.isFixedVector()) {
SmallVector<APInt> Csts =
ConstantFoldVectorBinop(MI.getOpcode(), Op1, Op2, MRI);
if (Csts.empty())
@@ -5361,7 +5357,7 @@ bool CombinerHelper::matchConstantFoldFPBinOp(MachineInstr &MI,
Register Op2 = MI.getOperand(2).getReg();
LLT DstTy = MRI.getType(Dst);
- if (DstTy.isVector()) {
+ if (DstTy.isFixedVector()) {
SmallVector<APFloat> Csts =
ConstantFoldVectorFPBinop(MI.getOpcode(), Op1, Op2, MRI);
if (Csts.empty())
@@ -5384,8 +5380,9 @@ bool CombinerHelper::matchConstantFoldFPBinOp(MachineInstr &MI,
bool CombinerHelper::matchConstantFoldFPUnary(MachineInstr &MI,
BuildFnTy &MatchInfo) const {
Register Dst = MI.getOperand(0).getReg();
+ LLT DstTy = MRI.getType(Dst);
SmallVector<APFloat> Csts = ConstantFoldUnaryFPOp(
- MI.getOpcode(), MRI.getType(Dst), MI.getOperand(1).getReg(), MRI);
+ MI.getOpcode(), DstTy, MI.getOperand(1).getReg(), MRI);
if (Csts.empty())
return false;
MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
@@ -5412,7 +5409,7 @@ bool CombinerHelper::matchConstantFoldFMA(MachineInstr &MI,
return Res;
};
- if (DstTy.isVector()) {
+ if (DstTy.isFixedVector()) {
auto *BV1 = getOpcodeDef<GBuildVector>(Op1, MRI);
auto *BV2 = getOpcodeDef<GBuildVector>(Op2, MRI);
auto *BV3 = getOpcodeDef<GBuildVector>(Op3, MRI);
@@ -7431,16 +7428,6 @@ bool CombinerHelper::matchShiftsTooBig(MachineInstr &MI,
}
int64_t Val = *Result;
- if (ResTy.isVector()) {
- SmallVector<APInt> Csts(ResTy.getNumElements(),
- APInt(ResTy.getScalarSizeInBits(), Val,
- /*isSigned=*/true));
- MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
- B.buildBuildVectorConstant(Dst, Csts);
- };
- return true;
- }
-
MatchInfo = [Dst, Val](MachineIRBuilder &B) { B.buildConstant(Dst, Val); };
return true;
}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir
index 9fbab67265f34..afdf88a9f12e7 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir
@@ -532,3 +532,55 @@ body: |
RET_ReallyLR implicit $q0
...
+---
+name: udiv_vector_by_zero
+legalized: true
+body: |
+ bb.1.entry:
+ ; A zero divisor in any lane must refuse the whole vector fold.
+ ; CHECK-LABEL: name: udiv_vector_by_zero
+ ; CHECK: %a0:_(i64) = G_CONSTANT i64 40
+ ; CHECK-NEXT: %a1:_(i64) = G_CONSTANT i64 10
+ ; CHECK-NEXT: %b0:_(i64) = G_CONSTANT i64 2
+ ; CHECK-NEXT: %b1:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: %av:_(<2 x i64>) = G_BUILD_VECTOR %a0(i64), %a1(i64)
+ ; CHECK-NEXT: %bv:_(<2 x i64>) = G_BUILD_VECTOR %b0(i64), %b1(i64)
+ ; CHECK-NEXT: %res:_(<2 x i64>) = G_UDIV %av, %bv
+ ; CHECK-NEXT: $q0 = COPY %res(<2 x i64>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %a0:_(i64) = G_CONSTANT i64 40
+ %a1:_(i64) = G_CONSTANT i64 10
+ %b0:_(i64) = G_CONSTANT i64 2
+ %b1:_(i64) = G_CONSTANT i64 0
+ %av:_(<2 x i64>) = G_BUILD_VECTOR %a0(i64), %a1(i64)
+ %bv:_(<2 x i64>) = G_BUILD_VECTOR %b0(i64), %b1(i64)
+ %res:_(<2 x i64>) = G_UDIV %av, %bv
+ $q0 = COPY %res(<2 x i64>)
+ RET_ReallyLR implicit $q0
+
+...
+---
+name: add_vector_undef_lane
+legalized: true
+body: |
+ bb.1.entry:
+ ; An undef lane must refuse the whole vector fold.
+ ; CHECK-LABEL: name: add_vector_undef_lane
+ ; CHECK: %a0:_(i64) = G_CONSTANT i64 40
+ ; CHECK-NEXT: %a1:_(i64) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: %b:_(i64) = G_CONSTANT i64 2
+ ; CHECK-NEXT: %av:_(<2 x i64>) = G_BUILD_VECTOR %a0(i64), %a1(i64)
+ ; CHECK-NEXT: %bv:_(<2 x i64>) = G_BUILD_VECTOR %b(i64), %b(i64)
+ ; CHECK-NEXT: %res:_(<2 x i64>) = G_ADD %av, %bv
+ ; CHECK-NEXT: $q0 = COPY %res(<2 x i64>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %a0:_(i64) = G_CONSTANT i64 40
+ %a1:_(i64) = G_IMPLICIT_DEF
+ %b:_(i64) = G_CONSTANT i64 2
+ %av:_(<2 x i64>) = G_BUILD_VECTOR %a0(i64), %a1(i64)
+ %bv:_(<2 x i64>) = G_BUILD_VECTOR %b(i64), %b(i64)
+ %res:_(<2 x i64>) = G_ADD %av, %bv
+ $q0 = COPY %res(<2 x i64>)
+ RET_ReallyLR implicit $q0
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir
index 5ac9d99af28ba..e5db8de4e4ef8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir
@@ -375,17 +375,15 @@ body: |
; GFX6: liveins: $vgpr0
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: %shl:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C1]](s32)
- ; GFX6-NEXT: $vgpr0_vgpr1 = COPY %shl(<2 x s32>)
+ ; GFX6-NEXT: %extend:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32)
+ ; GFX6-NEXT: $vgpr0_vgpr1 = COPY %extend(<2 x s32>)
;
; GFX9-LABEL: name: do_not_shl_v2s32_zero_by_16_from_zext_v2s16
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: %shl:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C1]](s32)
- ; GFX9-NEXT: $vgpr0_vgpr1 = COPY %shl(<2 x s32>)
+ ; GFX9-NEXT: %extend:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32)
+ ; GFX9-NEXT: $vgpr0_vgpr1 = COPY %extend(<2 x s32>)
%zero:_(s16) = G_CONSTANT i16 0
%zerovector:_(<2 x s16>) = G_BUILD_VECTOR %zero, %zero:_(s16)
%shiftamt:_(s16) = G_CONSTANT i16 16
>From eccc351691cfb5f098836f13162f11f1491e2ffb Mon Sep 17 00:00:00 2001
From: Vikash Gupta <Vikash.Gupta at amd.com>
Date: Tue, 29 Sep 2026 11:44:18 +0530
Subject: [PATCH 7/7] Apply batched suggestions from code review
Co-authored-by: Tim Gymnich <tim at gymni.ch>
---
llvm/lib/Analysis/ConstantFolding.cpp | 4 ++--
llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp | 4 ++--
llvm/lib/CodeGen/GlobalISel/Utils.cpp | 4 ++--
3 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index ae384104f293f..cb4c23e9c19dd 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -4703,7 +4703,7 @@ Constant *llvm::ConstantFoldFP(double (*NativeFP)(double), const APFloat &V,
return nullptr;
llvm_fenv_clearexcept();
- auto Input = FlushWithDenormKind(V, DenormMode.Input);
+ APFloat Input = FlushWithDenormKind(V, DenormMode.Input);
double Result = NativeFP(Input.convertToDouble());
if (llvm_fenv_testexcept()) {
llvm_fenv_clearexcept();
@@ -4714,7 +4714,7 @@ Constant *llvm::ConstantFoldFP(double (*NativeFP)(double), const APFloat &V,
if (DenormMode.Output == DenormalMode::DenormalModeKind::IEEE)
return Output;
const auto *CFP = static_cast<ConstantFP *>(Output);
- const auto Res = FlushWithDenormKind(CFP->getValueAPF(), DenormMode.Output);
+ APFloat Res = FlushWithDenormKind(CFP->getValueAPF(), DenormMode.Output);
return ConstantFP::get(Ty->getContext(), Res);
}
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 554514b3d5570..d6702f4562239 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -5304,8 +5304,8 @@ bool CombinerHelper::matchConstantFoldUnaryIntOp(MachineInstr &MI,
Register Dst = MI.getOperand(0).getReg();
LLT DstTy = MRI.getType(Dst);
- auto Csts = ConstantFoldUnaryIntOp(MI.getOpcode(), DstTy,
- MI.getOperand(1).getReg(), MRI);
+ SmallVector<APInt> Csts = ConstantFoldUnaryIntOp(
+ MI.getOpcode(), DstTy, MI.getOperand(1).getReg(), MRI);
if (Csts.empty())
return false;
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index c935272e12ed7..323e85b42176f 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -836,11 +836,11 @@ SmallVector<APFloat>
llvm::ConstantFoldVectorFPBinop(unsigned Opcode, const Register Op1,
const Register Op2,
const MachineRegisterInfo &MRI) {
- auto *SrcVec2 = getBuildVectorLikeDef(Op2, MRI);
+ GBuildVector *SrcVec2 = getBuildVectorLikeDef(Op2, MRI);
if (!SrcVec2)
return {};
- auto *SrcVec1 = getBuildVectorLikeDef(Op1, MRI);
+ GBuildVector *SrcVec1 = getBuildVectorLikeDef(Op1, MRI);
if (!SrcVec1)
return {};
More information about the llvm-commits
mailing list