[llvm] [GlobalISel] Support fixed vectors in constant-fold combines. (PR #220876)
Vikash Gupta via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 6 02:09:29 PDT 2026
https://github.com/vg0204 updated https://github.com/llvm/llvm-project/pull/220876
>From a13d3434b28c757f736eff0bed6189d25ae98651 Mon Sep 17 00:00:00 2001
From: vg0204 <Vikash.Gupta at amd.com>
Date: Thu, 3 Sep 2026 16:26:58 +0530
Subject: [PATCH 1/8] [GlobalISel] Support fixed vectors in constant-fold
combines.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This patch Extend the GlobalISel constant-folding combines to
element-wise fold **fixed vectors**, in addition to scalars.
Previously these rules only fired when the operands were scalar
constants; a `G_BUILD_VECTOR` of constants was left un-folded.
This covers all five families:
- Integer/pointer binops (`constant_fold_binop`)
- FP binops (`constant_fold_fp_binop`)
- FMA (`constant_fold_fma`)
- Integer casts — `G_ZEXT`/`G_SEXT`/`G_ANYEXT` (`constant_fold_cast_op`)
- FP unary ops — `G_FNEG`, `G_FABS`, `G_FSQRT`, `G_FLOG2`, `G_FPEXT`,
`G_FPTRUNC`, `G_FCEIL`, `G_FFLOOR`,
`G_INTRINSIC_TRUNC/ROUND/ROUNDEVEN`,
`G_FRINT`, `G_FNEARBYINT` (`constant_fold_unary_fp_op_rule`)
---
.../llvm/CodeGen/GlobalISel/CombinerHelper.h | 16 +-
.../CodeGen/GlobalISel/MachineIRBuilder.h | 5 +
llvm/include/llvm/CodeGen/GlobalISel/Utils.h | 14 ++
.../include/llvm/Target/GlobalISel/Combine.td | 23 +-
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 215 ++++++++++--------
.../CodeGen/GlobalISel/MachineIRBuilder.cpp | 11 +
llvm/lib/CodeGen/GlobalISel/Utils.cpp | 106 +++++++++
.../GlobalISel/combine-logic-of-compare.mir | 5 +-
.../AArch64/GlobalISel/combine-overflow.mir | 5 +-
.../AArch64/GlobalISel/combine-udiv.ll | 13 +-
.../postlegalizer-combiner-constant-fold.mir | 108 +++++++++
...relegalizer-combiner-constant-fold-fma.mir | 58 +++++
llvm/test/CodeGen/AArch64/arm64-vabs.ll | 66 +++---
llvm/test/CodeGen/AArch64/combine-sdiv.ll | 174 +++++++-------
.../CodeGen/AArch64/extract-vector-elt.ll | 6 -
llvm/test/CodeGen/AArch64/fcvt_combine.ll | 38 ++--
llvm/test/CodeGen/AArch64/hadd-combine.ll | 56 ++---
.../half-precision-signof-no-assert.ll | 12 +-
.../AArch64/neon-compare-instructions.ll | 35 +--
.../GlobalISel/combine-or-redundant.mir | 9 +-
.../GlobalISel/combine-redundant-and.mir | 7 +-
...mbine-shl-from-extend-narrow.postlegal.mir | 6 +-
...ombine-shl-from-extend-narrow.prelegal.mir | 12 +-
.../GlobalISel/postlegalizercombiner-ashr.mir | 9 +-
.../GlobalISel/postlegalizercombiner-lshr.mir | 9 +-
.../GlobalISel/postlegalizercombiner-shl.mir | 9 +-
26 files changed, 643 insertions(+), 384 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-constant-fold-fma.mir
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index 9f020ba7be528d..482466181e307c 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -436,10 +436,6 @@ class CombinerHelper {
LLVM_ABI bool matchCombineUnmergeZExtToZExt(MachineInstr &MI) const;
LLVM_ABI void applyCombineUnmergeZExtToZExt(MachineInstr &MI) const;
- /// Transform fp_instr(cst) to constant result of the fp operation.
- LLVM_ABI void applyCombineConstantFoldFpUnary(MachineInstr &MI,
- const ConstantFP *Cst) const;
-
/// Constant fold a unary integer op (G_CTLZ, G_CTTZ, G_CTPOP and their
/// _ZERO_POISON variants, G_ABS, G_BSWAP, G_BITREVERSE) when the operand is
/// a scalar constant or a G_BUILD_VECTOR of constants.
@@ -735,19 +731,23 @@ class CombinerHelper {
/// Do constant folding when opportunities are exposed after MIR building.
LLVM_ABI bool matchConstantFoldCastOp(MachineInstr &MI,
- APInt &MatchInfo) const;
+ BuildFnTy &MatchInfo) const;
/// Do constant folding when opportunities are exposed after MIR building.
LLVM_ABI bool matchConstantFoldBinOp(MachineInstr &MI,
- APInt &MatchInfo) const;
+ BuildFnTy &MatchInfo) const;
/// Do constant FP folding when opportunities are exposed after MIR building.
LLVM_ABI bool matchConstantFoldFPBinOp(MachineInstr &MI,
- ConstantFP *&MatchInfo) const;
+ BuildFnTy &MatchInfo) const;
+
+ /// Constant fold a unary FP operation when the source is constant.
+ LLVM_ABI bool matchConstantFoldFPUnary(MachineInstr &MI,
+ BuildFnTy &MatchInfo) const;
/// Constant fold G_FMA/G_FMAD.
LLVM_ABI bool matchConstantFoldFMA(MachineInstr &MI,
- ConstantFP *&MatchInfo) const;
+ BuildFnTy &MatchInfo) const;
/// \returns true if it is possible to narrow the width of a scalar binop
/// feeding a G_AND instruction \p MI.
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index 0816186507ca1b..3126be704837cb 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -1191,6 +1191,11 @@ class LLVM_ABI MachineIRBuilder {
MachineInstrBuilder buildBuildVectorConstant(const DstOp &Res,
ArrayRef<APInt> Ops);
+ /// Build and insert \p Res = G_BUILD_VECTOR \p Op0, ... where each OpN is
+ /// built with G_FCONSTANT.
+ MachineInstrBuilder buildBuildVectorFConstant(const DstOp &Res,
+ ArrayRef<APFloat> Ops);
+
/// Build and insert \p Res = G_BUILD_VECTOR with \p Src replicated to fill
/// the number of elements
MachineInstrBuilder buildSplatBuildVector(const DstOp &Res, const SrcOp &Src);
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
index 30af31e103e6b8..53d80e9a329832 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -321,6 +321,12 @@ LLVM_ABI SmallVector<APInt>
ConstantFoldVectorBinop(unsigned Opcode, const Register Op1, const Register Op2,
const MachineRegisterInfo &MRI);
+/// Tries to constant fold a vector floating-point binop with sources \p Op1 and
+/// \p Op2. Returns an empty vector on failure.
+LLVM_ABI SmallVector<APFloat>
+ConstantFoldVectorFPBinop(unsigned Opcode, const Register Op1,
+ const Register Op2, const MachineRegisterInfo &MRI);
+
LLVM_ABI std::optional<APInt>
ConstantFoldCastOp(unsigned Opcode, LLT DstTy, const Register Op0,
const MachineRegisterInfo &MRI);
@@ -341,6 +347,14 @@ LLVM_ABI SmallVector<APInt>
ConstantFoldUnaryIntOp(unsigned Opcode, LLT DstTy, Register Src,
const MachineRegisterInfo &MRI);
+/// Tries to constant fold a unary floating-point operation (G_FNEG, G_FABS,
+/// G_FSQRT, G_FLOG2, G_FPEXT, G_FPTRUNC, G_FCEIL, G_FFLOOR, G_INTRINSIC_TRUNC,
+/// G_INTRINSIC_ROUND, G_INTRINSIC_ROUNDEVEN, G_FRINT, G_FNEARBYINT) on \p Src.
+/// If \p Src is a vector then it tries to do an element-wise constant fold.
+LLVM_ABI SmallVector<APFloat>
+ConstantFoldUnaryFPOp(unsigned Opcode, LLT DstTy, Register Src,
+ const MachineRegisterInfo &MRI);
+
LLVM_ABI std::optional<SmallVector<APInt>>
ConstantFoldICmp(unsigned Pred, const Register Op1, const Register Op2,
unsigned DstScalarSizeInBits, unsigned ExtOp,
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index b61e229a3a44b8..7f5d47276b648b 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -867,9 +867,10 @@ def simplify_add_to_sub : GICombineRule <
// Fold fp_op(cst) to the constant result of the floating point operation.
class constant_fold_unary_fp_op_rule<Instruction opcode> : GICombineRule <
- (defs root:$dst),
- (match (opcode $dst, $src0):$root, (G_FCONSTANT $src0, $cst)),
- (apply [{ Helper.applyCombineConstantFoldFpUnary(*${root}, ${cst}.getFPImm()); }])
+ (defs root:$dst, build_fn_matchinfo:$matchinfo),
+ (match (opcode $dst, $src0):$root,
+ [{ return Helper.matchConstantFoldFPUnary(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])
>;
def constant_fold_fneg : constant_fold_unary_fp_op_rule<G_FNEG>;
@@ -1641,12 +1642,12 @@ def reassocs : GICombineGroup<[reassoc_ptradd, reassoc_comm_binops]>;
// Constant fold operations.
def constant_fold_binop : GICombineRule<
- (defs root:$d, apint_matchinfo:$matchinfo),
+ (defs root:$d, build_fn_matchinfo:$matchinfo),
(match (wip_match_opcode G_ADD, G_PTR_ADD, G_AND, G_ASHR, G_LSHR, G_MUL, G_OR,
G_SHL, G_SUB, G_XOR, G_UDIV, G_SDIV, G_UREM, G_SREM,
G_SMIN, G_SMAX, G_UMIN, G_UMAX):$d,
[{ return Helper.matchConstantFoldBinOp(*${d}, ${matchinfo}); }]),
- (apply [{ Helper.replaceInstWithConstant(*${d}, ${matchinfo}); }])>;
+ (apply [{ Helper.applyBuildFn(*${d}, ${matchinfo}); }])>;
def constant_fold_fp_binop_frags : GICombinePatFrag<
(outs root:$dst), (ins),
@@ -1657,10 +1658,10 @@ def constant_fold_fp_binop_frags : GICombinePatFrag<
(pattern (op $dst, $src0, $src1)))>;
def constant_fold_fp_binop : GICombineRule<
- (defs root:$d, constantfp_matchinfo:$matchinfo),
+ (defs root:$d, build_fn_matchinfo:$matchinfo),
(match (constant_fold_fp_binop_frags $d):$mi,
[{ return Helper.matchConstantFoldFPBinOp(*${mi}, ${matchinfo}); }]),
- (apply [{ Helper.replaceInstWithFConstant(*${mi}, ${matchinfo}); }])>;
+ (apply [{ Helper.applyBuildFn(*${mi}, ${matchinfo}); }])>;
def constant_fold_fma_frags : GICombinePatFrag<
@@ -1668,20 +1669,20 @@ def constant_fold_fma_frags : GICombinePatFrag<
!foreach(op, [G_FMAD, G_FMA], (pattern (op $dst, $src0, $src1, $src2)))>;
def constant_fold_fma : GICombineRule<
- (defs root:$dst, constantfp_matchinfo:$matchinfo),
+ (defs root:$dst, build_fn_matchinfo:$matchinfo),
(match (constant_fold_fma_frags $dst):$d,
[{ return Helper.matchConstantFoldFMA(*${d}, ${matchinfo}); }]),
- (apply [{ Helper.replaceInstWithFConstant(*${d}, ${matchinfo}); }])>;
+ (apply [{ Helper.applyBuildFn(*${d}, ${matchinfo}); }])>;
def constant_fold_cast_op_frags : GICombinePatFrag<
(outs root:$dst), (ins),
!foreach(op, [G_ZEXT, G_SEXT, G_ANYEXT], (pattern (op $dst, $src)))>;
def constant_fold_cast_op : GICombineRule<
- (defs root:$dst, apint_matchinfo:$matchinfo),
+ (defs root:$dst, build_fn_matchinfo:$matchinfo),
(match (constant_fold_cast_op_frags $dst):$d,
[{ return Helper.matchConstantFoldCastOp(*${d}, ${matchinfo}); }]),
- (apply [{ Helper.replaceInstWithConstant(*${d}, ${matchinfo}); }])>;
+ (apply [{ Helper.applyBuildFn(*${d}, ${matchinfo}); }])>;
def unary_int_op_frags : GICombinePatFrag<
(outs root:$dst), (ins),
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 5e4efabf01c97c..53eea4d968341f 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -1742,80 +1742,6 @@ bool CombinerHelper::tryCombineMemCpyFamily(MachineInstr &MI,
LegalizerHelper::LegalizeResult::Legalized;
}
-static APFloat constantFoldFpUnary(const MachineInstr &MI,
- const MachineRegisterInfo &MRI,
- const APFloat &Val) {
- APFloat Result(Val);
- switch (MI.getOpcode()) {
- default:
- llvm_unreachable("Unexpected opcode!");
- case TargetOpcode::G_FNEG: {
- Result.changeSign();
- return Result;
- }
- case TargetOpcode::G_FABS: {
- Result.clearSign();
- return Result;
- }
- case TargetOpcode::G_FCEIL:
- Result.roundToIntegral(APFloat::rmTowardPositive);
- return Result;
- case TargetOpcode::G_FFLOOR:
- Result.roundToIntegral(APFloat::rmTowardNegative);
- return Result;
- case TargetOpcode::G_INTRINSIC_TRUNC:
- Result.roundToIntegral(APFloat::rmTowardZero);
- return Result;
- case TargetOpcode::G_INTRINSIC_ROUND:
- Result.roundToIntegral(APFloat::rmNearestTiesToAway);
- return Result;
- case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
- Result.roundToIntegral(APFloat::rmNearestTiesToEven);
- return Result;
- case TargetOpcode::G_FRINT:
- case TargetOpcode::G_FNEARBYINT:
- // Use default rounding mode (round to nearest, ties to even)
- Result.roundToIntegral(APFloat::rmNearestTiesToEven);
- return Result;
- case TargetOpcode::G_FPEXT:
- case TargetOpcode::G_FPTRUNC: {
- bool Unused;
- LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
- Result.convert(getFltSemanticForLLT(DstTy), APFloat::rmNearestTiesToEven,
- &Unused);
- return Result;
- }
- case TargetOpcode::G_FSQRT: {
- bool Unused;
- Result.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
- &Unused);
- Result = APFloat(sqrt(Result.convertToDouble()));
- break;
- }
- case TargetOpcode::G_FLOG2: {
- bool Unused;
- Result.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
- &Unused);
- Result = APFloat(log2(Result.convertToDouble()));
- break;
- }
- }
- // Convert `APFloat` to appropriate IEEE type depending on `DstTy`. Otherwise,
- // `buildFConstant` will assert on size mismatch. Only `G_FSQRT`, and
- // `G_FLOG2` reach here.
- bool Unused;
- Result.convert(Val.getSemantics(), APFloat::rmNearestTiesToEven, &Unused);
- return Result;
-}
-
-void CombinerHelper::applyCombineConstantFoldFpUnary(
- MachineInstr &MI, const ConstantFP *Cst) const {
- APFloat Folded = constantFoldFpUnary(MI, MRI, Cst->getValue());
- const ConstantFP *NewCst = ConstantFP::get(Builder.getContext(), Folded);
- Builder.buildFConstant(MI.getOperand(0), *NewCst);
- MI.eraseFromParent();
-}
-
bool CombinerHelper::matchPtrAddImmedChain(MachineInstr &MI,
PtrAddChain &MatchInfo) const {
// We're trying to match the following pattern:
@@ -5193,12 +5119,33 @@ bool CombinerHelper::matchReassocCommBinOp(MachineInstr &MI,
}
bool CombinerHelper::matchConstantFoldCastOp(MachineInstr &MI,
- APInt &MatchInfo) const {
- LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
+ BuildFnTy &MatchInfo) const {
+ Register Dst = MI.getOperand(0).getReg();
Register SrcOp = MI.getOperand(1).getReg();
+ LLT DstTy = MRI.getType(Dst);
+ unsigned Opc = MI.getOpcode();
+
+ if (DstTy.isVector()) {
+ auto *BV = getOpcodeDef<GBuildVector>(SrcOp, MRI);
+ if (!BV)
+ return false;
+ SmallVector<APInt> Csts;
+ for (unsigned I = 0, E = BV->getNumSources(); I != E; ++I) {
+ auto MaybeCst = ConstantFoldCastOp(Opc, DstTy, BV->getSourceReg(I), MRI);
+ if (!MaybeCst)
+ return false;
+ Csts.push_back(*MaybeCst);
+ }
+ MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
+ B.buildBuildVectorConstant(Dst, Csts);
+ };
+ return true;
+ }
- if (auto MaybeCst = ConstantFoldCastOp(MI.getOpcode(), DstTy, SrcOp, MRI)) {
- MatchInfo = *MaybeCst;
+ if (auto MaybeCst = ConstantFoldCastOp(Opc, DstTy, SrcOp, MRI)) {
+ MatchInfo = [Dst, Cst = *MaybeCst](MachineIRBuilder &B) {
+ B.buildConstant(Dst, Cst);
+ };
return true;
}
@@ -5223,50 +5170,128 @@ bool CombinerHelper::matchConstantFoldUnaryIntOp(MachineInstr &MI,
}
bool CombinerHelper::matchConstantFoldBinOp(MachineInstr &MI,
- APInt &MatchInfo) const {
+ BuildFnTy &MatchInfo) const {
+ Register Dst = MI.getOperand(0).getReg();
Register Op1 = MI.getOperand(1).getReg();
Register Op2 = MI.getOperand(2).getReg();
+ LLT DstTy = MRI.getType(Dst);
+
+ if (DstTy.isVector()) {
+ // Pointer-element vectors (e.g. <n x ptr> from a vector G_PTR_ADD) have no
+ // G_BUILD_VECTOR of G_CONSTANT representation.
+ if (DstTy.getElementType().isPointer())
+ return false;
+ SmallVector<APInt> Csts =
+ ConstantFoldVectorBinop(MI.getOpcode(), Op1, Op2, MRI);
+ if (Csts.empty())
+ return false;
+ MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
+ B.buildBuildVectorConstant(Dst, Csts);
+ };
+ return true;
+ }
+
auto MaybeCst = ConstantFoldBinOp(MI.getOpcode(), Op1, Op2, MRI);
if (!MaybeCst)
return false;
- MatchInfo = *MaybeCst;
+ MatchInfo = [Dst, Cst = *MaybeCst](MachineIRBuilder &B) {
+ B.buildConstant(Dst, Cst);
+ };
return true;
}
bool CombinerHelper::matchConstantFoldFPBinOp(MachineInstr &MI,
- ConstantFP *&MatchInfo) const {
+ BuildFnTy &MatchInfo) const {
+ Register Dst = MI.getOperand(0).getReg();
Register Op1 = MI.getOperand(1).getReg();
Register Op2 = MI.getOperand(2).getReg();
+ LLT DstTy = MRI.getType(Dst);
+
+ if (DstTy.isVector()) {
+ SmallVector<APFloat> Csts =
+ ConstantFoldVectorFPBinop(MI.getOpcode(), Op1, Op2, MRI);
+ if (Csts.empty())
+ return false;
+ MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
+ B.buildBuildVectorFConstant(Dst, Csts);
+ };
+ return true;
+ }
+
auto MaybeCst = ConstantFoldFPBinOp(MI.getOpcode(), Op1, Op2, MRI);
if (!MaybeCst)
return false;
- MatchInfo =
- ConstantFP::get(MI.getMF()->getFunction().getContext(), *MaybeCst);
+ MatchInfo = [Dst, Cst = *MaybeCst](MachineIRBuilder &B) {
+ B.buildFConstant(Dst, Cst);
+ };
+ return true;
+}
+
+bool CombinerHelper::matchConstantFoldFPUnary(MachineInstr &MI,
+ BuildFnTy &MatchInfo) const {
+ Register Dst = MI.getOperand(0).getReg();
+ SmallVector<APFloat> Csts = ConstantFoldUnaryFPOp(
+ MI.getOpcode(), MRI.getType(Dst), MI.getOperand(1).getReg(), MRI);
+ if (Csts.empty())
+ return false;
+ MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
+ if (Csts.size() == 1)
+ B.buildFConstant(Dst, Csts[0]);
+ else
+ B.buildBuildVectorFConstant(Dst, Csts);
+ };
return true;
}
bool CombinerHelper::matchConstantFoldFMA(MachineInstr &MI,
- ConstantFP *&MatchInfo) const {
+ BuildFnTy &MatchInfo) const {
assert(MI.getOpcode() == TargetOpcode::G_FMA ||
MI.getOpcode() == TargetOpcode::G_FMAD);
+ Register Dst = MI.getOperand(0).getReg();
auto [_, Op1, Op2, Op3] = MI.getFirst4Regs();
+ LLT DstTy = MRI.getType(Dst);
- const ConstantFP *Op3Cst = getConstantFPVRegVal(Op3, MRI);
- if (!Op3Cst)
- return false;
+ auto FoldFMA = [](const APFloat &A, const APFloat &B,
+ const APFloat &C) -> APFloat {
+ APFloat Res(A);
+ Res.fusedMultiplyAdd(B, C, APFloat::rmNearestTiesToEven);
+ return Res;
+ };
- const ConstantFP *Op2Cst = getConstantFPVRegVal(Op2, MRI);
- if (!Op2Cst)
- return false;
+ if (DstTy.isVector()) {
+ auto *BV1 = getOpcodeDef<GBuildVector>(Op1, MRI);
+ auto *BV2 = getOpcodeDef<GBuildVector>(Op2, MRI);
+ auto *BV3 = getOpcodeDef<GBuildVector>(Op3, MRI);
+ if (!BV1 || !BV2 || !BV3)
+ return false;
+ unsigned NumElts = BV1->getNumSources();
+ if (BV2->getNumSources() != NumElts || BV3->getNumSources() != NumElts)
+ return false;
+ SmallVector<APFloat> Csts;
+ for (unsigned I = 0; I != NumElts; ++I) {
+ const ConstantFP *C1 = getConstantFPVRegVal(BV1->getSourceReg(I), MRI);
+ const ConstantFP *C2 = getConstantFPVRegVal(BV2->getSourceReg(I), MRI);
+ const ConstantFP *C3 = getConstantFPVRegVal(BV3->getSourceReg(I), MRI);
+ if (!C1 || !C2 || !C3)
+ return false;
+ Csts.push_back(
+ FoldFMA(C1->getValueAPF(), C2->getValueAPF(), C3->getValueAPF()));
+ }
+ MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
+ B.buildBuildVectorFConstant(Dst, Csts);
+ };
+ return true;
+ }
+ const ConstantFP *Op3Cst = getConstantFPVRegVal(Op3, MRI);
+ const ConstantFP *Op2Cst = getConstantFPVRegVal(Op2, MRI);
const ConstantFP *Op1Cst = getConstantFPVRegVal(Op1, MRI);
- if (!Op1Cst)
+ if (!Op1Cst || !Op2Cst || !Op3Cst)
return false;
- APFloat Op1F = Op1Cst->getValueAPF();
- Op1F.fusedMultiplyAdd(Op2Cst->getValueAPF(), Op3Cst->getValueAPF(),
- APFloat::rmNearestTiesToEven);
- MatchInfo = ConstantFP::get(MI.getMF()->getFunction().getContext(), Op1F);
+ APFloat Res = FoldFMA(Op1Cst->getValueAPF(), Op2Cst->getValueAPF(),
+ Op3Cst->getValueAPF());
+ MatchInfo = [Dst, Res](MachineIRBuilder &B) { B.buildFConstant(Dst, Res); };
return true;
}
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index 24996a85f4cf15..0eb46a0b4f5f31 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -801,6 +801,17 @@ MachineIRBuilder::buildBuildVectorConstant(const DstOp &Res,
return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec);
}
+MachineInstrBuilder
+MachineIRBuilder::buildBuildVectorFConstant(const DstOp &Res,
+ ArrayRef<APFloat> Ops) {
+ SmallVector<SrcOp> TmpVec;
+ TmpVec.reserve(Ops.size());
+ LLT EltTy = Res.getLLTTy(*getMRI()).getElementType();
+ for (const auto &Op : Ops)
+ TmpVec.push_back(buildFConstant(EltTy, Op));
+ return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec);
+}
+
MachineInstrBuilder MachineIRBuilder::buildSplatBuildVector(const DstOp &Res,
const SrcOp &Src) {
SmallVector<SrcOp, 8> TmpVec(Res.getLLTTy(*getMRI()).getNumElements(), Src);
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 9a369bbdced1ad..e80a82bc19482a 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -37,6 +37,7 @@
#include "llvm/Support/UndefPoison.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/Utils/SizeOpts.h"
+#include <cmath>
#include <limits>
#include <numeric>
#include <optional>
@@ -828,6 +829,32 @@ llvm::ConstantFoldVectorBinop(unsigned Opcode, const Register Op1,
return FoldedElements;
}
+SmallVector<APFloat>
+llvm::ConstantFoldVectorFPBinop(unsigned Opcode, const Register Op1,
+ const Register Op2,
+ const MachineRegisterInfo &MRI) {
+ auto *SrcVec2 = getBuildVectorLikeDef(Op2, MRI);
+ if (!SrcVec2)
+ return SmallVector<APFloat>();
+
+ auto *SrcVec1 = getBuildVectorLikeDef(Op1, MRI);
+ if (!SrcVec1)
+ return SmallVector<APFloat>();
+
+ if (SrcVec1->getNumSources() != SrcVec2->getNumSources())
+ return SmallVector<APFloat>();
+
+ SmallVector<APFloat> FoldedElements;
+ for (unsigned Idx = 0, E = SrcVec1->getNumSources(); Idx < E; ++Idx) {
+ auto MaybeCst = ConstantFoldFPBinOp(Opcode, SrcVec1->getSourceReg(Idx),
+ SrcVec2->getSourceReg(Idx), MRI);
+ if (!MaybeCst)
+ return SmallVector<APFloat>();
+ FoldedElements.push_back(*MaybeCst);
+ }
+ return FoldedElements;
+}
+
Align llvm::inferAlignFromPtrInfo(MachineFunction &MF,
const MachinePointerInfo &MPO) {
auto PSV = dyn_cast_if_present<const PseudoSourceValue *>(MPO.V);
@@ -979,6 +1006,85 @@ llvm::ConstantFoldUnaryIntOp(unsigned Opcode, LLT DstTy, Register Src,
return {};
}
+SmallVector<APFloat>
+llvm::ConstantFoldUnaryFPOp(unsigned Opcode, LLT DstTy, Register Src,
+ const MachineRegisterInfo &MRI) {
+ LLT DstEltTy = DstTy.getScalarType();
+ auto Fold = [Opcode, DstEltTy](const APFloat &V) -> APFloat {
+ APFloat Result(V);
+ bool Unused;
+ switch (Opcode) {
+ case TargetOpcode::G_FNEG:
+ Result.changeSign();
+ return Result;
+ case TargetOpcode::G_FABS:
+ Result.clearSign();
+ return Result;
+ case TargetOpcode::G_FCEIL:
+ Result.roundToIntegral(APFloat::rmTowardPositive);
+ return Result;
+ case TargetOpcode::G_FFLOOR:
+ Result.roundToIntegral(APFloat::rmTowardNegative);
+ return Result;
+ case TargetOpcode::G_INTRINSIC_TRUNC:
+ Result.roundToIntegral(APFloat::rmTowardZero);
+ return Result;
+ case TargetOpcode::G_INTRINSIC_ROUND:
+ Result.roundToIntegral(APFloat::rmNearestTiesToAway);
+ return Result;
+ case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
+ case TargetOpcode::G_FRINT:
+ case TargetOpcode::G_FNEARBYINT:
+ Result.roundToIntegral(APFloat::rmNearestTiesToEven);
+ return Result;
+ case TargetOpcode::G_FPEXT:
+ case TargetOpcode::G_FPTRUNC:
+ Result.convert(getFltSemanticForLLT(DstEltTy),
+ APFloat::rmNearestTiesToEven, &Unused);
+ return Result;
+ case TargetOpcode::G_FSQRT:
+ Result.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
+ &Unused);
+ Result = APFloat(std::sqrt(Result.convertToDouble()));
+ break;
+ case TargetOpcode::G_FLOG2:
+ Result.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
+ &Unused);
+ Result = APFloat(std::log2(Result.convertToDouble()));
+ break;
+ default:
+ llvm_unreachable("unexpected opcode in ConstantFoldUnaryFPOp");
+ }
+ // Only G_FSQRT and G_FLOG2 reach here; convert the double result back to
+ // the source (== destination) semantics.
+ Result.convert(V.getSemantics(), APFloat::rmNearestTiesToEven, &Unused);
+ return Result;
+ };
+
+ auto tryFoldScalar = [&](Register R) -> std::optional<APFloat> {
+ if (const ConstantFP *Cst = getConstantFPVRegVal(R, MRI))
+ return Fold(Cst->getValueAPF());
+ return std::nullopt;
+ };
+ if (MRI.getType(Src).isVector()) {
+ auto *BV = getOpcodeDef<GBuildVector>(Src, MRI);
+ if (!BV)
+ return {};
+ SmallVector<APFloat> Folded;
+ for (unsigned SrcIdx = 0; SrcIdx < BV->getNumSources(); ++SrcIdx) {
+ if (auto MaybeFold = tryFoldScalar(BV->getSourceReg(SrcIdx))) {
+ Folded.emplace_back(std::move(*MaybeFold));
+ continue;
+ }
+ return {};
+ }
+ return Folded;
+ }
+ if (auto MaybeCst = tryFoldScalar(Src))
+ return {std::move(*MaybeCst)};
+ return {};
+}
+
std::optional<SmallVector<APInt>>
llvm::ConstantFoldICmp(unsigned Pred, const Register Op1, const Register Op2,
unsigned DstScalarSizeInBits, unsigned ExtOp,
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-logic-of-compare.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-logic-of-compare.mir
index 7bb166e0312c28..92d1a81098e1d9 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-logic-of-compare.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-logic-of-compare.mir
@@ -386,9 +386,8 @@ body: |
; CHECK-LABEL: name: test_fcmp_and_fcmp_with_vectors
; CHECK: liveins: $x0, $x1
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(i1) = G_CONSTANT i1 false
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x i1>) = G_BUILD_VECTOR [[C]](i1), [[C]](i1)
- ; CHECK-NEXT: %zext:_(<2 x i64>) = G_ZEXT [[BUILD_VECTOR]](<2 x i1>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: %zext:_(<2 x i64>) = G_BUILD_VECTOR [[C]](i64), [[C]](i64)
; CHECK-NEXT: $q0 = COPY %zext(<2 x i64>)
%0:_(f64) = COPY $x0
%1:_(f64) = COPY $x1
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-overflow.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-overflow.mir
index df2cdb15ef8994..9b98cd45da0dee 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-overflow.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-overflow.mir
@@ -157,9 +157,8 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $w0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $w1
; CHECK-NEXT: %bv0:_(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY]](i32), [[COPY1]](i32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(i1) = G_CONSTANT i1 false
- ; CHECK-NEXT: %o:_(<4 x i1>) = G_BUILD_VECTOR [[C]](i1), [[C]](i1), [[C]](i1), [[C]](i1)
- ; CHECK-NEXT: %o_wide:_(<4 x i32>) = G_ZEXT %o(<4 x i1>)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: %o_wide:_(<4 x i32>) = G_BUILD_VECTOR [[C]](i32), [[C]](i32), [[C]](i32), [[C]](i32)
; CHECK-NEXT: $q0 = COPY %bv0(<4 x i32>)
; CHECK-NEXT: $q1 = COPY %o_wide(<4 x i32>)
; CHECK-NEXT: RET_ReallyLR implicit $w0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll b/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll
index 461a7ef67e9e0b..66fcd1d0c02e0e 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll
@@ -210,24 +210,23 @@ define <8 x i16> @pr38477(<8 x i16> %a0) {
; GISEL-LABEL: pr38477:
; GISEL: // %bb.0:
; GISEL-NEXT: adrp x8, .LCPI5_3
+; GISEL-NEXT: adrp x9, .LCPI5_0
; GISEL-NEXT: ldr q1, [x8, :lo12:.LCPI5_3]
; GISEL-NEXT: adrp x8, .LCPI5_2
; GISEL-NEXT: ldr q3, [x8, :lo12:.LCPI5_2]
-; GISEL-NEXT: adrp x8, .LCPI5_0
+; GISEL-NEXT: adrp x8, .LCPI5_1
; GISEL-NEXT: umull2 v2.4s, v0.8h, v1.8h
; GISEL-NEXT: umull v1.4s, v0.4h, v1.4h
; GISEL-NEXT: uzp2 v1.8h, v1.8h, v2.8h
; GISEL-NEXT: sub v2.8h, v0.8h, v1.8h
; GISEL-NEXT: umull2 v4.4s, v2.8h, v3.8h
; GISEL-NEXT: umull v2.4s, v2.4h, v3.4h
-; GISEL-NEXT: ldr d3, [x8, :lo12:.LCPI5_0]
-; GISEL-NEXT: adrp x8, .LCPI5_1
-; GISEL-NEXT: ushll v3.8h, v3.8b, #0
+; GISEL-NEXT: ldr q3, [x8, :lo12:.LCPI5_1]
; GISEL-NEXT: uzp2 v2.8h, v2.8h, v4.8h
-; GISEL-NEXT: ldr q4, [x8, :lo12:.LCPI5_1]
-; GISEL-NEXT: shl v3.8h, v3.8h, #15
+; GISEL-NEXT: ldr q4, [x9, :lo12:.LCPI5_0]
; GISEL-NEXT: add v1.8h, v2.8h, v1.8h
-; GISEL-NEXT: neg v2.8h, v4.8h
+; GISEL-NEXT: neg v2.8h, v3.8h
+; GISEL-NEXT: shl v3.8h, v4.8h, #15
; GISEL-NEXT: ushl v1.8h, v1.8h, v2.8h
; GISEL-NEXT: cmlt v2.8h, v3.8h, #0
; GISEL-NEXT: bif v0.16b, v1.16b, v2.16b
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir
index 9840360c418385..66b1b9eb6caea6 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir
@@ -408,3 +408,111 @@ body: |
RET_ReallyLR implicit $s0
...
+---
+name: add_vector
+legalized: true
+liveins:
+ - { reg: '$q0' }
+body: |
+ bb.1.entry:
+ liveins: $q0
+
+ ; Element-wise fold of an integer vector binop: <40, 10> + <2, 2> = <42, 12>.
+ ; CHECK-LABEL: name: add_vector
+ ; CHECK: liveins: $q0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 42
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 12
+ ; CHECK-NEXT: %res:_(<2 x i64>) = G_BUILD_VECTOR [[C]](i64), [[C1]](i64)
+ ; CHECK-NEXT: $q0 = COPY %res(<2 x i64>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %a0:_(i64) = G_CONSTANT i64 40
+ %a1:_(i64) = G_CONSTANT i64 10
+ %b:_(i64) = G_CONSTANT i64 2
+ %av:_(<2 x i64>) = G_BUILD_VECTOR %a0(i64), %a1(i64)
+ %bv:_(<2 x i64>) = G_BUILD_VECTOR %b(i64), %b(i64)
+ %res:_(<2 x i64>) = G_ADD %av, %bv
+ $q0 = COPY %res(<2 x i64>)
+ RET_ReallyLR implicit $q0
+
+...
+---
+name: fadd_vector
+legalized: true
+liveins:
+ - { reg: '$q0' }
+body: |
+ bb.1.entry:
+ liveins: $q0
+
+ ; Element-wise fold of an FP vector binop: <40.0, 1.0> + <2.0, 2.0>.
+ ; CHECK-LABEL: name: fadd_vector
+ ; CHECK: liveins: $q0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double 4.200000e+01
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(f64) = G_FCONSTANT double 3.000000e+00
+ ; CHECK-NEXT: %res:_(<2 x f64>) = G_BUILD_VECTOR [[C]](f64), [[C1]](f64)
+ ; CHECK-NEXT: $q0 = COPY %res(<2 x f64>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %a0:_(f64) = G_FCONSTANT double 40.0
+ %a1:_(f64) = G_FCONSTANT double 1.0
+ %b:_(f64) = G_FCONSTANT double 2.0
+ %av:_(<2 x f64>) = G_BUILD_VECTOR %a0(f64), %a1(f64)
+ %bv:_(<2 x f64>) = G_BUILD_VECTOR %b(f64), %b(f64)
+ %res:_(<2 x f64>) = G_FADD %av, %bv
+ $q0 = COPY %res(<2 x f64>)
+ RET_ReallyLR implicit $q0
+
+...
+---
+name: zext_vector
+legalized: true
+liveins:
+ - { reg: '$q0' }
+body: |
+ bb.1.entry:
+ liveins: $q0
+
+ ; Element-wise fold of a vector cast: zext(<7, 8> : <2 x i32>) to <2 x i64>.
+ ; CHECK-LABEL: name: zext_vector
+ ; CHECK: liveins: $q0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 7
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
+ ; CHECK-NEXT: %res:_(<2 x i64>) = G_BUILD_VECTOR [[C]](i64), [[C1]](i64)
+ ; CHECK-NEXT: $q0 = COPY %res(<2 x i64>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %a0:_(i32) = G_CONSTANT i32 7
+ %a1:_(i32) = G_CONSTANT i32 8
+ %src:_(<2 x i32>) = G_BUILD_VECTOR %a0(i32), %a1(i32)
+ %res:_(<2 x i64>) = G_ZEXT %src(<2 x i32>)
+ $q0 = COPY %res(<2 x i64>)
+ RET_ReallyLR implicit $q0
+
+...
+---
+name: fneg_vector
+legalized: true
+liveins:
+ - { reg: '$q0' }
+body: |
+ bb.1.entry:
+ liveins: $q0
+
+ ; Element-wise fold of a unary FP op: fneg(<3.0, -4.0>) = <-3.0, 4.0>.
+ ; CHECK-LABEL: name: fneg_vector
+ ; CHECK: liveins: $q0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double -3.000000e+00
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(f64) = G_FCONSTANT double 4.000000e+00
+ ; CHECK-NEXT: %res:_(<2 x f64>) = G_BUILD_VECTOR [[C]](f64), [[C1]](f64)
+ ; CHECK-NEXT: $q0 = COPY %res(<2 x f64>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %a0:_(f64) = G_FCONSTANT double 3.0
+ %a1:_(f64) = G_FCONSTANT double -4.0
+ %src:_(<2 x f64>) = G_BUILD_VECTOR %a0(f64), %a1(f64)
+ %res:_(<2 x f64>) = G_FNEG %src(<2 x f64>)
+ $q0 = COPY %res(<2 x f64>)
+ RET_ReallyLR implicit $q0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-constant-fold-fma.mir b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-constant-fold-fma.mir
new file mode 100644
index 00000000000000..83ed5c03ecd39d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-constant-fold-fma.mir
@@ -0,0 +1,58 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64 -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name: fma_scalar
+legalized: true
+liveins:
+ - { reg: '$d0' }
+body: |
+ bb.1.entry:
+ liveins: $d0
+
+ ; fma(3.0, 4.0, 5.0) = 17.0
+ ; CHECK-LABEL: name: fma_scalar
+ ; CHECK: liveins: $d0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %res:_(f64) = G_FCONSTANT double 1.700000e+01
+ ; CHECK-NEXT: $d0 = COPY %res(f64)
+ ; CHECK-NEXT: RET_ReallyLR implicit $d0
+ %a:_(f64) = G_FCONSTANT double 3.0
+ %b:_(f64) = G_FCONSTANT double 4.0
+ %c:_(f64) = G_FCONSTANT double 5.0
+ %res:_(f64) = G_FMA %a, %b, %c
+ $d0 = COPY %res(f64)
+ RET_ReallyLR implicit $d0
+
+...
+---
+name: fma_vector
+legalized: true
+liveins:
+ - { reg: '$q0' }
+body: |
+ bb.1.entry:
+ liveins: $q0
+
+ ; Element-wise fma: fma(<3.0, 2.0>, <4.0, 4.0>, <5.0, 1.0>) = <17.0, 9.0>.
+ ; CHECK-LABEL: name: fma_vector
+ ; CHECK: liveins: $q0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double 1.700000e+01
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(f64) = G_FCONSTANT double 9.000000e+00
+ ; CHECK-NEXT: %res:_(<2 x f64>) = G_BUILD_VECTOR [[C]](f64), [[C1]](f64)
+ ; CHECK-NEXT: $q0 = COPY %res(<2 x f64>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %a0:_(f64) = G_FCONSTANT double 3.0
+ %a1:_(f64) = G_FCONSTANT double 2.0
+ %b:_(f64) = G_FCONSTANT double 4.0
+ %c0:_(f64) = G_FCONSTANT double 5.0
+ %c1:_(f64) = G_FCONSTANT double 1.0
+ %av:_(<2 x f64>) = G_BUILD_VECTOR %a0(f64), %a1(f64)
+ %bv:_(<2 x f64>) = G_BUILD_VECTOR %b(f64), %b(f64)
+ %cv:_(<2 x f64>) = G_BUILD_VECTOR %c0(f64), %c1(f64)
+ %res:_(<2 x f64>) = G_FMA %av, %bv, %cv
+ $q0 = COPY %res(<2 x f64>)
+ RET_ReallyLR implicit $q0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/arm64-vabs.ll b/llvm/test/CodeGen/AArch64/arm64-vabs.ll
index d175247cca3d96..bf69869b192b48 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vabs.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vabs.ll
@@ -2032,25 +2032,26 @@ define <16 x i16> @uabd16b_i16_const_select(<16 x i8> %a) {
;
; CHECK-GI-LABEL: uabd16b_i16_const_select:
; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI106_3
+; CHECK-GI-NEXT: ushll.8h v2, v0, #0
+; CHECK-GI-NEXT: ushll2.8h v4, v0, #0
+; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI106_3]
; CHECK-GI-NEXT: adrp x8, .LCPI106_1
-; CHECK-GI-NEXT: mov d3, v0[1]
-; CHECK-GI-NEXT: ushll.8h v4, v0, #0
-; CHECK-GI-NEXT: ldr d1, [x8, :lo12:.LCPI106_1]
+; CHECK-GI-NEXT: ldr q3, [x8, :lo12:.LCPI106_1]
; CHECK-GI-NEXT: adrp x8, .LCPI106_0
-; CHECK-GI-NEXT: ushll2.8h v6, v0, #0
-; CHECK-GI-NEXT: ldr d2, [x8, :lo12:.LCPI106_0]
-; CHECK-GI-NEXT: ushll.8h v5, v1, #0
-; CHECK-GI-NEXT: usubl.8h v16, v0, v1
-; CHECK-GI-NEXT: usubl.8h v1, v1, v0
-; CHECK-GI-NEXT: ushll.8h v7, v2, #0
-; CHECK-GI-NEXT: usubl.8h v2, v3, v2
-; CHECK-GI-NEXT: cmhi.8h v3, v5, v4
-; CHECK-GI-NEXT: cmhi.8h v4, v7, v6
-; CHECK-GI-NEXT: usubw2.8h v5, v7, v0
-; CHECK-GI-NEXT: mov.16b v0, v3
-; CHECK-GI-NEXT: bsl.16b v0, v1, v16
+; CHECK-GI-NEXT: ldr q5, [x8, :lo12:.LCPI106_0]
+; CHECK-GI-NEXT: adrp x8, .LCPI106_2
+; CHECK-GI-NEXT: cmhi.8h v2, v1, v2
+; CHECK-GI-NEXT: ldr q6, [x8, :lo12:.LCPI106_2]
+; CHECK-GI-NEXT: uaddw.8h v3, v3, v0
+; CHECK-GI-NEXT: usubw.8h v1, v1, v0
+; CHECK-GI-NEXT: uaddw2.8h v5, v5, v0
+; CHECK-GI-NEXT: cmhi.8h v4, v6, v4
+; CHECK-GI-NEXT: usubw2.8h v6, v6, v0
+; CHECK-GI-NEXT: mov.16b v0, v2
+; CHECK-GI-NEXT: bsl.16b v0, v1, v3
; CHECK-GI-NEXT: mov.16b v1, v4
-; CHECK-GI-NEXT: bsl.16b v1, v5, v2
+; CHECK-GI-NEXT: bsl.16b v1, v6, v5
; CHECK-GI-NEXT: ret
%aext = zext <16 x i8> %a to <16 x i16>
%bext = zext <16 x i8> <i8 39, i8 42, i8 51, i8 51, i8 0, i8 0, i8 54, i8 57, i8 66, i8 69, i8 75, i8 69, i8 75, i8 81, i8 255, i8 99> to <16 x i16>
@@ -2074,25 +2075,26 @@ define <16 x i16> @sabd16b_i16_const_select(<16 x i8> %a) {
;
; CHECK-GI-LABEL: sabd16b_i16_const_select:
; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: adrp x8, .LCPI107_3
+; CHECK-GI-NEXT: sshll.8h v2, v0, #0
+; CHECK-GI-NEXT: sshll2.8h v4, v0, #0
+; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI107_3]
; CHECK-GI-NEXT: adrp x8, .LCPI107_1
-; CHECK-GI-NEXT: mov d3, v0[1]
-; CHECK-GI-NEXT: sshll.8h v4, v0, #0
-; CHECK-GI-NEXT: ldr d1, [x8, :lo12:.LCPI107_1]
+; CHECK-GI-NEXT: ldr q3, [x8, :lo12:.LCPI107_1]
; CHECK-GI-NEXT: adrp x8, .LCPI107_0
-; CHECK-GI-NEXT: sshll2.8h v6, v0, #0
-; CHECK-GI-NEXT: ldr d2, [x8, :lo12:.LCPI107_0]
-; CHECK-GI-NEXT: sshll.8h v5, v1, #0
-; CHECK-GI-NEXT: ssubl.8h v16, v0, v1
-; CHECK-GI-NEXT: ssubl.8h v1, v1, v0
-; CHECK-GI-NEXT: sshll.8h v7, v2, #0
-; CHECK-GI-NEXT: ssubl.8h v2, v3, v2
-; CHECK-GI-NEXT: cmgt.8h v3, v5, v4
-; CHECK-GI-NEXT: cmgt.8h v4, v7, v6
-; CHECK-GI-NEXT: ssubw2.8h v5, v7, v0
-; CHECK-GI-NEXT: mov.16b v0, v3
-; CHECK-GI-NEXT: bsl.16b v0, v1, v16
+; CHECK-GI-NEXT: ldr q5, [x8, :lo12:.LCPI107_0]
+; CHECK-GI-NEXT: adrp x8, .LCPI107_2
+; CHECK-GI-NEXT: cmgt.8h v2, v1, v2
+; CHECK-GI-NEXT: ldr q6, [x8, :lo12:.LCPI107_2]
+; CHECK-GI-NEXT: saddw.8h v3, v3, v0
+; CHECK-GI-NEXT: ssubw.8h v1, v1, v0
+; CHECK-GI-NEXT: saddw2.8h v5, v5, v0
+; CHECK-GI-NEXT: cmgt.8h v4, v6, v4
+; CHECK-GI-NEXT: ssubw2.8h v6, v6, v0
+; CHECK-GI-NEXT: mov.16b v0, v2
+; CHECK-GI-NEXT: bsl.16b v0, v1, v3
; CHECK-GI-NEXT: mov.16b v1, v4
-; CHECK-GI-NEXT: bsl.16b v1, v5, v2
+; CHECK-GI-NEXT: bsl.16b v1, v6, v5
; CHECK-GI-NEXT: ret
%aext = sext <16 x i8> %a to <16 x i16>
%bext = sext <16 x i8> <i8 -39, i8 42, i8 -51, i8 51, i8 0, i8 0, i8 -54, i8 57, i8 -66, i8 69, i8 75, i8 69, i8 75, i8 81, i8 -128, i8 99> to <16 x i16>
diff --git a/llvm/test/CodeGen/AArch64/combine-sdiv.ll b/llvm/test/CodeGen/AArch64/combine-sdiv.ll
index b160333edb4873..ac33cf38cae1cf 100644
--- a/llvm/test/CodeGen/AArch64/combine-sdiv.ll
+++ b/llvm/test/CodeGen/AArch64/combine-sdiv.ll
@@ -260,19 +260,18 @@ define <8 x i16> @combine_vec_sdiv_by_pow2b_v8i16(<8 x i16> %x) {
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: adrp x8, .LCPI15_1
; CHECK-GI-NEXT: cmlt v2.8h, v0.8h, #0
+; CHECK-GI-NEXT: adrp x9, .LCPI15_0
; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI15_1]
-; CHECK-GI-NEXT: adrp x8, .LCPI15_0
-; CHECK-GI-NEXT: ldr d3, [x8, :lo12:.LCPI15_0]
; CHECK-GI-NEXT: adrp x8, .LCPI15_2
+; CHECK-GI-NEXT: ldr q3, [x9, :lo12:.LCPI15_0]
; CHECK-GI-NEXT: neg v1.8h, v1.8h
+; CHECK-GI-NEXT: shl v3.8h, v3.8h, #15
; CHECK-GI-NEXT: ushl v1.8h, v2.8h, v1.8h
-; CHECK-GI-NEXT: ushll v2.8h, v3.8b, #0
-; CHECK-GI-NEXT: ldr q3, [x8, :lo12:.LCPI15_2]
-; CHECK-GI-NEXT: neg v3.8h, v3.8h
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI15_2]
+; CHECK-GI-NEXT: neg v2.8h, v2.8h
; CHECK-GI-NEXT: add v1.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT: shl v2.8h, v2.8h, #15
-; CHECK-GI-NEXT: sshl v1.8h, v1.8h, v3.8h
-; CHECK-GI-NEXT: cmlt v2.8h, v2.8h, #0
+; CHECK-GI-NEXT: sshl v1.8h, v1.8h, v2.8h
+; CHECK-GI-NEXT: cmlt v2.8h, v3.8h, #0
; CHECK-GI-NEXT: bif v0.16b, v1.16b, v2.16b
; CHECK-GI-NEXT: ret
%1 = sdiv <8 x i16> %x, <i16 1, i16 4, i16 2, i16 16, i16 8, i16 32, i16 64, i16 2>
@@ -306,15 +305,14 @@ define <16 x i16> @combine_vec_sdiv_by_pow2b_v16i16(<16 x i16> %x) {
; CHECK-GI-NEXT: cmlt v3.8h, v0.8h, #0
; CHECK-GI-NEXT: cmlt v4.8h, v1.8h, #0
; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI16_1]
-; CHECK-GI-NEXT: adrp x8, .LCPI16_0
-; CHECK-GI-NEXT: ldr d5, [x8, :lo12:.LCPI16_0]
; CHECK-GI-NEXT: adrp x8, .LCPI16_2
+; CHECK-GI-NEXT: adrp x9, .LCPI16_0
+; CHECK-GI-NEXT: ldr q5, [x9, :lo12:.LCPI16_0]
; CHECK-GI-NEXT: neg v2.8h, v2.8h
-; CHECK-GI-NEXT: ushll v5.8h, v5.8b, #0
+; CHECK-GI-NEXT: shl v5.8h, v5.8h, #15
; CHECK-GI-NEXT: ushl v3.8h, v3.8h, v2.8h
; CHECK-GI-NEXT: ushl v2.8h, v4.8h, v2.8h
; CHECK-GI-NEXT: ldr q4, [x8, :lo12:.LCPI16_2]
-; CHECK-GI-NEXT: shl v5.8h, v5.8h, #15
; CHECK-GI-NEXT: neg v4.8h, v4.8h
; CHECK-GI-NEXT: add v3.8h, v0.8h, v3.8h
; CHECK-GI-NEXT: add v2.8h, v1.8h, v2.8h
@@ -365,33 +363,32 @@ define <32 x i16> @combine_vec_sdiv_by_pow2b_v32i16(<32 x i16> %x) {
; CHECK-GI-NEXT: cmlt v5.8h, v0.8h, #0
; CHECK-GI-NEXT: cmlt v6.8h, v1.8h, #0
; CHECK-GI-NEXT: ldr q4, [x8, :lo12:.LCPI17_1]
-; CHECK-GI-NEXT: adrp x8, .LCPI17_0
; CHECK-GI-NEXT: cmlt v7.8h, v2.8h, #0
; CHECK-GI-NEXT: cmlt v16.8h, v3.8h, #0
-; CHECK-GI-NEXT: ldr d17, [x8, :lo12:.LCPI17_0]
; CHECK-GI-NEXT: adrp x8, .LCPI17_2
; CHECK-GI-NEXT: neg v4.8h, v4.8h
-; CHECK-GI-NEXT: ushll v17.8h, v17.8b, #0
+; CHECK-GI-NEXT: ldr q17, [x8, :lo12:.LCPI17_2]
+; CHECK-GI-NEXT: adrp x8, .LCPI17_0
+; CHECK-GI-NEXT: neg v17.8h, v17.8h
; CHECK-GI-NEXT: ushl v5.8h, v5.8h, v4.8h
; CHECK-GI-NEXT: ushl v6.8h, v6.8h, v4.8h
; CHECK-GI-NEXT: ushl v7.8h, v7.8h, v4.8h
; CHECK-GI-NEXT: ushl v4.8h, v16.8h, v4.8h
-; CHECK-GI-NEXT: ldr q16, [x8, :lo12:.LCPI17_2]
-; CHECK-GI-NEXT: shl v17.8h, v17.8h, #15
-; CHECK-GI-NEXT: neg v16.8h, v16.8h
+; CHECK-GI-NEXT: ldr q16, [x8, :lo12:.LCPI17_0]
+; CHECK-GI-NEXT: shl v16.8h, v16.8h, #15
; CHECK-GI-NEXT: add v5.8h, v0.8h, v5.8h
; CHECK-GI-NEXT: add v6.8h, v1.8h, v6.8h
; CHECK-GI-NEXT: add v7.8h, v2.8h, v7.8h
; CHECK-GI-NEXT: add v4.8h, v3.8h, v4.8h
-; CHECK-GI-NEXT: cmlt v17.8h, v17.8h, #0
-; CHECK-GI-NEXT: sshl v5.8h, v5.8h, v16.8h
-; CHECK-GI-NEXT: sshl v6.8h, v6.8h, v16.8h
-; CHECK-GI-NEXT: sshl v7.8h, v7.8h, v16.8h
-; CHECK-GI-NEXT: sshl v4.8h, v4.8h, v16.8h
-; CHECK-GI-NEXT: bif v0.16b, v5.16b, v17.16b
-; CHECK-GI-NEXT: bif v1.16b, v6.16b, v17.16b
-; CHECK-GI-NEXT: bif v2.16b, v7.16b, v17.16b
-; CHECK-GI-NEXT: bif v3.16b, v4.16b, v17.16b
+; CHECK-GI-NEXT: cmlt v16.8h, v16.8h, #0
+; CHECK-GI-NEXT: sshl v5.8h, v5.8h, v17.8h
+; CHECK-GI-NEXT: sshl v6.8h, v6.8h, v17.8h
+; CHECK-GI-NEXT: sshl v7.8h, v7.8h, v17.8h
+; CHECK-GI-NEXT: sshl v4.8h, v4.8h, v17.8h
+; CHECK-GI-NEXT: bif v0.16b, v5.16b, v16.16b
+; CHECK-GI-NEXT: bif v1.16b, v6.16b, v16.16b
+; CHECK-GI-NEXT: bif v2.16b, v7.16b, v16.16b
+; CHECK-GI-NEXT: bif v3.16b, v4.16b, v16.16b
; CHECK-GI-NEXT: ret
%1 = sdiv <32 x i16> %x, <i16 1, i16 4, i16 2, i16 16, i16 8, i16 32, i16 64, i16 2, i16 1, i16 4, i16 2, i16 16, i16 8, i16 32, i16 64, i16 2, i16 1, i16 4, i16 2, i16 16, i16 8, i16 32, i16 64, i16 2, i16 1, i16 4, i16 2, i16 16, i16 8, i16 32, i16 64, i16 2>
ret <32 x i16> %1
@@ -463,22 +460,25 @@ define <8 x i32> @combine_vec_sdiv_by_pow2b_v8i32(<8 x i32> %x) {
; CHECK-GI-NEXT: cmlt v4.4s, v1.4s, #0
; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI19_1]
; CHECK-GI-NEXT: adrp x8, .LCPI19_0
-; CHECK-GI-NEXT: ldr d5, [x8, :lo12:.LCPI19_0]
+; CHECK-GI-NEXT: ldr q5, [x8, :lo12:.LCPI19_0]
; CHECK-GI-NEXT: adrp x8, .LCPI19_2
; CHECK-GI-NEXT: neg v2.4s, v2.4s
-; CHECK-GI-NEXT: ushll v5.4s, v5.4h, #0
+; CHECK-GI-NEXT: ldr q6, [x8, :lo12:.LCPI19_2]
+; CHECK-GI-NEXT: neg v6.4s, v6.4s
; CHECK-GI-NEXT: ushl v3.4s, v3.4s, v2.4s
; CHECK-GI-NEXT: ushl v2.4s, v4.4s, v2.4s
-; CHECK-GI-NEXT: ldr q4, [x8, :lo12:.LCPI19_2]
-; CHECK-GI-NEXT: shl v5.4s, v5.4s, #31
-; CHECK-GI-NEXT: neg v4.4s, v4.4s
+; CHECK-GI-NEXT: ushll v4.4s, v5.4h, #0
+; CHECK-GI-NEXT: ushll2 v5.4s, v5.8h, #0
; CHECK-GI-NEXT: add v3.4s, v0.4s, v3.4s
+; CHECK-GI-NEXT: shl v4.4s, v4.4s, #31
; CHECK-GI-NEXT: add v2.4s, v1.4s, v2.4s
-; CHECK-GI-NEXT: sshl v3.4s, v3.4s, v4.4s
-; CHECK-GI-NEXT: sshl v2.4s, v2.4s, v4.4s
-; CHECK-GI-NEXT: cmlt v4.4s, v5.4s, #0
+; CHECK-GI-NEXT: shl v5.4s, v5.4s, #31
+; CHECK-GI-NEXT: sshl v3.4s, v3.4s, v6.4s
+; CHECK-GI-NEXT: sshl v2.4s, v2.4s, v6.4s
+; CHECK-GI-NEXT: cmlt v4.4s, v4.4s, #0
+; CHECK-GI-NEXT: cmlt v5.4s, v5.4s, #0
; CHECK-GI-NEXT: bif v0.16b, v3.16b, v4.16b
-; CHECK-GI-NEXT: bif v1.16b, v2.16b, v4.16b
+; CHECK-GI-NEXT: bif v1.16b, v2.16b, v5.16b
; CHECK-GI-NEXT: ret
%1 = sdiv <8 x i32> %x, <i32 1, i32 4, i32 8, i32 16, i32 1, i32 4, i32 8, i32 16>
ret <8 x i32> %1
@@ -522,32 +522,35 @@ define <16 x i32> @combine_vec_sdiv_by_pow2b_v16i32(<16 x i32> %x) {
; CHECK-GI-NEXT: cmlt v6.4s, v1.4s, #0
; CHECK-GI-NEXT: ldr q4, [x8, :lo12:.LCPI20_1]
; CHECK-GI-NEXT: adrp x8, .LCPI20_0
-; CHECK-GI-NEXT: cmlt v7.4s, v2.4s, #0
-; CHECK-GI-NEXT: cmlt v16.4s, v3.4s, #0
-; CHECK-GI-NEXT: ldr d17, [x8, :lo12:.LCPI20_0]
+; CHECK-GI-NEXT: cmlt v16.4s, v2.4s, #0
+; CHECK-GI-NEXT: cmlt v17.4s, v3.4s, #0
+; CHECK-GI-NEXT: ldr q7, [x8, :lo12:.LCPI20_0]
; CHECK-GI-NEXT: adrp x8, .LCPI20_2
; CHECK-GI-NEXT: neg v4.4s, v4.4s
-; CHECK-GI-NEXT: ushll v17.4s, v17.4h, #0
+; CHECK-GI-NEXT: ldr q18, [x8, :lo12:.LCPI20_2]
+; CHECK-GI-NEXT: ushll v19.4s, v7.4h, #0
+; CHECK-GI-NEXT: ushll2 v7.4s, v7.8h, #0
; CHECK-GI-NEXT: ushl v5.4s, v5.4s, v4.4s
; CHECK-GI-NEXT: ushl v6.4s, v6.4s, v4.4s
-; CHECK-GI-NEXT: ushl v7.4s, v7.4s, v4.4s
-; CHECK-GI-NEXT: ushl v4.4s, v16.4s, v4.4s
-; CHECK-GI-NEXT: ldr q16, [x8, :lo12:.LCPI20_2]
-; CHECK-GI-NEXT: shl v17.4s, v17.4s, #31
-; CHECK-GI-NEXT: neg v16.4s, v16.4s
+; CHECK-GI-NEXT: ushl v16.4s, v16.4s, v4.4s
+; CHECK-GI-NEXT: ushl v4.4s, v17.4s, v4.4s
+; CHECK-GI-NEXT: neg v17.4s, v18.4s
+; CHECK-GI-NEXT: shl v18.4s, v19.4s, #31
+; CHECK-GI-NEXT: shl v7.4s, v7.4s, #31
; CHECK-GI-NEXT: add v5.4s, v0.4s, v5.4s
; CHECK-GI-NEXT: add v6.4s, v1.4s, v6.4s
-; CHECK-GI-NEXT: add v7.4s, v2.4s, v7.4s
+; CHECK-GI-NEXT: add v16.4s, v2.4s, v16.4s
; CHECK-GI-NEXT: add v4.4s, v3.4s, v4.4s
-; CHECK-GI-NEXT: cmlt v17.4s, v17.4s, #0
-; CHECK-GI-NEXT: sshl v5.4s, v5.4s, v16.4s
-; CHECK-GI-NEXT: sshl v6.4s, v6.4s, v16.4s
-; CHECK-GI-NEXT: sshl v7.4s, v7.4s, v16.4s
-; CHECK-GI-NEXT: sshl v4.4s, v4.4s, v16.4s
-; CHECK-GI-NEXT: bif v0.16b, v5.16b, v17.16b
-; CHECK-GI-NEXT: bif v1.16b, v6.16b, v17.16b
-; CHECK-GI-NEXT: bif v2.16b, v7.16b, v17.16b
-; CHECK-GI-NEXT: bif v3.16b, v4.16b, v17.16b
+; CHECK-GI-NEXT: cmlt v18.4s, v18.4s, #0
+; CHECK-GI-NEXT: cmlt v7.4s, v7.4s, #0
+; CHECK-GI-NEXT: sshl v5.4s, v5.4s, v17.4s
+; CHECK-GI-NEXT: sshl v6.4s, v6.4s, v17.4s
+; CHECK-GI-NEXT: sshl v16.4s, v16.4s, v17.4s
+; CHECK-GI-NEXT: sshl v4.4s, v4.4s, v17.4s
+; CHECK-GI-NEXT: bif v0.16b, v5.16b, v18.16b
+; CHECK-GI-NEXT: bif v1.16b, v6.16b, v7.16b
+; CHECK-GI-NEXT: bif v2.16b, v16.16b, v18.16b
+; CHECK-GI-NEXT: bif v3.16b, v4.16b, v7.16b
; CHECK-GI-NEXT: ret
%1 = sdiv <16 x i32> %x, <i32 1, i32 4, i32 8, i32 16, i32 1, i32 4, i32 8, i32 16, i32 1, i32 4, i32 8, i32 16, i32 1, i32 4, i32 8, i32 16>
ret <16 x i32> %1
@@ -687,45 +690,49 @@ define <8 x i64> @combine_vec_sdiv_by_pow2b_v8i64(<8 x i64> %x) {
; CHECK-GI-NEXT: cmlt v7.2d, v0.2d, #0
; CHECK-GI-NEXT: ldr q4, [x8, :lo12:.LCPI23_2]
; CHECK-GI-NEXT: adrp x8, .LCPI23_1
-; CHECK-GI-NEXT: ldr d5, [x9, :lo12:.LCPI23_0]
-; CHECK-GI-NEXT: ldr q6, [x8, :lo12:.LCPI23_1]
+; CHECK-GI-NEXT: ldr q6, [x9, :lo12:.LCPI23_0]
+; CHECK-GI-NEXT: ldr q5, [x8, :lo12:.LCPI23_1]
; CHECK-GI-NEXT: cmlt v16.2d, v1.2d, #0
; CHECK-GI-NEXT: cmlt v17.2d, v2.2d, #0
; CHECK-GI-NEXT: neg v4.2d, v4.2d
-; CHECK-GI-NEXT: ushll v5.4s, v5.4h, #0
; CHECK-GI-NEXT: cmlt v19.2d, v3.2d, #0
-; CHECK-GI-NEXT: neg v6.2d, v6.2d
+; CHECK-GI-NEXT: ushll v20.4s, v6.4h, #0
+; CHECK-GI-NEXT: neg v5.2d, v5.2d
+; CHECK-GI-NEXT: ushll2 v6.4s, v6.8h, #0
; CHECK-GI-NEXT: adrp x8, .LCPI23_4
; CHECK-GI-NEXT: ldr q18, [x8, :lo12:.LCPI23_4]
; CHECK-GI-NEXT: adrp x8, .LCPI23_3
; CHECK-GI-NEXT: ushl v7.2d, v7.2d, v4.2d
-; CHECK-GI-NEXT: ushll v20.2d, v5.2s, #0
; CHECK-GI-NEXT: ushl v4.2d, v17.2d, v4.2d
-; CHECK-GI-NEXT: ushl v16.2d, v16.2d, v6.2d
-; CHECK-GI-NEXT: ushll v17.2d, v5.2s, #0
-; CHECK-GI-NEXT: ushl v6.2d, v19.2d, v6.2d
-; CHECK-GI-NEXT: ushll2 v5.2d, v5.4s, #0
-; CHECK-GI-NEXT: ldr q19, [x8, :lo12:.LCPI23_3]
-; CHECK-GI-NEXT: neg v18.2d, v18.2d
+; CHECK-GI-NEXT: ushll v17.2d, v20.2s, #0
+; CHECK-GI-NEXT: ushl v16.2d, v16.2d, v5.2d
+; CHECK-GI-NEXT: ushl v5.2d, v19.2d, v5.2d
+; CHECK-GI-NEXT: ushll2 v19.2d, v20.4s, #0
+; CHECK-GI-NEXT: ushll v20.2d, v6.2s, #0
+; CHECK-GI-NEXT: ushll2 v6.2d, v6.4s, #0
+; CHECK-GI-NEXT: ldr q21, [x8, :lo12:.LCPI23_3]
; CHECK-GI-NEXT: add v7.2d, v0.2d, v7.2d
-; CHECK-GI-NEXT: shl v20.2d, v20.2d, #63
; CHECK-GI-NEXT: add v4.2d, v2.2d, v4.2d
-; CHECK-GI-NEXT: add v1.2d, v1.2d, v16.2d
-; CHECK-GI-NEXT: shl v16.2d, v17.2d, #63
-; CHECK-GI-NEXT: neg v17.2d, v19.2d
-; CHECK-GI-NEXT: add v3.2d, v3.2d, v6.2d
-; CHECK-GI-NEXT: shl v5.2d, v5.2d, #63
-; CHECK-GI-NEXT: sshl v6.2d, v7.2d, v18.2d
-; CHECK-GI-NEXT: cmlt v7.2d, v20.2d, #0
+; CHECK-GI-NEXT: neg v18.2d, v18.2d
+; CHECK-GI-NEXT: add v16.2d, v1.2d, v16.2d
+; CHECK-GI-NEXT: add v5.2d, v3.2d, v5.2d
+; CHECK-GI-NEXT: neg v21.2d, v21.2d
+; CHECK-GI-NEXT: shl v17.2d, v17.2d, #63
+; CHECK-GI-NEXT: shl v19.2d, v19.2d, #63
+; CHECK-GI-NEXT: shl v20.2d, v20.2d, #63
+; CHECK-GI-NEXT: shl v6.2d, v6.2d, #63
+; CHECK-GI-NEXT: sshl v7.2d, v7.2d, v18.2d
; CHECK-GI-NEXT: sshl v4.2d, v4.2d, v18.2d
-; CHECK-GI-NEXT: cmlt v16.2d, v16.2d, #0
-; CHECK-GI-NEXT: sshl v1.2d, v1.2d, v17.2d
-; CHECK-GI-NEXT: sshl v3.2d, v3.2d, v17.2d
-; CHECK-GI-NEXT: cmlt v5.2d, v5.2d, #0
-; CHECK-GI-NEXT: bif v0.16b, v6.16b, v7.16b
-; CHECK-GI-NEXT: bif v2.16b, v4.16b, v16.16b
-; CHECK-GI-NEXT: bic v1.16b, v1.16b, v5.16b
-; CHECK-GI-NEXT: bic v3.16b, v3.16b, v5.16b
+; CHECK-GI-NEXT: sshl v16.2d, v16.2d, v21.2d
+; CHECK-GI-NEXT: sshl v5.2d, v5.2d, v21.2d
+; CHECK-GI-NEXT: cmlt v17.2d, v17.2d, #0
+; CHECK-GI-NEXT: cmlt v18.2d, v19.2d, #0
+; CHECK-GI-NEXT: cmlt v19.2d, v20.2d, #0
+; CHECK-GI-NEXT: cmlt v6.2d, v6.2d, #0
+; CHECK-GI-NEXT: bif v0.16b, v7.16b, v17.16b
+; CHECK-GI-NEXT: bif v1.16b, v16.16b, v18.16b
+; CHECK-GI-NEXT: bif v2.16b, v4.16b, v19.16b
+; CHECK-GI-NEXT: bif v3.16b, v5.16b, v6.16b
; CHECK-GI-NEXT: ret
%1 = sdiv <8 x i64> %x, <i64 1, i64 4, i64 8, i64 16, i64 1, i64 4, i64 8, i64 16>
ret <8 x i64> %1
@@ -1156,8 +1163,7 @@ define <8 x i16> @combine_vec_sdiv_nonuniform7(<8 x i16> %x) {
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: adrp x8, .LCPI34_0
; CHECK-GI-NEXT: neg v2.8h, v0.8h
-; CHECK-GI-NEXT: ldr d1, [x8, :lo12:.LCPI34_0]
-; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI34_0]
; CHECK-GI-NEXT: shl v1.8h, v1.8h, #15
; CHECK-GI-NEXT: cmlt v1.8h, v1.8h, #0
; CHECK-GI-NEXT: bit v0.16b, v2.16b, v1.16b
diff --git a/llvm/test/CodeGen/AArch64/extract-vector-elt.ll b/llvm/test/CodeGen/AArch64/extract-vector-elt.ll
index 3ab413b50e17d8..515c5e45ad965a 100644
--- a/llvm/test/CodeGen/AArch64/extract-vector-elt.ll
+++ b/llvm/test/CodeGen/AArch64/extract-vector-elt.ll
@@ -694,9 +694,6 @@ define i32 @extract_v4i32_abs_const(<4 x float> %a, i32 %c) {
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: adrp x8, .LCPI25_0
; CHECK-GI-NEXT: ldr q0, [x8, :lo12:.LCPI25_0]
-; CHECK-GI-NEXT: frintp v0.4s, v0.4s
-; CHECK-GI-NEXT: frintm v0.4s, v0.4s
-; CHECK-GI-NEXT: fabs v0.4s, v0.4s
; CHECK-GI-NEXT: fcvtzs v0.4s, v0.4s
; CHECK-GI-NEXT: abs v0.4s, v0.4s
; CHECK-GI-NEXT: mov s0, v0.s[1]
@@ -736,9 +733,6 @@ define i32 @extract_v4i32_abs_half_const(<4 x float> %a, i32 %c) {
; CHECK-GI-NEXT: ldr q0, [x8, :lo12:.LCPI26_0]
; CHECK-GI-NEXT: mov w8, w0
; CHECK-GI-NEXT: and x8, x8, #0x3
-; CHECK-GI-NEXT: frintp v0.4s, v0.4s
-; CHECK-GI-NEXT: frintm v0.4s, v0.4s
-; CHECK-GI-NEXT: fabs v0.4s, v0.4s
; CHECK-GI-NEXT: fcvtzs v0.4s, v0.4s
; CHECK-GI-NEXT: abs v0.4s, v0.4s
; CHECK-GI-NEXT: str q0, [sp]
diff --git a/llvm/test/CodeGen/AArch64/fcvt_combine.ll b/llvm/test/CodeGen/AArch64/fcvt_combine.ll
index 70220cbaa99aad..3a5654ca2ba827 100644
--- a/llvm/test/CodeGen/AArch64/fcvt_combine.ll
+++ b/llvm/test/CodeGen/AArch64/fcvt_combine.ll
@@ -356,12 +356,11 @@ define <8 x i16> @test_v8f16(<8 x half> %in) {
;
; CHECK-NO16-GI-LABEL: test_v8f16:
; CHECK-NO16-GI: // %bb.0:
-; CHECK-NO16-GI-NEXT: movi v1.4h, #68, lsl #8
; CHECK-NO16-GI-NEXT: fcvtl v2.4s, v0.4h
; CHECK-NO16-GI-NEXT: fcvtl2 v0.4s, v0.8h
-; CHECK-NO16-GI-NEXT: fcvtl v1.4s, v1.4h
-; CHECK-NO16-GI-NEXT: fmul v2.4s, v2.4s, v1.4s
-; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-NO16-GI-NEXT: fmov s1, #4.00000000
+; CHECK-NO16-GI-NEXT: fmul v2.4s, v2.4s, v1.s[0]
+; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.s[0]
; CHECK-NO16-GI-NEXT: fcvtn v1.4h, v2.4s
; CHECK-NO16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-NO16-GI-NEXT: fcvtl v1.4s, v1.4h
@@ -394,10 +393,9 @@ define <4 x i16> @test_v4f16(<4 x half> %in) {
;
; CHECK-NO16-GI-LABEL: test_v4f16:
; CHECK-NO16-GI: // %bb.0:
-; CHECK-NO16-GI-NEXT: movi v1.4h, #68, lsl #8
; CHECK-NO16-GI-NEXT: fcvtl v0.4s, v0.4h
-; CHECK-NO16-GI-NEXT: fcvtl v1.4s, v1.4h
-; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-NO16-GI-NEXT: fmov s1, #4.00000000
+; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.s[0]
; CHECK-NO16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-NO16-GI-NEXT: fcvtl v0.4s, v0.4h
; CHECK-NO16-GI-NEXT: fcvtzu v0.4s, v0.4s
@@ -429,10 +427,9 @@ define <4 x i32> @test_v4f16_i32(<4 x half> %in) {
;
; CHECK-NO16-GI-LABEL: test_v4f16_i32:
; CHECK-NO16-GI: // %bb.0:
-; CHECK-NO16-GI-NEXT: movi v1.4h, #68, lsl #8
; CHECK-NO16-GI-NEXT: fcvtl v0.4s, v0.4h
-; CHECK-NO16-GI-NEXT: fcvtl v1.4s, v1.4h
-; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-NO16-GI-NEXT: fmov s1, #4.00000000
+; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.s[0]
; CHECK-NO16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-NO16-GI-NEXT: fcvtl v0.4s, v0.4h
; CHECK-NO16-GI-NEXT: fcvtzs v0.4s, v0.4s
@@ -470,10 +467,10 @@ define <8 x i16> @test_v8f16_nan(<8 x half> %a) {
;
; CHECK-NO16-GI-LABEL: test_v8f16_nan:
; CHECK-NO16-GI: // %bb.0: // %entry
-; CHECK-NO16-GI-NEXT: movi v1.4h, #126, lsl #8
+; CHECK-NO16-GI-NEXT: mvni v1.4s, #63, msl #16
; CHECK-NO16-GI-NEXT: fcvtl v2.4s, v0.4h
; CHECK-NO16-GI-NEXT: fcvtl2 v0.4s, v0.8h
-; CHECK-NO16-GI-NEXT: fcvtl v1.4s, v1.4h
+; CHECK-NO16-GI-NEXT: fneg v1.4s, v1.4s
; CHECK-NO16-GI-NEXT: fmul v2.4s, v2.4s, v1.4s
; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
; CHECK-NO16-GI-NEXT: fcvtn v1.4h, v2.4s
@@ -946,12 +943,11 @@ define <8 x i16> @test_v8f16_sat(<8 x half> %in) {
;
; CHECK-NO16-GI-LABEL: test_v8f16_sat:
; CHECK-NO16-GI: // %bb.0:
-; CHECK-NO16-GI-NEXT: movi v1.4h, #68, lsl #8
; CHECK-NO16-GI-NEXT: fcvtl v2.4s, v0.4h
+; CHECK-NO16-GI-NEXT: fmov s1, #4.00000000
; CHECK-NO16-GI-NEXT: fcvtl2 v0.4s, v0.8h
-; CHECK-NO16-GI-NEXT: fcvtl v1.4s, v1.4h
-; CHECK-NO16-GI-NEXT: fmul v2.4s, v2.4s, v1.4s
-; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-NO16-GI-NEXT: fmul v2.4s, v2.4s, v1.s[0]
+; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.s[0]
; CHECK-NO16-GI-NEXT: fcvtn v1.4h, v2.4s
; CHECK-NO16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-NO16-GI-NEXT: fcvtl v1.4s, v1.4h
@@ -985,10 +981,9 @@ define <4 x i16> @test_v4f16_sat(<4 x half> %in) {
;
; CHECK-NO16-GI-LABEL: test_v4f16_sat:
; CHECK-NO16-GI: // %bb.0:
-; CHECK-NO16-GI-NEXT: movi v1.4h, #68, lsl #8
; CHECK-NO16-GI-NEXT: fcvtl v0.4s, v0.4h
-; CHECK-NO16-GI-NEXT: fcvtl v1.4s, v1.4h
-; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-NO16-GI-NEXT: fmov s1, #4.00000000
+; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.s[0]
; CHECK-NO16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-NO16-GI-NEXT: fcvtl v0.4s, v0.4h
; CHECK-NO16-GI-NEXT: fcvtzu v0.4s, v0.4s
@@ -1020,10 +1015,9 @@ define <4 x i32> @test_v4f16_i32_sat(<4 x half> %in) {
;
; CHECK-NO16-GI-LABEL: test_v4f16_i32_sat:
; CHECK-NO16-GI: // %bb.0:
-; CHECK-NO16-GI-NEXT: movi v1.4h, #68, lsl #8
; CHECK-NO16-GI-NEXT: fcvtl v0.4s, v0.4h
-; CHECK-NO16-GI-NEXT: fcvtl v1.4s, v1.4h
-; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-NO16-GI-NEXT: fmov s1, #4.00000000
+; CHECK-NO16-GI-NEXT: fmul v0.4s, v0.4s, v1.s[0]
; CHECK-NO16-GI-NEXT: fcvtn v0.4h, v0.4s
; CHECK-NO16-GI-NEXT: fcvtl v0.4s, v0.4h
; CHECK-NO16-GI-NEXT: fcvtzs v0.4s, v0.4s
diff --git a/llvm/test/CodeGen/AArch64/hadd-combine.ll b/llvm/test/CodeGen/AArch64/hadd-combine.ll
index 450069cd274280..6dbe5b2c3ecf76 100644
--- a/llvm/test/CodeGen/AArch64/hadd-combine.ll
+++ b/llvm/test/CodeGen/AArch64/hadd-combine.ll
@@ -95,19 +95,10 @@ define <8 x i16> @haddu_const_both() {
}
define <8 x i16> @haddu_const_bothhigh() {
-; CHECK-SD-LABEL: haddu_const_bothhigh:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: mvni v0.8h, #1
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: haddu_const_bothhigh:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi d0, #0xffffffffffffffff
-; CHECK-GI-NEXT: mvni v1.4h, #1
-; CHECK-GI-NEXT: uaddl v1.4s, v1.4h, v0.4h
-; CHECK-GI-NEXT: shrn v0.4h, v1.4s, #1
-; CHECK-GI-NEXT: shrn2 v0.8h, v1.4s, #1
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: haddu_const_bothhigh:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mvni v0.8h, #1
+; CHECK-NEXT: ret
%ext1 = zext <8 x i16> <i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534> to <8 x i32>
%ext2 = zext <8 x i16> <i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535> to <8 x i32>
%add = add <8 x i32> %ext1, %ext2
@@ -341,9 +332,7 @@ define <8 x i16> @hadds_const_bothhigh() {
; CHECK-GI-LABEL: hadds_const_bothhigh:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: adrp x8, .LCPI19_0
-; CHECK-GI-NEXT: mvni v0.8h, #128, lsl #8
-; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI19_0]
-; CHECK-GI-NEXT: shadd v0.8h, v1.8h, v0.8h
+; CHECK-GI-NEXT: ldr q0, [x8, :lo12:.LCPI19_0]
; CHECK-GI-NEXT: ret
%ext1 = sext <8 x i16> <i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766> to <8 x i32>
%ext2 = sext <8 x i16> <i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767> to <8 x i32>
@@ -615,21 +604,10 @@ define <8 x i16> @rhaddu_const_both() {
}
define <8 x i16> @rhaddu_const_bothhigh() {
-; CHECK-SD-LABEL: rhaddu_const_bothhigh:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: movi v0.2d, #0xffffffffffffffff
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: rhaddu_const_bothhigh:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi d0, #0xffffffffffffffff
-; CHECK-GI-NEXT: mvni v1.4h, #1
-; CHECK-GI-NEXT: movi v2.4s, #1
-; CHECK-GI-NEXT: uaddl v0.4s, v1.4h, v0.4h
-; CHECK-GI-NEXT: add v1.4s, v0.4s, v2.4s
-; CHECK-GI-NEXT: shrn v0.4h, v1.4s, #1
-; CHECK-GI-NEXT: shrn2 v0.8h, v1.4s, #1
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: rhaddu_const_bothhigh:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v0.2d, #0xffffffffffffffff
+; CHECK-NEXT: ret
%ext1 = zext <8 x i16> <i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534, i16 65534> to <8 x i32>
%ext2 = zext <8 x i16> <i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535, i16 65535> to <8 x i32>
%add1 = add <8 x i32> %ext1, %ext2
@@ -863,18 +841,10 @@ define <8 x i16> @rhadds_const_both() {
}
define <8 x i16> @rhadds_const_bothhigh() {
-; CHECK-SD-LABEL: rhadds_const_bothhigh:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: mvni v0.8h, #128, lsl #8
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: rhadds_const_bothhigh:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: adrp x8, .LCPI49_0
-; CHECK-GI-NEXT: mvni v0.8h, #128, lsl #8
-; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI49_0]
-; CHECK-GI-NEXT: srhadd v0.8h, v1.8h, v0.8h
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: rhadds_const_bothhigh:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mvni v0.8h, #128, lsl #8
+; CHECK-NEXT: ret
%ext1 = sext <8 x i16> <i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766, i16 32766> to <8 x i32>
%ext2 = sext <8 x i16> <i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767, i16 32767> to <8 x i32>
%add1 = add <8 x i32> %ext1, %ext2
diff --git a/llvm/test/CodeGen/AArch64/half-precision-signof-no-assert.ll b/llvm/test/CodeGen/AArch64/half-precision-signof-no-assert.ll
index b5831e32366229..d538004d5cd9a9 100644
--- a/llvm/test/CodeGen/AArch64/half-precision-signof-no-assert.ll
+++ b/llvm/test/CodeGen/AArch64/half-precision-signof-no-assert.ll
@@ -34,15 +34,13 @@ define ptr @fn(ptr %in, ptr %out) {
;
; CHECK-GI-LABEL: fn:
; CHECK-GI: // %bb.0: // %fn
-; CHECK-GI-NEXT: movi v0.2d, #0000000000000000
; CHECK-GI-NEXT: ldr d1, [x0]
+; CHECK-GI-NEXT: movi v0.4s, #1
; CHECK-GI-NEXT: fcvtl v1.4s, v1.4h
-; CHECK-GI-NEXT: fcvtl v0.4s, v0.4h
-; CHECK-GI-NEXT: fcmgt v2.4s, v1.4s, v0.4s
-; CHECK-GI-NEXT: fcmgt v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: movi v1.4s, #1
-; CHECK-GI-NEXT: orr v0.16b, v0.16b, v2.16b
-; CHECK-GI-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT: fcmgt v2.4s, v1.4s, #0.0
+; CHECK-GI-NEXT: fcmlt v1.4s, v1.4s, #0.0
+; CHECK-GI-NEXT: orr v1.16b, v1.16b, v2.16b
+; CHECK-GI-NEXT: and v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT: mvni v1.4h, #128, lsl #8
; CHECK-GI-NEXT: ucvtf v0.4s, v0.4s
; CHECK-GI-NEXT: fcvtn v0.4h, v0.4s
diff --git a/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll b/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll
index 017f7ea0de8913..ada83c54680a73 100644
--- a/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll
@@ -2528,9 +2528,7 @@ define <2 x i32> @fcmal2xfloat(<2 x float> %A, <2 x float> %B) {
;
; CHECK-GI-LABEL: fcmal2xfloat:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi v0.2s, #1
-; CHECK-GI-NEXT: shl v0.2s, v0.2s, #31
-; CHECK-GI-NEXT: cmlt v0.2s, v0.2s, #0
+; CHECK-GI-NEXT: movi d0, #0xffffffffffffffff
; CHECK-GI-NEXT: ret
%tmp3 = fcmp true <2 x float> %A, %B
%tmp4 = sext <2 x i1> %tmp3 to <2 x i32>
@@ -2538,34 +2536,19 @@ define <2 x i32> @fcmal2xfloat(<2 x float> %A, <2 x float> %B) {
}
define <4 x i32> @fcmal4xfloat(<4 x float> %A, <4 x float> %B) {
-; CHECK-SD-LABEL: fcmal4xfloat:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: movi v0.2d, #0xffffffffffffffff
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: fcmal4xfloat:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: movi v0.4s, #1
-; CHECK-GI-NEXT: shl v0.4s, v0.4s, #31
-; CHECK-GI-NEXT: cmlt v0.4s, v0.4s, #0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: fcmal4xfloat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v0.2d, #0xffffffffffffffff
+; CHECK-NEXT: ret
%tmp3 = fcmp true <4 x float> %A, %B
%tmp4 = sext <4 x i1> %tmp3 to <4 x i32>
ret <4 x i32> %tmp4
}
define <2 x i64> @fcmal2xdouble(<2 x double> %A, <2 x double> %B) {
-; CHECK-SD-LABEL: fcmal2xdouble:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: movi v0.2d, #0xffffffffffffffff
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: fcmal2xdouble:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: adrp x8, .LCPI221_0
-; CHECK-GI-NEXT: ldr q0, [x8, :lo12:.LCPI221_0]
-; CHECK-GI-NEXT: shl v0.2d, v0.2d, #63
-; CHECK-GI-NEXT: cmlt v0.2d, v0.2d, #0
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: fcmal2xdouble:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v0.2d, #0xffffffffffffffff
+; CHECK-NEXT: ret
%tmp3 = fcmp true <2 x double> %A, %B
%tmp4 = sext <2 x i1> %tmp3 to <2 x i64>
ret <2 x i64> %tmp4
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-or-redundant.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-or-redundant.mir
index e75a811c0bee69..6e3d2f76398872 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-or-redundant.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-or-redundant.mir
@@ -201,12 +201,9 @@ tracksRegLiveness: true
body: |
bb.0:
; CHECK-LABEL: name: vector_dont_combine_const_too_wide
- ; CHECK: %fifteen:_(s16) = G_CONSTANT i16 15
- ; CHECK-NEXT: %too_wide:_(s16) = G_CONSTANT i16 257
- ; CHECK-NEXT: %mask:_(s16) = G_CONSTANT i16 255
- ; CHECK-NEXT: %c1:_(<2 x s16>) = G_BUILD_VECTOR %fifteen(s16), %too_wide(s16)
- ; CHECK-NEXT: %c2:_(<2 x s16>) = G_BUILD_VECTOR %mask(s16), %mask(s16)
- ; CHECK-NEXT: %and:_(<2 x s16>) = G_OR %c1, %c2
+ ; CHECK: %mask:_(s16) = G_CONSTANT i16 255
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 511
+ ; CHECK-NEXT: %and:_(<2 x s16>) = G_BUILD_VECTOR %mask(s16), [[C]](s16)
; CHECK-NEXT: $vgpr0 = COPY %and(<2 x s16>)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%fifteen:_(s16) = G_CONSTANT i16 15
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-redundant-and.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-redundant-and.mir
index 70bcd3d95c6620..a476ca669cc005 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-redundant-and.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-redundant-and.mir
@@ -215,11 +215,8 @@ body: |
bb.0:
; CHECK-LABEL: name: vector_dont_combine_const_too_wide
; CHECK: %fifteen:_(s16) = G_CONSTANT i16 15
- ; CHECK-NEXT: %too_wide:_(s16) = G_CONSTANT i16 257
- ; CHECK-NEXT: %mask:_(s16) = G_CONSTANT i16 255
- ; CHECK-NEXT: %c1:_(<2 x s16>) = G_BUILD_VECTOR %fifteen(s16), %too_wide(s16)
- ; CHECK-NEXT: %c2:_(<2 x s16>) = G_BUILD_VECTOR %mask(s16), %mask(s16)
- ; CHECK-NEXT: %and:_(<2 x s16>) = G_AND %c1, %c2
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 1
+ ; CHECK-NEXT: %and:_(<2 x s16>) = G_BUILD_VECTOR %fifteen(s16), [[C]](s16)
; CHECK-NEXT: $vgpr0 = COPY %and(<2 x s16>)
; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%fifteen:_(s16) = G_CONSTANT i16 15
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir
index 5a337b251bab6f..5ac9d99af28ba3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir
@@ -375,14 +375,16 @@ body: |
; GFX6: liveins: $vgpr0
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: %shl:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: %shl:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C1]](s32)
; GFX6-NEXT: $vgpr0_vgpr1 = COPY %shl(<2 x s32>)
;
; GFX9-LABEL: name: do_not_shl_v2s32_zero_by_16_from_zext_v2s16
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: %shl:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32)
+ ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX9-NEXT: %shl:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C1]](s32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY %shl(<2 x s32>)
%zero:_(s16) = G_CONSTANT i16 0
%zerovector:_(<2 x s16>) = G_BUILD_VECTOR %zero, %zero:_(s16)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.prelegal.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.prelegal.mir
index 25feacdca01cec..628ac494b13047 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.prelegal.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.prelegal.mir
@@ -246,16 +246,16 @@ body: |
; GFX6-LABEL: name: do_not_shl_v2s32_zero_by_16_from_zext_v2s16
; GFX6: liveins: $vgpr0, $vgpr1
; GFX6-NEXT: {{ $}}
- ; GFX6-NEXT: %6:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: %shl:_(<2 x s32>) = G_BUILD_VECTOR %6(s32), %6(s32)
- ; GFX6-NEXT: $vgpr0_vgpr1 = COPY %shl(<2 x s32>)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: %extend:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32)
+ ; GFX6-NEXT: $vgpr0_vgpr1 = COPY %extend(<2 x s32>)
;
; GFX9-LABEL: name: do_not_shl_v2s32_zero_by_16_from_zext_v2s16
; GFX9: liveins: $vgpr0, $vgpr1
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: %6:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: %shl:_(<2 x s32>) = G_BUILD_VECTOR %6(s32), %6(s32)
- ; GFX9-NEXT: $vgpr0_vgpr1 = COPY %shl(<2 x s32>)
+ ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; GFX9-NEXT: %extend:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32)
+ ; GFX9-NEXT: $vgpr0_vgpr1 = COPY %extend(<2 x s32>)
%zero:_(s16) = G_CONSTANT i16 0
%zerovector:_(<2 x s16>) = G_BUILD_VECTOR %zero, %zero:_(s16)
%shiftamt:_(s16) = G_CONSTANT i16 16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-ashr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-ashr.mir
index 95009fb1b686e6..14b5a8833d9dc7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-ashr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-ashr.mir
@@ -59,12 +59,9 @@ tracksRegLiveness: true
body: |
bb.0:
; CHECK-LABEL: name: constant_fold_ashr_v2s16_v2s16
- ; CHECK: %val0:_(s16) = G_CONSTANT i16 1234
- ; CHECK-NEXT: %val1:_(s16) = G_CONSTANT i16 -5678
- ; CHECK-NEXT: %val:_(<2 x s16>) = G_BUILD_VECTOR %val0(s16), %val1(s16)
- ; CHECK-NEXT: %shift_amt_elt:_(s16) = G_CONSTANT i16 8
- ; CHECK-NEXT: %shift_amt:_(<2 x s16>) = G_BUILD_VECTOR %shift_amt_elt(s16), %shift_amt_elt(s16)
- ; CHECK-NEXT: %shift:_(<2 x s16>) = G_ASHR %val, %shift_amt(<2 x s16>)
+ ; CHECK: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 4
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 -23
+ ; CHECK-NEXT: %shift:_(<2 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C1]](s16)
; CHECK-NEXT: $vgpr0 = COPY %shift(<2 x s16>)
%val0:_(s16) = G_CONSTANT i16 1234
%val1:_(s16) = G_CONSTANT i16 -5678
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-lshr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-lshr.mir
index 09844b944eea21..645c0333ffe096 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-lshr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-lshr.mir
@@ -59,12 +59,9 @@ tracksRegLiveness: true
body: |
bb.0:
; CHECK-LABEL: name: constant_fold_lshr_v2s16_v2s16
- ; CHECK: %val0:_(s16) = G_CONSTANT i16 1234
- ; CHECK-NEXT: %val1:_(s16) = G_CONSTANT i16 -5678
- ; CHECK-NEXT: %val:_(<2 x s16>) = G_BUILD_VECTOR %val0(s16), %val1(s16)
- ; CHECK-NEXT: %shift_amt_elt:_(s16) = G_CONSTANT i16 8
- ; CHECK-NEXT: %shift_amt:_(<2 x s16>) = G_BUILD_VECTOR %shift_amt_elt(s16), %shift_amt_elt(s16)
- ; CHECK-NEXT: %shift:_(<2 x s16>) = G_LSHR %val, %shift_amt(<2 x s16>)
+ ; CHECK: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 4
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 233
+ ; CHECK-NEXT: %shift:_(<2 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C1]](s16)
; CHECK-NEXT: $vgpr0 = COPY %shift(<2 x s16>)
%val0:_(s16) = G_CONSTANT i16 1234
%val1:_(s16) = G_CONSTANT i16 -5678
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-shl.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-shl.mir
index ae2c63942dbffd..a90a5025bc48dd 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-shl.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-shl.mir
@@ -59,12 +59,9 @@ tracksRegLiveness: true
body: |
bb.0:
; CHECK-LABEL: name: constant_fold_shl_v2s16_v2s16
- ; CHECK: %val0:_(s16) = G_CONSTANT i16 1234
- ; CHECK-NEXT: %val1:_(s16) = G_CONSTANT i16 5678
- ; CHECK-NEXT: %val:_(<2 x s16>) = G_BUILD_VECTOR %val0(s16), %val1(s16)
- ; CHECK-NEXT: %shift_amt_elt:_(s16) = G_CONSTANT i16 8
- ; CHECK-NEXT: %shift_amt:_(<2 x s16>) = G_BUILD_VECTOR %shift_amt_elt(s16), %shift_amt_elt(s16)
- ; CHECK-NEXT: %shift:_(<2 x s16>) = G_SHL %val, %shift_amt(<2 x s16>)
+ ; CHECK: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 -11776
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 11776
+ ; CHECK-NEXT: %shift:_(<2 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C1]](s16)
; CHECK-NEXT: $vgpr0 = COPY %shift(<2 x s16>)
%val0:_(s16) = G_CONSTANT i16 1234
%val1:_(s16) = G_CONSTANT i16 5678
>From 6fe046829c22ba17dd8772d571ce399333b7645e Mon Sep 17 00:00:00 2001
From: vg0204 <Vikash.Gupta at amd.com>
Date: Mon, 7 Sep 2026 13:07:03 +0530
Subject: [PATCH 2/8] Added ConstantFoldFP for host-libm and addressed reviewed
changes.
---
llvm/include/llvm/Analysis/ConstantFolding.h | 8 +++
llvm/lib/Analysis/ConstantFolding.cpp | 46 ++++++------
llvm/lib/CodeGen/GlobalISel/Utils.cpp | 31 ++++----
.../combine-constant-fold-fp-rounding.mir | 28 ++++++++
.../postlegalizer-combiner-constant-fold.mir | 72 +++++++++++--------
...relegalizer-combiner-constant-fold-fma.mir | 18 +----
6 files changed, 123 insertions(+), 80 deletions(-)
diff --git a/llvm/include/llvm/Analysis/ConstantFolding.h b/llvm/include/llvm/Analysis/ConstantFolding.h
index 57c18108cb22b8..ba971bce1bf74d 100644
--- a/llvm/include/llvm/Analysis/ConstantFolding.h
+++ b/llvm/include/llvm/Analysis/ConstantFolding.h
@@ -19,6 +19,7 @@
#ifndef LLVM_ANALYSIS_CONSTANTFOLDING_H
#define LLVM_ANALYSIS_CONSTANTFOLDING_H
+#include "llvm/ADT/APFloat.h"
#include "llvm/Support/Compiler.h"
#include <stdint.h>
@@ -108,6 +109,13 @@ ConstantFoldFPInstOperands(unsigned Opcode, Constant *LHS, Constant *RHS,
const DataLayout &DL, const Instruction *I,
bool AllowNonDeterministic = true);
+/// Evaluate \p NativeFP on \p V using the host libm, then convert the result to
+/// \p Ty. Returns nullptr if the host call raised a non-inexact floating-point
+/// exception, or if \p DenormMode is invalid or dynamic.
+LLVM_ABI Constant *
+ConstantFoldFP(double (*NativeFP)(double), const APFloat &V, Type *Ty,
+ DenormalMode DenormMode = DenormalMode::getIEEE());
+
/// Attempt to flush float point constant according to denormal mode set in the
/// instruction's parent function attributes. If so, return a zero with the
/// correct sign, otherwise return the original constant. Inputs and outputs to
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index b09f348a3b30a5..389dc028a68b91 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -2292,29 +2292,6 @@ static APFloat FlushWithDenormKind(const APFloat &V,
}
}
-Constant *ConstantFoldFP(double (*NativeFP)(double), const APFloat &V, Type *Ty,
- DenormalMode DenormMode = DenormalMode::getIEEE()) {
- if (!DenormMode.isValid() ||
- DenormMode.Input == DenormalMode::DenormalModeKind::Dynamic ||
- DenormMode.Output == DenormalMode::DenormalModeKind::Dynamic)
- return nullptr;
-
- llvm_fenv_clearexcept();
- auto Input = FlushWithDenormKind(V, DenormMode.Input);
- double Result = NativeFP(Input.convertToDouble());
- if (llvm_fenv_testexcept()) {
- llvm_fenv_clearexcept();
- return nullptr;
- }
-
- Constant *Output = GetConstantFoldFPValue(Result, Ty);
- if (DenormMode.Output == DenormalMode::DenormalModeKind::IEEE)
- return Output;
- const auto *CFP = static_cast<ConstantFP *>(Output);
- const auto Res = FlushWithDenormKind(CFP->getValueAPF(), DenormMode.Output);
- return ConstantFP::get(Ty->getContext(), Res);
-}
-
#if defined(HAS_IEE754_FLOAT128) && defined(HAS_LOGF128)
Constant *ConstantFoldFP128(float128 (*NativeFP)(float128), const APFloat &V,
Type *Ty) {
@@ -4785,6 +4762,29 @@ ConstantFoldStructCall(StringRef Name, Intrinsic::ID IntrinsicID,
} // end anonymous namespace
+Constant *llvm::ConstantFoldFP(double (*NativeFP)(double), const APFloat &V,
+ Type *Ty, DenormalMode DenormMode) {
+ if (!DenormMode.isValid() ||
+ DenormMode.Input == DenormalMode::DenormalModeKind::Dynamic ||
+ DenormMode.Output == DenormalMode::DenormalModeKind::Dynamic)
+ return nullptr;
+
+ llvm_fenv_clearexcept();
+ auto Input = FlushWithDenormKind(V, DenormMode.Input);
+ double Result = NativeFP(Input.convertToDouble());
+ if (llvm_fenv_testexcept()) {
+ llvm_fenv_clearexcept();
+ return nullptr;
+ }
+
+ Constant *Output = GetConstantFoldFPValue(Result, Ty);
+ if (DenormMode.Output == DenormalMode::DenormalModeKind::IEEE)
+ return Output;
+ const auto *CFP = static_cast<ConstantFP *>(Output);
+ const auto Res = FlushWithDenormKind(CFP->getValueAPF(), DenormMode.Output);
+ return ConstantFP::get(Ty->getContext(), Res);
+}
+
Constant *llvm::ConstantFoldIntrinsic(Intrinsic::ID ID,
ArrayRef<Constant *> Ops, Type *Ty,
const DataLayout &DL,
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index e80a82bc19482a..30520dc5815dc7 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -12,6 +12,7 @@
#include "llvm/CodeGen/GlobalISel/Utils.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
+#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/CodeGenCommonISel.h"
#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
@@ -34,6 +35,8 @@
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/IR/Constants.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Type.h"
#include "llvm/Support/UndefPoison.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/Utils/SizeOpts.h"
@@ -1010,7 +1013,8 @@ SmallVector<APFloat>
llvm::ConstantFoldUnaryFPOp(unsigned Opcode, LLT DstTy, Register Src,
const MachineRegisterInfo &MRI) {
LLT DstEltTy = DstTy.getScalarType();
- auto Fold = [Opcode, DstEltTy](const APFloat &V) -> APFloat {
+ auto Fold = [Opcode, DstEltTy,
+ &MRI](const APFloat &V) -> std::optional<APFloat> {
APFloat Result(V);
bool Unused;
switch (Opcode) {
@@ -1043,22 +1047,21 @@ llvm::ConstantFoldUnaryFPOp(unsigned Opcode, LLT DstTy, Register Src,
APFloat::rmNearestTiesToEven, &Unused);
return Result;
case TargetOpcode::G_FSQRT:
- Result.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
- &Unused);
- Result = APFloat(std::sqrt(Result.convertToDouble()));
- break;
- case TargetOpcode::G_FLOG2:
- Result.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
- &Unused);
- Result = APFloat(std::log2(Result.convertToDouble()));
- break;
+ case TargetOpcode::G_FLOG2: {
+ Type *Ty =
+ Type::getFloatingPointTy(MRI.getMF().getFunction().getContext(),
+ getFltSemanticForLLT(DstEltTy));
+ Constant *C = ConstantFoldFP(Opcode == TargetOpcode::G_FSQRT
+ ? static_cast<double (*)(double)>(sqrt)
+ : static_cast<double (*)(double)>(log2),
+ V, Ty);
+ if (!C)
+ return std::nullopt;
+ return cast<ConstantFP>(C)->getValueAPF();
+ }
default:
llvm_unreachable("unexpected opcode in ConstantFoldUnaryFPOp");
}
- // Only G_FSQRT and G_FLOG2 reach here; convert the double result back to
- // the source (== destination) semantics.
- Result.convert(V.getSemantics(), APFloat::rmNearestTiesToEven, &Unused);
- return Result;
};
auto tryFoldScalar = [&](Register R) -> std::optional<APFloat> {
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-constant-fold-fp-rounding.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-constant-fold-fp-rounding.mir
index e483f86097c43d..b44a37b326621c 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-constant-fold-fp-rounding.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-constant-fold-fp-rounding.mir
@@ -165,3 +165,31 @@ body: |
RET_ReallyLR implicit $d0
...
+---
+name: fsqrt
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: fsqrt
+ ; CHECK: %res:_(f64) = G_FCONSTANT double 2.000000e+00
+ ; CHECK-NEXT: $d0 = COPY %res(f64)
+ ; CHECK-NEXT: RET_ReallyLR implicit $d0
+ %a:_(f64) = G_FCONSTANT double 4.0
+ %res:_(f64) = G_FSQRT %a
+ $d0 = COPY %res(f64)
+ RET_ReallyLR implicit $d0
+
+...
+---
+name: flog2
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: flog2
+ ; CHECK: %res:_(f64) = G_FCONSTANT double 3.000000e+00
+ ; CHECK-NEXT: $d0 = COPY %res(f64)
+ ; CHECK-NEXT: RET_ReallyLR implicit $d0
+ %a:_(f64) = G_FCONSTANT double 8.0
+ %res:_(f64) = G_FLOG2 %a
+ $d0 = COPY %res(f64)
+ RET_ReallyLR implicit $d0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir
index 66b1b9eb6caea6..9fbab67265f34a 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir
@@ -411,17 +411,11 @@ body: |
---
name: add_vector
legalized: true
-liveins:
- - { reg: '$q0' }
body: |
bb.1.entry:
- liveins: $q0
-
; Element-wise fold of an integer vector binop: <40, 10> + <2, 2> = <42, 12>.
; CHECK-LABEL: name: add_vector
- ; CHECK: liveins: $q0
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 42
+ ; CHECK: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 42
; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 12
; CHECK-NEXT: %res:_(<2 x i64>) = G_BUILD_VECTOR [[C]](i64), [[C1]](i64)
; CHECK-NEXT: $q0 = COPY %res(<2 x i64>)
@@ -439,17 +433,11 @@ body: |
---
name: fadd_vector
legalized: true
-liveins:
- - { reg: '$q0' }
body: |
bb.1.entry:
- liveins: $q0
-
; Element-wise fold of an FP vector binop: <40.0, 1.0> + <2.0, 2.0>.
; CHECK-LABEL: name: fadd_vector
- ; CHECK: liveins: $q0
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double 4.200000e+01
+ ; CHECK: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double 4.200000e+01
; CHECK-NEXT: [[C1:%[0-9]+]]:_(f64) = G_FCONSTANT double 3.000000e+00
; CHECK-NEXT: %res:_(<2 x f64>) = G_BUILD_VECTOR [[C]](f64), [[C1]](f64)
; CHECK-NEXT: $q0 = COPY %res(<2 x f64>)
@@ -467,17 +455,11 @@ body: |
---
name: zext_vector
legalized: true
-liveins:
- - { reg: '$q0' }
body: |
bb.1.entry:
- liveins: $q0
-
; Element-wise fold of a vector cast: zext(<7, 8> : <2 x i32>) to <2 x i64>.
; CHECK-LABEL: name: zext_vector
- ; CHECK: liveins: $q0
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 7
+ ; CHECK: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 7
; CHECK-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; CHECK-NEXT: %res:_(<2 x i64>) = G_BUILD_VECTOR [[C]](i64), [[C1]](i64)
; CHECK-NEXT: $q0 = COPY %res(<2 x i64>)
@@ -493,17 +475,11 @@ body: |
---
name: fneg_vector
legalized: true
-liveins:
- - { reg: '$q0' }
body: |
bb.1.entry:
- liveins: $q0
-
; Element-wise fold of a unary FP op: fneg(<3.0, -4.0>) = <-3.0, 4.0>.
; CHECK-LABEL: name: fneg_vector
- ; CHECK: liveins: $q0
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double -3.000000e+00
+ ; CHECK: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double -3.000000e+00
; CHECK-NEXT: [[C1:%[0-9]+]]:_(f64) = G_FCONSTANT double 4.000000e+00
; CHECK-NEXT: %res:_(<2 x f64>) = G_BUILD_VECTOR [[C]](f64), [[C1]](f64)
; CHECK-NEXT: $q0 = COPY %res(<2 x f64>)
@@ -516,3 +492,43 @@ body: |
RET_ReallyLR implicit $q0
...
+---
+name: fsqrt_vector
+legalized: true
+body: |
+ bb.1.entry:
+ ; Element-wise fold of a host-libm unary FP op: fsqrt(<4.0, 9.0>) = <2.0, 3.0>.
+ ; CHECK-LABEL: name: fsqrt_vector
+ ; CHECK: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double 2.000000e+00
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(f64) = G_FCONSTANT double 3.000000e+00
+ ; CHECK-NEXT: %res:_(<2 x f64>) = G_BUILD_VECTOR [[C]](f64), [[C1]](f64)
+ ; CHECK-NEXT: $q0 = COPY %res(<2 x f64>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %a0:_(f64) = G_FCONSTANT double 4.0
+ %a1:_(f64) = G_FCONSTANT double 9.0
+ %src:_(<2 x f64>) = G_BUILD_VECTOR %a0(f64), %a1(f64)
+ %res:_(<2 x f64>) = G_FSQRT %src(<2 x f64>)
+ $q0 = COPY %res(<2 x f64>)
+ RET_ReallyLR implicit $q0
+
+...
+---
+name: flog2_vector
+legalized: true
+body: |
+ bb.1.entry:
+ ; Element-wise fold of a host-libm unary FP op: flog2(<8.0, 4.0>) = <3.0, 2.0>.
+ ; CHECK-LABEL: name: flog2_vector
+ ; CHECK: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double 3.000000e+00
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(f64) = G_FCONSTANT double 2.000000e+00
+ ; CHECK-NEXT: %res:_(<2 x f64>) = G_BUILD_VECTOR [[C]](f64), [[C1]](f64)
+ ; CHECK-NEXT: $q0 = COPY %res(<2 x f64>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %a0:_(f64) = G_FCONSTANT double 8.0
+ %a1:_(f64) = G_FCONSTANT double 4.0
+ %src:_(<2 x f64>) = G_BUILD_VECTOR %a0(f64), %a1(f64)
+ %res:_(<2 x f64>) = G_FLOG2 %src(<2 x f64>)
+ $q0 = COPY %res(<2 x f64>)
+ RET_ReallyLR implicit $q0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-constant-fold-fma.mir b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-constant-fold-fma.mir
index 83ed5c03ecd39d..e6bfb8597a0593 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-constant-fold-fma.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-constant-fold-fma.mir
@@ -1,20 +1,14 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple aarch64 -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple aarch64 -run-pass=aarch64-prelegalizer-combiner %s -o - | FileCheck %s
---
name: fma_scalar
legalized: true
-liveins:
- - { reg: '$d0' }
body: |
bb.1.entry:
- liveins: $d0
-
; fma(3.0, 4.0, 5.0) = 17.0
; CHECK-LABEL: name: fma_scalar
- ; CHECK: liveins: $d0
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: %res:_(f64) = G_FCONSTANT double 1.700000e+01
+ ; CHECK: %res:_(f64) = G_FCONSTANT double 1.700000e+01
; CHECK-NEXT: $d0 = COPY %res(f64)
; CHECK-NEXT: RET_ReallyLR implicit $d0
%a:_(f64) = G_FCONSTANT double 3.0
@@ -28,17 +22,11 @@ body: |
---
name: fma_vector
legalized: true
-liveins:
- - { reg: '$q0' }
body: |
bb.1.entry:
- liveins: $q0
-
; Element-wise fma: fma(<3.0, 2.0>, <4.0, 4.0>, <5.0, 1.0>) = <17.0, 9.0>.
; CHECK-LABEL: name: fma_vector
- ; CHECK: liveins: $q0
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double 1.700000e+01
+ ; CHECK: [[C:%[0-9]+]]:_(f64) = G_FCONSTANT double 1.700000e+01
; CHECK-NEXT: [[C1:%[0-9]+]]:_(f64) = G_FCONSTANT double 9.000000e+00
; CHECK-NEXT: %res:_(<2 x f64>) = G_BUILD_VECTOR [[C]](f64), [[C1]](f64)
; CHECK-NEXT: $q0 = COPY %res(<2 x f64>)
>From 73059144f0210be621228f9cca5f813c041b24b1 Mon Sep 17 00:00:00 2001
From: vg0204 <Vikash.Gupta at amd.com>
Date: Wed, 9 Sep 2026 12:21:07 +0530
Subject: [PATCH 3/8] Addresse the reviewed changes.
---
llvm/lib/CodeGen/GlobalISel/Utils.cpp | 7 +++----
.../GlobalISel/prelegalizercombiner-commute-shift.mir | 5 +++--
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 30520dc5815dc7..09d18997f59b06 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -1051,10 +1051,9 @@ llvm::ConstantFoldUnaryFPOp(unsigned Opcode, LLT DstTy, Register Src,
Type *Ty =
Type::getFloatingPointTy(MRI.getMF().getFunction().getContext(),
getFltSemanticForLLT(DstEltTy));
- Constant *C = ConstantFoldFP(Opcode == TargetOpcode::G_FSQRT
- ? static_cast<double (*)(double)>(sqrt)
- : static_cast<double (*)(double)>(log2),
- V, Ty);
+ Constant *C = Opcode == TargetOpcode::G_FSQRT
+ ? ConstantFoldFP(sqrt, V, Ty)
+ : ConstantFoldFP(log2, V, Ty);
if (!C)
return std::nullopt;
return cast<ConstantFP>(C)->getValueAPF();
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-commute-shift.mir b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-commute-shift.mir
index 7bc1503e9da32f..8f78c1dacaf2ad 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-commute-shift.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-commute-shift.mir
@@ -108,8 +108,9 @@ body: |
; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 2
; CHECK-NEXT: %veccst2:_(<4 x i32>) = G_BUILD_VECTOR [[C]](i32), [[C]](i32), [[C]](i32), [[C]](i32)
; CHECK-NEXT: [[SHL:%[0-9]+]]:_(<4 x i32>) = G_SHL %xvec, %veccst2(<4 x i32>)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(<4 x i32>) = G_SHL %veccst2, %veccst2(<4 x i32>)
- ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(<4 x i32>) = G_ADD [[SHL]], [[SHL1]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[C1]](i32), [[C1]](i32), [[C1]](i32), [[C1]](i32)
+ ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(<4 x i32>) = G_ADD [[SHL]], [[BUILD_VECTOR]]
; CHECK-NEXT: G_STORE [[ADD]](<4 x i32>), [[COPY]](p0) :: (store (<4 x i32>))
; CHECK-NEXT: RET_ReallyLR
%0:_(p0) = COPY $x0
>From 2c1afafa1185dcf9f7a0a165352635cc729dfcd0 Mon Sep 17 00:00:00 2001
From: vg0204 <Vikash.Gupta at amd.com>
Date: Wed, 9 Sep 2026 14:17:39 +0530
Subject: [PATCH 4/8] Extend the same BuildFnTy-based scalar/vector branching
used by matchConstantFoldBinOp/matchConstantFoldUnaryIntOp to three more
matchinfo-driven combines, so they no longer rely on replaceInstWithConstant.
They are as follows :
- matchICmpToTrueFalseKnownBits
- matchShiftsTooBig
- matchCastOfInteger
---
.../llvm/CodeGen/GlobalISel/CombinerHelper.h | 7 +-
.../include/llvm/Target/GlobalISel/Combine.td | 19 ++----
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 64 ++++++++++++++-----
.../GlobalISel/CombinerHelperCasts.cpp | 10 ++-
4 files changed, 65 insertions(+), 35 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index 482466181e307c..73c5ee7d45321d 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -682,7 +682,7 @@ class CombinerHelper {
/// \returns true if a G_ICMP instruction \p MI can be replaced with a true
/// or false constant based off of KnownBits information.
LLVM_ABI bool matchICmpToTrueFalseKnownBits(MachineInstr &MI,
- int64_t &MatchInfo) const;
+ BuildFnTy &MatchInfo) const;
/// \returns true if a G_ICMP \p MI can be replaced with its LHS based off of
/// KnownBits information.
@@ -925,8 +925,7 @@ class CombinerHelper {
/// Match shifts greater or equal to the range (the bitwidth of the result
/// datatype, or the effective bitwidth of the source value).
- LLVM_ABI bool matchShiftsTooBig(MachineInstr &MI,
- std::optional<int64_t> &MatchInfo) const;
+ LLVM_ABI bool matchShiftsTooBig(MachineInstr &MI, BuildFnTy &MatchInfo) const;
/// Match constant LHS ops that should be commuted.
LLVM_ABI bool matchCommuteConstantToRHS(MachineInstr &MI) const;
@@ -972,7 +971,7 @@ class CombinerHelper {
BuildFnTy &MatchInfo) const;
LLVM_ABI bool matchCastOfInteger(const MachineInstr &CastMI,
- APInt &MatchInfo) const;
+ BuildFnTy &MatchInfo) const;
/// Combine addos.
LLVM_ABI bool matchAddOverflow(MachineInstr &MI, BuildFnTy &MatchInfo) const;
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index 7f5d47276b648b..ac819aab2830d9 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -358,18 +358,11 @@ def shift_const_op : GICombinePatFrag<
!foreach(op,
[G_SHL, G_ASHR, G_LSHR],
(pattern (op $dst, $shifted, $amt)))>;
-def shift_result_matchdata : GIDefMatchData<"std::optional<int64_t>">;
def shifts_too_big : GICombineRule<
- (defs root:$root, shift_result_matchdata:$matchinfo),
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
(match (shift_const_op $root):$mi,
[{ return Helper.matchShiftsTooBig(*${mi}, ${matchinfo}); }]),
- (apply [{
- if (${matchinfo}) {
- Helper.replaceInstWithConstant(*${mi}, *${matchinfo});
- } else {
- Helper.replaceInstWithUndef(*${mi});
- }
- }])>;
+ (apply [{ Helper.applyBuildFn(*${mi}, ${matchinfo}); }])>;
// Fold shift (shift base x), y -> shift base, (x+y), if shifts are same
def shift_immed_matchdata : GIDefMatchData<"RegisterImmPair">;
@@ -1425,10 +1418,10 @@ def rotate_out_of_range : GICombineRule<
>;
def icmp_to_true_false_known_bits : GICombineRule<
- (defs root:$d, int64_matchinfo:$matchinfo),
+ (defs root:$d, build_fn_matchinfo:$matchinfo),
(match (G_ICMP $dst, $pred, $src1, $src2):$d,
[{ return Helper.matchICmpToTrueFalseKnownBits(*${d}, ${matchinfo}); }]),
- (apply [{ Helper.replaceInstWithConstant(*${d}, ${matchinfo}); }])>;
+ (apply [{ Helper.applyBuildFn(*${d}, ${matchinfo}); }])>;
def icmp_to_lhs_known_bits : GICombineRule<
(defs root:$root, build_fn_matchinfo:$info),
@@ -2634,11 +2627,11 @@ def narrow_binop_xor : narrow_binop_opcode<G_XOR>;
// Cast of integer.
class integer_of_opcode<Instruction castOpcode> : GICombineRule <
- (defs root:$root, apint_matchinfo:$matchinfo),
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
(match (G_CONSTANT $int, $imm),
(castOpcode $root, $int):$Cast,
[{ return Helper.matchCastOfInteger(*${Cast}, ${matchinfo}); }]),
- (apply [{ Helper.replaceInstWithConstant(*${Cast}, ${matchinfo}); }])>;
+ (apply [{ Helper.applyBuildFn(*${Cast}, ${matchinfo}); }])>;
def integer_of_truncate : integer_of_opcode<G_TRUNC>;
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 53eea4d968341f..0bff186d6e1130 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -4511,7 +4511,7 @@ void CombinerHelper::applyRotateOutOfRange(MachineInstr &MI) const {
}
bool CombinerHelper::matchICmpToTrueFalseKnownBits(MachineInstr &MI,
- int64_t &MatchInfo) const {
+ BuildFnTy &MatchInfo) const {
assert(MI.getOpcode() == TargetOpcode::G_ICMP);
auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
@@ -4544,13 +4544,25 @@ bool CombinerHelper::matchICmpToTrueFalseKnownBits(MachineInstr &MI,
if (!KnownVal)
return false;
- MatchInfo =
- *KnownVal
- ? getICmpTrueVal(getTargetLowering(),
- /*IsVector = */
- MRI.getType(MI.getOperand(0).getReg()).isVector(),
- /* IsFP = */ false)
- : 0;
+
+ Register Dst = MI.getOperand(0).getReg();
+ LLT DstTy = MRI.getType(Dst);
+ int64_t Val = *KnownVal ? getICmpTrueVal(getTargetLowering(),
+ /*IsVector = */ DstTy.isVector(),
+ /* IsFP = */ false)
+ : 0;
+
+ if (DstTy.isVector()) {
+ SmallVector<APInt> Csts(DstTy.getNumElements(),
+ APInt(DstTy.getScalarSizeInBits(), Val,
+ /*isSigned=*/true));
+ MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
+ B.buildBuildVectorConstant(Dst, Csts);
+ };
+ return true;
+ }
+
+ MatchInfo = [Dst, Val](MachineIRBuilder &B) { B.buildConstant(Dst, Val); };
return true;
}
@@ -7233,24 +7245,46 @@ getMinUselessShift(KnownBits ValueKB, unsigned Opcode,
return ValueKB.getBitWidth() - SignificantBits;
}
-bool CombinerHelper::matchShiftsTooBig(
- MachineInstr &MI, std::optional<int64_t> &MatchInfo) const {
+bool CombinerHelper::matchShiftsTooBig(MachineInstr &MI,
+ BuildFnTy &MatchInfo) const {
+ Register Dst = MI.getOperand(0).getReg();
Register ShiftVal = MI.getOperand(1).getReg();
Register ShiftReg = MI.getOperand(2).getReg();
- LLT ResTy = MRI.getType(MI.getOperand(0).getReg());
+ LLT ResTy = MRI.getType(Dst);
+ std::optional<int64_t> Result;
auto IsShiftTooBig = [&](const Constant *C) {
auto *CI = dyn_cast<ConstantInt>(C);
if (!CI)
return false;
if (CI->uge(ResTy.getScalarSizeInBits())) {
- MatchInfo = std::nullopt;
+ Result = std::nullopt;
return true;
}
- auto OptMaxUsefulShift = getMinUselessShift(VT->getKnownBits(ShiftVal),
- MI.getOpcode(), MatchInfo);
+ auto OptMaxUsefulShift =
+ getMinUselessShift(VT->getKnownBits(ShiftVal), MI.getOpcode(), Result);
return OptMaxUsefulShift && CI->uge(*OptMaxUsefulShift);
};
- return matchUnaryPredicate(MRI, ShiftReg, IsShiftTooBig);
+ if (!matchUnaryPredicate(MRI, ShiftReg, IsShiftTooBig))
+ return false;
+
+ if (!Result) {
+ MatchInfo = [Dst](MachineIRBuilder &B) { B.buildUndef(Dst); };
+ return true;
+ }
+
+ int64_t Val = *Result;
+ if (ResTy.isVector()) {
+ SmallVector<APInt> Csts(ResTy.getNumElements(),
+ APInt(ResTy.getScalarSizeInBits(), Val,
+ /*isSigned=*/true));
+ MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
+ B.buildBuildVectorConstant(Dst, Csts);
+ };
+ return true;
+ }
+
+ MatchInfo = [Dst, Val](MachineIRBuilder &B) { B.buildConstant(Dst, Val); };
+ return true;
}
bool CombinerHelper::matchCommuteConstantToRHS(MachineInstr &MI) const {
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelperCasts.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelperCasts.cpp
index a310cb8174f435..43c48303a4deab 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelperCasts.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelperCasts.cpp
@@ -362,19 +362,23 @@ bool CombinerHelper::matchNarrowBinop(const MachineInstr &TruncMI,
}
bool CombinerHelper::matchCastOfInteger(const MachineInstr &CastMI,
- APInt &MatchInfo) const {
+ BuildFnTy &MatchInfo) const {
const GExtOrTruncOp *Cast = cast<GExtOrTruncOp>(&CastMI);
APInt Input = getIConstantFromReg(Cast->getSrcReg(), MRI);
- LLT DstTy = MRI.getType(Cast->getReg(0));
+ Register Dst = Cast->getReg(0);
+ LLT DstTy = MRI.getType(Dst);
if (!isConstantLegalOrBeforeLegalizer(DstTy))
return false;
switch (Cast->getOpcode()) {
case TargetOpcode::G_TRUNC: {
- MatchInfo = Input.trunc(DstTy.getScalarSizeInBits());
+ APInt Result = Input.trunc(DstTy.getScalarSizeInBits());
+ MatchInfo = [Dst, Result](MachineIRBuilder &B) {
+ B.buildConstant(Dst, Result);
+ };
return true;
}
default:
>From 981e6bb8205ef50f866fa3462b0f2dac97617199 Mon Sep 17 00:00:00 2001
From: vg0204 <Vikash.Gupta at amd.com>
Date: Tue, 15 Sep 2026 14:27:01 +0530
Subject: [PATCH 5/8] Addressed the reviewed changes.
---
llvm/lib/CodeGen/GlobalISel/Utils.cpp | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 09d18997f59b06..9cbb65b645a285 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -838,21 +838,18 @@ llvm::ConstantFoldVectorFPBinop(unsigned Opcode, const Register Op1,
const MachineRegisterInfo &MRI) {
auto *SrcVec2 = getBuildVectorLikeDef(Op2, MRI);
if (!SrcVec2)
- return SmallVector<APFloat>();
+ return {};
auto *SrcVec1 = getBuildVectorLikeDef(Op1, MRI);
if (!SrcVec1)
- return SmallVector<APFloat>();
-
- if (SrcVec1->getNumSources() != SrcVec2->getNumSources())
- return SmallVector<APFloat>();
+ return {};
SmallVector<APFloat> FoldedElements;
for (unsigned Idx = 0, E = SrcVec1->getNumSources(); Idx < E; ++Idx) {
auto MaybeCst = ConstantFoldFPBinOp(Opcode, SrcVec1->getSourceReg(Idx),
SrcVec2->getSourceReg(Idx), MRI);
if (!MaybeCst)
- return SmallVector<APFloat>();
+ return {};
FoldedElements.push_back(*MaybeCst);
}
return FoldedElements;
>From 879e65eeb202b8c06ea428506c053a23e5cc6e65 Mon Sep 17 00:00:00 2001
From: vg0204 <Vikash.Gupta at amd.com>
Date: Fri, 18 Sep 2026 17:46:31 +0530
Subject: [PATCH 6/8] Addressed review: restrict constant folding to fixed
vectors
---
.../llvm/CodeGen/GlobalISel/CombinerHelper.h | 8 ++-
.../include/llvm/Target/GlobalISel/Combine.td | 4 +-
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 69 ++++++++-----------
.../postlegalizer-combiner-constant-fold.mir | 52 ++++++++++++++
...mbine-shl-from-extend-narrow.postlegal.mir | 10 ++-
5 files changed, 91 insertions(+), 52 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index 73c5ee7d45321d..6b9c0f5357016e 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -167,8 +167,10 @@ class CombinerHelper {
/// perform a FewerElements action on the target.
LLVM_ABI bool isLegalOrHasFewerElements(const LegalityQuery &Query) const;
- /// \return true if the combine is running prior to legalization, or if \p Ty
- /// is a legal integer constant type on the target.
+ /// \return true if the combine is running prior to legalization, or if a
+ /// scalar \p Ty G_CONSTANT is legal. For fixed vectors, also requires
+ /// G_BUILD_VECTOR of those scalar constants to be legal. Scalable vectors are
+ /// not constant folded and return false.
LLVM_ABI bool isConstantLegalOrBeforeLegalizer(const LLT Ty) const;
/// MachineRegisterInfo::replaceRegWith() and inform the observer of the changes
@@ -682,7 +684,7 @@ class CombinerHelper {
/// \returns true if a G_ICMP instruction \p MI can be replaced with a true
/// or false constant based off of KnownBits information.
LLVM_ABI bool matchICmpToTrueFalseKnownBits(MachineInstr &MI,
- BuildFnTy &MatchInfo) const;
+ int64_t &MatchInfo) const;
/// \returns true if a G_ICMP \p MI can be replaced with its LHS based off of
/// KnownBits information.
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index ac819aab2830d9..70bafad21c4bbd 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -1418,10 +1418,10 @@ def rotate_out_of_range : GICombineRule<
>;
def icmp_to_true_false_known_bits : GICombineRule<
- (defs root:$d, build_fn_matchinfo:$matchinfo),
+ (defs root:$d, int64_matchinfo:$matchinfo),
(match (G_ICMP $dst, $pred, $src1, $src2):$d,
[{ return Helper.matchICmpToTrueFalseKnownBits(*${d}, ${matchinfo}); }]),
- (apply [{ Helper.applyBuildFn(*${d}, ${matchinfo}); }])>;
+ (apply [{ Helper.replaceInstWithConstant(*${d}, ${matchinfo}); }])>;
def icmp_to_lhs_known_bits : GICombineRule<
(defs root:$root, build_fn_matchinfo:$info),
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 0bff186d6e1130..f2a785b254bf5c 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -174,9 +174,11 @@ bool CombinerHelper::isLegalOrHasFewerElements(
bool CombinerHelper::isConstantLegalOrBeforeLegalizer(const LLT Ty) const {
if (!Ty.isVector())
return isLegalOrBeforeLegalizer({TargetOpcode::G_CONSTANT, {Ty}});
- // Vector constants are represented as a G_BUILD_VECTOR of scalar G_CONSTANTs.
if (isPreLegalize())
return true;
+ // Scalable vectors cannot be constant folded here.
+ if (!Ty.isFixedVector())
+ return false;
LLT EltTy = Ty.getElementType();
return isLegal({TargetOpcode::G_BUILD_VECTOR, {Ty, EltTy}}) &&
isLegal({TargetOpcode::G_CONSTANT, {EltTy}});
@@ -4511,7 +4513,7 @@ void CombinerHelper::applyRotateOutOfRange(MachineInstr &MI) const {
}
bool CombinerHelper::matchICmpToTrueFalseKnownBits(MachineInstr &MI,
- BuildFnTy &MatchInfo) const {
+ int64_t &MatchInfo) const {
assert(MI.getOpcode() == TargetOpcode::G_ICMP);
auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
@@ -4544,25 +4546,13 @@ bool CombinerHelper::matchICmpToTrueFalseKnownBits(MachineInstr &MI,
if (!KnownVal)
return false;
-
- Register Dst = MI.getOperand(0).getReg();
- LLT DstTy = MRI.getType(Dst);
- int64_t Val = *KnownVal ? getICmpTrueVal(getTargetLowering(),
- /*IsVector = */ DstTy.isVector(),
- /* IsFP = */ false)
- : 0;
-
- if (DstTy.isVector()) {
- SmallVector<APInt> Csts(DstTy.getNumElements(),
- APInt(DstTy.getScalarSizeInBits(), Val,
- /*isSigned=*/true));
- MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
- B.buildBuildVectorConstant(Dst, Csts);
- };
- return true;
- }
-
- MatchInfo = [Dst, Val](MachineIRBuilder &B) { B.buildConstant(Dst, Val); };
+ MatchInfo =
+ *KnownVal
+ ? getICmpTrueVal(getTargetLowering(),
+ /*IsVector = */
+ MRI.getType(MI.getOperand(0).getReg()).isVector(),
+ /* IsFP = */ false)
+ : 0;
return true;
}
@@ -5137,7 +5127,10 @@ bool CombinerHelper::matchConstantFoldCastOp(MachineInstr &MI,
LLT DstTy = MRI.getType(Dst);
unsigned Opc = MI.getOpcode();
- if (DstTy.isVector()) {
+ if (!isConstantLegalOrBeforeLegalizer(DstTy))
+ return false;
+
+ if (DstTy.isFixedVector()) {
auto *BV = getOpcodeDef<GBuildVector>(SrcOp, MRI);
if (!BV)
return false;
@@ -5167,7 +5160,9 @@ bool CombinerHelper::matchConstantFoldCastOp(MachineInstr &MI,
bool CombinerHelper::matchConstantFoldUnaryIntOp(MachineInstr &MI,
BuildFnTy &MatchInfo) const {
Register Dst = MI.getOperand(0).getReg();
- auto Csts = ConstantFoldUnaryIntOp(MI.getOpcode(), MRI.getType(Dst),
+ LLT DstTy = MRI.getType(Dst);
+
+ auto Csts = ConstantFoldUnaryIntOp(MI.getOpcode(), DstTy,
MI.getOperand(1).getReg(), MRI);
if (Csts.empty())
return false;
@@ -5188,11 +5183,12 @@ bool CombinerHelper::matchConstantFoldBinOp(MachineInstr &MI,
Register Op2 = MI.getOperand(2).getReg();
LLT DstTy = MRI.getType(Dst);
- if (DstTy.isVector()) {
- // Pointer-element vectors (e.g. <n x ptr> from a vector G_PTR_ADD) have no
- // G_BUILD_VECTOR of G_CONSTANT representation.
- if (DstTy.getElementType().isPointer())
- return false;
+ // Pointer-element vectors have no G_BUILD_VECTOR of G_CONSTANT
+ // representation.
+ if (DstTy.isVector() && DstTy.getElementType().isPointer())
+ return false;
+
+ if (DstTy.isFixedVector()) {
SmallVector<APInt> Csts =
ConstantFoldVectorBinop(MI.getOpcode(), Op1, Op2, MRI);
if (Csts.empty())
@@ -5219,7 +5215,7 @@ bool CombinerHelper::matchConstantFoldFPBinOp(MachineInstr &MI,
Register Op2 = MI.getOperand(2).getReg();
LLT DstTy = MRI.getType(Dst);
- if (DstTy.isVector()) {
+ if (DstTy.isFixedVector()) {
SmallVector<APFloat> Csts =
ConstantFoldVectorFPBinop(MI.getOpcode(), Op1, Op2, MRI);
if (Csts.empty())
@@ -5242,8 +5238,9 @@ bool CombinerHelper::matchConstantFoldFPBinOp(MachineInstr &MI,
bool CombinerHelper::matchConstantFoldFPUnary(MachineInstr &MI,
BuildFnTy &MatchInfo) const {
Register Dst = MI.getOperand(0).getReg();
+ LLT DstTy = MRI.getType(Dst);
SmallVector<APFloat> Csts = ConstantFoldUnaryFPOp(
- MI.getOpcode(), MRI.getType(Dst), MI.getOperand(1).getReg(), MRI);
+ MI.getOpcode(), DstTy, MI.getOperand(1).getReg(), MRI);
if (Csts.empty())
return false;
MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
@@ -5270,7 +5267,7 @@ bool CombinerHelper::matchConstantFoldFMA(MachineInstr &MI,
return Res;
};
- if (DstTy.isVector()) {
+ if (DstTy.isFixedVector()) {
auto *BV1 = getOpcodeDef<GBuildVector>(Op1, MRI);
auto *BV2 = getOpcodeDef<GBuildVector>(Op2, MRI);
auto *BV3 = getOpcodeDef<GBuildVector>(Op3, MRI);
@@ -7273,16 +7270,6 @@ bool CombinerHelper::matchShiftsTooBig(MachineInstr &MI,
}
int64_t Val = *Result;
- if (ResTy.isVector()) {
- SmallVector<APInt> Csts(ResTy.getNumElements(),
- APInt(ResTy.getScalarSizeInBits(), Val,
- /*isSigned=*/true));
- MatchInfo = [Dst, Csts = std::move(Csts)](MachineIRBuilder &B) {
- B.buildBuildVectorConstant(Dst, Csts);
- };
- return true;
- }
-
MatchInfo = [Dst, Val](MachineIRBuilder &B) { B.buildConstant(Dst, Val); };
return true;
}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir
index 9fbab67265f34a..afdf88a9f12e7d 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-constant-fold.mir
@@ -532,3 +532,55 @@ body: |
RET_ReallyLR implicit $q0
...
+---
+name: udiv_vector_by_zero
+legalized: true
+body: |
+ bb.1.entry:
+ ; A zero divisor in any lane must refuse the whole vector fold.
+ ; CHECK-LABEL: name: udiv_vector_by_zero
+ ; CHECK: %a0:_(i64) = G_CONSTANT i64 40
+ ; CHECK-NEXT: %a1:_(i64) = G_CONSTANT i64 10
+ ; CHECK-NEXT: %b0:_(i64) = G_CONSTANT i64 2
+ ; CHECK-NEXT: %b1:_(i64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: %av:_(<2 x i64>) = G_BUILD_VECTOR %a0(i64), %a1(i64)
+ ; CHECK-NEXT: %bv:_(<2 x i64>) = G_BUILD_VECTOR %b0(i64), %b1(i64)
+ ; CHECK-NEXT: %res:_(<2 x i64>) = G_UDIV %av, %bv
+ ; CHECK-NEXT: $q0 = COPY %res(<2 x i64>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %a0:_(i64) = G_CONSTANT i64 40
+ %a1:_(i64) = G_CONSTANT i64 10
+ %b0:_(i64) = G_CONSTANT i64 2
+ %b1:_(i64) = G_CONSTANT i64 0
+ %av:_(<2 x i64>) = G_BUILD_VECTOR %a0(i64), %a1(i64)
+ %bv:_(<2 x i64>) = G_BUILD_VECTOR %b0(i64), %b1(i64)
+ %res:_(<2 x i64>) = G_UDIV %av, %bv
+ $q0 = COPY %res(<2 x i64>)
+ RET_ReallyLR implicit $q0
+
+...
+---
+name: add_vector_undef_lane
+legalized: true
+body: |
+ bb.1.entry:
+ ; An undef lane must refuse the whole vector fold.
+ ; CHECK-LABEL: name: add_vector_undef_lane
+ ; CHECK: %a0:_(i64) = G_CONSTANT i64 40
+ ; CHECK-NEXT: %a1:_(i64) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: %b:_(i64) = G_CONSTANT i64 2
+ ; CHECK-NEXT: %av:_(<2 x i64>) = G_BUILD_VECTOR %a0(i64), %a1(i64)
+ ; CHECK-NEXT: %bv:_(<2 x i64>) = G_BUILD_VECTOR %b(i64), %b(i64)
+ ; CHECK-NEXT: %res:_(<2 x i64>) = G_ADD %av, %bv
+ ; CHECK-NEXT: $q0 = COPY %res(<2 x i64>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %a0:_(i64) = G_CONSTANT i64 40
+ %a1:_(i64) = G_IMPLICIT_DEF
+ %b:_(i64) = G_CONSTANT i64 2
+ %av:_(<2 x i64>) = G_BUILD_VECTOR %a0(i64), %a1(i64)
+ %bv:_(<2 x i64>) = G_BUILD_VECTOR %b(i64), %b(i64)
+ %res:_(<2 x i64>) = G_ADD %av, %bv
+ $q0 = COPY %res(<2 x i64>)
+ RET_ReallyLR implicit $q0
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir
index 5ac9d99af28ba3..e5db8de4e4ef85 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shl-from-extend-narrow.postlegal.mir
@@ -375,17 +375,15 @@ body: |
; GFX6: liveins: $vgpr0
; GFX6-NEXT: {{ $}}
; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: %shl:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C1]](s32)
- ; GFX6-NEXT: $vgpr0_vgpr1 = COPY %shl(<2 x s32>)
+ ; GFX6-NEXT: %extend:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32)
+ ; GFX6-NEXT: $vgpr0_vgpr1 = COPY %extend(<2 x s32>)
;
; GFX9-LABEL: name: do_not_shl_v2s32_zero_by_16_from_zext_v2s16
; GFX9: liveins: $vgpr0
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: %shl:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C1]](s32)
- ; GFX9-NEXT: $vgpr0_vgpr1 = COPY %shl(<2 x s32>)
+ ; GFX9-NEXT: %extend:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32)
+ ; GFX9-NEXT: $vgpr0_vgpr1 = COPY %extend(<2 x s32>)
%zero:_(s16) = G_CONSTANT i16 0
%zerovector:_(<2 x s16>) = G_BUILD_VECTOR %zero, %zero:_(s16)
%shiftamt:_(s16) = G_CONSTANT i16 16
>From ed281c9310528e15271ff78d4d955cf022b246bf Mon Sep 17 00:00:00 2001
From: Vikash Gupta <Vikash.Gupta at amd.com>
Date: Tue, 29 Sep 2026 11:44:18 +0530
Subject: [PATCH 7/8] Apply batched suggestions from code review
Co-authored-by: Tim Gymnich <tim at gymni.ch>
---
llvm/lib/Analysis/ConstantFolding.cpp | 4 ++--
llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp | 4 ++--
llvm/lib/CodeGen/GlobalISel/Utils.cpp | 4 ++--
llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll | 4 ++--
4 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 389dc028a68b91..4f69294dd59acf 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -4770,7 +4770,7 @@ Constant *llvm::ConstantFoldFP(double (*NativeFP)(double), const APFloat &V,
return nullptr;
llvm_fenv_clearexcept();
- auto Input = FlushWithDenormKind(V, DenormMode.Input);
+ APFloat Input = FlushWithDenormKind(V, DenormMode.Input);
double Result = NativeFP(Input.convertToDouble());
if (llvm_fenv_testexcept()) {
llvm_fenv_clearexcept();
@@ -4781,7 +4781,7 @@ Constant *llvm::ConstantFoldFP(double (*NativeFP)(double), const APFloat &V,
if (DenormMode.Output == DenormalMode::DenormalModeKind::IEEE)
return Output;
const auto *CFP = static_cast<ConstantFP *>(Output);
- const auto Res = FlushWithDenormKind(CFP->getValueAPF(), DenormMode.Output);
+ APFloat Res = FlushWithDenormKind(CFP->getValueAPF(), DenormMode.Output);
return ConstantFP::get(Ty->getContext(), Res);
}
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index f2a785b254bf5c..01bfe87eb5ade6 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -5162,8 +5162,8 @@ bool CombinerHelper::matchConstantFoldUnaryIntOp(MachineInstr &MI,
Register Dst = MI.getOperand(0).getReg();
LLT DstTy = MRI.getType(Dst);
- auto Csts = ConstantFoldUnaryIntOp(MI.getOpcode(), DstTy,
- MI.getOperand(1).getReg(), MRI);
+ SmallVector<APInt> Csts = ConstantFoldUnaryIntOp(
+ MI.getOpcode(), DstTy, MI.getOperand(1).getReg(), MRI);
if (Csts.empty())
return false;
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 9cbb65b645a285..7ea70e2eb637e5 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -836,11 +836,11 @@ SmallVector<APFloat>
llvm::ConstantFoldVectorFPBinop(unsigned Opcode, const Register Op1,
const Register Op2,
const MachineRegisterInfo &MRI) {
- auto *SrcVec2 = getBuildVectorLikeDef(Op2, MRI);
+ GBuildVector *SrcVec2 = getBuildVectorLikeDef(Op2, MRI);
if (!SrcVec2)
return {};
- auto *SrcVec1 = getBuildVectorLikeDef(Op1, MRI);
+ GBuildVector *SrcVec1 = getBuildVectorLikeDef(Op1, MRI);
if (!SrcVec1)
return {};
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll
index f957345c53a2e2..1e05868267f3b1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.bf16.ll
@@ -50,7 +50,7 @@ define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vc(<2 x bfloat> %a) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_pk_add_bf16 v0, v0, 2.0 op_sel:[0,1] neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, -2.0 op_sel:[0,1]
; GFX1250-NEXT: ; return to shader part epilog
%result = fsub <2 x bfloat> %a, <bfloat 2.0, bfloat 2.0>
ret <2 x bfloat> %result
@@ -63,7 +63,7 @@ define amdgpu_ps <2 x bfloat> @fsub_v2bf16_vl(<2 x bfloat> %a) {
; GFX1250-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_pk_add_bf16 v0, 0x42c83f80, v0 neg_lo:[1,0] neg_hi:[1,0]
+; GFX1250-NEXT: v_pk_add_bf16 v0, 0xc2c8bf80, v0
; GFX1250-NEXT: ; return to shader part epilog
%result = fsub <2 x bfloat> %a, <bfloat 1.0, bfloat 100.0>
ret <2 x bfloat> %result
>From 110b95e93a4f4b1e7ab0f7186eb514f99067466b Mon Sep 17 00:00:00 2001
From: vg0204 <Vikash.Gupta at amd.com>
Date: Thu, 1 Oct 2026 15:00:50 +0530
Subject: [PATCH 8/8] Add early exit for scalable vectors that may create empty
smallVectors.
---
llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 01bfe87eb5ade6..82565ef6c56f2d 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -5130,6 +5130,9 @@ bool CombinerHelper::matchConstantFoldCastOp(MachineInstr &MI,
if (!isConstantLegalOrBeforeLegalizer(DstTy))
return false;
+ if (DstTy.isScalableVector())
+ return false;
+
if (DstTy.isFixedVector()) {
auto *BV = getOpcodeDef<GBuildVector>(SrcOp, MRI);
if (!BV)
@@ -5183,9 +5186,10 @@ bool CombinerHelper::matchConstantFoldBinOp(MachineInstr &MI,
Register Op2 = MI.getOperand(2).getReg();
LLT DstTy = MRI.getType(Dst);
- // Pointer-element vectors have no G_BUILD_VECTOR of G_CONSTANT
- // representation.
- if (DstTy.isVector() && DstTy.getElementType().isPointer())
+ // Scalable and pointer-element vectors have no G_BUILD_VECTOR of
+ // G_CONSTANT to fold to.
+ if (DstTy.isScalableVector() ||
+ (DstTy.isVector() && DstTy.getElementType().isPointer()))
return false;
if (DstTy.isFixedVector()) {
@@ -5215,6 +5219,9 @@ bool CombinerHelper::matchConstantFoldFPBinOp(MachineInstr &MI,
Register Op2 = MI.getOperand(2).getReg();
LLT DstTy = MRI.getType(Dst);
+ if (DstTy.isScalableVector())
+ return false;
+
if (DstTy.isFixedVector()) {
SmallVector<APFloat> Csts =
ConstantFoldVectorFPBinop(MI.getOpcode(), Op1, Op2, MRI);
@@ -5267,6 +5274,9 @@ bool CombinerHelper::matchConstantFoldFMA(MachineInstr &MI,
return Res;
};
+ if (DstTy.isScalableVector())
+ return false;
+
if (DstTy.isFixedVector()) {
auto *BV1 = getOpcodeDef<GBuildVector>(Op1, MRI);
auto *BV2 = getOpcodeDef<GBuildVector>(Op2, MRI);
More information about the llvm-commits
mailing list